UTF-8 compatibility

centrallix-lib/include/mtlexer.h

@@ -55,6 +55,8 @@ typedef struct _LX
     char**		ReservedWords;	/* reserved words */
     int			LineNumber;
     int			(*ReadFn)();
+    int         (*ValidateFn)(char *); /* pointer to a validate function */
+    int         (*CharFit)(char, int, int); /* pointer to see if a multibyte char will be split */
     void*		ReadArg;
     }
     LxSession, *pLxSession;
@@ -143,6 +145,8 @@ int mlxSetOffset(pLxSession this, unsigned long new_offset);
 #define MLX_F_SSTRING		(1<<22)	/* Differentiate "" and '' strings */
 #define MLX_F_PROCLINE		(1<<23)	/* (int) Line has been processed. */
 #define MLX_F_ALLOWNUL		(1<<24)	/* Allow nul (\0) bytes in input. */
+#define MLX_F_ENFORCEUTF8	(1<<25) /* validate all strings as UTF-8. Incompatable with MLX_F_ENFORCEASCII */
+#define MLX_F_ENFORCEASCII	(1<<26) /* validate all strings as ASCII. Incompatable with MLX_F_ENFORCEUTF8 */
 
 #define MLX_F_PUBLIC		(MLX_F_ICASE | MLX_F_POUNDCOMM | MLX_F_CCOMM | MLX_F_CPPCOMM | MLX_F_SEMICOMM | MLX_F_DASHCOMM | MLX_F_EOL | MLX_F_EOF | MLX_F_IFSONLY | MLX_F_DASHKW | MLX_F_NODISCARD | MLX_F_FILENAMES | MLX_F_DBLBRACE | MLX_F_LINEONLY | MLX_F_SYMBOLMODE | MLX_F_TOKCOMM | MLX_F_NOUNESC | MLX_F_SSTRING | MLX_F_ALLOWNUL)
 #define	MLX_F_PRIVATE		(MLX_F_INSTRING | MLX_F_NOFILE | MLX_F_FOUNDEOL | MLX_F_INCOMM | MLX_F_PROCLINE)

centrallix-lib/include/qprintf.h

@@ -37,9 +37,13 @@ typedef int (*qpf_grow_fn_t)(char**, size_t*, size_t, void*, size_t);
 typedef struct _QPS
     {
     unsigned int	Errors;		/* QPF_ERR_T_xxx */
+    unsigned int    Flags;
     }
     QPSession, *pQPSession;
 
+
+#define QPF_F_ENFORCE_UTF8  1 /* use UTF-8 validation */
+
 #define QPF_ERR_T_NOTIMPL	1	/* unimplemented feature */
 #define QPF_ERR_T_BUFOVERFLOW	2	/* dest buffer too small */
 #define QPF_ERR_T_INSOVERFLOW	4	/* NLEN or *LEN restriction occurred */
@@ -54,11 +58,15 @@ typedef struct _QPS
 #define QPF_ERR_T_BADFILE	2048	/* Bad filename for &FILE filter */
 #define QPF_ERR_T_BADPATH	4096	/* Bad pathname for &PATH filter */
 #define QPF_ERR_T_BADCHAR	8192	/* Bad character for filter (e.g. an octothorpe for &DB64) */
+#define QPF_ERR_T_TRUNC	    16384	/* To avoid splitting a utf-8 char, not all of the space was used */
 
 #define QPERR(x) (s->Errors |= (x))
 
+
 /*** QPrintf methods ***/
+void qpfInitializeDefaultFlags(int isUTF8);
 pQPSession qpfOpenSession();
+pQPSession qpfOpenSessionFlags(unsigned int flags);
 int qpfCloseSession(pQPSession s);
 int qpfClearErrors(pQPSession s);
 unsigned int qpfErrors(pQPSession s);

centrallix-lib/include/util.h

@@ -32,3 +32,42 @@ extern "C" {
 
 #endif	/* UTILITY_H */
 
+#define UTIL_VALID_CHAR (size_t)-1 /** for use with verifyUTF8 **/
+#define UTIL_INVALID_CHAR (size_t)-2
+#define UTIL_INVALID_ARGUMENT (size_t)-3
+/** states for validate **/
+#define UTIL_STATE_START  0
+#define UTIL_STATE_3_BYTE 1 /** 3 bytes left; was 4 total **/
+#define UTIL_STATE_2_BYTE 2 /** 2 bytes left; a leats 3 total **/
+#define UTIL_STATE_1_BYTE 3 /** 1 byte left; was at least 2 total **/
+#define UTIL_STATE_ERROR  4
+#define UTIL_STATE_E_SURROGATE 5
+#define UTIL_STATE_E_OVERLONG 6 /** starts with E0, check for overlong **/
+#define UTIL_STATE_F_OVERLONG 7 /** starts with F0, check for overlong **/
+#define UTIL_STATE_TOO_LARGE 8  /** starts with F4, check for too long **/
+
+/** \brief This function ensures that a string contains valid UTF-8.
+ \param string The string to verify.
+ \return The index of the first byte of the first invald char, or a 
+  code if not applicable  */
+int verifyUTF8(char* str);
+
+/** \brief This function ensures that all of the bytes of a string are 
+     valid ASCII.
+ \param string The string to verify.
+ \return returns index of the first invalid string, or a code if not 
+  applicable */
+int verifyASCII(char * str);
+
+/** \brief This function ensures that all of the bytes of a string are 
+     valid ASCII. Does not require a NULL terminator.
+ \param str The string to verify.
+ \param len the length of the suplied string
+ \return returns index of the first invalid string, or a code if not 
+  applicable */
+int nVerifyUTF8(char* str, int len);
+
+/** \brief Computes the number of bytes in a utf-8 char based on the first byte.
+ * \param byte the character to be checked
+ * \return the number of bytes in the character, or -1 on error. */
+ int numBytesInChar(char byte);

centrallix-lib/include/xstring.h

@@ -26,7 +26,7 @@
 /*		realloc'ing string data structure.			*/
 /************************************************************************/
 
-
+#include <stdlib.h>
 #include <stdarg.h>
 
 #define XS_BLK_SIZ	256
@@ -61,10 +61,15 @@ int xsRTrim(pXString this);
 int xsLTrim(pXString this);
 int xsTrim(pXString this);
 int xsFind(pXString this,char* find,int findlen, int offset);
+int xsFindWithCharOffset(pXString this, char* find, int findlen, int offset);
 int xsFindRev(pXString this,char* find,int findlen, int offset);
+int xsFindRevWithCharOffset(pXString this, char* find, int findlen, int offset);
 int xsSubst(pXString this, int offset, int len, char* rep, int replen);
+int xsSubstWithCharOffset(pXString this, int offset, int len, char* rep, int replen);
 int xsReplace(pXString this, char* find, int findlen, int offset, char* rep, int replen);
+int xsReplaceWithCharOffset(pXString this, char* find, int findlen, int offset, char* rep, int replen);
 int xsInsertAfter(pXString this, char* ins, int inslen, int offset);
+int xsInsertAfterWithCharOffset(pXString this, char* ins, int inslen, int offset);
 int xsGenPrintf(int (*write_fn)(), void* write_arg, char** buf, int* buf_size, const char* fmt, ...);
 int xsGenPrintf_va(int (*write_fn)(), void* write_arg, char** buf, int* buf_size, const char* fmt, va_list va);
 int xsQPrintf(pXString this, char* fmt, ...);
@@ -73,6 +78,9 @@ int xsConcatQPrintf(pXString this, char* fmt, ...);
 pXString xsNew();
 void xsFree(pXString this);
 
+/** Needed utiliy function **/
+size_t chrCharLength(char* string);
+
 #define XS_U_SEEK	2
 
 #endif /* _XSTRING_H */