/*---------------------------------------------------------------------------- %%File: validate.c %%Unit: fechmap %%Contact: jpick "Rolling" state machines that allow interactive verification of DBCS and EUC files. Currently, separate tables are stored for each encoding so that the state machines can be run in parallel (i.e., multiple parse streams). These routines are used by auto-detection and if caller wants conversion routines to return errors on invalid characters. Following is a description of the structure of the DBCS and EUC encodings handled by this module. This information is taken from CJK.INF (maintained by Ken Lunde, author of _Understanding Japanese Information Processing_). This information governs the structure of the class and validation state tables used in this module. Big5 Two-byte Standard Characters Encoding Ranges first byte range 0xA1-0xFE second byte ranges 0x40-0x7E, 0xA1-0xFE One-byte Characters Encoding Range ASCII 0x21-0x7E GBK Two-byte Standard Characters Encoding Ranges first byte range 0x81-0xFE second byte ranges 0x40-0x7E and 0x80-0xFE One-byte Characters Encoding Range ASCII 0x21-0x7E HZ (information from HZ spec Fung F. Lee (lee@umunhum.stanford.edu)) One-byte characters Encoding Ranges first GB byte range 0x21-0x77 second GB byte range 0x21-0x7E ASCII 0x21-0x7E Mode switching Encoding sequence escape sequence from GB to ASCII 0x7E followed by 0x7B ("~{") escape sequence from ASCII to GB 0x7E followed by 0x7D ("~}") line continuation marker 0x7E followed by 0x0A (Note: ASCII mode is the default mode) Shift-Jis Two-byte Standard Characters Encoding Ranges first byte ranges 0x81-0x9F, 0xE0-0xEF second byte ranges 0x40-0x7E, 0x80-0xFC Two-byte User-defined Dharacters Encoding Ranges first byte range 0xF0-0xFC second byte ranges 0x40-0x7E, 0x80-0xFC One-byte Characters Encoding Range Half-width katakana 0xA1-0xDF ASCII/JIS-Roman 0x21-0x7E Wansung Two-byte Standard Characters Encoding Ranges first byte range 0x81-0xFE second byte ranges 0x40-0x7E and 0x80-0xFE One-byte Characters Encoding Range ASCII 0x21-0x7E EUC-Cn Code set 0 (ASCII or GB 1988-89): 0x21-0x7E Code set 1 (GB 2312-80): 0xA1A1-0xFEFE Code set 2: unused Code set 3: unused EUC-Jp Code set 0 (ASCII or JIS X 0201-1976 Roman): 0x21-0x7E Code set 1 (JIS X 0208): 0xA1A1-0xFEFE Code set 2 (half-width katakana): 0x8EA1-0x8EDF Code set 3 (JIS X 0212-1990): 0x8FA1A1-0x8FFEFE EUC-Kr Code set 0 (ASCII or KS C 5636-1993): 0x21-0x7E Code set 1 (KS C 5601-1992): 0xA1A1-0xFEFE Code set 2: unused Code set 3: unused EUC-Tw Code set 0 (ASCII): 0x21-0x7E Code set 1 (CNS 11643-1992 Plane 1): 0xA1A1-0xFEFE Code set 2 (CNS 11643-1992 Planes 1-16): 0x8EA1A1A1-0x8EB0FEFE Code set 3: unused UTF-7 (information from the RFC2152 by D.Goldsmith) One-byte characters Encoding Ranges Direct and Optionally direct 0x21-0x2A, 0x2C-0x5B, 0x5D-0x60, 0x7B-0x7D 0x09, 0x0A, 0x0D, 0x20 Modified Base64 0x2B, 0x2F-39, 0x41-0x5A, 0x61-0x7A Mode switching escape sequence from D/O to M. Base64 0x2B escape sequence from M. Base64 to D/O 0x2D (or any control character) ----------------------------------------------------------------------------*/ #include #include #include "private.h" #include "fechmap_.h" #include "lexint_.h" /*---------------------------------------------------------------------------- Common Defs for all Sequence Validation ----------------------------------------------------------------------------*/ // Characters are broken down into ranges -- the smallest ranges that // are treated as important by either EUC or DBCS (all flavors). In // some cases, the smallest range is a single character. It saves // some space to avoid having two class tables (even though more states // are added to the state machines), so both encodings share these // tokens. // Common Tokens // #define ollow 0 // "other" legal low ascii character #define x000a 1 // 0x0a ("\n") #define x212a 2 // characters in range 0x21-0x2a #define x002b 3 // 0x2b ("+") #define x002c 4 // 0x2c (",") #define x002d 5 // 0x2d ("-") #define x002e 6 // 0x2e ("\") #define x2f39 7 // characters in range 0x2f-0x39 #define x3a3f 8 // characters in range 0x3a-0x3f #define x0040 9 // 0x40 #define x415a 10 // characters in range 0x41-0x5a #define x005b 11 // 0x5b ("[") #define x005c 12 // 0x5c ("\") #define x5d60 13 // characters in range 0x5d-0x60 #define x6177 14 // characters in range 0x61-0x77 #define x787a 15 // characters in range 0x78-0x7a #define x007b 16 // 0x7b ("{") #define x007c 17 // 0x7c ("|") #define x007d 18 // 0x7d ("}") #define x007e 19 // 0x7e ("~") #define x007f 20 // 0x7f (DEL) #define x0080 21 // 0x80 #define x818d 22 // characters in range 0x81-0x8d #define x008e 23 // 0x8e #define x008f 24 // 0x8f #define x909f 25 // characters in range 0x90-0x9f #define x00a0 26 // 0xa0 #define xa1b0 27 // characters in range 0xa1-0xb0 #define xb1df 28 // characters in range 0xb1-0xdf #define xe0ef 29 // characters in range 0xe0-0xef #define xf0fc 30 // characters in range 0xf0-0xfc #define xfdfe 31 // characters in range 0xfd-0xfe #define ateof 32 // end-of-file #define other 33 // character not covered by above tokens #define nTokens 34 // // Class table // static char _rgchCharClass[256] = // 0 1 2 3 4 5 6 7 8 9 a b c d e f { // 0 nul soh stx etx eot enq ack bel bs tab lf vt np cr so si 0 other, other, other, other, other, other, other, other, other, ollow, x000a, other, other, ollow, other, other, // 1 dle dc1 dc2 dc3 dc4 nak syn etb can em eof esc fs gs rs us 1 other, other, other, other, other, other, other, other, other, other, ollow, other, other, other, other, other, // 2 sp ! " # $ % & ' ( ) * + , - . / 2 ollow, x212a, x212a, x212a, x212a, x212a, x212a, x212a, x212a, x212a, x212a, x002b, x002c, x002d, x002e, x2f39, // 3 0 1 2 3 4 5 6 7 8 9 : ; < = > ? 3 x2f39, x2f39, x2f39, x2f39, x2f39, x2f39, x2f39, x2f39, x2f39, x2f39, x3a3f, x3a3f, x3a3f, x3a3f, x3a3f, x3a3f, // 4 @ A B C D E F G H I J K L M N O 4 x0040, x415a, x415a, x415a, x415a, x415a, x415a, x415a, x415a, x415a, x415a, x415a, x415a, x415a, x415a, x415a, // 5 P Q R S T U V W X Y Z [ \ ] ^ _ 5 x415a, x415a, x415a, x415a, x415a, x415a, x415a, x415a, x415a, x415a, x415a, x005b, x005c, x5d60, x5d60, x5d60, // 6 ` a b c d e f g h i j k l m n o 6 x5d60, x6177, x6177, x6177, x6177, x6177, x6177, x6177, x6177, x6177, x6177, x6177, x6177, x6177, x6177, x6177, // 7 p q r s t u v w x y z { | } ~ del 7 x6177, x6177, x6177, x6177, x6177, x6177, x6177, x6177, x787a, x787a, x787a, x007b, x007c, x007d, x007e, x007f, // 8 8 x0080, x818d, x818d, x818d, x818d, x818d, x818d, x818d, x818d, x818d, x818d, x818d, x818d, x818d, x008e, x008f, // 9 9 x909f, x909f, x909f, x909f, x909f, x909f, x909f, x909f, x909f, x909f, x909f, x909f, x909f, x909f, x909f, x909f, // a a x00a0, xa1b0, xa1b0, xa1b0, xa1b0, xa1b0, xa1b0, xa1b0, xa1b0, xa1b0, xa1b0, xa1b0, xa1b0, xa1b0, xa1b0, xa1b0, // b b xa1b0, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, // c c xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, // d d xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, xb1df, // e e xe0ef, xe0ef, xe0ef, xe0ef, xe0ef, xe0ef, xe0ef, xe0ef, xe0ef, xe0ef, xe0ef, xe0ef, xe0ef, xe0ef, xe0ef, xe0ef, // f f xf0fc, xf0fc, xf0fc, xf0fc, xf0fc, xf0fc, xf0fc, xf0fc, xf0fc, xf0fc, xf0fc, xf0fc, xf0fc, xfdfe, xfdfe, other, // 0 1 2 3 4 5 6 7 8 9 a b c d e f }; // Common States -- All SM's use these // #define ACC 0x4e #define ERR 0x7f // Other States -- All SM's use some of these, not all use all // #define ST0 0x00 #define ST0c 0x40 #define ST1 0x01 #define ST1c 0x41 #define ST2 0x02 #define ST2c 0x42 #define ST3 0x03 #define ST3c 0x43 #define ST4 0x04 #define ST4c 0x44 // Each state can have a corresponding counting stata i.e. stata with // with the same transitions but during which we look for special sequences. // #define FTstCounting(tst) (((tst) & 0x40) != 0) // If the state is counting (including ACC) #define TstNotCountingFromTst(tst) ((tst) & 0x3f) // Obtain the real state from the counting /*---------------------------------------------------------------------------- DBCS character sequence validation ----------------------------------------------------------------------------*/ #define nSJisStates 2 static signed char _rgchSJisNextState[nSJisStates][nTokens] = { // o x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x a o // l 0 2 0 0 0 0 2 3 0 4 0 0 5 6 7 0 0 0 0 0 0 8 0 0 9 0 a b e f f t t // l 0 1 0 0 0 0 e a 0 1 0 0 d 1 8 0 0 0 0 0 0 1 0 0 0 0 1 1 0 0 d e h // o 0 2 2 2 2 2 3 3 4 5 5 5 6 7 7 7 7 7 7 7 8 8 8 8 9 a b d e f f o e // w a a b c d e 9 f 0 a b c 0 7 a b c d e f 0 d e f f 0 0 f f c e f r //------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ // // DBCS State 0 -- start (look for legal single byte or lead byte) ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ERR, ERR, ST1, ST1, ST1, ST1, ERR, ACC, ACC, ST1, ST1, ERR, ACC, ERR, // DBCS State 1 -- saw lead byte, need legal trail byte ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ERR, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ERR, ERR, ERR, }; #define nBig5States 2 static signed char _rgchBig5NextState[nBig5States][nTokens] = { // // o x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x a o // l 0 2 0 0 0 0 2 3 0 4 0 0 5 6 7 0 0 0 0 0 0 8 0 0 9 0 a b e f f t t // l 0 1 0 0 0 0 f a 0 1 0 0 d 1 8 0 0 0 0 0 0 1 0 0 0 0 1 1 0 0 d e h // o 0 2 2 2 2 2 3 3 4 5 5 5 6 7 7 7 7 7 7 7 8 8 8 8 9 a b d e f f o e // w a a b c d e 9 f 0 a b c 0 7 a b c d e f 0 d e f f 0 0 f f c e f r //------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ // // DBCS State 0 -- start (look for legal single byte or lead byte) ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ERR, ERR, ST1, ST1, ST1, ST1, ST1, ST1, ST1, ST1, ST1, ST1, ACC, ERR, // DBCS State 1 -- saw lead byte, need legal trail byte ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ACC, ACC, ACC, ACC, ACC, ERR, ERR, }; #define nGbkWanStates 2 static signed char _rgchGbkWanNextState[nGbkWanStates][nTokens] = { // // o x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x a o // l 0 2 0 0 0 0 2 3 0 4 0 0 5 6 7 0 0 0 0 0 0 8 0 0 9 0 a b e f f t t // l 0 1 0 0 0 0 f a 0 1 0 0 d 1 8 0 0 0 0 0 0 1 0 0 0 0 1 1 0 0 d e h // o 0 2 2 2 2 2 3 3 4 5 5 5 6 7 7 7 7 7 7 7 8 8 8 8 9 a b d e f f o e // w a a b c d e 9 f 0 a b c 0 7 a b c d e f 0 d e f f 0 0 f f c e f r //----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- // // DBCS State 0 -- start (look for legal single byte or lead byte) ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ERR, ERR, ST1, ST1, ST1, ST1, ST1, ST1, ST1, ST1, ST1, ST1, ACC, ERR, // DBCS State 1 -- saw lead byte, need legal trail byte ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ERR, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ERR, ERR, }; /*---------------------------------------------------------------------------- EUC character sequence validation ----------------------------------------------------------------------------*/ #define nEucJpStates 4 static signed char _rgchEucJpNextState[nEucJpStates][nTokens] = { // // o x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x a o // l 0 2 0 0 0 0 2 3 0 4 0 0 5 6 7 0 0 0 0 0 0 8 0 0 9 0 a b e f f t t // l 0 1 0 0 0 0 f a 0 1 0 0 d 1 8 0 0 0 0 0 0 1 0 0 0 0 1 1 0 0 d e h // o 0 2 2 2 2 2 3 3 4 5 5 5 6 7 7 7 7 7 7 7 8 8 8 8 9 a b d e f f o e // w a a b c d e 9 f 0 a b c 0 7 a b c d e f 0 d e f f 0 0 f f c e f r //----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- // // EUC State 0 -- start ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ERR, ERR, ERR, ST2, ST3, ERR, ERR, ST1, ST1, ST1, ST1, ST1, ACC, ERR, // EUC State 1 -- saw a1fe, need one more a1fe ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ACC, ACC, ACC, ACC, ACC, ERR, ERR, // EUC State 2 -- saw 8e, need a1df ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ACC, ACC, ERR, ERR, ERR, ERR, ERR, // EUC State 3 -- saw 8f, need 2 a1fe ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ST1, ST1, ST1, ST1, ST1, ERR, ERR, }; #define nEucKrCnStates 2 static signed char _rgchEucKrCnNextState[nEucKrCnStates][nTokens] = { // // o x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x a o // l 0 2 0 0 0 0 2 3 0 4 0 0 5 6 7 0 0 0 0 0 0 8 0 0 9 0 a b e f f t t // l 0 1 0 0 0 0 f a 0 1 0 0 d 1 8 0 0 0 0 0 0 1 0 0 0 0 1 1 0 0 d e h // o 0 2 2 2 2 2 3 3 4 5 5 5 6 7 7 7 7 7 7 7 8 8 8 8 9 a b d e f f o e // w a a b c d e 9 f 0 a b c 0 7 a b c d e f 0 d e f f 0 0 f f c e f r //----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- // // EUC State 0 -- start ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ST1, ST1, ST1, ST1, ST1, ACC, ERR, // EUC State 1 -- saw a1fe, need one more a1fe ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ACC, ACC, ACC, ACC, ACC, ERR, ERR, }; #define nEucTwStates 4 static signed char _rgchEucTwNextState[nEucTwStates][nTokens] = { // // o x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x a o // l 0 2 0 0 0 0 2 3 0 4 0 0 5 6 7 0 0 0 0 0 0 8 0 0 9 0 a b e f f t t // l 0 1 0 0 0 0 f a 0 1 0 0 d 1 8 0 0 0 0 0 0 1 0 0 0 0 1 1 0 0 d e h // o 0 2 2 2 2 2 3 3 4 5 5 5 6 7 7 7 7 7 7 7 8 8 8 8 9 a b d e f f o e // w a a b c d e 9 f 0 a b c 0 7 a b c d e f 0 d e f f 0 0 f f c e f r //----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- // // EUC State 0 -- start ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ERR, ERR, ERR, ST2, ERR, ERR, ERR, ST1, ST1, ST1, ST1, ST1, ACC, ERR, // EUC State 1 -- saw a1fe, need one more a1fe ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ACC, ACC, ACC, ACC, ACC, ERR, ERR, // EUC State 2 -- saw 8e, need a1b0 ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ST3, ERR, ERR, ERR, ERR, ERR, ERR, // EUC State 3 -- saw 8e, a1b0; need 2 a1fe ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ST1, ST1, ST1, ST1, ST1, ERR, ERR, }; /*----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- HZ character sequence validation ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------*/ // Currently some of the rules for HZ encoding outlined above are a bit loosened up. // (e.g. the range for the first GB byte is expanded) The rules were adjusted based on real data. #define nHzStates 5 static signed char _rgchHzNextState[nHzStates][nTokens] = { // // o x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x a o // l 0 2 0 0 0 0 2 3 0 4 0 0 5 6 7 0 0 0 0 0 0 8 0 0 9 0 a b e f f t t // l 0 1 0 0 0 0 f a 0 1 0 0 d 1 8 0 0 0 0 0 0 1 0 0 0 0 1 1 0 0 d e h // o 0 2 2 2 2 2 3 3 4 5 5 5 6 7 7 7 7 7 7 7 8 8 8 8 9 a b d e f f o e // w a a b c d e 9 f 0 a b c 0 7 a b c d e f 0 d e f f 0 0 f f c e f r //----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- // // HZ State 0 -- ASCII ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ST1c, ACC, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ACC, ERR, // HZ State 1 -- saw "~," looking for "{" to make transition to GB mode ERR, ACC, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ST2c, ERR, ERR, ACC, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, // HZ State 2 -- just saw "{," expecting GB byte ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ERR, ERR, ERR, ST4c, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, // HZ State 3 -- expecting GB byte ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST4c, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, // HZ State 4 -- saw "~," looking for "}" to make transition to ASCII mode ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ST3, ACC, ST3, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, }; /*----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- UTF-7 character sequence validation ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------*/ #define nUtf7States 3 static signed char _rgchUtf7NextState[nUtf7States][nTokens] = { // // o x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x a o // l 0 2 0 0 0 0 2 3 0 4 0 0 5 6 7 0 0 0 0 0 0 8 0 0 9 0 a b e f f t t // l 0 1 0 0 0 0 f a 0 1 0 0 d 1 8 0 0 0 0 0 0 1 0 0 0 0 1 1 0 0 d e h // o 0 2 2 2 2 2 3 3 4 5 5 5 6 7 7 7 7 7 7 7 8 8 8 8 9 a b d e f f o e // w a a b c d e 9 f 0 a b c 0 7 a b c d e f 0 d e f f 0 0 f f c e f r //----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- // // UTF7 State 0 -- Direct/optionally direct ACSII mode, state transition can happen on "+" ACC, ACC, ACC, ST1c, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ACC, ERR, ACC, ACC, ACC, ACC, ACC, ACC, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ACC, ERR, // UTF7 State 1 -- Expecting first character from Modified Base64 alphabet ERR, ERR, ERR, ST2, ERR, ACC, ERR, ST2, ERR, ERR, ST2, ERR, ERR, ERR, ST2, ST2, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, // UTF7 State 2 -- Modified Base64 alphabet mode, can be exited with "-" or any control character. ACC, ACC, ERR, ST2, ERR, ACC, ERR, ST2, ERR, ERR, ST2, ERR, ERR, ERR, ST2, ST2, ERR, ERR, ERR, ERR, ACC, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ACC, ERR, }; /*---------------------------------------------------------------------------- UTF-8 character sequence validation ----------------------------------------------------------------------------*/ static int _nUtf8Tb = 0; #define BIT7(a) ((a) & 0x80) #define BIT6(a) ((a) & 0x40) /* N U T F 8 */ /*---------------------------------------------------------------------------- %%Function: _NUtf8 %%Contact: jpick UTF-8 doesn't require a state table for validation, just a count of the number of expected trail bytes. See utf8lex.c for an explanation of this code. ----------------------------------------------------------------------------*/ static int __inline NUtf8(UCHAR uch, BOOL fEoi) { // BIT7(uch) == 0 implies single ASCII byte. // BIT6(uch) == 0 implies one of n trail bytes. // Otherwise, lead byte, with number of bits set // up to first 0 equal to the total number bytes // in the sequence. // // REVIEW: _nUtf8Tb *is* really the state of this // validator -- use nState in structure? // if (fEoi && (_nUtf8Tb != 0)) { return 0; // unexpected end-of-input } else if (BIT7(uch) == 0) { if (_nUtf8Tb != 0) // unexpected single byte return 0; return 1; } else if (BIT6(uch) == 0) { if (_nUtf8Tb == 0) // unexpected trail byte return 0; if ((--_nUtf8Tb) == 0) return 1; } else { if (_nUtf8Tb != 0) // unexpected lead byte return 0; while (BIT7(uch) != 0) { uch <<= 1; _nUtf8Tb++; } _nUtf8Tb--; // don't count lead byte } return -1; } /*---------------------------------------------------------------------------- Character Mapping Defs ----------------------------------------------------------------------------*/ // If caller wants us to check characters as part of validation // typedef BOOL (*PFNCHECKCHAR)(ICET icetIn); #define cchMaxBuff 5 typedef struct _cc { int nCp; // code page int cchBuff; // fill count of character buffer PFNCHECKCHAR pfnCheckChar; // character check routine char rgchBuff[cchMaxBuff]; // character buffer } CC; // Character validation prototypes // static BOOL _FDbcsCheckChar(ICET icetIn); // DBCS character checker structures // // Big5 static CC _ccBig5 = { nCpTaiwan, 0, _FDbcsCheckChar, }; // Gbk static CC _ccGbk = { nCpChina, 0, _FDbcsCheckChar, }; // ShiftJis static CC _ccSJis = { nCpJapan, 0, _FDbcsCheckChar, }; // Wansung static CC _ccWan = { nCpKorea, 0, _FDbcsCheckChar, }; // Character checker structures just used as buffers. // // Euc-Jp static CC _ccEucJp = { 0, 0, 0, }; // Hz static CC _ccHz = { 0, 0, 0, }; // Utf7 static CC _ccUtf7 = { 0, 0, 0, }; /*---------------------------------------------------------------------------- Character Occurrence Counters ----------------------------------------------------------------------------*/ // If calling app wants us to track occurrences of common character // sequences during validation (used only by auto-detection, so far). // typedef struct _coce { int cHits; short cwch; WCHAR rgwch[2]; } COCE; typedef struct _coc { BOOL fMatching; short nCoceCurr; short nCoceIndex; int ccoce; COCE *rgcoce; } COC; // Big5 // static COCE _rgcoceBig5[] = { {0, 2, {(WCHAR)0xa7da, (WCHAR)0xadcc},}, // "wo men" {0, 2, {(WCHAR)0xa8e4, (WCHAR)0xb9ea},}, // "qi shi" {0, 2, {(WCHAR)0xa65d, (WCHAR)0xacb0},}, // "yin wei" {0, 2, {(WCHAR)0xb8ea, (WCHAR)0xb054},}, // "zi xun" {0, 2, {(WCHAR)0xb971, (WCHAR)0xb8a3},}, // "diam nao" {0, 2, {(WCHAR)0xbaf4, (WCHAR)0xb8f4},}, // "wang lu" {0, 2, {(WCHAR)0xbd75, (WCHAR)0xa457},}, // "xian shang" {0, 2, {(WCHAR)0xc577, (WCHAR)0xaaef},}, // "huan ying" {0, 2, {(WCHAR)0xa477, (WCHAR)0xb867},}, // "yi jing" }; static COC _cocBig5 = { fFalse, // fMatching 0, // nCoceCurr 0, // nCoceIndex sizeof(_rgcoceBig5) / sizeof(_rgcoceBig5[0]), // ccoce _rgcoceBig5, // rgcoce }; // Euc-Cn // static COCE _rgcoceEucCn[] = { {0, 2, {(WCHAR)0xcbfb, (WCHAR)0xc3c7},}, // "ta men" {0, 2, {(WCHAR)0xced2, (WCHAR)0xc3c7},}, // "wo men" {0, 2, {(WCHAR)0xd2f2, (WCHAR)0xb4cb},}, // "yin ci" {0, 2, {(WCHAR)0xcab2, (WCHAR)0xc3b4},}, // "shen mo" {0, 2, {(WCHAR)0xc8e7, (WCHAR)0xb9fb},}, // "ru guo" {0, 2, {(WCHAR)0xd2f2, (WCHAR)0xceaa},}, // "yin wei" {0, 2, {(WCHAR)0xcbf9, (WCHAR)0xd2d4},}, // "suo yi" {0, 2, {(WCHAR)0xbbb6, (WCHAR)0xd3ad},}, // "huan ying" {0, 2, {(WCHAR)0xcdf8, (WCHAR)0xc2e7},}, // "wang luo" {0, 2, {(WCHAR)0xd0c5, (WCHAR)0xcfa2},}, // "xin xi" {0, 2, {(WCHAR)0xbcc6, (WCHAR)0xcbe3},}, // "ji guan" }; static COC _cocEucCn = { fFalse, // fMatching 0, // nCoceCurr 0, // nCoceIndex sizeof(_rgcoceEucCn) / sizeof(_rgcoceEucCn[0]), // ccoce _rgcoceEucCn, // rgcoce }; // Euc-Kr // static COCE _rgcoceEucKr[] = { {0, 2, {(WCHAR)0xb0a1, (WCHAR)0x0020},}, {0, 2, {(WCHAR)0xb0a1, (WCHAR)0xa1a1},}, {0, 2, {(WCHAR)0xb4c2, (WCHAR)0x0020},}, {0, 2, {(WCHAR)0xb4c2, (WCHAR)0xa1a1},}, {0, 2, {(WCHAR)0xb4d9, (WCHAR)0x002e},}, {0, 2, {(WCHAR)0xb4d9, (WCHAR)0xa3ae},}, {0, 2, {(WCHAR)0xb8a6, (WCHAR)0x0020},}, {0, 2, {(WCHAR)0xb8a6, (WCHAR)0xa1a1},}, {0, 2, {(WCHAR)0xc0ba, (WCHAR)0x0020},}, {0, 2, {(WCHAR)0xc0ba, (WCHAR)0xa1a1},}, {0, 2, {(WCHAR)0xc0bb, (WCHAR)0x0020},}, {0, 2, {(WCHAR)0xc0bb, (WCHAR)0xa1a1},}, {0, 2, {(WCHAR)0xc0cc, (WCHAR)0x0020},}, {0, 2, {(WCHAR)0xc0cc, (WCHAR)0xa1a1},}, }; static COC _cocEucKr = { fFalse, // fMatching 0, // nCoceCurr 0, // nCoceIndex sizeof(_rgcoceEucKr) / sizeof(_rgcoceEucKr[0]), // ccoce _rgcoceEucKr, // rgcoce }; // EUC-Jp // static COCE _rgcoceEucJp[] = { {0, 2, {(WCHAR)0xa4c7, (WCHAR)0xa4b9},}, // "de su" {0, 2, {(WCHAR)0xa4c0, (WCHAR)0xa1a3},}, // "da ." {0, 2, {(WCHAR)0xa4a4, (WCHAR)0xa4eb},}, // "i ru" {0, 2, {(WCHAR)0xa4de, (WCHAR)0xa4b9},}, // "ma su" {0, 2, {(WCHAR)0xa4b7, (WCHAR)0xa4bf},}, // "shi ta" {0, 2, {(WCHAR)0xa4b9, (WCHAR)0xa4eb},}, // "su ru" {0, 2, {(WCHAR)0xa4bf, (WCHAR)0xa1a3},}, // "ta ." {0, 2, {(WCHAR)0xa4eb, (WCHAR)0xa1a3},}, // "ru ." }; static COC _cocEucJp = { fFalse, // fMatching 0, // nCoceCurr 0, // nCoceIndex sizeof(_rgcoceEucJp) / sizeof(_rgcoceEucJp[0]), // ccoce _rgcoceEucJp, // rgcoce }; // GBK // static COCE _rgcoceGbk[] = { {0, 2, {(WCHAR)0xcbfb, (WCHAR)0xc3c7},}, // "ta men" {0, 2, {(WCHAR)0xced2, (WCHAR)0xc3c7},}, // "wo men" {0, 2, {(WCHAR)0xd2f2, (WCHAR)0xb4cb},}, // "yin ci" {0, 2, {(WCHAR)0xcab2, (WCHAR)0xc3b4},}, // "shen mo" {0, 2, {(WCHAR)0xc8e7, (WCHAR)0xb9fb},}, // "ru guo" {0, 2, {(WCHAR)0xd2f2, (WCHAR)0xceaa},}, // "yin wei" {0, 2, {(WCHAR)0xcbf9, (WCHAR)0xd2d4},}, // "suo yi" {0, 2, {(WCHAR)0xbbb6, (WCHAR)0xd3ad},}, // "huan ying" {0, 2, {(WCHAR)0xcdf8, (WCHAR)0xc2e7},}, // "wang luo" {0, 2, {(WCHAR)0xd0c5, (WCHAR)0xcfa2},}, // "xin xi" {0, 2, {(WCHAR)0xbcc6, (WCHAR)0xcbe3},}, // "ji guan" }; static COC _cocGbk = { fFalse, // fMatching 0, // nCoceCurr 0, // nCoceIndex sizeof(_rgcoceGbk) / sizeof(_rgcoceGbk[0]), // ccoce _rgcoceGbk, // rgcoce }; // Shift-JIS // static COCE _rgcoceSJis[] = { {0, 2, {(WCHAR)0x82c5, (WCHAR)0x82b7},}, // "de su" {0, 2, {(WCHAR)0x82be, (WCHAR)0x8142},}, // "da ." {0, 2, {(WCHAR)0x82a2, (WCHAR)0x82e9},}, // "i ru" {0, 2, {(WCHAR)0x82dc, (WCHAR)0x82b7},}, // "ma su" {0, 2, {(WCHAR)0x82b5, (WCHAR)0x82bd},}, // "shi ta" {0, 2, {(WCHAR)0x82b7, (WCHAR)0x82e9},}, // "su ru" {0, 2, {(WCHAR)0x82bd, (WCHAR)0x8142},}, // "ta ." {0, 2, {(WCHAR)0x82e9, (WCHAR)0x8142},}, // "ru ." }; static COC _cocSJis = { fFalse, // fMatching 0, // nCoceCurr 0, // nCoceIndex sizeof(_rgcoceSJis) / sizeof(_rgcoceSJis[0]), // ccoce _rgcoceSJis, // rgcoce }; // Wansung // // REVIEW: bug (1/2 this table is being ignored) // static COCE _rgcoceWan[] = { {0, 2, {(WCHAR)0xb0a1, (WCHAR)0x0020},}, {0, 2, {(WCHAR)0xb0a1, (WCHAR)0xa1a1},}, {0, 2, {(WCHAR)0xb4c2, (WCHAR)0x0020},}, {0, 2, {(WCHAR)0xb4c2, (WCHAR)0xa1a1},}, {0, 2, {(WCHAR)0xb4d9, (WCHAR)0x002e},}, {0, 2, {(WCHAR)0xb4d9, (WCHAR)0xa3ae},}, {0, 2, {(WCHAR)0xb8a6, (WCHAR)0x0020},}, {0, 2, {(WCHAR)0xb8a6, (WCHAR)0xa1a1},}, {0, 2, {(WCHAR)0xc0ba, (WCHAR)0x0020},}, {0, 2, {(WCHAR)0xc0ba, (WCHAR)0xa1a1},}, {0, 2, {(WCHAR)0xc0bb, (WCHAR)0x0020},}, {0, 2, {(WCHAR)0xc0bb, (WCHAR)0xa1a1},}, {0, 2, {(WCHAR)0xc0cc, (WCHAR)0x0020},}, {0, 2, {(WCHAR)0xc0cc, (WCHAR)0xa1a1},}, }; static COC _cocWan = { fFalse, // fMatching 0, // nCoceCurr 0, // nCoceIndex sizeof(_rgcoceWan) / sizeof(_rgcoceWan[0]), // ccoce _rgcoceWan, // rgcoce }; // Hz // static COCE _rgcoceHz[] = { {0, 2, {(WCHAR)0x007e, (WCHAR)0x007b},}, // ~{ {0, 2, {(WCHAR)0x007e, (WCHAR)0x007d},}, // ~} }; static COC _cocHz = { fFalse, // fMatching 0, // nCoceCurr 0, // nCoceIndex sizeof(_rgcoceHz) / sizeof(_rgcoceHz[0]), // ccoce _rgcoceHz, // rgcoce }; // Utf7 // static COCE _rgcoceUtf7[] = { {0, 2, {(WCHAR)0x002b, (WCHAR)0x002d},}, // +- }; static COC _cocUtf7 = { fFalse, // fMatching 0, // nCoceCurr 0, // nCoceIndex sizeof(_rgcoceUtf7) / sizeof(_rgcoceUtf7[0]), // ccoce _rgcoceUtf7, // rgcoce }; // Character counter prototype. // static void _CountChars(ICET icetIn); /*---------------------------------------------------------------------------- Main Definitions ----------------------------------------------------------------------------*/ // Structure to keep state, state machine and other associated // information for a given character set "parse stream." // typedef struct _vr { BOOL fInUse; DWORD dwFlags; int nState; CC *ccCheck; signed char (*rgchNextState)[nTokens]; } VR; // Array of validation records. We allow multiple, active parse // streams for auto-detect -- this way, it can concurrently keep // a parse stream for each encoding type, without needing to read // its input multiple times. // static VR _mpicetvr[icetCount] = { {fTrue, 0, ST0, 0, _rgchEucKrCnNextState,}, // icetEucCn {fTrue, 0, ST0, &_ccEucJp, _rgchEucJpNextState,}, // icetEucJp {fTrue, 0, ST0, 0, _rgchEucKrCnNextState,}, // icetEucKr {fTrue, 0, ST0, 0, _rgchEucTwNextState,}, // icetEucTw {fFalse, 0, ST0, 0, 0,}, // icetIso2022Cn {fFalse, 0, ST0, 0, 0,}, // icetIso2022Jp {fFalse, 0, ST0, 0, 0,}, // icetIso2022Kr {fFalse, 0, ST0, 0, 0,}, // icetIso2022Tw {fTrue, 0, ST0, &_ccBig5, _rgchBig5NextState,}, // icetBig5 {fTrue, 0, ST0, &_ccGbk, _rgchGbkWanNextState,}, // icetGbk {fTrue, 0, ST0, &_ccHz, _rgchHzNextState,}, // icetHz {fTrue, 0, ST0, &_ccSJis, _rgchSJisNextState,}, // icetShiftJis {fTrue, 0, ST0, &_ccWan, _rgchGbkWanNextState,}, // icetWansung {fTrue, 0, ST0, &_ccUtf7, _rgchUtf7NextState,}, // icetUtf7 {fTrue, 0, ST0, 0, 0,}, // icetUtf8 }; // Array of character sequence counters, one per encoding type. // static COC *_mpicetlpcoc[icetCount] = { &_cocEucCn, // icetEucCn &_cocEucJp, // icetEucJp &_cocEucKr, // icetEucKr 0, // icetEucTw 0, // icetIso2022Cn 0, // icetIso2022Jp 0, // icetIso2022Kr 0, // icetIso2022Tw &_cocBig5, // icetBig5 &_cocGbk, // icetGbk &_cocHz, // icetHz &_cocSJis, // icetShiftJis &_cocWan, // icetWansung &_cocUtf7, // icetUtf7 0, // icetUtf8 }; /* V A L I D A T E I N I T */ /*---------------------------------------------------------------------------- %%Function: ValidateInit %%Contact: jpick Initialize the state machine for the given character set (set its state to ST0 (the start state) and store its parsing options). ----------------------------------------------------------------------------*/ void ValidateInit(ICET icetIn, DWORD dwFlags) { // Initialize the character occurrence counter, if caller wants // us to count common character sequences (auto-detect, only, // for now). Turn off the count-common-chars flag if we're not // set up to count sequences (meaning we don't have a set of // common characters for this encoding type or have no place // to buffer them). // if (dwFlags & grfCountCommonChars) { if ((_mpicetlpcoc[icetIn]) && (_mpicetvr[icetIn].ccCheck)) { int i; for (i = 0; i < _mpicetlpcoc[icetIn]->ccoce; i++) _mpicetlpcoc[icetIn]->rgcoce[i].cHits = 0; _mpicetlpcoc[icetIn]->fMatching = fFalse; } else { dwFlags &= ~grfCountCommonChars; } } // If validation not supported for the encoding type, there's // nothing else for us to do here. // if (!_mpicetvr[icetIn].fInUse) return; _mpicetvr[icetIn].nState = ST0; // Can't do character mapping validation without character // checker information. (If we do have the character checker, // initialize its buffer length to 0). // if (_mpicetvr[icetIn].ccCheck) _mpicetvr[icetIn].ccCheck->cchBuff = 0; else dwFlags &= ~grfValidateCharMapping; // It's also impossible without a valid code page. // if ((dwFlags & grfValidateCharMapping) && !IsValidCodePage(_mpicetvr[icetIn].ccCheck->nCp)) dwFlags &= ~grfValidateCharMapping; _mpicetvr[icetIn].dwFlags = dwFlags; if (icetIn == icetUtf8) _nUtf8Tb = 0; } /* V A L I D A T E R E S E T A L L*/ /*---------------------------------------------------------------------------- %%Function: ValidateInitAll %%Contact: jpick Initialize the state machines for all character sets (set their states to ST0 (the start state) and store their parsing options). ----------------------------------------------------------------------------*/ void ValidateInitAll(DWORD dwFlags) { int i; for (i = 0 ; i < icetCount; i++) { if (!_mpicetvr[i].fInUse) continue; ValidateInit((ICET)i, dwFlags); } } /* V A L I D A T E R E S E T */ /*---------------------------------------------------------------------------- %%Function: ValidateReset %%Contact: jpick Reset the state machine for the given character set (set its state to ST0 (the start state)). ----------------------------------------------------------------------------*/ void ValidateReset(ICET icetIn) { // Initialize the character occurrence counter, if caller wants // us to count common character sequences (auto-detect, only, // for now). We're guaranteed to have the structures if the // flag is set by ValidateInit(), above. // if (_mpicetvr[icetIn].dwFlags & grfCountCommonChars) { int i; for (i = 0; i < _mpicetlpcoc[icetIn]->ccoce; i++) _mpicetlpcoc[icetIn]->rgcoce[i].cHits = 0; _mpicetlpcoc[icetIn]->fMatching = fFalse; } // If validation not supported for the encoding type, there's // nothing else for us to do here. // if (!_mpicetvr[icetIn].fInUse) return; _mpicetvr[icetIn].nState = ST0; if (_mpicetvr[icetIn].ccCheck) _mpicetvr[icetIn].ccCheck->cchBuff = 0; if (icetIn == icetUtf8) _nUtf8Tb = 0; } /* V A L I D A T E R E S E T A L L */ /*---------------------------------------------------------------------------- %%Function: ValidateResetAll %%Contact: jpick Reset the state machines for all character sets (set their states to ST0 (the start state)). ----------------------------------------------------------------------------*/ void ValidateResetAll(void) { int i; for (i=0 ; i < icetCount; i++) { if (!_mpicetvr[i].fInUse) continue; ValidateReset((ICET)i); } } /* N V A L I D A T E U C H */ /*---------------------------------------------------------------------------- %%Function: NValidateUch %%Contact: jpick Single step parser, takes one transition through the state table for the given character set. Current state is kept for each character set's parse stream. Routine returns -1 if it does not reach a final state on this transition; 0 if transitioned to ERR(or) and 1 if transtioned to ACC(ept). If final state is ACC(ept), machine reset to ST0 (start state). (i.e., there's no need to manually reset on ACC(ept)). Routine is also a convenient collection point for certain statistics (currently only the counting of occurrences of common character sequences (defined for character sets, above)). ----------------------------------------------------------------------------*/ int NValidateUch(ICET icetIn, UCHAR uch, BOOL fEoi) { int nToken; int nPrevState; int rc = -1; // If not validating this icet, nothing to do (so say // we accept the character). // if (!_mpicetvr[icetIn].fInUse) return 1; if (_mpicetvr[icetIn].nState == ERR) return 0; // Ignore all zeros in the detection file. if (!uch && !fEoi) { goto _LRet; } // Hack -- want to validate UTF-8, but don't need a state // table to do so. Treat as special case here and return. // if (icetIn == icetUtf8) { if ((rc = NUtf8(uch, fEoi)) == 0) _mpicetvr[icetIn].nState = ERR; return rc; } // Classify the character... // nPrevState = _mpicetvr[icetIn].nState; nToken = fEoi ? ateof : _rgchCharClass[uch]; // First obtain a real number for a state based on the counting state... // Then do the transition... // _mpicetvr[icetIn].nState = (_mpicetvr[icetIn].rgchNextState)[TstNotCountingFromTst(_mpicetvr[icetIn].nState)][nToken]; #if 0 if (_mpicetvr[icetIn].nState == ERR) printf("Character 0x%.2x; Going from state %.2x to state %.2x\n", uch, nPrevState, _mpicetvr[icetIn].nState); #endif // If we're in an error state or have seen end-of-input, return. // if ((_mpicetvr[icetIn].nState == ERR) || (nToken == ateof)) goto _LRet; // Are we to do character mapping validation? (If this flag // is set, we're guaranteed to have a character checker // structure). How about character occurrence counting? // (This also guarantees us a character checker structure). // if (!(_mpicetvr[icetIn].dwFlags & grfValidateCharMapping) && !(_mpicetvr[icetIn].dwFlags & grfCountCommonChars)) { goto _LRet; } // Buffer the current character (trusting that we'll never get // more than the max amount -- present tables enforce this) // (if it's Utf7 or Hz, buffer only if we are in the counting state // if (FTstCounting(_mpicetvr[icetIn].nState) || (icetIn != icetHz && icetIn != icetUtf7)) _mpicetvr[icetIn].ccCheck->rgchBuff[_mpicetvr[icetIn].ccCheck->cchBuff++] = uch; // Return if we are not in the counting state // if (!(FTstCounting(_mpicetvr[icetIn].nState))) goto _LRet; // Call the character checker, if we have one. // if (_mpicetvr[icetIn].dwFlags & grfValidateCharMapping) { if (_mpicetvr[icetIn].ccCheck->pfnCheckChar && !(_mpicetvr[icetIn].ccCheck->pfnCheckChar)(icetIn)) { _mpicetvr[icetIn].nState = ERR; goto _LRet; } } // If we're counting common characters, do so now. // if (_mpicetvr[icetIn].dwFlags & grfCountCommonChars) _CountChars(icetIn); // Reset the character checker/counter buffer. // _mpicetvr[icetIn].ccCheck->cchBuff = 0; _LRet: // Return the appropriate code. // switch (_mpicetvr[icetIn].nState) { case ERR: return 0; case ACC: _mpicetvr[icetIn].nState = ST0; // Reset return 1; default: return -1; // need more data } } /* F V A L I D A T E C H A R C O U N T */ /*---------------------------------------------------------------------------- %%Function: FValidateCharCount %%Contact: jpick Return the number of matched special character sequences for the given character set. If we're not keeping track of these sequences for the character set, either because we don't have the necessary static data or because the flag wasn't set by the calling routine, return fFalse. Otherwise, return the count in *lpcMatch and return fTrue; (We track the counts separately for each sequence, just in case we want to weight them differently in the future. Return the total, here). ----------------------------------------------------------------------------*/ BOOL FValidateCharCount(ICET icetIn, int *lpcMatch) { int i; COC *lpcoc = _mpicetlpcoc[icetIn]; VR *lpvr = &_mpicetvr[icetIn]; if (!lpcoc || !lpvr->fInUse || !(lpvr->dwFlags & grfCountCommonChars)) return fFalse; for (i = 0, *lpcMatch = 0; i < lpcoc->ccoce; i++) *lpcMatch += lpcoc->rgcoce[i].cHits; return fTrue; } /* _ C O U N T C H A R S */ /*---------------------------------------------------------------------------- %%Function: _CountChars %%Contact: jpick We've just completed a legal character for the given character set. Match it against the set of special character sequences for the character set, if we have them. Update match counts and current match indices (since sequences can span multiple legal characters) as needed. ----------------------------------------------------------------------------*/ static void _CountChars(ICET icetIn) { WCHAR wch; int i; BOOL fFound; // Anything to do? // if (!_mpicetlpcoc[icetIn] || !_mpicetvr[icetIn].ccCheck) return; // Build the WCHAR. // switch (_mpicetvr[icetIn].ccCheck->cchBuff) { case 1: wch = WchFromUchUch(0, _mpicetvr[icetIn].ccCheck->rgchBuff[0]); break; case 2: wch = WchFromUchUch(_mpicetvr[icetIn].ccCheck->rgchBuff[0], _mpicetvr[icetIn].ccCheck->rgchBuff[1]); break; case 3: wch = WchFromUchUch(_mpicetvr[icetIn].ccCheck->rgchBuff[1], _mpicetvr[icetIn].ccCheck->rgchBuff[2]); break; case 4: wch = WchFromUchUch(_mpicetvr[icetIn].ccCheck->rgchBuff[2], _mpicetvr[icetIn].ccCheck->rgchBuff[3]); break; default: return; } // Are we currently working on matching a sequence? // if ((_mpicetlpcoc[icetIn]->fMatching) && (wch == _mpicetlpcoc[icetIn]->rgcoce[_mpicetlpcoc[icetIn]->nCoceCurr].rgwch[_mpicetlpcoc[icetIn]->nCoceIndex])) { // Did we just match the entire sequence? If so, increment the // hit count and reset. // if (++_mpicetlpcoc[icetIn]->nCoceIndex >= _mpicetlpcoc[icetIn]->rgcoce[_mpicetlpcoc[icetIn]->nCoceCurr].cwch) { ++_mpicetlpcoc[icetIn]->rgcoce[_mpicetlpcoc[icetIn]->nCoceCurr].cHits; _mpicetlpcoc[icetIn]->fMatching = fFalse; } // All done. // return; } // If we need to start matching again (either because we're not // currently in a sequence or because a 2nd or later character // didn't match), try the current character as a lead character. // // REVIEW: wrong for sequences longer than 2 wchars. // for (i = 0, fFound = fFalse; (!fFound && (i < _mpicetlpcoc[icetIn]->ccoce)); i++) { if (wch == _mpicetlpcoc[icetIn]->rgcoce[i].rgwch[0]) fFound = fTrue; } // Any luck? // if (!fFound) { _mpicetlpcoc[icetIn]->fMatching = fFalse; return; } // Store the matching state. // _mpicetlpcoc[icetIn]->fMatching = fTrue; _mpicetlpcoc[icetIn]->nCoceCurr = i - 1; _mpicetlpcoc[icetIn]->nCoceIndex = 1; // where to look next } /* _ D B C S C H E C K C H A R */ /*---------------------------------------------------------------------------- %%Function: _DbcsCheckChar %%Contact: jpick Character validator for DBCS formats. Attempts to round-trip a legal multi-byte sequence to ensure that its valid for the given character set. REVIEW: Slow, slow, slow -- do we really gain anything from the round-trip check, or is conversion *to* Unicode a sufficient test? ----------------------------------------------------------------------------*/ static WCHAR _rgwBuff[10]; static UCHAR _rgchBuff[30]; static BOOL _FDbcsCheckChar(ICET icetIn) { int cCvt; // skip 1 byte characters, mostly uninteresting (Shift-Jis ??). // if (_mpicetvr[icetIn].ccCheck->cchBuff == 1) return fTrue; if (!(cCvt = MultiByteToWideChar(_mpicetvr[icetIn].ccCheck->nCp, MB_ERR_INVALID_CHARS, _mpicetvr[icetIn].ccCheck->rgchBuff, _mpicetvr[icetIn].ccCheck->cchBuff, _rgwBuff, 10))) { if (GetLastError() == ERROR_NO_UNICODE_TRANSLATION) return fFalse; } return fTrue; // probably not always right }