From: Zhiyi Zhang <zzhang@codeweavers.com> Fix The Heroic Legend of America: Second Revolutionary War (SteamID: 2340720) failing to start. The game has a line of `var i、unit, dataList;` in one of its .js file. The `、`(\u3001, category Po) is a punctuation mark used in CJK languages. This patch adds support for Unicode characters in identifiers according ECMA-262 3rd edition 7.6. <ZWNJ> (\u200c) and <ZWJ> (\u200d) are not allowed so it's not using ES5. However, other Unicode categories other than those specified in the spec are supported in IdentifierPart according tests so it's not fully ECMA-262 compliant. This non-compliant behavior is needed for the game, as demonstrated by the \u3001 in the Po category. Note that make_unicode uses Unicode 17.0.0 to generate the allowed Unicode characters ranges and some characters got moved to different categories compared to Unicode 2.1 used by ECMA-262. However, I don't think it's worth adding another Unicode data file. We can adjust it when it's needed by real-world applications. Conditional compilation identifiers also support some Unicode characters. For example, the following code is valid. ```js @cc_on @set @π = 3.14; WScript.Echo(@π); ``` However, conditional compilation identifiers do not support Unicode escape sequences. So the previous is_identifier_first_char() and is_identifier_char() were wrong as well. With that said, I doubt any application would use Unicode escape sequences for conditional compilation identifiers. So let's use the same parser helpers for standard identifiers for now. --- dlls/jscript/Makefile.in | 1 + dlls/jscript/identifier_letters.c | 195 ++++++++++++++++++++++++++++++ dlls/jscript/identifier_letters.h | 28 +++++ dlls/jscript/lex.c | 143 +++++++++++++++++++--- tools/make_unicode | 107 ++++++++++++++++ 5 files changed, 454 insertions(+), 20 deletions(-) create mode 100644 dlls/jscript/identifier_letters.c create mode 100644 dlls/jscript/identifier_letters.h diff --git a/dlls/jscript/Makefile.in b/dlls/jscript/Makefile.in index ef1a809951c..15ca01adfa0 100644 --- a/dlls/jscript/Makefile.in +++ b/dlls/jscript/Makefile.in @@ -19,6 +19,7 @@ SOURCES = \ error.c \ function.c \ global.c \ + identifier_letters.c \ jscript.c \ jscript.rc \ jscript_classes.idl \ diff --git a/dlls/jscript/identifier_letters.c b/dlls/jscript/identifier_letters.c new file mode 100644 index 00000000000..d05316eaaca --- /dev/null +++ b/dlls/jscript/identifier_letters.c @@ -0,0 +1,195 @@ +/* Unicode letter ranges allowed in jscript identifiers */ +/* Automatically generated; DO NOT EDIT!! */ + +#include "identifier_letters.h" + +const unicode_range_t identifier_start_letters[381] = +{ + {0x0041, 0x005a}, {0x0061, 0x007a}, {0x00aa, 0x00aa}, {0x00b5, 0x00b5}, + {0x00ba, 0x00ba}, {0x00c0, 0x00d6}, {0x00d8, 0x00f6}, {0x00f8, 0x02c1}, + {0x02c6, 0x02d1}, {0x02e0, 0x02e4}, {0x02ec, 0x02ec}, {0x02ee, 0x02ee}, + {0x0370, 0x0374}, {0x0376, 0x0377}, {0x037a, 0x037d}, {0x037f, 0x037f}, + {0x0386, 0x0386}, {0x0388, 0x038a}, {0x038c, 0x038c}, {0x038e, 0x03a1}, + {0x03a3, 0x03f5}, {0x03f7, 0x0481}, {0x048a, 0x052f}, {0x0531, 0x0556}, + {0x0559, 0x0559}, {0x0560, 0x0588}, {0x05d0, 0x05ea}, {0x05ef, 0x05f2}, + {0x0620, 0x064a}, {0x066e, 0x066f}, {0x0671, 0x06d3}, {0x06d5, 0x06d5}, + {0x06e5, 0x06e6}, {0x06ee, 0x06ef}, {0x06fa, 0x06fc}, {0x06ff, 0x06ff}, + {0x0710, 0x0710}, {0x0712, 0x072f}, {0x074d, 0x07a5}, {0x07b1, 0x07b1}, + {0x07ca, 0x07ea}, {0x07f4, 0x07f5}, {0x07fa, 0x07fa}, {0x0800, 0x0815}, + {0x081a, 0x081a}, {0x0824, 0x0824}, {0x0828, 0x0828}, {0x0840, 0x0858}, + {0x0860, 0x086a}, {0x0870, 0x0887}, {0x0889, 0x088f}, {0x08a0, 0x08c9}, + {0x0904, 0x0939}, {0x093d, 0x093d}, {0x0950, 0x0950}, {0x0958, 0x0961}, + {0x0971, 0x0980}, {0x0985, 0x098c}, {0x098f, 0x0990}, {0x0993, 0x09a8}, + {0x09aa, 0x09b0}, {0x09b2, 0x09b2}, {0x09b6, 0x09b9}, {0x09bd, 0x09bd}, + {0x09ce, 0x09ce}, {0x09dc, 0x09dd}, {0x09df, 0x09e1}, {0x09f0, 0x09f1}, + {0x09fc, 0x09fc}, {0x0a05, 0x0a0a}, {0x0a0f, 0x0a10}, {0x0a13, 0x0a28}, + {0x0a2a, 0x0a30}, {0x0a32, 0x0a33}, {0x0a35, 0x0a36}, {0x0a38, 0x0a39}, + {0x0a59, 0x0a5c}, {0x0a5e, 0x0a5e}, {0x0a72, 0x0a74}, {0x0a85, 0x0a8d}, + {0x0a8f, 0x0a91}, {0x0a93, 0x0aa8}, {0x0aaa, 0x0ab0}, {0x0ab2, 0x0ab3}, + {0x0ab5, 0x0ab9}, {0x0abd, 0x0abd}, {0x0ad0, 0x0ad0}, {0x0ae0, 0x0ae1}, + {0x0af9, 0x0af9}, {0x0b05, 0x0b0c}, {0x0b0f, 0x0b10}, {0x0b13, 0x0b28}, + {0x0b2a, 0x0b30}, {0x0b32, 0x0b33}, {0x0b35, 0x0b39}, {0x0b3d, 0x0b3d}, + {0x0b5c, 0x0b5d}, {0x0b5f, 0x0b61}, {0x0b71, 0x0b71}, {0x0b83, 0x0b83}, + {0x0b85, 0x0b8a}, {0x0b8e, 0x0b90}, {0x0b92, 0x0b95}, {0x0b99, 0x0b9a}, + {0x0b9c, 0x0b9c}, {0x0b9e, 0x0b9f}, {0x0ba3, 0x0ba4}, {0x0ba8, 0x0baa}, + {0x0bae, 0x0bb9}, {0x0bd0, 0x0bd0}, {0x0c05, 0x0c0c}, {0x0c0e, 0x0c10}, + {0x0c12, 0x0c28}, {0x0c2a, 0x0c39}, {0x0c3d, 0x0c3d}, {0x0c58, 0x0c5a}, + {0x0c5c, 0x0c5d}, {0x0c60, 0x0c61}, {0x0c80, 0x0c80}, {0x0c85, 0x0c8c}, + {0x0c8e, 0x0c90}, {0x0c92, 0x0ca8}, {0x0caa, 0x0cb3}, {0x0cb5, 0x0cb9}, + {0x0cbd, 0x0cbd}, {0x0cdc, 0x0cde}, {0x0ce0, 0x0ce1}, {0x0cf1, 0x0cf2}, + {0x0d04, 0x0d0c}, {0x0d0e, 0x0d10}, {0x0d12, 0x0d3a}, {0x0d3d, 0x0d3d}, + {0x0d4e, 0x0d4e}, {0x0d54, 0x0d56}, {0x0d5f, 0x0d61}, {0x0d7a, 0x0d7f}, + {0x0d85, 0x0d96}, {0x0d9a, 0x0db1}, {0x0db3, 0x0dbb}, {0x0dbd, 0x0dbd}, + {0x0dc0, 0x0dc6}, {0x0e01, 0x0e30}, {0x0e32, 0x0e33}, {0x0e40, 0x0e46}, + {0x0e81, 0x0e82}, {0x0e84, 0x0e84}, {0x0e86, 0x0e8a}, {0x0e8c, 0x0ea3}, + {0x0ea5, 0x0ea5}, {0x0ea7, 0x0eb0}, {0x0eb2, 0x0eb3}, {0x0ebd, 0x0ebd}, + {0x0ec0, 0x0ec4}, {0x0ec6, 0x0ec6}, {0x0edc, 0x0edf}, {0x0f00, 0x0f00}, + {0x0f40, 0x0f47}, {0x0f49, 0x0f6c}, {0x0f88, 0x0f8c}, {0x1000, 0x102a}, + {0x103f, 0x103f}, {0x1050, 0x1055}, {0x105a, 0x105d}, {0x1061, 0x1061}, + {0x1065, 0x1066}, {0x106e, 0x1070}, {0x1075, 0x1081}, {0x108e, 0x108e}, + {0x10a0, 0x10c5}, {0x10c7, 0x10c7}, {0x10cd, 0x10cd}, {0x10d0, 0x10fa}, + {0x10fc, 0x1248}, {0x124a, 0x124d}, {0x1250, 0x1256}, {0x1258, 0x1258}, + {0x125a, 0x125d}, {0x1260, 0x1288}, {0x128a, 0x128d}, {0x1290, 0x12b0}, + {0x12b2, 0x12b5}, {0x12b8, 0x12be}, {0x12c0, 0x12c0}, {0x12c2, 0x12c5}, + {0x12c8, 0x12d6}, {0x12d8, 0x1310}, {0x1312, 0x1315}, {0x1318, 0x135a}, + {0x1380, 0x138f}, {0x13a0, 0x13f5}, {0x13f8, 0x13fd}, {0x1401, 0x166c}, + {0x166f, 0x167f}, {0x1681, 0x169a}, {0x16a0, 0x16ea}, {0x16ee, 0x16f8}, + {0x1700, 0x1711}, {0x171f, 0x1731}, {0x1740, 0x1751}, {0x1760, 0x176c}, + {0x176e, 0x1770}, {0x1780, 0x17b3}, {0x17d7, 0x17d7}, {0x17dc, 0x17dc}, + {0x1820, 0x1878}, {0x1880, 0x1884}, {0x1887, 0x18a8}, {0x18aa, 0x18aa}, + {0x18b0, 0x18f5}, {0x1900, 0x191e}, {0x1950, 0x196d}, {0x1970, 0x1974}, + {0x1980, 0x19ab}, {0x19b0, 0x19c9}, {0x1a00, 0x1a16}, {0x1a20, 0x1a54}, + {0x1aa7, 0x1aa7}, {0x1b05, 0x1b33}, {0x1b45, 0x1b4c}, {0x1b83, 0x1ba0}, + {0x1bae, 0x1baf}, {0x1bba, 0x1be5}, {0x1c00, 0x1c23}, {0x1c4d, 0x1c4f}, + {0x1c5a, 0x1c7d}, {0x1c80, 0x1c8a}, {0x1c90, 0x1cba}, {0x1cbd, 0x1cbf}, + {0x1ce9, 0x1cec}, {0x1cee, 0x1cf3}, {0x1cf5, 0x1cf6}, {0x1cfa, 0x1cfa}, + {0x1d00, 0x1dbf}, {0x1e00, 0x1f15}, {0x1f18, 0x1f1d}, {0x1f20, 0x1f45}, + {0x1f48, 0x1f4d}, {0x1f50, 0x1f57}, {0x1f59, 0x1f59}, {0x1f5b, 0x1f5b}, + {0x1f5d, 0x1f5d}, {0x1f5f, 0x1f7d}, {0x1f80, 0x1fb4}, {0x1fb6, 0x1fbc}, + {0x1fbe, 0x1fbe}, {0x1fc2, 0x1fc4}, {0x1fc6, 0x1fcc}, {0x1fd0, 0x1fd3}, + {0x1fd6, 0x1fdb}, {0x1fe0, 0x1fec}, {0x1ff2, 0x1ff4}, {0x1ff6, 0x1ffc}, + {0x2071, 0x2071}, {0x207f, 0x207f}, {0x2090, 0x209c}, {0x2102, 0x2102}, + {0x2107, 0x2107}, {0x210a, 0x2113}, {0x2115, 0x2115}, {0x2119, 0x211d}, + {0x2124, 0x2124}, {0x2126, 0x2126}, {0x2128, 0x2128}, {0x212a, 0x212d}, + {0x212f, 0x2139}, {0x213c, 0x213f}, {0x2145, 0x2149}, {0x214e, 0x214e}, + {0x2160, 0x2188}, {0x2c00, 0x2ce4}, {0x2ceb, 0x2cee}, {0x2cf2, 0x2cf3}, + {0x2d00, 0x2d25}, {0x2d27, 0x2d27}, {0x2d2d, 0x2d2d}, {0x2d30, 0x2d67}, + {0x2d6f, 0x2d6f}, {0x2d80, 0x2d96}, {0x2da0, 0x2da6}, {0x2da8, 0x2dae}, + {0x2db0, 0x2db6}, {0x2db8, 0x2dbe}, {0x2dc0, 0x2dc6}, {0x2dc8, 0x2dce}, + {0x2dd0, 0x2dd6}, {0x2dd8, 0x2dde}, {0x2e2f, 0x2e2f}, {0x3005, 0x3007}, + {0x3021, 0x3029}, {0x3031, 0x3035}, {0x3038, 0x303c}, {0x3041, 0x3096}, + {0x309d, 0x309f}, {0x30a1, 0x30fa}, {0x30fc, 0x30ff}, {0x3105, 0x312f}, + {0x3131, 0x318e}, {0x31a0, 0x31bf}, {0x31f0, 0x31ff}, {0x3400, 0x3400}, + {0x4dbf, 0x4dbf}, {0x4e00, 0x4e00}, {0x9fff, 0xa48c}, {0xa4d0, 0xa4fd}, + {0xa500, 0xa60c}, {0xa610, 0xa61f}, {0xa62a, 0xa62b}, {0xa640, 0xa66e}, + {0xa67f, 0xa69d}, {0xa6a0, 0xa6ef}, {0xa717, 0xa71f}, {0xa722, 0xa788}, + {0xa78b, 0xa7dc}, {0xa7f1, 0xa801}, {0xa803, 0xa805}, {0xa807, 0xa80a}, + {0xa80c, 0xa822}, {0xa840, 0xa873}, {0xa882, 0xa8b3}, {0xa8f2, 0xa8f7}, + {0xa8fb, 0xa8fb}, {0xa8fd, 0xa8fe}, {0xa90a, 0xa925}, {0xa930, 0xa946}, + {0xa960, 0xa97c}, {0xa984, 0xa9b2}, {0xa9cf, 0xa9cf}, {0xa9e0, 0xa9e4}, + {0xa9e6, 0xa9ef}, {0xa9fa, 0xa9fe}, {0xaa00, 0xaa28}, {0xaa40, 0xaa42}, + {0xaa44, 0xaa4b}, {0xaa60, 0xaa76}, {0xaa7a, 0xaa7a}, {0xaa7e, 0xaaaf}, + {0xaab1, 0xaab1}, {0xaab5, 0xaab6}, {0xaab9, 0xaabd}, {0xaac0, 0xaac0}, + {0xaac2, 0xaac2}, {0xaadb, 0xaadd}, {0xaae0, 0xaaea}, {0xaaf2, 0xaaf4}, + {0xab01, 0xab06}, {0xab09, 0xab0e}, {0xab11, 0xab16}, {0xab20, 0xab26}, + {0xab28, 0xab2e}, {0xab30, 0xab5a}, {0xab5c, 0xab69}, {0xab70, 0xabe2}, + {0xac00, 0xac00}, {0xd7a3, 0xd7a3}, {0xd7b0, 0xd7c6}, {0xd7cb, 0xd7fb}, + {0xf900, 0xfa6d}, {0xfa70, 0xfad9}, {0xfb00, 0xfb06}, {0xfb13, 0xfb17}, + {0xfb1d, 0xfb1d}, {0xfb1f, 0xfb28}, {0xfb2a, 0xfb36}, {0xfb38, 0xfb3c}, + {0xfb3e, 0xfb3e}, {0xfb40, 0xfb41}, {0xfb43, 0xfb44}, {0xfb46, 0xfbb1}, + {0xfbd3, 0xfd3d}, {0xfd50, 0xfd8f}, {0xfd92, 0xfdc7}, {0xfdf0, 0xfdfb}, + {0xfe70, 0xfe74}, {0xfe76, 0xfefc}, {0xff21, 0xff3a}, {0xff41, 0xff5a}, + {0xff66, 0xffbe}, {0xffc2, 0xffc7}, {0xffca, 0xffcf}, {0xffd2, 0xffd7}, + {0xffda, 0xffdc} +}; +const int identifier_start_letters_size = 381; + +const unicode_range_t identifier_part_letters[333] = +{ + {0x0030, 0x0039}, {0x005f, 0x005f}, {0x00a1, 0x00a9}, {0x00ab, 0x00ac}, + {0x00ae, 0x00b4}, {0x00b6, 0x00b9}, {0x00bb, 0x00bf}, {0x00d7, 0x00d7}, + {0x00f7, 0x00f7}, {0x02c2, 0x02c5}, {0x02d2, 0x02df}, {0x02e5, 0x02eb}, + {0x02ed, 0x02ed}, {0x02ef, 0x036f}, {0x0375, 0x0375}, {0x037e, 0x037e}, + {0x0384, 0x0385}, {0x0387, 0x0387}, {0x03f6, 0x03f6}, {0x0482, 0x0489}, + {0x055a, 0x055f}, {0x0589, 0x058a}, {0x058d, 0x058f}, {0x0591, 0x05c7}, + {0x05f3, 0x05f4}, {0x0606, 0x061b}, {0x061d, 0x061f}, {0x064b, 0x066d}, + {0x0670, 0x0670}, {0x06d4, 0x06d4}, {0x06d6, 0x06dc}, {0x06de, 0x06e4}, + {0x06e7, 0x06ed}, {0x06f0, 0x06f9}, {0x06fd, 0x06fe}, {0x0700, 0x070d}, + {0x0711, 0x0711}, {0x0730, 0x074a}, {0x07a6, 0x07b0}, {0x07c0, 0x07c9}, + {0x07eb, 0x07f3}, {0x07f6, 0x07f9}, {0x07fd, 0x07ff}, {0x0816, 0x0819}, + {0x081b, 0x0823}, {0x0825, 0x0827}, {0x0829, 0x082d}, {0x0830, 0x083e}, + {0x0859, 0x085b}, {0x085e, 0x085e}, {0x0888, 0x0888}, {0x0897, 0x089f}, + {0x08ca, 0x08e1}, {0x08e3, 0x0903}, {0x093a, 0x093c}, {0x093e, 0x094f}, + {0x0951, 0x0957}, {0x0962, 0x0970}, {0x0981, 0x0983}, {0x09bc, 0x09bc}, + {0x09be, 0x09c4}, {0x09c7, 0x09c8}, {0x09cb, 0x09cd}, {0x09d7, 0x09d7}, + {0x09e2, 0x09e3}, {0x09e6, 0x09ef}, {0x09f2, 0x09fb}, {0x09fd, 0x09fe}, + {0x0a01, 0x0a03}, {0x0a3c, 0x0a3c}, {0x0a3e, 0x0a42}, {0x0a47, 0x0a48}, + {0x0a4b, 0x0a4d}, {0x0a51, 0x0a51}, {0x0a66, 0x0a71}, {0x0a75, 0x0a76}, + {0x0a81, 0x0a83}, {0x0abc, 0x0abc}, {0x0abe, 0x0ac5}, {0x0ac7, 0x0ac9}, + {0x0acb, 0x0acd}, {0x0ae2, 0x0ae3}, {0x0ae6, 0x0af1}, {0x0afa, 0x0aff}, + {0x0b01, 0x0b03}, {0x0b3c, 0x0b3c}, {0x0b3e, 0x0b44}, {0x0b47, 0x0b48}, + {0x0b4b, 0x0b4d}, {0x0b55, 0x0b57}, {0x0b62, 0x0b63}, {0x0b66, 0x0b70}, + {0x0b72, 0x0b77}, {0x0b82, 0x0b82}, {0x0bbe, 0x0bc2}, {0x0bc6, 0x0bc8}, + {0x0bca, 0x0bcd}, {0x0bd7, 0x0bd7}, {0x0be6, 0x0bfa}, {0x0c00, 0x0c04}, + {0x0c3c, 0x0c3c}, {0x0c3e, 0x0c44}, {0x0c46, 0x0c48}, {0x0c4a, 0x0c4d}, + {0x0c55, 0x0c56}, {0x0c62, 0x0c63}, {0x0c66, 0x0c6f}, {0x0c77, 0x0c7f}, + {0x0c81, 0x0c84}, {0x0cbc, 0x0cbc}, {0x0cbe, 0x0cc4}, {0x0cc6, 0x0cc8}, + {0x0cca, 0x0ccd}, {0x0cd5, 0x0cd6}, {0x0ce2, 0x0ce3}, {0x0ce6, 0x0cef}, + {0x0cf3, 0x0cf3}, {0x0d00, 0x0d03}, {0x0d3b, 0x0d3c}, {0x0d3e, 0x0d44}, + {0x0d46, 0x0d48}, {0x0d4a, 0x0d4d}, {0x0d4f, 0x0d4f}, {0x0d57, 0x0d5e}, + {0x0d62, 0x0d63}, {0x0d66, 0x0d79}, {0x0d81, 0x0d83}, {0x0dca, 0x0dca}, + {0x0dcf, 0x0dd4}, {0x0dd6, 0x0dd6}, {0x0dd8, 0x0ddf}, {0x0de6, 0x0def}, + {0x0df2, 0x0df4}, {0x0e31, 0x0e31}, {0x0e34, 0x0e3a}, {0x0e3f, 0x0e3f}, + {0x0e47, 0x0e5b}, {0x0eb1, 0x0eb1}, {0x0eb4, 0x0ebc}, {0x0ec8, 0x0ece}, + {0x0ed0, 0x0ed9}, {0x0f01, 0x0f3f}, {0x0f71, 0x0f87}, {0x0f8d, 0x0f97}, + {0x0f99, 0x0fbc}, {0x0fbe, 0x0fcc}, {0x0fce, 0x0fda}, {0x102b, 0x103e}, + {0x1040, 0x104f}, {0x1056, 0x1059}, {0x105e, 0x1060}, {0x1062, 0x1064}, + {0x1067, 0x106d}, {0x1071, 0x1074}, {0x1082, 0x108d}, {0x108f, 0x109f}, + {0x10fb, 0x10fb}, {0x135d, 0x137c}, {0x1390, 0x1399}, {0x1400, 0x1400}, + {0x166d, 0x166e}, {0x169b, 0x169c}, {0x16eb, 0x16ed}, {0x1712, 0x1715}, + {0x1732, 0x1736}, {0x1752, 0x1753}, {0x1772, 0x1773}, {0x17b4, 0x17d6}, + {0x17d8, 0x17db}, {0x17dd, 0x17dd}, {0x17e0, 0x17e9}, {0x17f0, 0x17f9}, + {0x1800, 0x180d}, {0x180f, 0x1819}, {0x1885, 0x1886}, {0x18a9, 0x18a9}, + {0x1920, 0x192b}, {0x1930, 0x193b}, {0x1940, 0x1940}, {0x1944, 0x194f}, + {0x19d0, 0x19da}, {0x19de, 0x19ff}, {0x1a17, 0x1a1b}, {0x1a1e, 0x1a1f}, + {0x1a55, 0x1a5e}, {0x1a60, 0x1a7c}, {0x1a7f, 0x1a89}, {0x1a90, 0x1a99}, + {0x1aa0, 0x1aa6}, {0x1aa8, 0x1aad}, {0x1ab0, 0x1add}, {0x1ae0, 0x1aeb}, + {0x1b00, 0x1b04}, {0x1b34, 0x1b44}, {0x1b4e, 0x1b82}, {0x1ba1, 0x1bad}, + {0x1bb0, 0x1bb9}, {0x1be6, 0x1bf3}, {0x1bfc, 0x1bff}, {0x1c24, 0x1c37}, + {0x1c3b, 0x1c49}, {0x1c50, 0x1c59}, {0x1c7e, 0x1c7f}, {0x1cc0, 0x1cc7}, + {0x1cd0, 0x1ce8}, {0x1ced, 0x1ced}, {0x1cf4, 0x1cf4}, {0x1cf7, 0x1cf9}, + {0x1dc0, 0x1dff}, {0x1fbd, 0x1fbd}, {0x1fbf, 0x1fc1}, {0x1fcd, 0x1fcf}, + {0x1fdd, 0x1fdf}, {0x1fed, 0x1fef}, {0x1ffd, 0x1ffe}, {0x2010, 0x2027}, + {0x2030, 0x205e}, {0x2070, 0x2070}, {0x2074, 0x207e}, {0x2080, 0x208e}, + {0x20a0, 0x20c1}, {0x20d0, 0x20f0}, {0x2100, 0x2101}, {0x2103, 0x2106}, + {0x2108, 0x2109}, {0x2114, 0x2114}, {0x2116, 0x2118}, {0x211e, 0x2123}, + {0x2125, 0x2125}, {0x2127, 0x2127}, {0x2129, 0x2129}, {0x212e, 0x212e}, + {0x213a, 0x213b}, {0x2140, 0x2144}, {0x214a, 0x214d}, {0x214f, 0x215f}, + {0x2189, 0x218b}, {0x2190, 0x2429}, {0x2440, 0x244a}, {0x2460, 0x2b73}, + {0x2b76, 0x2bff}, {0x2ce5, 0x2cea}, {0x2cef, 0x2cf1}, {0x2cf9, 0x2cff}, + {0x2d70, 0x2d70}, {0x2d7f, 0x2d7f}, {0x2de0, 0x2e2e}, {0x2e30, 0x2e5d}, + {0x2e80, 0x2e99}, {0x2e9b, 0x2ef3}, {0x2f00, 0x2fd5}, {0x2ff0, 0x2fff}, + {0x3001, 0x3004}, {0x3008, 0x3020}, {0x302a, 0x3030}, {0x3036, 0x3037}, + {0x303d, 0x303f}, {0x3099, 0x309c}, {0x30a0, 0x30a0}, {0x30fb, 0x30fb}, + {0x3190, 0x319f}, {0x31c0, 0x31e5}, {0x31ef, 0x31ef}, {0x3200, 0x321e}, + {0x3220, 0x33ff}, {0x4dc0, 0x4dff}, {0xa490, 0xa4c6}, {0xa4fe, 0xa4ff}, + {0xa60d, 0xa60f}, {0xa620, 0xa629}, {0xa66f, 0xa67e}, {0xa69e, 0xa69f}, + {0xa6f0, 0xa6f7}, {0xa700, 0xa716}, {0xa720, 0xa721}, {0xa789, 0xa78a}, + {0xa802, 0xa802}, {0xa806, 0xa806}, {0xa80b, 0xa80b}, {0xa823, 0xa82c}, + {0xa830, 0xa839}, {0xa874, 0xa877}, {0xa880, 0xa881}, {0xa8b4, 0xa8c5}, + {0xa8ce, 0xa8d9}, {0xa8e0, 0xa8f1}, {0xa8f8, 0xa8fa}, {0xa8fc, 0xa8fc}, + {0xa8ff, 0xa909}, {0xa926, 0xa92f}, {0xa947, 0xa953}, {0xa95f, 0xa95f}, + {0xa980, 0xa983}, {0xa9b3, 0xa9cd}, {0xa9d0, 0xa9d9}, {0xa9de, 0xa9df}, + {0xa9e5, 0xa9e5}, {0xa9f0, 0xa9f9}, {0xaa29, 0xaa36}, {0xaa43, 0xaa43}, + {0xaa4c, 0xaa4d}, {0xaa50, 0xaa59}, {0xaa5c, 0xaa5f}, {0xaa77, 0xaa79}, + {0xaa7b, 0xaa7d}, {0xaab0, 0xaab0}, {0xaab2, 0xaab4}, {0xaab7, 0xaab8}, + {0xaabe, 0xaabf}, {0xaac1, 0xaac1}, {0xaade, 0xaadf}, {0xaaeb, 0xaaf1}, + {0xaaf5, 0xaaf6}, {0xab5b, 0xab5b}, {0xab6a, 0xab6b}, {0xabe3, 0xabed}, + {0xabf0, 0xabf9}, {0xfb1e, 0xfb1e}, {0xfb29, 0xfb29}, {0xfbb2, 0xfbd2}, + {0xfd3e, 0xfd4f}, {0xfd90, 0xfd91}, {0xfdc8, 0xfdcf}, {0xfdfc, 0xfe19}, + {0xfe20, 0xfe52}, {0xfe54, 0xfe66}, {0xfe68, 0xfe6b}, {0xff01, 0xff20}, + {0xff3b, 0xff40}, {0xff5b, 0xff65}, {0xffe0, 0xffe6}, {0xffe8, 0xffee}, + {0xfffc, 0xfffd} +}; +const int identifier_part_letters_size = 333; + diff --git a/dlls/jscript/identifier_letters.h b/dlls/jscript/identifier_letters.h new file mode 100644 index 00000000000..e7b6e3c3feb --- /dev/null +++ b/dlls/jscript/identifier_letters.h @@ -0,0 +1,28 @@ +/* + * Copyright 2026 Zhiyi Zhang for CodeWeavers + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA + */ + +typedef struct _unicode_range +{ + unsigned short start; + unsigned short end; +} unicode_range_t; + +extern const int identifier_start_letters_size; +extern const int identifier_part_letters_size; +extern const unicode_range_t identifier_start_letters[]; +extern const unicode_range_t identifier_part_letters[]; diff --git a/dlls/jscript/lex.c b/dlls/jscript/lex.c index ee29bff1e37..f47d378465b 100644 --- a/dlls/jscript/lex.c +++ b/dlls/jscript/lex.c @@ -27,6 +27,7 @@ #include "parser.h" #include "parser.tab.h" +#include "identifier_letters.h" #include "wine/debug.h" @@ -79,15 +80,44 @@ static int lex_error(parser_ctx_t *ctx, HRESULT hres) return -1; } -/* ECMA-262 3rd Edition 7.6 */ -BOOL is_identifier_char(WCHAR c) +static BOOL search_ranges(WCHAR c, const unicode_range_t *ranges, int count) { - return iswalnum(c) || c == '$' || c == '_' || c == '\\'; + int left = 0, right = count - 1; + + while(left <= right) { + int middle = left + (right - left) / 2; + + if(c >= ranges[middle].start && c <= ranges[middle].end) + return TRUE; + + if(c < ranges[middle].start) + right = middle - 1; + else + left = middle + 1; + } + + return FALSE; } +/* ECMA-262 3rd Edition 7.6 */ static BOOL is_identifier_first_char(WCHAR c) { - return iswalpha(c) || c == '$' || c == '_' || c == '\\'; + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '$' || c == '_' || c == '\\') + return TRUE; + + return search_ranges(c, identifier_start_letters, identifier_start_letters_size); +} + +/* More Unicode categories are allowed in IdentifierPart than those specified in ECMA-262 3rd + * edition according to tests */ +BOOL is_identifier_char(WCHAR c) +{ + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') + || c == '$' || c == '_' || c == '\\') + return TRUE; + + return search_ranges(c, identifier_start_letters, identifier_start_letters_size) + || search_ranges(c, identifier_part_letters, identifier_part_letters_size); } static int check_keyword(parser_ctx_t *ctx, const WCHAR *word, const WCHAR **lval) @@ -131,19 +161,20 @@ int hex_to_int(WCHAR c) return -1; } -static int check_keywords(parser_ctx_t *ctx, const WCHAR **lval) +static int check_keywords(parser_ctx_t *ctx, const WCHAR *identifier, const WCHAR **lval) { int min = 0, max = ARRAY_SIZE(keywords)-1, r, i; while(min <= max) { i = (min+max)/2; - r = check_keyword(ctx, keywords[i].word, lval); + r = wcscmp(identifier, keywords[i].word); if(!r) { + if(lval) + *lval = keywords[i].word; if(ctx->script->version < keywords[i].min_version) { TRACE("ignoring keyword %s in incompatible mode\n", debugstr_w(keywords[i].word)); - ctx->ptr -= lstrlenW(keywords[i].word); return 0; } ctx->implicit_nl_semicolon = keywords[i].no_nl; @@ -318,22 +349,94 @@ BOOL unescape(WCHAR *str, size_t *len) return TRUE; } +static BOOL unescape_identifier(WCHAR *str, size_t *len) +{ + WCHAR *pd, *p, c, *end = str + *len; + int i; + + pd = p = str; + while(p < end) { + if(*p != '\\') { + *pd++ = *p++; + continue; + } + + if(++p == end) + return FALSE; + + if(*p != 'u' || p + 4 >= end) + return FALSE; + + i = hex_to_int(*++p); + if(i == -1) + return FALSE; + c = i << 12; + + i = hex_to_int(*++p); + if(i == -1) + return FALSE; + c += i << 8; + + i = hex_to_int(*++p); + if(i == -1) + return FALSE; + c += i << 4; + + i = hex_to_int(*++p); + if(i == -1) + return FALSE; + c += i; + + if(pd == str && !is_identifier_first_char(c)) + return FALSE; + else if(!is_identifier_char(c)) + return FALSE; + + *pd++ = c; + p++; + } + + *len = pd - str; + return TRUE; +} + static int parse_identifier(parser_ctx_t *ctx, const WCHAR **ret) { const WCHAR *ptr = ctx->ptr++; - WCHAR *wstr; - int len; + BOOL needs_unescape = FALSE; + WCHAR *wstr, *unescape_str; + size_t len; + + if(*ptr== '\\') + needs_unescape = TRUE; + + while(ctx->ptr < ctx->end && is_identifier_char(*ctx->ptr)) { + if(*ctx->ptr == '\\') + needs_unescape = TRUE; - while(ctx->ptr < ctx->end && is_identifier_char(*ctx->ptr)) ctx->ptr++; + } len = ctx->ptr-ptr; + if(needs_unescape) { + unescape_str = parser_alloc(ctx, len*sizeof(WCHAR)); + if(!unescape_str) + return lex_error(ctx, E_OUTOFMEMORY); + + memcpy(unescape_str, ptr, len*sizeof(WCHAR)); + if(!unescape_identifier(unescape_str, &len)) { + WARN("unescape identifier failed\n"); + return lex_error(ctx, E_FAIL); + } + + ptr = unescape_str; + } + *ret = wstr = parser_alloc(ctx, (len+1)*sizeof(WCHAR)); memcpy(wstr, ptr, len*sizeof(WCHAR)); wstr[len] = 0; - /* FIXME: unescape */ return tIdentifier; } @@ -559,12 +662,15 @@ static int next_token(parser_ctx_t *ctx, unsigned *loc, void *lval) ctx->implicit_nl_semicolon = FALSE; } - if(iswalpha(*ctx->ptr)) { - int ret = check_keywords(ctx, lval); - if(ret) - return ret; + if(is_identifier_first_char(*ctx->ptr)) { + int ret = parse_identifier(ctx, lval); + if(ret == tIdentifier) { + int token = check_keywords(ctx, *(WCHAR **)lval, lval); + if(token) + return token; + } - return parse_identifier(ctx, lval); + return ret; } if(is_digit(*ctx->ptr)) { @@ -795,10 +901,6 @@ static int next_token(parser_ctx_t *ctx, unsigned *loc, void *lval) case '\'': return parse_string_literal(ctx, lval, *ctx->ptr); - case '_': - case '$': - return parse_identifier(ctx, lval); - case '@': return '@'; } @@ -893,6 +995,7 @@ static BOOL parse_cc_identifier(parser_ctx_t *ctx, const WCHAR **ret, unsigned * return FALSE; } + /* FIXME: Unicode escape sequences are not allowed */ if(!is_identifier_first_char(*++ctx->ptr)) { lex_error(ctx, JS_E_EXPECTED_IDENTIFIER); return FALSE; diff --git a/tools/make_unicode b/tools/make_unicode index 6dda5573c98..1c5ee6d1005 100755 --- a/tools/make_unicode +++ b/tools/make_unicode @@ -1973,6 +1973,8 @@ my @decomp_compat_table = (); my @comp_exclusions = (); my @idna_decomp_table = (); my @idna_disallowed = (); +my @jscript_identifier_start_letters = (); +my @jscript_identifier_part_letters = (); my %registry_keys; my $default_char; my $default_wchar; @@ -2213,6 +2215,30 @@ sub load_data() $initial_joining_table[$src] = $joining_types{"U"}; } + # Only characters in the Basic Multilingual Plane are supported + if ($src < 0x10000) + { + # IdentifierStart at ECMA-262 3rd edition 7.6 + if ($cat eq "Lu" || $cat eq "Ll" || $cat eq "Lt" || $cat eq "Lm" || $cat eq "Lo" || $cat eq "Nl") + { + push(@jscript_identifier_start_letters, $src); + } + # IdentifierPart at ECMA-262 3rd edition 7.6 + elsif ($cat eq "Mn" || $cat eq "Mc" || $cat eq "Nd" || $cat eq "Pc") + { + push(@jscript_identifier_part_letters, $src); + } + # More Unicode categories are allowed in IdentifierPart than those specified in ECMA-262 + # 3rd edition according to tests. Exclude code points <= 0x7F because things like \u0022 + # (Quotation Mark) also falls in Po. + elsif ($src > 0x7F && ($cat eq "Me" || $cat eq "No" || $cat eq "Pd" || $cat eq "Ps" + || $cat eq "Pe" || $cat eq "Pi" || $cat eq "Pf" || $cat eq "Po" || $cat eq "Sm" + || $cat eq "Sc" || $cat eq "Sk" || $cat eq "So")) + { + push(@jscript_identifier_part_letters, $src); + } + } + if ($lower ne "") { $tolower_table[$src] = hex $lower; @@ -3145,6 +3171,86 @@ sub dump_scripts($) save_file($filename); } +################################################################ +# get Unicode code point range pairs in an array +sub get_ranges +{ + my @array = @_; + my @ranges; + return @ranges unless @array; + my $start = $array[0]; + my $end = $array[0]; + for (my $i = 1; $i < @array; $i++) + { + if ($array[$i] == $end + 1) + { + $end = $array[$i]; + } + else + { + push(@ranges, $start, $end); + $start = $end = $array[$i]; + } + } + push(@ranges, $start, $end); + return @ranges; +} + +################################################################ +# dump Unicode point ranges +sub dump_range_struct($@) +{ + my ($bit_width, @array) = @_; + my $format = sprintf "{0x%%0%ux, 0x%%0%ux}", $bit_width / 4, $bit_width / 4; + my $ret = ""; + for (my $i = 0; $i < @array; $i += 2) + { + $ret .= " " if ($i % 8 == 0); + $ret .= sprintf($format, $array[$i], $array[$i+1]); + if ($i < @array - 2) + { + if ($i % 8 == 6) + { + $ret .= ",\n"; + } + else + { + $ret .= ", "; + } + } + } + return $ret; +} + +################################################################ +# dump allowed jscript identifier letters +sub dump_jscript_identifier_letters($) +{ + my $filename = shift; + + open OUTPUT,">$filename.new" or die "Cannot create $filename"; + print "Building $filename\n"; + print OUTPUT "/* Unicode letter ranges allowed in jscript identifiers */\n"; + printf OUTPUT "/* Automatically generated; DO NOT EDIT!! */\n\n"; + + print OUTPUT "#include \"identifier_letters.h\"\n\n"; + + my @start_ranges = get_ranges(@jscript_identifier_start_letters); + printf OUTPUT "const unicode_range_t identifier_start_letters[%d] =\n{\n", @start_ranges / 2; + print OUTPUT dump_range_struct( 16, @start_ranges ); + print OUTPUT "\n};\n"; + printf OUTPUT "const int identifier_start_letters_size = %d;\n\n", @start_ranges / 2; + + my @part_ranges = get_ranges(@jscript_identifier_part_letters); + printf OUTPUT "const unicode_range_t identifier_part_letters[%d] =\n{\n", @part_ranges / 2; + print OUTPUT dump_range_struct( 16, @part_ranges ); + print OUTPUT "\n};\n"; + printf OUTPUT "const int identifier_part_letters_size = %d;\n\n", @part_ranges / 2;; + + close OUTPUT; + save_file($filename); +} + ################################################################ # dump the BiDi mirroring table sub dump_mirroring($) @@ -6271,6 +6377,7 @@ dump_arabic_shaping( "dlls/dwrite/shapers/arabic_table.c" ); dump_linebreak( "dlls/gdi32/uniscribe/linebreak.c" ); dump_linebreak( "dlls/dwrite/linebreak.c" ); dump_scripts( "dlls/dwrite/scripts" ); +dump_jscript_identifier_letters( "dlls/jscript/identifier_letters.c" ); dump_indic( "dlls/gdi32/uniscribe/indicsyllable.c" ); dump_vertical( "dlls/win32u/vertical.c", 1 ); dump_vertical( "dlls/wineps.drv/vertical.c", 0 ); -- GitLab https://gitlab.winehq.org/wine/wine/-/merge_requests/11452