May 2022 - Wine-Commits - list.winehq.org

Alexandre Julliard : kernelbase: Reimplement sortkey generation using the sortdefault.nls data.
by Alexandre Julliard May 30, 2022

May 30, 2022

Module: wine Branch: master Commit: 681489dbb68166b493ee1cae2f2ca4464a85c029 URL: https://source.winehq.org/git/wine.git/?a=commit;h=681489dbb68166b493ee1cae… Author: Alexandre Julliard <julliard(a)winehq.org> Date: Mon May 30 12:56:40 2022 +0200 kernelbase: Reimplement sortkey generation using the sortdefault.nls data. Signed-off-by: Alexandre Julliard <julliard(a)winehq.org> --- dlls/kernelbase/locale.c | 387 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 283 insertions(+), 104 deletions(-) Diff: https://source.winehq.org/git/wine.git/?a=commitdiff;h=681489dbb68166b493ee…

1 0

Alexandre Julliard : kernelbase: Add support for character expansion in sortkeys.
by Alexandre Julliard May 30, 2022

May 30, 2022

Module: wine Branch: master Commit: 9605f5870ad70076f70d29c78091cb9f9ca6ac92 URL: https://source.winehq.org/git/wine.git/?a=commit;h=9605f5870ad70076f70d29c7… Author: Alexandre Julliard <julliard(a)winehq.org> Date: Mon May 30 13:02:09 2022 +0200 kernelbase: Add support for character expansion in sortkeys. Signed-off-by: Alexandre Julliard <julliard(a)winehq.org> --- dlls/kernelbase/locale.c | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/dlls/kernelbase/locale.c b/dlls/kernelbase/locale.c index 08842cf2c08..8eaa5f5fb3c 100644 --- a/dlls/kernelbase/locale.c +++ b/dlls/kernelbase/locale.c @@ -337,6 +337,11 @@ struct sortguid #define FLAG_DOUBLECOMPRESSION 0x20 #define FLAG_INVERSECASING 0x40 +struct sort_expansion +{ + WCHAR exp[2]; +}; + union char_weights { UINT val; @@ -388,11 +393,13 @@ static struct { UINT version; /* NLS version */ UINT guid_count; /* number of sort GUIDs */ + UINT exp_count; /* number of character expansions */ const UINT *keys; /* sortkey table, indexed by char */ const USHORT *casemap; /* casemap table, in l_intl.nls format */ const WORD *ctypes; /* CT_CTYPE1,2,3 values */ const BYTE *ctype_idx; /* index to map char to ctypes array entry */ const struct sortguid *guids; /* table of sort GUIDs */ + const struct sort_expansion *expansions; /* character expansions */ } sort; static CRITICAL_SECTION locale_section; @@ -475,6 +482,10 @@ static void load_sortdefault_nls(void) sort.guid_count = table[1]; sort.guids = (struct sortguid *)(table + 2); + table = (UINT *)(sort.guids + sort.guid_count); + sort.exp_count = table[0]; + sort.expansions = (struct sort_expansion *)(table + 1); + locale_sorts = RtlAllocateHeap( GetProcessHeap(), HEAP_ZERO_MEMORY, locale_table->nb_lcnames * sizeof(*locale_sorts) ); } @@ -3207,6 +3218,23 @@ static void append_nonspace_weights( struct sortkey *key, union char_weights wei else append_sortkey( key, weights.diacritic ); } +static void append_expansion_weights( const struct sortguid *sortid, struct sortkey *key_primary, + struct sortkey *key_diacritic, struct sortkey *key_case, + union char_weights weights, DWORD flags, BOOL is_compare ) +{ + /* sortkey and comparison behave differently here */ + if (is_compare) + { + if (weights.script == SCRIPT_UNSORTABLE) return; + if (weights.script == SCRIPT_NONSPACE_MARK) + { + append_nonspace_weights( key_diacritic, weights, flags ); + return; + } + } + append_normal_weights( sortid, key_primary, key_diacritic, key_case, weights, flags ); +} + /* put one of the elements of a sortkey into the dst buffer */ static int put_sortkey( BYTE *dst, int dstlen, int pos, const struct sortkey *key, BYTE terminator ) { @@ -3290,6 +3318,7 @@ static int append_weights( const struct sortguid *sortid, DWORD flags, const WCHAR *compr_tables[8], struct sortkey_state *s, BOOL is_compare ) { union char_weights weights = get_char_weights( src[pos], except ); + WCHAR idx = (weights.val >> 16) & ~(CASE_COMPR_6 << 8); /* expansion index */ int ret = 1; weights._case &= case_mask; @@ -3303,6 +3332,21 @@ static int append_weights( const struct sortguid *sortid, DWORD flags, append_nonspace_weights( &s->key_diacritic, weights, flags ); break; + case SCRIPT_EXPANSION: + while (weights.script == SCRIPT_EXPANSION) + { + weights = get_char_weights( sort.expansions[idx].exp[0], except ); + weights._case &= case_mask; + append_expansion_weights( sortid, &s->key_primary, &s->key_diacritic, + &s->key_case, weights, flags, is_compare ); + weights = get_char_weights( sort.expansions[idx].exp[1], except ); + idx = weights.val >> 16; + weights._case &= case_mask; + } + append_expansion_weights( sortid, &s->key_primary, &s->key_diacritic, + &s->key_case, weights, flags, is_compare ); + break; + case SCRIPT_EXTENSION_A: append_sortkey( &s->key_primary, 0xfd ); append_sortkey( &s->key_primary, 0xff );

1 0

Alexandre Julliard : kernelbase: Add support for character compression in sortkeys.
by Alexandre Julliard May 30, 2022

May 30, 2022

Module: wine Branch: master Commit: 5c324cb8c96a1cf54a4a2e7a7326490b6b515410 URL: https://source.winehq.org/git/wine.git/?a=commit;h=5c324cb8c96a1cf54a4a2e7a… Author: Alexandre Julliard <julliard(a)winehq.org> Date: Mon May 30 13:06:26 2022 +0200 kernelbase: Add support for character compression in sortkeys. Signed-off-by: Alexandre Julliard <julliard(a)winehq.org> --- dlls/kernelbase/locale.c | 67 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/dlls/kernelbase/locale.c b/dlls/kernelbase/locale.c index 8eaa5f5fb3c..03b34bf99db 100644 --- a/dlls/kernelbase/locale.c +++ b/dlls/kernelbase/locale.c @@ -342,6 +342,15 @@ struct sort_expansion WCHAR exp[2]; }; +struct sort_compression +{ + UINT offset; + WCHAR minchar, maxchar; + WORD len[8]; +}; + +static inline int compression_size( int len ) { return 2 + len + (len & 1); } + union char_weights { UINT val; @@ -394,12 +403,15 @@ static struct UINT version; /* NLS version */ UINT guid_count; /* number of sort GUIDs */ UINT exp_count; /* number of character expansions */ + UINT compr_count; /* number of compression tables */ const UINT *keys; /* sortkey table, indexed by char */ const USHORT *casemap; /* casemap table, in l_intl.nls format */ const WORD *ctypes; /* CT_CTYPE1,2,3 values */ const BYTE *ctype_idx; /* index to map char to ctypes array entry */ const struct sortguid *guids; /* table of sort GUIDs */ const struct sort_expansion *expansions; /* character expansions */ + const struct sort_compression *compressions; /* character compression tables */ + const WCHAR *compr_data; /* data for individual compressions */ } sort; static CRITICAL_SECTION locale_section; @@ -486,6 +498,11 @@ static void load_sortdefault_nls(void) sort.exp_count = table[0]; sort.expansions = (struct sort_expansion *)(table + 1); + table = (UINT *)(sort.expansions + sort.exp_count); + sort.compr_count = table[0]; + sort.compressions = (struct sort_compression *)(table + 1); + sort.compr_data = (WCHAR *)(sort.compressions + sort.compr_count); + locale_sorts = RtlAllocateHeap( GetProcessHeap(), HEAP_ZERO_MEMORY, locale_table->nb_lcnames * sizeof(*locale_sorts) ); } @@ -3235,6 +3252,53 @@ static void append_expansion_weights( const struct sortguid *sortid, struct sort append_normal_weights( sortid, key_primary, key_diacritic, key_case, weights, flags ); } +static const UINT *find_compression( const WCHAR *src, const WCHAR *table, int count, int len ) +{ + int elem_size = compression_size( len ), min = 0, max = count - 1; + + while (min <= max) + { + int pos = (min + max) / 2; + int res = wcsncmp( src, table + pos * elem_size, len ); + if (!res) return (UINT *)(table + (pos + 1) * elem_size) - 1; + if (res > 0) min = pos + 1; + else max = pos - 1; + } + return NULL; +} + +/* find a compression for a char sequence */ +/* return the number of extra chars to skip */ +static int get_compression_weights( UINT compression, const WCHAR *compr_tables[8], + const WCHAR *src, int srclen, union char_weights *weights ) +{ + const struct sort_compression *compr = sort.compressions + compression; + const UINT *ret; + BYTE size = weights->_case & CASE_COMPR_6; + int i, maxlen = 1; + + if (compression >= sort.compr_count) return 0; + if (size == CASE_COMPR_6) maxlen = 8; + else if (size == CASE_COMPR_4) maxlen = 5; + else if (size == CASE_COMPR_2) maxlen = 3; + maxlen = min( maxlen, srclen ); + for (i = 0; i < maxlen; i++) if (src[i] < compr->minchar || src[i] > compr->maxchar) break; + maxlen = i; + if (!compr_tables[0]) + { + compr_tables[0] = sort.compr_data + compr->offset; + for (i = 1; i <= maxlen - 2; i++) + compr_tables[i] = compr_tables[i - 1] + compr->len[i - 1] * compression_size( i + 1 ); + } + for (i = maxlen - 2; i >= 0; i--) + { + if (!(ret = find_compression( src, compr_tables[i], compr->len[i], i + 2 ))) continue; + weights->val = *ret; + return i + 1; + } + return 0; +} + /* put one of the elements of a sortkey into the dst buffer */ static int put_sortkey( BYTE *dst, int dstlen, int pos, const struct sortkey *key, BYTE terminator ) { @@ -3321,6 +3385,9 @@ static int append_weights( const struct sortguid *sortid, DWORD flags, WCHAR idx = (weights.val >> 16) & ~(CASE_COMPR_6 << 8); /* expansion index */ int ret = 1; + if (weights._case & CASE_COMPR_6) + ret += get_compression_weights( sortid->compr, compr_tables, src + pos, srclen - pos, &weights ); + weights._case &= case_mask; switch (weights.script)

1 0

Alexandre Julliard : kernelbase: Add support for Hangul composition in sortkeys.
by Alexandre Julliard May 30, 2022

May 30, 2022

Module: wine Branch: master Commit: abc93d0809cfcccff28511327af3f9eccee65d33 URL: https://source.winehq.org/git/wine.git/?a=commit;h=abc93d0809cfcccff2851132… Author: Alexandre Julliard <julliard(a)winehq.org> Date: Mon May 30 13:09:26 2022 +0200 kernelbase: Add support for Hangul composition in sortkeys. Signed-off-by: Alexandre Julliard <julliard(a)winehq.org> --- dlls/kernelbase/locale.c | 94 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/dlls/kernelbase/locale.c b/dlls/kernelbase/locale.c index 03b34bf99db..d189cb004f0 100644 --- a/dlls/kernelbase/locale.c +++ b/dlls/kernelbase/locale.c @@ -342,6 +342,16 @@ struct sort_expansion WCHAR exp[2]; }; +struct jamo_sort +{ + BYTE is_old; + BYTE leading; + BYTE vowel; + BYTE trailing; + BYTE weight; + BYTE pad[3]; +}; + struct sort_compression { UINT offset; @@ -412,6 +422,7 @@ static struct const struct sort_expansion *expansions; /* character expansions */ const struct sort_compression *compressions; /* character compression tables */ const WCHAR *compr_data; /* data for individual compressions */ + const struct jamo_sort *jamo; /* table for Jamo compositions */ } sort; static CRITICAL_SECTION locale_section; @@ -478,7 +489,9 @@ static void load_sortdefault_nls(void) const WORD *ctype; const UINT *table; + UINT i; SIZE_T size; + const struct sort_compression *last_compr; NtGetNlsSectionPtr( 9, 0, NULL, (void **)&header, &size ); @@ -503,6 +516,12 @@ static void load_sortdefault_nls(void) sort.compressions = (struct sort_compression *)(table + 1); sort.compr_data = (WCHAR *)(sort.compressions + sort.compr_count); + last_compr = sort.compressions + sort.compr_count - 1; + table = (UINT *)(sort.compr_data + last_compr->offset); + for (i = 0; i < 7; i++) table += last_compr->len[i] * ((i + 5) / 2); + table += 1 + table[0] / 2; /* skip multiple weights */ + sort.jamo = (struct jamo_sort *)(table + 1); + locale_sorts = RtlAllocateHeap( GetProcessHeap(), HEAP_ZERO_MEMORY, locale_table->nb_lcnames * sizeof(*locale_sorts) ); } @@ -3299,6 +3318,75 @@ static int get_compression_weights( UINT compression, const WCHAR *compr_tables[ return 0; } + +#define HANGUL_SBASE 0xac00 +#define HANGUL_LCOUNT 19 +#define HANGUL_VCOUNT 21 +#define HANGUL_TCOUNT 28 + +static int append_hangul_weights( struct sortkey *key, const WCHAR *src, int srclen, UINT except ) +{ + int leading_idx = 0x115f - 0x1100; /* leading filler */ + int vowel_idx = 0x1160 - 0x1100; /* vowel filler */ + int trailing_idx = -1; + BYTE leading_off, vowel_off, trailing_off; + union char_weights weights; + WCHAR composed; + BYTE filler_mask = 0; + int pos = 0; + + /* leading */ + if (src[pos] >= 0x1100 && src[pos] <= 0x115f) leading_idx = src[pos++] - 0x1100; + else if (src[pos] >= 0xa960 && src[pos] <= 0xa97c) leading_idx = src[pos++] - (0xa960 - 0x100); + + /* vowel */ + if (srclen > pos) + { + if (src[pos] >= 0x1160 && src[pos] <= 0x11a7) vowel_idx = src[pos++] - 0x1100; + else if (src[pos] >= 0xd7b0 && src[pos] <= 0xd7c6) vowel_idx = src[pos++] - (0xd7b0 - 0x11d); + } + + /* trailing */ + if (srclen > pos) + { + if (src[pos] >= 0x11a8 && src[pos] <= 0x11ff) trailing_idx = src[pos++] - 0x1100; + else if (src[pos] >= 0xd7cb && src[pos] <= 0xd7fb) trailing_idx = src[pos++] - (0xd7cb - 0x134); + } + + if (!sort.jamo[leading_idx].is_old && !sort.jamo[vowel_idx].is_old && + (trailing_idx == -1 || !sort.jamo[trailing_idx].is_old)) + { + /* not old Hangul, only use leading char; vowel and trailing will be handled in the next pass */ + pos = 1; + vowel_idx = 0x1160 - 0x1100; + trailing_idx = -1; + } + + leading_off = max( sort.jamo[leading_idx].leading, sort.jamo[vowel_idx].leading ); + vowel_off = max( sort.jamo[leading_idx].vowel, sort.jamo[vowel_idx].vowel ); + trailing_off = max( sort.jamo[leading_idx].trailing, sort.jamo[vowel_idx].trailing ); + if (trailing_idx != -1) trailing_off = max( trailing_off, sort.jamo[trailing_idx].trailing ); + composed = HANGUL_SBASE + (leading_off * HANGUL_VCOUNT + vowel_off) * HANGUL_TCOUNT + trailing_off; + + if (leading_idx == 0x115f - 0x1100 || vowel_idx == 0x1160 - 0x1100) + { + filler_mask = 0x80; + composed--; + } + if (composed < HANGUL_SBASE) composed = 0x3260; + + weights = get_char_weights( composed, except ); + append_sortkey( key, weights.script ); + append_sortkey( key, weights.primary ); + append_sortkey( key, 0xff ); + append_sortkey( key, sort.jamo[leading_idx].weight | filler_mask ); + append_sortkey( key, 0xff ); + append_sortkey( key, sort.jamo[vowel_idx].weight ); + append_sortkey( key, 0xff ); + append_sortkey( key, trailing_idx != -1 ? sort.jamo[trailing_idx].weight : 2 ); + return pos - 1; +} + /* put one of the elements of a sortkey into the dst buffer */ static int put_sortkey( BYTE *dst, int dstlen, int pos, const struct sortkey *key, BYTE terminator ) { @@ -3414,6 +3502,12 @@ static int append_weights( const struct sortguid *sortid, DWORD flags, &s->key_case, weights, flags, is_compare ); break; + case SCRIPT_JAMO_SPECIAL: + ret += append_hangul_weights( &s->key_primary, src + pos, srclen - pos, except ); + append_sortkey( &s->key_diacritic, 2 ); + append_sortkey( &s->key_case, 2 ); + break; + case SCRIPT_EXTENSION_A: append_sortkey( &s->key_primary, 0xfd ); append_sortkey( &s->key_primary, 0xff );

1 0

Alexandre Julliard : kernelbase: Add support for Kana extra weights in sortkeys.
by Alexandre Julliard May 30, 2022

May 30, 2022

Module: wine Branch: master Commit: b90f0be0e70d1e25e23c410f2e971dcff6638cc3 URL: https://source.winehq.org/git/wine.git/?a=commit;h=b90f0be0e70d1e25e23c410f… Author: Alexandre Julliard <julliard(a)winehq.org> Date: Mon May 30 13:11:41 2022 +0200 kernelbase: Add support for Kana extra weights in sortkeys. Signed-off-by: Alexandre Julliard <julliard(a)winehq.org> --- dlls/kernelbase/locale.c | 84 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 81 insertions(+), 3 deletions(-) diff --git a/dlls/kernelbase/locale.c b/dlls/kernelbase/locale.c index d189cb004f0..179f0436578 100644 --- a/dlls/kernelbase/locale.c +++ b/dlls/kernelbase/locale.c @@ -3318,6 +3318,48 @@ static int get_compression_weights( UINT compression, const WCHAR *compr_tables[ return 0; } +/* append the extra weights for kana prolonged sound / repeat marks */ +static int append_extra_kana_weights( struct sortkey keys[4], const WCHAR *src, int pos, UINT except, + BYTE case_mask, union char_weights *weights ) +{ + BYTE extra1 = 3, case_weight = weights->_case; + + if (weights->primary <= 1) + { + while (pos > 0) + { + union char_weights prev = get_char_weights( src[--pos], except ); + if (prev.script == SCRIPT_UNSORTABLE || prev.script == SCRIPT_NONSPACE_MARK) continue; + if (prev.script == SCRIPT_EXPANSION) return 0; + if (prev.script != SCRIPT_EASTASIA_SPECIAL) + { + *weights = prev; + return 1; + } + if (prev.primary <= 1) continue; + + case_weight = prev._case & case_mask; + if (weights->primary == 1) /* prolonged sound mark */ + { + prev.primary &= 0x87; + case_weight &= ~CASE_FULLWIDTH; + case_weight |= weights->_case & CASE_FULLWIDTH; + } + extra1 = 4 + weights->primary; + weights->primary = prev.primary; + goto done; + } + return 0; + } +done: + append_sortkey( &keys[0], 0xc4 | (case_weight & CASE_FULLSIZE) ); + append_sortkey( &keys[1], extra1 ); + append_sortkey( &keys[2], 0xc4 | (case_weight & CASE_KATAKANA) ); + append_sortkey( &keys[3], 0xc4 | (case_weight & CASE_FULLWIDTH) ); + weights->script = SCRIPT_KANA; + return 1; +} + #define HANGUL_SBASE 0xac00 #define HANGUL_LCOUNT 19 @@ -3405,6 +3447,7 @@ struct sortkey_state struct sortkey key_diacritic; struct sortkey key_case; struct sortkey key_special; + struct sortkey key_extra[4]; UINT primary_pos; BYTE buffer[3 * 128]; }; @@ -3438,15 +3481,18 @@ static void init_sortkey_state( struct sortkey_state *s, DWORD flags, UINT srcle s->key_primary.max = srclen * 8; s->key_case.max = srclen * 3; s->key_special.max = srclen * 4; + s->key_extra[2].max = s->key_extra[3].max = srclen; if (!(flags & NORM_IGNORENONSPACE)) { s->key_diacritic.max = srclen * 3; + s->key_extra[0].max = s->key_extra[1].max = srclen; } } -static void remove_unneeded_weights( const struct sortguid *sortid, struct sortkey_state *s ) +static BOOL remove_unneeded_weights( const struct sortguid *sortid, struct sortkey_state *s ) { - int i; + const BYTE ignore[4] = { 0xc4 | CASE_FULLSIZE, 0x03, 0xc4 | CASE_KATAKANA, 0xc4 | CASE_FULLWIDTH }; + int i, j; if (sortid->flags & FLAG_REVERSEDIACRITICS) reverse_sortkey( &s->key_diacritic ); @@ -3455,6 +3501,15 @@ static void remove_unneeded_weights( const struct sortguid *sortid, struct sortk for (i = s->key_case.len; i > 0; i--) if (s->key_case.buf[i - 1] > 2) break; s->key_case.len = i; + + if (!s->key_extra[2].len) return FALSE; + + for (i = 0; i < 4; i++) + { + for (j = s->key_extra[i].len; j > 0; j--) if (s->key_extra[i].buf[j - 1] != ignore[i]) break; + s->key_extra[i].len = j; + } + return TRUE; } static void free_sortkey_state( struct sortkey_state *s ) @@ -3463,6 +3518,10 @@ static void free_sortkey_state( struct sortkey_state *s ) RtlFreeHeap( GetProcessHeap(), 0, s->key_diacritic.new_buf ); RtlFreeHeap( GetProcessHeap(), 0, s->key_case.new_buf ); RtlFreeHeap( GetProcessHeap(), 0, s->key_special.new_buf ); + RtlFreeHeap( GetProcessHeap(), 0, s->key_extra[0].new_buf ); + RtlFreeHeap( GetProcessHeap(), 0, s->key_extra[1].new_buf ); + RtlFreeHeap( GetProcessHeap(), 0, s->key_extra[2].new_buf ); + RtlFreeHeap( GetProcessHeap(), 0, s->key_extra[3].new_buf ); } static int append_weights( const struct sortguid *sortid, DWORD flags, @@ -3502,6 +3561,17 @@ static int append_weights( const struct sortguid *sortid, DWORD flags, &s->key_case, weights, flags, is_compare ); break; + case SCRIPT_EASTASIA_SPECIAL: + if (!append_extra_kana_weights( s->key_extra, src, pos, except, case_mask, &weights )) + { + append_sortkey( &s->key_primary, 0xff ); + append_sortkey( &s->key_primary, 0xff ); + break; + } + weights._case = 2; + append_normal_weights( sortid, &s->key_primary, &s->key_diacritic, &s->key_case, weights, flags ); + break; + case SCRIPT_JAMO_SPECIAL: ret += append_hangul_weights( &s->key_primary, src + pos, srclen - pos, except ); append_sortkey( &s->key_diacritic, 2 ); @@ -3559,6 +3629,7 @@ static int get_sortkey( const struct sortguid *sortid, DWORD flags, struct sortkey_state s; BYTE primary_buf[256]; int ret = 0, pos = 0; + BOOL have_extra; BYTE case_mask = 0x3f; UINT except = sortid->except; const WCHAR *compr_tables[8]; @@ -3574,12 +3645,19 @@ static int get_sortkey( const struct sortguid *sortid, DWORD flags, while (pos < srclen) pos += append_weights( sortid, flags, src, srclen, pos, case_mask, except, compr_tables, &s, FALSE ); - remove_unneeded_weights( sortid, &s ); + have_extra = remove_unneeded_weights( sortid, &s ); ret = put_sortkey( dst, dstlen, ret, &s.key_primary, 0x01 ); ret = put_sortkey( dst, dstlen, ret, &s.key_diacritic, 0x01 ); ret = put_sortkey( dst, dstlen, ret, &s.key_case, 0x01 ); + if (have_extra) + { + ret = put_sortkey( dst, dstlen, ret, &s.key_extra[0], 0xff ); + ret = put_sortkey( dst, dstlen, ret, &s.key_extra[1], 0x02 ); + ret = put_sortkey( dst, dstlen, ret, &s.key_extra[2], 0xff ); + ret = put_sortkey( dst, dstlen, ret, &s.key_extra[3], 0xff ); + } if (dstlen > ret) dst[ret] = 0x01; ret++;

1 0

Alexandre Julliard : kernelbase: Add support for SORT_DIGITSASNUMBERS in sortkeys.
by Alexandre Julliard May 30, 2022

May 30, 2022

Module: wine Branch: master Commit: bc440c9c6dcb819929d7fd3cee93531c3eafd852 URL: https://source.winehq.org/git/wine.git/?a=commit;h=bc440c9c6dcb819929d7fd3c… Author: Alexandre Julliard <julliard(a)winehq.org> Date: Mon May 30 13:13:03 2022 +0200 kernelbase: Add support for SORT_DIGITSASNUMBERS in sortkeys. Signed-off-by: Alexandre Julliard <julliard(a)winehq.org> --- dlls/kernelbase/locale.c | 63 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/dlls/kernelbase/locale.c b/dlls/kernelbase/locale.c index 179f0436578..ec6e7047be6 100644 --- a/dlls/kernelbase/locale.c +++ b/dlls/kernelbase/locale.c @@ -3318,6 +3318,56 @@ static int get_compression_weights( UINT compression, const WCHAR *compr_tables[ return 0; } +/* get the zero digit for the digit character range that contains 'ch' */ +static WCHAR get_digit_zero_char( WCHAR ch ) +{ + static const WCHAR zeroes[] = + { + 0x0030, 0x0660, 0x06f0, 0x0966, 0x09e6, 0x0a66, 0x0ae6, 0x0b66, 0x0be6, 0x0c66, + 0x0ce6, 0x0d66, 0x0e50, 0x0ed0, 0x0f20, 0x1040, 0x1090, 0x17e0, 0x1810, 0x1946, + 0x1bb0, 0x1c40, 0x1c50, 0xa620, 0xa8d0, 0xa900, 0xaa50, 0xff10 + }; + int min = 0, max = ARRAY_SIZE( zeroes ) - 1; + + while (min <= max) + { + int pos = (min + max) / 2; + if (zeroes[pos] <= ch && zeroes[pos] + 9 >= ch) return zeroes[pos]; + if (zeroes[pos] < ch) min = pos + 1; + else max = pos - 1; + } + return 0; +} + +/* append weights for digits when using SORT_DIGITSASNUMBERS */ +/* return the number of extra chars to skip */ +static int append_digit_weights( struct sortkey *key, const WCHAR *src, UINT srclen ) +{ + UINT i, zero, len, lzero; + BYTE val, values[19]; + + if (!(zero = get_digit_zero_char( *src ))) return -1; + + values[0] = *src - zero; + for (len = 1; len < ARRAY_SIZE(values) && len < srclen; len++) + { + if (src[len] < zero || src[len] > zero + 9) break; + values[len] = src[len] - zero; + } + for (lzero = 0; lzero < len; lzero++) if (values[lzero]) break; + + append_sortkey( key, SCRIPT_DIGIT ); + append_sortkey( key, 2 ); + append_sortkey( key, 2 + len - lzero ); + for (i = lzero, val = 2; i < len; i++) + { + if ((len - i) % 2) append_sortkey( key, (val << 4) + values[i] + 2 ); + else val = values[i] + 2; + } + append_sortkey( key, 0xfe - lzero ); + return len - 1; +} + /* append the extra weights for kana prolonged sound / repeat marks */ static int append_extra_kana_weights( struct sortkey keys[4], const WCHAR *src, int pos, UINT except, BYTE case_mask, union char_weights *weights ) @@ -3614,6 +3664,19 @@ static int append_weights( const struct sortguid *sortid, DWORD flags, append_sortkey( &s->key_case, weights._case ); break; + case SCRIPT_DIGIT: + if (flags & SORT_DIGITSASNUMBERS) + { + int len = append_digit_weights( &s->key_primary, src + pos, srclen - pos ); + if (len >= 0) + { + ret += len; + append_sortkey( &s->key_diacritic, weights.diacritic ); + append_sortkey( &s->key_case, weights._case ); + break; + } + } + /* fall through */ default: append_normal_weights( sortid, &s->key_primary, &s->key_diacritic, &s->key_case, weights, flags ); break;

1 0

Francois Gouget : testbot: Use GetSortedItems() to sort the Steps.
by Alexandre Julliard May 30, 2022

May 30, 2022

Module: tools Branch: master Commit: 068575b74b9d655869a1e43473f01b0b34ba588f URL: https://source.winehq.org/git/tools.git/?a=commit;h=068575b74b9d655869a1e43… Author: Francois Gouget <fgouget(a)codeweavers.com> Date: Mon May 30 18:44:06 2022 +0200 testbot: Use GetSortedItems() to sort the Steps. Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com> Signed-off-by: Alexandre Julliard <julliard(a)winehq.org> --- testbot/lib/WineTestBot/Jobs.pm | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/testbot/lib/WineTestBot/Jobs.pm b/testbot/lib/WineTestBot/Jobs.pm index 82cac66..778a0f4 100644 --- a/testbot/lib/WineTestBot/Jobs.pm +++ b/testbot/lib/WineTestBot/Jobs.pm @@ -218,8 +218,7 @@ sub UpdateStatus($) my %Has; my $Steps = $self->Steps->Clone(); - my @SortedSteps = sort { $a->No <=> $b->No } @{$Steps->GetItems()}; - foreach my $Step (@SortedSteps) + foreach my $Step (@{$Steps->GetSortedItems()}) { my $Skip; if ($Step->PreviousNo) @@ -419,8 +418,7 @@ sub Restart($) } my $Steps = $self->Steps; - my @SortedSteps = sort { $a->No <=> $b->No } @{$Steps->GetItems()}; - foreach my $Step (@SortedSteps) + foreach my $Step (@{$Steps->GetSortedItems()}) { my $Tasks = $Step->Tasks; foreach my $Task (@{$Tasks->GetItems()})

1 0

Francois Gouget : testbot: Fix the type of inherited auto-increment columns.
by Alexandre Julliard May 30, 2022

May 30, 2022

Module: tools Branch: master Commit: 0833f114a3db28343f89a03b27748ff2d9306ac9 URL: https://source.winehq.org/git/tools.git/?a=commit;h=0833f114a3db28343f89a03… Author: Francois Gouget <fgouget(a)codeweavers.com> Date: Mon May 30 18:44:40 2022 +0200 testbot: Fix the type of inherited auto-increment columns. Some objects, such as jobs, use a database-generated auto-increment integer as their primary key. However objects that 'inherit' such primary keys, such as Steps, do not. So the corresponding property descriptor should be a regular integer. Otherwise the DBIBackend will try to fetch a unique id that does not exist. Fortunately this issue has so far been masked by the 'master columns' mechanism. Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com> Signed-off-by: Alexandre Julliard <julliard(a)winehq.org> --- testbot/lib/WineTestBot/Records.pm | 2 +- testbot/lib/WineTestBot/Steps.pm | 2 +- testbot/lib/WineTestBot/Tasks.pm | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/testbot/lib/WineTestBot/Records.pm b/testbot/lib/WineTestBot/Records.pm index fd8e574..6002d1c 100644 --- a/testbot/lib/WineTestBot/Records.pm +++ b/testbot/lib/WineTestBot/Records.pm @@ -83,7 +83,7 @@ my @PropertyDescriptors = ( CreateBasicPropertyDescriptor("Value", "Value", !1, !1, "A", 64), ); my @FlatPropertyDescriptors = ( - CreateBasicPropertyDescriptor("RecordGroupId", "Group id", 1, 1, "S", 10), + CreateBasicPropertyDescriptor("RecordGroupId", "Group id", 1, 1, "N", 10), @PropertyDescriptors ); diff --git a/testbot/lib/WineTestBot/Steps.pm b/testbot/lib/WineTestBot/Steps.pm index 0f65d63..860dc49 100644 --- a/testbot/lib/WineTestBot/Steps.pm +++ b/testbot/lib/WineTestBot/Steps.pm @@ -247,7 +247,7 @@ my @PropertyDescriptors = ( ); SetDetailrefKeyPrefix("Step", @PropertyDescriptors); my @FlatPropertyDescriptors = ( - CreateBasicPropertyDescriptor("JobId", "Job id", 1, 1, "S", 10), + CreateBasicPropertyDescriptor("JobId", "Job id", 1, 1, "N", 10), @PropertyDescriptors ); diff --git a/testbot/lib/WineTestBot/Tasks.pm b/testbot/lib/WineTestBot/Tasks.pm index 5cc2cc6..50ce960 100644 --- a/testbot/lib/WineTestBot/Tasks.pm +++ b/testbot/lib/WineTestBot/Tasks.pm @@ -368,7 +368,7 @@ my @PropertyDescriptors = ( CreateBasicPropertyDescriptor("TestFailures", "Failures", !1, !1, "N", 6), ); my @FlatPropertyDescriptors = ( - CreateBasicPropertyDescriptor("JobId", "Job id", 1, 1, "S", 10), + CreateBasicPropertyDescriptor("JobId", "Job id", 1, 1, "N", 10), CreateBasicPropertyDescriptor("StepNo", "Step no", 1, 1, "N", 2), @PropertyDescriptors );

1 0

Francois Gouget : testbot/orm: Add documentation to the database backend.
by Alexandre Julliard May 30, 2022

May 30, 2022

Module: tools Branch: master Commit: 9aa433e3178d138cb9b4e0cbb74167292e5ca450 URL: https://source.winehq.org/git/tools.git/?a=commit;h=9aa433e3178d138cb9b4e0c… Author: Francois Gouget <fgouget(a)codeweavers.com> Date: Mon May 30 18:44:22 2022 +0200 testbot/orm: Add documentation to the database backend. Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com> Signed-off-by: Alexandre Julliard <julliard(a)winehq.org> --- testbot/lib/ObjectModel/DBIBackEnd.pm | 107 ++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/testbot/lib/ObjectModel/DBIBackEnd.pm b/testbot/lib/ObjectModel/DBIBackEnd.pm index dc527d4..94e4be6 100644 --- a/testbot/lib/ObjectModel/DBIBackEnd.pm +++ b/testbot/lib/ObjectModel/DBIBackEnd.pm @@ -63,6 +63,17 @@ sub GetDb($) return $self->{Db}; } +=pod +=over 12 + +=item C<ToDb()> + +Convert the property value to a format suitable for the database. +This is only needed for boolean and timestamp properties. + +=back +=cut + sub ToDb($$$) { my ($self, $Value, $PropertyDescriptor) = @_; @@ -101,6 +112,16 @@ sub ToDb($$$) return $Value; } +=pod +=over 12 + +=item C<FromDb()> + +Convert the database values to a format suitable for Perl. See ToDb(). + +=back +=cut + sub FromDb($$$) { my ($self, $Value, $PropertyDescriptor) = @_; @@ -132,6 +153,18 @@ sub FromDb($$$) return $Value; } +=pod +=over 12 + +=item C<BuildKeyWhere()> + +Augments the specified WHERE clause with statements to match the key columns. +It is up to the caller to then append the corresponding key components to the +query's data array. + +=back +=cut + sub BuildKeyWhere($$$) { my ($self, $PropertyDescriptors, $Where) = @_; @@ -154,6 +187,17 @@ sub BuildKeyWhere($$$) return $Where; } +=pod +=over 12 + +=item C<BuildFieldList()> + +Build the comma-separated SELECT list of columns to load from the database. +Note that only the non-master columns are loaded. + +=back +=cut + sub BuildFieldList($$) { my ($self, $PropertyDescriptors) = @_; @@ -277,6 +321,19 @@ sub GetFilterWhere($$$) die "unsupported '$Filter->{Type}' filter type"; } +=pod +=over 12 + +=item C<LoadCollection()> + +Loads the collection from the corresponding database table, filtering it on +both the master columns and the collection's filter if any. + +Note that only the non-master columns are loaded (see BuildFieldList()). + +=back +=cut + sub LoadCollection($$) { my ($self, $Collection) = @_; @@ -346,6 +403,8 @@ By design this method will not check if the Item is already present in the Collection scope. Such checks belong in the higher levels. This method will however put the Item in the Collection's scope. +Note that only the non-master columns are loaded (see BuildFieldList()). + =back =cut @@ -395,6 +454,17 @@ sub LoadItem($$$) return $Item; } +=pod +=over 12 + +=item C<BuildInsertStatement()> + +Builds the SQL statement for saving new Items to the database. +This can be reused multiple times when saving a collection. + +=back +=cut + sub BuildInsertStatement($$$$) { my ($self, $TableName, $PropertyDescriptors, $MasterColNames) = @_; @@ -432,6 +502,18 @@ sub BuildInsertStatement($$$$) return "INSERT INTO $TableName ($Fields) VALUES($PlaceHolders)"; } +=pod +=over 12 + +=item C<GetInsertData()> + +Builds a data array suitable for use with the SQL statement returned by +BuildInsertStatement() for the specified Item. This simply includes all +of the object's properties. + +=back +=cut + sub GetInsertData($$$) { my ($self, $MasterColValues, $Item) = @_; @@ -454,6 +536,17 @@ sub GetInsertData($$$) return \@Data; } +=pod +=over 12 + +=item C<BuildUpdateStatement()> + +Builds the SQL statement for saving *modified* Items to the database. +This can be reused multiple times when saving a collection. + +=back +=cut + sub BuildUpdateStatement($$$$) { my ($self, $TableName, $PropertyDescriptors, $MasterColNames) = @_; @@ -484,6 +577,20 @@ sub BuildUpdateStatement($$$$) return "UPDATE $TableName SET $Fields WHERE $Where"; } +=pod +=over 12 + +=item C<GetUpdateData()> + +Returns a data array suitable for use with the SQL statement returned by +BuildUpdateStatement() for the specified Item. It is composed of: +* The non-key properties for the SET part of the statement since only those + could have been modified. +* The key properties for the WHERE part of the statement. + +=back +=cut + sub GetUpdateData($$$) { my ($self, $MasterColValues, $Item) = @_;

1 0

Francois Gouget : testbot: Allow resuming jobs after stopping them.
by Alexandre Julliard May 30, 2022

May 30, 2022

Module: tools Branch: master Commit: d673d98e959adce6795541b413ab9a4638d49106 URL: https://source.winehq.org/git/tools.git/?a=commit;h=d673d98e959adce6795541b… Author: Francois Gouget <fgouget(a)codeweavers.com> Date: Mon May 30 18:07:29 2022 +0200 testbot: Allow resuming jobs after stopping them. This renames the 'cancel' action to 'stop' since it's reversible now. Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com> Signed-off-by: Alexandre Julliard <julliard(a)winehq.org> --- testbot/bin/Engine.pl | 6 ++-- testbot/lib/WineTestBot/Jobs.pm | 65 ++++++++++++++++++++++++++++++++++++++--- testbot/web/JobDetails.pl | 15 ++++++---- 3 files changed, 74 insertions(+), 12 deletions(-) diff --git a/testbot/bin/Engine.pl b/testbot/bin/Engine.pl index da72ffb..8885273 100755 --- a/testbot/bin/Engine.pl +++ b/testbot/bin/Engine.pl @@ -333,7 +333,7 @@ sub HandleJobControl($) { my ($Action, $JobKey) = @_; - if ($Action !~ /^(cancel|restart)$/) + if ($Action !~ /^(stop|resume|restart)$/) { LogMsg "Invalid $Action JobControl command\n"; return "0Invalid $Action JobControl command"; @@ -350,7 +350,9 @@ sub HandleJobControl($) $JobKey =~ m/^(.*)$/; $JobKey = $1; - my $ErrMessage = $Action eq "cancel" ? $Job->Cancel() : $Job->Restart(); + my $ErrMessage = $Action eq "stop" ? $Job->Stop() : + $Action eq "resume" ? $Job->Resume() : + $Job->Restart(); if (defined($ErrMessage)) { LogMsg "$Action problem: $ErrMessage\n"; diff --git a/testbot/lib/WineTestBot/Jobs.pm b/testbot/lib/WineTestBot/Jobs.pm index c082a36..82cac66 100644 --- a/testbot/lib/WineTestBot/Jobs.pm +++ b/testbot/lib/WineTestBot/Jobs.pm @@ -75,7 +75,7 @@ Once all the Steps have completed the Job's Status is updated to reflect the overall result: completed, badpatch, etc. =item * -If the Job is canceled by the user, then the Status field is set to canceled. +If the Job is stopped by the user, then the Status field is set to canceled. =back @@ -280,9 +280,9 @@ sub UpdateStatus($) =pod =over 12 -=item C<Cancel()> +=item C<Stop()> -Cancels the Job, preserving existing results. +Stops the Job, preserving existing results. More precisely, goes through all of that Job's 'queued' and 'running' tasks, killing all the running ones and marking them, and all the queued tasks, as @@ -294,7 +294,7 @@ Returns undef if successful, the error message otherwise. =back =cut -sub Cancel($) +sub Stop($) { my ($self) = @_; my $ErrMessage; @@ -339,6 +339,63 @@ sub Cancel($) =pod =over 12 +=item C<Resume()> + +Resume running the Job. + +More precisely, marks the 'skipped' and 'canceled' tasks as 'queued' +again and updates the job status accordingly. This preserves existing +results unlike Restart(). + +Returns undef if successful, the error message otherwise. + +=back +=cut + +sub Resume($) +{ + my ($self) = @_; + + if ($self->Status ne "canceled") + { + return "The job has not been stopped and is currently ". $self->Status; + } + + my $Steps = $self->Steps; + $Steps->AddFilter("Status", ["canceled", "skipped"]); + foreach my $Step (@{$Steps->GetSortedItems()}) + { + if ($Step->PreviousNo) + { + my $PrevStatus = $Steps->GetItem($Step->PreviousNo)->Status; + if ($PrevStatus ne "queued" && $PrevStatus ne "running" && + $PrevStatus ne "completed") + { + # The previous step was supposed to provide binaries but it failed. + # So this one cannot resume. + continue; + } + } + + my $Tasks = $Step->Tasks; + $Tasks->AddFilter("Status", ["canceled", "skipped"]); + foreach my $Task (@{$Tasks->GetItems()}) + { + $Task->Status("queued"); + $Task->Started(undef); + } + $Step->Status("queued"); + } + # Mark the job as queued and let UpdateStatus() set it to running if + # appropriate. + $self->Status("queued"); + $self->UpdateStatus(); + return undef; +} + +=pod +=over 12 + =item C<Restart()> Restarts the Job from scratch. diff --git a/testbot/web/JobDetails.pl b/testbot/web/JobDetails.pl index 87a5694..329ab15 100644 --- a/testbot/web/JobDetails.pl +++ b/testbot/web/JobDetails.pl @@ -173,7 +173,8 @@ sub CanDoJobControl($$) } my %AllowedStatus = ( - "cancel" => {"queued" => 1, "running" => 1}, + "stop" => {"queued" => 1, "running" => 1}, + "resume" => {"canceled" => 1}, "restart" => {"boterror" => 1, "canceled" => 1}, ); return $AllowedStatus{$Action}->{$Job->Status} ? undef : @@ -210,17 +211,19 @@ sub GetActions($) { my ($self) = @_; - # These are mutually exclusive - return ["Cancel job"] if (!defined $self->CanDoJobControl("cancel")); - return ["Restart job"] if (!defined $self->CanDoJobControl("restart")); - return []; + my @Actions; + push @Actions, "Stop job" if (!$self->CanDoJobControl("stop")); + push @Actions, "Resume job" if (!$self->CanDoJobControl("resume")); + push @Actions, "Restart job" if (!$self->CanDoJobControl("restart")); + return \@Actions; } sub OnAction($$) { my ($self, $Action) = @_; - return $Action eq "Cancel job" ? $self->OnJobControl("cancel") : + return $Action eq "Stop job" ? $self->OnJobControl("stop") : + $Action eq "Resume job" ? $self->OnJobControl("resume") : $Action eq "Restart job" ? $self->OnJobControl("restart") : $self->SUPER::OnAction($Action); }

1 0