[PATCH 0/5] MR2424: widl: Cleanup lexer a bit and simplify string lexing.

List overview All Threads

newer

older

[PATCH 0/5] MR2422: win32u:...

[PATCH 0/1] MR2429: Add stub...

Rémi Bernon (＠rbernon)

17 Mar 2023 17 Mar '23

8:45 a.m.

-- https://gitlab.winehq.org/wine/wine/-/merge_requests/2424

Show replies by date

Rémi Bernon

17 Mar 17 Mar

8:45 a.m.

New subject: [PATCH 1/5] widl: Group <INITIAL,ATTR> tokens together.

From: Rémi Bernon rbernon@codeweavers.com

--- tools/widl/parser.l | 52 +++++++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 23 deletions(-)

diff --git a/tools/widl/parser.l b/tools/widl/parser.l index 92955516e9e..8698f084354 100644 --- a/tools/widl/parser.l +++ b/tools/widl/parser.l @@ -140,7 +140,6 @@ struct uuid *parse_uuid(const char *u) */ %% <INITIAL>^{ws}*#{ws}*pragma{ws}+ yy_push_state(PP_PRAGMA); -<INITIAL,ATTR>^{ws}*#{ws}* yy_push_state(PP_LINE); <PP_LINE>[^\n]* { int lineno; char *cptr, *fname; @@ -202,40 +201,47 @@ struct uuid *parse_uuid(const char *u) <SQUOTE>\' addcchar(yytext[1]); <QUOTE,WSTRQUOTE,SQUOTE>\. addcchar('\'); addcchar(yytext[1]); <QUOTE,WSTRQUOTE,SQUOTE>. addcchar(yytext[0]); -<INITIAL,ATTR>[ yy_push_state(ATTR); return '['; <ATTR>] yy_pop_state(); return ']'; <ATTR>{cident} return attr_token(yytext, yylval); <ATTR>{uuid} { yylval->uuid = parse_uuid(yytext); return aUUID; } -<INITIAL,ATTR>{hex} { - yylval->num = xstrtoul(yytext, NULL, 0); - return aHEXNUM; - } -<INITIAL,ATTR>{int} { - yylval->num = xstrtoul(yytext, NULL, 0); - return aNUM; - } <INITIAL>{double} { yylval->dbl = strtod(yytext, NULL); return aDOUBLE; } SAFEARRAY{ws}*/( return tSAFEARRAY; {cident} return kw_token(yytext, yylval); -<INITIAL,ATTR>\n line_number++; -<INITIAL,ATTR>{ws} -<INITIAL,ATTR><< return SHL; -<INITIAL,ATTR>>> return SHR; -<INITIAL,ATTR>-> return MEMBERPTR; -<INITIAL,ATTR>== return EQUALITY; -<INITIAL,ATTR>!= return INEQUALITY; -<INITIAL,ATTR>>= return GREATEREQUAL; -<INITIAL,ATTR><= return LESSEQUAL; -<INITIAL,ATTR>|| return LOGICALOR; -<INITIAL,ATTR>&& return LOGICALAND; -<INITIAL,ATTR>... return ELLIPSIS; -<INITIAL,ATTR>. return yytext[0]; + +<INITIAL,ATTR>{ + ^{ws}*#{ws}* { yy_push_state(PP_LINE); } + [ { yy_push_state(ATTR); return '['; } + + {hex} { + yylval->num = xstrtoul(yytext, NULL, 0); + return aHEXNUM; + } + {int} { + yylval->num = xstrtoul(yytext, NULL, 0); + return aNUM; + } + + \n { line_number++; } + {ws} {} + << { return SHL; } + >> { return SHR; } + -> { return MEMBERPTR; } + == { return EQUALITY; } + != { return INEQUALITY; } + >= { return GREATEREQUAL; } + <= { return LESSEQUAL; } + || { return LOGICALOR; } + && { return LOGICALAND; } + ... { return ELLIPSIS; } + . { return yytext[0]; } +} + <<EOF>> { if (!list_empty( &import_stack )) return aEOF;

-- GitLab https://gitlab.winehq.org/wine/wine/-/merge_requests/2424

Rémi Bernon

8:45 a.m.

New subject: [PATCH 2/5] widl: Group <INITIAL> tokens together.

From: Rémi Bernon rbernon@codeweavers.com

--- tools/widl/parser.l | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/tools/widl/parser.l b/tools/widl/parser.l index 8698f084354..d534b23bf02 100644 --- a/tools/widl/parser.l +++ b/tools/widl/parser.l @@ -139,7 +139,6 @@ struct uuid *parse_uuid(const char *u) ************************************************************************** */ %% -<INITIAL>^{ws}*#{ws}*pragma{ws}+ yy_push_state(PP_PRAGMA); <PP_LINE>[^\n]* { int lineno; char *cptr, *fname; @@ -177,7 +176,6 @@ struct uuid *parse_uuid(const char *u) yy_pop_state(); } <PP_PRAGMA>[^\n]* yylval->str = xstrdup(yytext); yy_pop_state(); return aPRAGMA; -<INITIAL>^{ws}*midl_pragma{ws}+warning return tPRAGMA_WARNING; <INITIAL,ATTR>" yy_push_state(QUOTE); cbufidx = 0; <QUOTE>" { yy_pop_state(); @@ -207,10 +205,17 @@ struct uuid *parse_uuid(const char *u) yylval->uuid = parse_uuid(yytext); return aUUID; } -<INITIAL>{double} { - yylval->dbl = strtod(yytext, NULL); - return aDOUBLE; - } + +<INITIAL>{ + ^{ws}*#{ws}*pragma{ws}+ { yy_push_state( PP_PRAGMA ); } + ^{ws}*midl_pragma{ws}+warning { return tPRAGMA_WARNING; } + + {double} { + yylval->dbl = strtod( yytext, NULL ); + return aDOUBLE; + } +} + SAFEARRAY{ws}*/( return tSAFEARRAY; {cident} return kw_token(yytext, yylval);

-- GitLab https://gitlab.winehq.org/wine/wine/-/merge_requests/2424

Rémi Bernon

8:45 a.m.

New subject: [PATCH 3/5] widl: Group <ATTR> tokens together.

From: Rémi Bernon rbernon@codeweavers.com

--- tools/widl/parser.l | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/tools/widl/parser.l b/tools/widl/parser.l index d534b23bf02..02db1cfeeb1 100644 --- a/tools/widl/parser.l +++ b/tools/widl/parser.l @@ -199,12 +199,16 @@ struct uuid *parse_uuid(const char *u) <SQUOTE>\' addcchar(yytext[1]); <QUOTE,WSTRQUOTE,SQUOTE>\. addcchar('\'); addcchar(yytext[1]); <QUOTE,WSTRQUOTE,SQUOTE>. addcchar(yytext[0]); -<ATTR>] yy_pop_state(); return ']'; -<ATTR>{cident} return attr_token(yytext, yylval); -<ATTR>{uuid} { - yylval->uuid = parse_uuid(yytext); - return aUUID; - } + +<ATTR>{ + ] { yy_pop_state(); return ']'; } + + {cident} { return attr_token( yytext, yylval ); } + {uuid} { + yylval->uuid = parse_uuid( yytext ); + return aUUID; + } +}

<INITIAL>{ ^{ws}*#{ws}*pragma{ws}+ { yy_push_state( PP_PRAGMA ); }

-- GitLab https://gitlab.winehq.org/wine/wine/-/merge_requests/2424

Rémi Bernon

8:45 a.m.

New subject: [PATCH 4/5] widl: Introduce a new helper to produce num tokens.

From: Rémi Bernon rbernon@codeweavers.com

--- tools/widl/parser.l | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/tools/widl/parser.l b/tools/widl/parser.l index 02db1cfeeb1..539e6f381ed 100644 --- a/tools/widl/parser.l +++ b/tools/widl/parser.l @@ -131,6 +131,12 @@ struct uuid *parse_uuid(const char *u) return uuid; }

+static int token_num( int token, const char *yytext, YYSTYPE *yylval ) +{ + yylval->num = xstrtoul( yytext, NULL, 0 ); + return token; +} + %}

/* @@ -227,14 +233,8 @@ SAFEARRAY{ws}*/( return tSAFEARRAY; ^{ws}*#{ws}* { yy_push_state(PP_LINE); } [ { yy_push_state(ATTR); return '['; }

- {hex} { - yylval->num = xstrtoul(yytext, NULL, 0); - return aHEXNUM; - } - {int} { - yylval->num = xstrtoul(yytext, NULL, 0); - return aNUM; - } + {hex} { return token_num( aHEXNUM, yytext, yylval ); } + {int} { return token_num( aNUM, yytext, yylval ); }

\n { line_number++; } {ws} {}

-- GitLab https://gitlab.winehq.org/wine/wine/-/merge_requests/2424

Rémi Bernon

8:45 a.m.

New subject: [PATCH 5/5] widl: Simplify string literals lexing.

From: Rémi Bernon rbernon@codeweavers.com

--- tools/widl/parser.l | 75 +++++++++++++++------------------------------ 1 file changed, 24 insertions(+), 51 deletions(-)

diff --git a/tools/widl/parser.l b/tools/widl/parser.l index 539e6f381ed..066d61a5a84 100644 --- a/tools/widl/parser.l +++ b/tools/widl/parser.l @@ -35,12 +35,9 @@ hex 0(x|X){hexd}+({l_suffix}?{u_suffix}?|{u_suffix}?{l_suffix}?)? uuid {hexd}{8}-{hexd}{4}-{hexd}{4}-{hexd}{4}-{hexd}{12} double [0-9]+.[0-9]+([eE][+-]?[0-9]+)*

-%x QUOTE -%x WSTRQUOTE %x ATTR %x PP_LINE %x PP_PRAGMA -%x SQUOTE

@@ -67,13 +64,6 @@ double [0-9]+.[0-9]+([eE][+-]?[0-9]+)*

#include "parser.tab.h"

-static void addcchar(char c); -static char *get_buffered_cstring(void); - -static char *cbuffer; -static int cbufidx; -static int cbufalloc = 0; - static int kw_token(const char *kw, YYSTYPE *yylval); static int attr_token(const char *kw, YYSTYPE *yylval);

@@ -131,6 +121,26 @@ struct uuid *parse_uuid(const char *u) return uuid; }

+static int token_str( int token, const char *str, YYSTYPE *yylval ) +{ + char *tmp = xstrdup( str ); + + if (token == aWSTRING || token == aSTRING || token == aSQSTRING) + { + char *src, *dst; + src = dst = ++tmp; /* skip first quote */ + while (*src) + { + if (*src == '\') src++; + *dst++ = *src++; + } + dst[-1] = 0; /* strip last quote */ + } + + yylval->str = tmp; + return token; +} + static int token_num( int token, const char *yytext, YYSTYPE *yylval ) { yylval->num = xstrtoul( yytext, NULL, 0 ); @@ -182,29 +192,6 @@ static int token_num( int token, const char *yytext, YYSTYPE *yylval ) yy_pop_state(); } <PP_PRAGMA>[^\n]* yylval->str = xstrdup(yytext); yy_pop_state(); return aPRAGMA; -<INITIAL,ATTR>" yy_push_state(QUOTE); cbufidx = 0; -<QUOTE>" { - yy_pop_state(); - yylval->str = get_buffered_cstring(); - return aSTRING; - } -<INITIAL,ATTR>L" yy_push_state(WSTRQUOTE); cbufidx = 0; -<WSTRQUOTE>" { - yy_pop_state(); - yylval->str = get_buffered_cstring(); - return aWSTRING; - } -<INITIAL,ATTR>' yy_push_state(SQUOTE); cbufidx = 0; -<SQUOTE>' { - yy_pop_state(); - yylval->str = get_buffered_cstring(); - return aSQSTRING; - } -<QUOTE,WSTRQUOTE,SQUOTE>\\ | -<QUOTE,WSTRQUOTE>\" addcchar(yytext[1]); -<SQUOTE>\' addcchar(yytext[1]); -<QUOTE,WSTRQUOTE,SQUOTE>\. addcchar('\'); addcchar(yytext[1]); -<QUOTE,WSTRQUOTE,SQUOTE>. addcchar(yytext[0]);

<ATTR>{ ] { yy_pop_state(); return ']'; } @@ -236,6 +223,10 @@ SAFEARRAY{ws}*/( return tSAFEARRAY; {hex} { return token_num( aHEXNUM, yytext, yylval ); } {int} { return token_num( aNUM, yytext, yylval ); }

+ L"(\.|[^"\])*" { return token_str( aWSTRING, yytext + 1, yylval ); } + "(\.|[^"\])*" { return token_str( aSTRING, yytext, yylval ); } + '(\.|[^'\])*' { return token_str( aSQSTRING, yytext, yylval ); } + \n { line_number++; } {ws} {} << { return SHL; } @@ -508,24 +499,6 @@ static int attr_token(const char *kw, YYSTYPE *yylval) return kw_token(kw, yylval); }

-static void addcchar(char c) -{ - if(cbufidx >= cbufalloc) - { - cbufalloc += 1024; - cbuffer = xrealloc(cbuffer, cbufalloc * sizeof(cbuffer[0])); - if(cbufalloc > 65536) - parser_warning("Reallocating string buffer larger than 64kB\n"); - } - cbuffer[cbufidx++] = c; -} - -static char *get_buffered_cstring(void) -{ - addcchar(0); - return xstrdup(cbuffer); -} - void pop_import(void) { struct list *entry = list_head( &import_stack );

-- GitLab https://gitlab.winehq.org/wine/wine/-/merge_requests/2424

Jacek Caban (＠jacek)

1:21 p.m.

New subject: [PATCH 0/5] MR2424: widl: Cleanup lexer a bit and simplify string lexing. - approved

This merge request was approved by Jacek Caban.

-- https://gitlab.winehq.org/wine/wine/-/merge_requests/2424

693

Age (days ago)

693

Last active (days ago)

wine-gitlab@winehq.org

6 comments

3 participants

tags (0)

participants (3)

Jacek Caban (＠jacek)
Rémi Bernon
Rémi Bernon (＠rbernon)