[PATCH vkd3d 1/5] vkd3d-shader: Implement macro body expansion.
Signed-off-by: Zebediah Figura <zfigura(a)codeweavers.com> --- Makefile.am | 1 - libs/vkd3d-shader/preproc.h | 18 ++++++++ libs/vkd3d-shader/preproc.l | 83 +++++++++++++++++++++++++++++++------ libs/vkd3d-shader/preproc.y | 24 +++++++++-- tests/hlsl_d3d12.c | 9 ++-- 5 files changed, 113 insertions(+), 22 deletions(-) diff --git a/Makefile.am b/Makefile.am index bf1d7bfa..514f0506 100644 --- a/Makefile.am +++ b/Makefile.am @@ -243,7 +243,6 @@ XFAIL_TESTS = \ tests/hlsl-vector-indexing.shader_test \ tests/hlsl-vector-indexing-uniform.shader_test \ tests/math.shader_test \ - tests/preproc-if.shader_test \ tests/preproc-ifdef.shader_test \ tests/preproc-if-expr.shader_test \ tests/preproc-invalid.shader_test \ diff --git a/libs/vkd3d-shader/preproc.h b/libs/vkd3d-shader/preproc.h index 16b579e0..7d6bda2a 100644 --- a/libs/vkd3d-shader/preproc.h +++ b/libs/vkd3d-shader/preproc.h @@ -50,10 +50,24 @@ struct preproc_file size_t if_count, if_stack_size; }; +struct preproc_text +{ + struct vkd3d_string_buffer text; + struct vkd3d_shader_location location; +}; + +struct preproc_expansion +{ + struct preproc_buffer buffer; + const struct preproc_text *text; +}; + struct preproc_macro { struct rb_entry entry; char *name; + + struct preproc_text body; }; struct preproc_ctx @@ -67,6 +81,9 @@ struct preproc_ctx struct preproc_file *file_stack; size_t file_count, file_stack_size; + struct preproc_expansion *expansion_stack; + size_t expansion_count, expansion_stack_size; + struct rb_tree macros; int current_directive; @@ -78,6 +95,7 @@ struct preproc_ctx }; void preproc_close_include(struct preproc_ctx *ctx, const struct vkd3d_shader_code *code) DECLSPEC_HIDDEN; +struct preproc_macro *preproc_find_macro(struct preproc_ctx *ctx, const char *name) DECLSPEC_HIDDEN; void preproc_free_macro(struct preproc_macro *macro) DECLSPEC_HIDDEN; bool preproc_push_include(struct preproc_ctx *ctx, char *filename, const struct vkd3d_shader_code *code) DECLSPEC_HIDDEN; void preproc_warning(struct preproc_ctx *ctx, const struct vkd3d_shader_location *loc, diff --git a/libs/vkd3d-shader/preproc.l b/libs/vkd3d-shader/preproc.l index 476c1e59..86ca4733 100644 --- a/libs/vkd3d-shader/preproc.l +++ b/libs/vkd3d-shader/preproc.l @@ -169,27 +169,41 @@ static void update_location(struct preproc_ctx *ctx) static void preproc_pop_buffer(struct preproc_ctx *ctx) { - struct preproc_file *file = preproc_get_top_file(ctx); + if (ctx->expansion_count) + { + struct preproc_expansion *exp = &ctx->expansion_stack[ctx->expansion_count - 1]; - if (ctx->file_count > 1) - preproc_close_include(ctx, &file->code); + yy_delete_buffer(exp->buffer.lexer_buffer, ctx->scanner); - if (file->if_count) + --ctx->expansion_count; + TRACE("Expansion stack size is now %zu.\n", ctx->expansion_count); + } + else { - const struct vkd3d_shader_location loc = {.source_name = file->filename}; + struct preproc_file *file = preproc_get_top_file(ctx); - preproc_warning(ctx, &loc, VKD3D_SHADER_WARNING_PP_UNTERMINATED_IF, "Unterminated #if block."); - } - vkd3d_free(file->if_stack); + if (ctx->file_count > 1) + preproc_close_include(ctx, &file->code); + + if (file->if_count) + { + const struct vkd3d_shader_location loc = {.source_name = file->filename}; - vkd3d_free(file->filename); + preproc_warning(ctx, &loc, VKD3D_SHADER_WARNING_PP_UNTERMINATED_IF, "Unterminated #if block."); + } + vkd3d_free(file->if_stack); - yy_delete_buffer(file->buffer.lexer_buffer, ctx->scanner); + vkd3d_free(file->filename); - --ctx->file_count; - TRACE("File stack size is now %zu.\n", ctx->file_count); + yy_delete_buffer(file->buffer.lexer_buffer, ctx->scanner); - if (ctx->file_count) + --ctx->file_count; + TRACE("File stack size is now %zu.\n", ctx->file_count); + } + + if (ctx->expansion_count) + yy_switch_to_buffer(ctx->expansion_stack[ctx->expansion_count - 1].buffer.lexer_buffer, ctx->scanner); + else if (ctx->file_count) yy_switch_to_buffer(ctx->file_stack[ctx->file_count - 1].buffer.lexer_buffer, ctx->scanner); } @@ -218,6 +232,21 @@ static int return_token(int token, YYSTYPE *lval, const char *text) return token; } +static bool preproc_push_expansion(struct preproc_ctx *ctx, const struct preproc_text *text) +{ + struct preproc_expansion *exp; + + if (!vkd3d_array_reserve((void **)&ctx->expansion_stack, &ctx->expansion_stack_size, + ctx->expansion_count + 1, sizeof(*ctx->expansion_stack))) + return false; + exp = &ctx->expansion_stack[ctx->expansion_count++]; + exp->text = text; + exp->buffer.lexer_buffer = yy_scan_bytes(text->text.buffer, text->text.content_size, ctx->scanner); + exp->buffer.location = text->location; + TRACE("Expansion stack size is now %zu.\n", ctx->expansion_count); + return true; +} + int yylex(YYSTYPE *lval, YYLTYPE *lloc, yyscan_t scanner) { struct preproc_ctx *ctx = yyget_extra(scanner); @@ -239,6 +268,10 @@ int yylex(YYSTYPE *lval, YYLTYPE *lloc, yyscan_t scanner) if (!(token = preproc_lexer_lex(lval, lloc, scanner))) { ctx->last_was_eof = true; + + /* If we have reached the end of an included file, inject a newline. */ + if (ctx->expansion_count) + continue; token = T_NEWLINE; text = "\n"; } @@ -288,6 +321,29 @@ int yylex(YYSTYPE *lval, YYLTYPE *lloc, yyscan_t scanner) continue; } + if (token == T_IDENTIFIER) + { + struct preproc_macro *macro; + + switch (ctx->current_directive) + { + case T_DEFINE: + case T_IFDEF: + case T_IFNDEF: + case T_UNDEF: + /* Return identifiers verbatim. */ + return return_token(token, lval, text); + } + + /* Otherwise, expand a macro if there is one. */ + + if ((macro = preproc_find_macro(ctx, text))) + { + preproc_push_expansion(ctx, ¯o->body); + continue; + } + } + if (ctx->current_directive) return return_token(token, lval, text); @@ -365,6 +421,7 @@ int preproc_lexer_parse(const struct vkd3d_shader_compile_info *compile_info, rb_destroy(&ctx.macros, preproc_macro_rb_free, NULL); vkd3d_free(ctx.file_stack); + vkd3d_free(ctx.expansion_stack); if (ctx.error) { diff --git a/libs/vkd3d-shader/preproc.y b/libs/vkd3d-shader/preproc.y index 53ce3e57..17bc6499 100644 --- a/libs/vkd3d-shader/preproc.y +++ b/libs/vkd3d-shader/preproc.y @@ -68,7 +68,7 @@ static void yyerror(const YYLTYPE *loc, void *scanner, struct preproc_ctx *ctx, preproc_error(ctx, loc, VKD3D_SHADER_ERROR_PP_INVALID_SYNTAX, "%s", string); } -static struct preproc_macro *preproc_find_macro(struct preproc_ctx *ctx, const char *name) +struct preproc_macro *preproc_find_macro(struct preproc_ctx *ctx, const char *name) { struct rb_entry *entry; @@ -77,7 +77,8 @@ static struct preproc_macro *preproc_find_macro(struct preproc_ctx *ctx, const c return NULL; } -static bool preproc_add_macro(struct preproc_ctx *ctx, const struct vkd3d_shader_location *loc, char *name) +static bool preproc_add_macro(struct preproc_ctx *ctx, const struct vkd3d_shader_location *loc, char *name, + const struct vkd3d_shader_location *body_loc, struct vkd3d_string_buffer *body) { struct preproc_macro *macro; int ret; @@ -94,6 +95,8 @@ static bool preproc_add_macro(struct preproc_ctx *ctx, const struct vkd3d_shader if (!(macro = vkd3d_malloc(sizeof(*macro)))) return false; macro->name = name; + macro->body.text = *body; + macro->body.location = *body_loc; ret = rb_put(&ctx->macros, name, ¯o->entry); assert(!ret); return true; @@ -102,6 +105,7 @@ static bool preproc_add_macro(struct preproc_ctx *ctx, const struct vkd3d_shader void preproc_free_macro(struct preproc_macro *macro) { vkd3d_free(macro->name); + vkd3d_string_buffer_cleanup(¯o->body.text); vkd3d_free(macro); } @@ -265,6 +269,7 @@ static const void *get_parent_data(struct preproc_ctx *ctx) { char *string; uint32_t integer; + struct vkd3d_string_buffer string_buffer; } %token <string> T_IDENTIFIER @@ -286,6 +291,7 @@ static const void *get_parent_data(struct preproc_ctx *ctx) %type <integer> expr %type <string> body_token +%type <string_buffer> body_text %% @@ -298,8 +304,16 @@ shader_text body_text : %empty + { + vkd3d_string_buffer_init(&$$); + } | body_text body_token { + if (vkd3d_string_buffer_printf(&$$, "%s ", $2) < 0) + { + vkd3d_free($2); + YYABORT; + } vkd3d_free($2); } @@ -311,8 +325,12 @@ body_token directive : T_DEFINE T_IDENTIFIER body_text T_NEWLINE { - if (!preproc_add_macro(ctx, &@$, $2)) + if (!preproc_add_macro(ctx, &@$, $2, &@3, &$3)) + { + vkd3d_free($2); + vkd3d_string_buffer_cleanup(&$3); YYABORT; + } } | T_UNDEF T_IDENTIFIER T_NEWLINE { diff --git a/tests/hlsl_d3d12.c b/tests/hlsl_d3d12.c index 54ab94de..ac616c23 100644 --- a/tests/hlsl_d3d12.c +++ b/tests/hlsl_d3d12.c @@ -349,10 +349,10 @@ static void test_preprocess(void) for (i = 0; i < ARRAY_SIZE(tests); ++i) { - if (i == 6 || i == 10 || i == 11) + if (i == 10 || i == 11) continue; vkd3d_test_set_context("Source \"%s\"", tests[i].source); - todo_if (i <= 4 || (i >= 9 && i <= 14)) + todo_if (i <= 4 || (i >= 9 && i <= 14) || i == 43) check_preprocess(tests[i].source, NULL, NULL, tests[i].present, tests[i].absent); } vkd3d_test_set_context(NULL); @@ -400,12 +400,11 @@ static void test_preprocess(void) macros[1].Definition = "KEY2"; todo check_preprocess("KEY", macros, NULL, "value", NULL); - if (0) - todo check_preprocess(test_include_top, NULL, &test_include, "pass", "fail"); + check_preprocess(test_include_top, NULL, &test_include, "pass", "fail"); ok(!refcount_file1, "Got %d references to file1.\n", refcount_file1); ok(!refcount_file2, "Got %d references to file1.\n", refcount_file2); ok(!refcount_file3, "Got %d references to file1.\n", refcount_file3); - todo ok(include_count_file2 == 2, "file2 was included %u times.\n", include_count_file2); + ok(include_count_file2 == 2, "file2 was included %u times.\n", include_count_file2); /* Macro invocation spread across multiple files. */ if (0) -- 2.30.0
Signed-off-by: Zebediah Figura <zfigura(a)codeweavers.com> --- libs/vkd3d-shader/preproc.l | 5 ++- libs/vkd3d-shader/preproc.y | 72 +++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/libs/vkd3d-shader/preproc.l b/libs/vkd3d-shader/preproc.l index 86ca4733..77f4f0dc 100644 --- a/libs/vkd3d-shader/preproc.l +++ b/libs/vkd3d-shader/preproc.l @@ -70,6 +70,7 @@ IDENTIFIER [A-Za-z_][A-Za-z0-9_]* <C_COMMENT,CXX_COMMENT><<EOF>> {yy_pop_state(yyscanner);} <C_COMMENT,CXX_COMMENT>. {} +<INITIAL>{IDENTIFIER}/\( {return T_IDENTIFIER_PAREN;} <INITIAL>{IDENTIFIER} {return T_IDENTIFIER;} /* We have no use for floats, but shouldn't parse them as integers. */ @@ -141,6 +142,7 @@ IDENTIFIER [A-Za-z_][A-Za-z0-9_]* } <INITIAL>{WS}+ {} +<INITIAL>[(),] {return yytext[0];} <INITIAL>. {return T_TEXT;} %% @@ -221,6 +223,7 @@ static int return_token(int token, YYSTYPE *lval, const char *text) switch (token) { case T_IDENTIFIER: + case T_IDENTIFIER_PAREN: case T_INTEGER: case T_STRING: case T_TEXT: @@ -321,7 +324,7 @@ int yylex(YYSTYPE *lval, YYLTYPE *lloc, yyscan_t scanner) continue; } - if (token == T_IDENTIFIER) + if (token == T_IDENTIFIER || token == T_IDENTIFIER_PAREN) { struct preproc_macro *macro; diff --git a/libs/vkd3d-shader/preproc.y b/libs/vkd3d-shader/preproc.y index 17bc6499..3cb7de61 100644 --- a/libs/vkd3d-shader/preproc.y +++ b/libs/vkd3d-shader/preproc.y @@ -28,6 +28,12 @@ #define PREPROC_YYLTYPE struct vkd3d_shader_location +struct parse_arg_names +{ + char **args; + size_t count; +}; + } %code provides @@ -254,6 +260,15 @@ static const void *get_parent_data(struct preproc_ctx *ctx) return preproc_get_top_file(ctx)->code.code; } +static void free_parse_arg_names(struct parse_arg_names *args) +{ + unsigned int i; + + for (i = 0; i < args->count; ++i) + vkd3d_free(args->args[i]); + vkd3d_free(args->args); +} + } %define api.prefix {preproc_yy} @@ -268,11 +283,14 @@ static const void *get_parent_data(struct preproc_ctx *ctx) %union { char *string; + const char *const_string; uint32_t integer; struct vkd3d_string_buffer string_buffer; + struct parse_arg_names arg_names; } %token <string> T_IDENTIFIER +%token <string> T_IDENTIFIER_PAREN %token <string> T_INTEGER %token <string> T_STRING %token <string> T_TEXT @@ -291,7 +309,9 @@ static const void *get_parent_data(struct preproc_ctx *ctx) %type <integer> expr %type <string> body_token +%type <const_string> body_token_const %type <string_buffer> body_text +%type <arg_names> identifier_list %% @@ -302,6 +322,28 @@ shader_text vkd3d_string_buffer_printf(&ctx->buffer, "\n"); } +identifier_list + : T_IDENTIFIER + { + if (!($$.args = vkd3d_malloc(sizeof(*$$.args)))) + YYABORT; + $$.args[0] = $1; + $$.count = 1; + } + | identifier_list ',' T_IDENTIFIER + { + char **new_array; + + if (!(new_array = vkd3d_realloc($1.args, ($1.count + 1) * sizeof(*$$.args)))) + { + free_parse_arg_names(&$1); + YYABORT; + } + $$.args = new_array; + $$.count = $1.count + 1; + $$.args[$1.count] = $3; + } + body_text : %empty { @@ -316,12 +358,32 @@ body_text } vkd3d_free($2); } + | body_text body_token_const + { + if (vkd3d_string_buffer_printf(&$$, "%s ", $2) < 0) + YYABORT; + } body_token : T_IDENTIFIER + | T_IDENTIFIER_PAREN | T_INTEGER | T_TEXT +body_token_const + : '(' + { + $$ = "("; + } + | ')' + { + $$ = ")"; + } + | ',' + { + $$ = ","; + } + directive : T_DEFINE T_IDENTIFIER body_text T_NEWLINE { @@ -332,6 +394,16 @@ directive YYABORT; } } + | T_DEFINE T_IDENTIFIER_PAREN '(' identifier_list ')' body_text T_NEWLINE + { + free_parse_arg_names(&$4); + if (!preproc_add_macro(ctx, &@6, $2, &@6, &$6)) + { + vkd3d_free($2); + vkd3d_string_buffer_cleanup(&$6); + YYABORT; + } + } | T_UNDEF T_IDENTIFIER T_NEWLINE { struct preproc_macro *macro; -- 2.30.0
Signed-off-by: Matteo Bruni <mbruni(a)codeweavers.com>
Signed-off-by: Henri Verbeet <hverbeet(a)codeweavers.com>
Signed-off-by: Zebediah Figura <zfigura(a)codeweavers.com> --- Makefile.am | 1 - libs/vkd3d-shader/preproc.h | 35 +++ libs/vkd3d-shader/preproc.l | 329 ++++++++++++++++++----- libs/vkd3d-shader/preproc.y | 46 +++- libs/vkd3d-shader/vkd3d_shader_private.h | 2 + tests/hlsl_d3d12.c | 2 +- 6 files changed, 338 insertions(+), 77 deletions(-) diff --git a/Makefile.am b/Makefile.am index 514f0506..22151adf 100644 --- a/Makefile.am +++ b/Makefile.am @@ -247,7 +247,6 @@ XFAIL_TESTS = \ tests/preproc-if-expr.shader_test \ tests/preproc-invalid.shader_test \ tests/preproc-macro.shader_test \ - tests/preproc-misc.shader_test \ tests/swizzle-0.shader_test \ tests/swizzle-1.shader_test \ tests/swizzle-2.shader_test \ diff --git a/libs/vkd3d-shader/preproc.h b/libs/vkd3d-shader/preproc.h index 7d6bda2a..0f2d8ba9 100644 --- a/libs/vkd3d-shader/preproc.h +++ b/libs/vkd3d-shader/preproc.h @@ -60,6 +60,9 @@ struct preproc_expansion { struct preproc_buffer buffer; const struct preproc_text *text; + /* Back-pointer to the macro, if this expansion a macro body. This is + * necessary so that argument tokens can be correctly replaced. */ + struct preproc_macro *macro; }; struct preproc_macro @@ -67,6 +70,10 @@ struct preproc_macro struct rb_entry entry; char *name; + char **arg_names; + size_t arg_count; + struct preproc_text *arg_values; + struct preproc_text body; }; @@ -86,8 +93,36 @@ struct preproc_ctx struct rb_tree macros; + /* It's possible to parse as many as two function-like macros at once: one + * in the main text, and another inside of #if directives. E.g. + * + * func1( + * #if func2(...) + * #endif + * ) + * + * It's not possible to parse more than two, however. In the case of nested + * calls like "func1(func2(...))", we store everything inside the outer + * parentheses as unparsed text, and then parse it once the argument is + * actually invoked. + */ + struct preproc_func_state + { + struct preproc_macro *macro; + size_t arg_count; + enum + { + STATE_NONE = 0, + STATE_IDENTIFIER, + STATE_ARGS, + } state; + unsigned int paren_depth; + } text_func, directive_func; + int current_directive; + int lookahead_token; + bool last_was_newline; bool last_was_eof; diff --git a/libs/vkd3d-shader/preproc.l b/libs/vkd3d-shader/preproc.l index 77f4f0dc..9043adee 100644 --- a/libs/vkd3d-shader/preproc.l +++ b/libs/vkd3d-shader/preproc.l @@ -142,7 +142,7 @@ IDENTIFIER [A-Za-z_][A-Za-z0-9_]* } <INITIAL>{WS}+ {} -<INITIAL>[(),] {return yytext[0];} +<INITIAL>[()\[\]{},] {return yytext[0];} <INITIAL>. {return T_TEXT;} %% @@ -169,6 +169,26 @@ static void update_location(struct preproc_ctx *ctx) } } +static bool preproc_is_writing(struct preproc_ctx *ctx) +{ + const struct preproc_file *file; + + /* This can happen while checking for unterminated macro invocation. */ + if (!ctx->file_count) + return true; + file = preproc_get_top_file(ctx); + if (!file->if_count) + return true; + return file->if_stack[file->if_count - 1].current_true; +} + +static struct preproc_macro *preproc_get_top_macro(struct preproc_ctx *ctx) +{ + if (!ctx->expansion_count) + return NULL; + return ctx->expansion_stack[ctx->expansion_count - 1].macro; +} + static void preproc_pop_buffer(struct preproc_ctx *ctx) { if (ctx->expansion_count) @@ -209,15 +229,6 @@ static void preproc_pop_buffer(struct preproc_ctx *ctx) yy_switch_to_buffer(ctx->file_stack[ctx->file_count - 1].buffer.lexer_buffer, ctx->scanner); } -static bool preproc_is_writing(struct preproc_ctx *ctx) -{ - const struct preproc_file *file = preproc_get_top_file(ctx); - - if (!file->if_count) - return true; - return file->if_stack[file->if_count - 1].current_true; -} - static int return_token(int token, YYSTYPE *lval, const char *text) { switch (token) @@ -235,7 +246,29 @@ static int return_token(int token, YYSTYPE *lval, const char *text) return token; } -static bool preproc_push_expansion(struct preproc_ctx *ctx, const struct preproc_text *text) +static const struct preproc_text *find_arg_expansion(struct preproc_ctx *ctx, const char *s) +{ + struct preproc_macro *macro; + unsigned int i; + + if ((macro = preproc_get_top_macro(ctx))) + { + for (i = 0; i < macro->arg_count; ++i) + { + if (!strcmp(s, macro->arg_names[i])) + return ¯o->arg_values[i]; + } + } + return NULL; +} + +static void preproc_text_add(struct preproc_text *text, const char *string) +{ + vkd3d_string_buffer_printf(&text->text, "%s", string); +} + +static bool preproc_push_expansion(struct preproc_ctx *ctx, + const struct preproc_text *text, struct preproc_macro *macro) { struct preproc_expansion *exp; @@ -246,6 +279,7 @@ static bool preproc_push_expansion(struct preproc_ctx *ctx, const struct preproc exp->text = text; exp->buffer.lexer_buffer = yy_scan_bytes(text->text.buffer, text->text.content_size, ctx->scanner); exp->buffer.location = text->location; + exp->macro = macro; TRACE("Expansion stack size is now %zu.\n", ctx->expansion_count); return true; } @@ -256,58 +290,72 @@ int yylex(YYSTYPE *lval, YYLTYPE *lloc, yyscan_t scanner) for (;;) { + struct preproc_func_state *func_state; const char *text; int token; - if (ctx->last_was_eof) - { - preproc_pop_buffer(ctx); - if (!ctx->file_count) - return 0; - } - ctx->last_was_eof = false; - - assert(ctx->file_count); - if (!(token = preproc_lexer_lex(lval, lloc, scanner))) + if (ctx->lookahead_token) { - ctx->last_was_eof = true; - - /* If we have reached the end of an included file, inject a newline. */ - if (ctx->expansion_count) - continue; - token = T_NEWLINE; - text = "\n"; + token = ctx->lookahead_token; + text = yyget_text(scanner); } else { - text = yyget_text(scanner); - } + if (ctx->last_was_eof) + { + preproc_pop_buffer(ctx); + if (!ctx->file_count) + return 0; + } + ctx->last_was_eof = false; - if (ctx->last_was_newline) - { - switch (token) + assert(ctx->file_count); + if (!(token = preproc_lexer_lex(lval, lloc, scanner))) + { + ctx->last_was_eof = true; + + /* If we have reached the end of an included file, inject a newline. */ + if (ctx->expansion_count) + continue; + token = T_NEWLINE; + text = "\n"; + } + else + { + text = yyget_text(scanner); + } + + if (ctx->last_was_newline) { - case T_DEFINE: - case T_ELIF: - case T_ELSE: - case T_ENDIF: - case T_IF: - case T_IFDEF: - case T_IFNDEF: - case T_INCLUDE: - case T_UNDEF: - ctx->current_directive = token; - break; - - default: - ctx->current_directive = 0; + switch (token) + { + case T_DEFINE: + case T_ELIF: + case T_ELSE: + case T_ENDIF: + case T_IF: + case T_IFDEF: + case T_IFNDEF: + case T_INCLUDE: + case T_UNDEF: + ctx->current_directive = token; + break; + + default: + ctx->current_directive = 0; + } } + + ctx->last_was_newline = (token == T_NEWLINE); } - ctx->last_was_newline = (token == T_NEWLINE); + func_state = ctx->current_directive ? &ctx->directive_func : &ctx->text_func; + + TRACE("Parsing token %d%s, line %d, in directive %d, state %#x, string %s.\n", + token, ctx->lookahead_token ? " (lookahead)" : "", lloc->line, + ctx->current_directive, func_state->state, debugstr_a(text)); - TRACE("Parsing token %d, line %d, in directive %d, string %s.\n", - token, lloc->line, ctx->current_directive, debugstr_a(text)); + ctx->lookahead_token = 0; switch (ctx->current_directive) { @@ -324,33 +372,154 @@ int yylex(YYSTYPE *lval, YYLTYPE *lloc, yyscan_t scanner) continue; } - if (token == T_IDENTIFIER || token == T_IDENTIFIER_PAREN) + switch (func_state->state) { - struct preproc_macro *macro; - - switch (ctx->current_directive) - { - case T_DEFINE: - case T_IFDEF: - case T_IFNDEF: - case T_UNDEF: - /* Return identifiers verbatim. */ + case STATE_NONE: + if (token == T_IDENTIFIER || token == T_IDENTIFIER_PAREN) + { + const struct preproc_text *expansion; + struct preproc_macro *macro; + + switch (ctx->current_directive) + { + case T_DEFINE: + case T_IFDEF: + case T_IFNDEF: + case T_UNDEF: + /* Return identifiers verbatim. */ + return return_token(token, lval, text); + } + + /* Otherwise, expand a macro if there is one. */ + + if ((expansion = find_arg_expansion(ctx, text))) + { + preproc_push_expansion(ctx, expansion, NULL); + continue; + } + + if ((macro = preproc_find_macro(ctx, text))) + { + if (!macro->arg_count) + { + preproc_push_expansion(ctx, ¯o->body, macro); + } + else + { + func_state->state = STATE_IDENTIFIER; + func_state->macro = macro; + } + continue; + } + } + + if (ctx->current_directive) return return_token(token, lval, text); - } - /* Otherwise, expand a macro if there is one. */ + vkd3d_string_buffer_printf(&ctx->buffer, "%s ", text); + break; - if ((macro = preproc_find_macro(ctx, text))) + case STATE_IDENTIFIER: + if (token == '(') + { + struct preproc_text *first_arg = &func_state->macro->arg_values[0]; + unsigned int i; + + func_state->arg_count = 0; + func_state->paren_depth = 1; + func_state->state = STATE_ARGS; + for (i = 0; i < func_state->macro->arg_count; ++i) + func_state->macro->arg_values[i].text.content_size = 0; + + first_arg->location = *lloc; + } + else + { + const char *name = func_state->macro->name; + + ctx->lookahead_token = token; + func_state->macro = NULL; + func_state->state = STATE_NONE; + + if (ctx->current_directive) + return return_token(T_IDENTIFIER, lval, name); + + vkd3d_string_buffer_printf(&ctx->buffer, "%s ", name); + } + break; + + case STATE_ARGS: { - preproc_push_expansion(ctx, ¯o->body); - continue; + struct preproc_text *current_arg = NULL; + + assert(func_state->macro->arg_count); + + if (func_state->arg_count < func_state->macro->arg_count) + current_arg = &func_state->macro->arg_values[func_state->arg_count]; + + switch (token) + { + case T_NEWLINE: + if (current_arg) + preproc_text_add(current_arg, " "); + break; + + case ')': + case ']': + case '}': + if (!--func_state->paren_depth) + { + if (++func_state->arg_count == func_state->macro->arg_count) + { + preproc_push_expansion(ctx, &func_state->macro->body, func_state->macro); + } + else + { + preproc_warning(ctx, lloc, VKD3D_SHADER_WARNING_PP_ARGUMENT_COUNT_MISMATCH, + "Wrong number of arguments to macro \"%s\": expected %zu, got %zu.", + func_state->macro->name, func_state->macro->arg_count, func_state->arg_count); + + if (ctx->current_directive) + return return_token(T_IDENTIFIER, lval, func_state->macro->name); + + vkd3d_string_buffer_printf(&ctx->buffer, "%s ", func_state->macro->name); + } + func_state->macro = NULL; + func_state->state = STATE_NONE; + } + else + { + if (current_arg) + preproc_text_add(current_arg, text); + } + break; + + case ',': + if (func_state->paren_depth == 1) + { + ++func_state->arg_count; + if (current_arg) + current_arg->location = *lloc; + } + else if (current_arg) + { + preproc_text_add(current_arg, text); + } + break; + + case '(': + case '[': + case '{': + ++func_state->paren_depth; + /* fall through */ + + default: + if (current_arg) + preproc_text_add(current_arg, text); + } + break; } } - - if (ctx->current_directive) - return return_token(token, lval, text); - - vkd3d_string_buffer_printf(&ctx->buffer, "%s ", text); } } @@ -418,6 +587,26 @@ int preproc_lexer_parse(const struct vkd3d_shader_compile_info *compile_info, preproc_yyparse(ctx.scanner, &ctx); + switch (ctx.text_func.state) + { + case STATE_NONE: + break; + + case STATE_ARGS: + { + const struct vkd3d_shader_location loc = {.source_name = source_name}; + + preproc_warning(&ctx, &loc, VKD3D_SHADER_WARNING_PP_UNTERMINATED_MACRO, + "Unterminated macro invocation."); + } + /* fall through */ + + case STATE_IDENTIFIER: + if (preproc_is_writing(&ctx)) + vkd3d_string_buffer_printf(&ctx.buffer, "%s ", ctx.text_func.macro->name); + break; + } + while (ctx.file_count) preproc_pop_buffer(&ctx); yylex_destroy(ctx.scanner); diff --git a/libs/vkd3d-shader/preproc.y b/libs/vkd3d-shader/preproc.y index 3cb7de61..4173af0f 100644 --- a/libs/vkd3d-shader/preproc.y +++ b/libs/vkd3d-shader/preproc.y @@ -84,9 +84,10 @@ struct preproc_macro *preproc_find_macro(struct preproc_ctx *ctx, const char *na } static bool preproc_add_macro(struct preproc_ctx *ctx, const struct vkd3d_shader_location *loc, char *name, - const struct vkd3d_shader_location *body_loc, struct vkd3d_string_buffer *body) + char **arg_names, size_t arg_count, const struct vkd3d_shader_location *body_loc, struct vkd3d_string_buffer *body) { struct preproc_macro *macro; + unsigned int i; int ret; if ((macro = preproc_find_macro(ctx, name))) @@ -96,11 +97,21 @@ static bool preproc_add_macro(struct preproc_ctx *ctx, const struct vkd3d_shader preproc_free_macro(macro); } - TRACE("Defining new macro %s.\n", debugstr_a(name)); + TRACE("Defining new macro %s with %zu arguments.\n", debugstr_a(name), arg_count); if (!(macro = vkd3d_malloc(sizeof(*macro)))) return false; macro->name = name; + macro->arg_names = arg_names; + macro->arg_count = arg_count; + macro->arg_values = NULL; + if (arg_count && !(macro->arg_values = vkd3d_calloc(arg_count, sizeof(*macro->arg_values)))) + { + vkd3d_free(macro); + return false; + } + for (i = 0; i < arg_count; ++i) + vkd3d_string_buffer_init(¯o->arg_values[i].text); macro->body.text = *body; macro->body.location = *body_loc; ret = rb_put(&ctx->macros, name, ¯o->entry); @@ -110,7 +121,16 @@ static bool preproc_add_macro(struct preproc_ctx *ctx, const struct vkd3d_shader void preproc_free_macro(struct preproc_macro *macro) { + unsigned int i; + vkd3d_free(macro->name); + for (i = 0; i < macro->arg_count; ++i) + { + vkd3d_string_buffer_cleanup(¯o->arg_values[i].text); + vkd3d_free(macro->arg_names[i]); + } + vkd3d_free(macro->arg_names); + vkd3d_free(macro->arg_values); vkd3d_string_buffer_cleanup(¯o->body.text); vkd3d_free(macro); } @@ -379,6 +399,22 @@ body_token_const { $$ = ")"; } + | '[' + { + $$ = "["; + } + | ']' + { + $$ = "]"; + } + | '{' + { + $$ = "{"; + } + | '}' + { + $$ = "}"; + } | ',' { $$ = ","; @@ -387,7 +423,7 @@ body_token_const directive : T_DEFINE T_IDENTIFIER body_text T_NEWLINE { - if (!preproc_add_macro(ctx, &@$, $2, &@3, &$3)) + if (!preproc_add_macro(ctx, &@$, $2, NULL, 0, &@3, &$3)) { vkd3d_free($2); vkd3d_string_buffer_cleanup(&$3); @@ -396,10 +432,10 @@ directive } | T_DEFINE T_IDENTIFIER_PAREN '(' identifier_list ')' body_text T_NEWLINE { - free_parse_arg_names(&$4); - if (!preproc_add_macro(ctx, &@6, $2, &@6, &$6)) + if (!preproc_add_macro(ctx, &@6, $2, $4.args, $4.count, &@6, &$6)) { vkd3d_free($2); + free_parse_arg_names(&$4); vkd3d_string_buffer_cleanup(&$6); YYABORT; } diff --git a/libs/vkd3d-shader/vkd3d_shader_private.h b/libs/vkd3d-shader/vkd3d_shader_private.h index 5224c929..157dc5e2 100644 --- a/libs/vkd3d-shader/vkd3d_shader_private.h +++ b/libs/vkd3d-shader/vkd3d_shader_private.h @@ -85,7 +85,9 @@ enum vkd3d_shader_error VKD3D_SHADER_WARNING_PP_ALREADY_DEFINED = 4300, VKD3D_SHADER_WARNING_PP_INVALID_DIRECTIVE = 4301, + VKD3D_SHADER_WARNING_PP_ARGUMENT_COUNT_MISMATCH = 4302, VKD3D_SHADER_WARNING_PP_UNKNOWN_DIRECTIVE = 4303, + VKD3D_SHADER_WARNING_PP_UNTERMINATED_MACRO = 4304, VKD3D_SHADER_WARNING_PP_UNTERMINATED_IF = 4305, }; diff --git a/tests/hlsl_d3d12.c b/tests/hlsl_d3d12.c index ac616c23..6980b998 100644 --- a/tests/hlsl_d3d12.c +++ b/tests/hlsl_d3d12.c @@ -352,7 +352,7 @@ static void test_preprocess(void) if (i == 10 || i == 11) continue; vkd3d_test_set_context("Source \"%s\"", tests[i].source); - todo_if (i <= 4 || (i >= 9 && i <= 14) || i == 43) + todo_if (i <= 4 || (i >= 9 && i <= 14)) check_preprocess(tests[i].source, NULL, NULL, tests[i].present, tests[i].absent); } vkd3d_test_set_context(NULL); -- 2.30.0
Signed-off-by: Matteo Bruni <mbruni(a)codeweavers.com>
On Tue, Jan 12, 2021 at 11:15 PM Zebediah Figura <zfigura(a)codeweavers.com> wrote:
Signed-off-by: Zebediah Figura <zfigura(a)codeweavers.com> --- Makefile.am | 1 - libs/vkd3d-shader/preproc.h | 35 +++ libs/vkd3d-shader/preproc.l | 329 ++++++++++++++++++----- libs/vkd3d-shader/preproc.y | 46 +++- libs/vkd3d-shader/vkd3d_shader_private.h | 2 + tests/hlsl_d3d12.c | 2 +- 6 files changed, 338 insertions(+), 77 deletions(-)
diff --git a/libs/vkd3d-shader/preproc.h b/libs/vkd3d-shader/preproc.h index 7d6bda2a..0f2d8ba9 100644 --- a/libs/vkd3d-shader/preproc.h +++ b/libs/vkd3d-shader/preproc.h @@ -60,6 +60,9 @@ struct preproc_expansion { struct preproc_buffer buffer; const struct preproc_text *text; + /* Back-pointer to the macro, if this expansion a macro body. This is + * necessary so that argument tokens can be correctly replaced. */ + struct preproc_macro *macro; };
This comment seems to be missing a verb. Something that I noticed and isn't specifically related to this patch: it looks like sometimes there is a "spurious" whitespace at the start of a line in the output, maybe after macro expansion. Just to mention it, I don't know if you noticed already or if that's actually supposed to happen in the first place.
On 1/14/21 7:08 AM, Matteo Bruni wrote:
On Tue, Jan 12, 2021 at 11:15 PM Zebediah Figura <zfigura(a)codeweavers.com> wrote:
Signed-off-by: Zebediah Figura <zfigura(a)codeweavers.com> --- Makefile.am | 1 - libs/vkd3d-shader/preproc.h | 35 +++ libs/vkd3d-shader/preproc.l | 329 ++++++++++++++++++----- libs/vkd3d-shader/preproc.y | 46 +++- libs/vkd3d-shader/vkd3d_shader_private.h | 2 + tests/hlsl_d3d12.c | 2 +- 6 files changed, 338 insertions(+), 77 deletions(-)
diff --git a/libs/vkd3d-shader/preproc.h b/libs/vkd3d-shader/preproc.h index 7d6bda2a..0f2d8ba9 100644 --- a/libs/vkd3d-shader/preproc.h +++ b/libs/vkd3d-shader/preproc.h @@ -60,6 +60,9 @@ struct preproc_expansion { struct preproc_buffer buffer; const struct preproc_text *text; + /* Back-pointer to the macro, if this expansion a macro body. This is + * necessary so that argument tokens can be correctly replaced. */ + struct preproc_macro *macro; };
This comment seems to be missing a verb.
Something that I noticed and isn't specifically related to this patch: it looks like sometimes there is a "spurious" whitespace at the start of a line in the output, maybe after macro expansion. Just to mention it, I don't know if you noticed already or if that's actually supposed to happen in the first place.
I tried to be careful to ensure there was enough whitespace to prevent merging of tokens, but I wasn't really careful to match whitespace closely with native (though the approach of emitting whitespace after every token seems indeed to match pretty closely). I think native may emit spurious whitespace on an empty line in some cases.
Signed-off-by: Henri Verbeet <hverbeet(a)codeweavers.com>
Signed-off-by: Zebediah Figura <zfigura(a)codeweavers.com> --- libs/vkd3d-shader/preproc.l | 22 ++++++++++++++++++++++ libs/vkd3d-shader/preproc.y | 6 ++++++ tests/hlsl_d3d12.c | 7 ++----- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/libs/vkd3d-shader/preproc.l b/libs/vkd3d-shader/preproc.l index 9043adee..18a59b70 100644 --- a/libs/vkd3d-shader/preproc.l +++ b/libs/vkd3d-shader/preproc.l @@ -83,6 +83,8 @@ IDENTIFIER [A-Za-z_][A-Za-z0-9_]* <INITIAL>0[0-7]*[ul]{0,2} {return T_INTEGER;} <INITIAL>[1-9][0-9]*[ul]{0,2} {return T_INTEGER;} +<INITIAL>## {return T_CONCAT;} + <INITIAL>"&&" {return T_TEXT;} <INITIAL>"||" {return T_TEXT;} <INITIAL>"++" {return T_TEXT;} @@ -189,6 +191,18 @@ static struct preproc_macro *preproc_get_top_macro(struct preproc_ctx *ctx) return ctx->expansion_stack[ctx->expansion_count - 1].macro; } +/* Concatenation is not done for object-like macros, but is done for both + * function-like macro bodies and their arguments. */ +static bool should_concat(struct preproc_ctx *ctx) +{ + struct preproc_macro *macro; + + if (!ctx->expansion_count) + return false; + macro = ctx->expansion_stack[ctx->expansion_count - 1].macro; + return !macro || macro->arg_count; +} + static void preproc_pop_buffer(struct preproc_ctx *ctx) { if (ctx->expansion_count) @@ -375,6 +389,14 @@ int yylex(YYSTYPE *lval, YYLTYPE *lloc, yyscan_t scanner) switch (func_state->state) { case STATE_NONE: + if (token == T_CONCAT && should_concat(ctx)) + { + while (ctx->buffer.content_size + && strchr(" \t\r\n", ctx->buffer.buffer[ctx->buffer.content_size - 1])) + --ctx->buffer.content_size; + break; + } + if (token == T_IDENTIFIER || token == T_IDENTIFIER_PAREN) { const struct preproc_text *expansion; diff --git a/libs/vkd3d-shader/preproc.y b/libs/vkd3d-shader/preproc.y index 4173af0f..45577f23 100644 --- a/libs/vkd3d-shader/preproc.y +++ b/libs/vkd3d-shader/preproc.y @@ -327,6 +327,8 @@ static void free_parse_arg_names(struct parse_arg_names *args) %token T_INCLUDE "#include" %token T_UNDEF "#undef" +%token T_CONCAT "##" + %type <integer> expr %type <string> body_token %type <const_string> body_token_const @@ -419,6 +421,10 @@ body_token_const { $$ = ","; } + | T_CONCAT + { + $$ = "##"; + } directive : T_DEFINE T_IDENTIFIER body_text T_NEWLINE diff --git a/tests/hlsl_d3d12.c b/tests/hlsl_d3d12.c index 6980b998..e7267986 100644 --- a/tests/hlsl_d3d12.c +++ b/tests/hlsl_d3d12.c @@ -349,10 +349,8 @@ static void test_preprocess(void) for (i = 0; i < ARRAY_SIZE(tests); ++i) { - if (i == 10 || i == 11) - continue; vkd3d_test_set_context("Source \"%s\"", tests[i].source); - todo_if (i <= 4 || (i >= 9 && i <= 14)) + todo_if (i <= 4 || i == 9 || (i >= 12 && i <= 14)) check_preprocess(tests[i].source, NULL, NULL, tests[i].present, tests[i].absent); } vkd3d_test_set_context(NULL); @@ -407,8 +405,7 @@ static void test_preprocess(void) ok(include_count_file2 == 2, "file2 was included %u times.\n", include_count_file2); /* Macro invocation spread across multiple files. */ - if (0) - todo check_preprocess(test_include2_top, NULL, &test_include, "pass", NULL); + check_preprocess(test_include2_top, NULL, &test_include, "pass", NULL); blob = errors = (ID3D10Blob *)0xdeadbeef; hr = D3DPreprocess(test_include_top, strlen(test_include_top), NULL, NULL, &test_include_fail, &blob, &errors); -- 2.30.0
Signed-off-by: Matteo Bruni <mbruni(a)codeweavers.com>
Signed-off-by: Henri Verbeet <hverbeet(a)codeweavers.com>
Signed-off-by: Zebediah Figura <zfigura(a)codeweavers.com> --- libs/vkd3d-shader/preproc.l | 41 ++++++++++++++++++++++++++++++++++++- libs/vkd3d-shader/preproc.y | 4 +++- tests/hlsl_d3d12.c | 2 +- 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/libs/vkd3d-shader/preproc.l b/libs/vkd3d-shader/preproc.l index 18a59b70..388b77f6 100644 --- a/libs/vkd3d-shader/preproc.l +++ b/libs/vkd3d-shader/preproc.l @@ -104,7 +104,7 @@ IDENTIFIER [A-Za-z_][A-Za-z0-9_]* const char *p; if (!ctx->last_was_newline) - return T_TEXT; + return T_HASHSTRING; for (p = yytext + 1; strchr(" \t", *p); ++p) ; @@ -247,6 +247,7 @@ static int return_token(int token, YYSTYPE *lval, const char *text) { switch (token) { + case T_HASHSTRING: case T_IDENTIFIER: case T_IDENTIFIER_PAREN: case T_INTEGER: @@ -389,6 +390,9 @@ int yylex(YYSTYPE *lval, YYLTYPE *lloc, yyscan_t scanner) switch (func_state->state) { case STATE_NONE: + { + struct preproc_macro *macro; + if (token == T_CONCAT && should_concat(ctx)) { while (ctx->buffer.content_size @@ -397,6 +401,40 @@ int yylex(YYSTYPE *lval, YYLTYPE *lloc, yyscan_t scanner) break; } + /* Stringification, however, is only done for function-like + * macro bodies. */ + if (token == T_HASHSTRING && (macro = preproc_get_top_macro(ctx)) && macro->arg_count) + { + const struct preproc_text *expansion; + const char *p = text + 1; + unsigned int i; + + if (ctx->current_directive) + return return_token(token, lval, text); + + while (*p == ' ' || *p == '\t') + ++p; + + vkd3d_string_buffer_printf(&ctx->buffer, "\""); + if ((expansion = find_arg_expansion(ctx, p))) + { + for (i = 0; i < expansion->text.content_size; ++i) + { + char c = expansion->text.buffer[i]; + + if (c == '\\' || c == '"') + vkd3d_string_buffer_printf(&ctx->buffer, "\\"); + vkd3d_string_buffer_printf(&ctx->buffer, "%c", c); + } + } + else + { + vkd3d_string_buffer_printf(&ctx->buffer, "%s", p); + } + vkd3d_string_buffer_printf(&ctx->buffer, "\""); + break; + } + if (token == T_IDENTIFIER || token == T_IDENTIFIER_PAREN) { const struct preproc_text *expansion; @@ -440,6 +478,7 @@ int yylex(YYSTYPE *lval, YYLTYPE *lloc, yyscan_t scanner) vkd3d_string_buffer_printf(&ctx->buffer, "%s ", text); break; + } case STATE_IDENTIFIER: if (token == '(') diff --git a/libs/vkd3d-shader/preproc.y b/libs/vkd3d-shader/preproc.y index 45577f23..72d6486e 100644 --- a/libs/vkd3d-shader/preproc.y +++ b/libs/vkd3d-shader/preproc.y @@ -309,6 +309,7 @@ static void free_parse_arg_names(struct parse_arg_names *args) struct parse_arg_names arg_names; } +%token <string> T_HASHSTRING %token <string> T_IDENTIFIER %token <string> T_IDENTIFIER_PAREN %token <string> T_INTEGER @@ -387,7 +388,8 @@ body_text } body_token - : T_IDENTIFIER + : T_HASHSTRING + | T_IDENTIFIER | T_IDENTIFIER_PAREN | T_INTEGER | T_TEXT diff --git a/tests/hlsl_d3d12.c b/tests/hlsl_d3d12.c index e7267986..9d8d16d9 100644 --- a/tests/hlsl_d3d12.c +++ b/tests/hlsl_d3d12.c @@ -350,7 +350,7 @@ static void test_preprocess(void) for (i = 0; i < ARRAY_SIZE(tests); ++i) { vkd3d_test_set_context("Source \"%s\"", tests[i].source); - todo_if (i <= 4 || i == 9 || (i >= 12 && i <= 14)) + todo_if (i == 9 || (i >= 12 && i <= 14)) check_preprocess(tests[i].source, NULL, NULL, tests[i].present, tests[i].absent); } vkd3d_test_set_context(NULL); -- 2.30.0
Signed-off-by: Matteo Bruni <mbruni(a)codeweavers.com>
Signed-off-by: Henri Verbeet <hverbeet(a)codeweavers.com>
participants (5)
-
Henri Verbeet -
Matteo Bruni -
Matteo Bruni -
Zebediah Figura -
Zebediah Figura (she/her)