[PATCH 0/4] MR418: tests/shader-runner: Improvements, part 1.

[Francisco Casas (＠fcasas)]

First part of a series that allows the shader-runner to test SM1 compilation on [shader] directives even if a backend to run these tests is not available and prepare they way for SM1-only tests.

Specifically:
- Compile [shader] directives separately from backend execution (on part 1).
- Allow specifying more detailed shader ranges for todo(), fail() and notimpl() qualifiers (on part 2).
- Always test shader compilation with SM1 profiles (on part 2).

The `minimum_shader_model` and `maximum_shader_model` parameters for run_shader_tests() are interpreted in the following way:
> Ask the pertaining backend to run the shader_test file so that each test is executed with the lowest profile it can support within the range of shader models specified, if any.

This allows us to control how many shader models we want to test for each backend, e.g.:
```
run_shader_tests(..., SHADER_MODEL_2_0, SHADER_MODEL_3_0)
run_shader_tests(..., SHADER_MODEL_4_0, SHADER_MODEL_5_1)
run_shader_tests(..., SHADER_MODEL_6_0, SHADER_MODEL_6_0)
```
versus
```
for (i = SHADER_MODEL_2_0, i <= SHADER_MODEL_6_0, ++i)
    run_shader_tests(..., i, i);
```

Also, to allow to compile [shader] directives, which are not backend-specific, separately from the [test] directives, which are, the run_shader_tests() function is modified to skip backend specific directives if the shader_runner_ops is NULL.

Following patches are on my [master6i](https://gitlab.winehq.org/fcasas/vkd3d/-/commits/master6i) branch.

[PATCH 1/4] tests/shader-runner: Add missing requirement checks for backends.

[Francisco Casas]

From: Francisco Casas

We need these checks to properly handle tests that require target
profiles 3.0 and lower, or tests that require target profile 6.0
and higher.

Since all backeds require an ops->check_requirements now, expect it to
always be defined.
---
 tests/shader_runner.c        |  3 ++-
 tests/shader_runner.h        |  4 ++--
 tests/shader_runner_d3d11.c  | 13 +++++++++++++
 tests/shader_runner_d3d12.c  | 14 ++++++++++++++
 tests/shader_runner_vulkan.c | 13 +++++++++++++
 5 files changed, 44 insertions(+), 3 deletions(-)

diff --git a/tests/shader_runner.c b/tests/shader_runner.c
index cae0bf026..96fed19b6 100644
--- a/tests/shader_runner.c
+++ b/tests/shader_runner.c
@@ -1073,8 +1073,9 @@ void run_shader_tests(struct shader_runner *runner, const struct shader_runner_o
                     break;

case STATE_REQUIRE:
+                    assert(runner->ops->check_requirements);
                     if (runner->maximum_shader_model < runner->minimum_shader_model
-                            || (runner->ops->check_requirements && !runner->ops->check_requirements(runner)))
+                            || !runner->ops->check_requirements(runner))
                     {
                         skip_tests = true;
                     }
diff --git a/tests/shader_runner.h b/tests/shader_runner.h
index 3efbcf362..f6ef44a71 100644
--- a/tests/shader_runner.h
+++ b/tests/shader_runner.h
@@ -140,9 +140,9 @@ struct shader_runner

struct shader_runner_ops
 {
-    /* Returns false if unable to run the given tests. If NULL, all tests are
-     * run. */
+    /* Returns false if unable to run the given tests. */
     bool (*check_requirements)(struct shader_runner *runner);
+
     struct resource *(*create_resource)(struct shader_runner *runner, const struct resource_params *params);
     void (*destroy_resource)(struct shader_runner *runner, struct resource *resource);
     bool (*draw)(struct shader_runner *runner, D3D_PRIMITIVE_TOPOLOGY primitive_topology, unsigned int vertex_count);
diff --git a/tests/shader_runner_d3d11.c b/tests/shader_runner_d3d11.c
index bd9d363ce..7a8f8dac0 100644
--- a/tests/shader_runner_d3d11.c
+++ b/tests/shader_runner_d3d11.c
@@ -100,6 +100,18 @@ static ID3D10Blob *compile_shader(const struct d3d11_shader_runner *runner, cons
     return blob;
 }

+static bool d3d11_runner_check_requirements(struct shader_runner *r)
+{
+    struct d3d11_shader_runner *runner = d3d11_shader_runner(r);
+
+    if (runner->r.maximum_shader_model < SHADER_MODEL_4_0)
+        return false;
+    if (runner->r.minimum_shader_model >= SHADER_MODEL_6_0)
+        return false;
+
+    return true;
+}
+
 static IDXGIAdapter *create_adapter(void)
 {
     IDXGIFactory4 *factory4;
@@ -724,6 +736,7 @@ static void d3d11_runner_release_readback(struct shader_runner *r, struct resour

static const struct shader_runner_ops d3d11_runner_ops =
 {
+    .check_requirements = d3d11_runner_check_requirements,
     .create_resource = d3d11_runner_create_resource,
     .destroy_resource = d3d11_runner_destroy_resource,
     .dispatch = d3d11_runner_dispatch,
diff --git a/tests/shader_runner_d3d12.c b/tests/shader_runner_d3d12.c
index 8453617ef..c54cb5384 100644
--- a/tests/shader_runner_d3d12.c
+++ b/tests/shader_runner_d3d12.c
@@ -96,6 +96,19 @@ static ID3D10Blob *compile_shader(const struct d3d12_shader_runner *runner, cons
     return blob;
 }

+static bool d3d12_runner_check_requirements(struct shader_runner *r)
+{
+    struct d3d12_shader_runner *runner = d3d12_shader_runner(r);
+
+    if (runner->r.maximum_shader_model < SHADER_MODEL_4_0)
+        return false;
+
+    if (runner->r.minimum_shader_model >= SHADER_MODEL_6_0 && !runner->dxc_compiler)
+        return false;
+
+    return true;
+}
+
 #define MAX_RESOURCE_DESCRIPTORS (MAX_RESOURCES * 2)

static struct resource *d3d12_runner_create_resource(struct shader_runner *r, const struct resource_params *params)
@@ -567,6 +580,7 @@ static void d3d12_runner_release_readback(struct shader_runner *r, struct resour

static const struct shader_runner_ops d3d12_runner_ops =
 {
+    .check_requirements = d3d12_runner_check_requirements,
     .create_resource = d3d12_runner_create_resource,
     .destroy_resource = d3d12_runner_destroy_resource,
     .dispatch = d3d12_runner_dispatch,
diff --git a/tests/shader_runner_vulkan.c b/tests/shader_runner_vulkan.c
index ebb15eba9..ab9f6ecfc 100644
--- a/tests/shader_runner_vulkan.c
+++ b/tests/shader_runner_vulkan.c
@@ -83,6 +83,18 @@ static struct vulkan_shader_runner *vulkan_shader_runner(struct shader_runner *r

#define VK_CALL(f) (runner->f)

+static bool vulkan_runner_check_requirements(struct shader_runner *r)
+{
+    struct vulkan_shader_runner *runner = vulkan_shader_runner(r);
+
+    if (runner->r.maximum_shader_model < SHADER_MODEL_4_0)
+        return false;
+    if (runner->r.minimum_shader_model >= SHADER_MODEL_6_0)
+        return false;
+
+    return true;
+}
+
 static void begin_command_buffer(struct vulkan_shader_runner *runner)
 {
     VkCommandBufferBeginInfo buffer_begin_desc = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};
@@ -1155,6 +1167,7 @@ static void vulkan_runner_release_readback(struct shader_runner *r, struct resou

static const struct shader_runner_ops vulkan_runner_ops =
 {
+    .check_requirements = vulkan_runner_check_requirements,
     .create_resource = vulkan_runner_create_resource,
     .destroy_resource = vulkan_runner_destroy_resource,
     .dispatch = vulkan_runner_dispatch,

[PATCH 2/4] tests/shader-runner: Discern between the minimum_shader_model and the selected_shader_model.

[Francisco Casas]

From: Francisco Casas

In case that the backend selects a shader model that is higher than the
minimum in the provided range, it doesn't make sense to check todos(...),
fails(...), or notimpl(...) against the minimum.

So the backend must report the selected model for the given range (if
any). This is done through the check_requirements() function.
---
 tests/shader_runner.c        | 23 +++++++++++++----------
 tests/shader_runner.h        |  6 ++++--
 tests/shader_runner_d3d11.c  |  8 ++++----
 tests/shader_runner_d3d12.c  | 12 ++++++------
 tests/shader_runner_d3d9.c   |  6 ++++--
 tests/shader_runner_vulkan.c |  8 ++++----
 6 files changed, 35 insertions(+), 28 deletions(-)

diff --git a/tests/shader_runner.c b/tests/shader_runner.c
index 96fed19b6..e0ab9eb63 100644
--- a/tests/shader_runner.c
+++ b/tests/shader_runner.c
@@ -499,9 +499,9 @@ static void parse_test_directive(struct shader_runner *runner, const char *line)
     if (match_string(line, "todo", &line))
         runner->is_todo = true;
     else if (match_string(line, "todo(sm<6)", &line))
-        runner->is_todo = runner->minimum_shader_model < SHADER_MODEL_6_0;
+        runner->is_todo = runner->selected_shader_model < SHADER_MODEL_6_0;
     else if (match_string(line, "todo(sm>=6)", &line))
-        runner->is_todo = runner->minimum_shader_model >= SHADER_MODEL_6_0;
+        runner->is_todo = runner->selected_shader_model >= SHADER_MODEL_6_0;

if (match_string(line, "dispatch", &line))
     {
@@ -937,7 +937,7 @@ result_release:
 static void compile_shader(struct shader_runner *runner, IDxcCompiler3 *dxc_compiler, const char *source, size_t len,
         enum shader_type type, HRESULT expect)
 {
-    bool use_dxcompiler = runner->minimum_shader_model >= SHADER_MODEL_6_0;
+    bool use_dxcompiler = runner->selected_shader_model >= SHADER_MODEL_6_0;
     ID3D10Blob *blob = NULL, *errors = NULL;
     char profile[7];
     HRESULT hr;
@@ -960,7 +960,7 @@ static void compile_shader(struct shader_runner *runner, IDxcCompiler3 *dxc_comp
     }
     else
     {
-        sprintf(profile, "%s_%s", shader_type_string(type), shader_models[runner->minimum_shader_model]);
+        sprintf(profile, "%s_%s", shader_type_string(type), shader_models[runner->selected_shader_model]);
         hr = D3DCompile(source, len, NULL, NULL, NULL, "main", profile, runner->compile_options, 0, &blob, &errors);
     }
     hr = map_unidentified_hrs(hr);
@@ -990,7 +990,7 @@ static enum parse_state read_shader_directive(struct shader_runner *runner, enum
         /* 'todo' is not meaningful when dxcompiler is in use, so it has no '(sm<6) qualifier. */
         if (match_directive_substring(src, "todo", &src))
         {
-            if (runner->minimum_shader_model >= SHADER_MODEL_6_0)
+            if (runner->selected_shader_model >= SHADER_MODEL_6_0)
                 continue;
             if (state == STATE_SHADER_COMPUTE)
                 state = STATE_SHADER_COMPUTE_TODO;
@@ -1005,17 +1005,17 @@ static enum parse_state read_shader_directive(struct shader_runner *runner, enum
         }
         else if (match_directive_substring(src, "fail(sm<6)", &src))
         {
-            if (runner->minimum_shader_model < SHADER_MODEL_6_0)
+            if (runner->selected_shader_model < SHADER_MODEL_6_0)
                 *expect_hr = E_FAIL;
         }
         else if (match_directive_substring(src, "fail(sm>=6)", &src))
         {
-            if (runner->minimum_shader_model >= SHADER_MODEL_6_0)
+            if (runner->selected_shader_model >= SHADER_MODEL_6_0)
                 *expect_hr = E_FAIL;
         }
         else if (match_directive_substring(src, "notimpl(sm<6)", &src))
         {
-            if (runner->minimum_shader_model < SHADER_MODEL_6_0)
+            if (runner->selected_shader_model < SHADER_MODEL_6_0)
                 *expect_hr = E_NOTIMPL;
         }
         else
@@ -1055,6 +1055,9 @@ void run_shader_tests(struct shader_runner *runner, const struct shader_runner_o
     runner->minimum_shader_model = minimum_shader_model;
     runner->maximum_shader_model = maximum_shader_model;

+    assert(runner->ops->check_requirements);
+    skip_tests = !runner->ops->check_requirements(runner, &runner->selected_shader_model);
+
     for (;;)
     {
         char *ret = fgets(line_buffer, sizeof(line_buffer), f);
@@ -1073,9 +1076,8 @@ void run_shader_tests(struct shader_runner *runner, const struct shader_runner_o
                     break;

case STATE_REQUIRE:
-                    assert(runner->ops->check_requirements);
                     if (runner->maximum_shader_model < runner->minimum_shader_model
-                            || !runner->ops->check_requirements(runner))
+                            || !runner->ops->check_requirements(runner, &runner->selected_shader_model))
                     {
                         skip_tests = true;
                     }
@@ -1212,6 +1214,7 @@ void run_shader_tests(struct shader_runner *runner, const struct shader_runner_o
                 state = STATE_REQUIRE;
                 runner->minimum_shader_model = minimum_shader_model;
                 runner->maximum_shader_model = maximum_shader_model;
+                runner->selected_shader_model = minimum_shader_model;
                 runner->compile_options = 0;
                 skip_tests = false;
             }
diff --git a/tests/shader_runner.h b/tests/shader_runner.h
index f6ef44a71..e52a44596 100644
--- a/tests/shader_runner.h
+++ b/tests/shader_runner.h
@@ -120,6 +120,7 @@ struct shader_runner
     char *cs_source;
     enum shader_model minimum_shader_model;
     enum shader_model maximum_shader_model;
+    enum shader_model selected_shader_model;

bool last_render_failed;

@@ -140,8 +141,9 @@ struct shader_runner

struct shader_runner_ops
 {
-    /* Returns false if unable to run the given tests. */
-    bool (*check_requirements)(struct shader_runner *runner);
+    /* Returns false if unable to run the given tests. Otherwise, it also outputs the lowest model
+     * supported within the range provided in the runner. */
+    bool (*check_requirements)(struct shader_runner *runner, enum shader_model *model);

struct resource *(*create_resource)(struct shader_runner *runner, const struct resource_params *params);
     void (*destroy_resource)(struct shader_runner *runner, struct resource *resource);
diff --git a/tests/shader_runner_d3d11.c b/tests/shader_runner_d3d11.c
index 7a8f8dac0..df704da8c 100644
--- a/tests/shader_runner_d3d11.c
+++ b/tests/shader_runner_d3d11.c
@@ -80,15 +80,13 @@ static ID3D10Blob *compile_shader(const struct d3d11_shader_runner *runner, cons

static const char *const shader_models[] =
     {
-        [SHADER_MODEL_2_0] = "4_0",
-        [SHADER_MODEL_3_0] = "4_0",
         [SHADER_MODEL_4_0] = "4_0",
         [SHADER_MODEL_4_1] = "4_1",
         [SHADER_MODEL_5_0] = "5_0",
         [SHADER_MODEL_5_1] = "5_1",
     };

-    sprintf(profile, "%s_%s", type, shader_models[runner->r.minimum_shader_model]);
+    sprintf(profile, "%s_%s", type, shader_models[runner->r.selected_shader_model]);
     hr = D3DCompile(source, strlen(source), NULL, NULL, NULL, "main", profile, runner->r.compile_options, 0, &blob, &errors);
     ok(hr == S_OK, "Failed to compile shader, hr %#lx.\n", hr);
     if (errors)
@@ -100,7 +98,7 @@ static ID3D10Blob *compile_shader(const struct d3d11_shader_runner *runner, cons
     return blob;
 }

-static bool d3d11_runner_check_requirements(struct shader_runner *r)
+static bool d3d11_runner_check_requirements(struct shader_runner *r, enum shader_model *model)
 {
     struct d3d11_shader_runner *runner = d3d11_shader_runner(r);

@@ -109,6 +107,8 @@ static bool d3d11_runner_check_requirements(struct shader_runner *r)
     if (runner->r.minimum_shader_model >= SHADER_MODEL_6_0)
         return false;

+    *model = max(SHADER_MODEL_4_0, runner->r.minimum_shader_model);
+
     return true;
 }

diff --git a/tests/shader_runner_d3d12.c b/tests/shader_runner_d3d12.c
index c54cb5384..ccac95189 100644
--- a/tests/shader_runner_d3d12.c
+++ b/tests/shader_runner_d3d12.c
@@ -67,8 +67,6 @@ static ID3D10Blob *compile_shader(const struct d3d12_shader_runner *runner, cons

static const char *const shader_models[] =
     {
-        [SHADER_MODEL_2_0] = "4_0",
-        [SHADER_MODEL_3_0] = "4_0",
         [SHADER_MODEL_4_0] = "4_0",
         [SHADER_MODEL_4_1] = "4_1",
         [SHADER_MODEL_5_0] = "5_0",
@@ -76,14 +74,14 @@ static ID3D10Blob *compile_shader(const struct d3d12_shader_runner *runner, cons
         [SHADER_MODEL_6_0] = "6_0",
     };

-    if (runner->r.minimum_shader_model >= SHADER_MODEL_6_0)
+    if (runner->r.selected_shader_model >= SHADER_MODEL_6_0)
     {
         assert(runner->dxc_compiler);
         hr = dxc_compiler_compile_shader(runner->dxc_compiler, type, runner->r.compile_options, source, &blob, &errors);
     }
     else
     {
-        sprintf(profile, "%s_%s", shader_type_string(type), shader_models[runner->r.minimum_shader_model]);
+        sprintf(profile, "%s_%s", shader_type_string(type), shader_models[runner->r.selected_shader_model]);
         hr = D3DCompile(source, strlen(source), NULL, NULL, NULL, "main", profile, runner->r.compile_options, 0, &blob, &errors);
     }
     ok(FAILED(hr) == !blob, "Got unexpected hr %#x, blob %p.\n", hr, blob);
@@ -96,7 +94,7 @@ static ID3D10Blob *compile_shader(const struct d3d12_shader_runner *runner, cons
     return blob;
 }

-static bool d3d12_runner_check_requirements(struct shader_runner *r)
+static bool d3d12_runner_check_requirements(struct shader_runner *r, enum shader_model *model)
 {
     struct d3d12_shader_runner *runner = d3d12_shader_runner(r);

@@ -106,6 +104,8 @@ static bool d3d12_runner_check_requirements(struct shader_runner *r)
     if (runner->r.minimum_shader_model >= SHADER_MODEL_6_0 && !runner->dxc_compiler)
         return false;

+    *model = max(SHADER_MODEL_4_0, runner->r.minimum_shader_model);
+
     return true;
 }

@@ -336,7 +336,7 @@ static bool d3d12_runner_dispatch(struct shader_runner *r, unsigned int x, unsig
     size_t i;

cs_code = compile_shader(runner, runner->r.cs_source, SHADER_TYPE_CS);
-    todo_if(runner->r.is_todo && runner->r.minimum_shader_model < SHADER_MODEL_6_0) ok(cs_code, "Failed to compile shader.\n");
+    todo_if(runner->r.is_todo && runner->r.selected_shader_model < SHADER_MODEL_6_0) ok(cs_code, "Failed to compile shader.\n");
     if (!cs_code)
         return false;

diff --git a/tests/shader_runner_d3d9.c b/tests/shader_runner_d3d9.c
index 0b33c1c53..fbd7d6b5b 100644
--- a/tests/shader_runner_d3d9.c
+++ b/tests/shader_runner_d3d9.c
@@ -68,7 +68,7 @@ static ID3D10Blob *compile_shader(const struct d3d9_shader_runner *runner, const
         [SHADER_MODEL_3_0] = "3_0",
     };

-    sprintf(profile, "%s_%s", type, shader_models[runner->r.minimum_shader_model]);
+    sprintf(profile, "%s_%s", type, shader_models[runner->r.selected_shader_model]);
     hr = D3DCompile(source, strlen(source), NULL, NULL, NULL, "main", profile, runner->r.compile_options, 0, &blob, &errors);
     ok(hr == S_OK, "Failed to compile shader, hr %#lx.\n", hr);
     if (errors)
@@ -183,13 +183,15 @@ static D3DTEXTUREADDRESS sampler_address_to_d3d9(D3D12_TEXTURE_ADDRESS_MODE addr
     vkd3d_unreachable();
 }

-static bool d3d9_runner_check_requirements(struct shader_runner *r)
+static bool d3d9_runner_check_requirements(struct shader_runner *r, enum shader_model *model)
 {
     struct d3d9_shader_runner *runner = d3d9_shader_runner(r);

if (runner->r.minimum_shader_model >= SHADER_MODEL_4_0)
         return false;

+    *model = runner->r.minimum_shader_model;
+
     return true;
 }

diff --git a/tests/shader_runner_vulkan.c b/tests/shader_runner_vulkan.c
index ab9f6ecfc..97b7f6280 100644
--- a/tests/shader_runner_vulkan.c
+++ b/tests/shader_runner_vulkan.c
@@ -83,7 +83,7 @@ static struct vulkan_shader_runner *vulkan_shader_runner(struct shader_runner *r

#define VK_CALL(f) (runner->f)

-static bool vulkan_runner_check_requirements(struct shader_runner *r)
+static bool vulkan_runner_check_requirements(struct shader_runner *r, enum shader_model *model)
 {
     struct vulkan_shader_runner *runner = vulkan_shader_runner(r);

@@ -92,6 +92,8 @@ static bool vulkan_runner_check_requirements(struct shader_runner *r)
     if (runner->r.minimum_shader_model >= SHADER_MODEL_6_0)
         return false;

+    *model = max(SHADER_MODEL_4_0, runner->r.minimum_shader_model);
+
     return true;
 }

@@ -407,8 +409,6 @@ static bool compile_shader(const struct vulkan_shader_runner *runner, const char

static const char *const shader_models[] =
     {
-        [SHADER_MODEL_2_0] = "4_0",
-        [SHADER_MODEL_3_0] = "4_0",
         [SHADER_MODEL_4_0] = "4_0",
         [SHADER_MODEL_4_1] = "4_1",
         [SHADER_MODEL_5_0] = "5_0",
@@ -451,7 +451,7 @@ static bool compile_shader(const struct vulkan_shader_runner *runner, const char
     }

hlsl_info.entry_point = "main";
-    sprintf(profile, "%s_%s", type, shader_models[runner->r.minimum_shader_model]);
+    sprintf(profile, "%s_%s", type, shader_models[runner->r.selected_shader_model]);
     hlsl_info.profile = profile;

ret = vkd3d_shader_compile(&info, dxbc, &messages);

[PATCH 3/4] tests/shader-runner: Decouple run_shader_tests() ranges from those supported by the backend.

[Francisco Casas]

From: Francisco Casas

So, the idea is that running run_shader_tests() means to ask the backend
to run the shader_test file so that each test is executed with the
lowest profile the backend actually supports (or not excuted at all if
there isn't any).

Considering that in the future we may want to test all profiles for all
backends, for instance, for Windows:

for(m = SHADER_MODEL_2_0; m <= SHADER_MODEL_6_0; ++m)
    {
        run_shader_tests_d3d9(m, m);
        run_shader_tests_d3d11(m, m)
        run_shader_tests_d3d12(m, m, dxc_compiler);
    }
---
 tests/shader_runner.c        | 25 ++++++++++++++-----------
 tests/shader_runner.h        | 10 +++++-----
 tests/shader_runner_d3d11.c  |  4 ++--
 tests/shader_runner_d3d12.c  |  4 ++--
 tests/shader_runner_d3d9.c   |  4 ++--
 tests/shader_runner_vulkan.c |  4 ++--
 6 files changed, 27 insertions(+), 24 deletions(-)

diff --git a/tests/shader_runner.c b/tests/shader_runner.c
index e0ab9eb63..7d92d5757 100644
--- a/tests/shader_runner.c
+++ b/tests/shader_runner.c
@@ -955,7 +955,8 @@ static void compile_shader(struct shader_runner *runner, IDxcCompiler3 *dxc_comp

if (use_dxcompiler)
     {
-        assert(dxc_compiler);
+        if (!dxc_compiler)
+            return;
         hr = dxc_compiler_compile_shader(dxc_compiler, type, runner->compile_options, source, &blob, &errors);
     }
     else
@@ -1030,6 +1031,8 @@ static enum parse_state read_shader_directive(struct shader_runner *runner, enum
     return state;
 }

+/* Ask the backend to run the shader_test file so that each test is executed with the lowest profile
+ * it can support within the range of shader models specified, if any. */
 void run_shader_tests(struct shader_runner *runner, const struct shader_runner_ops *ops, void *dxc_compiler,
         enum shader_model minimum_shader_model, enum shader_model maximum_shader_model)
 {
@@ -1500,13 +1503,13 @@ START_TEST(shader_runner)
     trace("Running tests from a Windows cross build\n");

trace("Compiling shaders with d3dcompiler_47.dll and executing with d3d9.dll\n");
-    run_shader_tests_d3d9();
+    run_shader_tests_d3d9(SHADER_MODEL_2_0, SHADER_MODEL_6_0);

trace("Compiling shaders with d3dcompiler_47.dll and executing with d3d11.dll\n");
-    run_shader_tests_d3d11();
+    run_shader_tests_d3d11(SHADER_MODEL_2_0, SHADER_MODEL_6_0);

trace("Compiling shaders with d3dcompiler_47.dll and executing with d3d12.dll\n");
-    run_shader_tests_d3d12(NULL, SHADER_MODEL_4_0, SHADER_MODEL_5_1);
+    run_shader_tests_d3d12(SHADER_MODEL_2_0, SHADER_MODEL_6_0, NULL);

print_dll_version("d3dcompiler_47.dll");
     print_dll_version("dxgi.dll");
@@ -1517,18 +1520,18 @@ START_TEST(shader_runner)
     trace("Running tests from a Windows non-cross build\n");

trace("Compiling shaders with vkd3d-shader and executing with d3d9.dll\n");
-    run_shader_tests_d3d9();
+    run_shader_tests_d3d9(SHADER_MODEL_2_0, SHADER_MODEL_6_0);

trace("Compiling shaders with vkd3d-shader and executing with d3d11.dll\n");
-    run_shader_tests_d3d11();
+    run_shader_tests_d3d11(SHADER_MODEL_2_0, SHADER_MODEL_6_0);

trace("Compiling shaders with vkd3d-shader and executing with vkd3d\n");
-    run_shader_tests_d3d12(NULL, SHADER_MODEL_4_0, SHADER_MODEL_5_1);
+    run_shader_tests_d3d12(SHADER_MODEL_2_0, SHADER_MODEL_6_0, NULL);

if ((dxc_compiler = dxcompiler_create()))
     {
         trace("Compiling shaders with dxcompiler and executing with vkd3d\n");
-        run_shader_tests_d3d12(dxc_compiler, SHADER_MODEL_6_0, SHADER_MODEL_6_0);
+        run_shader_tests_d3d12(SHADER_MODEL_6_0, SHADER_MODEL_6_0, dxc_compiler);
         IDxcCompiler3_Release(dxc_compiler);
         print_dll_version(SONAME_LIBDXCOMPILER);
     }
@@ -1539,15 +1542,15 @@ START_TEST(shader_runner)
     trace("Running tests from a Unix build\n");

trace("Compiling shaders with vkd3d-shader and executing with Vulkan\n");
-    run_shader_tests_vulkan();
+    run_shader_tests_vulkan(SHADER_MODEL_2_0, SHADER_MODEL_6_0);

trace("Compiling shaders with vkd3d-shader and executing with vkd3d\n");
-    run_shader_tests_d3d12(NULL, SHADER_MODEL_4_0, SHADER_MODEL_5_1);
+    run_shader_tests_d3d12(SHADER_MODEL_2_0, SHADER_MODEL_6_0, NULL);

if ((dxc_compiler = dxcompiler_create()))
     {
         trace("Compiling shaders with dxcompiler and executing with vkd3d\n");
-        run_shader_tests_d3d12(dxc_compiler, SHADER_MODEL_6_0, SHADER_MODEL_6_0);
+        run_shader_tests_d3d12(SHADER_MODEL_6_0, SHADER_MODEL_6_0, dxc_compiler);
         IDxcCompiler3_Release(dxc_compiler);
     }
 #endif
diff --git a/tests/shader_runner.h b/tests/shader_runner.h
index e52a44596..b91e23d08 100644
--- a/tests/shader_runner.h
+++ b/tests/shader_runner.h
@@ -169,10 +169,10 @@ void run_shader_tests(struct shader_runner *runner, const struct shader_runner_o
         enum shader_model minimum_shader_model, enum shader_model maximum_shader_model);

#ifdef _WIN32
-void run_shader_tests_d3d9(void);
-void run_shader_tests_d3d11(void);
+void run_shader_tests_d3d9(enum shader_model minimum_shader_model, enum shader_model maximum_shader_model);
+void run_shader_tests_d3d11(enum shader_model minimum_shader_model, enum shader_model maximum_shader_model);
 #else
-void run_shader_tests_vulkan(void);
+void run_shader_tests_vulkan(enum shader_model minimum_shader_model, enum shader_model maximum_shader_model);
 #endif
-void run_shader_tests_d3d12(void *dxc_compiler, enum shader_model minimum_shader_model,
-        enum shader_model maximum_shader_model);
+void run_shader_tests_d3d12(enum shader_model minimum_shader_model, enum shader_model maximum_shader_model,
+        void *dxc_compiler);
diff --git a/tests/shader_runner_d3d11.c b/tests/shader_runner_d3d11.c
index df704da8c..978dd471e 100644
--- a/tests/shader_runner_d3d11.c
+++ b/tests/shader_runner_d3d11.c
@@ -745,7 +745,7 @@ static const struct shader_runner_ops d3d11_runner_ops =
     .release_readback = d3d11_runner_release_readback,
 };

-void run_shader_tests_d3d11(void)
+void run_shader_tests_d3d11(enum shader_model minimum_shader_model, enum shader_model maximum_shader_model)
 {
     struct d3d11_shader_runner runner;
     HMODULE dxgi_module, d3d11_module;
@@ -760,7 +760,7 @@ void run_shader_tests_d3d11(void)
         init_adapter_info();
         if (init_test_context(&runner))
         {
-            run_shader_tests(&runner.r, &d3d11_runner_ops, NULL, SHADER_MODEL_2_0, SHADER_MODEL_5_1);
+            run_shader_tests(&runner.r, &d3d11_runner_ops, NULL, minimum_shader_model, maximum_shader_model);
             destroy_test_context(&runner);
         }
     }
diff --git a/tests/shader_runner_d3d12.c b/tests/shader_runner_d3d12.c
index ccac95189..98e3e7c0f 100644
--- a/tests/shader_runner_d3d12.c
+++ b/tests/shader_runner_d3d12.c
@@ -589,8 +589,8 @@ static const struct shader_runner_ops d3d12_runner_ops =
     .release_readback = d3d12_runner_release_readback,
 };

-void run_shader_tests_d3d12(void *dxc_compiler, enum shader_model minimum_shader_model,
-        enum shader_model maximum_shader_model)
+void run_shader_tests_d3d12(enum shader_model minimum_shader_model, enum shader_model maximum_shader_model,
+        void *dxc_compiler)
 {
     static const struct test_context_desc desc =
     {
diff --git a/tests/shader_runner_d3d9.c b/tests/shader_runner_d3d9.c
index fbd7d6b5b..b05d178b2 100644
--- a/tests/shader_runner_d3d9.c
+++ b/tests/shader_runner_d3d9.c
@@ -531,7 +531,7 @@ static const struct shader_runner_ops d3d9_runner_ops =
     .release_readback = d3d9_runner_release_readback,
 };

-void run_shader_tests_d3d9(void)
+void run_shader_tests_d3d9(enum shader_model minimum_shader_model, enum shader_model maximum_shader_model)
 {
     struct d3d9_shader_runner runner;
     HMODULE d3d9_module;
@@ -543,7 +543,7 @@ void run_shader_tests_d3d9(void)

init_adapter_info();
         init_test_context(&runner);
-        run_shader_tests(&runner.r, &d3d9_runner_ops, NULL, SHADER_MODEL_2_0, SHADER_MODEL_3_0);
+        run_shader_tests(&runner.r, &d3d9_runner_ops, NULL, minimum_shader_model, maximum_shader_model);
         destroy_test_context(&runner);
     }
     FreeLibrary(d3d9_module);
diff --git a/tests/shader_runner_vulkan.c b/tests/shader_runner_vulkan.c
index 97b7f6280..6a332bdf0 100644
--- a/tests/shader_runner_vulkan.c
+++ b/tests/shader_runner_vulkan.c
@@ -1387,14 +1387,14 @@ static void cleanup_vulkan_runner(struct vulkan_shader_runner *runner)
     VK_CALL(vkDestroyInstance(runner->instance, NULL));
 }

-void run_shader_tests_vulkan(void)
+void run_shader_tests_vulkan(enum shader_model minimum_shader_model, enum shader_model maximum_shader_model)
 {
     struct vulkan_shader_runner runner = {0};

if (!init_vulkan_runner(&runner))
         return;

-    run_shader_tests(&runner.r, &vulkan_runner_ops, NULL, SHADER_MODEL_2_0, SHADER_MODEL_5_1);
+    run_shader_tests(&runner.r, &vulkan_runner_ops, NULL, minimum_shader_model, maximum_shader_model);

cleanup_vulkan_runner(&runner);
 }

[PATCH 4/4] tests/shader-runner: Compile [shader] directives separately from backend execution.

[Francisco Casas]

From: Francisco Casas

This would allow us to test compilation of SM1 shaders even if we don't
have an available backend that can run them.
---
 tests/shader_runner.c        | 62 ++++++++++++++++++++++++++++++------
 tests/shader_runner.h        |  2 +-
 tests/shader_runner_d3d11.c  |  2 +-
 tests/shader_runner_d3d12.c  |  2 +-
 tests/shader_runner_d3d9.c   |  2 +-
 tests/shader_runner_vulkan.c |  2 +-
 6 files changed, 57 insertions(+), 15 deletions(-)

diff --git a/tests/shader_runner.c b/tests/shader_runner.c
index 7d92d5757..690066ab4 100644
--- a/tests/shader_runner.c
+++ b/tests/shader_runner.c
@@ -1032,9 +1032,11 @@ static enum parse_state read_shader_directive(struct shader_runner *runner, enum
 }

/* Ask the backend to run the shader_test file so that each test is executed with the lowest profile
- * it can support within the range of shader models specified, if any. */
+ * it can support within the range of shader models specified, if any.
+ * If check_syntax is true, then the shaders are compiled on the [shader] directives using the
+ * available compiler. If ops is NULL, execution tests are skipped. */
 void run_shader_tests(struct shader_runner *runner, const struct shader_runner_ops *ops, void *dxc_compiler,
-        enum shader_model minimum_shader_model, enum shader_model maximum_shader_model)
+        enum shader_model minimum_shader_model, enum shader_model maximum_shader_model, bool check_syntax)
 {
     size_t shader_source_size = 0, shader_source_len = 0;
     struct resource_params current_resource;
@@ -1058,8 +1060,16 @@ void run_shader_tests(struct shader_runner *runner, const struct shader_runner_o
     runner->minimum_shader_model = minimum_shader_model;
     runner->maximum_shader_model = maximum_shader_model;

-    assert(runner->ops->check_requirements);
-    skip_tests = !runner->ops->check_requirements(runner, &runner->selected_shader_model);
+    if (runner->ops)
+    {
+        skip_tests = !runner->ops->check_requirements(runner, &runner->selected_shader_model);
+    }
+    else
+    {
+        /* TODO: Make runner->selected_shader_model = runner->minimum_shader_model so that
+         * check_syntax is allowed to compile SM1 profiles. */
+        runner->selected_shader_model = max(SHADER_MODEL_4_0, runner->minimum_shader_model);
+    }

for (;;)
     {
@@ -1079,21 +1089,29 @@ void run_shader_tests(struct shader_runner *runner, const struct shader_runner_o
                     break;

case STATE_REQUIRE:
+                    assert(!runner->ops || runner->ops->check_requirements);
                     if (runner->maximum_shader_model < runner->minimum_shader_model
-                            || !runner->ops->check_requirements(runner, &runner->selected_shader_model))
+                            || (runner->ops && !runner->ops->check_requirements(runner, &runner->selected_shader_model)))
                     {
                         skip_tests = true;
                     }
+                    if (!runner->ops)
+                    {
+                        /* TODO: Make runner->selected_shader_model = runner->minimum_shader_model so that
+                         * check_syntax is allowed to compile SM1 profiles. */
+                        runner->selected_shader_model = max(SHADER_MODEL_4_0, runner->minimum_shader_model);
+                    }
                     break;

case STATE_RESOURCE:
-                    set_resource(runner, runner->ops->create_resource(runner, &current_resource));
+                    if (runner->ops)
+                        set_resource(runner, runner->ops->create_resource(runner, &current_resource));
                     free(current_resource.data);
                     break;

case STATE_SHADER_COMPUTE:
                 case STATE_SHADER_COMPUTE_TODO:
-                    if (!skip_tests)
+                    if (check_syntax && !skip_tests)
                     {
                         todo_if (state == STATE_SHADER_COMPUTE_TODO)
                         compile_shader(runner, dxc_compiler, shader_source, shader_source_len, SHADER_TYPE_CS,
@@ -1108,7 +1126,7 @@ void run_shader_tests(struct shader_runner *runner, const struct shader_runner_o

case STATE_SHADER_PIXEL:
                 case STATE_SHADER_PIXEL_TODO:
-                    if (!skip_tests)
+                    if (check_syntax && !skip_tests)
                     {
                         todo_if (state == STATE_SHADER_PIXEL_TODO)
                         compile_shader(runner, dxc_compiler, shader_source, shader_source_len, SHADER_TYPE_PS,
@@ -1123,7 +1141,7 @@ void run_shader_tests(struct shader_runner *runner, const struct shader_runner_o

case STATE_SHADER_VERTEX:
                 case STATE_SHADER_VERTEX_TODO:
-                    if (!skip_tests)
+                    if (check_syntax && !skip_tests)
                     {
                         todo_if (state == STATE_SHADER_VERTEX_TODO)
                         compile_shader(runner, dxc_compiler, shader_source, shader_source_len, SHADER_TYPE_VS,
@@ -1380,7 +1398,7 @@ void run_shader_tests(struct shader_runner *runner, const struct shader_runner_o
                 case STATE_TEST:
                     /* Compilation which fails with dxcompiler is not 'todo', therefore the tests are
                      * not 'todo' either. They cannot run, so skip them entirely. */
-                    if (!skip_tests && SUCCEEDED(expect_hr))
+                    if (runner->ops && !skip_tests && SUCCEEDED(expect_hr))
                         parse_test_directive(runner, line);
                     break;
             }
@@ -1394,6 +1412,7 @@ void run_shader_tests(struct shader_runner *runner, const struct shader_runner_o
     free(runner->ps_source);
     for (i = 0; i < runner->resource_count; ++i)
     {
+        assert(runner->ops);
         if (runner->resources[i])
             runner->ops->destroy_resource(runner, runner->resources[i]);
     }
@@ -1401,6 +1420,14 @@ void run_shader_tests(struct shader_runner *runner, const struct shader_runner_o
     fclose(f);
 }

+static void run_shader_compilation_tests(enum shader_model minimum_shader_model,
+        enum shader_model maximum_shader_model, void *dxc_compiler)
+{
+    struct shader_runner runner = {};
+
+    return run_shader_tests(&runner, NULL, dxc_compiler, minimum_shader_model, maximum_shader_model, true);
+}
+
 #ifdef _WIN32
 static void print_dll_version(const char *file_name)
 {
@@ -1502,6 +1529,9 @@ START_TEST(shader_runner)
 #if defined(VKD3D_CROSSTEST)
     trace("Running tests from a Windows cross build\n");

+    trace("Compiling shaders with d3dcompiler_47.dll\n");
+    run_shader_compilation_tests(SHADER_MODEL_2_0, SHADER_MODEL_6_0, NULL);
+
     trace("Compiling shaders with d3dcompiler_47.dll and executing with d3d9.dll\n");
     run_shader_tests_d3d9(SHADER_MODEL_2_0, SHADER_MODEL_6_0);

@@ -1519,6 +1549,9 @@ START_TEST(shader_runner)
 #elif defined(_WIN32)
     trace("Running tests from a Windows non-cross build\n");

+    trace("Compiling shaders with vkd3d-shader\n");
+    run_shader_compilation_tests(SHADER_MODEL_2_0, SHADER_MODEL_6_0, NULL);
+
     trace("Compiling shaders with vkd3d-shader and executing with d3d9.dll\n");
     run_shader_tests_d3d9(SHADER_MODEL_2_0, SHADER_MODEL_6_0);

@@ -1530,6 +1563,9 @@ START_TEST(shader_runner)

if ((dxc_compiler = dxcompiler_create()))
     {
+        trace("Compiling shaders with dxcompiler\n");
+        run_shader_compilation_tests(SHADER_MODEL_6_0, SHADER_MODEL_6_0, dxc_compiler);
+
         trace("Compiling shaders with dxcompiler and executing with vkd3d\n");
         run_shader_tests_d3d12(SHADER_MODEL_6_0, SHADER_MODEL_6_0, dxc_compiler);
         IDxcCompiler3_Release(dxc_compiler);
@@ -1541,6 +1577,9 @@ START_TEST(shader_runner)
 #else
     trace("Running tests from a Unix build\n");

+    trace("Compiling shaders with vkd3d-shader\n");
+    run_shader_compilation_tests(SHADER_MODEL_2_0, SHADER_MODEL_6_0, NULL);
+
     trace("Compiling shaders with vkd3d-shader and executing with Vulkan\n");
     run_shader_tests_vulkan(SHADER_MODEL_2_0, SHADER_MODEL_6_0);

@@ -1549,6 +1588,9 @@ START_TEST(shader_runner)

if ((dxc_compiler = dxcompiler_create()))
     {
+        trace("Compiling shaders with dxcompiler\n");
+        run_shader_compilation_tests(SHADER_MODEL_6_0, SHADER_MODEL_6_0, dxc_compiler);
+
         trace("Compiling shaders with dxcompiler and executing with vkd3d\n");
         run_shader_tests_d3d12(SHADER_MODEL_6_0, SHADER_MODEL_6_0, dxc_compiler);
         IDxcCompiler3_Release(dxc_compiler);
diff --git a/tests/shader_runner.h b/tests/shader_runner.h
index b91e23d08..9763b3661 100644
--- a/tests/shader_runner.h
+++ b/tests/shader_runner.h
@@ -166,7 +166,7 @@ HRESULT dxc_compiler_compile_shader(void *dxc_compiler, enum shader_type type, u
         const char *hlsl, ID3D10Blob **blob_out, ID3D10Blob **errors_out);

void run_shader_tests(struct shader_runner *runner, const struct shader_runner_ops *ops, void *dxc_compiler,
-        enum shader_model minimum_shader_model, enum shader_model maximum_shader_model);
+        enum shader_model minimum_shader_model, enum shader_model maximum_shader_model, bool check_syntax);

#ifdef _WIN32
 void run_shader_tests_d3d9(enum shader_model minimum_shader_model, enum shader_model maximum_shader_model);
diff --git a/tests/shader_runner_d3d11.c b/tests/shader_runner_d3d11.c
index 978dd471e..ab7eb94fa 100644
--- a/tests/shader_runner_d3d11.c
+++ b/tests/shader_runner_d3d11.c
@@ -760,7 +760,7 @@ void run_shader_tests_d3d11(enum shader_model minimum_shader_model, enum shader_
         init_adapter_info();
         if (init_test_context(&runner))
         {
-            run_shader_tests(&runner.r, &d3d11_runner_ops, NULL, minimum_shader_model, maximum_shader_model);
+            run_shader_tests(&runner.r, &d3d11_runner_ops, NULL, minimum_shader_model, maximum_shader_model, false);
             destroy_test_context(&runner);
         }
     }
diff --git a/tests/shader_runner_d3d12.c b/tests/shader_runner_d3d12.c
index 98e3e7c0f..8f15f1bf8 100644
--- a/tests/shader_runner_d3d12.c
+++ b/tests/shader_runner_d3d12.c
@@ -622,7 +622,7 @@ void run_shader_tests_d3d12(enum shader_model minimum_shader_model, enum shader_
             runner.compute_allocator, NULL, &IID_ID3D12GraphicsCommandList, (void **)&runner.compute_list);
     ok(hr == S_OK, "Failed to create command list, hr %#x.\n", hr);

-    run_shader_tests(&runner.r, &d3d12_runner_ops, dxc_compiler, minimum_shader_model, maximum_shader_model);
+    run_shader_tests(&runner.r, &d3d12_runner_ops, dxc_compiler, minimum_shader_model, maximum_shader_model, false);

ID3D12GraphicsCommandList_Release(runner.compute_list);
     ID3D12CommandAllocator_Release(runner.compute_allocator);
diff --git a/tests/shader_runner_d3d9.c b/tests/shader_runner_d3d9.c
index b05d178b2..3c81f70d5 100644
--- a/tests/shader_runner_d3d9.c
+++ b/tests/shader_runner_d3d9.c
@@ -543,7 +543,7 @@ void run_shader_tests_d3d9(enum shader_model minimum_shader_model, enum shader_m

init_adapter_info();
         init_test_context(&runner);
-        run_shader_tests(&runner.r, &d3d9_runner_ops, NULL, minimum_shader_model, maximum_shader_model);
+        run_shader_tests(&runner.r, &d3d9_runner_ops, NULL, minimum_shader_model, maximum_shader_model, false);
         destroy_test_context(&runner);
     }
     FreeLibrary(d3d9_module);
diff --git a/tests/shader_runner_vulkan.c b/tests/shader_runner_vulkan.c
index 6a332bdf0..2d0a9a432 100644
--- a/tests/shader_runner_vulkan.c
+++ b/tests/shader_runner_vulkan.c
@@ -1394,7 +1394,7 @@ void run_shader_tests_vulkan(enum shader_model minimum_shader_model, enum shader
     if (!init_vulkan_runner(&runner))
         return;

-    run_shader_tests(&runner.r, &vulkan_runner_ops, NULL, minimum_shader_model, maximum_shader_model);
+    run_shader_tests(&runner.r, &vulkan_runner_ops, NULL, minimum_shader_model, maximum_shader_model, false);

cleanup_vulkan_runner(&runner);
 }