[PATCH vkd3d 3/8] vkd3d: Do not disable depth-stencil tests for PSOs with invalid DSV format.

[Józef Kucia]

From: Józef Kucia <jkucia(a)codeweavers.com>

This is invalid usage according to the D3D12 validation layer. However,
Shadow of the Tomb Raider uses PSOs with DSVFormat equal to
DXGI_FORMAT_UNKNOWN and enabled depth-stencil tests. Moreover, the test
in the next commit passes on Windows with AMD, Intel and Nvidia GPUs.

Signed-off-by: Józef Kucia <jkucia(a)codeweavers.com>
---
 libs/vkd3d/command.c       | 14 +++++--
 libs/vkd3d/state.c         | 78 ++++++++++++++++++++++++++++----------
 libs/vkd3d/vkd3d_private.h |  5 ++-
 3 files changed, 72 insertions(+), 25 deletions(-)

diff --git a/libs/vkd3d/command.c b/libs/vkd3d/command.c
index 8c7057b36340..861c8405c3e6 100644
--- a/libs/vkd3d/command.c
+++ b/libs/vkd3d/command.c
@@ -2251,11 +2251,13 @@ static void d3d12_command_list_reset_state(struct d3d12_command_list *list,
     list->fb_width = 0;
     list->fb_height = 0;
     list->fb_layer_count = 0;
+    list->dsv_format = VK_FORMAT_UNDEFINED;
 
     list->xfb_enabled = false;
 
     list->current_framebuffer = VK_NULL_HANDLE;
     list->current_pipeline = VK_NULL_HANDLE;
+    list->pso_render_pass = VK_NULL_HANDLE;
     list->current_render_pass = VK_NULL_HANDLE;
 
     memset(list->pipeline_bindings, 0, sizeof(list->pipeline_bindings));
@@ -2370,7 +2372,7 @@ static bool d3d12_command_list_update_current_framebuffer(struct d3d12_command_l
     fb_desc.sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO;
     fb_desc.pNext = NULL;
     fb_desc.flags = 0;
-    fb_desc.renderPass = graphics->render_pass;
+    fb_desc.renderPass = list->pso_render_pass;
     fb_desc.attachmentCount = view_count;
     fb_desc.pAttachments = views;
     d3d12_command_list_get_fb_extent(list, &fb_desc.width, &fb_desc.height, &fb_desc.layers);
@@ -2407,9 +2409,12 @@ static bool d3d12_command_list_update_current_pipeline(struct d3d12_command_list
     }
 
     if (!(vk_pipeline = d3d12_pipeline_state_get_or_create_pipeline(list->state,
-            list->primitive_topology, list->strides)))
+            list->primitive_topology, list->strides, list->dsv_format, &list->pso_render_pass)))
         return false;
 
+    if (!list->pso_render_pass)
+        list->pso_render_pass = list->state->u.graphics.render_pass;
+
     VK_CALL(vkCmdBindPipeline(list->vk_command_buffer, list->state->vk_bind_point, vk_pipeline));
     list->current_pipeline = vk_pipeline;
 
@@ -2797,7 +2802,8 @@ static bool d3d12_command_list_begin_render_pass(struct d3d12_command_list *list
     if (list->current_render_pass != VK_NULL_HANDLE)
         return true;
 
-    vk_render_pass = list->state->u.graphics.render_pass;
+    vk_render_pass = list->pso_render_pass;
+    assert(vk_render_pass);
 
     begin_desc.sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO;
     begin_desc.pNext = NULL;
@@ -4373,11 +4379,13 @@ static void STDMETHODCALLTYPE d3d12_command_list_OMSetRenderTargets(ID3D12Graphi
             list->fb_width = max(list->fb_width, dsv_desc->width);
             list->fb_height = max(list->fb_height, dsv_desc->height);
             list->fb_layer_count = max(list->fb_layer_count, dsv_desc->layer_count);
+            list->dsv_format = dsv_desc->format;
         }
         else
         {
             WARN("DSV descriptor is not initialized.\n");
             list->views[0] = VK_NULL_HANDLE;
+            list->dsv_format = VK_FORMAT_UNDEFINED;
         }
     }
 
diff --git a/libs/vkd3d/state.c b/libs/vkd3d/state.c
index b1f4ef9acac1..457f3d9bc84b 100644
--- a/libs/vkd3d/state.c
+++ b/libs/vkd3d/state.c
@@ -1236,6 +1236,7 @@ struct vkd3d_pipeline_key
 {
     D3D12_PRIMITIVE_TOPOLOGY topology;
     uint32_t strides[D3D12_IA_VERTEX_INPUT_RESOURCE_SLOT_COUNT];
+    VkFormat dsv_format;
 };
 
 struct vkd3d_compiled_pipeline
@@ -1243,6 +1244,7 @@ struct vkd3d_compiled_pipeline
     struct list entry;
     struct vkd3d_pipeline_key key;
     VkPipeline vk_pipeline;
+    VkRenderPass vk_render_pass;
 };
 
 /* ID3D12PipelineState */
@@ -2052,7 +2054,8 @@ bool d3d12_pipeline_state_is_render_pass_compatible(const struct d3d12_pipeline_
 STATIC_ASSERT(sizeof(struct vkd3d_shader_transform_feedback_element) == sizeof(D3D12_SO_DECLARATION_ENTRY));
 
 static HRESULT d3d12_graphics_pipeline_state_create_render_pass(
-        struct d3d12_graphics_pipeline_state *graphics, struct d3d12_device *device)
+        struct d3d12_graphics_pipeline_state *graphics, struct d3d12_device *device,
+        VkFormat dynamic_dsv_format, VkRenderPass *vk_render_pass)
 {
     struct vkd3d_render_pass_key key;
     unsigned int i;
@@ -2064,7 +2067,12 @@ static HRESULT d3d12_graphics_pipeline_state_create_render_pass(
         key.stencil_enable = graphics->ds_desc.stencilTestEnable;
         key.depth_stencil_write = graphics->ds_desc.depthWriteEnable
                 || graphics->ds_desc.front.writeMask;
-        key.vk_formats[0] = graphics->dsv_format;
+
+        if (!(key.vk_formats[0] = graphics->dsv_format))
+            key.vk_formats[0] = dynamic_dsv_format;
+
+        if (!key.vk_formats[0])
+            FIXME("Compiling with DXGI_FORMAT_UNKNOWN.\n");
     }
     else
     {
@@ -2082,7 +2090,7 @@ static HRESULT d3d12_graphics_pipeline_state_create_render_pass(
     key.padding = 0;
     key.sample_count = graphics->ms_desc.rasterizationSamples;
 
-    return vkd3d_render_pass_cache_find(&device->render_pass_cache, device, &key, &graphics->render_pass);
+    return vkd3d_render_pass_cache_find(&device->render_pass_cache, device, &key, vk_render_pass);
 }
 
 static HRESULT d3d12_pipeline_state_init_graphics(struct d3d12_pipeline_state *state,
@@ -2177,26 +2185,29 @@ static HRESULT d3d12_pipeline_state_init_graphics(struct d3d12_pipeline_state *s
         rt_count = ARRAY_SIZE(graphics->blend_attachments);
     }
 
-    if (desc->DSVFormat == DXGI_FORMAT_UNKNOWN
-            && (desc->DepthStencilState.DepthEnable || desc->DepthStencilState.StencilEnable))
-        FIXME("DSV format is DXGI_FORMAT_UNKNOWN, disabling depth/stencil tests.\n");
-
     graphics->rt_idx = 0;
     graphics->null_attachment_mask = 0;
-    if (desc->DSVFormat != DXGI_FORMAT_UNKNOWN
-            && (desc->DepthStencilState.DepthEnable || desc->DepthStencilState.StencilEnable))
+    if (desc->DepthStencilState.DepthEnable || desc->DepthStencilState.StencilEnable)
     {
-        if (!(format = vkd3d_get_format(device, desc->DSVFormat, true)))
+        if (desc->DSVFormat == DXGI_FORMAT_UNKNOWN)
+        {
+            WARN("DSV format is DXGI_FORMAT_UNKNOWN.\n");
+            graphics->dsv_format = VK_FORMAT_UNDEFINED;
+        }
+        else if ((format = vkd3d_get_format(device, desc->DSVFormat, true)))
+        {
+            if (!(format->vk_aspect_mask & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)))
+                FIXME("Format %#x is not depth/stencil format.\n", format->dxgi_format);
+
+            graphics->dsv_format = format->vk_format;
+        }
+        else
         {
             WARN("Invalid DSV format %#x.\n", desc->DSVFormat);
             hr = E_INVALIDARG;
             goto fail;
         }
 
-        if (!(format->vk_aspect_mask & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)))
-            FIXME("Format %#x is not depth/stencil format.\n", format->dxgi_format);
-
-        graphics->dsv_format = format->vk_format;
         ++graphics->rt_idx;
 
         if (!desc->PS.pShaderBytecode)
@@ -2538,7 +2549,10 @@ static HRESULT d3d12_pipeline_state_init_graphics(struct d3d12_pipeline_state *s
 
     ds_desc_from_d3d12(&graphics->ds_desc, &desc->DepthStencilState);
 
-    if (FAILED(hr = d3d12_graphics_pipeline_state_create_render_pass(graphics, device)))
+    if (graphics->dsv_format == VK_FORMAT_UNDEFINED)
+        graphics->render_pass = VK_NULL_HANDLE;
+    else if (FAILED(hr = d3d12_graphics_pipeline_state_create_render_pass(graphics,
+            device, 0, &graphics->render_pass)))
         goto fail;
 
     graphics->root_signature = root_signature;
@@ -2640,7 +2654,7 @@ static enum VkPrimitiveTopology vk_topology_from_d3d12_topology(D3D12_PRIMITIVE_
 }
 
 static VkPipeline d3d12_pipeline_state_find_compiled_pipeline(const struct d3d12_pipeline_state *state,
-        const struct vkd3d_pipeline_key *key)
+        const struct vkd3d_pipeline_key *key, VkRenderPass *vk_render_pass)
 {
     const struct d3d12_graphics_pipeline_state *graphics = &state->u.graphics;
     struct d3d12_device *device = state->device;
@@ -2648,6 +2662,8 @@ static VkPipeline d3d12_pipeline_state_find_compiled_pipeline(const struct d3d12
     struct vkd3d_compiled_pipeline *current;
     int rc;
 
+    *vk_render_pass = VK_NULL_HANDLE;
+
     if (!(rc = pthread_mutex_lock(&device->mutex)))
     {
         LIST_FOR_EACH_ENTRY(current, &graphics->compiled_pipelines, struct vkd3d_compiled_pipeline, entry)
@@ -2655,6 +2671,7 @@ static VkPipeline d3d12_pipeline_state_find_compiled_pipeline(const struct d3d12
             if (!memcmp(¤t->key, key, sizeof(*key)))
             {
                 vk_pipeline = current->vk_pipeline;
+                *vk_render_pass = current->vk_render_pass;
                 break;
             }
         }
@@ -2669,7 +2686,7 @@ static VkPipeline d3d12_pipeline_state_find_compiled_pipeline(const struct d3d12
 }
 
 static bool d3d12_pipeline_state_put_pipeline_to_cache(struct d3d12_pipeline_state *state,
-        const struct vkd3d_pipeline_key *key, VkPipeline vk_pipeline)
+        const struct vkd3d_pipeline_key *key, VkPipeline vk_pipeline, VkRenderPass vk_render_pass)
 {
     struct d3d12_graphics_pipeline_state *graphics = &state->u.graphics;
     struct vkd3d_compiled_pipeline *compiled_pipeline, *current;
@@ -2681,6 +2698,7 @@ static bool d3d12_pipeline_state_put_pipeline_to_cache(struct d3d12_pipeline_sta
 
     compiled_pipeline->key = *key;
     compiled_pipeline->vk_pipeline = vk_pipeline;
+    compiled_pipeline->vk_render_pass = vk_render_pass;
 
     if ((rc = pthread_mutex_lock(&device->mutex)))
     {
@@ -2707,7 +2725,8 @@ static bool d3d12_pipeline_state_put_pipeline_to_cache(struct d3d12_pipeline_sta
 }
 
 VkPipeline d3d12_pipeline_state_get_or_create_pipeline(struct d3d12_pipeline_state *state,
-        D3D12_PRIMITIVE_TOPOLOGY topology, const uint32_t *strides)
+        D3D12_PRIMITIVE_TOPOLOGY topology, const uint32_t *strides, VkFormat dsv_format,
+        VkRenderPass *vk_render_pass)
 {
     VkVertexInputBindingDescription bindings[D3D12_IA_VERTEX_INPUT_RESOURCE_SLOT_COUNT];
     const struct vkd3d_vk_device_procs *vk_procs = &state->device->vk_procs;
@@ -2725,6 +2744,7 @@ VkPipeline d3d12_pipeline_state_get_or_create_pipeline(struct d3d12_pipeline_sta
     unsigned int i;
     uint32_t mask;
     VkResult vr;
+    HRESULT hr;
 
     static const VkPipelineViewportStateCreateInfo vp_desc =
     {
@@ -2754,6 +2774,8 @@ VkPipeline d3d12_pipeline_state_get_or_create_pipeline(struct d3d12_pipeline_sta
 
     assert(d3d12_pipeline_state_is_graphics(state));
 
+    *vk_render_pass = VK_NULL_HANDLE;
+
     memset(&pipeline_key, 0, sizeof(pipeline_key));
     pipeline_key.topology = topology;
 
@@ -2783,7 +2805,9 @@ VkPipeline d3d12_pipeline_state_get_or_create_pipeline(struct d3d12_pipeline_sta
         ++binding_count;
     }
 
-    if ((vk_pipeline = d3d12_pipeline_state_find_compiled_pipeline(state, &pipeline_key)))
+    pipeline_key.dsv_format = dsv_format;
+
+    if ((vk_pipeline = d3d12_pipeline_state_find_compiled_pipeline(state, &pipeline_key, vk_render_pass)))
         return vk_pipeline;
 
     input_desc.sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO;
@@ -2846,6 +2870,18 @@ VkPipeline d3d12_pipeline_state_get_or_create_pipeline(struct d3d12_pipeline_sta
     pipeline_desc.subpass = 0;
     pipeline_desc.basePipelineHandle = VK_NULL_HANDLE;
     pipeline_desc.basePipelineIndex = -1;
+
+    /* Create a render pass for pipelines with DXGI_FORMAT_UNKNOWN. */
+    if (!pipeline_desc.renderPass)
+    {
+        TRACE("Compiling %p with DSV format %#x.\n", state, dsv_format);
+        if (FAILED(hr = d3d12_graphics_pipeline_state_create_render_pass(graphics, device, dsv_format,
+                &pipeline_desc.renderPass)))
+            return VK_NULL_HANDLE;
+
+        *vk_render_pass = pipeline_desc.renderPass;
+    }
+
     if ((vr = VK_CALL(vkCreateGraphicsPipelines(device->vk_device, device->vk_pipeline_cache,
             1, &pipeline_desc, NULL, &vk_pipeline))) < 0)
     {
@@ -2853,12 +2889,12 @@ VkPipeline d3d12_pipeline_state_get_or_create_pipeline(struct d3d12_pipeline_sta
         return VK_NULL_HANDLE;
     }
 
-    if (d3d12_pipeline_state_put_pipeline_to_cache(state, &pipeline_key, vk_pipeline))
+    if (d3d12_pipeline_state_put_pipeline_to_cache(state, &pipeline_key, vk_pipeline, pipeline_desc.renderPass))
         return vk_pipeline;
 
     /* Other thread compiled the pipeline before us. */
     VK_CALL(vkDestroyPipeline(device->vk_device, vk_pipeline, NULL));
-    vk_pipeline = d3d12_pipeline_state_find_compiled_pipeline(state, &pipeline_key);
+    vk_pipeline = d3d12_pipeline_state_find_compiled_pipeline(state, &pipeline_key, vk_render_pass);
     if (!vk_pipeline)
         ERR("Could not get the pipeline compiled by other thread from the cache.\n");
     return vk_pipeline;
diff --git a/libs/vkd3d/vkd3d_private.h b/libs/vkd3d/vkd3d_private.h
index 4ae6afe128db..3fd1eacbfc77 100644
--- a/libs/vkd3d/vkd3d_private.h
+++ b/libs/vkd3d/vkd3d_private.h
@@ -741,7 +741,8 @@ HRESULT d3d12_pipeline_state_create_compute(struct d3d12_device *device,
 HRESULT d3d12_pipeline_state_create_graphics(struct d3d12_device *device,
         const D3D12_GRAPHICS_PIPELINE_STATE_DESC *desc, struct d3d12_pipeline_state **state) DECLSPEC_HIDDEN;
 VkPipeline d3d12_pipeline_state_get_or_create_pipeline(struct d3d12_pipeline_state *state,
-        D3D12_PRIMITIVE_TOPOLOGY topology, const uint32_t *strides) DECLSPEC_HIDDEN;
+        D3D12_PRIMITIVE_TOPOLOGY topology, const uint32_t *strides, VkFormat dsv_format,
+        VkRenderPass *vk_render_pass) DECLSPEC_HIDDEN;
 bool d3d12_pipeline_state_is_render_pass_compatible(const struct d3d12_pipeline_state *state_a,
         const struct d3d12_pipeline_state *state_b) DECLSPEC_HIDDEN;
 struct d3d12_pipeline_state *unsafe_impl_from_ID3D12PipelineState(ID3D12PipelineState *iface) DECLSPEC_HIDDEN;
@@ -861,11 +862,13 @@ struct d3d12_command_list
     unsigned int fb_width;
     unsigned int fb_height;
     unsigned int fb_layer_count;
+    VkFormat dsv_format;
 
     bool xfb_enabled;
 
     VkFramebuffer current_framebuffer;
     VkPipeline current_pipeline;
+    VkRenderPass pso_render_pass;
     VkRenderPass current_render_pass;
     struct vkd3d_pipeline_bindings pipeline_bindings[VK_PIPELINE_BIND_POINT_RANGE_SIZE];
 
-- 
2.21.0

[Henri Verbeet]

Signed-off-by: Henri Verbeet <hverbeet(a)codeweavers.com>