[PATCH v3 5/9] vkd3d-shader/spirv: Return an error if an invalid handler is encountered.

[Conor McCarthy]

From: Conor McCarthy

Prevents assertions while handling later instructions, and prevents
return of an invalid SPIR-V module.
---
 libs/vkd3d-shader/spirv.c | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/libs/vkd3d-shader/spirv.c b/libs/vkd3d-shader/spirv.c
index d2621ffa1..2f88ecb31 100644
--- a/libs/vkd3d-shader/spirv.c
+++ b/libs/vkd3d-shader/spirv.c
@@ -6657,7 +6657,7 @@ static void spirv_compiler_emit_bool_cast(struct spirv_compiler *compiler,
     spirv_compiler_emit_store_dst(compiler, dst, val_id);
 }

-static void spirv_compiler_emit_alu_instruction(struct spirv_compiler *compiler,
+static enum vkd3d_result spirv_compiler_emit_alu_instruction(struct spirv_compiler *compiler,
         const struct vkd3d_shader_instruction *instruction)
 {
     struct vkd3d_spirv_builder *builder = &compiler->spirv_builder;
@@ -6681,7 +6681,7 @@ static void spirv_compiler_emit_alu_instruction(struct spirv_compiler *compiler,
             /* VSIR supports cast from bool to signed/unsigned integer types and floating point types,
              * where bool is treated as a 1-bit integer and a signed 'true' value converts to -1. */
             spirv_compiler_emit_bool_cast(compiler, instruction);
-            return;
+            return VKD3D_OK;
         }
     }
     else
@@ -6694,7 +6694,7 @@ static void spirv_compiler_emit_alu_instruction(struct spirv_compiler *compiler,
         ERR("Unexpected instruction %#x.\n", instruction->handler_idx);
         spirv_compiler_error(compiler, VKD3D_SHADER_ERROR_SPV_INVALID_HANDLER,
                 "Encountered invalid/unhandled instruction handler %#x.", instruction->handler_idx);
-        return;
+        return VKD3D_ERROR_INVALID_SHADER;
     }

assert(instruction->dst_count == 1);
@@ -6726,6 +6726,7 @@ static void spirv_compiler_emit_alu_instruction(struct spirv_compiler *compiler,
         vkd3d_spirv_build_op_decorate(builder, val_id, SpvDecorationNoContraction, NULL, 0);

spirv_compiler_emit_store_dst(compiler, dst, val_id);
+    return VKD3D_OK;
 }

static enum GLSLstd450 spirv_compiler_map_ext_glsl_instruction(
@@ -9506,7 +9507,7 @@ static int spirv_compiler_handle_instruction(struct spirv_compiler *compiler,
         case VKD3DSIH_UTOF:
         case VKD3DSIH_UTOU:
         case VKD3DSIH_XOR:
-            spirv_compiler_emit_alu_instruction(compiler, instruction);
+            ret = spirv_compiler_emit_alu_instruction(compiler, instruction);
             break;
         case VKD3DSIH_DFMA:
         case VKD3DSIH_DMAX:
@@ -9727,6 +9728,9 @@ static int spirv_compiler_handle_instruction(struct spirv_compiler *compiler,
             break;
         default:
             FIXME("Unhandled instruction %#x.\n", instruction->handler_idx);
+            spirv_compiler_error(compiler, VKD3D_SHADER_ERROR_SPV_INVALID_HANDLER,
+                    "Encountered invalid/unhandled instruction handler %#x.", instruction->handler_idx);
+            return VKD3D_ERROR_INVALID_SHADER;
     }

return ret;