[PATCH v2 6/7] vkd3d-shader/ir: Transform clip/cull inputs into an array.

[Conor McCarthy]

From: Conor McCarthy

Up to eight clip/cull values are supported, and the DXBC pattern of
spreading these across two vector signature elements is a hacky solution,
so converting these into an array is best for VSIR. SPIR-V requires these
to be an array.
---
 libs/vkd3d-shader/ir.c                   | 312 ++++++++++++++++++++++-
 libs/vkd3d-shader/spirv.c                |   7 +-
 libs/vkd3d-shader/vkd3d_shader_private.h |   2 +
 3 files changed, 317 insertions(+), 4 deletions(-)

diff --git a/libs/vkd3d-shader/ir.c b/libs/vkd3d-shader/ir.c
index 3f8a6f192..192511b67 100644
--- a/libs/vkd3d-shader/ir.c
+++ b/libs/vkd3d-shader/ir.c
@@ -1255,6 +1255,313 @@ static enum vkd3d_result shader_normalise_io_registers(struct vkd3d_shader_parse
     return VKD3D_OK;
 }

+struct clip_cull_normaliser_signature
+{
+    struct shader_signature *s;
+    unsigned int base_element_idx;
+    unsigned int offsets[MAX_REG_OUTPUT];
+};
+
+static void shader_signature_transform_clip_cull(struct clip_cull_normaliser_signature *signature,
+        enum vkd3d_shader_sysval_semantic target_sysval, struct vkd3d_shader_parser *parser)
+{
+    unsigned int i, j, base, array_size, reg_mask;
+    struct shader_signature *s = signature->s;
+    struct signature_element *e;
+
+    /* Up to two vec4 clip/cull elements are allowed. Merge these into an array,
+     * and track the location and array offset of the second one. */
+    for (i = 0, array_size = 0, reg_mask = 0, base = 0; i < s->element_count; ++i)
+    {
+        e = &s->elements[i];
+
+        if (e->sysval_semantic != target_sysval)
+        {
+            reg_mask |= ((1u << e->register_count) - 1) << e->register_index;
+            continue;
+        }
+
+        if (!array_size)
+            base = i;
+
+        signature->offsets[i] = array_size;
+        array_size += vsir_write_mask_component_count(e->mask);
+    }
+
+    if (!array_size)
+        return;
+
+    signature->base_element_idx = base;
+    e = &s->elements[base];
+
+    /* Compile a map of used registers. */
+    for (i = base, j = ((1u << array_size) - 1) << base; i < MAX_REG_OUTPUT; ++i, j <<= 1)
+    {
+        if (!(reg_mask & j))
+            break;
+    }
+
+    /* Signature locations are not used for sysvals, so if it proves necessary, it should be
+     * ok to raise MAX_REG_OUTPUT to 40 and validate that non-sysval register indices are < 32. */
+    if (array_size > MAX_REG_OUTPUT - i)
+    {
+        FIXME("Too many registers; array size %u.\n", array_size);
+        vkd3d_shader_parser_error(parser, VKD3D_SHADER_ERROR_VSIR_UNSUPPORTED_SIGNATURE,
+                "Clip or cull array size %u exceeds limit of 32 registers.", array_size);
+        return;
+    }
+
+    e->register_index = i;
+    e->register_count = array_size;
+    e->mask = 1;
+    e->used_mask = 1;
+
+    /* Delete the second clip/cull element. It will be remapped to the first by checking the offset table. */
+    j = base + 1;
+    for (i = j; i < s->element_count; ++i)
+    {
+        e = &s->elements[i];
+
+        if (e->sysval_semantic != target_sysval)
+            s->elements[j++] = *e;
+    }
+    s->element_count = j;
+}
+
+struct clip_cull_normaliser
+{
+    struct vkd3d_shader_parser *parser;
+
+    struct vkd3d_shader_location location;
+    enum vkd3d_result result;
+
+    struct vkd3d_shader_instruction *instructions;
+    size_t instruction_capacity;
+    size_t instruction_count;
+
+    struct clip_cull_normaliser_signature input_signature;
+
+    unsigned int temp_idx;
+    unsigned int temp_count;
+};
+
+static struct vkd3d_shader_instruction *clip_cull_normaliser_require_space(struct clip_cull_normaliser *normaliser,
+        size_t count)
+{
+    if (!vkd3d_array_reserve((void **)&normaliser->instructions, &normaliser->instruction_capacity,
+            normaliser->instruction_count + count, sizeof(*normaliser->instructions)))
+    {
+        ERR("Failed to allocate instructions.\n");
+        normaliser->result = VKD3D_ERROR_OUT_OF_MEMORY;
+        return NULL;
+    }
+    return &normaliser->instructions[normaliser->instruction_count];
+}
+
+static struct vkd3d_shader_dst_param *clip_cull_normaliser_emit_mov(struct clip_cull_normaliser *normaliser,
+        struct vkd3d_shader_src_param *src_param)
+{
+    struct vkd3d_shader_instruction *ins;
+
+    if (!(ins = clip_cull_normaliser_require_space(normaliser, 1)))
+        return NULL;
+    vsir_instruction_init(ins, &normaliser->location, VKD3DSIH_MOV);
+
+    ins->src = src_param;
+    ins->src_count = 1;
+
+    if (!(ins->dst = shader_parser_get_dst_params(normaliser->parser, 1)))
+    {
+        ERR("Failed to allocate instructions.\n");
+        normaliser->result = VKD3D_ERROR_OUT_OF_MEMORY;
+        return NULL;
+    }
+    ins->dst_count = 1;
+
+    ++normaliser->instruction_count;
+    return ins->dst;
+}
+
+static void clip_cull_normaliser_copy_instruction(struct clip_cull_normaliser *normaliser,
+        struct vkd3d_shader_instruction *ins)
+{
+    struct vkd3d_shader_instruction *dst_ins;
+
+    if (!(dst_ins = clip_cull_normaliser_require_space(normaliser, 1)))
+        return;
+    *dst_ins = *ins;
+    ++normaliser->instruction_count;
+}
+
+static void shader_src_param_clip_cull_normalise(struct vkd3d_shader_src_param *src_param,
+        unsigned int write_mask, struct clip_cull_normaliser *normaliser)
+{
+    struct vkd3d_shader_parser *parser = normaliser->parser;
+    unsigned int i, element_idx, component_idx, array_offset;
+    const struct clip_cull_normaliser_signature *signature;
+    struct vkd3d_shader_register *reg = &src_param->reg;
+    struct vkd3d_shader_dst_param *dst_param;
+    struct vkd3d_shader_src_param *mov_src;
+    const struct signature_element *e;
+    const struct shader_signature *s;
+
+    switch (reg->type)
+    {
+        case VKD3DSPR_INPUT:
+            if (parser->shader_version.type == VKD3D_SHADER_TYPE_DOMAIN)
+                return;
+            signature = &normaliser->input_signature;
+            break;
+        default:
+            return;
+    }
+
+    s = signature->s;
+    element_idx = reg->idx[reg->idx_count - 1].offset;
+    if ((array_offset = signature->offsets[element_idx]))
+    {
+        element_idx = signature->base_element_idx;
+        reg->idx[reg->idx_count - 1].offset = element_idx;
+    }
+    e = &s->elements[element_idx];
+
+    if (e->sysval_semantic != VKD3D_SHADER_SV_CLIP_DISTANCE && e->sysval_semantic != VKD3D_SHADER_SV_CULL_DISTANCE)
+        return;
+
+    if (!write_mask)
+    {
+        WARN("Unexpected zero write mask.\n");
+        vkd3d_shader_parser_warning(parser, VKD3D_SHADER_ERROR_VSIR_INVALID_WRITE_MASK,
+                "Write mask for a clip/cull load is zero.");
+        normaliser->result = VKD3D_ERROR_INVALID_SHADER;
+        return;
+    }
+
+    memmove(&reg->idx[1], &reg->idx[0], reg->idx_count * sizeof(reg->idx[0]));
+    memset(&reg->idx[0], 0, sizeof(reg->idx[0]));
+    ++reg->idx_count;
+
+    if (vsir_write_mask_component_count(write_mask) == 1)
+    {
+        reg->idx[0].offset = array_offset + vsir_swizzle_get_component(src_param->swizzle,
+                vsir_write_mask_get_component_idx(write_mask));
+        return;
+    }
+
+    for (i = 0; i < VKD3D_VEC4_SIZE; ++i)
+    {
+        if (!(write_mask & (1u << i)))
+            continue;
+
+        /* For each component, emit a MOV from the clip/cull array to a temp. */
+
+        if (!(mov_src = shader_parser_get_src_params(parser, 1)))
+        {
+            ERR("Failed to allocate instructions.\n");
+            normaliser->result = VKD3D_ERROR_OUT_OF_MEMORY;
+            return;
+        }
+        mov_src->reg = *reg;
+        mov_src->reg.idx[0].offset = array_offset + i;
+        component_idx = vsir_swizzle_get_component(src_param->swizzle, i);
+        mov_src->swizzle = vkd3d_shader_create_swizzle(component_idx, component_idx, component_idx, component_idx);
+        mov_src->modifiers = 0;
+
+        if (!(dst_param = clip_cull_normaliser_emit_mov(normaliser, mov_src)))
+            return;
+        vsir_register_init(&dst_param->reg, VKD3DSPR_TEMP, reg->data_type, 1);
+        dst_param->reg.dimension = reg->dimension;
+        dst_param->reg.idx[0].offset = normaliser->temp_idx;
+        normaliser->temp_count = 1;
+        dst_param->write_mask = 1u << i;
+        dst_param->modifiers = 0;
+        dst_param->shift = 0;
+    }
+
+    /* Substitute the temp for the vector clip/cull source. */
+    vsir_register_init(reg, VKD3DSPR_TEMP, reg->data_type, 1);
+    reg->dimension = VSIR_DIMENSION_VEC4;
+    reg->idx[0].offset = normaliser->temp_idx;
+}
+
+static void shader_instruction_normalise_clip_cull_params(struct vkd3d_shader_instruction *ins,
+        struct clip_cull_normaliser *normaliser)
+{
+    unsigned int i, write_mask;
+
+    normaliser->location = ins->location;
+
+    if (shader_instruction_is_dcl(ins))
+    {
+        clip_cull_normaliser_copy_instruction(normaliser, ins);
+        return;
+    }
+
+    for (i = 0, write_mask = 0; i < ins->dst_count; ++i)
+        if (ins->dst[i].reg.type != VKD3DSPR_NULL)
+            write_mask |= ins->dst[i].write_mask;
+    for (i = 0; i < ins->src_count; ++i)
+        shader_src_param_clip_cull_normalise(&ins->src[i], write_mask, normaliser);
+
+    clip_cull_normaliser_copy_instruction(normaliser, ins);
+}
+
+static enum vkd3d_result shader_normalise_clip_cull(struct vkd3d_shader_parser *parser)
+{
+    struct clip_cull_normaliser normaliser = {0};
+    unsigned int i;
+
+    normaliser.parser = parser;
+    normaliser.input_signature.s = &parser->shader_desc.input_signature;
+    normaliser.temp_idx = parser->shader_desc.temp_count;
+
+    if (parser->shader_version.type != VKD3D_SHADER_TYPE_DOMAIN)
+    {
+        shader_signature_transform_clip_cull(&normaliser.input_signature, VKD3D_SHADER_SV_CLIP_DISTANCE, parser);
+        shader_signature_transform_clip_cull(&normaliser.input_signature, VKD3D_SHADER_SV_CULL_DISTANCE, parser);
+    }
+    if (parser->failed)
+        normaliser.result = VKD3D_ERROR_INVALID_SHADER;
+
+    if (clip_cull_normaliser_require_space(&normaliser, parser->instructions.count))
+    {
+        for (i = 0; i < parser->instructions.count; ++i)
+            shader_instruction_normalise_clip_cull_params(&parser->instructions.elements[i], &normaliser);
+    }
+    else
+    {
+        normaliser.result = VKD3D_ERROR_OUT_OF_MEMORY;
+    }
+
+    if (normaliser.result >= 0)
+    {
+        vkd3d_free(parser->instructions.elements);
+        parser->instructions.elements = normaliser.instructions;
+        parser->instructions.capacity = normaliser.instruction_capacity;
+        parser->instructions.count = normaliser.instruction_count;
+
+        if (!normaliser.temp_count)
+            return VKD3D_OK;
+
+        parser->shader_desc.temp_count += normaliser.temp_count;
+        for (i = 0; i < parser->instructions.count; ++i)
+            if (parser->instructions.elements[i].handler_idx == VKD3DSIH_DCL_TEMPS)
+                parser->instructions.elements[i].declaration.count = parser->shader_desc.temp_count;
+
+        return VKD3D_OK;
+    }
+    else
+    {
+        if (normaliser.result == VKD3D_ERROR_OUT_OF_MEMORY)
+        {
+            vkd3d_shader_parser_error(parser, VKD3D_SHADER_ERROR_VSIR_OUT_OF_MEMORY,
+                    "Out of memory allocating clip/cull normalization instructions.\n");
+        }
+        vkd3d_free(normaliser.instructions);
+        return normaliser.result;
+    }
+}
+
 struct flat_constant_def
 {
     enum vkd3d_shader_d3dbc_constant_register set;
@@ -1543,7 +1850,7 @@ enum vkd3d_result vkd3d_shader_normalise(struct vkd3d_shader_parser *parser,
     enum vkd3d_result result = VKD3D_OK;

if (parser->shader_desc.is_dxil)
-        return result;
+        return shader_normalise_clip_cull(parser);

if (parser->shader_version.type != VKD3D_SHADER_TYPE_PIXEL
             && (result = remap_output_signature(parser, compile_info)) < 0)
@@ -1561,6 +1868,9 @@ enum vkd3d_result vkd3d_shader_normalise(struct vkd3d_shader_parser *parser,
     if ((result = shader_normalise_io_registers(parser)) < 0)
         return result;

+    if ((result = shader_normalise_clip_cull(parser)) < 0)
+        return result;
+
     if ((result = instruction_array_normalise_flat_constants(parser)) < 0)
         return result;

diff --git a/libs/vkd3d-shader/spirv.c b/libs/vkd3d-shader/spirv.c
index 23f249ee3..dc3a89382 100644
--- a/libs/vkd3d-shader/spirv.c
+++ b/libs/vkd3d-shader/spirv.c
@@ -4844,7 +4844,7 @@ static uint32_t spirv_compiler_emit_input(struct spirv_compiler *compiler,
     array_sizes[0] = signature_element->register_count;
     array_sizes[1] = (reg_type == VKD3DSPR_PATCHCONST ? 0 : compiler->input_control_point_count);
     if (array_sizes[0] == 1 && !vsir_sysval_semantic_is_tess_factor(signature_element->sysval_semantic)
-            && (!vsir_sysval_semantic_is_clip_cull(signature_element->sysval_semantic) || array_sizes[1]))
+            && (!vsir_sysval_semantic_is_clip_cull(sysval) || array_sizes[1]))
     {
         array_sizes[0] = 0;
     }
@@ -9953,8 +9953,6 @@ static int spirv_compiler_generate_spirv(struct spirv_compiler *compiler,
     enum vkd3d_result result = VKD3D_OK;
     unsigned int i;

-    if (parser->shader_desc.temp_count)
-        spirv_compiler_emit_temps(compiler, parser->shader_desc.temp_count);
     if (parser->shader_desc.ssa_count)
         spirv_compiler_allocate_ssa_register_ids(compiler, parser->shader_desc.ssa_count);

@@ -9966,6 +9964,9 @@ static int spirv_compiler_generate_spirv(struct spirv_compiler *compiler,
     if ((result = vkd3d_shader_normalise(parser, compile_info)) < 0)
         return result;

+    if (parser->shader_desc.temp_count)
+        spirv_compiler_emit_temps(compiler, parser->shader_desc.temp_count);
+
     instructions = parser->instructions;
     memset(&parser->instructions, 0, sizeof(parser->instructions));

diff --git a/libs/vkd3d-shader/vkd3d_shader_private.h b/libs/vkd3d-shader/vkd3d_shader_private.h
index b85441aeb..2ed5c0057 100644
--- a/libs/vkd3d-shader/vkd3d_shader_private.h
+++ b/libs/vkd3d-shader/vkd3d_shader_private.h
@@ -217,6 +217,8 @@ enum vkd3d_shader_error
     VKD3D_SHADER_ERROR_VSIR_INVALID_INDEX               = 9015,
     VKD3D_SHADER_ERROR_VSIR_INVALID_INSTRUCTION_NESTING = 9016,
     VKD3D_SHADER_ERROR_VSIR_INVALID_SSA_USAGE           = 9017,
+    VKD3D_SHADER_ERROR_VSIR_OUT_OF_MEMORY               = 9018,
+    VKD3D_SHADER_ERROR_VSIR_UNSUPPORTED_SIGNATURE       = 9019,

VKD3D_SHADER_WARNING_VSIR_DYNAMIC_DESCRIPTOR_ARRAY  = 9300,
 };