[PATCH v7 06/10] vkd3d-shader/hlsl: Use hlsl_ir_index for array and record access.

[Francisco Casas]

From: Francisco Casas

From this point on, it is no longer true that only hlsl_ir_loads can
return objects, because an object can also come from chain of
hlsl_ir_indexes that ends in an hlsl_ir_load.

The lower_index_loads pass takes care of lowering all hlsl_ir_indexes
into hlsl_ir_loads.

For this reason, hlsl_resource_load_params now expects both the resource
as the sampler to be just an hlsl_ir_node pointer instead of a pointer
to a more specific hlsl_ir_load.
---
 libs/vkd3d-shader/hlsl.c               |  88 ++++++++++++++++-
 libs/vkd3d-shader/hlsl.h               |   5 +-
 libs/vkd3d-shader/hlsl.y               | 128 ++++++++-----------------
 libs/vkd3d-shader/hlsl_codegen.c       |  77 +++++++++++++++
 tests/hlsl-matrix-indexing.shader_test |   4 +-
 tests/side-effects.shader_test         |   2 +-
 6 files changed, 205 insertions(+), 99 deletions(-)

diff --git a/libs/vkd3d-shader/hlsl.c b/libs/vkd3d-shader/hlsl.c
index c5050c23..cf6c13d1 100644
--- a/libs/vkd3d-shader/hlsl.c
+++ b/libs/vkd3d-shader/hlsl.c
@@ -437,6 +437,71 @@ static bool init_deref(struct hlsl_ctx *ctx, struct hlsl_deref *deref, struct hl
     return true;
 }

+bool hlsl_init_deref_from_index_chain(struct hlsl_ctx *ctx, struct hlsl_deref *deref, struct hlsl_ir_node *chain)
+{
+    struct hlsl_ir_index *index;
+    struct hlsl_ir_load *load;
+    unsigned int chain_len, i;
+    struct hlsl_ir_node *ptr;
+
+    deref->path = NULL;
+    deref->path_len = 0;
+    deref->offset.node = NULL;
+
+    assert(chain);
+    if (chain->type == HLSL_IR_INDEX)
+        assert(!hlsl_index_is_noncontiguous(hlsl_ir_index(chain)));
+
+    /* Find the length of the index chain */
+    chain_len = 0;
+    ptr = chain;
+    while (ptr->type == HLSL_IR_INDEX)
+    {
+        index = hlsl_ir_index(ptr);
+
+        chain_len++;
+        ptr = index->val.node;
+    }
+
+    if (ptr->type != HLSL_IR_LOAD)
+    {
+        hlsl_error(ctx, &chain->loc, VKD3D_SHADER_ERROR_HLSL_INVALID_LVALUE, "Invalid l-value.");
+        return false;
+    }
+    load = hlsl_ir_load(ptr);
+
+    if (!init_deref(ctx, deref, load->src.var, load->src.path_len + chain_len))
+        return false;
+
+    for (i = 0; i < load->src.path_len; ++i)
+        hlsl_src_from_node(&deref->path[i], load->src.path[i].node);
+
+    chain_len = 0;
+    ptr = chain;
+    while (ptr->type == HLSL_IR_INDEX)
+    {
+        unsigned int p = deref->path_len - 1 - chain_len;
+
+        index = hlsl_ir_index(ptr);
+        if (hlsl_index_is_noncontiguous(index))
+        {
+            hlsl_src_from_node(&deref->path[p], deref->path[p + 1].node);
+            hlsl_src_remove(&deref->path[p + 1]);
+            hlsl_src_from_node(&deref->path[p + 1], index->idx.node);
+        }
+        else
+        {
+            hlsl_src_from_node(&deref->path[p], index->idx.node);
+        }
+
+        chain_len++;
+        ptr = index->val.node;
+    }
+    assert(deref->path_len == load->src.path_len + chain_len);
+
+    return true;
+}
+
 struct hlsl_type *hlsl_deref_get_type(struct hlsl_ctx *ctx, const struct hlsl_deref *deref)
 {
     struct hlsl_type *type;
@@ -1231,17 +1296,23 @@ struct hlsl_ir_resource_load *hlsl_new_resource_load(struct hlsl_ctx *ctx,
         return NULL;
     init_node(&load->node, HLSL_IR_RESOURCE_LOAD, params->format, loc);
     load->load_type = params->type;
-    if (!hlsl_copy_deref(ctx, &load->resource, &params->resource))
+
+    if (!hlsl_init_deref_from_index_chain(ctx, &load->resource, params->resource))
     {
         vkd3d_free(load);
         return NULL;
     }
-    if (!hlsl_copy_deref(ctx, &load->sampler, &params->sampler))
+
+    if (params->sampler)
     {
-        hlsl_cleanup_deref(&load->resource);
-        vkd3d_free(load);
-        return NULL;
+        if (!hlsl_init_deref_from_index_chain(ctx, &load->sampler, params->sampler))
+        {
+            hlsl_cleanup_deref(&load->resource);
+            vkd3d_free(load);
+            return NULL;
+        }
     }
+
     hlsl_src_from_node(&load->coords, params->coords);
     hlsl_src_from_node(&load->texel_offset, params->texel_offset);
     hlsl_src_from_node(&load->lod, params->lod);
@@ -1280,6 +1351,13 @@ struct hlsl_ir_swizzle *hlsl_new_swizzle(struct hlsl_ctx *ctx, DWORD s, unsigned
     return swizzle;
 }

+bool hlsl_index_is_noncontiguous(struct hlsl_ir_index *index)
+{
+    struct hlsl_type *type = index->val.node->data_type;
+
+    return type->class == HLSL_CLASS_MATRIX && !hlsl_type_is_row_major(type);
+}
+
 struct hlsl_ir_node *hlsl_new_index(struct hlsl_ctx *ctx, struct hlsl_ir_node *val,
         struct hlsl_ir_node *idx, const struct vkd3d_shader_location *loc)
 {
diff --git a/libs/vkd3d-shader/hlsl.h b/libs/vkd3d-shader/hlsl.h
index 6ccf7d7e..e7f09c68 100644
--- a/libs/vkd3d-shader/hlsl.h
+++ b/libs/vkd3d-shader/hlsl.h
@@ -794,7 +794,7 @@ struct hlsl_resource_load_params
 {
     struct hlsl_type *format;
     enum hlsl_resource_load_type type;
-    struct hlsl_deref resource, sampler;
+    struct hlsl_ir_node *resource, *sampler;
     struct hlsl_ir_node *coords, *lod, *texel_offset;
 };

@@ -1006,6 +1006,7 @@ void hlsl_dump_function(struct hlsl_ctx *ctx, const struct hlsl_ir_function_decl
 int hlsl_emit_bytecode(struct hlsl_ctx *ctx, struct hlsl_ir_function_decl *entry_func,
         enum vkd3d_shader_target_type target_type, struct vkd3d_shader_code *out);

+bool hlsl_init_deref_from_index_chain(struct hlsl_ctx *ctx, struct hlsl_deref *deref, struct hlsl_ir_node *chain);
 bool hlsl_copy_deref(struct hlsl_ctx *ctx, struct hlsl_deref *deref, const struct hlsl_deref *other);

void hlsl_cleanup_deref(struct hlsl_deref *deref);
@@ -1070,6 +1071,8 @@ struct hlsl_ir_store *hlsl_new_store_index(struct hlsl_ctx *ctx, const struct hl
 struct hlsl_ir_store *hlsl_new_store_component(struct hlsl_ctx *ctx, struct hlsl_block *block,
         const struct hlsl_deref *lhs, unsigned int comp, struct hlsl_ir_node *rhs);

+bool hlsl_index_is_noncontiguous(struct hlsl_ir_index *index);
+
 struct hlsl_ir_node *hlsl_new_index(struct hlsl_ctx *ctx, struct hlsl_ir_node *val,
         struct hlsl_ir_node *idx, const struct vkd3d_shader_location *loc);
 struct hlsl_ir_loop *hlsl_new_loop(struct hlsl_ctx *ctx, struct vkd3d_shader_location loc);
diff --git a/libs/vkd3d-shader/hlsl.y b/libs/vkd3d-shader/hlsl.y
index 743205e3..64191f5a 100644
--- a/libs/vkd3d-shader/hlsl.y
+++ b/libs/vkd3d-shader/hlsl.y
@@ -685,9 +685,10 @@ static struct hlsl_ir_load *add_load_component(struct hlsl_ctx *ctx, struct list
     return load;
 }

-static bool add_record_load(struct hlsl_ctx *ctx, struct list *instrs, struct hlsl_ir_node *record,
+static bool add_record_access(struct hlsl_ctx *ctx, struct list *instrs, struct hlsl_ir_node *record,
         unsigned int idx, const struct vkd3d_shader_location loc)
 {
+    struct hlsl_ir_node *index;
     struct hlsl_ir_constant *c;

assert(idx < record->data_type->e.record.field_count);
@@ -696,60 +697,17 @@ static bool add_record_load(struct hlsl_ctx *ctx, struct list *instrs, struct hl
         return false;
     list_add_tail(instrs, &c->node.entry);

-    return !!add_load_index(ctx, instrs, record, &c->node, &loc);
+    if (!(index = hlsl_new_index(ctx, record, &c->node, &loc)))
+        return false;
+    list_add_tail(instrs, &index->entry);
+
+    return true;
 }

static struct hlsl_ir_node *add_binary_arithmetic_expr(struct hlsl_ctx *ctx, struct list *instrs,
         enum hlsl_ir_expr_op op, struct hlsl_ir_node *arg1, struct hlsl_ir_node *arg2,
         const struct vkd3d_shader_location *loc);

-static bool add_matrix_index(struct hlsl_ctx *ctx, struct list *instrs,
-        struct hlsl_ir_node *matrix, struct hlsl_ir_node *index, const struct vkd3d_shader_location *loc)
-{
-    struct hlsl_type *mat_type = matrix->data_type, *ret_type;
-    struct hlsl_deref var_deref;
-    struct hlsl_ir_load *load;
-    struct hlsl_ir_var *var;
-    unsigned int i;
-
-    if (hlsl_type_is_row_major(mat_type))
-        return add_load_index(ctx, instrs, matrix, index, loc);
-
-    ret_type = hlsl_get_vector_type(ctx, mat_type->base_type, mat_type->dimx);
-
-    if (!(var = hlsl_new_synthetic_var(ctx, "index", ret_type, loc)))
-        return false;
-    hlsl_init_simple_deref_from_var(&var_deref, var);
-
-    for (i = 0; i < mat_type->dimx; ++i)
-    {
-        struct hlsl_ir_load *column, *value;
-        struct hlsl_ir_store *store;
-        struct hlsl_ir_constant *c;
-        struct hlsl_block block;
-
-        if (!(c = hlsl_new_uint_constant(ctx, i, loc)))
-            return false;
-        list_add_tail(instrs, &c->node.entry);
-
-        if (!(column = add_load_index(ctx, instrs, matrix, &c->node, loc)))
-            return false;
-
-        if (!(value = add_load_index(ctx, instrs, &column->node, index, loc)))
-            return false;
-
-        if (!(store = hlsl_new_store_component(ctx, &block, &var_deref, i, &value->node)))
-            return false;
-        list_move_tail(instrs, &block.instrs);
-    }
-
-    if (!(load = hlsl_new_var_load(ctx, var, *loc)))
-        return false;
-    list_add_tail(instrs, &load->node.entry);
-
-    return true;
-}
-
 static struct hlsl_ir_node *add_zero_mipmap_level(struct hlsl_ctx *ctx, struct list *instrs,
         struct hlsl_ir_node *index, unsigned int dim_count, const struct vkd3d_shader_location *loc)
 {
@@ -783,10 +741,11 @@ static struct hlsl_ir_node *add_zero_mipmap_level(struct hlsl_ctx *ctx, struct l
     return &coords_load->node;
 }

-static bool add_array_load(struct hlsl_ctx *ctx, struct list *instrs, struct hlsl_ir_node *array,
+static bool add_array_access(struct hlsl_ctx *ctx, struct list *instrs, struct hlsl_ir_node *array,
         struct hlsl_ir_node *index, const struct vkd3d_shader_location *loc)
 {
     const struct hlsl_type *expr_type = array->data_type, *index_type = index->data_type;
+    struct hlsl_ir_node *return_index;
     struct hlsl_ir_expr *cast;

if (expr_type->class == HLSL_CLASS_OBJECT
@@ -795,8 +754,6 @@ static bool add_array_load(struct hlsl_ctx *ctx, struct list *instrs, struct hls
     {
         struct hlsl_resource_load_params load_params = {.type = HLSL_RESOURCE_LOAD};
         unsigned int dim_count = hlsl_sampler_dim_count(expr_type->sampler_dim);
-        /* Only HLSL_IR_LOAD can return an object. */
-        struct hlsl_ir_load *object_load = hlsl_ir_load(array);
         struct hlsl_ir_resource_load *resource_load;

if (index_type->class > HLSL_CLASS_VECTOR || index_type->dimx != dim_count)
@@ -818,7 +775,7 @@ static bool add_array_load(struct hlsl_ctx *ctx, struct list *instrs, struct hls
             return false;

load_params.format = expr_type->e.resource_format;
-        load_params.resource = object_load->src;
+        load_params.resource = array;
         load_params.coords = index;

if (!(resource_load = hlsl_new_resource_load(ctx, &load_params, loc)))
@@ -838,10 +795,7 @@ static bool add_array_load(struct hlsl_ctx *ctx, struct list *instrs, struct hls
     list_add_tail(instrs, &cast->node.entry);
     index = &cast->node;

-    if (expr_type->class == HLSL_CLASS_MATRIX)
-        return add_matrix_index(ctx, instrs, array, index, loc);
-
-    if (expr_type->class != HLSL_CLASS_ARRAY && expr_type->class != HLSL_CLASS_VECTOR)
+    if (expr_type->class != HLSL_CLASS_ARRAY && expr_type->class != HLSL_CLASS_VECTOR && expr_type->class != HLSL_CLASS_MATRIX)
     {
         if (expr_type->class == HLSL_CLASS_SCALAR)
             hlsl_error(ctx, loc, VKD3D_SHADER_ERROR_HLSL_INVALID_INDEX, "Scalar expressions cannot be array-indexed.");
@@ -850,8 +804,9 @@ static bool add_array_load(struct hlsl_ctx *ctx, struct list *instrs, struct hls
         return false;
     }

-    if (!add_load_index(ctx, instrs, array, index, loc))
+    if (!(return_index = hlsl_new_index(ctx, array, index, loc)))
         return false;
+    list_add_tail(instrs, &return_index->entry);

return true;
 }
@@ -1784,7 +1739,7 @@ static struct hlsl_ir_node *add_assignment(struct hlsl_ctx *ctx, struct list *in
     if (!(rhs = add_implicit_conversion(ctx, instrs, rhs, lhs_type, &rhs->loc)))
         return NULL;

-    while (lhs->type != HLSL_IR_LOAD && lhs->type != HLSL_IR_RESOURCE_LOAD)
+    while (lhs->type != HLSL_IR_LOAD && lhs->type != HLSL_IR_RESOURCE_LOAD && lhs->type != HLSL_IR_INDEX)
     {
         if (lhs->type == HLSL_IR_EXPR && hlsl_ir_expr(lhs)->op == HLSL_OP1_CAST)
         {
@@ -1857,13 +1812,26 @@ static struct hlsl_ir_node *add_assignment(struct hlsl_ctx *ctx, struct list *in
             return NULL;
         list_add_tail(instrs, &store->node.entry);
     }
+    else if (lhs->type == HLSL_IR_INDEX && hlsl_index_is_noncontiguous(hlsl_ir_index(lhs)))
+    {
+        hlsl_fixme(ctx, &lhs->loc, "Column-major matrix index store.");
+        return NULL;
+    }
     else
     {
         struct hlsl_ir_store *store;
+        struct hlsl_deref deref;

-        if (!(store = hlsl_new_store_index(ctx, &hlsl_ir_load(lhs)->src, NULL, rhs, writemask, &rhs->loc)))
+        if (!hlsl_init_deref_from_index_chain(ctx, &deref, lhs))
+            return NULL;
+
+        if (!(store = hlsl_new_store_index(ctx, &deref, NULL, rhs, writemask, &rhs->loc)))
+        {
+            hlsl_cleanup_deref(&deref);
             return NULL;
+        }
         list_add_tail(instrs, &store->node.entry);
+        hlsl_cleanup_deref(&deref);
     }

/* Don't use the instruction itself as a source, as this makes structure
@@ -3156,7 +3124,6 @@ static bool intrinsic_tex(struct hlsl_ctx *ctx, const struct parse_initializer *
     struct hlsl_resource_load_params load_params = {.type = HLSL_RESOURCE_SAMPLE};
     const struct hlsl_type *sampler_type;
     struct hlsl_ir_resource_load *load;
-    struct hlsl_ir_load *sampler_load;
     struct hlsl_ir_node *coords;

if (params->args_count != 2 && params->args_count != 4)
@@ -3185,10 +3152,7 @@ static bool intrinsic_tex(struct hlsl_ctx *ctx, const struct parse_initializer *
     }
     else
     {
-        /* Only HLSL_IR_LOAD can return an object. */
-        sampler_load = hlsl_ir_load(params->args[0]);
-
-        load_params.resource = sampler_load->src;
+        load_params.resource = params->args[0];
     }

if (!(coords = add_implicit_conversion(ctx, params->instrs, params->args[1],
@@ -3524,7 +3488,6 @@ static bool add_method_call(struct hlsl_ctx *ctx, struct list *instrs, struct hl
         const char *name, const struct parse_initializer *params, const struct vkd3d_shader_location *loc)
 {
     const struct hlsl_type *object_type = object->data_type;
-    struct hlsl_ir_load *object_load;

if (object_type->class != HLSL_CLASS_OBJECT || object_type->base_type != HLSL_TYPE_TEXTURE
             || object_type->sampler_dim == HLSL_SAMPLER_DIM_GENERIC)
@@ -3538,9 +3501,6 @@ static bool add_method_call(struct hlsl_ctx *ctx, struct list *instrs, struct hl
         return false;
     }

-    /* Only HLSL_IR_LOAD can return an object. */
-    object_load = hlsl_ir_load(object);
-
     if (!strcmp(name, "Load")
             && object_type->sampler_dim != HLSL_SAMPLER_DIM_CUBE
             && object_type->sampler_dim != HLSL_SAMPLER_DIM_CUBEARRAY)
@@ -3584,7 +3544,7 @@ static bool add_method_call(struct hlsl_ctx *ctx, struct list *instrs, struct hl
             return false;

load_params.format = object_type->e.resource_format;
-        load_params.resource = object_load->src;
+        load_params.resource = object;

if (!(load = hlsl_new_resource_load(ctx, &load_params, loc)))
             return false;
@@ -3600,7 +3560,6 @@ static bool add_method_call(struct hlsl_ctx *ctx, struct list *instrs, struct hl
         struct hlsl_resource_load_params load_params = {.type = HLSL_RESOURCE_SAMPLE};
         const struct hlsl_type *sampler_type;
         struct hlsl_ir_resource_load *load;
-        struct hlsl_ir_load *sampler_load;

if (params->args_count < 2 || params->args_count > 4 + !!offset_dim)
         {
@@ -3623,9 +3582,6 @@ static bool add_method_call(struct hlsl_ctx *ctx, struct list *instrs, struct hl
             return false;
         }

-        /* Only HLSL_IR_LOAD can return an object. */
-        sampler_load = hlsl_ir_load(params->args[0]);
-
         if (!(load_params.coords = add_implicit_conversion(ctx, instrs, params->args[1],
                 hlsl_get_vector_type(ctx, HLSL_TYPE_FLOAT, sampler_dim), loc)))
             return false;
@@ -3643,8 +3599,8 @@ static bool add_method_call(struct hlsl_ctx *ctx, struct list *instrs, struct hl
             hlsl_fixme(ctx, loc, "Tiled resource status argument.");

load_params.format = object_type->e.resource_format;
-        load_params.resource = object_load->src;
-        load_params.sampler = sampler_load->src;
+        load_params.resource = object;
+        load_params.sampler = params->args[0];

if (!(load = hlsl_new_resource_load(ctx, &load_params, loc)))
             return false;
@@ -3664,7 +3620,6 @@ static bool add_method_call(struct hlsl_ctx *ctx, struct list *instrs, struct hl
         struct hlsl_resource_load_params load_params = {0};
         const struct hlsl_type *sampler_type;
         struct hlsl_ir_resource_load *load;
-        struct hlsl_ir_load *sampler_load;
         unsigned int read_channel;

if (!strcmp(name, "GatherGreen"))
@@ -3740,16 +3695,13 @@ static bool add_method_call(struct hlsl_ctx *ctx, struct list *instrs, struct hl
             return false;
         }

-        /* Only HLSL_IR_LOAD can return an object. */
-        sampler_load = hlsl_ir_load(params->args[0]);
-
         if (!(load_params.coords = add_implicit_conversion(ctx, instrs, params->args[1],
                 hlsl_get_vector_type(ctx, HLSL_TYPE_FLOAT, sampler_dim), loc)))
             return false;

load_params.format = hlsl_get_vector_type(ctx, object_type->e.resource_format->base_type, 4);
-        load_params.resource = object_load->src;
-        load_params.sampler = sampler_load->src;
+        load_params.resource = object;
+        load_params.sampler = params->args[0];

if (!(load = hlsl_new_resource_load(ctx, &load_params, loc)))
             return false;
@@ -3765,7 +3717,6 @@ static bool add_method_call(struct hlsl_ctx *ctx, struct list *instrs, struct hl
         const unsigned int offset_dim = hlsl_offset_dim_count(object_type->sampler_dim);
         const struct hlsl_type *sampler_type;
         struct hlsl_ir_resource_load *load;
-        struct hlsl_ir_load *sampler_load;

if (params->args_count < 3 || params->args_count > 4 + !!offset_dim)
         {
@@ -3788,9 +3739,6 @@ static bool add_method_call(struct hlsl_ctx *ctx, struct list *instrs, struct hl
             return false;
         }

-        /* Only HLSL_IR_LOAD can return an object. */
-        sampler_load = hlsl_ir_load(params->args[0]);
-
         if (!(load_params.coords = add_implicit_conversion(ctx, instrs, params->args[1],
                 hlsl_get_vector_type(ctx, HLSL_TYPE_FLOAT, sampler_dim), loc)))
             load_params.coords = params->args[1];
@@ -3810,8 +3758,8 @@ static bool add_method_call(struct hlsl_ctx *ctx, struct list *instrs, struct hl
             hlsl_fixme(ctx, loc, "Tiled resource status argument.");

load_params.format = object_type->e.resource_format;
-        load_params.resource = object_load->src;
-        load_params.sampler = sampler_load->src;
+        load_params.resource = object;
+        load_params.sampler = params->args[0];

if (!(load = hlsl_new_resource_load(ctx, &load_params, loc)))
             return false;
@@ -5457,7 +5405,7 @@ postfix_expr:
                 }

field_idx = field - type->e.record.fields;
-                if (!add_record_load(ctx, $1, node, field_idx, @2))
+                if (!add_record_access(ctx, $1, node, field_idx, @2))
                     YYABORT;
                 $$ = $1;
             }
@@ -5486,7 +5434,7 @@ postfix_expr:
             list_move_tail($1, $3);
             vkd3d_free($3);

-            if (!add_array_load(ctx, $1, array, index, &@2))
+            if (!add_array_access(ctx, $1, array, index, &@2))
             {
                 destroy_instr_list($1);
                 YYABORT;
diff --git a/libs/vkd3d-shader/hlsl_codegen.c b/libs/vkd3d-shader/hlsl_codegen.c
index accc7492..a5568be0 100644
--- a/libs/vkd3d-shader/hlsl_codegen.c
+++ b/libs/vkd3d-shader/hlsl_codegen.c
@@ -731,6 +731,81 @@ static bool lower_calls(struct hlsl_ctx *ctx, struct hlsl_ir_node *instr, void *
     return true;
 }

+/* hlsl_ir_index nodes are a parse-time construct used to represent array indexing and struct
+ * record access before knowing if they will be used in the lhs of an assignment --in which case
+ * they are lowered into a deref-- or as the load of an element within a larger value.
+ * For the latter case, this pass takes care of lowering hlsl_ir_indexes into individual
+ * hlsl_ir_loads. */
+static bool lower_index_loads(struct hlsl_ctx *ctx, struct hlsl_ir_node *instr, void *context)
+{
+    struct hlsl_deref var_deref;
+    struct hlsl_ir_index *index;
+    struct hlsl_ir_store *store;
+    struct hlsl_ir_load *load;
+    struct hlsl_ir_node *val;
+    struct hlsl_ir_var *var;
+
+    if (instr->type != HLSL_IR_INDEX)
+        return false;
+    index = hlsl_ir_index(instr);
+    val = index->val.node;
+
+    if (!(var = hlsl_new_synthetic_var(ctx, "index-val", val->data_type, &instr->loc)))
+        return false;
+    hlsl_init_simple_deref_from_var(&var_deref, var);
+
+    if (!(store = hlsl_new_simple_store(ctx, var, val)))
+        return false;
+    list_add_before(&instr->entry, &store->node.entry);
+
+    if (hlsl_index_is_noncontiguous(index))
+    {
+        struct hlsl_ir_node *mat = index->val.node;
+        struct hlsl_deref row_deref;
+        unsigned int i;
+
+        assert(!hlsl_type_is_row_major(mat->data_type));
+
+        if (!(var = hlsl_new_synthetic_var(ctx, "row", instr->data_type, &instr->loc)))
+            return false;
+        hlsl_init_simple_deref_from_var(&row_deref, var);
+
+        for (i = 0; i < mat->data_type->dimx; ++i)
+        {
+            struct hlsl_ir_constant *c;
+
+            if (!(c = hlsl_new_uint_constant(ctx, i, &instr->loc)))
+                return false;
+            list_add_before(&instr->entry, &c->node.entry);
+
+            if (!(load = hlsl_new_load_index(ctx, &var_deref, &c->node, &instr->loc)))
+                return false;
+            list_add_before(&instr->entry, &load->node.entry);
+
+            if (!(load = hlsl_new_load_index(ctx, &load->src, index->idx.node, &instr->loc)))
+                return false;
+            list_add_before(&instr->entry, &load->node.entry);
+
+            if (!(store = hlsl_new_store_index(ctx, &row_deref, &c->node, &load->node, 0, &instr->loc)))
+                return false;
+            list_add_before(&instr->entry, &store->node.entry);
+        }
+
+        if (!(load = hlsl_new_var_load(ctx, var, instr->loc)))
+            return false;
+        list_add_before(&instr->entry, &load->node.entry);
+        hlsl_replace_node(instr, &load->node);
+    }
+    else
+    {
+        if (!(load = hlsl_new_load_index(ctx, &var_deref, index->idx.node, &instr->loc)))
+            return false;
+        list_add_before(&instr->entry, &load->node.entry);
+        hlsl_replace_node(instr, &load->node);
+    }
+    return true;
+}
+
 /* Lower casts from vec1 to vecN to swizzles. */
 static bool lower_broadcasts(struct hlsl_ctx *ctx, struct hlsl_ir_node *instr, void *context)
 {
@@ -3375,6 +3450,8 @@ int hlsl_emit_bytecode(struct hlsl_ctx *ctx, struct hlsl_ir_function_decl *entry

while (transform_ir(ctx, lower_calls, body, NULL));

+    transform_ir(ctx, lower_index_loads, body, NULL);
+
     LIST_FOR_EACH_ENTRY(var, &ctx->globals->vars, struct hlsl_ir_var, scope_entry)
     {
         if (var->storage_modifiers & HLSL_STORAGE_UNIFORM)
diff --git a/tests/hlsl-matrix-indexing.shader_test b/tests/hlsl-matrix-indexing.shader_test
index 74152d17..e4731639 100644
--- a/tests/hlsl-matrix-indexing.shader_test
+++ b/tests/hlsl-matrix-indexing.shader_test
@@ -78,7 +78,7 @@ draw quad
 probe all rgba (1.0, 5.0, 7.0, 12.0)

-[pixel shader]
+[pixel shader todo]
 float4 main() : SV_TARGET
 {
     float3x2 m = {1, 2, 3, 4, 5, 6};
@@ -89,7 +89,7 @@ float4 main() : SV_TARGET
 }

[test]
-draw quad
+todo draw quad
 todo probe all rgba (30.0, 40.0, 5.0, 6.0)

diff --git a/tests/side-effects.shader_test b/tests/side-effects.shader_test
index b64a9dd4..f631db64 100644
--- a/tests/side-effects.shader_test
+++ b/tests/side-effects.shader_test
@@ -24,7 +24,7 @@ float4 main() : sv_target

[test]
 draw quad
-todo probe all rgba (11.0, 11.0, 11.0, 11.0)
+probe all rgba (11.0, 11.0, 11.0, 11.0)

[pixel shader]