Giovanni Mascellani : vkd3d-shader/hlsl: Correctly fold casts from float.

[Alexandre Julliard]

Module: vkd3d
Branch: master
Commit: 49bbd98a0463b0e67fe1c778f33b60be8b41d547
URL:    https://gitlab.winehq.org/wine/vkd3d/-/commit/49bbd98a0463b0e67fe1c778f33b60...

Author: Giovanni Mascellani 
Date:   Sun Sep 10 22:50:01 2023 +0200

vkd3d-shader/hlsl: Correctly fold casts from float.

I.e., without invoking undefined behavior in the compiler. The rules
are desumed from the the MSDN documentation for ftoi and ftou.

---

libs/vkd3d-shader/hlsl_constant_ops.c | 29 +++++++++++++++++++++++++++--
 tests/hlsl/cast-to-uint.shader_test   |  2 +-
 2 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/libs/vkd3d-shader/hlsl_constant_ops.c b/libs/vkd3d-shader/hlsl_constant_ops.c
index 0e4e3b32..31a2faef 100644
--- a/libs/vkd3d-shader/hlsl_constant_ops.c
+++ b/libs/vkd3d-shader/hlsl_constant_ops.c
@@ -63,6 +63,31 @@ static bool fold_abs(struct hlsl_ctx *ctx, struct hlsl_constant_value *dst,
     return true;
 }

+static uint32_t float_to_uint(float x)
+{
+    if (isnan(x) || x <= 0)
+        return 0;
+
+    if (x >= 4294967296.0f)
+        return UINT32_MAX;
+
+    return x;
+}
+
+static int32_t float_to_int(float x)
+{
+    if (isnan(x))
+        return 0;
+
+    if (x <= -2147483648.0f)
+        return INT32_MIN;
+
+    if (x >= 2147483648.0f)
+        return INT32_MAX;
+
+    return x;
+}
+
 static bool fold_cast(struct hlsl_ctx *ctx, struct hlsl_constant_value *dst,
         const struct hlsl_type *dst_type, const struct hlsl_ir_constant *src)
 {
@@ -86,8 +111,8 @@ static bool fold_cast(struct hlsl_ctx *ctx, struct hlsl_constant_value *dst,
         {
             case HLSL_TYPE_FLOAT:
             case HLSL_TYPE_HALF:
-                u = src->value.u[k].f;
-                i = src->value.u[k].f;
+                u = float_to_uint(src->value.u[k].f);
+                i = float_to_int(src->value.u[k].f);
                 f = src->value.u[k].f;
                 d = src->value.u[k].f;
                 break;
diff --git a/tests/hlsl/cast-to-uint.shader_test b/tests/hlsl/cast-to-uint.shader_test
index 4ffc041a..93862a36 100644
--- a/tests/hlsl/cast-to-uint.shader_test
+++ b/tests/hlsl/cast-to-uint.shader_test
@@ -41,4 +41,4 @@ float4 main() : sv_target

[test]
 draw quad
-todo probe all rgba (0.5, 0.5, 0.5, 0.5)
+probe all rgba (0.5, 0.5, 0.5, 0.5)