Petrichor Park : wined3d: Implement a zero-safe normalize function for FFP.

[Alexandre Julliard]

Module: wine
Branch: master
Commit: b79c4e7de60d63aec257764d20cfe69f9943c81b
URL:    https://gitlab.winehq.org/wine/wine/-/commit/b79c4e7de60d63aec257764d20cfe69...

Author: Petrichor Park 
Date:   Thu Jul 20 09:49:33 2023 -0500

wined3d: Implement a zero-safe normalize function for FFP.

DirectX's FFP normalizes the zero vector to the zero vector, but GLSL
normalizes them to all NaN.

This patch creates ffp_normalize, which normalizes vectors but
has DirectX's behavior on 0 vectors. Further patches in this set switch
other calls from GLSL's normalize to the 0-safe version.

Wine-Bug: https://bugs.winehq.org/show_bug.cgi?id=36564

---

dlls/wined3d/glsl_shader.c | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/dlls/wined3d/glsl_shader.c b/dlls/wined3d/glsl_shader.c
index f7eb611df98..bbfea62f56a 100644
--- a/dlls/wined3d/glsl_shader.c
+++ b/dlls/wined3d/glsl_shader.c
@@ -9090,6 +9090,11 @@ static GLuint shader_glsl_generate_ffp_vertex_shader(struct shader_glsl_priv *pr
         declare_out_varying(gl_info, buffer, FALSE, "float ffp_varying_fogcoord;\n");
     }

+    shader_addline(buffer, "\nvec3 ffp_normalize(vec3 n)\n{\n");
+    shader_addline(buffer, "    float lensq = dot(n, n);\n");
+    shader_addline(buffer, "    return lensq == 0.0 ? n : (n * inversesqrt(lensq));\n");
+    shader_addline(buffer, "}\n");
+
     shader_addline(buffer, "\nvoid main()\n{\n");
     shader_addline(buffer, "float m;\n");
     shader_addline(buffer, "vec3 r;\n");
@@ -9151,7 +9156,7 @@ static GLuint shader_glsl_generate_ffp_vertex_shader(struct shader_glsl_priv *pr
         }

if (settings->normalize)
-            shader_addline(buffer, "normal = normalize(normal);\n");
+            shader_addline(buffer, "normal = ffp_normalize(normal);\n");
     }

shader_glsl_ffp_vertex_lighting(buffer, settings, legacy_lighting);
@@ -9192,11 +9197,11 @@ static GLuint shader_glsl_generate_ffp_vertex_shader(struct shader_glsl_priv *pr

case WINED3DTSS_TCI_CAMERASPACEREFLECTIONVECTOR:
                 shader_addline(buffer, "ffp_varying_texcoord[%u] = ffp_texture_matrix[%u]"
-                        " * vec4(reflect(normalize(ec_pos.xyz), normal), 1.0);\n", i, i);
+                        " * vec4(reflect(ffp_normalize(ec_pos.xyz), normal), 1.0);\n", i, i);
                 break;

case WINED3DTSS_TCI_SPHEREMAP:
-                shader_addline(buffer, "r = reflect(normalize(ec_pos.xyz), normal);\n");
+                shader_addline(buffer, "r = reflect(ffp_normalize(ec_pos.xyz), normal);\n");
                 shader_addline(buffer, "m = 2.0 * length(vec3(r.x, r.y, r.z + 1.0));\n");
                 shader_addline(buffer, "ffp_varying_texcoord[%u] = ffp_texture_matrix[%u]"
                         " * vec4(r.x / m + 0.5, r.y / m + 0.5, 0.0, 1.0);\n", i, i);