[PATCH v2 2/2] oleaut32: Fix VarAnd string conversion length handling

[Maotong Zhang]

From: Maotong Zhang

Determine type (I2, I4, I8) based on BSTR length.

Wine-Bug: https://bugs.winehq.org/show_bug.cgi?id=56280
---
 dlls/oleaut32/variant.c | 109 +++++++++++++++++++++++++++++++++-------
 1 file changed, 91 insertions(+), 18 deletions(-)

diff --git a/dlls/oleaut32/variant.c b/dlls/oleaut32/variant.c
index fbe035a5576..4653e5c3138 100644
--- a/dlls/oleaut32/variant.c
+++ b/dlls/oleaut32/variant.c
@@ -3106,34 +3106,107 @@ HRESULT WINAPI VarAnd(LPVARIANT left, LPVARIANT right, LPVARIANT result)
         V_VT(&varLeft) = VT_I4; /* Don't overflow */
     else
     {
-        DOUBLE d;
+        if (V_VT(&varLeft) == VT_BSTR)
+        {
+            LONGLONG val64;
+            HRESULT tmpres = VarI8FromStr(V_BSTR(&varLeft), LOCALE_USER_DEFAULT, 0, &val64);
+            if (SUCCEEDED(tmpres))
+            {
+                VARIANT tmpvar;
+                VariantInit(&tmpvar);

-        if (V_VT(&varLeft) == VT_BSTR &&
-            FAILED(VarR8FromStr(V_BSTR(&varLeft),
-            LOCALE_USER_DEFAULT, 0, &d)))
-            hres = VariantChangeType(&varLeft,&varLeft,
-            VARIANT_LOCALBOOL, VT_BOOL);
+                if (val64 >= SHRT_MIN && val64 <= SHRT_MAX)
+                {
+                    V_VT(&tmpvar) = VT_I2;
+                    V_I2(&tmpvar) = (SHORT)val64;
+                    resvt = VT_I2;
+                }
+                else if (val64 >= LONG_MIN && val64 <= LONG_MAX)
+                {
+                    V_VT(&tmpvar) = VT_I4;
+                    V_I4(&tmpvar) = (LONG)val64;
+                    resvt = VT_I4;
+                }
+                else
+                {
+                    V_VT(&tmpvar) = VT_I8;
+                    V_I8(&tmpvar) = val64;
+                    resvt = VT_I8;
+                }
+                VariantClear(&varLeft);
+                hres = VariantCopy(&varLeft, &tmpvar);
+                VariantClear(&tmpvar);
+            }
+            else
+            {
+                double d;
+                if (FAILED(VarR8FromStr(V_BSTR(&varLeft), LOCALE_USER_DEFAULT, 0, &d)))
+                    hres = VariantChangeType(&varLeft, &varLeft, VARIANT_LOCALBOOL, VT_BOOL);
+            }
+            if (FAILED(hres)) goto VarAnd_Exit;
+        }
         if (SUCCEEDED(hres) && V_VT(&varLeft) != resvt)
-            hres = VariantChangeType(&varLeft,&varLeft,0,resvt);
+            hres = VariantChangeType(&varLeft, &varLeft, 0, resvt);
         if (FAILED(hres)) goto VarAnd_Exit;
     }

-    if (resvt == VT_I4 && V_VT(&varRight) == VT_UI4)
-        V_VT(&varRight) = VT_I4; /* Don't overflow */
-    else
+    if (V_VT(&varRight) == VT_BSTR)
     {
-        DOUBLE d;
+        LONGLONG val64;
+        HRESULT tmpres = VarI8FromStr(V_BSTR(&varRight), LOCALE_USER_DEFAULT, 0, &val64);
+        if (SUCCEEDED(tmpres))
+        {
+            VARIANT tmpvar;
+            VariantInit(&tmpvar);

-        if (V_VT(&varRight) == VT_BSTR &&
-            FAILED(VarR8FromStr(V_BSTR(&varRight),
-            LOCALE_USER_DEFAULT, 0, &d)))
-            hres = VariantChangeType(&varRight, &varRight,
-                VARIANT_LOCALBOOL, VT_BOOL);
-        if (SUCCEEDED(hres) && V_VT(&varRight) != resvt)
-            hres = VariantChangeType(&varRight, &varRight, 0, resvt);
+            if (val64 >= SHRT_MIN && val64 <= SHRT_MAX)
+            {
+                V_VT(&tmpvar) = VT_I2;
+                V_I2(&tmpvar) = (SHORT)val64;
+                resvt = VT_I2;
+            }
+            else if (val64 >= LONG_MIN && val64 <= LONG_MAX)
+            {
+                V_VT(&tmpvar) = VT_I4;
+                V_I4(&tmpvar) = (LONG)val64;
+                resvt = VT_I4;
+            }
+            else
+            {
+                V_VT(&tmpvar) = VT_I8;
+                V_I8(&tmpvar) = val64;
+                resvt = VT_I8;
+            }
+            VariantClear(&varRight);
+            hres = VariantCopy(&varRight, &tmpvar);
+            VariantClear(&tmpvar);
+        }
+        else
+        {
+            double d;
+            if (FAILED(VarR8FromStr(V_BSTR(&varRight), LOCALE_USER_DEFAULT, 0, &d)))
+                hres = VariantChangeType(&varRight, &varRight, VARIANT_LOCALBOOL, VT_BOOL);
+        }
         if (FAILED(hres)) goto VarAnd_Exit;
     }

+    if (SUCCEEDED(hres) && (V_VT(&varRight) != resvt || V_VT(&varLeft) != resvt))
+    {
+        if(V_VT(&varLeft) != resvt && (V_VT(&varLeft) < V_VT(&varRight)))
+        {
+            hres = VariantChangeType(&varLeft, &varLeft, 0, resvt);
+            if (FAILED(hres)) goto VarAnd_Exit;
+        }
+        if (V_VT(&varRight) != resvt)
+        {
+            VARTYPE target_vt = resvt;
+            if (V_VT(&varLeft) == resvt && (resvt > V_VT(&varRight)))
+                target_vt = V_VT(&varLeft);
+            hres = VariantChangeType(&varRight, &varRight, 0, target_vt);
+            if (FAILED(hres)) goto VarAnd_Exit;
+        }
+    }
+
     V_VT(result) = resvt;
     switch(resvt)
     {