Robert Wilhelm : oleaut32: Rescale result after overflow in VarDecAdd.

[Alexandre Julliard]

Module: wine
Branch: master
Commit: b440573d2a9228495a4814ccba0483e05bab14e6
URL:    https://source.winehq.org/git/wine.git/?a=commit;h=b440573d2a9228495a4814ccb...

Author: Robert Wilhelm 
Date:   Tue Oct 12 22:25:08 2021 +0200

oleaut32: Rescale result after overflow in VarDecAdd.

Signed-off-by: Robert Wilhelm 
Signed-off-by: Alexandre Julliard

---

dlls/oleaut32/tests/vartype.c |  4 ++--
 dlls/oleaut32/vartype.c       | 37 +++++++++++++++++++++++++++++++++++++
 2 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/dlls/oleaut32/tests/vartype.c b/dlls/oleaut32/tests/vartype.c
index bf31fde2b7a..995fd254208 100644
--- a/dlls/oleaut32/tests/vartype.c
+++ b/dlls/oleaut32/tests/vartype.c
@@ -3967,14 +3967,14 @@ static void test_VarDecAdd(void)
      S(U(out)).scale, S(U(out)).sign, out.Hi32, S1(U1(out)).Mid32, S1(U1(out)).Lo32, hres);

SETDEC64(l,1,0,0xffffffff,0xffffffff,0xffffffff);SETDEC(r,1,0,0,1); MATH2(VarDecAdd);
-  todo_wine EXPECTDEC64(0,0,0x19999999,0x99999999,0x9999999A);
+  EXPECTDEC64(0,0,0x19999999,0x99999999,0x9999999A);

SETDEC64(l,0,0,0xe22ea493,0xb30310a7,0x70000000);SETDEC64(r,0,0,0xe22ea493,0xb30310a7,0x70000000); MATH2(VarDecAdd);
   ok(hres == DISP_E_OVERFLOW,"Expected overflow, got (%d,%d,%d,(%8x,%8x)x) hres 0x%08x\n",
      S(U(out)).scale, S(U(out)).sign, out.Hi32, S1(U1(out)).Mid32, S1(U1(out)).Lo32, hres);

SETDEC64(l,1,0,0xe22ea493,0xb30310a7,0x70000000);SETDEC64(r,1,0,0xe22ea493,0xb30310a7,0x70000000); MATH2(VarDecAdd);
-  todo_wine EXPECTDEC64(0,0,0x2d3c8750,0xbd670354,0xb0000000);
+  EXPECTDEC64(0,0,0x2d3c8750,0xbd670354,0xb0000000);

SETDEC(l,3,128,0,123456); SETDEC64(r,0,0,0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF);
   MATH2(VarDecAdd); EXPECTDEC64(0,0,-1,0xFFFFFFFF,0xFFFFFF84);
diff --git a/dlls/oleaut32/vartype.c b/dlls/oleaut32/vartype.c
index dfdb0a15c51..0c6249d48db 100644
--- a/dlls/oleaut32/vartype.c
+++ b/dlls/oleaut32/vartype.c
@@ -4643,6 +4643,43 @@ VarDecAdd_AsPositive:
       DEC_LO32(pDecOut)  = VARIANT_Add(DEC_LO32(pDecLeft),  DEC_LO32(pDecRight),  &overflow);
       DEC_MID32(pDecOut) = VARIANT_Add(DEC_MID32(pDecLeft), DEC_MID32(pDecRight), &overflow);
       DEC_HI32(pDecOut)  = VARIANT_Add(DEC_HI32(pDecLeft),  DEC_HI32(pDecRight),  &overflow);
+
+      if (overflow)
+      {
+        int i;
+        DWORD n[4];
+        unsigned char remainder;
+
+        if (!DEC_SCALE(pDecLeft))
+          return DISP_E_OVERFLOW;
+
+        DEC_SCALE(pDecOut) = DEC_SCALE(pDecLeft) - 1;
+        DEC_SIGN(pDecOut) = sign;
+
+        n[0] = DEC_LO32(pDecOut);
+        n[1] = DEC_MID32(pDecOut);
+        n[2] = DEC_HI32(pDecOut);
+        n[3] = overflow;
+
+        remainder = VARIANT_int_divbychar(n,4,10);
+
+        /* round up the result */
+        if (remainder >= 5)
+        {
+          for (remainder = 1, i = 0; i < ARRAY_SIZE(n) && remainder; i++)
+          {
+            ULONGLONG digit = n[i] + 1;
+            remainder = (digit > 0xFFFFFFFF) ? 1 : 0;
+            n[i] = digit & 0xFFFFFFFF;
+          }
+        }
+
+        DEC_LO32(pDecOut) = n[0] ;
+        DEC_MID32(pDecOut) = n[1];
+        DEC_HI32(pDecOut) = n[2];
+
+        return S_OK;
+      }
     }

if (overflow)