[PATCH 0/1] MR968: crypt32: Set correct exponent length in decoded BCRYPT_RSAKEY_BLOB.

[Paul Gofman (＠gofman)]

Nvidia's Streamline library depends on that (https://github.com/NVIDIAGameWorks/Streamline/).

I modified an exponent value in tests to a non sensual one which has non-symmetric byte sequence so the byte order also gets tested.

-- 
https://gitlab.winehq.org/wine/wine/-/merge_requests/968

[PATCH 1/1] crypt32: Set correct exponent length in decoded BCRYPT_RSAKEY_BLOB.

[Paul Gofman]

From: Paul Gofman <pgofman(a)codeweavers.com>

---
 dlls/crypt32/decode.c       | 37 +++++++++++++++++++++++++++----------
 dlls/crypt32/tests/encode.c | 20 ++++++++++----------
 2 files changed, 37 insertions(+), 20 deletions(-)

diff --git a/dlls/crypt32/decode.c b/dlls/crypt32/decode.c
index 60f9868ed67..356ad9d539e 100644
--- a/dlls/crypt32/decode.c
+++ b/dlls/crypt32/decode.c
@@ -3874,7 +3874,7 @@ static BOOL WINAPI CRYPT_AsnDecodeCertPolicyConstraints(
 
 struct DECODED_RSA_PUB_KEY
 {
-    DWORD              pubexp;
+    CRYPT_INTEGER_BLOB pubexp;
     CRYPT_INTEGER_BLOB modulus;
 };
 
@@ -3893,12 +3893,18 @@ static BOOL CRYPT_raw_decode_rsa_pub_key(struct DECODED_RSA_PUB_KEY **decodedKey
        FALSE, TRUE, offsetof(struct DECODED_RSA_PUB_KEY, modulus.pbData),
        0 },
      { ASN_INTEGER, offsetof(struct DECODED_RSA_PUB_KEY, pubexp),
-       CRYPT_AsnDecodeIntInternal, sizeof(DWORD), FALSE, FALSE, 0, 0 },
+       CRYPT_AsnDecodeUnsignedIntegerInternal, sizeof(CRYPT_INTEGER_BLOB),
+       FALSE, TRUE, offsetof(struct DECODED_RSA_PUB_KEY, pubexp.pbData),
+       0 },
     };
 
     ret = CRYPT_AsnDecodeSequence(items, ARRAY_SIZE(items),
      pbEncoded, cbEncoded, CRYPT_DECODE_ALLOC_FLAG, NULL, decodedKey,
      size, NULL, NULL);
+
+    if (ret && (*decodedKey)->pubexp.cbData > sizeof(DWORD))
+        WARN("Unexpected exponent length %lu.\n", (*decodedKey)->pubexp.cbData);
+
     return ret;
 }
 
@@ -3920,7 +3926,7 @@ static BOOL WINAPI CRYPT_AsnDecodeRsaPubKey_Bcrypt(DWORD dwCertEncodingType,
         if (ret)
         {
             /* Header, exponent, and modulus */
-            DWORD bytesNeeded = sizeof(BCRYPT_RSAKEY_BLOB) + sizeof(DWORD) +
+            DWORD bytesNeeded = sizeof(BCRYPT_RSAKEY_BLOB) + decodedKey->pubexp.cbData +
              decodedKey->modulus.cbData;
 
             if (!pvStructInfo)
@@ -3939,7 +3945,7 @@ static BOOL WINAPI CRYPT_AsnDecodeRsaPubKey_Bcrypt(DWORD dwCertEncodingType,
                 hdr = pvStructInfo;
                 hdr->Magic = BCRYPT_RSAPUBLIC_MAGIC;
                 hdr->BitLength = decodedKey->modulus.cbData * 8;
-                hdr->cbPublicExp = sizeof(DWORD);
+                hdr->cbPublicExp = decodedKey->pubexp.cbData;
                 hdr->cbModulus = decodedKey->modulus.cbData;
                 hdr->cbPrime1 = 0;
                 hdr->cbPrime2 = 0;
@@ -3947,9 +3953,9 @@ static BOOL WINAPI CRYPT_AsnDecodeRsaPubKey_Bcrypt(DWORD dwCertEncodingType,
                  * in big-endian format, so we need to convert from little-endian
                  */
                 CRYPT_CopyReversed((BYTE *)pvStructInfo + sizeof(BCRYPT_RSAKEY_BLOB),
-                 (BYTE *)&decodedKey->pubexp, sizeof(DWORD));
+                 decodedKey->pubexp.pbData, hdr->cbPublicExp);
                 CRYPT_CopyReversed((BYTE *)pvStructInfo + sizeof(BCRYPT_RSAKEY_BLOB) +
-                 sizeof(DWORD), decodedKey->modulus.pbData,
+                 hdr->cbPublicExp, decodedKey->modulus.pbData,
                  decodedKey->modulus.cbData);
             }
             LocalFree(decodedKey);
@@ -3969,6 +3975,7 @@ static BOOL WINAPI CRYPT_AsnDecodeRsaPubKey(DWORD dwCertEncodingType,
  LPCSTR lpszStructType, const BYTE *pbEncoded, DWORD cbEncoded, DWORD dwFlags,
  PCRYPT_DECODE_PARA pDecodePara, void *pvStructInfo, DWORD *pcbStructInfo)
 {
+    RSAPUBKEY *rsaPubKey;
     BOOL ret;
 
     __TRY
@@ -3981,16 +3988,21 @@ static BOOL WINAPI CRYPT_AsnDecodeRsaPubKey(DWORD dwCertEncodingType,
             DWORD bytesNeeded = sizeof(BLOBHEADER) + sizeof(RSAPUBKEY) +
              decodedKey->modulus.cbData;
 
-            if (!pvStructInfo)
+            if (decodedKey->pubexp.cbData > sizeof(rsaPubKey->pubexp))
+            {
+                WARN("Unexpected exponent length %lu.\n", decodedKey->pubexp.cbData);
+                SetLastError(CRYPT_E_ASN1_LARGE);
+                ret = FALSE;
+            }
+            else if (!pvStructInfo)
             {
                 *pcbStructInfo = bytesNeeded;
-                ret = TRUE;
             }
             else if ((ret = CRYPT_DecodeEnsureSpace(dwFlags, pDecodePara,
              pvStructInfo, pcbStructInfo, bytesNeeded)))
             {
                 BLOBHEADER *hdr;
-                RSAPUBKEY *rsaPubKey;
+                unsigned int i;
 
                 if (dwFlags & CRYPT_DECODE_ALLOC_FLAG)
                     pvStructInfo = *(BYTE **)pvStructInfo;
@@ -4002,7 +4014,12 @@ static BOOL WINAPI CRYPT_AsnDecodeRsaPubKey(DWORD dwCertEncodingType,
                 rsaPubKey = (RSAPUBKEY *)((BYTE *)pvStructInfo +
                  sizeof(BLOBHEADER));
                 rsaPubKey->magic = RSA1_MAGIC;
-                rsaPubKey->pubexp = decodedKey->pubexp;
+                rsaPubKey->pubexp = 0;
+                for (i = 0; i < decodedKey->pubexp.cbData; ++i)
+                {
+                    rsaPubKey->pubexp |= decodedKey->pubexp.pbData[i] << (i * 8);
+                }
+
                 rsaPubKey->bitlen = decodedKey->modulus.cbData * 8;
                 memcpy((BYTE *)pvStructInfo + sizeof(BLOBHEADER) +
                  sizeof(RSAPUBKEY), decodedKey->modulus.pbData,
diff --git a/dlls/crypt32/tests/encode.c b/dlls/crypt32/tests/encode.c
index 3c37f41d504..565afb4c9e0 100644
--- a/dlls/crypt32/tests/encode.c
+++ b/dlls/crypt32/tests/encode.c
@@ -2315,10 +2315,10 @@ static const BYTE modulus1[] = { 0,0,0,1,1,1,1,1 };
 static const BYTE modulus2[] = { 1,1,1,1,1,0,0,0 };
 static const BYTE modulus3[] = { 0x80,1,1,1,1,0,0,0 };
 static const BYTE modulus4[] = { 1,1,1,1,1,0,0,0x80 };
-static const BYTE mod1_encoded[] = { 0x30,0x0f,0x02,0x08,0x01,0x01,0x01,0x01,0x01,0x00,0x00,0x00,0x02,0x03,0x01,0x00,0x01 };
-static const BYTE mod2_encoded[] = { 0x30,0x0c,0x02,0x05,0x01,0x01,0x01,0x01,0x01,0x02,0x03,0x01,0x00,0x01 };
-static const BYTE mod3_encoded[] = { 0x30,0x0c,0x02,0x05,0x01,0x01,0x01,0x01,0x80,0x02,0x03,0x01,0x00,0x01 };
-static const BYTE mod4_encoded[] = { 0x30,0x10,0x02,0x09,0x00,0x80,0x00,0x00,0x01,0x01,0x01,0x01,0x01,0x02,0x03,0x01,0x00,0x01 };
+static const BYTE mod1_encoded[] = { 0x30,0x0f,0x02,0x08,0x01,0x01,0x01,0x01,0x01,0x00,0x00,0x00,0x02,0x03,0x02,0x00,0x01 };
+static const BYTE mod2_encoded[] = { 0x30,0x0c,0x02,0x05,0x01,0x01,0x01,0x01,0x01,0x02,0x03,0x02,0x00,0x01 };
+static const BYTE mod3_encoded[] = { 0x30,0x0c,0x02,0x05,0x01,0x01,0x01,0x01,0x80,0x02,0x03,0x02,0x00,0x01 };
+static const BYTE mod4_encoded[] = { 0x30,0x10,0x02,0x09,0x00,0x80,0x00,0x00,0x01,0x01,0x01,0x01,0x01,0x02,0x03,0x02,0x00,0x01 };
 
 struct EncodedRSAPubKey
 {
@@ -2351,7 +2351,7 @@ static void test_encodeRsaPublicKey(DWORD dwEncoding)
     hdr->aiKeyAlg = CALG_RSA_KEYX;
     rsaPubKey->magic = 0x31415352;
     rsaPubKey->bitlen = sizeof(modulus1) * 8;
-    rsaPubKey->pubexp = 65537;
+    rsaPubKey->pubexp = 131073;
     memcpy(toEncode + sizeof(BLOBHEADER) + sizeof(RSAPUBKEY), modulus1,
      sizeof(modulus1));
 
@@ -2480,7 +2480,7 @@ static void test_decodeRsaPublicKey(DWORD dwEncoding)
              "Expected magic RSA1, got %08lx\n", rsaPubKey->magic);
             ok(rsaPubKey->bitlen == rsaPubKeys[i].decodedModulusLen * 8,
              "Wrong bit len %ld\n", rsaPubKey->bitlen);
-            ok(rsaPubKey->pubexp == 65537, "Expected pubexp 65537, got %ld\n",
+            ok(rsaPubKey->pubexp == 131073, "Expected pubexp 131073, got %ld\n",
              rsaPubKey->pubexp);
             ok(!memcmp(buf + sizeof(BLOBHEADER) + sizeof(RSAPUBKEY),
              rsaPubKeys[i].modulus, rsaPubKeys[i].decodedModulusLen),
@@ -2497,7 +2497,7 @@ static void test_encodeRsaPublicKey_Bcrypt(DWORD dwEncoding)
     BOOL ret;
     BYTE *buf = NULL;
     DWORD bufSize = 0, i;
-    BYTE pubexp[] = {0x01,0x00,0x01,0x00}; /* 65537 */
+    BYTE pubexp[] = {0x01,0x00,0x02,0x00}; /* 131073 */
 
     /* Verify that the Magic value doesn't matter */
     hdr->Magic = 1;
@@ -2568,7 +2568,7 @@ static void test_decodeRsaPublicKey_Bcrypt(DWORD dwEncoding)
         if (ret)
         {
             BCRYPT_RSAKEY_BLOB *hdr = (BCRYPT_RSAKEY_BLOB *)buf;
-            BYTE pubexp[] = {0xff,0xff,0xff,0xff}, pubexp_expected[] = {0x01,0x00,0x01};
+            BYTE pubexp[] = {0xff,0xff,0xff,0xcc}, pubexp_expected[] = {0x01,0x00,0x02};
             /* CNG_RSA_PUBLIC_KEY_BLOB stores the exponent
              * in big-endian format, so we need to convert it to little-endian
              */
@@ -2584,13 +2584,13 @@ static void test_decodeRsaPublicKey_Bcrypt(DWORD dwEncoding)
             /* Windows decodes the exponent to 3 bytes, since it will fit.
              * Our implementation currently unconditionally decodes to a DWORD (4 bytes)
              */
-            todo_wine ok(hdr->cbPublicExp == 3, "Expected cbPublicExp 3, got %ld\n", hdr->cbPublicExp);
+            ok(hdr->cbPublicExp == 3, "Expected cbPublicExp 3, got %ld\n", hdr->cbPublicExp);
             ok(hdr->cbModulus == rsaPubKeys[i].decodedModulusLen,
               "Wrong modulus len %ld\n", hdr->cbModulus);
             ok(hdr->cbPrime1 == 0,"Wrong cbPrime1 %ld\n", hdr->cbPrime1);
             ok(hdr->cbPrime2 == 0,"Wrong cbPrime2 %ld\n", hdr->cbPrime2);
             ok(!memcmp(pubexp, pubexp_expected, sizeof(pubexp_expected)), "Wrong exponent\n");
-            todo_wine ok(pubexp[3] == 0xff, "Got %02x\n", pubexp[3]);
+            ok(pubexp[3] == 0xcc, "Got %02x\n", pubexp[3]);
 
             leModulus = malloc(hdr->cbModulus);
              /*
-- 
GitLab

https://gitlab.winehq.org/wine/wine/-/merge_requests/968