On 3/15/20 4:00 PM, Aaro Altonen wrote:
Add CRYPT_STRING_HEX format implementation for CryptStringToBinaryW()

Wine-Bug: https://bugs.winehq.org/show_bug.cgi?id=48487
Signed-off-by: Aaro Altonen <a.altonen@hotmail.com>

Hi.

---
 dlls/crypt32/base64.c       | 91 +++++++++++++++++++++++++++++++++++++
 dlls/crypt32/tests/base64.c | 66 +++++++++++++--------------
 2 files changed, 124 insertions(+), 33 deletions(-)

diff --git a/dlls/crypt32/base64.c b/dlls/crypt32/base64.c
index 73619421ab..9856dead03 100644
--- a/dlls/crypt32/base64.c
+++ b/dlls/crypt32/base64.c
@@ -1036,6 +1036,95 @@ static LONG DecodeAnyW(LPCWSTR pszString, DWORD cchString,
     return ret;
 }
 
+static BYTE char_to_byte(BYTE byte)
+{
+    if (byte >= '0' && byte <= '9')
+        return byte - '0';
+    if (byte >= 'a' && byte <='f')
+        return byte - 'a' + 10;
+    else if (byte >= 'A' && byte <='F')
+        return byte - 'A' + 10;
+
+    return 0xff;
+}
What's a point of testing halves?
+
+static DWORD hex_to_binary(WCHAR *str, BYTE *out, DWORD *outlen, DWORD *processed)
+{
+    DWORD ret = ERROR_SUCCESS;
+
+    while (*str && *(str + 1))
+    {
+        BYTE un = char_to_byte(*str);
+        BYTE ln = char_to_byte(*(str + 1));
+
+        if (un == 0xff || ln == 0xff)
+            return ERROR_INVALID_DATA;
+
+        if (*processed >= *outlen)
+        {
+            if (out)
+                return ERROR_MORE_DATA;
+            ret = ERROR_MORE_DATA;
+        }
+        else if (out)
+            out[*processed] = (un << 4) | ln;
+
+        (*processed)++;
+        str += 2;
+    }
+
+    return ret;
+}
+
+static LONG DecodeBinaryToHexW(const WCHAR *instr, DWORD slen,
+    BYTE *out, DWORD *outlen, DWORD *skip, DWORD *flags)
+{
+    const WCHAR delim[] = { ' ', '\n', '\r', '\t', '\0' };
+    WCHAR *duped, *ptr, *tok, *saveptr;
+    LONG ret = ERROR_SUCCESS;
+    DWORD processed = 0;
+
+    if (!outlen)
+        return ERROR_INVALID_PARAMETER;
+
+    duped = ptr = HeapAlloc(GetProcessHeap(), 0, (lstrlenW(instr) + 1) * sizeof(WCHAR));
+    if (duped)
+        lstrcpyW(duped, instr);
+    else
+        return ERROR_NOT_ENOUGH_MEMORY;
Do you need to duplicate input string?
+
+    /* force strtok_sW() to parse at most slen - 1 bytes (including white spaces) */
+    duped[slen] = '\0';
+
+    while ((tok = strtok_sW(ptr, delim, &saveptr)))
It's probably easier to duplicate wcstok(). 
+    {
+        if (strlenW(tok) % 2)
+        {
+            ret = ERROR_INVALID_DATA;
+            break;
+        }
+
+        if ((ret = hex_to_binary(tok, out, outlen, &processed)))
+        {
+            if (ret == ERROR_INVALID_DATA)
+                break;
+        }
+
+        ptr = NULL;
+    }
+
+    if (ret == ERROR_INVALID_DATA)
+    {
+        if (!out)
+            *outlen = 0;
+    }
+    else
+        *outlen = processed;
+
+    HeapFree(GetProcessHeap(), 0, duped);
+    return ret;
+}
+
 BOOL WINAPI CryptStringToBinaryW(LPCWSTR pszString,
  DWORD cchString, DWORD dwFlags, BYTE *pbBinary, DWORD *pcbBinary,
  DWORD *pdwSkip, DWORD *pdwFlags)
@@ -1081,6 +1170,8 @@ BOOL WINAPI CryptStringToBinaryW(LPCWSTR pszString,
         decoder = DecodeAnyW;
         break;
     case CRYPT_STRING_HEX:
+        decoder = DecodeBinaryToHexW;
+        break;
Helper name is backwards, as it decodes from hex string to binary buffer.
     case CRYPT_STRING_HEXASCII:
     case CRYPT_STRING_HEXADDR:
     case CRYPT_STRING_HEXASCIIADDR:
diff --git a/dlls/crypt32/tests/base64.c b/dlls/crypt32/tests/base64.c
index 8af0c5bae0..b789a0a010 100644
--- a/dlls/crypt32/tests/base64.c
+++ b/dlls/crypt32/tests/base64.c
@@ -813,8 +813,8 @@ static void testStringToBinaryW(void)
     data_len = 0xdeadbeef;
     ret = CryptStringToBinaryW(input, 0, CRYPT_STRING_HEX, NULL, &data_len, &skip_, &flags);
     ok(!ret, "Got %u, expected zero\n", ret);
-    todo_wine ok(GetLastError() == ERROR_INVALID_DATA, "Got %d, expected 13\n", GetLastError());
-    todo_wine ok(data_len == 0, "Got %u, expected 0\n", data_len);
+    ok(GetLastError() == ERROR_INVALID_DATA, "Got %d, expected 13\n", GetLastError());
+    ok(data_len == 0, "Got %u, expected 0\n", data_len);
     heap_free(input);
 
     /* length is uneven -> 13 */
@@ -823,8 +823,8 @@ static void testStringToBinaryW(void)
     data_len = 0xdeadbeef;
     ret = CryptStringToBinaryW(input, 3, CRYPT_STRING_HEX, NULL, &data_len, &skip_, &flags);
     ok(!ret, "Got %u, expected zero\n", ret);
-    todo_wine ok(GetLastError() == ERROR_INVALID_DATA, "Got %d, expected 13\n", GetLastError());
-    todo_wine ok(data_len == 0, "Got %u, expected 0\n", data_len);
+    ok(GetLastError() == ERROR_INVALID_DATA, "Got %d, expected 13\n", GetLastError());
+    ok(data_len == 0, "Got %u, expected 0\n", data_len);
     heap_free(input);
 
     /* invalid 0x prefix -> 13 */
@@ -833,8 +833,8 @@ static void testStringToBinaryW(void)
     data_len = 0xdeadbeef;
     ret = CryptStringToBinaryW(input, 0, CRYPT_STRING_HEX, NULL, &data_len, &skip_, &flags);
     ok(!ret, "Got %u, expected zero\n", ret);
-    todo_wine ok(GetLastError() == ERROR_INVALID_DATA, "Got %d, expected 13\n", GetLastError());
-    todo_wine ok(data_len == 0, "Got %u, expected 0\n", data_len);
+    ok(GetLastError() == ERROR_INVALID_DATA, "Got %d, expected 13\n", GetLastError());
+    ok(data_len == 0, "Got %u, expected 0\n", data_len);
     heap_free(input);
 
     /* invalid characters -> 13 */
@@ -843,8 +843,8 @@ static void testStringToBinaryW(void)
     data_len = 0xdeadbeef;
     ret = CryptStringToBinaryW(input, 0, CRYPT_STRING_HEX, NULL, &data_len, NULL, NULL);
     ok(!ret, "Got %u, expected zero\n", ret);
-    todo_wine ok(GetLastError() == ERROR_INVALID_DATA, "Got %d, expected 13\n", GetLastError());
-    todo_wine ok(data_len == 0, "Got %u, expected 0\n", data_len);
+    ok(GetLastError() == ERROR_INVALID_DATA, "Got %d, expected 13\n", GetLastError());
+    ok(data_len == 0, "Got %u, expected 0\n", data_len);
     heap_free(input);
 
     /* insufficient buffer -> 234 */
@@ -858,9 +858,9 @@ static void testStringToBinaryW(void)
     input = strdupAtoW("213c73796d6c696e6b3efffe");
     ret = CryptStringToBinaryW(input, 24, CRYPT_STRING_HEX, out, &data_len, &skip_, &flags);
     ok(!ret, "Got %u, expected zero\n", ret);
-    todo_wine ok(GetLastError() == ERROR_MORE_DATA, "Got %d, expected 234\n", GetLastError());
+    ok(GetLastError() == ERROR_MORE_DATA, "Got %d, expected 234\n", GetLastError());
     ok(data_len == 4, "Got %u, expected 4\n", data_len);
-    todo_wine ok(!memcmp(out, expected, 4), "Invalid output from CryptStringToBinaryW()!\n");
+    ok(!memcmp(out, expected, 4), "Invalid output from CryptStringToBinaryW()!\n");
     heap_free(input);
 
     /* valid data */
@@ -868,9 +868,9 @@ static void testStringToBinaryW(void)
     input = strdupAtoW("213c73796d6c696e6b3efffe");
     data_len = 0xdeadbeef;
     ret = CryptStringToBinaryW(input, 24, CRYPT_STRING_HEX, NULL, &data_len, &skip_, &flags);
-    todo_wine ok(ret, "Got %u, expected one\n", ret);
-    todo_wine ok(GetLastError() == 0xdeadbeef, "Got %x, expected 0xdeadbeef\n", GetLastError());
-    todo_wine ok(data_len == 12, "Got %u, expected 12\n", data_len);
+    ok(ret, "Got %u, expected one\n", ret);
+    ok(GetLastError() == 0xdeadbeef, "Got %x, expected 0xdeadbeef\n", GetLastError());
+    ok(data_len == 12, "Got %u, expected 12\n", data_len);
     heap_free(input);
 
     /* valid data with white spaces  */
@@ -879,9 +879,9 @@ static void testStringToBinaryW(void)
     data_len = 0xdeadbeef;
     SetLastError(0xdeadbeef);
     ret = CryptStringToBinaryW(input, 25, CRYPT_STRING_HEX, NULL, &data_len, &skip_, &flags);
-    todo_wine ok(ret, "Got %u, expected one\n", ret);
-    todo_wine ok(GetLastError() == 0xdeadbeef, "Got %d, expected 0xdeadbeef\n", GetLastError());
-    todo_wine ok(data_len == 7, "Got %u, expected 7\n", data_len);
+    ok(ret, "Got %u, expected one\n", ret);
+    ok(GetLastError() == 0xdeadbeef, "Got %d, expected 0xdeadbeef\n", GetLastError());
+    ok(data_len == 7, "Got %u, expected 7\n", data_len);
     heap_free(input);
 
     /* valid data with white spaces but spacing breaks the valid data into invalid chunks */
@@ -890,8 +890,8 @@ static void testStringToBinaryW(void)
     data_len = 0xdeadbeef;
     ret = CryptStringToBinaryW(input, 0, CRYPT_STRING_HEX, NULL, &data_len, &skip_, &flags);
     ok(!ret, "Got %u, expected zero\n", ret);
-    todo_wine ok(GetLastError() == ERROR_INVALID_DATA, "Got %d, expected 13\n", GetLastError());
-    todo_wine ok(data_len == 0, "Got %u, expected 0\n", data_len);
+    ok(GetLastError() == ERROR_INVALID_DATA, "Got %d, expected 13\n", GetLastError());
+    ok(data_len == 0, "Got %u, expected 0\n", data_len);
     heap_free(input);
 
     /* if "input" contains both valid and invalid data and "out" is valid, "out" shall contain all valid bytes
@@ -902,7 +902,7 @@ static void testStringToBinaryW(void)
     input = strdupAtoW("21 3 c ff");
     ret = CryptStringToBinaryW(input, 0, CRYPT_STRING_HEX, out, &data_len, &skip_, &flags);
     ok(!ret, "Got %u, expected zero\n", ret);
-    todo_wine ok(GetLastError() == ERROR_INVALID_DATA, "Got %d, expected 13\n", GetLastError());
+    ok(GetLastError() == ERROR_INVALID_DATA, "Got %d, expected 13\n", GetLastError());
     ok(data_len == 4, "Got %u, expected 4\n", data_len);
     heap_free(input);
 
@@ -915,10 +915,10 @@ static void testStringToBinaryW(void)
     input = strdupAtoW("213c73796d6c696e6b3efffe");
     data_len = 256;
     ret = CryptStringToBinaryW(input, 24, CRYPT_STRING_HEX, out, &data_len, &skip_, &flags);
-    todo_wine ok(ret, "Got %u, expected one\n", ret);
-    todo_wine ok(GetLastError() == 0xdeadbeef, "Got %x, expected 0xdeadbeef\n", GetLastError());
-    todo_wine ok(data_len == 12, "Got %u, expected 12\n", data_len);
-    todo_wine ok(!memcmp(out, expected, 12), "Invalid output from CryptStringToBinaryW()!\n");
+    ok(ret, "Got %u, expected one\n", ret);
+    ok(GetLastError() == 0xdeadbeef, "Got %x, expected 0xdeadbeef\n", GetLastError());
+    ok(data_len == 12, "Got %u, expected 12\n", data_len);
+    ok(!memcmp(out, expected, 12), "Invalid output from CryptStringToBinaryW()!\n");
     heap_free(input);
 
     /* invalid data but length small enough that it's never detected */
@@ -926,9 +926,9 @@ static void testStringToBinaryW(void)
     input = strdupAtoW("abcdefhhh");
     data_len = 0xdeadbeef;
     ret = CryptStringToBinaryW(input, 4, CRYPT_STRING_HEX, NULL, &data_len, NULL, NULL);
-    todo_wine ok(ret, "Got %u, expected one\n", ret);
-    todo_wine ok(GetLastError() == 0xdeadbeef, "Got %x, expected 0xdeadbeef\n", GetLastError());
-    todo_wine ok(data_len == 2, "Got %u, expected 2\n", data_len);
+    ok(ret, "Got %u, expected one\n", ret);
+    ok(GetLastError() == 0xdeadbeef, "Got %x, expected 0xdeadbeef\n", GetLastError());
+    ok(data_len == 2, "Got %u, expected 2\n", data_len);
     heap_free(input);
 
     /* invalid data but length small enough that it's never detected, with whitespaces */
@@ -937,9 +937,9 @@ static void testStringToBinaryW(void)
     input = strdupAtoW("\t\t21 fe");
     data_len = 256;
     ret = CryptStringToBinaryW(input, 5, CRYPT_STRING_HEX, out, &data_len, &skip_, &flags);
-    todo_wine ok(ret, "Got %u, expected one\n", ret);
-    todo_wine ok(GetLastError() == 0xdeadbeef, "Got %x, expected 0xdeadbeef\n", GetLastError());
-    todo_wine ok(data_len == 1, "Got %u, expected 1\n", data_len);
+    ok(ret, "Got %u, expected one\n", ret);
+    ok(GetLastError() == 0xdeadbeef, "Got %x, expected 0xdeadbeef\n", GetLastError());
+    ok(data_len == 1, "Got %u, expected 1\n", data_len);
     heap_free(input);
 
     /* valid data but parse only the first 6 bytes (12 chars) */
@@ -950,10 +950,10 @@ static void testStringToBinaryW(void)
     input = strdupAtoW("213c73796d6c696e6b3efffe");
     data_len = 256;
     ret = CryptStringToBinaryW(input, 12, CRYPT_STRING_HEX, out, &data_len, &skip_, &flags);
-    todo_wine ok(ret, "Got %u, expected one\n", ret);
-    todo_wine ok(GetLastError() == 0xdeadbeef, "Got %x, expected 0xdeadbeef\n", GetLastError());
-    todo_wine ok(data_len == 6, "Got %u, expected 6\n", data_len);
-    todo_wine ok(!memcmp(out, expected, 6), "Invalid output from CryptStringToBinaryW()!\n");
+    ok(ret, "Got %u, expected one\n", ret);
+    ok(GetLastError() == 0xdeadbeef, "Got %x, expected 0xdeadbeef\n", GetLastError());
+    ok(data_len == 6, "Got %u, expected 6\n", data_len);
+    ok(!memcmp(out, expected, 6), "Invalid output from CryptStringToBinaryW()!\n");
     heap_free(input);
 }