[PATCH 1/3] ntdll: Use Vista's algorithm for RtlEncode/DecodePointer.

[Myah Caron]

Signed-off-by: Myah Caron <qsniyg(a)protonmail.com>
---
The algorithm is an implementation of
https://web.archive.org/web/20100603042315/http://blogs.msdn.com/b/michael_h...

I'm sending in this patchset without the tests as I realize the tests are non-deterministic,
and therefore probably not acceptable.

However, in case it is acceptable I will also send a separate v2 patch for the tests (v1 is
194268), fixing the algorithm according to the linked MSDN post.

 dlls/ntdll/rtl.c | 39 +++++++++++++++++++++++++++++++++++++--
 1 file changed, 37 insertions(+), 2 deletions(-)

diff --git a/dlls/ntdll/rtl.c b/dlls/ntdll/rtl.c
index ca4fea84209..154de807501 100644
--- a/dlls/ntdll/rtl.c
+++ b/dlls/ntdll/rtl.c
@@ -1542,19 +1542,54 @@ static DWORD_PTR get_pointer_obfuscator( void )
     return pointer_obfuscator;
 }

+/***********************************************************************
+ * rotl_ptr (internal)
+ */
+#ifdef _WIN64
+#define ROT_BITS 64
+#else
+#define ROT_BITS 32
+#endif
+
+static DWORD_PTR rotl_ptr( DWORD_PTR num, int shift )
+{
+    shift &= ROT_BITS - 1;
+    return (num << shift) | (num >> (ROT_BITS-shift));
+}
+
+/***********************************************************************
+ * rotr_ptr (internal)
+ */
+static DWORD_PTR rotr_ptr( DWORD_PTR num, int shift )
+{
+    shift &= ROT_BITS - 1;
+    return (num >> shift) | (num << (ROT_BITS-shift));
+}
+
+#undef ROT_BITS
+
 /*************************************************************************
  * RtlEncodePointer   [NTDLL.@]
  */
 PVOID WINAPI RtlEncodePointer( PVOID ptr )
 {
+
     DWORD_PTR ptrval = (DWORD_PTR) ptr;
-    return (PVOID)(ptrval ^ get_pointer_obfuscator());
+    DWORD_PTR cookie = get_pointer_obfuscator();
+
+    /* http://blogs.msdn.com/b/michael_howard/archive/2006/08/16/702707.aspx */
+
+    ptrval = (ptrval ^ cookie);
+    return (PVOID)rotr_ptr(ptrval, cookie);
 }

 PVOID WINAPI RtlDecodePointer( PVOID ptr )
 {
     DWORD_PTR ptrval = (DWORD_PTR) ptr;
-    return (PVOID)(ptrval ^ get_pointer_obfuscator());
+    DWORD_PTR cookie = get_pointer_obfuscator();
+
+    ptrval = rotl_ptr(ptrval, cookie);
+    return (PVOID)(ptrval ^ cookie);
 }

 /*************************************************************************
--
2.28.0