[PATCH 1/5] include: Add InterlockedAnd().

[Zebediah Figura]

Signed-off-by: Zebediah Figura 
---
 dlls/msvcrt/lock.c | 10 ----------
 include/winnt.h    | 12 ++++++++++++
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/dlls/msvcrt/lock.c b/dlls/msvcrt/lock.c
index c10445becda..74156aa7db3 100644
--- a/dlls/msvcrt/lock.c
+++ b/dlls/msvcrt/lock.c
@@ -1157,16 +1157,6 @@ static inline void spin_wait_for_next_rwl(rwl_queue *q)
     SpinWait_dtor(&sw);
 }

-static LONG InterlockedAnd(LONG *d, LONG v)
-{
-    LONG l = *d, old;
-    while ((l & v) != l) {
-        if((old = InterlockedCompareExchange(d, l&v, l)) == l) break;
-        l = old;
-    }
-    return l;
-}
-
 /* ?lock@reader_writer_lock@Concurrency@@QAEXXZ */
 /* ?lock@reader_writer_lock@Concurrency@@QEAAXXZ */
 DEFINE_THISCALL_WRAPPER(reader_writer_lock_lock, 4)
diff --git a/include/winnt.h b/include/winnt.h
index 8834ced73e6..b03f7bb3646 100644
--- a/include/winnt.h
+++ b/include/winnt.h
@@ -6896,6 +6896,7 @@ static inline BOOLEAN BitScanReverse(DWORD *index, DWORD mask)

#ifdef _MSC_VER

+#pragma intrinsic(_InterlockedAnd)
 #pragma intrinsic(_InterlockedCompareExchange)
 #pragma intrinsic(_InterlockedCompareExchange64)
 #ifdef _WIN64
@@ -6907,6 +6908,7 @@ static inline BOOLEAN BitScanReverse(DWORD *index, DWORD mask)
 #pragma intrinsic(_InterlockedDecrement)
 #pragma intrinsic(_InterlockedOr)

+long      _InterlockedAnd(long volatile *,long);
 long      _InterlockedCompareExchange(long volatile*,long,long);
 long long _InterlockedCompareExchange64(long long volatile*,long long,long long);
 #ifdef _WIN64
@@ -6918,6 +6920,11 @@ long      _InterlockedExchangeAdd(long volatile*,long);
 long      _InterlockedIncrement(long volatile*);
 long      _InterlockedOr(long volatile *,long);

+static FORCEINLINE LONG WINAPI InterlockedAnd( LONG volatile *dest, LONG val )
+{
+    return _InterlockedAnd( (long volatile *)dest, val );
+}
+
 static FORCEINLINE LONG WINAPI InterlockedCompareExchange( LONG volatile *dest, LONG xchg, LONG compare )
 {
     return _InterlockedCompareExchange( (long volatile *)dest, xchg, compare );
@@ -7030,6 +7037,11 @@ static FORCEINLINE void MemoryBarrier(void)

#elif defined(__GNUC__)

+static FORCEINLINE LONG WINAPI InterlockedAnd( LONG volatile *dest, LONG val )
+{
+    return __sync_fetch_and_and( dest, val );
+}
+
 static FORCEINLINE LONG WINAPI InterlockedCompareExchange( LONG volatile *dest, LONG xchg, LONG compare )
 {
     return __sync_val_compare_and_swap( dest, compare, xchg );

[PATCH 2/5] include: Add InterlockedIncrement16() and InterlockedDecrement16().

[Zebediah Figura]

Signed-off-by: Zebediah Figura 
---
 include/winnt.h | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/include/winnt.h b/include/winnt.h
index b03f7bb3646..d8abb8bde70 100644
--- a/include/winnt.h
+++ b/include/winnt.h
@@ -6905,7 +6905,9 @@ static inline BOOLEAN BitScanReverse(DWORD *index, DWORD mask)
 #pragma intrinsic(_InterlockedExchange)
 #pragma intrinsic(_InterlockedExchangeAdd)
 #pragma intrinsic(_InterlockedIncrement)
+#pragma intrinsic(_InterlockedIncrement16)
 #pragma intrinsic(_InterlockedDecrement)
+#pragma intrinsic(_InterlockedDecrement16)
 #pragma intrinsic(_InterlockedOr)

long      _InterlockedAnd(long volatile *,long);
@@ -6915,9 +6917,11 @@ long long _InterlockedCompareExchange64(long long volatile*,long long,long long)
 unsigned char _InterlockedCompareExchange128(volatile __int64 *, __int64, __int64, __int64 *);
 #endif
 long      _InterlockedDecrement(long volatile*);
+short     _InterlockedDecrement16(short volatile*);
 long      _InterlockedExchange(long volatile*,long);
 long      _InterlockedExchangeAdd(long volatile*,long);
 long      _InterlockedIncrement(long volatile*);
+short     _InterlockedIncrement16(short volatile*);
 long      _InterlockedOr(long volatile *,long);

static FORCEINLINE LONG WINAPI InterlockedAnd( LONG volatile *dest, LONG val )
@@ -6957,11 +6961,21 @@ static FORCEINLINE LONG WINAPI InterlockedIncrement( LONG volatile *dest )
     return _InterlockedIncrement( (long volatile *)dest );
 }

+static FORCEINLINE short WINAPI InterlockedIncrement16( short volatile *dest )
+{
+    return _InterlockedIncrement16( (long volatile *)dest );
+}
+
 static FORCEINLINE LONG WINAPI InterlockedDecrement( LONG volatile *dest )
 {
     return _InterlockedDecrement( (long volatile *)dest );
 }

+static FORCEINLINE short WINAPI InterlockedDecrement16( short volatile *dest )
+{
+    return _InterlockedDecrement16( (long volatile *)dest );
+}
+
 static FORCEINLINE LONG WINAPI InterlockedOr( LONG volatile *dest, LONG val )
 {
     return _InterlockedOr( (long volatile *)dest, val );
@@ -7098,11 +7112,21 @@ static FORCEINLINE LONG WINAPI InterlockedIncrement( LONG volatile *dest )
     return __sync_add_and_fetch( dest, 1 );
 }

+static FORCEINLINE short WINAPI InterlockedIncrement16( short volatile *dest )
+{
+    return __sync_add_and_fetch( dest, 1 );
+}
+
 static FORCEINLINE LONG WINAPI InterlockedDecrement( LONG volatile *dest )
 {
     return __sync_add_and_fetch( dest, -1 );
 }

+static FORCEINLINE short WINAPI InterlockedDecrement16( short volatile *dest )
+{
+    return __sync_add_and_fetch( dest, -1 );
+}
+
 static FORCEINLINE void * WINAPI InterlockedExchangePointer( void *volatile *dest, void *val )
 {
     void *ret;

[PATCH 3/5] include: Add YieldProcessor().

[Zebediah Figura]

Signed-off-by: Zebediah Figura 
---
 include/winnt.h | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/include/winnt.h b/include/winnt.h
index d8abb8bde70..d933b26c0ec 100644
--- a/include/winnt.h
+++ b/include/winnt.h
@@ -7154,6 +7154,19 @@ static FORCEINLINE void MemoryBarrier(void)

#endif  /* __GNUC__ */

+static FORCEINLINE void YieldProcessor(void)
+{
+#ifdef __GNUC__
+#if defined(__i386__) || defined(__x86_64__)
+    __asm__ __volatile__( "rep; nop" : : : "memory" );
+#elif defined(__arm__) || defined(__aarch64__)
+    __asm__ __volatile__( "dmb ishst; yield" : : : "memory" );
+#else
+    __asm__ __volatile__( "" : : : "memory" );
+#endif
+#endif
+}
+
 #ifdef __cplusplus
 }
 #endif

[PATCH 4/5] wined3d: Use YieldProcessor().

[Zebediah Figura]

Signed-off-by: Zebediah Figura 
---
 dlls/wined3d/cs.c              | 4 ++--
 dlls/wined3d/wined3d_private.h | 9 +--------
 2 files changed, 3 insertions(+), 10 deletions(-)

diff --git a/dlls/wined3d/cs.c b/dlls/wined3d/cs.c
index 0b9c5f70043..56df760145a 100644
--- a/dlls/wined3d/cs.c
+++ b/dlls/wined3d/cs.c
@@ -646,7 +646,7 @@ void wined3d_cs_emit_present(struct wined3d_cs *cs, struct wined3d_swapchain *sw
      * ahead of the worker thread. */
     while (pending >= swapchain->max_frame_latency)
     {
-        wined3d_pause();
+        YieldProcessor();
         pending = InterlockedCompareExchange(&cs->pending_presents, 0, 0);
     }
 }
@@ -2822,7 +2822,7 @@ static void wined3d_cs_mt_finish(struct wined3d_cs *cs, enum wined3d_cs_queue_id
         return wined3d_cs_st_finish(cs, queue_id);

while (cs->queue[queue_id].head != *(volatile LONG *)&cs->queue[queue_id].tail)
-        wined3d_pause();
+        YieldProcessor();
 }

static const struct wined3d_cs_ops wined3d_cs_mt_ops =
diff --git a/dlls/wined3d/wined3d_private.h b/dlls/wined3d/wined3d_private.h
index 37fe3a314cb..e010deb1264 100644
--- a/dlls/wined3d/wined3d_private.h
+++ b/dlls/wined3d/wined3d_private.h
@@ -424,13 +424,6 @@ static inline unsigned int wined3d_popcount(unsigned int x)
 #endif
 }

-static inline void wined3d_pause(void)
-{
-#if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))
-    __asm__ __volatile__( "rep;nop" : : : "memory" );
-#endif
-}
-
 #define ORM_BACKBUFFER  0
 #define ORM_FBO         1

@@ -4817,7 +4810,7 @@ static inline void wined3d_resource_wait_idle(struct wined3d_resource *resource)
         return;

while (InterlockedCompareExchange(&resource->access_count, 0, 0))
-        wined3d_pause();
+        YieldProcessor();
 }

/* TODO: Add tests and support for FLOAT16_4 POSITIONT, D3DCOLOR position, other

[PATCH 4/5] wined3d: Use YieldProcessor().

[Henri Verbeet]

Signed-off-by: Henri Verbeet

[PATCH 5/5] ntoskrnl: Use YieldProcessor().

[Zebediah Figura]

Signed-off-by: Zebediah Figura 
---
 dlls/ntoskrnl.exe/sync.c | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

diff --git a/dlls/ntoskrnl.exe/sync.c b/dlls/ntoskrnl.exe/sync.c
index 671398594e8..2aa60a0f2a0 100644
--- a/dlls/ntoskrnl.exe/sync.c
+++ b/dlls/ntoskrnl.exe/sync.c
@@ -540,15 +540,6 @@ void WINAPI KeInitializeSpinLock( KSPIN_LOCK *lock )
     *lock = 0;
 }

-static inline void small_pause(void)
-{
-#ifdef __x86_64__
-    __asm__ __volatile__( "rep;nop" : : : "memory" );
-#else
-    __asm__ __volatile__( "" : : : "memory" );
-#endif
-}
-
 /***********************************************************************
  *           KeAcquireSpinLockAtDpcLevel (NTOSKRNL.EXE.@)
  */
@@ -556,7 +547,7 @@ void WINAPI KeAcquireSpinLockAtDpcLevel( KSPIN_LOCK *lock )
 {
     TRACE("lock %p.\n", lock);
     while (InterlockedCompareExchangePointer( (void **)lock, (void *)1, (void *)0 ))
-        small_pause();
+        YieldProcessor();
 }

/***********************************************************************
@@ -592,7 +583,7 @@ void FASTCALL KeAcquireInStackQueuedSpinLockAtDpcLevel( KSPIN_LOCK *lock, KLOCK_
         while (!((ULONG_PTR)InterlockedCompareExchangePointer( (void **)&queue->LockQueue.Lock, 0, 0 )
                  & QUEUED_SPINLOCK_OWNED))
         {
-            small_pause();
+            YieldProcessor();
         }
     }
 }
@@ -619,7 +610,7 @@ void FASTCALL KeReleaseInStackQueuedSpinLockFromDpcLevel( KLOCK_QUEUE_HANDLE *qu
         /* Otherwise, someone just queued themselves, but hasn't yet set
          * themselves as successor. Spin waiting for them to do so. */
         while (!(next = queue->LockQueue.Next))
-            small_pause();
+            YieldProcessor();
     }

InterlockedExchangePointer( (void **)&next->Lock, (KSPIN_LOCK *)((ULONG_PTR)lock | QUEUED_SPINLOCK_OWNED) );