From c5870ac276100746dc733aa86ec601b207c659cd Mon Sep 17 00:00:00 2001
From: wj32 <wj32@21ef857c-d57f-4fe0-8362-d861dc6d29cd>
Date: Wed, 24 Feb 2010 07:35:53 +0000
Subject: [PATCH] added fast path variants of queued lock functions

git-svn-id: svn://svn.code.sf.net/p/processhacker/code@2873 21ef857c-d57f-4fe0-8362-d861dc6d29cd
---
 2.x/trunk/ProcessHacker/basesup.c            |  8 +-
 2.x/trunk/ProcessHacker/include/queuedlock.h | 98 ++++++++++++++++++++
 2.x/trunk/ProcessHacker/queuedlock.c         | 34 +++++++
 3 files changed, 136 insertions(+), 4 deletions(-)

diff --git a/2.x/trunk/ProcessHacker/basesup.c b/2.x/trunk/ProcessHacker/basesup.c
index 30a416cd8..5641107cd 100644
--- a/2.x/trunk/ProcessHacker/basesup.c
+++ b/2.x/trunk/ProcessHacker/basesup.c
@@ -2550,7 +2550,7 @@ PVOID PhAllocateFromFreeList(
 
     // TODO: Implement as lock-free singly linked list.
 
-    PhAcquireQueuedLockExclusive(&FreeList->Lock);
+    PhAcquireQueuedLockExclusiveFast(&FreeList->Lock);
 
     if (FreeList->Count != 0)
     {
@@ -2562,7 +2562,7 @@ PVOID PhAllocateFromFreeList(
         memory = PhAllocate(FreeList->Size);
     }
 
-    PhReleaseQueuedLockExclusive(&FreeList->Lock);
+    PhReleaseQueuedLockExclusiveFast(&FreeList->Lock);
 
     return memory;
 }
@@ -2578,7 +2578,7 @@ VOID PhFreeToFreeList(
     __in PVOID Memory
     )
 {
-    PhAcquireQueuedLockExclusive(&FreeList->Lock);
+    PhAcquireQueuedLockExclusiveFast(&FreeList->Lock);
 
     if (FreeList->Count < FreeList->MaximumCount)
     {
@@ -2589,7 +2589,7 @@ VOID PhFreeToFreeList(
         PhFree(Memory);
     }
 
-    PhReleaseQueuedLockExclusive(&FreeList->Lock);
+    PhReleaseQueuedLockExclusiveFast(&FreeList->Lock);
 }
 
 /**
diff --git a/2.x/trunk/ProcessHacker/include/queuedlock.h b/2.x/trunk/ProcessHacker/include/queuedlock.h
index 2f087bc03..ea674d07b 100644
--- a/2.x/trunk/ProcessHacker/include/queuedlock.h
+++ b/2.x/trunk/ProcessHacker/include/queuedlock.h
@@ -2,6 +2,7 @@
 #define QUEUEDLOCK_H
 
 #define PH_QUEUED_LOCK_OWNED ((ULONG_PTR)0x1)
+#define PH_QUEUED_LOCK_OWNED_SHIFT 0
 #define PH_QUEUED_LOCK_WAITERS ((ULONG_PTR)0x2)
 
 // Valid only if Waiters = 0
@@ -78,4 +79,101 @@ VOID FASTCALL PhfReleaseQueuedLockShared(
     __inout PPH_QUEUED_LOCK QueuedLock
     );
 
+#define PhTryWakePushLock PhfTryWakePushLock
+VOID FASTCALL PhfTryWakePushLock(
+    __inout PPH_QUEUED_LOCK QueuedLock
+    );
+
+// Inline functions
+
+FORCEINLINE VOID PhAcquireQueuedLockExclusiveFast(
+    __inout PPH_QUEUED_LOCK QueuedLock
+    )
+{
+#ifdef _M_IX86
+    if (_interlockedbittestandset((PLONG)&QueuedLock->Value, PH_QUEUED_LOCK_OWNED_SHIFT))
+#else
+    if (_interlockedbittestandset64((PLONG64)&QueuedLock->Value, PH_QUEUED_LOCK_OWNED_SHIFT))
+#endif
+    {
+        // Owned bit was already set. Slow path.
+        PhAcquireQueuedLockExclusive(QueuedLock);
+    }
+}
+
+FORCEINLINE VOID PhAcquireQueuedLockSharedFast(
+    __inout PPH_QUEUED_LOCK QueuedLock
+    )
+{
+    ULONG_PTR value;
+
+    value = QueuedLock->Value;
+
+    if ((ULONG_PTR)_InterlockedCompareExchangePointer(
+        (PPVOID)&QueuedLock->Value,
+        (PVOID)(PH_QUEUED_LOCK_OWNED | PH_QUEUED_LOCK_SHARED_INC),
+        (PVOID)value
+        ) != value)
+    {
+        PhAcquireQueuedLockShared(QueuedLock);
+    }
+}
+
+FORCEINLINE BOOLEAN PhTryAcquirePushLockExclusive(
+    __inout PPH_QUEUED_LOCK QueuedLock
+    )
+{
+#ifdef _M_IX86
+    if (!_interlockedbittestandset((PLONG)&QueuedLock->Value, PH_QUEUED_LOCK_OWNED_SHIFT))
+#else
+    if (!_interlockedbittestandset64((PLONG64)&QueuedLock->Value, PH_QUEUED_LOCK_OWNED_SHIFT))
+#endif
+    {
+        return TRUE; 
+    }
+    else
+    {
+        return FALSE;
+    }
+}
+
+FORCEINLINE VOID PhReleaseQueuedLockExclusiveFast(
+    __inout PPH_QUEUED_LOCK QueuedLock
+    )
+{
+    ULONG_PTR value;
+
+#ifdef _M_IX86
+    value = (ULONG_PTR)_InterlockedExchangeAdd((PLONG)&QueuedLock->Value, -PH_QUEUED_LOCK_OWNED);
+#else
+    value = (ULONG_PTR)_InterlockedExchangeAdd((PLONG64)&QueuedLock->Value, -PH_QUEUED_LOCK_OWNED);
+#endif
+
+    if (
+        (value & PH_QUEUED_LOCK_WAITERS) &&
+        !(value & PH_QUEUED_LOCK_TRAVERSING)
+        )
+    {
+        PhTryWakePushLock(QueuedLock);
+    }
+}
+
+FORCEINLINE VOID PhReleaseQueuedLockSharedFast(
+    __inout PPH_QUEUED_LOCK QueuedLock
+    )
+{
+    ULONG_PTR value;
+
+    value = PH_QUEUED_LOCK_OWNED | PH_QUEUED_LOCK_SHARED_INC;
+
+    if ((ULONG_PTR)_InterlockedCompareExchangePointer(
+        (PPVOID)&QueuedLock->Value,
+        (PVOID)0,
+        (PVOID)value
+        ) != value)
+    {
+        PhReleaseQueuedLockShared(QueuedLock);
+    }
+}
+
 #endif
diff --git a/2.x/trunk/ProcessHacker/queuedlock.c b/2.x/trunk/ProcessHacker/queuedlock.c
index c1193c75f..334186cd2 100644
--- a/2.x/trunk/ProcessHacker/queuedlock.c
+++ b/2.x/trunk/ProcessHacker/queuedlock.c
@@ -794,3 +794,37 @@ VOID FASTCALL PhfReleaseQueuedLockShared(
         value = newValue;
     }
 }
+
+/**
+ * Wakes waiters in a queued lock, making no assumptions 
+ * about the state of the lock.
+ *
+ * \param QueuedLock A queued lock.
+ */
+VOID FASTCALL PhfTryWakePushLock(
+    __inout PPH_QUEUED_LOCK QueuedLock
+    )
+{
+    ULONG_PTR value;
+    ULONG_PTR newValue;
+
+    value = QueuedLock->Value;
+
+    if (
+        !(value & PH_QUEUED_LOCK_WAITERS) ||
+        (value & PH_QUEUED_LOCK_TRAVERSING) ||
+        (value & PH_QUEUED_LOCK_OWNED)
+        )
+        return;
+
+    newValue = value + PH_QUEUED_LOCK_TRAVERSING;
+
+    if ((ULONG_PTR)_InterlockedCompareExchangePointer(
+        (PPVOID)&QueuedLock->Value,
+        (PVOID)newValue,
+        (PVOID)value
+        ) == value)
+    {
+        PhpfWakeQueuedLock(QueuedLock, newValue);
+    }
+}