added fast path variants of queued lock functions

git-svn-id: svn://svn.code.sf.net/p/processhacker/code@2873 21ef857c-d57f-4fe0-8362-d861dc6d29cd

2.x/trunk/ProcessHacker/basesup.c

@@ -2550,7 +2550,7 @@ PVOID PhAllocateFromFreeList(
 
     // TODO: Implement as lock-free singly linked list.
 
-    PhAcquireQueuedLockExclusive(&FreeList->Lock);
+    PhAcquireQueuedLockExclusiveFast(&FreeList->Lock);
 
     if (FreeList->Count != 0)
     {
@@ -2562,7 +2562,7 @@ PVOID PhAllocateFromFreeList(
         memory = PhAllocate(FreeList->Size);
     }
 
-    PhReleaseQueuedLockExclusive(&FreeList->Lock);
+    PhReleaseQueuedLockExclusiveFast(&FreeList->Lock);
 
     return memory;
 }
@@ -2578,7 +2578,7 @@ VOID PhFreeToFreeList(
     __in PVOID Memory
     )
 {
-    PhAcquireQueuedLockExclusive(&FreeList->Lock);
+    PhAcquireQueuedLockExclusiveFast(&FreeList->Lock);
 
     if (FreeList->Count < FreeList->MaximumCount)
     {
@@ -2589,7 +2589,7 @@ VOID PhFreeToFreeList(
         PhFree(Memory);
     }
 
-    PhReleaseQueuedLockExclusive(&FreeList->Lock);
+    PhReleaseQueuedLockExclusiveFast(&FreeList->Lock);
 }
 
 /**

2.x/trunk/ProcessHacker/include/queuedlock.h

@@ -2,6 +2,7 @@
 #define QUEUEDLOCK_H
 
 #define PH_QUEUED_LOCK_OWNED ((ULONG_PTR)0x1)
+#define PH_QUEUED_LOCK_OWNED_SHIFT 0
 #define PH_QUEUED_LOCK_WAITERS ((ULONG_PTR)0x2)
 
 // Valid only if Waiters = 0
@@ -78,4 +79,101 @@ VOID FASTCALL PhfReleaseQueuedLockShared(
     __inout PPH_QUEUED_LOCK QueuedLock
     );
 
+#define PhTryWakePushLock PhfTryWakePushLock
+VOID FASTCALL PhfTryWakePushLock(
+    __inout PPH_QUEUED_LOCK QueuedLock
+    );
+
+// Inline functions
+
+FORCEINLINE VOID PhAcquireQueuedLockExclusiveFast(
+    __inout PPH_QUEUED_LOCK QueuedLock
+    )
+{
+#ifdef _M_IX86
+    if (_interlockedbittestandset((PLONG)&QueuedLock->Value, PH_QUEUED_LOCK_OWNED_SHIFT))
+#else
+    if (_interlockedbittestandset64((PLONG64)&QueuedLock->Value, PH_QUEUED_LOCK_OWNED_SHIFT))
+#endif
+    {
+        // Owned bit was already set. Slow path.
+        PhAcquireQueuedLockExclusive(QueuedLock);
+    }
+}
+
+FORCEINLINE VOID PhAcquireQueuedLockSharedFast(
+    __inout PPH_QUEUED_LOCK QueuedLock
+    )
+{
+    ULONG_PTR value;
+
+    value = QueuedLock->Value;
+
+    if ((ULONG_PTR)_InterlockedCompareExchangePointer(
+        (PPVOID)&QueuedLock->Value,
+        (PVOID)(PH_QUEUED_LOCK_OWNED | PH_QUEUED_LOCK_SHARED_INC),
+        (PVOID)value
+        ) != value)
+    {
+        PhAcquireQueuedLockShared(QueuedLock);
+    }
+}
+
+FORCEINLINE BOOLEAN PhTryAcquirePushLockExclusive(
+    __inout PPH_QUEUED_LOCK QueuedLock
+    )
+{
+#ifdef _M_IX86
+    if (!_interlockedbittestandset((PLONG)&QueuedLock->Value, PH_QUEUED_LOCK_OWNED_SHIFT))
+#else
+    if (!_interlockedbittestandset64((PLONG64)&QueuedLock->Value, PH_QUEUED_LOCK_OWNED_SHIFT))
+#endif
+    {
+        return TRUE; 
+    }
+    else
+    {
+        return FALSE;
+    }
+}
+
+FORCEINLINE VOID PhReleaseQueuedLockExclusiveFast(
+    __inout PPH_QUEUED_LOCK QueuedLock
+    )
+{
+    ULONG_PTR value;
+
+#ifdef _M_IX86
+    value = (ULONG_PTR)_InterlockedExchangeAdd((PLONG)&QueuedLock->Value, -PH_QUEUED_LOCK_OWNED);
+#else
+    value = (ULONG_PTR)_InterlockedExchangeAdd((PLONG64)&QueuedLock->Value, -PH_QUEUED_LOCK_OWNED);
+#endif
+
+    if (
+        (value & PH_QUEUED_LOCK_WAITERS) &&
+        !(value & PH_QUEUED_LOCK_TRAVERSING)
+        )
+    {
+        PhTryWakePushLock(QueuedLock);
+    }
+}
+
+FORCEINLINE VOID PhReleaseQueuedLockSharedFast(
+    __inout PPH_QUEUED_LOCK QueuedLock
+    )
+{
+    ULONG_PTR value;
+
+    value = PH_QUEUED_LOCK_OWNED | PH_QUEUED_LOCK_SHARED_INC;
+
+    if ((ULONG_PTR)_InterlockedCompareExchangePointer(
+        (PPVOID)&QueuedLock->Value,
+        (PVOID)0,
+        (PVOID)value
+        ) != value)
+    {
+        PhReleaseQueuedLockShared(QueuedLock);
+    }
+}
+
 #endif

2.x/trunk/ProcessHacker/queuedlock.c

@@ -794,3 +794,37 @@ VOID FASTCALL PhfReleaseQueuedLockShared(
         value = newValue;
     }
 }
+
+/**
+ * Wakes waiters in a queued lock, making no assumptions 
+ * about the state of the lock.
+ *
+ * \param QueuedLock A queued lock.
+ */
+VOID FASTCALL PhfTryWakePushLock(
+    __inout PPH_QUEUED_LOCK QueuedLock
+    )
+{
+    ULONG_PTR value;
+    ULONG_PTR newValue;
+
+    value = QueuedLock->Value;
+
+    if (
+        !(value & PH_QUEUED_LOCK_WAITERS) ||
+        (value & PH_QUEUED_LOCK_TRAVERSING) ||
+        (value & PH_QUEUED_LOCK_OWNED)
+        )
+        return;
+
+    newValue = value + PH_QUEUED_LOCK_TRAVERSING;
+
+    if ((ULONG_PTR)_InterlockedCompareExchangePointer(
+        (PPVOID)&QueuedLock->Value,
+        (PVOID)newValue,
+        (PVOID)value
+        ) == value)
+    {
+        PhpfWakeQueuedLock(QueuedLock, newValue);
+    }
+}