diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs index 55e5f4c77a7680..584f6cd3e15965 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs @@ -392,6 +392,11 @@ internal static unsafe void CopyConstruct(T* dest, T* src) where T : unmanage internal static ref byte GetRawData(this object obj) => ref Unsafe.As(obj).Data; + [DebuggerHidden] + [DebuggerStepThrough] + internal static ref nint GetMethodTableRef(this object obj) + => ref Unsafe.Subtract(ref Unsafe.As(ref GetRawData(obj)), 1); + [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static unsafe nuint GetRawObjectDataSize(object obj) { diff --git a/src/coreclr/System.Private.CoreLib/src/System/Threading/ManagedThreadId.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Threading/ManagedThreadId.CoreCLR.cs index cb5a56cb27e7b7..5f6040cae1e52d 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Threading/ManagedThreadId.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Threading/ManagedThreadId.CoreCLR.cs @@ -10,8 +10,16 @@ internal static class ManagedThreadId // This will be initialized by the runtime. [ThreadStatic] private static int t_currentManagedThreadId; + internal static int CurrentManagedThreadIdUnchecked => t_currentManagedThreadId; - public static int Current => t_currentManagedThreadId; + public static int Current + { + get + { + Debug.Assert(t_currentManagedThreadId != 0, "The runtime should have initialized the thread id by now."); + return t_currentManagedThreadId; + } + } } } diff --git a/src/coreclr/System.Private.CoreLib/src/System/Threading/Monitor.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Threading/Monitor.CoreCLR.cs index ab5c33950ab010..23e13f9c538a22 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Threading/Monitor.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Threading/Monitor.CoreCLR.cs @@ -54,55 +54,31 @@ static Lock GetLockObjectFallback(object obj) #endregion #region Public Enter/Exit methods + + [MethodImpl(MethodImplOptions.NoInlining)] public static void Enter(object obj) { - ObjectHeader.HeaderLockResult result = ObjectHeader.TryAcquireThinLock(obj); - if (result == ObjectHeader.HeaderLockResult.Success) - return; - - GetLockObject(obj).Enter(); + ObjectHeader.AcquireThinLock(obj); } + [MethodImpl(MethodImplOptions.NoInlining)] public static bool TryEnter(object obj) { - ObjectHeader.HeaderLockResult result = ObjectHeader.TryAcquireThinLock(obj); - if (result == ObjectHeader.HeaderLockResult.Success) - return true; - - if (result == ObjectHeader.HeaderLockResult.Failure) - return false; - - return GetLockObject(obj).TryEnter(); + return ObjectHeader.TryAcquireThinLock(obj); } + [MethodImpl(MethodImplOptions.NoInlining)] public static bool TryEnter(object obj, int millisecondsTimeout) { ArgumentOutOfRangeException.ThrowIfLessThan(millisecondsTimeout, -1); - - ObjectHeader.HeaderLockResult result = ObjectHeader.TryAcquireThinLock(obj); - if (result == ObjectHeader.HeaderLockResult.Success) - return true; - - return GetLockObject(obj).TryEnter(millisecondsTimeout); + return ObjectHeader.TryAcquireThinLock(obj, millisecondsTimeout); } [MethodImpl(MethodImplOptions.NoInlining)] public static void Exit(object obj) { ArgumentNullException.ThrowIfNull(obj); - ObjectHeader.HeaderLockResult result = ObjectHeader.Release(obj); - - if (result == ObjectHeader.HeaderLockResult.Success) - { - return; - } - - if (result == ObjectHeader.HeaderLockResult.Failure) - { - throw new SynchronizationLockException(); - } - - GetLockObject(obj).Exit(); + ObjectHeader.Release(obj); } // Marked no-inlining to prevent recursive inlining of IsAcquired. @@ -122,14 +98,7 @@ public static bool IsEntered(object obj) internal static void SynchronizedMethodEnter(object obj, ref bool lockTaken) { - ObjectHeader.HeaderLockResult result = ObjectHeader.TryAcquireThinLock(obj); - if (result == ObjectHeader.HeaderLockResult.Success) - { - lockTaken = true; - return; - } - - GetLockObject(obj).Enter(); + ObjectHeader.AcquireThinLock(obj); lockTaken = true; } @@ -139,19 +108,7 @@ internal static void SynchronizedMethodExit(object obj, ref bool lockTaken) if (!lockTaken) return; - ObjectHeader.HeaderLockResult result = ObjectHeader.Release(obj); - - if (result == ObjectHeader.HeaderLockResult.Success) - { - return; - } - - if (result == ObjectHeader.HeaderLockResult.Failure) - { - throw new SynchronizationLockException(); - } - - GetLockObject(obj).Exit(); + ObjectHeader.Release(obj); } } } diff --git a/src/coreclr/System.Private.CoreLib/src/System/Threading/ObjectHeader.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Threading/ObjectHeader.CoreCLR.cs index 7a91dffbc677ae..b079fc24987480 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Threading/ObjectHeader.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Threading/ObjectHeader.CoreCLR.cs @@ -48,7 +48,6 @@ internal static class ObjectHeader private const int SBLK_MASK_LOCK_RECLEVEL = 0x003F0000; // 64 recursion levels private const int SBLK_LOCK_RECLEVEL_INC = 0x00010000; // each level is this much higher than the previous one - // These must match the values in syncblk.h public enum HeaderLockResult { Success = 0, @@ -56,17 +55,11 @@ public enum HeaderLockResult UseSlowPath = 2 }; - [MethodImpl(MethodImplOptions.InternalCall)] - private static extern HeaderLockResult AcquireInternal(object obj); - - [MethodImpl(MethodImplOptions.InternalCall)] - public static extern HeaderLockResult Release(object obj); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static unsafe int* GetHeaderPtr(byte* ppObjectData) + private static unsafe int* GetHeaderPtr(nint* ppMethodTable) { - // The header is the 4 bytes before a pointer-sized chunk before the object data pointer. - return (int*)(ppObjectData - sizeof(void*) - sizeof(int)); + // The header is 4 bytes before MT pointer on all architectures + return (int*)ppMethodTable - 1; } // @@ -99,36 +92,135 @@ public enum HeaderLockResult // back-off as it favors micro-contention scenario, which we expect. // - // Try acquiring the thin-lock - public static unsafe HeaderLockResult TryAcquireThinLock(object obj) + // Try acquiring the thin-lock. + // The common cases (free lock, fat lock) are handled inline. A thread id that doesn't fit, + // recursive acquisition, contention and lost races are rarer and less performance sensitive, + // so they are handled out of line in AcquireUncommon to keep this inlined fast path small. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static unsafe bool TryAcquireThinLock(object obj, int millisecondsTimeout = 0) { ArgumentNullException.ThrowIfNull(obj); - HeaderLockResult result = AcquireInternal(obj); - if (result == HeaderLockResult.Failure) + // for an object used in locking there are two common cases: + // - header bits are unused or + // - there is a sync entry + int oldBits; + fixed (nint* ppMethodTable = &obj.GetMethodTableRef()) + { + int* pHeader = GetHeaderPtr(ppMethodTable); + oldBits = *pHeader; + + // Common case: the header is clean. Take it by storing our thread id with a single CAS. + if (oldBits == 0) + { + // Thread IDs are allocated sequentially starting from 1 and recycled, so it's + // unusual to have a thread ID that doesn't fit in the thin-lock field. + // Check here rather than at entry to keep the hot path as tight as possible. + int currentThreadID = ManagedThreadId.Current; + if ((uint)currentThreadID <= (uint)SBLK_MASK_LOCK_THREADID) + { + if (Interlocked.CompareExchange(pHeader, currentThreadID, oldBits) == oldBits) + { + return true; + } + } + } + } + + // Before checking uncommon cases, check if the lock is fat (or becoming fat). + // This is another common case. + if ((oldBits & (BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX | BIT_SBLK_SPIN_LOCK)) == 0) { - return TryAcquireThinLockSpin(obj); + HeaderLockResult result = TryAcquireUncommon(obj, millisecondsTimeout == 0); + + // If we acquired (or recursively re-acquired) the thin lock, we are done, + // regardless of the timeout. + if (result == HeaderLockResult.Success) + { + return true; + } + + // With no timeout, a Failure (lock owned by someone else) is definitive. + if (result == HeaderLockResult.Failure && millisecondsTimeout == 0) + { + return false; + } } - return result; + + Lock lck = Monitor.GetLockObject(obj); + return lck.TryEnter_Outlined(millisecondsTimeout); } - private static unsafe HeaderLockResult TryAcquireThinLockSpin(object obj) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static unsafe void AcquireThinLock(object obj) { - int currentThreadID = ManagedThreadId.Current; + ArgumentNullException.ThrowIfNull(obj); - // does thread ID fit? - if (currentThreadID > SBLK_MASK_LOCK_THREADID) - return HeaderLockResult.UseSlowPath; + // for an object used in locking there are two common cases: + // - header bits are unused or + // - there is a sync entry + int oldBits; + fixed (nint* ppMethodTable = &obj.GetMethodTableRef()) + { + int* pHeader = GetHeaderPtr(ppMethodTable); + oldBits = *pHeader; + + // Common case: the header is clean. Take it by storing our thread id with a single CAS. + if (oldBits == 0) + { + // Thread IDs are allocated sequentially starting from 1 and recycled, so it's + // unusual to have a thread ID that doesn't fit in the thin-lock field. + // Check here rather than at entry to keep the hot path as tight as possible. + // If the id doesn't fit, we fall through and call AcquireUncommon outside the + // fixed block to avoid keeping the object pinned while potentially spinning. + int currentThreadID = ManagedThreadId.Current; + if ((uint)currentThreadID <= (uint)SBLK_MASK_LOCK_THREADID) + { + if (Interlocked.CompareExchange(pHeader, currentThreadID, oldBits) == oldBits) + { + return; + } + } + } + } + + // Before checking uncommon cases, check if the lock is fat (or becoming fat). + // This is another common case. + if ((oldBits & (BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX | BIT_SBLK_SPIN_LOCK)) != 0 || + TryAcquireUncommon(obj, false) != HeaderLockResult.Success) + { + Lock lck = Monitor.GetLockObject(obj); + lck.Enter(); + } + } + + // handling uncommon cases here - recursive lock, contention, retries + // The public entry point spins by default (e.g. a blocking Monitor.Enter). Callers that want a + // single attempt (e.g. Monitor.TryEnter) pass isOneShot: true to succeed only if the lock is + // currently unowned. + [MethodImpl(MethodImplOptions.NoInlining)] + private static unsafe HeaderLockResult TryAcquireUncommon(object obj, bool isOneShot) + { + // A one-shot acquire does not spin while the lock is owned by another thread, but it still + // retries the rare CAS failures below where the lock is not owned by somebody else: a caller + // that knows the lock is unowned expects a one-shot acquire to succeed. + // Lock.IsSingleProcessor is lazily initialized at the first contended acquire; until then it + // is false and we assume a multicore machine. + int retries = isOneShot || Lock.IsSingleProcessor ? 0 : 16; - int retries = Lock.IsSingleProcessor ? 0 : 16; + int currentThreadID = ManagedThreadId.Current; + // A thin lock can only store a thread id that fits in the thread-id field. + // This check is deferred to here (rather than at entry) because it is unusual to be true. + if ((uint)currentThreadID > (uint)SBLK_MASK_LOCK_THREADID) + return HeaderLockResult.UseSlowPath; // retry when the lock is owned by somebody else. // this loop will spinwait between iterations. for (int i = 0; i <= retries; i++) { - fixed (byte* pObjectData = &obj.GetRawData()) + fixed (nint* ppMethodTable = &obj.GetMethodTableRef()) { - int* pHeader = GetHeaderPtr(pObjectData); + int* pHeader = GetHeaderPtr(ppMethodTable); // rare retries when lock is not owned by somebody else. // these do not count as iterations and do not spinwait. @@ -140,7 +232,6 @@ private static unsafe HeaderLockResult TryAcquireThinLockSpin(object obj) // we cannot use a thin-lock. if ((oldBits & (BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX | BIT_SBLK_SPIN_LOCK)) != 0) { - // Need to use a thick-lock. return HeaderLockResult.UseSlowPath; } // If we already own the lock, try incrementing recursion level. @@ -175,8 +266,7 @@ private static unsafe HeaderLockResult TryAcquireThinLockSpin(object obj) return HeaderLockResult.Success; } - // rare contention on lock. - // Try again in case the finalization bits were touched. + // Someone else touched the bits. Try again. continue; } else @@ -187,16 +277,66 @@ private static unsafe HeaderLockResult TryAcquireThinLockSpin(object obj) } } - if (retries != 0) + // lock is thin, but owned by somebody else. + // spin a bit before retrying (1 spinwait is roughly 35 nsec) + // the object is not pinned here + Thread.SpinWait(i); + } + + // the lock is thin, but owned by somebody else + return HeaderLockResult.Failure; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static unsafe void Release(object obj) + { + Debug.Assert(obj != null); + + fixed (nint* ppMethodTable = &obj.GetMethodTableRef()) + { + int* pHeader = GetHeaderPtr(ppMethodTable); + + // We may need to retry if we own the lock but the CAS races with another thread + // touching unrelated header bits. + while (true) { - // spin a bit before retrying (1 spinwait is roughly 35 nsec) - // the object is not pinned here - Thread.SpinWait(i); + int oldBits = *pHeader; + // is the lock thin? + if ((oldBits & (BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX | BIT_SBLK_SPIN_LOCK)) == 0) + { + // In CoreCLR, the managed ID is set by the runtime, thus we do not + // call CurrentManagedThreadIdUnchecked like we do in NativeAOT + int currentThreadID = ManagedThreadId.Current; + + // do we own the thin lock? + if ((oldBits & SBLK_MASK_LOCK_THREADID) == currentThreadID) + { + // decrement count or release entirely. + int newBits = (oldBits & SBLK_MASK_LOCK_RECLEVEL) != 0 ? + oldBits - SBLK_LOCK_RECLEVEL_INC : + oldBits & ~SBLK_MASK_LOCK_THREADID; + + if (Interlocked.CompareExchange(pHeader, newBits, oldBits) == oldBits) + { + return; + } + + // rare contention on owned thin lock, + // we still own the lock, try again + continue; + } + } + + // do slow path. + break; } } - // owned by somebody else - return HeaderLockResult.Failure; + // This is a case when we have: + // * a fat lock - the most likely case by far, or + // * we don't own the lock and need to throw and it is ok if the lock gets inflated. + // Let the slow path handle this. + Monitor.GetLockObject(obj).Exit(); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -204,9 +344,9 @@ public static unsafe HeaderLockResult IsAcquired(object obj) { ArgumentNullException.ThrowIfNull(obj); - fixed (byte* pObjectData = &obj.GetRawData()) + fixed (nint* ppMethodTable = &obj.GetMethodTableRef()) { - int* pHeader = GetHeaderPtr(pObjectData); + int* pHeader = GetHeaderPtr(ppMethodTable); // Ignore the spinlock here. // Either we'll read the thin-lock data in the header or we'll have a sync block. diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/Monitor.NativeAot.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/Monitor.NativeAot.cs index 00065f5257a42d..c4b4f1e3ac8ec7 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/Monitor.NativeAot.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/Monitor.NativeAot.cs @@ -36,64 +36,26 @@ internal static Lock GetLockObject(object obj) [MethodImpl(MethodImplOptions.NoInlining)] public static void Enter(object obj) { - int currentThreadID = ManagedThreadId.CurrentManagedThreadIdUnchecked; - int resultOrIndex = ObjectHeader.Acquire(obj, currentThreadID); - if (resultOrIndex < 0) - return; - - Lock lck = resultOrIndex == 0 ? - ObjectHeader.GetLockObject(obj) : - SyncTable.GetLockObject(resultOrIndex); - - lck.TryEnterSlow(Timeout.Infinite, currentThreadID); + ObjectHeader.AcquireThinLock(obj); } [MethodImpl(MethodImplOptions.NoInlining)] public static bool TryEnter(object obj) { - int currentThreadID = ManagedThreadId.CurrentManagedThreadIdUnchecked; - int resultOrIndex = ObjectHeader.TryAcquire(obj, currentThreadID); - if (resultOrIndex < 0) - return true; - - if (resultOrIndex == 0) - return false; - - Lock lck = SyncTable.GetLockObject(resultOrIndex); - - // The one-shot fast path is not covered by the slow path below for a zero timeout when the thread ID is - // initialized, so cover it here in case it wasn't already done - if (currentThreadID != Lock.UninitializedThreadId && lck.TryEnterOneShot(currentThreadID)) - return true; - - return lck.TryEnterSlow(0, currentThreadID) != Lock.UninitializedThreadId; + return ObjectHeader.TryAcquireThinLock(obj); } [MethodImpl(MethodImplOptions.NoInlining)] public static bool TryEnter(object obj, int millisecondsTimeout) { ArgumentOutOfRangeException.ThrowIfLessThan(millisecondsTimeout, -1); - - int currentThreadID = ManagedThreadId.CurrentManagedThreadIdUnchecked; - int resultOrIndex = ObjectHeader.TryAcquire(obj, currentThreadID); - if (resultOrIndex < 0) - return true; - - Lock lck = resultOrIndex == 0 ? - ObjectHeader.GetLockObject(obj) : - SyncTable.GetLockObject(resultOrIndex); - - // The one-shot fast path is not covered by the slow path below for a zero timeout when the thread ID is - // initialized, so cover it here in case it wasn't already done - if (millisecondsTimeout == 0 && currentThreadID != Lock.UninitializedThreadId && lck.TryEnterOneShot(currentThreadID)) - return true; - - return lck.TryEnterSlow(millisecondsTimeout, currentThreadID) != Lock.UninitializedThreadId; + return ObjectHeader.TryAcquireThinLock(obj, millisecondsTimeout); } [MethodImpl(MethodImplOptions.NoInlining)] public static void Exit(object obj) { + ArgumentNullException.ThrowIfNull(obj); ObjectHeader.Release(obj); } @@ -107,20 +69,7 @@ public static bool IsEntered(object obj) private static void SynchronizedMethodEnter(object obj, ref bool lockTaken) { - // Inlined Monitor.Enter with a few tweaks - int currentThreadID = ManagedThreadId.CurrentManagedThreadIdUnchecked; - int resultOrIndex = ObjectHeader.Acquire(obj, currentThreadID); - if (resultOrIndex < 0) - { - lockTaken = true; - return; - } - - Lock lck = resultOrIndex == 0 ? - ObjectHeader.GetLockObject(obj) : - SyncTable.GetLockObject(resultOrIndex); - - lck.TryEnterSlow(Timeout.Infinite, currentThreadID); + ObjectHeader.AcquireThinLock(obj); lockTaken = true; } diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/ObjectHeader.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/ObjectHeader.cs index b56054d9f164e0..8c40acac4e4d9a 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/ObjectHeader.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/ObjectHeader.cs @@ -280,76 +280,136 @@ public static unsafe void SetSyncEntryIndex(int* pHeader, int syncIndex) // back-off as it favors micro-contention scenario, which we expect. // - // Returs: - // -1 - success - // 0 - failed - // syncIndex - retry with the Lock - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe int Acquire(object obj, int currentThreadID) - { - return TryAcquire(obj, currentThreadID, oneShot: false); - } - - // -1 - success - // 0 - failed - // syncIndex - retry with the Lock + // Try acquiring the thin-lock. + // The common cases (free lock, fat lock) are handled inline. A thread id that doesn't fit, + // recursive acquisition, contention and lost races are rarer and less performance sensitive, + // so they are handled out of line in AcquireUncommon to keep this inlined fast path small. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe int TryAcquire(object obj, int currentThreadID, bool oneShot = true) + public static unsafe bool TryAcquireThinLock(object obj, int millisecondsTimeout = 0) { ArgumentNullException.ThrowIfNull(obj); - // if thread ID is uninitialized or too big, we do "uncommon" part. - if ((uint)(currentThreadID - 1) <= (uint)SBLK_MASK_LOCK_THREADID) + // for an object used in locking there are two common cases: + // - header bits are unused or + // - there is a sync entry + int oldBits; + fixed (MethodTable** ppMethodTable = &obj.GetMethodTableRef()) { - // for an object used in locking there are two common cases: - // - header bits are unused or - // - there is a sync entry - fixed (MethodTable** ppMethodTable = &obj.GetMethodTableRef()) + int* pHeader = GetHeaderPtr(ppMethodTable); + oldBits = *pHeader; + + // Common case: the header is clean. Take it by storing our thread id with a single CAS. + if (oldBits == 0) { - int* pHeader = GetHeaderPtr(ppMethodTable); - int oldBits = *pHeader; - // if unused for anything, try setting our thread id - // N.B. hashcode, thread ID and sync index are never 0, and hashcode is largest of all - if ((oldBits & MASK_HASHCODE_INDEX) == 0) + // Thread IDs are allocated sequentially starting from 1 and recycled, so it's + // unusual to have a thread ID that doesn't fit in the thin-lock field. + // Check here rather than at entry to keep the hot path as tight as possible. + // The uninitialized 0 id is also ruled out by this check. + int currentThreadID = ManagedThreadId.CurrentManagedThreadIdUnchecked; + if ((uint)(currentThreadID - 1) < (uint)SBLK_MASK_LOCK_THREADID) { - if (Interlocked.CompareExchange(pHeader, oldBits | currentThreadID, oldBits) == oldBits) + if (Interlocked.CompareExchange(pHeader, currentThreadID, oldBits) == oldBits) { - return -1; + return true; } } - else if (GetSyncEntryIndex(oldBits, out int syncIndex)) + } + } + + // Before checking uncommon cases, check if the lock is fat (or becoming fat). + // This is another common case. + int resultOrIndex = 0; + if ((oldBits & BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX) == 0) + { + resultOrIndex = TryAcquireUncommon(obj, millisecondsTimeout == 0); + + // If we acquired (or recursively re-acquired) the thin lock, we are done, + // regardless of the timeout. + if (resultOrIndex == -1) + { + return true; + } + + // With no timeout, a Failure (lock owned by someone else) is definitive. + if (resultOrIndex == 0 && millisecondsTimeout == 0) + { + return false; + } + } + + Lock lck = resultOrIndex == 0 ? + ObjectHeader.GetLockObject(obj) : + SyncTable.GetLockObject(resultOrIndex); + + return lck.TryEnter_Outlined(millisecondsTimeout); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static unsafe void AcquireThinLock(object obj) + { + ArgumentNullException.ThrowIfNull(obj); + + // for an object used in locking there are two common cases: + // - header bits are unused or + // - there is a sync entry + int oldBits; + fixed (MethodTable** ppMethodTable = &obj.GetMethodTableRef()) + { + int* pHeader = GetHeaderPtr(ppMethodTable); + oldBits = *pHeader; + + // Common case: the header is clean. Take it by storing our thread id with a single CAS. + if (oldBits == 0) + { + // Thread IDs are allocated sequentially starting from 1 and recycled, so it's + // unusual to have a thread ID that doesn't fit in the thin-lock field. + // Check here rather than at entry to keep the hot path as tight as possible. + // The uninitialized 0 id is also ruled out by this check. + int currentThreadID = ManagedThreadId.CurrentManagedThreadIdUnchecked; + if ((uint)(currentThreadID - 1) < (uint)SBLK_MASK_LOCK_THREADID) { - if (SyncTable.GetLockObject(syncIndex).TryEnterOneShot(currentThreadID)) + if (Interlocked.CompareExchange(pHeader, currentThreadID, oldBits) == oldBits) { - return -1; + return; } - - // has sync entry -> slow path - return syncIndex; } } } - return TryAcquireUncommon(obj, currentThreadID, oneShot); + // Before checking uncommon cases, check if the lock is fat (or becoming fat). + // This is another common case. + int resultOrIndex = 0; + if ((oldBits & BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX) != 0 || + (resultOrIndex = TryAcquireUncommon(obj, false)) != -1) + { + Lock lck = resultOrIndex == 0 ? + ObjectHeader.GetLockObject(obj) : + SyncTable.GetLockObject(resultOrIndex); + + lck.TryEnter_Outlined(Timeout.Infinite); + } } // handling uncommon cases here - recursive lock, contention, retries // -1 - success // 0 - failed // syncIndex - retry with the Lock - private static unsafe int TryAcquireUncommon(object obj, int currentThreadID, bool oneShot) + [MethodImpl(MethodImplOptions.NoInlining)] + private static unsafe int TryAcquireUncommon(object obj, bool isOneShot) { - if (currentThreadID == 0) - currentThreadID = Environment.CurrentManagedThreadId; - - // does thread ID fit? - if (currentThreadID > SBLK_MASK_LOCK_THREADID) + // A one-shot acquire does not spin while the lock is owned by another thread, but it still + // retries the rare CAS failures below where the lock is not owned by somebody else: a caller + // that knows the lock is unowned expects a one-shot acquire to succeed. + // Lock.IsSingleProcessor is lazily initialized at the first contended acquire; until then it + // is false and we assume a multicore machine. + int retries = isOneShot || Lock.IsSingleProcessor ? 0 : 16; + + int currentThreadID = ManagedThreadId.Current; + // A thin lock can only store a thread id that fits in the thread-id field. + // This check is deferred to here (rather than at entry) because it is unusual to be true. + if ((uint)currentThreadID > (uint)SBLK_MASK_LOCK_THREADID) return GetSyncIndex(obj); - // Lock.IsSingleProcessor gets a value that is lazy-initialized at the first contended acquire. - // Until then it is false and we assume we have multicore machine. - int retries = oneShot || Lock.IsSingleProcessor ? 0 : 16; - // retry when the lock is owned by somebody else. // this loop will spinwait between iterations. for (int i = 0; i <= retries; i++) @@ -364,35 +424,14 @@ private static unsafe int TryAcquireUncommon(object obj, int currentThreadID, bo { int oldBits = *pHeader; - // if unused for anything, try setting our thread id - // N.B. hashcode, thread ID and sync index are never 0, and hashcode is largest of all - if ((oldBits & MASK_HASHCODE_INDEX) == 0) - { - int newBits = oldBits | currentThreadID; - if (Interlocked.CompareExchange(pHeader, newBits, oldBits) == oldBits) - { - return -1; - } - - // contention on a lock that noone owned, - // but we do not know if there is an owner yet, so try again - continue; - } - - // has sync entry -> slow path - if (GetSyncEntryIndex(oldBits, out int syncIndex)) - { - return syncIndex; - } - - // has hashcode -> slow path + // If has a hash code or syncblock, + // we cannot use a thin-lock. if ((oldBits & BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX) != 0) { return SyncTable.AssignEntry(obj, pHeader); } - - // we own the lock already - if ((oldBits & SBLK_MASK_LOCK_THREADID) == currentThreadID) + // If we already own the lock, try incrementing recursion level. + else if ((oldBits & SBLK_MASK_LOCK_THREADID) == currentThreadID) { // try incrementing recursion level, check for overflow int newBits = oldBits + SBLK_LOCK_RECLEVEL_INC; @@ -410,62 +449,79 @@ private static unsafe int TryAcquireUncommon(object obj, int currentThreadID, bo } else { - // overflow, transition to a fat Lock + // overflow, need to transition to a fat Lock return SyncTable.AssignEntry(obj, pHeader); } } + // If no one owns the lock, try acquiring it. + else if ((oldBits & SBLK_MASK_LOCK_THREADID) == 0) + { + int newBits = oldBits | currentThreadID; + if (Interlocked.CompareExchange(pHeader, newBits, oldBits) == oldBits) + { + return -1; + } - // someone else owns. - break; + // Someone else touched the bits. Try again. + continue; + } + else + { + // Owned by somebody else. Now we spinwait and retry. + break; + } } } - if (retries != 0) - { - // spin a bit before retrying (1 spinwait is roughly 35 nsec) - // the object is not pinned here - Thread.SpinWaitInternal(i); - } + // lock is thin, but owned by somebody else. + // spin a bit before retrying (1 spinwait is roughly 35 nsec) + // the object is not pinned here + Thread.SpinWaitInternal(i); } - // owned by somebody else + // the lock is thin, but owned by somebody else return 0; } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static unsafe void Release(object obj) { - ArgumentNullException.ThrowIfNull(obj); - - int currentThreadID = ManagedThreadId.CurrentManagedThreadIdUnchecked; - // transform uninitialized ID into -1, so it will not match any possible lock owner - currentThreadID |= (currentThreadID - 1) >> 31; + Debug.Assert(obj != null); Lock fatLock; fixed (MethodTable** ppMethodTable = &obj.GetMethodTableRef()) { int* pHeader = GetHeaderPtr(ppMethodTable); + + // We may need to retry if we own the lock but the CAS races with another thread + // touching unrelated header bits. while (true) { int oldBits = *pHeader; - - // if we own the lock - if ((oldBits & SBLK_MASK_LOCK_THREADID) == currentThreadID && - (oldBits & BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX) == 0) + // is the lock thin? + if ((oldBits & (BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX)) == 0) { - // decrement count or release entirely. - int newBits = (oldBits & SBLK_MASK_LOCK_RECLEVEL) != 0 ? - oldBits - SBLK_LOCK_RECLEVEL_INC : - oldBits & ~SBLK_MASK_LOCK_THREADID; + int currentThreadID = ManagedThreadId.CurrentManagedThreadIdUnchecked; + // transform uninitialized ID into -1, so it will not match any possible lock owner + currentThreadID |= (currentThreadID - 1) >> 31; - if (Interlocked.CompareExchange(pHeader, newBits, oldBits) == oldBits) + // do we own the thin lock? + if ((oldBits & SBLK_MASK_LOCK_THREADID) == currentThreadID) { - return; - } + // decrement count or release entirely. + int newBits = (oldBits & SBLK_MASK_LOCK_RECLEVEL) != 0 ? + oldBits - SBLK_LOCK_RECLEVEL_INC : + oldBits & ~SBLK_MASK_LOCK_THREADID; - // rare contention on owned lock, - // we still own the lock, try again - continue; + if (Interlocked.CompareExchange(pHeader, newBits, oldBits) == oldBits) + { + return; + } + + // rare contention on owned lock, + // we still own the lock, try again + continue; + } } if (!GetSyncEntryIndex(oldBits, out int syncIndex)) @@ -474,12 +530,13 @@ public static unsafe void Release(object obj) throw new SynchronizationLockException(); } + // Get the fat lock. Must be done while still pinning the obj. fatLock = SyncTable.GetLockObject(syncIndex); break; } } - fatLock.Exit(currentThreadID); + fatLock.Exit(); } [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/src/coreclr/pal/inc/pal.h b/src/coreclr/pal/inc/pal.h index 2abfc57d0d3f3e..970d29fc769d78 100644 --- a/src/coreclr/pal/inc/pal.h +++ b/src/coreclr/pal/inc/pal.h @@ -3135,9 +3135,6 @@ Define_InterlockMethod( Exchange /* The value to be stored */) ) -#define InterlockedCompareExchangeAcquire InterlockedCompareExchange -#define InterlockedCompareExchangeRelease InterlockedCompareExchange - Define_InterlockMethod( LONGLONG, InterlockedCompareExchange64(IN OUT LONGLONG volatile *Destination, IN LONGLONG Exchange, IN LONGLONG Comperand), diff --git a/src/coreclr/vm/CMakeLists.txt b/src/coreclr/vm/CMakeLists.txt index bd14df4d4364f1..069ec7480f93d1 100644 --- a/src/coreclr/vm/CMakeLists.txt +++ b/src/coreclr/vm/CMakeLists.txt @@ -229,7 +229,6 @@ set(VM_HEADERS_DAC_AND_WKS_COMMON stublink.inl stubmgr.h syncblk.h - syncblk.inl threads.h threads.inl threadstatics.h diff --git a/src/coreclr/vm/common.h b/src/coreclr/vm/common.h index 61a7a486717027..756ef7e0817fbb 100644 --- a/src/coreclr/vm/common.h +++ b/src/coreclr/vm/common.h @@ -375,7 +375,6 @@ extern DummyGlobalContract ___contract; #include "object.inl" #include "clsload.inl" #include "method.inl" -#include "syncblk.inl" #include "threads.inl" #include "eehash.inl" #include "eventtrace.inl" diff --git a/src/coreclr/vm/comsynchronizable.cpp b/src/coreclr/vm/comsynchronizable.cpp index 09a611e7e3a2cf..c0da0b4245a5da 100644 --- a/src/coreclr/vm/comsynchronizable.cpp +++ b/src/coreclr/vm/comsynchronizable.cpp @@ -860,22 +860,6 @@ extern "C" void QCALLTYPE Monitor_GetOrCreateLockObject(QCall::ObjectHandleOnSta END_QCALL; } -FCIMPL1(ObjHeader::HeaderLockResult, ObjHeader_AcquireThinLock, Object* obj) -{ - FCALL_CONTRACT; - - return obj->GetHeader()->AcquireHeaderThinLock(GetThread()); -} -FCIMPLEND - -FCIMPL1(ObjHeader::HeaderLockResult, ObjHeader_ReleaseThinLock, Object* obj) -{ - FCALL_CONTRACT; - - return obj->GetHeader()->ReleaseHeaderThinLock(GetThread()); -} -FCIMPLEND - extern "C" INT32 QCALLTYPE ThreadNative_ReentrantWaitAny(BOOL alertable, INT32 timeout, INT32 count, HANDLE *handles) { QCALL_CONTRACT; diff --git a/src/coreclr/vm/comsynchronizable.h b/src/coreclr/vm/comsynchronizable.h index 7c23549fb4cde6..91a761c64a07b4 100644 --- a/src/coreclr/vm/comsynchronizable.h +++ b/src/coreclr/vm/comsynchronizable.h @@ -83,8 +83,5 @@ extern "C" void QCALLTYPE ThreadNative_DisableComObjectEagerCleanup(QCall::Threa extern "C" void QCALLTYPE Monitor_GetOrCreateLockObject(QCall::ObjectHandleOnStack obj, QCall::ObjectHandleOnStack lockObj); FCDECL1(OBJECTHANDLE, Monitor_GetLockHandleIfExists, Object* obj); - -FCDECL1(ObjHeader::HeaderLockResult, ObjHeader_AcquireThinLock, Object* obj); -FCDECL1(ObjHeader::HeaderLockResult, ObjHeader_ReleaseThinLock, Object* obj); #endif // _COMSYNCHRONIZABLE_H diff --git a/src/coreclr/vm/ecalllist.h b/src/coreclr/vm/ecalllist.h index a32d6969f32fbb..a23ae9e8d6a788 100644 --- a/src/coreclr/vm/ecalllist.h +++ b/src/coreclr/vm/ecalllist.h @@ -260,11 +260,6 @@ FCFuncStart(gThreadFuncs) FCFuncElement("get_OptimalMaxSpinWaitsPerSpinIteration", ThreadNative::GetOptimalMaxSpinWaitsPerSpinIteration) FCFuncEnd() -FCFuncStart(gObjectHeaderFuncs) - FCFuncElement("AcquireInternal", ObjHeader_AcquireThinLock) - FCFuncElement("Release", ObjHeader_ReleaseThinLock) -FCFuncEnd() - FCFuncStart(gMonitorFuncs) FCFuncElement("GetLockHandleIfExists", Monitor_GetLockHandleIfExists) FCFuncEnd() @@ -408,9 +403,6 @@ FCClassElement("MathF", "System", gMathFFuncs) FCClassElement("MetadataImport", "System.Reflection", gMetaDataImport) FCClassElement("MethodTable", "System.Runtime.CompilerServices", gMethodTableFuncs) FCClassElement("Monitor", "System.Threading", gMonitorFuncs) - -FCClassElement("ObjectHeader", "System.Threading", gObjectHeaderFuncs) - FCClassElement("RuntimeAssembly", "System.Reflection", gRuntimeAssemblyFuncs) FCClassElement("RuntimeFieldHandle", "System", gCOMFieldHandleNewFuncs) FCClassElement("RuntimeHelpers", "System.Runtime.CompilerServices", gRuntimeHelpers) diff --git a/src/coreclr/vm/syncblk.h b/src/coreclr/vm/syncblk.h index 238416cc583568..14fbbda2666683 100644 --- a/src/coreclr/vm/syncblk.h +++ b/src/coreclr/vm/syncblk.h @@ -912,17 +912,6 @@ class ObjHeader BOOL Validate (BOOL bVerifySyncBlkIndex = TRUE); - // These must match the values in ObjectHeader.CoreCLR.cs - enum class HeaderLockResult : int32_t { - Success = 0, - Failure = 1, - UseSlowPath = 2 - }; - - HeaderLockResult AcquireHeaderThinLock(Thread* pCurThread); - - HeaderLockResult ReleaseHeaderThinLock(Thread* pCurThread); - friend struct ::cdac_data; }; diff --git a/src/coreclr/vm/syncblk.inl b/src/coreclr/vm/syncblk.inl deleted file mode 100644 index da52c6af15c7b8..00000000000000 --- a/src/coreclr/vm/syncblk.inl +++ /dev/null @@ -1,135 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -#ifndef _SYNCBLK_INL_ -#define _SYNCBLK_INL_ - -FORCEINLINE ObjHeader::HeaderLockResult ObjHeader::AcquireHeaderThinLock(Thread* pCurThread) -{ - CONTRACTL - { - NOTHROW; - GC_NOTRIGGER; - MODE_COOPERATIVE; - } CONTRACTL_END; - - LONG oldValue = m_SyncBlockValue.LoadWithoutBarrier(); - - if ((oldValue & (BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX + - BIT_SBLK_SPIN_LOCK + - SBLK_MASK_LOCK_THREADID + - SBLK_MASK_LOCK_RECLEVEL)) == 0) - { - DWORD tid = pCurThread->GetThreadId(); - if (tid > SBLK_MASK_LOCK_THREADID) - { - return HeaderLockResult::UseSlowPath; - } - - LONG newValue = oldValue | tid; -#if defined(TARGET_WINDOWS) && defined(TARGET_ARM64) - if (FastInterlockedCompareExchangeAcquire((LONG*)&m_SyncBlockValue, newValue, oldValue) == oldValue) -#else - if (InterlockedCompareExchangeAcquire((LONG*)&m_SyncBlockValue, newValue, oldValue) == oldValue) -#endif - { - return HeaderLockResult::Success; - } - - return HeaderLockResult::Failure; - } - - // The header either has a sync block, will need a sync block to represent the lock - // or is in the process of transitioning to a sync block. In any of these cases, we need to take the slow path. - if (oldValue & (BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX | BIT_SBLK_SPIN_LOCK)) - { - return HeaderLockResult::UseSlowPath; - } - - // Here we know we have the "thin lock" layout, but the lock is not free. - // It could still be the recursion case - compare the thread id to check - if (pCurThread->GetThreadId() != (DWORD)(oldValue & SBLK_MASK_LOCK_THREADID)) - { - return HeaderLockResult::Failure; - } - - // Ok, the thread id matches, it's the recursion case. - // Bump up the recursion level and check for overflow - LONG newValue = oldValue + SBLK_LOCK_RECLEVEL_INC; - - if ((newValue & SBLK_MASK_LOCK_RECLEVEL) == 0) - { - return HeaderLockResult::UseSlowPath; - } - -#if defined(TARGET_WINDOWS) && defined(TARGET_ARM64) - if (FastInterlockedCompareExchangeAcquire((LONG*)&m_SyncBlockValue, newValue, oldValue) == oldValue) -#else - if (InterlockedCompareExchangeAcquire((LONG*)&m_SyncBlockValue, newValue, oldValue) == oldValue) -#endif - { - return HeaderLockResult::Success; - } - - // We failed to acquire because someone touched other bits in the header. - return HeaderLockResult::Failure; -} - -// Helper encapsulating the core logic for releasing monitor. Returns what kind of -// follow up action is necessary. This is FORCEINLINE to make it provide a very efficient implementation. -FORCEINLINE ObjHeader::HeaderLockResult ObjHeader::ReleaseHeaderThinLock(Thread* pCurThread) -{ - CONTRACTL { - NOTHROW; - GC_NOTRIGGER; - MODE_COOPERATIVE; - } CONTRACTL_END; - - DWORD syncBlockValue = m_SyncBlockValue.LoadWithoutBarrier(); - - if ((syncBlockValue & (BIT_SBLK_SPIN_LOCK + BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX)) == 0) - { - DWORD tid = pCurThread->GetThreadId(); - - if ((syncBlockValue & SBLK_MASK_LOCK_THREADID) != tid) - { - // This thread does not own the lock. - // We don't need to check if the thread id could be stored in the lock - // as we know we are in the thin lock case. - // The lock is definitely not being held by this thread. - return HeaderLockResult::Failure; - } - - if (!(syncBlockValue & SBLK_MASK_LOCK_RECLEVEL)) - { - // We are leaving the lock - DWORD newValue = (syncBlockValue & (~SBLK_MASK_LOCK_THREADID)); - -#if defined(TARGET_WINDOWS) && defined(TARGET_ARM64) - if (FastInterlockedCompareExchangeRelease((LONG*)&m_SyncBlockValue, newValue, syncBlockValue) == (LONG)syncBlockValue) -#else - if (InterlockedCompareExchangeRelease((LONG*)&m_SyncBlockValue, newValue, syncBlockValue) == (LONG)syncBlockValue) -#endif - { - return HeaderLockResult::Success; - } - } - else - { - // recursion and ThinLock - DWORD newValue = syncBlockValue - SBLK_LOCK_RECLEVEL_INC; -#if defined(TARGET_WINDOWS) && defined(TARGET_ARM64) - if (FastInterlockedCompareExchangeRelease((LONG*)&m_SyncBlockValue, newValue, syncBlockValue) == (LONG)syncBlockValue) -#else - if (InterlockedCompareExchangeRelease((LONG*)&m_SyncBlockValue, newValue, syncBlockValue) == (LONG)syncBlockValue) -#endif - { - return HeaderLockResult::Success; - } - } - } - - return HeaderLockResult::UseSlowPath; -} - -#endif // _SYNCBLK_INL_ diff --git a/src/coreclr/vm/util.hpp b/src/coreclr/vm/util.hpp index 1f09735dfff4fb..38ef31fa49eb65 100644 --- a/src/coreclr/vm/util.hpp +++ b/src/coreclr/vm/util.hpp @@ -68,66 +68,6 @@ BOOL inline FitsInU4(uint64_t val) return val == (uint64_t)(uint32_t)val; } -#if defined(DACCESS_COMPILE) -#define FastInterlockedCompareExchange InterlockedCompareExchange -#define FastInterlockedCompareExchangeAcquire InterlockedCompareExchangeAcquire -#define FastInterlockedCompareExchangeRelease InterlockedCompareExchangeRelease -#else - -#if defined(TARGET_WINDOWS) && defined(TARGET_ARM64) - -FORCEINLINE LONG FastInterlockedCompareExchange( - LONG volatile *Destination, - LONG Exchange, - LONG Comperand) -{ - if (g_arm64_atomics_present) - { - return (LONG) __casal32((unsigned __int32*) Destination, (unsigned __int32)Comperand, (unsigned __int32)Exchange); - } - else - { - return InterlockedCompareExchange(Destination, Exchange, Comperand); - } -} - -FORCEINLINE LONG FastInterlockedCompareExchangeAcquire( - IN OUT LONG volatile *Destination, - IN LONG Exchange, - IN LONG Comperand -) -{ - if (g_arm64_atomics_present) - { - return (LONG) __casa32((unsigned __int32*) Destination, (unsigned __int32)Comperand, (unsigned __int32)Exchange); - } - else - { - return InterlockedCompareExchangeAcquire(Destination, Exchange, Comperand); - } -} - -FORCEINLINE LONG FastInterlockedCompareExchangeRelease( - IN OUT LONG volatile *Destination, - IN LONG Exchange, - IN LONG Comperand -) -{ - if (g_arm64_atomics_present) - { - return (LONG) __casl32((unsigned __int32*) Destination, (unsigned __int32)Comperand, (unsigned __int32)Exchange); - } - else - { - return InterlockedCompareExchangeRelease(Destination, Exchange, Comperand); - } -} - -#endif // defined(TARGET_WINDOWS) && defined(TARGET_ARM64) - -#endif //defined(DACCESS_COMPILE) - - //************************************************************************ // CQuickHeap // diff --git a/src/libraries/System.Private.CoreLib/src/System/Threading/Lock.cs b/src/libraries/System.Private.CoreLib/src/System/Threading/Lock.cs index 7ea7244e1f99fa..436f08d1bbc203 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Threading/Lock.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Threading/Lock.cs @@ -272,7 +272,7 @@ public bool TryEnter(int millisecondsTimeout) public bool TryEnter(TimeSpan timeout) => TryEnter_Outlined(WaitHandle.ToTimeoutMilliseconds(timeout)); [MethodImpl(MethodImplOptions.NoInlining)] - private bool TryEnter_Outlined(int timeoutMs) => TryEnter_Inlined(timeoutMs) != UninitializedThreadId; + internal bool TryEnter_Outlined(int timeoutMs) => TryEnter_Inlined(timeoutMs) != UninitializedThreadId; [MethodImpl(MethodImplOptions.AggressiveInlining)] private int TryEnter_Inlined(int timeoutMs)