diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Max.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Max.cs index 8e42381c48e165..2f16e8b6b8d4f2 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Max.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Max.cs @@ -258,7 +258,7 @@ private static T MinMaxCore(ReadOnlySpan x) nanMask = IsNaN(result); if (nanMask != Vector512.Zero) { - return result.GetElement(IndexOfFirstMatch(nanMask)); + return result.GetElement(Vector512.IndexOfWhereAllBitsSet(nanMask)); } } @@ -277,7 +277,7 @@ private static T MinMaxCore(ReadOnlySpan x) nanMask = ~Vector512.Equals(current, current); if (nanMask != Vector512.Zero) { - return current.GetElement(IndexOfFirstMatch(nanMask)); + return current.GetElement(Vector512.IndexOfWhereAllBitsSet(nanMask)); } } @@ -296,7 +296,7 @@ private static T MinMaxCore(ReadOnlySpan x) nanMask = ~Vector512.Equals(current, current); if (nanMask != Vector512.Zero) { - return current.GetElement(IndexOfFirstMatch(nanMask)); + return current.GetElement(Vector512.IndexOfWhereAllBitsSet(nanMask)); } } @@ -323,7 +323,7 @@ private static T MinMaxCore(ReadOnlySpan x) nanMask = ~Vector256.Equals(result, result); if (nanMask != Vector256.Zero) { - return result.GetElement(IndexOfFirstMatch(nanMask)); + return result.GetElement(Vector256.IndexOfWhereAllBitsSet(nanMask)); } } @@ -342,7 +342,7 @@ private static T MinMaxCore(ReadOnlySpan x) nanMask = ~Vector256.Equals(current, current); if (nanMask != Vector256.Zero) { - return current.GetElement(IndexOfFirstMatch(nanMask)); + return current.GetElement(Vector256.IndexOfWhereAllBitsSet(nanMask)); } } @@ -362,7 +362,7 @@ private static T MinMaxCore(ReadOnlySpan x) nanMask = ~Vector256.Equals(current, current); if (nanMask != Vector256.Zero) { - return current.GetElement(IndexOfFirstMatch(nanMask)); + return current.GetElement(Vector256.IndexOfWhereAllBitsSet(nanMask)); } } @@ -389,7 +389,7 @@ private static T MinMaxCore(ReadOnlySpan x) nanMask = IsNaN(result); if (nanMask != Vector128.Zero) { - return result.GetElement(IndexOfFirstMatch(nanMask)); + return result.GetElement(Vector128.IndexOfWhereAllBitsSet(nanMask)); } } @@ -408,7 +408,7 @@ private static T MinMaxCore(ReadOnlySpan x) nanMask = IsNaN(current); if (nanMask != Vector128.Zero) { - return current.GetElement(IndexOfFirstMatch(nanMask)); + return current.GetElement(Vector128.IndexOfWhereAllBitsSet(nanMask)); } } @@ -427,7 +427,7 @@ private static T MinMaxCore(ReadOnlySpan x) nanMask = IsNaN(current); if (nanMask != Vector128.Zero) { - return current.GetElement(IndexOfFirstMatch(nanMask)); + return current.GetElement(Vector128.IndexOfWhereAllBitsSet(nanMask)); } } diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs index 31f9ba48151b8c..62c796b9d88f9b 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs @@ -530,8 +530,8 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) Vector128 search = Vector128.Load(searchSpace + offset); // Same method as below - uint matches = Vector128.Equals(Vector128.Zero, search).ExtractMostSignificantBits(); - if (matches == 0) + Vector128 cmp = Vector128.Equals(Vector128.Zero, search); + if (cmp == Vector128.Zero) { // Zero flags set so no matches offset += (nuint)Vector128.Count; @@ -539,7 +539,7 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) else { // Find bitflag offset of first match and add to current offset - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); + return (int)(offset + (uint)Vector128.IndexOfFirstMatch(cmp)); } } @@ -552,8 +552,8 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) Vector256 search = Vector256.Load(searchSpace + offset); // Same method as below - uint matches = Vector256.Equals(Vector256.Zero, search).ExtractMostSignificantBits(); - if (matches == 0) + Vector256 cmp = Vector256.Equals(Vector256.Zero, search); + if (cmp == Vector256.Zero) { // Zero flags set so no matches offset += (nuint)Vector256.Count; @@ -561,7 +561,7 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) else { // Find bitflag offset of first match and add to current offset - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); + return (int)(offset + (uint)Vector256.IndexOfFirstMatch(cmp)); } } lengthToExamine = GetByteVector512SpanLength(offset, Length); @@ -570,10 +570,8 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) do { Vector512 search = Vector512.Load(searchSpace + offset); - ulong matches = Vector512.Equals(Vector512.Zero, search).ExtractMostSignificantBits(); - // Note that MoveMask has converted the equal vector elements into a set of bit flags, - // So the bit position in 'matches' corresponds to the element offset. - if (matches == 0) + Vector512 cmp = Vector512.Equals(Vector512.Zero, search); + if (cmp == Vector512.Zero) { // Zero flags set so no matches offset += (nuint)Vector512.Count; @@ -581,7 +579,7 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) } // Find bitflag offset of first match and add to current offset - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); + return (int)(offset + (uint)Vector512.IndexOfFirstMatch(cmp)); } while (lengthToExamine > offset); } @@ -591,8 +589,8 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) Vector256 search = Vector256.Load(searchSpace + offset); // Same method as above - uint matches = Vector256.Equals(Vector256.Zero, search).ExtractMostSignificantBits(); - if (matches == 0) + Vector256 cmp = Vector256.Equals(Vector256.Zero, search); + if (cmp == Vector256.Zero) { // Zero flags set so no matches offset += (nuint)Vector256.Count; @@ -600,7 +598,7 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) else { // Find bitflag offset of first match and add to current offset - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); + return (int)(offset + (uint)Vector256.IndexOfFirstMatch(cmp)); } } @@ -610,8 +608,8 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) Vector128 search = Vector128.Load(searchSpace + offset); // Same method as above - uint matches = Vector128.Equals(Vector128.Zero, search).ExtractMostSignificantBits(); - if (matches == 0) + Vector128 cmp = Vector128.Equals(Vector128.Zero, search); + if (cmp == Vector128.Zero) { // Zero flags set so no matches offset += (nuint)Vector128.Count; @@ -619,7 +617,7 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) else { // Find bitflag offset of first match and add to current offset - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); + return (int)(offset + (uint)Vector128.IndexOfFirstMatch(cmp)); } } @@ -643,8 +641,8 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) Vector128 search = Vector128.Load(searchSpace + offset); // Same method as below - uint matches = Vector128.Equals(Vector128.Zero, search).ExtractMostSignificantBits(); - if (matches == 0) + Vector128 cmp = Vector128.Equals(Vector128.Zero, search); + if (cmp == Vector128.Zero) { // Zero flags set so no matches offset += (nuint)Vector128.Count; @@ -652,7 +650,7 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) else { // Find bitflag offset of first match and add to current offset - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); + return (int)(offset + (uint)Vector128.IndexOfFirstMatch(cmp)); } } @@ -662,10 +660,8 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) do { Vector256 search = Vector256.Load(searchSpace + offset); - uint matches = Vector256.Equals(Vector256.Zero, search).ExtractMostSignificantBits(); - // Note that MoveMask has converted the equal vector elements into a set of bit flags, - // So the bit position in 'matches' corresponds to the element offset. - if (matches == 0) + Vector256 cmp = Vector256.Equals(Vector256.Zero, search); + if (cmp == Vector256.Zero) { // Zero flags set so no matches offset += (nuint)Vector256.Count; @@ -673,7 +669,7 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) } // Find bitflag offset of first match and add to current offset - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); + return (int)(offset + (uint)Vector256.IndexOfFirstMatch(cmp)); } while (lengthToExamine > offset); } @@ -683,8 +679,8 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) Vector128 search = Vector128.Load(searchSpace + offset); // Same method as above - uint matches = Vector128.Equals(Vector128.Zero, search).ExtractMostSignificantBits(); - if (matches == 0) + Vector128 cmp = Vector128.Equals(Vector128.Zero, search); + if (cmp == Vector128.Zero) { // Zero flags set so no matches offset += (nuint)Vector128.Count; @@ -692,7 +688,7 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) else { // Find bitflag offset of first match and add to current offset - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); + return (int)(offset + (uint)Vector128.IndexOfFirstMatch(cmp)); } } @@ -723,8 +719,7 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) } // Find bitflag offset of first match and add to current offset - uint matches = compareResult.ExtractMostSignificantBits(); - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); + return (int)(offset + (uint)Vector128.IndexOfFirstMatch(compareResult)); } if (offset < (nuint)(uint)Length) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs index 8e53a21801ec29..7f562e59d626aa 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs @@ -3713,49 +3713,37 @@ private static int LastIndexOfAnyValueType(ref TValue searchSp [MethodImpl(MethodImplOptions.AggressiveInlining)] private static unsafe int ComputeFirstIndex(ref T searchSpace, ref T current, Vector128 equals) where T : struct { - uint notEqualsElements = equals.ExtractMostSignificantBits(); - int index = BitOperations.TrailingZeroCount(notEqualsElements); - return index + (int)((nuint)Unsafe.ByteOffset(ref searchSpace, ref current) / (nuint)sizeof(T)); + return Vector128.IndexOfFirstMatch(equals) + (int)((nuint)Unsafe.ByteOffset(ref searchSpace, ref current) / (nuint)sizeof(T)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static unsafe int ComputeFirstIndex(ref T searchSpace, ref T current, Vector256 equals) where T : struct { - uint notEqualsElements = equals.ExtractMostSignificantBits(); - int index = BitOperations.TrailingZeroCount(notEqualsElements); - return index + (int)((nuint)Unsafe.ByteOffset(ref searchSpace, ref current) / (nuint)sizeof(T)); + return Vector256.IndexOfFirstMatch(equals) + (int)((nuint)Unsafe.ByteOffset(ref searchSpace, ref current) / (nuint)sizeof(T)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static unsafe int ComputeFirstIndex(ref T searchSpace, ref T current, Vector512 equals) where T : struct { - ulong notEqualsElements = equals.ExtractMostSignificantBits(); - int index = BitOperations.TrailingZeroCount(notEqualsElements); - return index + (int)((nuint)Unsafe.ByteOffset(ref searchSpace, ref current) / (nuint)sizeof(T)); + return Vector512.IndexOfFirstMatch(equals) + (int)((nuint)Unsafe.ByteOffset(ref searchSpace, ref current) / (nuint)sizeof(T)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int ComputeLastIndex(nint offset, Vector128 equals) where T : struct { - uint notEqualsElements = equals.ExtractMostSignificantBits(); - int index = 31 - BitOperations.LeadingZeroCount(notEqualsElements); // 31 = 32 (bits in Int32) - 1 (indexing from zero) - return (int)offset + index; + return (int)offset + Vector128.IndexOfLastMatch(equals); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int ComputeLastIndex(nint offset, Vector256 equals) where T : struct { - uint notEqualsElements = equals.ExtractMostSignificantBits(); - int index = 31 - BitOperations.LeadingZeroCount(notEqualsElements); // 31 = 32 (bits in Int32) - 1 (indexing from zero) - return (int)offset + index; + return (int)offset + Vector256.IndexOfLastMatch(equals); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int ComputeLastIndex(nint offset, Vector512 equals) where T : struct { - ulong notEqualsElements = equals.ExtractMostSignificantBits(); - int index = 63 - BitOperations.LeadingZeroCount(notEqualsElements); // 31 = 32 (bits in Int32) - 1 (indexing from zero) - return (int)offset + index; + return (int)offset + Vector512.IndexOfLastMatch(equals); } internal interface INegator where T : struct @@ -4176,7 +4164,7 @@ public static unsafe int CountValueType(ref T current, T value, int length) w ref T oneVectorAwayFromEnd = ref Unsafe.Subtract(ref end, Vector512.Count); while (Unsafe.IsAddressLessThan(ref current, ref oneVectorAwayFromEnd)) { - count += BitOperations.PopCount(Vector512.Equals(Vector512.LoadUnsafe(ref current), targetVector).ExtractMostSignificantBits()); + count += Vector512.CountMatches(Vector512.Equals(Vector512.LoadUnsafe(ref current), targetVector)); current = ref Unsafe.Add(ref current, Vector512.Count); } @@ -4191,7 +4179,7 @@ public static unsafe int CountValueType(ref T current, T value, int length) w ref T oneVectorAwayFromEnd = ref Unsafe.Subtract(ref end, Vector256.Count); while (Unsafe.IsAddressLessThan(ref current, ref oneVectorAwayFromEnd)) { - count += BitOperations.PopCount(Vector256.Equals(Vector256.LoadUnsafe(ref current), targetVector).ExtractMostSignificantBits()); + count += Vector256.CountMatches(Vector256.Equals(Vector256.LoadUnsafe(ref current), targetVector)); current = ref Unsafe.Add(ref current, Vector256.Count); } @@ -4206,7 +4194,7 @@ public static unsafe int CountValueType(ref T current, T value, int length) w ref T oneVectorAwayFromEnd = ref Unsafe.Subtract(ref end, Vector128.Count); while (Unsafe.IsAddressLessThan(ref current, ref oneVectorAwayFromEnd)) { - count += BitOperations.PopCount(Vector128.Equals(Vector128.LoadUnsafe(ref current), targetVector).ExtractMostSignificantBits()); + count += Vector128.CountMatches(Vector128.Equals(Vector128.LoadUnsafe(ref current), targetVector)); current = ref Unsafe.Add(ref current, Vector128.Count); }