Skip to content

Commit a8ef873

Browse files
Gnbrkm41Gnbrkm41
authored
Implement Vector{Size}<T>.AllBitsSet (#33924)
1 parent 00c99a7 commit a8ef873

65 files changed

Lines changed: 3625 additions & 1091 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎src/coreclr/src/jit/hwintrinsic.h‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -659,9 +659,11 @@ struct HWIntrinsic final
659659
op1 = node->gtGetOp1();
660660
op2 = node->gtGetOp2();
661661

662-
assert(op1 != nullptr);
663-
664-
if (op1->OperIsList())
662+
if (op1 == nullptr)
663+
{
664+
numOperands = 0;
665+
}
666+
else if (op1->OperIsList())
665667
{
666668
assert(op2 == nullptr);
667669

‎src/coreclr/src/jit/hwintrinsicarm64.cpp‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -281,7 +281,9 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic,
281281

282282
if (!varTypeIsArithmetic(baseType))
283283
{
284-
assert((intrinsic == NI_Vector64_AsByte) || (intrinsic == NI_Vector128_As));
284+
assert((intrinsic == NI_Vector64_AsByte) || (intrinsic == NI_Vector128_As) ||
285+
(intrinsic == NI_Vector64_get_Zero) || (intrinsic == NI_Vector64_get_AllBitsSet) ||
286+
(intrinsic == NI_Vector128_get_Zero) || (intrinsic == NI_Vector128_get_AllBitsSet));
285287
return nullptr;
286288
}
287289
}
@@ -363,7 +365,16 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic,
363365
retNode = countNode;
364366
break;
365367
}
368+
case NI_Vector64_get_Zero:
369+
case NI_Vector64_get_AllBitsSet:
370+
case NI_Vector128_get_Zero:
371+
case NI_Vector128_get_AllBitsSet:
372+
{
373+
assert(!sig->hasThis());
374+
assert(numArgs == 0);
366375

376+
retNode = gtNewSimdHWIntrinsicNode(retType, intrinsic, baseType, simdSize);
377+
}
367378
default:
368379
{
369380
return nullptr;

‎src/coreclr/src/jit/hwintrinsiccodegenarm64.cpp‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,10 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
194194
op1Reg = intrin.op1->GetRegNum();
195195
break;
196196

197+
case 0:
198+
assert(HWIntrinsicInfo::lookupNumArgs(intrin.id) == 0);
199+
break;
200+
197201
default:
198202
unreached();
199203
}
@@ -268,7 +272,6 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
268272
else
269273
{
270274
instruction ins = INS_invalid;
271-
272275
switch (intrin.id)
273276
{
274277
case NI_Crc32_ComputeCrc32:
@@ -484,6 +487,17 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
484487
}
485488
break;
486489

490+
// mvni doesn't support the range of element types, so hard code the 'opts' value.
491+
case NI_Vector64_get_Zero:
492+
case NI_Vector64_get_AllBitsSet:
493+
GetEmitter()->emitIns_R_I(ins, emitSize, targetReg, 0, INS_OPTS_2S);
494+
break;
495+
496+
case NI_Vector128_get_Zero:
497+
case NI_Vector128_get_AllBitsSet:
498+
GetEmitter()->emitIns_R_I(ins, emitSize, targetReg, 0, INS_OPTS_4S);
499+
break;
500+
487501
default:
488502
unreached();
489503
}

‎src/coreclr/src/jit/hwintrinsiccodegenxarch.cpp‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1243,6 +1243,24 @@ void CodeGen::genBaseIntrinsic(GenTreeHWIntrinsic* node)
12431243
break;
12441244
}
12451245

1246+
case NI_Vector128_get_AllBitsSet:
1247+
case NI_Vector256_get_AllBitsSet:
1248+
{
1249+
assert(op1 == nullptr);
1250+
if (varTypeIsFloating(baseType) && compiler->compOpportunisticallyDependsOn(InstructionSet_AVX))
1251+
{
1252+
// The immediate 8 means Equal (unordered, non-signaling)
1253+
// This is not available without VEX prefix.
1254+
emit->emitIns_SIMD_R_R_R_I(ins, attr, targetReg, targetReg, targetReg, 8);
1255+
}
1256+
else
1257+
{
1258+
assert(varTypeIsIntegral(baseType) || !compiler->compIsaSupportedDebugOnly(InstructionSet_AVX));
1259+
emit->emitIns_SIMD_R_R_R(INS_pcmpeqd, attr, targetReg, targetReg, targetReg);
1260+
}
1261+
break;
1262+
}
1263+
12461264
default:
12471265
{
12481266
unreached();

‎src/coreclr/src/jit/hwintrinsiclistarm64.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,9 @@ HARDWARE_INTRINSIC(Vector64, AsSingle,
2424
HARDWARE_INTRINSIC(Vector64, AsUInt16, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg)
2525
HARDWARE_INTRINSIC(Vector64, AsUInt32, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg)
2626
HARDWARE_INTRINSIC(Vector64, CreateScalarUnsafe, 8, 1, {INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_invalid, INS_invalid, INS_fmov, INS_invalid}, HW_Category_SimpleSIMD, HW_Flag_SupportsContainment|HW_Flag_SpecialCodeGen)
27+
HARDWARE_INTRINSIC(Vector64, get_AllBitsSet, 8, 0, {INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen)
2728
HARDWARE_INTRINSIC(Vector64, get_Count, 8, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen)
29+
HARDWARE_INTRINSIC(Vector64, get_Zero, 8, 0, {INS_movi, INS_movi, INS_movi, INS_movi, INS_movi, INS_movi, INS_movi, INS_movi, INS_movi, INS_movi}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen)
2830

2931
// ***************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
3032
// ISA Function name SIMD size NumArg Instructions Category Flags
@@ -43,7 +45,9 @@ HARDWARE_INTRINSIC(Vector128, AsUInt16, 1
4345
HARDWARE_INTRINSIC(Vector128, AsUInt32, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg)
4446
HARDWARE_INTRINSIC(Vector128, AsUInt64, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg)
4547
HARDWARE_INTRINSIC(Vector128, CreateScalarUnsafe, 16, 1, {INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_fmov, INS_fmov}, HW_Category_SimpleSIMD, HW_Flag_SupportsContainment|HW_Flag_SpecialCodeGen)
48+
HARDWARE_INTRINSIC(Vector128, get_AllBitsSet, 16, 0, {INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni, INS_mvni}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen)
4649
HARDWARE_INTRINSIC(Vector128, get_Count, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen)
50+
HARDWARE_INTRINSIC(Vector128, get_Zero, 16, 0, {INS_movi, INS_movi, INS_movi, INS_movi, INS_movi, INS_movi, INS_movi, INS_movi, INS_movi, INS_movi}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen)
4751

4852
// ***************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
4953
// ISA Function name SIMD size NumArg Instructions Category Flags

‎src/coreclr/src/jit/hwintrinsiclistxarch.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ HARDWARE_INTRINSIC(Vector128, AsVector3,
4444
HARDWARE_INTRINSIC(Vector128, AsVector4, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics)
4545
HARDWARE_INTRINSIC(Vector128, AsVector128, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics)
4646
HARDWARE_INTRINSIC(Vector128, CreateScalarUnsafe, 16, 1, {INS_mov_i2xmm, INS_mov_i2xmm, INS_mov_i2xmm, INS_mov_i2xmm, INS_mov_i2xmm, INS_mov_i2xmm, INS_mov_i2xmm, INS_mov_i2xmm, INS_movss, INS_movsdsse2}, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics)
47+
// The instruction generated for float/double depends on which ISAs are supported
48+
HARDWARE_INTRINSIC(Vector128, get_AllBitsSet, 16, 0, {INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqd, INS_cmpps, INS_cmppd}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_NoRMWSemantics)
4749
HARDWARE_INTRINSIC(Vector128, get_Count, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_NoRMWSemantics)
4850
HARDWARE_INTRINSIC(Vector128, get_Zero, 16, 0, {INS_xorps, INS_xorps, INS_xorps, INS_xorps, INS_xorps, INS_xorps, INS_xorps, INS_xorps, INS_xorps, INS_xorps}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_NoRMWSemantics)
4951
HARDWARE_INTRINSIC(Vector128, GetElement, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_BaseTypeFromFirstArg)
@@ -70,6 +72,8 @@ HARDWARE_INTRINSIC(Vector256, AsUInt32,
7072
HARDWARE_INTRINSIC(Vector256, AsUInt64, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics)
7173
HARDWARE_INTRINSIC(Vector256, AsVector, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics)
7274
HARDWARE_INTRINSIC(Vector256, AsVector256, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics)
75+
// The instruction generated for float/double depends on which ISAs are supported
76+
HARDWARE_INTRINSIC(Vector256, get_AllBitsSet, 32, 0, {INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqd, INS_cmpps, INS_cmppd}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_NoRMWSemantics)
7377
HARDWARE_INTRINSIC(Vector256, get_Count, 32, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_NoRMWSemantics)
7478
HARDWARE_INTRINSIC(Vector256, get_Zero, 32, 0, {INS_xorps, INS_xorps, INS_xorps, INS_xorps, INS_xorps, INS_xorps, INS_xorps, INS_xorps, INS_xorps, INS_xorps}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_NoRMWSemantics)
7579
HARDWARE_INTRINSIC(Vector256, CreateScalarUnsafe, 32, 1, {INS_mov_i2xmm, INS_mov_i2xmm, INS_mov_i2xmm, INS_mov_i2xmm, INS_mov_i2xmm, INS_mov_i2xmm, INS_mov_i2xmm, INS_mov_i2xmm, INS_movss, INS_movsdsse2}, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics)

‎src/coreclr/src/jit/hwintrinsicxarch.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ static CORINFO_InstructionSet lookupInstructionSet(const char* className)
145145
}
146146

147147
//------------------------------------------------------------------------
148-
// lookupIsa: Gets the InstructionSet for a given class name and enclsoing class name
148+
// lookupIsa: Gets the InstructionSet for a given class name and enclosing class name
149149
//
150150
// Arguments:
151151
// className -- The name of the class associated with the InstructionSet to lookup
@@ -797,6 +797,7 @@ GenTree* Compiler::impBaseIntrinsic(NamedIntrinsic intrinsic,
797797
}
798798

799799
case NI_Vector128_get_Zero:
800+
case NI_Vector128_get_AllBitsSet:
800801
{
801802
assert(sig->numArgs == 0);
802803

@@ -842,6 +843,7 @@ GenTree* Compiler::impBaseIntrinsic(NamedIntrinsic intrinsic,
842843
}
843844

844845
case NI_Vector256_get_Zero:
846+
case NI_Vector256_get_AllBitsSet:
845847
{
846848
assert(sig->numArgs == 0);
847849

‎src/coreclr/src/jit/lsraarm64.cpp‎

Lines changed: 20 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1041,25 +1041,28 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree)
10411041

10421042
bool tgtPrefOp1 = false;
10431043

1044-
// If we have an RMW intrinsic, we want to preference op1Reg to the target if
1045-
// op1 is not contained.
1046-
if (isRMW)
1044+
if (intrin.op1 != nullptr)
10471045
{
1048-
tgtPrefOp1 = !intrin.op1->isContained();
1049-
}
1046+
// If we have an RMW intrinsic, we want to preference op1Reg to the target if
1047+
// op1 is not contained.
1048+
if (isRMW)
1049+
{
1050+
tgtPrefOp1 = !intrin.op1->isContained();
1051+
}
10501052

1051-
if (intrinsicTree->OperIsMemoryLoadOrStore())
1052-
{
1053-
srcCount += BuildAddrUses(intrin.op1);
1054-
}
1055-
else if (tgtPrefOp1)
1056-
{
1057-
tgtPrefUse = BuildUse(intrin.op1);
1058-
srcCount++;
1059-
}
1060-
else
1061-
{
1062-
srcCount += BuildOperandUses(intrin.op1);
1053+
if (intrinsicTree->OperIsMemoryLoadOrStore())
1054+
{
1055+
srcCount += BuildAddrUses(intrin.op1);
1056+
}
1057+
else if (tgtPrefOp1)
1058+
{
1059+
tgtPrefUse = BuildUse(intrin.op1);
1060+
srcCount++;
1061+
}
1062+
else
1063+
{
1064+
srcCount += BuildOperandUses(intrin.op1);
1065+
}
10631066
}
10641067

10651068
if (intrin.op2 != nullptr)

0 commit comments

Comments
 (0)