From 73f5d24b09a004f84b178a460e4cf359700b7766 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Thu, 9 Jul 2026 07:19:33 -0700 Subject: [PATCH 01/15] JIT: Remove floating-point CreateScalarUnsafe during xarch lowering Floating-point CreateScalarUnsafe is a pure reinterpret: the scalar value already resides in the lowest element of a SIMD register with the upper elements explicitly undefined. Rather than marking such nodes contained (which produces two-deep containment, genOperandDesc look-throughs, and special-cased size asserts), remove the node in LIR lowering and rewire its user to consume the underlying scalar directly. The scalar is intentionally left at its natural (scalar) type; retyping it would corrupt spill and memory-access sizes for other consumers. Correctness is preserved because a register consumer sees the full SIMD register and any memory-fold consumer is gated by the existing width-aware containment logic. Integral scalars (and decomposed longs) still require a real movd/movq and so keep the CreateScalarUnsafe node and fall through to standard containment. The scalar FMA negation loop is restructured accordingly: a removed FP CreateScalarUnsafe leaves a bare GT_NEG operand which is folded into the negated FMA variant. A codegen store size assert is relaxed to allow a narrow FP scalar feeding a wider SIMD local. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/codegenxarch.cpp | 7 +- src/coreclr/jit/lowerxarch.cpp | 131 +++++++++++++++++++------------ 2 files changed, 88 insertions(+), 50 deletions(-) diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 5308e680efa374..07fac81b60d6a1 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -5033,7 +5033,12 @@ void CodeGen::genCodeForStoreLclVar(GenTreeLclVar* lclNode) op1Type = op1VarDsc->GetRegisterType(op1LclVar); } assert(varTypeUsesSameRegType(targetType, op1Type)); - assert(varTypeUsesIntReg(targetType) || (emitTypeSize(targetType) == emitTypeSize(op1Type))); + // A floating-point CreateScalarUnsafe is removed during lowering, which can leave a narrower + // scalar (e.g. TYP_FLOAT) feeding a wider SIMD local. The scalar physically occupies a full + // SIMD register, so the store still copies the target-sized register; the upper elements are + // simply undefined, which matches the Unsafe contract. + assert(varTypeUsesIntReg(targetType) || (emitTypeSize(targetType) == emitTypeSize(op1Type)) || + (varTypeIsSIMD(targetType) && varTypeIsFloating(op1Type))); #endif #if !defined(TARGET_64BIT) diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index 0359b806c6e1bc..b5f194aa85e2d0 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -1281,69 +1281,60 @@ void Lowering::LowerFusedMultiplyOp(GenTreeHWIntrinsic* node) { GenTree* arg = node->Op(i); - if (!arg->OperIsHWIntrinsic()) - { - continue; - } - - GenTreeHWIntrinsic* hwArg = arg->AsHWIntrinsic(); - - switch (hwArg->GetHWIntrinsicId()) + if (isScalar && arg->OperIs(GT_NEG)) { - case NI_Vector_CreateScalarUnsafe: - { - GenTree*& argOp = hwArg->Op(1); + // For scalar FMA the CreateScalarUnsafe wrapper around each argument has already been + // removed during lowering (floating-point CreateScalarUnsafe is a no-op), so a negated + // scalar argument now appears as a bare GT_NEG. Fold that negation into the FMA variant + // and drop the GT_NEG node. - if (argOp->OperIs(GT_NEG)) - { - BlockRange().Remove(argOp); + GenTree* negOp = arg->gtGetOp1(); + BlockRange().Remove(arg); - argOp = argOp->gtGetOp1(); - argOp->ClearContained(); - ContainCheckHWIntrinsic(arg->AsHWIntrinsic()); + negOp->ClearContained(); + node->Op(i) = negOp; - negatedArgs[i - 1] ^= true; - } + negatedArgs[i - 1] ^= true; + continue; + } - break; - } + if (!arg->OperIsHWIntrinsic()) + { + continue; + } - default: - { - bool isScalarArg = false; - genTreeOps oper = hwArg->GetOperForHWIntrinsicId(&isScalarArg); + GenTreeHWIntrinsic* hwArg = arg->AsHWIntrinsic(); - if (oper != GT_XOR) - { - break; - } + bool isScalarArg = false; + genTreeOps oper = hwArg->GetOperForHWIntrinsicId(&isScalarArg); - GenTree* argOp = hwArg->Op(2); + if (oper != GT_XOR) + { + continue; + } - if (!argOp->isContained()) - { - // A constant should have already been contained - break; - } + GenTree* argOp = hwArg->Op(2); - // xor is bitwise and the actual xor node might be a different base type - // from the FMA node, so we check if its negative zero using the FMA base - // type since that's what the end negation would end up using + if (!argOp->isContained()) + { + // A constant should have already been contained + continue; + } - if (argOp->IsVectorNegativeZero(node->GetSimdBaseType())) - { - BlockRange().Remove(hwArg); - BlockRange().Remove(argOp); + // xor is bitwise and the actual xor node might be a different base type + // from the FMA node, so we check if its negative zero using the FMA base + // type since that's what the end negation would end up using - argOp = hwArg->Op(1); - argOp->ClearContained(); - node->Op(i) = argOp; + if (argOp->IsVectorNegativeZero(node->GetSimdBaseType())) + { + BlockRange().Remove(hwArg); + BlockRange().Remove(argOp); - negatedArgs[i - 1] ^= true; - } + argOp = hwArg->Op(1); + argOp->ClearContained(); + node->Op(i) = argOp; - break; - } + negatedArgs[i - 1] ^= true; } } @@ -1996,6 +1987,48 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) return LowerHWIntrinsicToScalar(node); } + case NI_Vector_CreateScalarUnsafe: + { + // A floating-point CreateScalarUnsafe is a pure reinterpret: the scalar value already + // resides in the lowest element of a SIMD register and the upper elements are explicitly + // undefined ("Unsafe"). There is therefore nothing for codegen to do, so we remove the + // node here in LIR and let its user consume the underlying scalar directly. + // + // The scalar is intentionally left at its natural (scalar) type - retyping it to a SIMD + // type would corrupt spill and memory-access sizes for other consumers. Correctness is + // preserved because any consumer that reads the value from a register sees the full SIMD + // register (with the undefined upper elements matching the Unsafe contract), while any + // consumer that could fold the scalar as a memory operand is gated by the width-aware + // containment logic in IsContainableHWIntrinsicOp (a 4/8-byte scalar cannot be contained + // where a wider operand is required). + // + // Integral scalars (and decomposed longs) still require an explicit movd/movq to move the + // value from a general-purpose register into a SIMD register, so those keep the + // CreateScalarUnsafe node and fall through to standard containment. + + if (varTypeIsFloating(node->GetSimdBaseType())) + { + LIR::Use use; + if (BlockRange().TryGetUse(node, &use)) + { + GenTree* op1 = node->Op(1); + GenTree* nextNode = node->gtNext; + + use.ReplaceWith(op1); + BlockRange().Remove(node); + return nextNode; + } + + // A node with no use is either already dead (and marked unused) or a transient + // CreateScalarUnsafe that Vector.Create lowering creates and lowers before wiring it + // to its parent. Leaving it in place is harmless: the dead case is DCE'd, and the + // transient builder case still produces the same code it does today. Removing the + // duplicative Vector.Create lowering (so it no longer manufactures these) is tracked + // as separate work. + } + break; + } + case NI_X86Base_Extract: { if (varTypeIsFloating(node->GetSimdBaseType())) From 4230fe99ae46d19f3ea96b5cb647a56dc31815c3 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Thu, 9 Jul 2026 09:53:13 -0700 Subject: [PATCH 02/15] JIT: stop builders manufacturing floating-point CreateScalarUnsafe Fold the floating-point CreateScalarUnsafe elision into the shared InsertNewSimdCreateScalarUnsafeNode: for a non-constant floating-point scalar it now returns the bare operand (the scalar already occupies the lowest element of a SIMD register with undefined upper elements, matching the Unsafe contract) instead of manufacturing a node that lowering would immediately remove. Integral and constant scalars still create a real node. LowerNode is folded into the shared helper so callers no longer lower the result themselves. This removes the xarch-only InsertNewSimdCreateScalarUnsafeNodeIfNeeded helper and applies the elision on arm64 as well (guarded to xarch/arm64; wasm is excluded because its SIMD model cannot hold a bare scalar in a v128 register). The user-level removal case is also tightened to always remove the node, transferring the unused marker to the operand when there is no use. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/lower.cpp | 28 ++++++++++++++++++++++++---- src/coreclr/jit/lowerarmarch.cpp | 1 - src/coreclr/jit/lowerwasm.cpp | 1 - src/coreclr/jit/lowerxarch.cpp | 32 ++++++++++++-------------------- 4 files changed, 36 insertions(+), 26 deletions(-) diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index ad2728678ca593..af3a8c2b5c60a2 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -13013,19 +13013,30 @@ void Lowering::ContainCheckConditionalCompare(GenTreeCCMP* cmp) #if defined(FEATURE_HW_INTRINSICS) //---------------------------------------------------------------------------------------------- -// Lowering::InsertNewSimdCreateScalarUnsafeNode: Inserts a new simd CreateScalarUnsafe node +// Lowering::InsertNewSimdCreateScalarUnsafeNode: Inserts a new simd CreateScalarUnsafe node, +// eliding it where it is a pure reinterpret // // Arguments: // simdType - The return type of SIMD node being created // op1 - The value of the lowest element of the simd value -// simdBaseJitType - the base JIT type of SIMD type of the intrinsic +// simdBaseType - the base type of the SIMD type of the intrinsic // simdSize - the size of the SIMD type of the intrinsic // // Returns: -// The inserted CreateScalarUnsafe node +// The node the consuming intrinsic should use as its operand. This is the newly created and +// lowered CreateScalarUnsafe node, or - where CreateScalarUnsafe is a pure reinterpret - the +// bare scalar op1 itself. // // Remarks: -// If the created node is a vector constant, op1 will be removed from the block range +// On targets where a floating-point scalar already occupies the lowest element of a SIMD +// register (with the upper elements undefined, matching the "Unsafe" contract), a non-constant +// floating-point CreateScalarUnsafe is a no-op reinterpret. In that case the bare scalar is +// returned and the consuming intrinsic reads it directly, avoiding a redundant node. +// +// Integral scalars still require a real movd/movq (or ins/dup) to move the value into a SIMD +// register, and a constant is materialized so it can fold to a vector constant, so those cases +// create, lower, and return a real CreateScalarUnsafe node. If that node is a vector constant, +// op1 is removed from the block range. // GenTree* Lowering::InsertNewSimdCreateScalarUnsafeNode(var_types simdType, GenTree* op1, @@ -13034,6 +13045,13 @@ GenTree* Lowering::InsertNewSimdCreateScalarUnsafeNode(var_types simdType, { assert(varTypeIsSIMD(simdType)); +#if defined(TARGET_XARCH) || defined(TARGET_ARM64) + if (varTypeIsFloating(simdBaseType) && !op1->IsCnsFltOrDbl()) + { + return op1; + } +#endif // TARGET_XARCH || TARGET_ARM64 + GenTree* result = m_compiler->gtNewSimdCreateScalarUnsafeNode(simdType, op1, simdBaseType, simdSize); BlockRange().InsertAfter(op1, result); @@ -13041,6 +13059,8 @@ GenTree* Lowering::InsertNewSimdCreateScalarUnsafeNode(var_types simdType, { BlockRange().Remove(op1); } + + LowerNode(result); return result; } diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index cdf88c1c0a707b..d70b0cd1a6b95a 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -2266,7 +2266,6 @@ GenTree* Lowering::LowerHWIntrinsicCreate(GenTreeHWIntrinsic* node) // ... GenTree* tmp1 = InsertNewSimdCreateScalarUnsafeNode(simdType, node->Op(1), simdBaseType, simdSize); - LowerNode(tmp1); // We will be constructing the following parts: // ... diff --git a/src/coreclr/jit/lowerwasm.cpp b/src/coreclr/jit/lowerwasm.cpp index 124a233e37cb3f..ea9005d773df57 100644 --- a/src/coreclr/jit/lowerwasm.cpp +++ b/src/coreclr/jit/lowerwasm.cpp @@ -1050,7 +1050,6 @@ GenTree* Lowering::LowerHWIntrinsicCreate(GenTreeHWIntrinsic* node) // ... GenTree* tmp1 = InsertNewSimdCreateScalarUnsafeNode(simdType, node->Op(1), simdBaseType, simdSize); - LowerNode(tmp1); // We will be constructing the following parts: // ... diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index b5f194aa85e2d0..67b496d5acdbe6 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -2008,23 +2008,24 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) if (varTypeIsFloating(node->GetSimdBaseType())) { + GenTree* op1 = node->Op(1); + GenTree* nextNode = node->gtNext; + LIR::Use use; if (BlockRange().TryGetUse(node, &use)) { - GenTree* op1 = node->Op(1); - GenTree* nextNode = node->gtNext; - use.ReplaceWith(op1); - BlockRange().Remove(node); - return nextNode; + } + else + { + // With no use, transfer the unused marker to op1 so it is DCE'd. + assert(node->IsUnusedValue()); + node->ClearUnusedValue(); + op1->SetUnusedValue(); } - // A node with no use is either already dead (and marked unused) or a transient - // CreateScalarUnsafe that Vector.Create lowering creates and lowers before wiring it - // to its parent. Leaving it in place is harmless: the dead case is DCE'd, and the - // transient builder case still produces the same code it does today. Removing the - // duplicative Vector.Create lowering (so it no longer manufactures these) is tracked - // as separate work. + BlockRange().Remove(node); + return nextNode; } break; } @@ -4035,7 +4036,6 @@ GenTree* Lowering::LowerHWIntrinsicCreate(GenTreeHWIntrinsic* node) // return Avx512.BroadcastScalarToVector512(tmp1); tmp1 = InsertNewSimdCreateScalarUnsafeNode(TYP_SIMD16, op1, simdBaseType, 16); - LowerNode(tmp1); node->ResetHWIntrinsicId(NI_AVX512_BroadcastScalarToVector512, tmp1); return LowerNode(node); } @@ -4059,7 +4059,6 @@ GenTree* Lowering::LowerHWIntrinsicCreate(GenTreeHWIntrinsic* node) // return Avx2.BroadcastScalarToVector256(tmp1); tmp1 = InsertNewSimdCreateScalarUnsafeNode(TYP_SIMD16, op1, simdBaseType, 16); - LowerNode(tmp1); node->ResetHWIntrinsicId(NI_AVX2_BroadcastScalarToVector256, tmp1); return LowerNode(node); @@ -4122,7 +4121,6 @@ GenTree* Lowering::LowerHWIntrinsicCreate(GenTreeHWIntrinsic* node) // ... tmp1 = InsertNewSimdCreateScalarUnsafeNode(TYP_SIMD16, op1, simdBaseType, 16); - LowerNode(tmp1); if ((simdBaseType != TYP_DOUBLE) && m_compiler->compOpportunisticallyDependsOn(InstructionSet_AVX2)) { @@ -4410,7 +4408,6 @@ GenTree* Lowering::LowerHWIntrinsicCreate(GenTreeHWIntrinsic* node) // ... tmp1 = InsertNewSimdCreateScalarUnsafeNode(TYP_SIMD16, op1, simdBaseType, 16); - LowerNode(tmp1); switch (simdBaseType) { @@ -4503,7 +4500,6 @@ GenTree* Lowering::LowerHWIntrinsicCreate(GenTreeHWIntrinsic* node) opN = node->Op(N + 1); tmp2 = InsertNewSimdCreateScalarUnsafeNode(TYP_SIMD16, opN, simdBaseType, 16); - LowerNode(tmp2); idx = m_compiler->gtNewIconNode(N << 4, TYP_INT); @@ -4537,7 +4533,6 @@ GenTree* Lowering::LowerHWIntrinsicCreate(GenTreeHWIntrinsic* node) opN = node->Op(argCnt); tmp2 = InsertNewSimdCreateScalarUnsafeNode(TYP_SIMD16, opN, simdBaseType, 16); - LowerNode(tmp2); idx = m_compiler->gtNewIconNode((argCnt - 1) << 4, TYP_INT); BlockRange().InsertAfter(tmp2, idx); @@ -4599,7 +4594,6 @@ GenTree* Lowering::LowerHWIntrinsicCreate(GenTreeHWIntrinsic* node) // return Sse.UnpackLow(tmp1, tmp2); tmp2 = InsertNewSimdCreateScalarUnsafeNode(TYP_SIMD16, op2, simdBaseType, 16); - LowerNode(tmp2); node->ResetHWIntrinsicId(NI_X86Base_UnpackLow, tmp1, tmp2); break; @@ -5319,7 +5313,6 @@ GenTree* Lowering::LowerHWIntrinsicWithElement(GenTreeHWIntrinsic* node) // tmp1 = Vector128.CreateScalarUnsafe(op3); tmp1 = InsertNewSimdCreateScalarUnsafeNode(TYP_SIMD16, op3, TYP_FLOAT, 16); - LowerNode(tmp1); imm8 = imm8 * 16; op3 = tmp1; @@ -5351,7 +5344,6 @@ GenTree* Lowering::LowerHWIntrinsicWithElement(GenTreeHWIntrinsic* node) // tmp1 = Vector128.CreateScalarUnsafe(op3); tmp1 = InsertNewSimdCreateScalarUnsafeNode(TYP_SIMD16, op3, TYP_DOUBLE, 16); - LowerNode(tmp1); result->ResetHWIntrinsicId((imm8 == 0) ? NI_X86Base_MoveScalar : NI_X86Base_UnpackLow, op1, tmp1); break; From ddaf8f8e685c8596ce7267a282a4bfbbb3e5566e Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Thu, 9 Jul 2026 10:27:54 -0700 Subject: [PATCH 03/15] JIT: remove intermediate CreateScalar when containing it during xarch lowering CreateScalar/CreateScalarUnsafe are not themselves memory operations. They were only marked contained so codegen could look through them to the underlying scalar, producing a second level of containment plus special handling in LSRA and codegen. Instead of containing such a node when it is consumed as a low-lane scalar by a parent hardware intrinsic (exactly the set IsContainableHWIntrinsicOp already selects), remove it in lowering and let the parent consume the scalar directly. The scalar keeps whatever contained/regOptional state it was already given, so codegen is unchanged (the memory operand or register is folded straight into the parent). Decomposed longs still require a real movd/movq and keep the node. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/lower.h | 1 + src/coreclr/jit/lowerxarch.cpp | 57 +++++++++++++++++++++++++++++----- 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/src/coreclr/jit/lower.h b/src/coreclr/jit/lower.h index b8435548fd8cf0..d6f7fa469dc8f0 100644 --- a/src/coreclr/jit/lower.h +++ b/src/coreclr/jit/lower.h @@ -126,6 +126,7 @@ class Lowering final : public Phase void ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node); #ifdef TARGET_XARCH void TryFoldCnsVecForEmbeddedBroadcast(GenTreeHWIntrinsic* parentNode, GenTreeVecCon* cnsVec); + void ContainHWIntrinsicOperand(GenTreeHWIntrinsic* parentNode, GenTree* childNode); #endif // TARGET_XARCH #endif // FEATURE_HW_INTRINSICS diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index 67b496d5acdbe6..193ff2a97b2871 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -8961,6 +8961,49 @@ void Lowering::TryFoldCnsVecForEmbeddedBroadcast(GenTreeHWIntrinsic* parentNode, MakeSrcContained(parentNode, broadcastNode); } +//------------------------------------------------------------------------ +// ContainHWIntrinsicOperand: Contains an operand of a hardware intrinsic node. +// +// Arguments: +// parentNode - The hardware intrinsic node which is the parent of 'childNode' +// childNode - The operand to contain; must already have been validated by +// IsContainableHWIntrinsicOp +// +// Notes: +// CreateScalar/CreateScalarUnsafe are not themselves memory operations - they were only +// selected as containable so codegen could look through them to the underlying scalar. Rather +// than contain such a node (which produces a second level of containment and requires special +// handling in LSRA and codegen), we remove it here and let the parent consume the scalar +// directly. The scalar keeps whatever contained/regOptional state it was already given. +// +void Lowering::ContainHWIntrinsicOperand(GenTreeHWIntrinsic* parentNode, GenTree* childNode) +{ + if (childNode->OperIsHWIntrinsic()) + { + NamedIntrinsic childId = childNode->AsHWIntrinsic()->GetHWIntrinsicId(); + + if (HWIntrinsicInfo::IsVectorCreateScalar(childId) || HWIntrinsicInfo::IsVectorCreateScalarUnsafe(childId)) + { + GenTree* scalar = childNode->AsHWIntrinsic()->Op(1); + + // A decomposed long requires a real movd/movq to move the value into a SIMD register, so + // it must keep the CreateScalar/CreateScalarUnsafe node and be contained normally. + if (!scalar->OperIsLong()) + { + LIR::Use use; + bool foundUse = BlockRange().TryGetUse(childNode, &use); + assert(foundUse); + + use.ReplaceWith(scalar); + BlockRange().Remove(childNode); + return; + } + } + } + + MakeSrcContained(parentNode, childNode); +} + //------------------------------------------------------------------------ // TryMakeSrcContainedOrRegOptional: Tries to make "childNode" a contained or regOptional node // @@ -8980,7 +9023,7 @@ void Lowering::TryMakeSrcContainedOrRegOptional(GenTreeHWIntrinsic* parentNode, } else { - MakeSrcContained(parentNode, childNode); + ContainHWIntrinsicOperand(parentNode, childNode); } } else if (supportsRegOptional) @@ -9406,7 +9449,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) } else { - MakeSrcContained(node, containedOperand); + ContainHWIntrinsicOperand(node, containedOperand); } } else if (regOptionalOperand != nullptr) @@ -9750,7 +9793,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) } else { - MakeSrcContained(node, containedOperand); + ContainHWIntrinsicOperand(node, containedOperand); } } else if (regOptionalOperand != nullptr) @@ -9834,7 +9877,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) } else { - MakeSrcContained(node, containedOperand); + ContainHWIntrinsicOperand(node, containedOperand); } } else if (regOptionalOperand != nullptr) @@ -10119,7 +10162,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) if (containedOperand != nullptr) { - MakeSrcContained(node, containedOperand); + ContainHWIntrinsicOperand(node, containedOperand); } else if (regOptionalOperand != nullptr) { @@ -10178,7 +10221,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) if (containedOperand != nullptr) { - MakeSrcContained(node, containedOperand); + ContainHWIntrinsicOperand(node, containedOperand); } else if (regOptionalOperand != nullptr) { @@ -10603,7 +10646,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) } else { - MakeSrcContained(node, containedOperand); + ContainHWIntrinsicOperand(node, containedOperand); } } else if (regOptionalOperand != nullptr) From eec529334916fa390559f2b2658f1f42c047b078 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Thu, 9 Jul 2026 11:05:45 -0700 Subject: [PATCH 04/15] JIT: remove dead contained-CreateScalarUnsafe support After lowering was changed to remove CreateScalar/CreateScalarUnsafe when a parent consumes it as a low-lane scalar (rather than containing the node), no CreateScalar/CreateScalarUnsafe is ever marked contained. Remove the now-dead support for a contained CreateScalar/Unsafe: - gentree.cpp: drop the CreateScalar/Unsafe case from isContainableHWIntrinsic. canBeContained() now returns false for these, so the DEBUG isContained() assert actively verifies none are ever contained on any target. - instr.cpp: remove the genOperandDesc look-through that treated a contained CreateScalar/Unsafe as a load from memory. - lsraxarch.cpp: remove SkipContainedUnaryOp (an identity function once no such node is contained) and read the operands directly. - Refresh stale comments in instr.cpp and hwintrinsiccodegenxarch.cpp that referenced the removed look-through. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/gentree.cpp | 7 --- src/coreclr/jit/hwintrinsiccodegenxarch.cpp | 7 ++- src/coreclr/jit/instr.cpp | 27 ++-------- src/coreclr/jit/lsraxarch.cpp | 57 +++------------------ 4 files changed, 14 insertions(+), 84 deletions(-) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 9ad468aa31c7b2..0419919532fe4a 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -22169,13 +22169,6 @@ bool GenTree::isContainableHWIntrinsic() const return node->GetSimdSize() == 16; } - case NI_Vector_CreateScalar: - case NI_Vector_CreateScalarUnsafe: - { - // These HWIntrinsic operations are contained as part of scalar ops - return true; - } - case NI_X86Base_LoadAndDuplicateToVector128: case NI_X86Base_MoveAndDuplicate: case NI_AVX_BroadcastScalarToVector128: diff --git a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp index 3e8c533ff6c825..2863faecba5df4 100644 --- a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp @@ -2274,10 +2274,9 @@ void CodeGen::genBaseIntrinsic(GenTreeHWIntrinsic* node, insOpts instOptions) case NI_Vector_AsVector3: case NI_Vector_ToScalar: { - // genOperandDesc looks through a contained CreateScalar/CreateScalarUnsafe to the operand it - // wraps, which may itself live in a register (e.g. Vector128.CreateScalarUnsafe(x).ToScalar()). - // We therefore use the descriptor's containment - not op1 directly - to decide instruction - // selection: only a true memory operand can be read with a plain integer load. + // op1 may be a contained memory operand or live in a register. We use the descriptor's + // containment - not op1 directly - to decide instruction selection: only a true memory + // operand can be read with a plain integer load. OperandDesc op1Desc = genOperandDesc(ins, op1); if (op1Desc.IsContained()) diff --git a/src/coreclr/jit/instr.cpp b/src/coreclr/jit/instr.cpp index 9876150b22e35e..1bdb55258c807e 100644 --- a/src/coreclr/jit/instr.cpp +++ b/src/coreclr/jit/instr.cpp @@ -1136,18 +1136,15 @@ CodeGen::OperandDesc CodeGen::genOperandDesc(instruction ins, GenTree* op) { assert(simdBaseType == TYP_DOUBLE); } - // If broadcast node is contained, should mean that we have some forms like - // Broadcast -> CreateScalarUnsafe -> Scalar. - // If so, directly emit scalar. - // In the code below, we specially handle the `Broadcast -> CNS_INT/CNS_LNG` form and - // handle other cases recursively. + // A contained broadcast wraps its scalar operand directly (any CreateScalar/Unsafe + // wrapper was removed during lowering). We specially handle the + // `Broadcast -> CNS_INT/CNS_LNG` form below and handle other cases recursively. GenTree* hwintrinsicChild = hwintrinsic->Op(1); assert(hwintrinsicChild->isContained()); if (hwintrinsicChild->IsIntegralConst()) { - // a special case is when the operand of CreateScalarUnsafe is an integer type, - // CreateScalarUnsafe node will be folded, so we directly match a pattern of - // broadcast -> LCL_VAR(TYP_(U)INT/LONG) + // A broadcast over an integral constant is materialized as a data constant + // that we can read directly from memory. INT64 scalarValue = hwintrinsicChild->AsIntConCommon()->IntegralValue(); UNATIVE_OFFSET cnum = emit->emitDataConst(&scalarValue, genTypeSize(simdBaseType), genTypeSize(simdBaseType), simdBaseType); @@ -1162,20 +1159,6 @@ CodeGen::OperandDesc CodeGen::genOperandDesc(instruction ins, GenTree* op) break; } - case NI_Vector_CreateScalar: - case NI_Vector_CreateScalarUnsafe: - { - // The hwintrinsic should be contained and its - // op1 should be either contained or spilled. This - // allows us to transparently "look through" the - // CreateScalar/Unsafe and treat it directly like - // a load from memory. - - assert(hwintrinsic->isContained()); - op = hwintrinsic->Op(1); - return genOperandDesc(ins, op); - } - default: { assert(hwintrinsic->OperIsMemoryLoad()); diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index 58a9e2cbef9cc6..a3b7f34d568027 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -2012,46 +2012,6 @@ int LinearScan::BuildIntrinsic(GenTree* tree) } #ifdef FEATURE_HW_INTRINSICS -//------------------------------------------------------------------------ -// SkipContainedUnaryOp: Skips a contained non-memory or const node -// and gets the underlying op1 instead -// -// Arguments: -// node - The node to handle -// -// Return Value: -// If node is a contained non-memory or const unary op, its op1 is returned; -// otherwise node is returned unchanged. -static GenTree* SkipContainedUnaryOp(GenTree* node) -{ - if (!node->isContained()) - { - return node; - } - - if (node->OperIsHWIntrinsic()) - { - GenTreeHWIntrinsic* hwintrinsic = node->AsHWIntrinsic(); - NamedIntrinsic intrinsicId = hwintrinsic->GetHWIntrinsicId(); - - switch (intrinsicId) - { - case NI_Vector_CreateScalar: - case NI_Vector_CreateScalarUnsafe: - { - return hwintrinsic->Op(1); - } - - default: - { - break; - } - } - } - - return node; -} - //------------------------------------------------------------------------ // BuildHWIntrinsic: Set the NodeInfo for a GT_HWINTRINSIC tree. // @@ -2106,47 +2066,42 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou } else { - // In a few cases, we contain an operand that isn't a load from memory or a constant. Instead, - // it is essentially a "transparent" node we're ignoring or handling specially in codegen - // to simplify the overall IR handling. As such, we need to "skip" such nodes when present and - // get the underlying op1 so that delayFreeUse and other preferencing remains correct. - GenTree* op1 = nullptr; GenTree* op2 = nullptr; GenTree* op3 = nullptr; GenTree* op4 = nullptr; GenTree* op5 = nullptr; - GenTree* lastOp = SkipContainedUnaryOp(intrinsicTree->Op(numArgs)); + GenTree* lastOp = intrinsicTree->Op(numArgs); switch (numArgs) { case 5: { - op5 = SkipContainedUnaryOp(intrinsicTree->Op(5)); + op5 = intrinsicTree->Op(5); FALLTHROUGH; } case 4: { - op4 = SkipContainedUnaryOp(intrinsicTree->Op(4)); + op4 = intrinsicTree->Op(4); FALLTHROUGH; } case 3: { - op3 = SkipContainedUnaryOp(intrinsicTree->Op(3)); + op3 = intrinsicTree->Op(3); FALLTHROUGH; } case 2: { - op2 = SkipContainedUnaryOp(intrinsicTree->Op(2)); + op2 = intrinsicTree->Op(2); FALLTHROUGH; } case 1: { - op1 = SkipContainedUnaryOp(intrinsicTree->Op(1)); + op1 = intrinsicTree->Op(1); break; } From cbe7ff1b8704933c9dabbd2acb66b3d2765cce50 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Thu, 9 Jul 2026 12:56:56 -0700 Subject: [PATCH 05/15] JIT: permit removed floating-point CreateScalarUnsafe operands in GetHWIntrinsicIdForBinOp Removing a floating-point CreateScalarUnsafe during lowering can leave a bare scalar operand feeding a SIMD binary op, such as a bitwise combine that is formed during lowering. The scalar occupies the low element of a SIMD register and is consumed at full register width, which matches the Unsafe contract. Relax the operand-shape asserts in GetHWIntrinsicIdForBinOp to permit such a floating scalar in addition to a full SIMD operand. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/gentree.cpp | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 0419919532fe4a..a0da42ea90dec7 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -31736,8 +31736,17 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, assert(varTypeIsArithmetic(simdBaseType)); assert(varTypeIsSIMD(simdType)); +#ifdef DEBUG + // A floating-point CreateScalarUnsafe is removed during lowering, which can leave a scalar + // operand occupying the low element of a SIMD register. Such an operand is consumed here at + // full register width, so it is valid in addition to a full SIMD operand. + auto isValidBinOpOperand = [=](GenTree* op) -> bool { + return op->TypeIs(simdType) || varTypeIsFloating(op); + }; +#endif // DEBUG + assert(op1 != nullptr); - assert(op1->TypeIs(simdType)); + assert(isValidBinOpOperand(op1)); assert(op2 != nullptr); #if defined(TARGET_XARCH) @@ -31763,7 +31772,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, { case GT_ADD: { - assert(op2->TypeIs(simdType)); + assert(isValidBinOpOperand(op2)); #if defined(TARGET_XARCH) if (simdSize == 64) @@ -31810,7 +31819,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, case GT_AND: { assert(!isScalar); - assert(op2->TypeIs(simdType)); + assert(isValidBinOpOperand(op2)); #if defined(TARGET_XARCH) if (simdSize == 64) @@ -31846,7 +31855,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, case GT_AND_NOT: { assert(!isScalar); - assert(op2->TypeIs(simdType)); + assert(isValidBinOpOperand(op2)); if (comp->fgNodeThreading != NodeThreading::LIR) { @@ -31897,7 +31906,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, #else assert(varTypeIsFloating(simdBaseType)); #endif - assert(op2->TypeIs(simdType)); + assert(isValidBinOpOperand(op2)); #if defined(TARGET_XARCH) if (varTypeIsFloating(simdBaseType)) @@ -31991,7 +32000,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, case GT_MUL: { #if defined(TARGET_XARCH) - assert(op2->TypeIs(simdType)); + assert(isValidBinOpOperand(op2)); if (simdSize == 64) { @@ -32058,7 +32067,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, case GT_OR: { assert(!isScalar); - assert(op2->TypeIs(simdType)); + assert(isValidBinOpOperand(op2)); #if defined(TARGET_XARCH) if (simdSize == 64) @@ -32238,7 +32247,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, case GT_SUB: { - assert(op2->TypeIs(simdType)); + assert(isValidBinOpOperand(op2)); #if defined(TARGET_XARCH) if (simdSize == 64) @@ -32285,7 +32294,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, case GT_XOR: { assert(!isScalar); - assert(op2->TypeIs(simdType)); + assert(isValidBinOpOperand(op2)); #if defined(TARGET_XARCH) if (simdSize == 64) From 2997855246ca78a2c34b5c67e33d99a1377accbe Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Thu, 9 Jul 2026 12:57:04 -0700 Subject: [PATCH 06/15] JIT: remove ToVector256Unsafe/ToVector512Unsafe during xarch lowering These intrinsics widen a narrower vector into a wider one where the upper bits are undefined. At the register level that is a no-op: the value already occupies the low bits of the wider register. Remove the node during lowering and let the consumer read the source operand directly, avoiding a register-to-register copy when the source is still live and keeping the node out of LIR. The consumer reads the operand from a register at its own wider size with the upper bits undefined, which is exactly the contract of these nodes. Containment performs its own size check, so the operand can never be widened into an undersized contained memory operand. Relax the genCodeForStoreLclVar size assert to allow a narrower SIMD source feeding a wider SIMD local. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/codegenxarch.cpp | 13 ++++++++----- src/coreclr/jit/lowerxarch.cpp | 27 +++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 07fac81b60d6a1..50997413ee6802 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -5033,12 +5033,15 @@ void CodeGen::genCodeForStoreLclVar(GenTreeLclVar* lclNode) op1Type = op1VarDsc->GetRegisterType(op1LclVar); } assert(varTypeUsesSameRegType(targetType, op1Type)); - // A floating-point CreateScalarUnsafe is removed during lowering, which can leave a narrower - // scalar (e.g. TYP_FLOAT) feeding a wider SIMD local. The scalar physically occupies a full - // SIMD register, so the store still copies the target-sized register; the upper elements are - // simply undefined, which matches the Unsafe contract. + // A floating-point CreateScalarUnsafe or a ToVector*Unsafe widen is removed during lowering, + // which can leave a narrower source (a TYP_FLOAT/TYP_DOUBLE scalar, or a smaller TYP_SIMD*) + // feeding a wider SIMD local. The source physically occupies the low bits of a full SIMD + // register, so the store still copies the target-sized register; the upper elements are simply + // undefined, which matches the Unsafe contract. assert(varTypeUsesIntReg(targetType) || (emitTypeSize(targetType) == emitTypeSize(op1Type)) || - (varTypeIsSIMD(targetType) && varTypeIsFloating(op1Type))); + (varTypeIsSIMD(targetType) && + (varTypeIsFloating(op1Type) || + (varTypeIsSIMD(op1Type) && (emitTypeSize(op1Type) <= emitTypeSize(targetType)))))); #endif #if !defined(TARGET_64BIT) diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index 193ff2a97b2871..e79b5a5450c34f 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -2562,6 +2562,33 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) break; } + case NI_Vector_ToVector256Unsafe: + case NI_Vector_ToVector512Unsafe: + { + // These widen a narrower vector into a wider one where the upper bits are undefined. + // At the register level that is a no-op: the value already occupies the low bits of + // the wider register. Remove the node and let the consumer read op1 directly, which + // avoids a register-to-register copy when the source is still live and keeps the node + // out of LIR entirely. + // + // The consumer reads op1 from a register at its own (wider) size, with the upper bits + // undefined, which is exactly the contract of these nodes. Nothing observes the node + // for sizing, and containment performs its own size check (operandSize >= expectedSize), + // so op1 can never be widened into an undersized contained memory operand. + + LIR::Use use; + if (BlockRange().TryGetUse(node, &use)) + { + GenTree* op1 = node->Op(1); + GenTree* next = node->gtNext; + + use.ReplaceWith(op1); + BlockRange().Remove(node); + return next; + } + break; + } + default: break; } From fde7f87e007e0619821f987d461f79512607458e Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Thu, 9 Jul 2026 13:10:11 -0700 Subject: [PATCH 07/15] JIT: remove GetLower/GetLower128 during xarch lowering GetLower and GetLower128 narrow a wider vector by reading only its low bits, which at the register level is a no-op: the value already occupies the low bits of a register. Remove the node during lowering and let the consumer read the source operand directly at its own (narrower) size, which avoids a register-to-register copy when the source is still live and keeps the node out of LIR. Containment performs its own size check, and the source of these nodes is always a full-width vector whose memory is fully allocated, so reading it at the consumer's size is always in bounds. Extend the genCodeForStoreLclVar size assert to also allow a wider SIMD source feeding a narrower SIMD local. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/codegenxarch.cpp | 15 +++++++-------- src/coreclr/jit/lowerxarch.cpp | 24 ++++++++++++++---------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 50997413ee6802..2b47da1d0427c0 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -5033,15 +5033,14 @@ void CodeGen::genCodeForStoreLclVar(GenTreeLclVar* lclNode) op1Type = op1VarDsc->GetRegisterType(op1LclVar); } assert(varTypeUsesSameRegType(targetType, op1Type)); - // A floating-point CreateScalarUnsafe or a ToVector*Unsafe widen is removed during lowering, - // which can leave a narrower source (a TYP_FLOAT/TYP_DOUBLE scalar, or a smaller TYP_SIMD*) - // feeding a wider SIMD local. The source physically occupies the low bits of a full SIMD - // register, so the store still copies the target-sized register; the upper elements are simply - // undefined, which matches the Unsafe contract. + // A floating-point CreateScalarUnsafe, a ToVector*Unsafe widen, or a GetLower/GetLower128 + // narrow is removed during lowering, which can leave a source whose size differs from the + // SIMD local: a TYP_FLOAT/TYP_DOUBLE scalar, a smaller TYP_SIMD*, or a larger TYP_SIMD*. The + // source physically occupies a full SIMD register, so the store copies the target-sized + // register; for a narrower source the upper elements are simply undefined, which matches the + // Unsafe contract. assert(varTypeUsesIntReg(targetType) || (emitTypeSize(targetType) == emitTypeSize(op1Type)) || - (varTypeIsSIMD(targetType) && - (varTypeIsFloating(op1Type) || - (varTypeIsSIMD(op1Type) && (emitTypeSize(op1Type) <= emitTypeSize(targetType)))))); + (varTypeIsSIMD(targetType) && (varTypeIsFloating(op1Type) || varTypeIsSIMD(op1Type)))); #endif #if !defined(TARGET_64BIT) diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index e79b5a5450c34f..2a3f73131d5688 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -2564,17 +2564,21 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) case NI_Vector_ToVector256Unsafe: case NI_Vector_ToVector512Unsafe: - { - // These widen a narrower vector into a wider one where the upper bits are undefined. - // At the register level that is a no-op: the value already occupies the low bits of - // the wider register. Remove the node and let the consumer read op1 directly, which - // avoids a register-to-register copy when the source is still live and keeps the node - // out of LIR entirely. + case NI_Vector_GetLower: + case NI_Vector_GetLower128: + { + // ToVector*Unsafe widens a narrower vector into a wider one where the upper bits are + // undefined; GetLower/GetLower128 narrows a wider vector by reading only its low bits. + // At the register level both are a no-op: the value already occupies the low bits of a + // register, so the consumer can read op1 directly at its own size. Remove the node, + // which avoids a register-to-register copy when the source is still live and keeps the + // node out of LIR entirely. // - // The consumer reads op1 from a register at its own (wider) size, with the upper bits - // undefined, which is exactly the contract of these nodes. Nothing observes the node - // for sizing, and containment performs its own size check (operandSize >= expectedSize), - // so op1 can never be widened into an undersized contained memory operand. + // The consumer reads op1 from a register at its own size; for the widening case the + // upper bits are undefined, which is exactly the contract of those nodes. Nothing + // observes the node for sizing, and containment performs its own size check + // (operandSize >= expectedSize), so op1 can never be read from an undersized contained + // memory operand. LIR::Use use; if (BlockRange().TryGetUse(node, &use)) From 802aa60970faafcfffae98d76cae7e6523d0b12c Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Thu, 9 Jul 2026 13:14:23 -0700 Subject: [PATCH 08/15] JIT: remove GetLower during arm64 lowering Mirror the xarch GetLower removal on arm64. GetLower narrows a Vector128 to a Vector64 by reading its low 64 bits, which at the register level is free because a Q register and its low D register alias the same physical register. Remove the node during lowering so the consumer reads the source directly at its own size. Release codegen is correct by construction: consumers size their operands from their own node and all SIMD values share the arm64 V register class, so no register-class mismatch or size assumption is broken. This change is code-reviewed only; the arm64 cross-compilation toolchain is not available in the authoring environment, so it relies on checked arm64 CI for DEBUG-assert validation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/lowerarmarch.cpp | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index d70b0cd1a6b95a..f6e7cd032f04dd 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -1727,6 +1727,31 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) break; } + case NI_Vector_GetLower: + { + // GetLower narrows a Vector128 by reading only its low 64 bits. At the register level + // this is a no-op: the value already occupies the low bits of the register (a Q register + // and its low D register alias), so the consumer can read op1 directly at its own size. + // Remove the node, which avoids a register-to-register copy when the source is still live + // and keeps the node out of LIR entirely. + // + // The consumer reads op1 from a register at its own size; nothing observes the node for + // sizing, and containment performs its own size check (operandSize >= expectedSize), so + // op1 can never be read from an undersized contained memory operand. + + LIR::Use use; + if (BlockRange().TryGetUse(node, &use)) + { + GenTree* op1 = node->Op(1); + GenTree* next = node->gtNext; + + use.ReplaceWith(op1); + BlockRange().Remove(node); + return next; + } + break; + } + case NI_AdvSimd_FusedMultiplyAddScalar: LowerHWIntrinsicFusedMultiplyAddScalar(node); break; From 89a3cdeb990a856ad91b1fa684c40803ad5f3376 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Thu, 9 Jul 2026 22:40:33 -0700 Subject: [PATCH 09/15] Only remove transparent SIMD reinterprets when consumed by a hwintrinsic The FP CreateScalarUnsafe, ToVector256Unsafe/ToVector512Unsafe, and GetLower/GetLower128 removals rewired every use to the underlying operand, including stores, returns, and call arguments. For those non-hwintrinsic consumers the value is materialized at the node's own type and size via the ABI, so replacing the node with a wider or narrower operand corrupts the copy size. This produced oversized reads (e.g. a 32-byte vmovups over a 16-byte Vector128 stack slot) that could read out of bounds and, on the SysV ABI where vectors larger than 16 bytes are passed through memory, crash with a SIGSEGV during the mis-sized argument copy. Gate each removal on the consumer being a GenTreeHWIntrinsic. Such a consumer reads the operand from a register at the intrinsic's own size (the widening cases rely on the upper bits being undefined, which matches the Unsafe contract) or as a contained memory operand whose size is independently validated by containment. Non-hwintrinsic consumers keep the node so the reinterpret is materialized at the correct size. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/lowerarmarch.cpp | 16 +++++++------ src/coreclr/jit/lowerxarch.cpp | 40 +++++++++++++++++++------------- 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index f6e7cd032f04dd..18feb2f22cb912 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -1731,16 +1731,18 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) { // GetLower narrows a Vector128 by reading only its low 64 bits. At the register level // this is a no-op: the value already occupies the low bits of the register (a Q register - // and its low D register alias), so the consumer can read op1 directly at its own size. - // Remove the node, which avoids a register-to-register copy when the source is still live - // and keeps the node out of LIR entirely. + // and its low D register alias), so the node can be removed and the consumer read op1 + // directly, avoiding a register-to-register copy when the source is still live. // - // The consumer reads op1 from a register at its own size; nothing observes the node for - // sizing, and containment performs its own size check (operandSize >= expectedSize), so - // op1 can never be read from an undersized contained memory operand. + // This is only valid when the consumer is another GenTreeHWIntrinsic. Such a consumer + // reads op1 as a register operand at the intrinsic's own size, and containment performs + // its own size check (operandSize >= expectedSize) so op1 can never be read from an + // undersized contained memory operand. Any other consumer (a store, return, or call + // argument) materializes a value of the node's own type and size via the ABI, so removing + // the node there would corrupt the copy size; keep the node for those. LIR::Use use; - if (BlockRange().TryGetUse(node, &use)) + if (BlockRange().TryGetUse(node, &use) && (use.User() != nullptr) && use.User()->OperIsHWIntrinsic()) { GenTree* op1 = node->Op(1); GenTree* next = node->gtNext; diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index 2a3f73131d5688..fdb934b5ceb40e 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -1991,16 +1991,19 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) { // A floating-point CreateScalarUnsafe is a pure reinterpret: the scalar value already // resides in the lowest element of a SIMD register and the upper elements are explicitly - // undefined ("Unsafe"). There is therefore nothing for codegen to do, so we remove the - // node here in LIR and let its user consume the underlying scalar directly. + // undefined ("Unsafe"). When the consumer is another GenTreeHWIntrinsic it reads the + // scalar directly from that register, so there is nothing for codegen to do and we remove + // the node here in LIR, letting the user consume the underlying scalar directly. // // The scalar is intentionally left at its natural (scalar) type - retyping it to a SIMD // type would corrupt spill and memory-access sizes for other consumers. Correctness is - // preserved because any consumer that reads the value from a register sees the full SIMD - // register (with the undefined upper elements matching the Unsafe contract), while any - // consumer that could fold the scalar as a memory operand is gated by the width-aware - // containment logic in IsContainableHWIntrinsicOp (a 4/8-byte scalar cannot be contained - // where a wider operand is required). + // preserved because a GenTreeHWIntrinsic consumer that reads the value from a register + // sees the full SIMD register (with the undefined upper elements matching the Unsafe + // contract), while any consumer that could fold the scalar as a memory operand is gated + // by the width-aware containment logic in IsContainableHWIntrinsicOp (a 4/8-byte scalar + // cannot be contained where a wider operand is required). A store, return, or call + // argument instead materializes a value of the node's SIMD type and size, so keep the + // node for those and let it fall through to standard containment and codegen. // // Integral scalars (and decomposed longs) still require an explicit movd/movq to move the // value from a general-purpose register into a SIMD register, so those keep the @@ -2014,6 +2017,10 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) LIR::Use use; if (BlockRange().TryGetUse(node, &use)) { + if ((use.User() == nullptr) || !use.User()->OperIsHWIntrinsic()) + { + break; + } use.ReplaceWith(op1); } else @@ -2570,18 +2577,19 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) // ToVector*Unsafe widens a narrower vector into a wider one where the upper bits are // undefined; GetLower/GetLower128 narrows a wider vector by reading only its low bits. // At the register level both are a no-op: the value already occupies the low bits of a - // register, so the consumer can read op1 directly at its own size. Remove the node, - // which avoids a register-to-register copy when the source is still live and keeps the - // node out of LIR entirely. + // register, so the node can be removed and the consumer read op1 directly, avoiding a + // register-to-register copy when the source is still live. // - // The consumer reads op1 from a register at its own size; for the widening case the - // upper bits are undefined, which is exactly the contract of those nodes. Nothing - // observes the node for sizing, and containment performs its own size check - // (operandSize >= expectedSize), so op1 can never be read from an undersized contained - // memory operand. + // This is only valid when the consumer is another GenTreeHWIntrinsic. Such a consumer + // reads op1 as a register operand at the intrinsic's own size (the widening case relies + // on the upper bits being undefined, which is exactly the contract of these nodes), and + // containment performs its own size check (operandSize >= expectedSize) so op1 can never + // be read from an undersized contained memory operand. Any other consumer (a store, + // return, or call argument) materializes a value of the node's own type and size via the + // ABI, so removing the node there would corrupt the copy size; keep the node for those. LIR::Use use; - if (BlockRange().TryGetUse(node, &use)) + if (BlockRange().TryGetUse(node, &use) && (use.User() != nullptr) && use.User()->OperIsHWIntrinsic()) { GenTree* op1 = node->Op(1); GenTree* next = node->gtNext; From e7d1bf32909cbbc5ef1f2df7ed530999cf9d7a26 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 10 Jul 2026 04:42:52 -0700 Subject: [PATCH 10/15] Tighten store asserts and drop redundant null checks now that removal is hwintrinsic-only Removal of transparent SIMD reinterprets is now gated on the use being a hwintrinsic, so a store consumer always keeps the node and its op1 matches the store's target type/size. The relaxed genCodeForStoreLclVar assert is therefore dead and is tightened back to the strict form. TryGetUse only returns true with a non-null user, so the use.User() null checks at the three lowering sites are redundant and removed. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/codegenxarch.cpp | 9 +-------- src/coreclr/jit/lowerarmarch.cpp | 2 +- src/coreclr/jit/lowerxarch.cpp | 4 ++-- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 2b47da1d0427c0..5308e680efa374 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -5033,14 +5033,7 @@ void CodeGen::genCodeForStoreLclVar(GenTreeLclVar* lclNode) op1Type = op1VarDsc->GetRegisterType(op1LclVar); } assert(varTypeUsesSameRegType(targetType, op1Type)); - // A floating-point CreateScalarUnsafe, a ToVector*Unsafe widen, or a GetLower/GetLower128 - // narrow is removed during lowering, which can leave a source whose size differs from the - // SIMD local: a TYP_FLOAT/TYP_DOUBLE scalar, a smaller TYP_SIMD*, or a larger TYP_SIMD*. The - // source physically occupies a full SIMD register, so the store copies the target-sized - // register; for a narrower source the upper elements are simply undefined, which matches the - // Unsafe contract. - assert(varTypeUsesIntReg(targetType) || (emitTypeSize(targetType) == emitTypeSize(op1Type)) || - (varTypeIsSIMD(targetType) && (varTypeIsFloating(op1Type) || varTypeIsSIMD(op1Type)))); + assert(varTypeUsesIntReg(targetType) || (emitTypeSize(targetType) == emitTypeSize(op1Type))); #endif #if !defined(TARGET_64BIT) diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index 18feb2f22cb912..e1ef2e42b7f786 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -1742,7 +1742,7 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) // the node there would corrupt the copy size; keep the node for those. LIR::Use use; - if (BlockRange().TryGetUse(node, &use) && (use.User() != nullptr) && use.User()->OperIsHWIntrinsic()) + if (BlockRange().TryGetUse(node, &use) && use.User()->OperIsHWIntrinsic()) { GenTree* op1 = node->Op(1); GenTree* next = node->gtNext; diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index fdb934b5ceb40e..feea5d98096e1f 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -2017,7 +2017,7 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) LIR::Use use; if (BlockRange().TryGetUse(node, &use)) { - if ((use.User() == nullptr) || !use.User()->OperIsHWIntrinsic()) + if (!use.User()->OperIsHWIntrinsic()) { break; } @@ -2589,7 +2589,7 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) // ABI, so removing the node there would corrupt the copy size; keep the node for those. LIR::Use use; - if (BlockRange().TryGetUse(node, &use) && (use.User() != nullptr) && use.User()->OperIsHWIntrinsic()) + if (BlockRange().TryGetUse(node, &use) && use.User()->OperIsHWIntrinsic()) { GenTree* op1 = node->Op(1); GenTree* next = node->gtNext; From ccf7e07192fe3fa2b931336efc090c47d5e29748 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 10 Jul 2026 05:19:18 -0700 Subject: [PATCH 11/15] Inline the GetHWIntrinsicIdForBinOp operand asserts Drop the isValidBinOpOperand lambda in favor of inlining the 'op->TypeIs(simdType) || varTypeIsFloating(op)' check directly at each assert site. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/gentree.cpp | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index a0da42ea90dec7..3004530a1d59bc 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -31736,17 +31736,12 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, assert(varTypeIsArithmetic(simdBaseType)); assert(varTypeIsSIMD(simdType)); -#ifdef DEBUG // A floating-point CreateScalarUnsafe is removed during lowering, which can leave a scalar // operand occupying the low element of a SIMD register. Such an operand is consumed here at // full register width, so it is valid in addition to a full SIMD operand. - auto isValidBinOpOperand = [=](GenTree* op) -> bool { - return op->TypeIs(simdType) || varTypeIsFloating(op); - }; -#endif // DEBUG assert(op1 != nullptr); - assert(isValidBinOpOperand(op1)); + assert(op1->TypeIs(simdType) || varTypeIsFloating(op1)); assert(op2 != nullptr); #if defined(TARGET_XARCH) @@ -31772,7 +31767,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, { case GT_ADD: { - assert(isValidBinOpOperand(op2)); + assert(op2->TypeIs(simdType) || varTypeIsFloating(op2)); #if defined(TARGET_XARCH) if (simdSize == 64) @@ -31819,7 +31814,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, case GT_AND: { assert(!isScalar); - assert(isValidBinOpOperand(op2)); + assert(op2->TypeIs(simdType) || varTypeIsFloating(op2)); #if defined(TARGET_XARCH) if (simdSize == 64) @@ -31855,7 +31850,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, case GT_AND_NOT: { assert(!isScalar); - assert(isValidBinOpOperand(op2)); + assert(op2->TypeIs(simdType) || varTypeIsFloating(op2)); if (comp->fgNodeThreading != NodeThreading::LIR) { @@ -31906,7 +31901,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, #else assert(varTypeIsFloating(simdBaseType)); #endif - assert(isValidBinOpOperand(op2)); + assert(op2->TypeIs(simdType) || varTypeIsFloating(op2)); #if defined(TARGET_XARCH) if (varTypeIsFloating(simdBaseType)) @@ -32000,7 +31995,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, case GT_MUL: { #if defined(TARGET_XARCH) - assert(isValidBinOpOperand(op2)); + assert(op2->TypeIs(simdType) || varTypeIsFloating(op2)); if (simdSize == 64) { @@ -32067,7 +32062,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, case GT_OR: { assert(!isScalar); - assert(isValidBinOpOperand(op2)); + assert(op2->TypeIs(simdType) || varTypeIsFloating(op2)); #if defined(TARGET_XARCH) if (simdSize == 64) @@ -32247,7 +32242,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, case GT_SUB: { - assert(isValidBinOpOperand(op2)); + assert(op2->TypeIs(simdType) || varTypeIsFloating(op2)); #if defined(TARGET_XARCH) if (simdSize == 64) @@ -32294,7 +32289,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, case GT_XOR: { assert(!isScalar); - assert(isValidBinOpOperand(op2)); + assert(op2->TypeIs(simdType) || varTypeIsFloating(op2)); #if defined(TARGET_XARCH) if (simdSize == 64) From 342eff94800eebb83f5d7c321ef915008f0176a8 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 10 Jul 2026 06:19:53 -0700 Subject: [PATCH 12/15] Relax the AdvSimd.Insert targetReg != op3Reg assert to allow aliased operands The assert assumed op3 could never share the target register. That held while the multi-element Vector.Create Insert-chain always fed a distinct CreateScalarUnsafe def as op1, but once a floating-point CreateScalarUnsafe is elided into a bare scalar the same enregistered value can be used as both op1 and op3 (e.g. Vector.Create(x, x)). op3 is delay-free, so a distinct op3 interval can never land on the def register; targetReg == op3Reg is only reachable when op3 aliases op1, which also forces targetReg == op1Reg. In that case the leading mov is skipped, so op3 is preserved and the ins is correct. Relax the assert to the precise safety condition accordingly. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 85d24554e9336c..4821d0389d3c43 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -1426,7 +1426,13 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) // fmov (scalar) zeros the upper bits and is not safe to use assert(!intrin.op3->isContainedFltOrDblImmed()); - assert(targetReg != op3Reg); + // The mov above copies op1 into targetReg and the ins below then reads op3. That is + // only unsafe when targetReg == op3Reg but targetReg != op1Reg, as the mov would then + // clobber op3 before it is read. LSRA marks op3 delayFree, so a distinct op3 can never + // share the def register; targetReg == op3Reg is only reachable when op3 aliases op1 + // (e.g. Vector.Create(x, ..., x, ...) after a floating-point CreateScalarUnsafe is + // elided into a bare scalar), in which case the mov is skipped and op3 is preserved. + assert((targetReg != op3Reg) || (targetReg == op1Reg)); HWIntrinsicImmOpHelper helper(this, intrin.op2, node); From 0de4361c75ed78d6e680f92c2204a22e96f63ee2 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 10 Jul 2026 06:20:01 -0700 Subject: [PATCH 13/15] Remove ToVector128Unsafe and floating-point CreateScalarUnsafe during arm64 lowering These are pure register-level reinterprets: ToVector128Unsafe widens a Vector64 into a Vector128 with undefined upper bits, and a floating-point CreateScalarUnsafe places a scalar in the low element of a SIMD register with undefined upper elements. When the consumer is another GenTreeHWIntrinsic it already reads the value from the low bits of the register, so the node is a no-op and can be removed in lowering, avoiding a register-to-register copy when the source is still live. The underlying operand is left at its natural type - retyping it would corrupt spill and memory-access sizes for other consumers. FusedMultiplyAddScalar is excluded because its lowering folds a size-8 CreateScalarUnsafe wrapping a GT_NEG into the negated fused-multiply variant. Integral CreateScalarUnsafe still needs an explicit fmov and is left unchanged. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/lowerarmarch.cpp | 63 ++++++++++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index e1ef2e42b7f786..4d1b10dbde29a4 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -1728,11 +1728,13 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) } case NI_Vector_GetLower: + case NI_Vector_ToVector128Unsafe: { - // GetLower narrows a Vector128 by reading only its low 64 bits. At the register level - // this is a no-op: the value already occupies the low bits of the register (a Q register - // and its low D register alias), so the node can be removed and the consumer read op1 - // directly, avoiding a register-to-register copy when the source is still live. + // GetLower narrows a Vector128 to its low 64 bits; ToVector128Unsafe widens a Vector64 + // into a Vector128 whose upper 64 bits are explicitly undefined ("Unsafe"). At the + // register level both are no-ops: the value already occupies the low bits of the register + // (a Q register and its low D register alias), so the node can be removed and the consumer + // read op1 directly, avoiding a register-to-register copy when the source is still live. // // This is only valid when the consumer is another GenTreeHWIntrinsic. Such a consumer // reads op1 as a register operand at the intrinsic's own size, and containment performs @@ -1758,6 +1760,59 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) LowerHWIntrinsicFusedMultiplyAddScalar(node); break; + case NI_Vector_CreateScalarUnsafe: + { + // A floating-point CreateScalarUnsafe is a pure reinterpret: the scalar value already + // resides in the lowest element of a SIMD register and the upper elements are explicitly + // undefined ("Unsafe"). When the consumer is another GenTreeHWIntrinsic it reads the + // scalar directly from that register, so there is nothing for codegen to do and we remove + // the node here in LIR, letting the user consume the underlying scalar directly. + // + // The scalar is intentionally left at its natural (scalar) type - retyping it to a SIMD + // type would corrupt spill and memory-access sizes for other consumers. Correctness is + // preserved because a GenTreeHWIntrinsic consumer that reads the value from a register + // sees the full SIMD register (with the undefined upper elements matching the Unsafe + // contract). A store, return, or call argument instead materializes a value of the node's + // SIMD type and size, so keep the node for those and let it fall through to standard + // containment and codegen. + // + // FusedMultiplyAddScalar is excluded: its lowering folds a GT_NEG wrapped in a size-8 + // CreateScalarUnsafe operand into the negated fused-multiply variant, so the node must + // survive for that fold to trigger. + // + // Integral scalars still require an explicit fmov to move the value from a general-purpose + // register into a SIMD register, so those keep the CreateScalarUnsafe node and fall + // through to standard containment. + + if (varTypeIsFloating(node->GetSimdBaseType())) + { + GenTree* op1 = node->Op(1); + GenTree* nextNode = node->gtNext; + + LIR::Use use; + if (BlockRange().TryGetUse(node, &use)) + { + if (!use.User()->OperIsHWIntrinsic() || + (use.User()->AsHWIntrinsic()->GetHWIntrinsicId() == NI_AdvSimd_FusedMultiplyAddScalar)) + { + break; + } + use.ReplaceWith(op1); + } + else + { + // With no use, transfer the unused marker to op1 so it is DCE'd. + assert(node->IsUnusedValue()); + node->ClearUnusedValue(); + op1->SetUnusedValue(); + } + + BlockRange().Remove(node); + return nextNode; + } + break; + } + case NI_Sve_ConditionalSelect: return LowerHWIntrinsicCndSel(node); From 6f6205464343ee9f7bfe18b12c3bdaece2106b92 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 10 Jul 2026 12:32:21 -0700 Subject: [PATCH 14/15] Relax gtNewSimdBinOpNode operand-shape asserts to be HIR-only The transparent-node elision during lowering can leave a size-changing SIMD reinterpret operand -- e.g. an elided GetLower or ToVectorXXXUnsafe -- feeding a reconstructed binop. In LIR the operand still occupies a full SIMD register and is consumed at the node's width, and containment validates the memory-operand size before allowing a load, so the exact-shape asserts only need to enforce in HIR. Gate them on fgNodeThreading == NodeThreading::LIR and treat any SIMD-typed operand as a full-vector operand in GetHWIntrinsicIdForBinOp. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/gentree.cpp | 58 ++++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 3004530a1d59bc..783f69de4cc264 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -22829,16 +22829,26 @@ GenTree* Compiler::gtNewSimdBinOpNode( assert(varTypeIsArithmetic(simdBaseType)); assert(op1 != nullptr); - assert(op1->TypeIs(type, simdBaseType, genActualType(simdBaseType)) || - (op1->TypeIs(TYP_SIMD12) && type == TYP_SIMD16)); - assert(op2 != nullptr); + // The operand-shape asserts below validate that the HIR builder is producing the exact vector + // shape we expect. Once in LIR, lowering is free to feed a size-changing reinterpret operand + // directly -- e.g. an elided GetLower or ToVectorXXXUnsafe -- since the binop operates on the low + // simdSize bytes and containment validates the memory-operand size (operandSize >= expectedSize) + // before allowing a load. Only enforce the shape in HIR. + bool isLIR = (fgNodeThreading == NodeThreading::LIR); + + if (!isLIR) + { + assert(op1->TypeIs(type, simdBaseType, genActualType(simdBaseType)) || + (op1->TypeIs(TYP_SIMD12) && type == TYP_SIMD16)); + } + if ((op == GT_LSH) || (op == GT_RSH) || (op == GT_RSZ)) { assert(genActualType(op2) == TYP_INT); } - else + else if (!isLIR) { assert((genActualType(op2) == genActualType(type)) || (genActualType(op2) == genActualType(simdBaseType)) || (op2->TypeIs(TYP_SIMD12) && (type == TYP_SIMD16))); @@ -31739,9 +31749,17 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, // A floating-point CreateScalarUnsafe is removed during lowering, which can leave a scalar // operand occupying the low element of a SIMD register. Such an operand is consumed here at // full register width, so it is valid in addition to a full SIMD operand. + // + // Likewise, once in LIR, lowering may feed a size-changing SIMD reinterpret operand directly -- + // e.g. an elided GetLower or ToVectorXXXUnsafe. It still occupies a full SIMD register and is + // consumed at the node's width, so treat any SIMD-typed operand as a full-vector operand rather + // than requiring an exact size match. In HIR the operand size must still be exact. + auto isFullVectorOp = [=](GenTree* op) -> bool { + return op->TypeIs(simdType) || ((comp->fgNodeThreading == NodeThreading::LIR) && varTypeIsSIMD(op)); + }; assert(op1 != nullptr); - assert(op1->TypeIs(simdType) || varTypeIsFloating(op1)); + assert(isFullVectorOp(op1) || varTypeIsFloating(op1)); assert(op2 != nullptr); #if defined(TARGET_XARCH) @@ -31767,7 +31785,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, { case GT_ADD: { - assert(op2->TypeIs(simdType) || varTypeIsFloating(op2)); + assert(isFullVectorOp(op2) || varTypeIsFloating(op2)); #if defined(TARGET_XARCH) if (simdSize == 64) @@ -31814,7 +31832,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, case GT_AND: { assert(!isScalar); - assert(op2->TypeIs(simdType) || varTypeIsFloating(op2)); + assert(isFullVectorOp(op2) || varTypeIsFloating(op2)); #if defined(TARGET_XARCH) if (simdSize == 64) @@ -31850,7 +31868,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, case GT_AND_NOT: { assert(!isScalar); - assert(op2->TypeIs(simdType) || varTypeIsFloating(op2)); + assert(isFullVectorOp(op2) || varTypeIsFloating(op2)); if (comp->fgNodeThreading != NodeThreading::LIR) { @@ -31901,7 +31919,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, #else assert(varTypeIsFloating(simdBaseType)); #endif - assert(op2->TypeIs(simdType) || varTypeIsFloating(op2)); + assert(isFullVectorOp(op2) || varTypeIsFloating(op2)); #if defined(TARGET_XARCH) if (varTypeIsFloating(simdBaseType)) @@ -31939,7 +31957,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, case GT_LSH: { assert(!isScalar); - assert(op2->TypeIs(simdType) || varTypeIsInt(op2)); + assert(isFullVectorOp(op2) || varTypeIsInt(op2)); assert(varTypeIsIntegral(simdBaseType)); #if defined(TARGET_XARCH) @@ -31995,7 +32013,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, case GT_MUL: { #if defined(TARGET_XARCH) - assert(op2->TypeIs(simdType) || varTypeIsFloating(op2)); + assert(isFullVectorOp(op2) || varTypeIsFloating(op2)); if (simdSize == 64) { @@ -32042,11 +32060,11 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, } else if (simdBaseType == TYP_DOUBLE) { - id = op2->TypeIs(simdType) ? NI_AdvSimd_Arm64_Multiply : NI_AdvSimd_Arm64_MultiplyByScalar; + id = isFullVectorOp(op2) ? NI_AdvSimd_Arm64_Multiply : NI_AdvSimd_Arm64_MultiplyByScalar; } else if (!varTypeIsLong(simdBaseType)) { - id = op2->TypeIs(simdType) ? NI_AdvSimd_Multiply : NI_AdvSimd_MultiplyByScalar; + id = isFullVectorOp(op2) ? NI_AdvSimd_Multiply : NI_AdvSimd_MultiplyByScalar; } #elif defined(TARGET_WASM) if (!varTypeIsByte(simdBaseType)) @@ -32062,7 +32080,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, case GT_OR: { assert(!isScalar); - assert(op2->TypeIs(simdType) || varTypeIsFloating(op2)); + assert(isFullVectorOp(op2) || varTypeIsFloating(op2)); #if defined(TARGET_XARCH) if (simdSize == 64) @@ -32098,7 +32116,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, case GT_ROL: { assert(!isScalar); - assert(op2->TypeIs(simdType) || varTypeIsInt(op2)); + assert(isFullVectorOp(op2) || varTypeIsInt(op2)); assert(varTypeIsIntegral(simdBaseType)); #if defined(TARGET_XARCH) @@ -32113,7 +32131,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, case GT_ROR: { assert(!isScalar); - assert(op2->TypeIs(simdType) || varTypeIsInt(op2)); + assert(isFullVectorOp(op2) || varTypeIsInt(op2)); assert(varTypeIsIntegral(simdBaseType)); #if defined(TARGET_XARCH) @@ -32128,7 +32146,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, case GT_RSH: { assert(!isScalar); - assert(op2->TypeIs(simdType) || varTypeIsInt(op2)); + assert(isFullVectorOp(op2) || varTypeIsInt(op2)); assert(varTypeIsIntegral(simdBaseType)); #if defined(TARGET_XARCH) @@ -32187,7 +32205,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, case GT_RSZ: { assert(!isScalar); - assert(op2->TypeIs(simdType) || varTypeIsInt(op2)); + assert(isFullVectorOp(op2) || varTypeIsInt(op2)); assert(varTypeIsIntegral(simdBaseType)); #if defined(TARGET_XARCH) @@ -32242,7 +32260,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, case GT_SUB: { - assert(op2->TypeIs(simdType) || varTypeIsFloating(op2)); + assert(isFullVectorOp(op2) || varTypeIsFloating(op2)); #if defined(TARGET_XARCH) if (simdSize == 64) @@ -32289,7 +32307,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForBinOp(Compiler* comp, case GT_XOR: { assert(!isScalar); - assert(op2->TypeIs(simdType) || varTypeIsFloating(op2)); + assert(isFullVectorOp(op2) || varTypeIsFloating(op2)); #if defined(TARGET_XARCH) if (simdSize == 64) From 6a9851aabc6e29c3fcd1d8ac393a2eb6eb2ee402 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 10 Jul 2026 17:43:29 -0700 Subject: [PATCH 15/15] Relax GetHWIntrinsicIdForCmpOp operand-shape asserts to be HIR-only The transparent-node elision during lowering can feed a size-changing SIMD reinterpret operand (e.g. an elided GetLower or ToVectorXXXUnsafe) into a reconstructed comparison via TryInvertMask. Such an operand still occupies a full SIMD register and is consumed at the node's width, so the exact operand size only needs to be enforced in HIR. Mirror the gtNewSimdBinOpNode relaxation using fgNodeThreading == NodeThreading::LIR, guarding the debug-only helper. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/gentree.cpp | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 783f69de4cc264..8e4dcfc377e72b 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -32382,8 +32382,18 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForCmpOp(Compiler* comp, assert(varTypeIsArithmetic(simdBaseType)); assert(varTypeIsSIMD(simdType)); +#ifdef DEBUG + // Once in LIR, lowering may feed a size-changing SIMD reinterpret operand directly -- e.g. an + // elided GetLower or ToVectorXXXUnsafe. It still occupies a full SIMD register and is consumed + // at the node's width, so treat any SIMD-typed operand as a full-vector operand rather than + // requiring an exact size match. In HIR the operand size must still be exact. + auto isFullVectorOp = [=](GenTree* op) -> bool { + return op->TypeIs(simdType) || ((comp->fgNodeThreading == NodeThreading::LIR) && varTypeIsSIMD(op)); + }; +#endif // DEBUG + assert(op1 != nullptr); - assert(op1->TypeIs(simdType)); + assert(isFullVectorOp(op1)); assert(op2 != nullptr); #if defined(TARGET_XARCH) @@ -32435,7 +32445,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForCmpOp(Compiler* comp, { case GT_EQ: { - assert(op2->TypeIs(simdType)); + assert(isFullVectorOp(op2)); #if defined(TARGET_XARCH) if (varTypeIsMask(type)) @@ -32477,7 +32487,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForCmpOp(Compiler* comp, case GT_GE: { - assert(op2->TypeIs(simdType)); + assert(isFullVectorOp(op2)); #if defined(TARGET_XARCH) if (varTypeIsMask(type)) @@ -32524,7 +32534,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForCmpOp(Compiler* comp, case GT_GT: { - assert(op2->TypeIs(simdType)); + assert(isFullVectorOp(op2)); #if defined(TARGET_XARCH) if (varTypeIsMask(type)) @@ -32585,7 +32595,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForCmpOp(Compiler* comp, case GT_LE: { - assert(op2->TypeIs(simdType)); + assert(isFullVectorOp(op2)); #if defined(TARGET_XARCH) if (varTypeIsMask(type)) @@ -32632,7 +32642,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForCmpOp(Compiler* comp, case GT_LT: { - assert(op2->TypeIs(simdType)); + assert(isFullVectorOp(op2)); // !GE @@ -32695,7 +32705,7 @@ NamedIntrinsic GenTreeHWIntrinsic::GetHWIntrinsicIdForCmpOp(Compiler* comp, case GT_NE: { - assert(op2->TypeIs(simdType)); + assert(isFullVectorOp(op2)); #if defined(TARGET_XARCH) if (varTypeIsMask(type))