diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 5dbdd4afd85703..9d9bac0a2770af 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1586,6 +1586,25 @@ class LinearScan : public LinearScanInterface PhasedVar availableFloatRegs; PhasedVar availableDoubleRegs; + // Register mask of argument registers currently occupied because we saw a + // PUTARG_REG node. Tracked between the PUTARG_REG and its corresponding + // CALL node and is used to avoid preferring these registers for locals + // which would otherwise force a spill. + regMaskTP placedArgRegs; + + struct PlacedLocal + { + unsigned VarIndex; + regNumber Reg; + }; + + // Locals that are currently placed in registers via PUTARG_REG. These + // locals are available due to the special PUTARG treatment, and we keep + // track of them between the PUTARG_REG and CALL to ensure we keep the + // register they are placed in in the preference set. + PlacedLocal placedArgLocals[REG_COUNT]; + size_t numPlacedArgLocals; + // The set of all register candidates. Note that this may be a subset of tracked vars. VARSET_TP registerCandidateVars; // Current set of live register candidate vars, used during building of RefPositions to determine @@ -1823,6 +1842,7 @@ class LinearScan : public LinearScanInterface // These methods return the number of sources. int BuildNode(GenTree* tree); + void UpdatePreferencesOfDyingLocal(Interval* interval); void getTgtPrefOperands(GenTree* tree, GenTree* op1, GenTree* op2, bool* prefOp1, bool* prefOp2); bool supportsSpecialPutArg(); diff --git a/src/coreclr/jit/lsraarmarch.cpp b/src/coreclr/jit/lsraarmarch.cpp index d41eef6ad2ed59..2c8a16af1a8646 100644 --- a/src/coreclr/jit/lsraarmarch.cpp +++ b/src/coreclr/jit/lsraarmarch.cpp @@ -384,6 +384,10 @@ int LinearScan::BuildCall(GenTreeCall* call) // Now generate defs and kills. regMaskTP killMask = getKillSetForCall(call); BuildDefsWithKills(call, dstCount, dstCandidates, killMask); + + // No args are placed in registers anymore. + placedArgRegs = RBM_NONE; + numPlacedArgLocals = 0; return srcCount; } diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 370bfc393f23ec..7ddde7f82b997b 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -1734,6 +1734,8 @@ void LinearScan::buildRefPositionsForNode(GenTree* tree, LsraLocation currentLoc assert(varDsc->lvTracked); unsigned varIndex = varDsc->lvVarIndex; VarSetOps::RemoveElemD(compiler, currentLiveVars, varIndex); + + UpdatePreferencesOfDyingLocal(getIntervalForLocalVar(varIndex)); } } #else // TARGET_XARCH @@ -2260,6 +2262,9 @@ void LinearScan::buildIntervals() intRegState->rsCalleeRegArgMaskLiveIn |= RBM_SECRET_STUB_PARAM; } + numPlacedArgLocals = 0; + placedArgRegs = RBM_NONE; + BasicBlock* predBlock = nullptr; BasicBlock* prevBlock = nullptr; @@ -2946,6 +2951,71 @@ void LinearScan::BuildDefsWithKills(GenTree* tree, int dstCount, regMaskTP dstCa BuildDefs(tree, dstCount, dstCandidates); } +//------------------------------------------------------------------------ +// UpdatePreferencesOfDyingLocal: Update the preference of a dying local. +// +// Arguments: +// interval - the interval for the local +// +// Notes: +// The "dying" information here is approximate, see the comment in BuildUse. +// +void LinearScan::UpdatePreferencesOfDyingLocal(Interval* interval) +{ + assert(!VarSetOps::IsMember(compiler, currentLiveVars, interval->getVarIndex(compiler))); + + // If we see a use of a local between placing a register and a call then we + // want to update that local's preferences to exclude the "placed" register. + // Picking the "placed" register is otherwise going to force a spill. + // + // We only need to do this on liveness updates because if the local is live + // _after_ the call, then we are going to prefer callee-saved registers for + // such local anyway, so there is no need to look at such local uses. + // + if (placedArgRegs == RBM_NONE) + { + return; + } + + // Write-thru locals are "free" to spill and we are quite conservative + // about allocating them to callee-saved registers, so leave them alone + // here. + if (interval->isWriteThru) + { + return; + } + + // Find the registers that we should remove from the preference set because + // they are occupied with argument values. + regMaskTP unpref = placedArgRegs; + unsigned varIndex = interval->getVarIndex(compiler); + for (size_t i = 0; i < numPlacedArgLocals; i++) + { + if (placedArgLocals[i].VarIndex == varIndex) + { + // This local's value is going to be available in this register so + // keep it in the preferences. + unpref &= ~genRegMask(placedArgLocals[i].Reg); + } + } + + if (unpref != RBM_NONE) + { +#ifdef DEBUG + if (VERBOSE) + { + printf("Last use of V%02u between PUTARG and CALL. Removing occupied arg regs from preferences: ", + compiler->lvaTrackedIndexToLclNum(varIndex)); + dumpRegMask(unpref); + printf("\n"); + } +#endif + + regMaskTP newPreferences = allRegs(interval->registerType) & ~unpref; + interval->updateRegisterPreferences(newPreferences); + } +} + //------------------------------------------------------------------------ // BuildUse: Remove the RefInfoListNode for the given multi-reg index of the given node from // the defList, and build a use RefPosition for the associated Interval. @@ -2985,6 +3055,7 @@ RefPosition* LinearScan::BuildUse(GenTree* operand, regMaskTP candidates, int mu { unsigned varIndex = interval->getVarIndex(compiler); VarSetOps::RemoveElemD(compiler, currentLiveVars, varIndex); + UpdatePreferencesOfDyingLocal(interval); } #if FEATURE_PARTIAL_SIMD_CALLEE_SAVE buildUpperVectorRestoreRefPosition(interval, currentLoc, operand, true); @@ -3883,6 +3954,9 @@ int LinearScan::BuildPutArgReg(GenTreeUnOp* node) regMaskTP argMask = genRegMask(argReg); RefPosition* use = BuildUse(op1, argMask); + // Record that this register is occupied by a register now. + placedArgRegs |= argMask; + if (supportsSpecialPutArg() && isCandidateLocalRef(op1) && ((op1->gtFlags & GTF_VAR_DEATH) == 0)) { // This is the case for a "pass-through" copy of a lclVar. In the case where it is a non-last-use, @@ -3893,6 +3967,14 @@ int LinearScan::BuildPutArgReg(GenTreeUnOp* node) // Preference the destination to the interval of the first register defined by the first operand. assert(use->getInterval()->isLocalVar); isSpecialPutArg = true; + + // Record that this local is available in the register to ensure we + // keep the register in its local set if we see it die before the call + // (see UpdatePreferencesOfDyingLocal). + assert(numPlacedArgLocals < ArrLen(placedArgLocals)); + placedArgLocals[numPlacedArgLocals].VarIndex = use->getInterval()->getVarIndex(compiler); + placedArgLocals[numPlacedArgLocals].Reg = argReg; + numPlacedArgLocals++; } #ifdef TARGET_ARM @@ -3917,6 +3999,7 @@ int LinearScan::BuildPutArgReg(GenTreeUnOp* node) def->getInterval()->assignRelatedInterval(use->getInterval()); } } + return srcCount; } diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index 3c5220ba1d9140..e248620825a9c1 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -1238,6 +1238,10 @@ int LinearScan::BuildCall(GenTreeCall* call) // Now generate defs and kills. regMaskTP killMask = getKillSetForCall(call); BuildDefsWithKills(call, dstCount, dstCandidates, killMask); + + // No args are placed in registers anymore. + placedArgRegs = RBM_NONE; + numPlacedArgLocals = 0; return srcCount; }