From 2997b19882c1a6f1922fb6f2cff693474724bf83 Mon Sep 17 00:00:00 2001 From: Andy Ayers Date: Tue, 8 Nov 2022 13:27:50 -0800 Subject: [PATCH 1/2] JIT: fix gc hole in peephole optimizations We cannot safely peephole instructions that straddle a gc enable boundary. Detecting when this might happen is a bit subtle; currently we rely on `emitForceNewIG` to be set. Add a new utility 'emitCanPeepholeLastIns` to centralize the logic that decides whether basing current emission on `emitLastIns` is safe. Closed #77661. --- src/coreclr/jit/emit.h | 7 +++++++ src/coreclr/jit/emitarm64.cpp | 10 ++++------ src/coreclr/jit/emitxarch.cpp | 18 +++++++----------- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h index 34379a6a3d05a3..394807e1d02a5a 100644 --- a/src/coreclr/jit/emit.h +++ b/src/coreclr/jit/emit.h @@ -2180,6 +2180,13 @@ class emitter instrDesc* emitLastIns; + // Check if a peephole opt involving emitLastIns is unsafe + // emitForceNewIG here prevents peepholes from crossing nogc boundaries. + bool emitCannotPeepholeLastIns() + { + return emitForceNewIG || ((emitCurIGinsCnt == 0) && ((emitCurIG->igFlags & IGF_EXTEND) == 0)); + } + #ifdef TARGET_ARMARCH instrDesc* emitLastMemBarrier; #endif diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 22b0075756d21e..84e7f65ea27a72 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -15957,7 +15957,7 @@ bool emitter::IsRedundantMov(instruction ins, emitAttr size, regNumber dst, regN return false; } - const bool isFirstInstrInBlock = (emitCurIGinsCnt == 0) && ((emitCurIG->igFlags & IGF_EXTEND) == 0); + const bool canOptimize = !emitCannotPeepholeLastIns(); if (dst == src) { @@ -15977,7 +15977,7 @@ bool emitter::IsRedundantMov(instruction ins, emitAttr size, regNumber dst, regN else if (isGeneralRegisterOrSP(dst) && (size == EA_4BYTE)) { // See if the previous instruction already cleared upper 4 bytes for us unintentionally - if (!isFirstInstrInBlock && (emitLastIns != nullptr) && (emitLastIns->idReg1() == dst) && + if (canOptimize && (emitLastIns != nullptr) && (emitLastIns->idReg1() == dst) && (emitLastIns->idOpSize() == size) && emitLastIns->idInsIs(INS_ldr, INS_ldrh, INS_ldrb)) { JITDUMP("\n -- suppressing mov because ldr already cleared upper 4 bytes\n"); @@ -15986,7 +15986,7 @@ bool emitter::IsRedundantMov(instruction ins, emitAttr size, regNumber dst, regN } } - if (!isFirstInstrInBlock && // Don't optimize if instruction is not the first instruction in IG. + if (canOptimize && // Don't optimize if instruction is not the first instruction in IG. (emitLastIns != nullptr) && (emitLastIns->idIns() == INS_mov) && // Don't optimize if last instruction was not 'mov'. (emitLastIns->idOpSize() == size)) // Don't optimize if operand size is different than previous instruction. @@ -16065,9 +16065,7 @@ bool emitter::IsRedundantMov(instruction ins, emitAttr size, regNumber dst, regN bool emitter::IsRedundantLdStr( instruction ins, regNumber reg1, regNumber reg2, ssize_t imm, emitAttr size, insFormat fmt) { - bool isFirstInstrInBlock = (emitCurIGinsCnt == 0) && ((emitCurIG->igFlags & IGF_EXTEND) == 0); - - if (((ins != INS_ldr) && (ins != INS_str)) || (isFirstInstrInBlock) || (emitLastIns == nullptr)) + if (((ins != INS_ldr) && (ins != INS_str)) || emitCannotPeepholeLastIns()) { return false; } diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index db6294890e7e3f..db808459679c18 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -489,7 +489,7 @@ bool emitter::AreUpper32BitsZero(regNumber reg) // If there are no instructions in this IG, we can look back at // the previous IG's instructions if this IG is an extension. // - if ((emitCurIGinsCnt == 0) && ((emitCurIG->igFlags & IGF_EXTEND) == 0)) + if (emitCannotPeepholeLastIns()) { return false; } @@ -569,8 +569,7 @@ bool emitter::AreFlagsSetToZeroCmp(regNumber reg, emitAttr opSize, genTreeOps tr return false; } - // Don't look back across IG boundaries (possible control flow) - if (emitCurIGinsCnt == 0 && ((emitCurIG->igFlags & IGF_EXTEND) == 0)) + if (emitCannotPeepholeLastIns()) { return false; } @@ -653,7 +652,7 @@ bool emitter::AreFlagsSetForSignJumpOpt(regNumber reg, emitAttr opSize, GenTree* } // Don't look back across IG boundaries (possible control flow) - if (emitCurIGinsCnt == 0 && ((emitCurIG->igFlags & IGF_EXTEND) == 0)) + if (emitCannotPeepholeLastIns()) { return false; } @@ -5717,12 +5716,10 @@ bool emitter::IsRedundantMov( return true; } - bool isFirstInstrInBlock = (emitCurIGinsCnt == 0) && ((emitCurIG->igFlags & IGF_EXTEND) == 0); - // TODO-XArch-CQ: Certain instructions, such as movaps vs movups, are equivalent in // functionality even if their actual identifier differs and we should optimize these - if (isFirstInstrInBlock || // Don't optimize if instruction is the first instruction in IG. + if (emitCannotPeepholeLastIns() || // Don't optimize if instruction is the first instruction in IG. (emitLastIns == nullptr) || // or if a last instruction doesn't exist (emitLastIns->idIns() != ins) || // or if the instruction is different from the last instruction (emitLastIns->idOpSize() != size) || // or if the operand size is different from the last instruction @@ -8410,13 +8407,10 @@ bool emitter::IsRedundantStackMov(instruction ins, insFormat fmt, emitAttr size, return false; } - bool hasSideEffect = HasSideEffect(ins, size); - - bool isFirstInstrInBlock = (emitCurIGinsCnt == 0) && ((emitCurIG->igFlags & IGF_EXTEND) == 0); // TODO-XArch-CQ: Certain instructions, such as movaps vs movups, are equivalent in // functionality even if their actual identifier differs and we should optimize these - if (isFirstInstrInBlock || // Don't optimize if instruction is the first instruction in IG. + if (emitCannotPeepholeLastIns() || // Don't optimize if instruction is the first instruction in IG. (emitLastIns == nullptr) || // or if a last instruction doesn't exist (emitLastIns->idIns() != ins) || // or if the instruction is different from the last instruction (emitLastIns->idOpSize() != size)) // or if the operand size is different from the last instruction @@ -8434,6 +8428,8 @@ bool emitter::IsRedundantStackMov(instruction ins, insFormat fmt, emitAttr size, int varNum = emitLastIns->idAddr()->iiaLclVar.lvaVarNum(); int lastOffs = emitLastIns->idAddr()->iiaLclVar.lvaOffset(); + const bool hasSideEffect = HasSideEffect(ins, size); + // Check if the last instruction and current instructions use the same register and local memory. if (varNum == varx && lastReg1 == ireg && lastOffs == offs) { From 6053fff453f0bdcf8a007ebefd91c4965e108c29 Mon Sep 17 00:00:00 2001 From: Andy Ayers Date: Tue, 8 Nov 2022 18:37:32 -0800 Subject: [PATCH 2/2] revise per feedback --- src/coreclr/jit/emit.h | 17 ++++++++++++----- src/coreclr/jit/emitarm64.cpp | 11 +++++------ src/coreclr/jit/emitxarch.cpp | 20 ++++++++++---------- 3 files changed, 27 insertions(+), 21 deletions(-) diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h index 394807e1d02a5a..dc4278a82e71eb 100644 --- a/src/coreclr/jit/emit.h +++ b/src/coreclr/jit/emit.h @@ -2180,11 +2180,18 @@ class emitter instrDesc* emitLastIns; - // Check if a peephole opt involving emitLastIns is unsafe - // emitForceNewIG here prevents peepholes from crossing nogc boundaries. - bool emitCannotPeepholeLastIns() - { - return emitForceNewIG || ((emitCurIGinsCnt == 0) && ((emitCurIG->igFlags & IGF_EXTEND) == 0)); + // Check if a peephole optimization involving emitLastIns is safe. + // + // We must have a lastInstr to consult. + // The emitForceNewIG check here prevents peepholes from crossing nogc boundaries. + // The final check prevents looking across an IG boundary unless we're in an extension IG. + bool emitCanPeepholeLastIns() + { + return (emitLastIns != nullptr) && // there is an emitLastInstr + !emitForceNewIG && // and we're not about to start a new IG + ((emitCurIGinsCnt > 0) || // and we're not at the start of a new IG + ((emitCurIG->igFlags & IGF_EXTEND) != 0)); // or we are at the start of a new IG, + // and it's an extension IG } #ifdef TARGET_ARMARCH diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 84e7f65ea27a72..20c722c6c3850e 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -15957,7 +15957,7 @@ bool emitter::IsRedundantMov(instruction ins, emitAttr size, regNumber dst, regN return false; } - const bool canOptimize = !emitCannotPeepholeLastIns(); + const bool canOptimize = emitCanPeepholeLastIns(); if (dst == src) { @@ -15977,8 +15977,8 @@ bool emitter::IsRedundantMov(instruction ins, emitAttr size, regNumber dst, regN else if (isGeneralRegisterOrSP(dst) && (size == EA_4BYTE)) { // See if the previous instruction already cleared upper 4 bytes for us unintentionally - if (canOptimize && (emitLastIns != nullptr) && (emitLastIns->idReg1() == dst) && - (emitLastIns->idOpSize() == size) && emitLastIns->idInsIs(INS_ldr, INS_ldrh, INS_ldrb)) + if (canOptimize && (emitLastIns->idReg1() == dst) && (emitLastIns->idOpSize() == size) && + emitLastIns->idInsIs(INS_ldr, INS_ldrh, INS_ldrb)) { JITDUMP("\n -- suppressing mov because ldr already cleared upper 4 bytes\n"); return true; @@ -15986,8 +15986,7 @@ bool emitter::IsRedundantMov(instruction ins, emitAttr size, regNumber dst, regN } } - if (canOptimize && // Don't optimize if instruction is not the first instruction in IG. - (emitLastIns != nullptr) && + if (canOptimize && // Don't optimize if unsafe. (emitLastIns->idIns() == INS_mov) && // Don't optimize if last instruction was not 'mov'. (emitLastIns->idOpSize() == size)) // Don't optimize if operand size is different than previous instruction. { @@ -16065,7 +16064,7 @@ bool emitter::IsRedundantMov(instruction ins, emitAttr size, regNumber dst, regN bool emitter::IsRedundantLdStr( instruction ins, regNumber reg1, regNumber reg2, ssize_t imm, emitAttr size, insFormat fmt) { - if (((ins != INS_ldr) && (ins != INS_str)) || emitCannotPeepholeLastIns()) + if (((ins != INS_ldr) && (ins != INS_str)) || !emitCanPeepholeLastIns()) { return false; } diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index db808459679c18..5dacf783fa5df1 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -486,10 +486,9 @@ bool emitter::IsFlagsAlwaysModified(instrDesc* id) bool emitter::AreUpper32BitsZero(regNumber reg) { - // If there are no instructions in this IG, we can look back at - // the previous IG's instructions if this IG is an extension. + // Only consider if safe // - if (emitCannotPeepholeLastIns()) + if (!emitCanPeepholeLastIns()) { return false; } @@ -569,7 +568,9 @@ bool emitter::AreFlagsSetToZeroCmp(regNumber reg, emitAttr opSize, genTreeOps tr return false; } - if (emitCannotPeepholeLastIns()) + // Only consider if safe + // + if (!emitCanPeepholeLastIns()) { return false; } @@ -651,8 +652,9 @@ bool emitter::AreFlagsSetForSignJumpOpt(regNumber reg, emitAttr opSize, GenTree* return false; } - // Don't look back across IG boundaries (possible control flow) - if (emitCannotPeepholeLastIns()) + // Only consider if safe + // + if (!emitCanPeepholeLastIns()) { return false; } @@ -5719,8 +5721,7 @@ bool emitter::IsRedundantMov( // TODO-XArch-CQ: Certain instructions, such as movaps vs movups, are equivalent in // functionality even if their actual identifier differs and we should optimize these - if (emitCannotPeepholeLastIns() || // Don't optimize if instruction is the first instruction in IG. - (emitLastIns == nullptr) || // or if a last instruction doesn't exist + if (!emitCanPeepholeLastIns() || // Don't optimize if unsafe (emitLastIns->idIns() != ins) || // or if the instruction is different from the last instruction (emitLastIns->idOpSize() != size) || // or if the operand size is different from the last instruction (emitLastIns->idInsFmt() != fmt)) // or if the format is different from the last instruction @@ -8410,8 +8411,7 @@ bool emitter::IsRedundantStackMov(instruction ins, insFormat fmt, emitAttr size, // TODO-XArch-CQ: Certain instructions, such as movaps vs movups, are equivalent in // functionality even if their actual identifier differs and we should optimize these - if (emitCannotPeepholeLastIns() || // Don't optimize if instruction is the first instruction in IG. - (emitLastIns == nullptr) || // or if a last instruction doesn't exist + if (!emitCanPeepholeLastIns() || // Don't optimize if unsafe (emitLastIns->idIns() != ins) || // or if the instruction is different from the last instruction (emitLastIns->idOpSize() != size)) // or if the operand size is different from the last instruction {