From ef3a03ee8c502b3abbde687b068591c846df9fbd Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 9 Feb 2026 17:44:26 +0100 Subject: [PATCH 1/6] JIT: Enable inlining of runtime async methods without awaits While inlining runtime async methods in general is tricky, it is relatively simple to allow inlining async methods that do not have any awaits in them, and I suspect this is a large fraction of them. --- src/coreclr/jit/async.cpp | 68 +++++++++++++++++++++---------- src/coreclr/jit/importercalls.cpp | 24 +++++++---- src/coreclr/jit/inline.def | 2 +- 3 files changed, 63 insertions(+), 31 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 705c3af4b4a49b..8e05c81b3bdd80 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -184,9 +184,17 @@ PhaseStatus Compiler::SaveAsyncContexts() // Insert RestoreContexts call in fault (exceptional case) // First argument: started = (continuation == null) - GenTree* continuation = gtNewLclvNode(lvaAsyncContinuationArg, TYP_REF); - GenTree* null = gtNewNull(); - GenTree* resumed = gtNewOperNode(GT_NE, TYP_INT, continuation, null); + GenTree* resumed; + if (compIsForInlining()) + { + resumed = gtNewFalse(); + } + else + { + GenTree* continuation = gtNewLclvNode(lvaAsyncContinuationArg, TYP_REF); + GenTree* null = gtNewNull(); + resumed = gtNewOperNode(GT_NE, TYP_INT, continuation, null); + } GenTreeCall* restoreCall = gtNewCallNode(CT_USER_FUNC, asyncInfo->restoreContextsMethHnd, TYP_VOID); restoreCall->gtArgs.PushFront(this, @@ -205,7 +213,10 @@ PhaseStatus Compiler::SaveAsyncContexts() for (BasicBlock* block : Blocks()) { - AddContextArgsToAsyncCalls(block); + if (!compIsForInlining()) + { + AddContextArgsToAsyncCalls(block); + } if (!block->KindIs(BBJ_RETURN) || (block == newReturnBB)) { @@ -220,23 +231,28 @@ PhaseStatus Compiler::SaveAsyncContexts() newReturnBB->inheritWeightPercentage(block, 0); } - // Store return value to common local - Statement* retStmt = block->lastStmt(); - assert((retStmt != nullptr) && retStmt->GetRootNode()->OperIs(GT_RETURN)); - - if (mergedReturnLcl != BAD_VAR_NUM) + // When inlining we do merging during import, so we do not need to do + // any storing there. + if (!compIsForInlining()) { - GenTree* retVal = retStmt->GetRootNode()->AsOp()->GetReturnValue(); - Statement* insertAfter = retStmt; - GenTree* storeRetVal = - gtNewTempStore(mergedReturnLcl, retVal, CHECK_SPILL_NONE, &insertAfter, retStmt->GetDebugInfo(), block); - Statement* storeStmt = fgNewStmtFromTree(storeRetVal); - fgInsertStmtAtEnd(block, storeStmt); - JITDUMP("Inserted store to common return local\n"); - DISPSTMT(storeStmt); - } + // Store return value to common local + Statement* retStmt = block->lastStmt(); + assert((retStmt != nullptr) && retStmt->GetRootNode()->OperIs(GT_RETURN)); - retStmt->GetRootNode()->gtBashToNOP(); + if (mergedReturnLcl != BAD_VAR_NUM) + { + GenTree* retVal = retStmt->GetRootNode()->AsOp()->GetReturnValue(); + Statement* insertAfter = retStmt; + GenTree* storeRetVal = + gtNewTempStore(mergedReturnLcl, retVal, CHECK_SPILL_NONE, &insertAfter, retStmt->GetDebugInfo(), block); + Statement* storeStmt = fgNewStmtFromTree(storeRetVal); + fgInsertStmtAtEnd(block, storeStmt); + JITDUMP("Inserted store to common return local\n"); + DISPSTMT(storeStmt); + } + + retStmt->GetRootNode()->gtBashToNOP(); + } // Jump to new shared restore + return block block->SetKindAndTargetEdge(BBJ_ALWAYS, fgAddRefPred(newReturnBB, block)); @@ -334,9 +350,17 @@ BasicBlock* Compiler::CreateReturnBB(unsigned* mergedReturnLcl) // Insert "restore" call CORINFO_ASYNC_INFO* asyncInfo = eeGetAsyncInfo(); - GenTree* continuation = gtNewLclvNode(lvaAsyncContinuationArg, TYP_REF); - GenTree* null = gtNewNull(); - GenTree* resumed = gtNewOperNode(GT_NE, TYP_INT, continuation, null); + GenTree* resumed; + if (compIsForInlining()) + { + resumed = gtNewFalse(); + } + else + { + GenTree* continuation = gtNewLclvNode(lvaAsyncContinuationArg, TYP_REF); + GenTree* null = gtNewNull(); + resumed = gtNewOperNode(GT_NE, TYP_INT, continuation, null); + } GenTreeCall* restoreCall = gtNewCallNode(CT_USER_FUNC, asyncInfo->restoreContextsMethHnd, TYP_VOID); restoreCall->gtArgs.PushFront(this, diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 1c90787d46ac1b..25b4d6de057570 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -404,6 +404,11 @@ var_types Compiler::impImportCall(OPCODE opcode, if (sig->isAsyncCall()) { impSetupAsyncCall(call->AsCall(), opcode, prefixFlags, di); + + if (compDonotInline()) + { + return TYP_UNDEF; + } } impPopCallArgs(sig, call->AsCall()); @@ -717,6 +722,11 @@ var_types Compiler::impImportCall(OPCODE opcode, { impSetupAsyncCall(call->AsCall(), opcode, prefixFlags, di); + if (compDonotInline()) + { + return TYP_UNDEF; + } + if (lvaNextCallAsyncContinuation != BAD_VAR_NUM) { asyncContinuation = gtNewLclVarNode(lvaNextCallAsyncContinuation); @@ -6868,6 +6878,12 @@ void Compiler::impCheckForPInvokeCall( // void Compiler::impSetupAsyncCall(GenTreeCall* call, OPCODE opcode, unsigned prefixFlags, const DebugInfo& callDI) { + if (compIsForInlining()) + { + compInlineResult->NoteFatal(InlineObservation::CALLEE_AWAIT); + return; + } + AsyncCallInfo asyncInfo; unsigned newSourceTypes = ICorDebugInfo::ASYNC; @@ -8123,14 +8139,6 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, return; } - if (call->IsAsync() && (call->GetAsyncInfo().ContinuationContextHandling != ContinuationContextHandling::None)) - { - // Cannot currently handle moving to captured context/thread pool when logically returning from inlinee. - // - inlineResult->NoteFatal(InlineObservation::CALLSITE_CONTINUATION_HANDLING); - return; - } - // Ignore indirect calls, unless they are indirect virtual stub calls with profile info. // if (call->gtCallType == CT_INDIRECT) diff --git a/src/coreclr/jit/inline.def b/src/coreclr/jit/inline.def index bb6d2226cea21a..4e52e7087cd715 100644 --- a/src/coreclr/jit/inline.def +++ b/src/coreclr/jit/inline.def @@ -56,6 +56,7 @@ INLINE_OBSERVATION(STACK_CRAWL_MARK, bool, "uses stack crawl mark", INLINE_OBSERVATION(STFLD_NEEDS_HELPER, bool, "stfld needs helper", FATAL, CALLEE) INLINE_OBSERVATION(TOO_MANY_ARGUMENTS, bool, "too many arguments", FATAL, CALLEE) INLINE_OBSERVATION(TOO_MANY_LOCALS, bool, "too many locals", FATAL, CALLEE) +INLINE_OBSERVATION(AWAIT, bool, "has await", FATAL, CALLEE) // ------ Callee Performance ------- @@ -161,7 +162,6 @@ INLINE_OBSERVATION(RETURN_TYPE_MISMATCH, bool, "return type mismatch", INLINE_OBSERVATION(STFLD_NEEDS_HELPER, bool, "stfld needs helper", FATAL, CALLSITE) INLINE_OBSERVATION(TOO_MANY_LOCALS, bool, "too many locals", FATAL, CALLSITE) INLINE_OBSERVATION(PINVOKE_EH, bool, "PInvoke call site with EH", FATAL, CALLSITE) -INLINE_OBSERVATION(CONTINUATION_HANDLING, bool, "Callsite needs continuation handling", FATAL, CALLSITE) // ------ Call Site Performance ------- From 259e2fd7feb49f186d15ccb381cff5511a4724df Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 9 Feb 2026 17:48:16 +0100 Subject: [PATCH 2/6] Run jit-format --- src/coreclr/jit/async.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 8e05c81b3bdd80..98fa557ce55966 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -193,7 +193,7 @@ PhaseStatus Compiler::SaveAsyncContexts() { GenTree* continuation = gtNewLclvNode(lvaAsyncContinuationArg, TYP_REF); GenTree* null = gtNewNull(); - resumed = gtNewOperNode(GT_NE, TYP_INT, continuation, null); + resumed = gtNewOperNode(GT_NE, TYP_INT, continuation, null); } GenTreeCall* restoreCall = gtNewCallNode(CT_USER_FUNC, asyncInfo->restoreContextsMethHnd, TYP_VOID); @@ -243,9 +243,9 @@ PhaseStatus Compiler::SaveAsyncContexts() { GenTree* retVal = retStmt->GetRootNode()->AsOp()->GetReturnValue(); Statement* insertAfter = retStmt; - GenTree* storeRetVal = - gtNewTempStore(mergedReturnLcl, retVal, CHECK_SPILL_NONE, &insertAfter, retStmt->GetDebugInfo(), block); - Statement* storeStmt = fgNewStmtFromTree(storeRetVal); + GenTree* storeRetVal = gtNewTempStore(mergedReturnLcl, retVal, CHECK_SPILL_NONE, &insertAfter, + retStmt->GetDebugInfo(), block); + Statement* storeStmt = fgNewStmtFromTree(storeRetVal); fgInsertStmtAtEnd(block, storeStmt); JITDUMP("Inserted store to common return local\n"); DISPSTMT(storeStmt); @@ -359,7 +359,7 @@ BasicBlock* Compiler::CreateReturnBB(unsigned* mergedReturnLcl) { GenTree* continuation = gtNewLclvNode(lvaAsyncContinuationArg, TYP_REF); GenTree* null = gtNewNull(); - resumed = gtNewOperNode(GT_NE, TYP_INT, continuation, null); + resumed = gtNewOperNode(GT_NE, TYP_INT, continuation, null); } GenTreeCall* restoreCall = gtNewCallNode(CT_USER_FUNC, asyncInfo->restoreContextsMethHnd, TYP_VOID); From a228333995fadec90e7c0b6223702516b0e990e8 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 9 Feb 2026 17:54:25 +0100 Subject: [PATCH 3/6] Account for extra EH clause --- src/coreclr/jit/fgbasic.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/fgbasic.cpp b/src/coreclr/jit/fgbasic.cpp index 9af008e90f3300..0ba1d827bfcfb5 100644 --- a/src/coreclr/jit/fgbasic.cpp +++ b/src/coreclr/jit/fgbasic.cpp @@ -3464,7 +3464,14 @@ void Compiler::fgFindBasicBlocks() // Verify we can expand the EH table as needed to incorporate the callee's EH clauses. // Failing here should be extremely rare. // - EHblkDsc* const dsc = fgTryAddEHTableEntries(0, info.compXcptnsCount, /* deferAdding */ true); + unsigned numEHEntries = info.compXcptnsCount; + // We will introduce another EH clause in the inlinee to restore async contexts + if ((info.compMethodInfo->options & CORINFO_ASYNC_SAVE_CONTEXTS) != 0) + { + numEHEntries++; + } + + EHblkDsc* const dsc = fgTryAddEHTableEntries(0, numEHEntries, /* deferAdding */ true); if (dsc == nullptr) { compInlineResult->NoteFatal(InlineObservation::CALLSITE_EH_TABLE_FULL); From f9e29793ecdcb8a4da60fd55d4fc18d13699540e Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 10 Feb 2026 13:10:20 +0100 Subject: [PATCH 4/6] Avoid inserting GT_RETURN --- src/coreclr/jit/async.cpp | 50 ++++++++++++++++++++----------------- src/coreclr/jit/fgbasic.cpp | 2 +- 2 files changed, 28 insertions(+), 24 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 98fa557ce55966..592b0dd8518712 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -379,42 +379,46 @@ BasicBlock* Compiler::CreateReturnBB(unsigned* mergedReturnLcl) JITDUMP("Inserted restore statement in return block\n"); DISPSTMT(restoreStmt); - *mergedReturnLcl = BAD_VAR_NUM; - - GenTree* ret; - if (compMethodHasRetVal()) + if (!compIsForInlining()) { - *mergedReturnLcl = lvaGrabTemp(false DEBUGARG("Async merged return local")); - - var_types retLclType = compMethodReturnsRetBufAddr() ? TYP_BYREF : genActualType(info.compRetType); + *mergedReturnLcl = BAD_VAR_NUM; - if (varTypeIsStruct(retLclType)) + GenTree* ret; + if (compMethodHasRetVal()) { - lvaSetStruct(*mergedReturnLcl, info.compMethodInfo->args.retTypeClass, false); + *mergedReturnLcl = lvaGrabTemp(false DEBUGARG("Async merged return local")); - if (compMethodReturnsMultiRegRetType()) + var_types retLclType = compMethodReturnsRetBufAddr() ? TYP_BYREF : genActualType(info.compRetType); + + if (varTypeIsStruct(retLclType)) { - lvaGetDesc(*mergedReturnLcl)->lvIsMultiRegRet = true; + lvaSetStruct(*mergedReturnLcl, info.compMethodInfo->args.retTypeClass, false); + + if (compMethodReturnsMultiRegRetType()) + { + lvaGetDesc(*mergedReturnLcl)->lvIsMultiRegRet = true; + } } + else + { + lvaGetDesc(*mergedReturnLcl)->lvType = retLclType; + } + + GenTree* retTemp = gtNewLclVarNode(*mergedReturnLcl); + ret = gtNewOperNode(GT_RETURN, retTemp->TypeGet(), retTemp); } else { - lvaGetDesc(*mergedReturnLcl)->lvType = retLclType; + ret = new (this, GT_RETURN) GenTreeOp(GT_RETURN, TYP_VOID); } - GenTree* retTemp = gtNewLclVarNode(*mergedReturnLcl); - ret = gtNewOperNode(GT_RETURN, retTemp->TypeGet(), retTemp); - } - else - { - ret = new (this, GT_RETURN) GenTreeOp(GT_RETURN, TYP_VOID); - } + Statement* retStmt = fgNewStmtFromTree(ret); - Statement* retStmt = fgNewStmtFromTree(ret); + fgInsertStmtAtEnd(newReturnBB, retStmt); + JITDUMP("Inserted return statement in return block\n"); + DISPSTMT(retStmt); + } - fgInsertStmtAtEnd(newReturnBB, retStmt); - JITDUMP("Inserted return statement in return block\n"); - DISPSTMT(retStmt); return newReturnBB; } diff --git a/src/coreclr/jit/fgbasic.cpp b/src/coreclr/jit/fgbasic.cpp index 0ba1d827bfcfb5..a3dfda0553f592 100644 --- a/src/coreclr/jit/fgbasic.cpp +++ b/src/coreclr/jit/fgbasic.cpp @@ -3465,7 +3465,7 @@ void Compiler::fgFindBasicBlocks() // Failing here should be extremely rare. // unsigned numEHEntries = info.compXcptnsCount; - // We will introduce another EH clause in the inlinee to restore async contexts + // We will introduce another EH clause before inlining finishes to restore async contexts if ((info.compMethodInfo->options & CORINFO_ASYNC_SAVE_CONTEXTS) != 0) { numEHEntries++; From 5d06522966a7c642ec245d2bebf9ce7658606749 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 10 Feb 2026 17:35:18 +0100 Subject: [PATCH 5/6] Feedback --- src/coreclr/jit/fgbasic.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/fgbasic.cpp b/src/coreclr/jit/fgbasic.cpp index a3dfda0553f592..ae64f2e754dffc 100644 --- a/src/coreclr/jit/fgbasic.cpp +++ b/src/coreclr/jit/fgbasic.cpp @@ -3455,7 +3455,7 @@ void Compiler::fgFindBasicBlocks() // Are there any exception handlers? // - if (info.compXcptnsCount > 0) + if (info.compXcptnsCount > 0 || ((info.compMethodInfo->options & CORINFO_ASYNC_SAVE_CONTEXTS) != 0)) { assert(!compIsForInlining() || opts.compInlineMethodsWithEH); From 0639eeabebbaf84cf9372ce62e52f034cebef3ad Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 10 Feb 2026 17:44:07 +0100 Subject: [PATCH 6/6] More feedback --- src/coreclr/jit/async.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 592b0dd8518712..dc9d4624c53e94 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -183,7 +183,7 @@ PhaseStatus Compiler::SaveAsyncContexts() } // Insert RestoreContexts call in fault (exceptional case) - // First argument: started = (continuation == null) + // First argument: resumed = (continuation != null) GenTree* resumed; if (compIsForInlining()) {