From ac7ca94a03463c9f5c6c205838646ffebb2943cc Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Sat, 11 Jul 2026 01:16:01 +0200 Subject: [PATCH 1/2] JIT: Coalesce stores from struct copies Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: f267b92e-719f-4b0d-b19f-eab8f97611f5 --- src/coreclr/jit/morphblock.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/coreclr/jit/morphblock.cpp b/src/coreclr/jit/morphblock.cpp index 322253d0b11d9a..fd176b79ce6e7e 100644 --- a/src/coreclr/jit/morphblock.cpp +++ b/src/coreclr/jit/morphblock.cpp @@ -1397,6 +1397,12 @@ GenTree* MorphCopyBlockHelper::CopyFieldByField() indirFlags |= GTF_IND_TGT_NOT_HEAP; } } + if (m_store->OperIsCopyBlkOp() && !varTypeIsGC(srcType)) + { + // This store is one piece of copying an entire struct from source to destination. + // Our memory model allows us to do this in a non-atomic way, so we can mark this store as such. + indirFlags |= GTF_IND_ALLOW_NON_ATOMIC; + } dstFldStore = m_compiler->gtNewStoreIndNode(srcType, fldAddr, srcFld, indirFlags); } } From 95a27d70107621a91ca2215b00e98387b44fd6a1 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Mon, 13 Jul 2026 14:24:19 +0200 Subject: [PATCH 2/2] JIT: Mark decomposed copy accesses non-atomic Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 572375cc-ea86-40c2-9aef-a17eda0d581a --- src/coreclr/jit/morphblock.cpp | 17 ++++++++--------- src/coreclr/jit/promotiondecomposition.cpp | 11 +++++++++-- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/coreclr/jit/morphblock.cpp b/src/coreclr/jit/morphblock.cpp index fd176b79ce6e7e..45d9c5a13127f9 100644 --- a/src/coreclr/jit/morphblock.cpp +++ b/src/coreclr/jit/morphblock.cpp @@ -1190,6 +1190,11 @@ GenTree* MorphCopyBlockHelper::CopyFieldByField() return tree; }; + auto getCopyIndirFlags = [](var_types type) { + // These accesses are pieces of a whole struct copy, so non-GC accesses do not need to be atomic. + return varTypeIsGC(type) ? GTF_EMPTY : GTF_IND_ALLOW_NON_ATOMIC; + }; + auto grabAddr = [=, &result](unsigned offs) { GenTree* addrClone = nullptr; // Need address of the source. @@ -1337,7 +1342,7 @@ GenTree* MorphCopyBlockHelper::CopyFieldByField() else { GenTree* fldAddr = grabAddr(fldOffset); - srcFld = m_compiler->gtNewIndir(destType, fldAddr); + srcFld = m_compiler->gtNewIndir(destType, fldAddr, getCopyIndirFlags(destType)); } } } @@ -1388,21 +1393,15 @@ GenTree* MorphCopyBlockHelper::CopyFieldByField() else { GenTree* fldAddr = grabAddr(srcFieldOffset); - GenTreeFlags indirFlags = GTF_EMPTY; + GenTreeFlags indirFlags = getCopyIndirFlags(srcType); if (m_store->OperIs(GT_STORE_BLK, GT_STOREIND)) { - indirFlags = m_store->gtFlags & (GTF_IND_TGT_NOT_HEAP | GTF_IND_TGT_HEAP); + indirFlags |= m_store->gtFlags & (GTF_IND_TGT_NOT_HEAP | GTF_IND_TGT_HEAP); if (m_store->OperIs(GT_STORE_BLK) && m_store->AsBlk()->GetLayout()->IsStackOnly(m_compiler)) { indirFlags |= GTF_IND_TGT_NOT_HEAP; } } - if (m_store->OperIsCopyBlkOp() && !varTypeIsGC(srcType)) - { - // This store is one piece of copying an entire struct from source to destination. - // Our memory model allows us to do this in a non-atomic way, so we can mark this store as such. - indirFlags |= GTF_IND_ALLOW_NON_ATOMIC; - } dstFldStore = m_compiler->gtNewStoreIndNode(srcType, fldAddr, srcFld, indirFlags); } } diff --git a/src/coreclr/jit/promotiondecomposition.cpp b/src/coreclr/jit/promotiondecomposition.cpp index a089fbfe370b5d..b7fce9154bbce9 100644 --- a/src/coreclr/jit/promotiondecomposition.cpp +++ b/src/coreclr/jit/promotiondecomposition.cpp @@ -1159,12 +1159,19 @@ class DecompositionPlan // GenTreeFlags GetIndirFlags(var_types type) { + GenTreeFlags flags = m_indirFlags; + if (!varTypeIsGC(type)) + { + // These accesses are pieces of a whole struct copy, so they do not need to be atomic. + flags |= GTF_IND_ALLOW_NON_ATOMIC; + } + if (genTypeSize(type) == 1) { - return m_indirFlags & ~GTF_IND_UNALIGNED; + flags &= ~GTF_IND_UNALIGNED; } - return m_indirFlags; + return flags; } };