From 350f08f2943fe2ad7175e3e36eccae344d5e3cc4 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 27 Jun 2026 00:24:04 +0300 Subject: [PATCH 1/3] ci(release): serialize release-plz runs to prevent release-PR race Two PRs merged back-to-back start two overlapping release-plz runs that race on the release-PR git refs: one closes the release PR while the other's recreate fails on the already-taken ref, leaving it closed with no replacement. Add a concurrency group so a second run queues behind the first; cancel-in-progress stays false so the publish step is never interrupted. --- .github/workflows/release-plz.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/release-plz.yml b/.github/workflows/release-plz.yml index da388468b..b42bc2f72 100644 --- a/.github/workflows/release-plz.yml +++ b/.github/workflows/release-plz.yml @@ -5,6 +5,16 @@ on: branches: - main +# Serialize release-plz runs. Two pushes landing within ~1 min (two PRs merged +# back-to-back) otherwise start two overlapping runs that race on the release-PR +# git refs: one closes the release PR, the other's recreate fails on the +# already-taken ref, leaving the release PR closed with no replacement. +# `cancel-in-progress: false` QUEUES the second run rather than cancelling it — +# the `release` (publish) step must never be interrupted mid-publish. +concurrency: + group: release-plz-${{ github.ref }} + cancel-in-progress: false + jobs: release-plz: runs-on: ubuntu-latest From 64edd42e868f93308bc44474cacc430633ca6ba8 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 27 Jun 2026 00:24:17 +0300 Subject: [PATCH 2/3] perf(decode): trim sequence-loop register pressure Two byte-identical op-reductions in the AVX2 sequence decoder, both perf-neutral on z000033 L1 (verified by perf stat instructions on i9; criterion without a c_ffi control arm gave false regressions): read OF/ML/LL as three separate pipelined reads instead of one serial PEXT-triple in the long-pipeline arm, and advance the FSE states before executing the current sequence in the short arm (mirrors upstream ZSTD_decodeSequence; the execute reads no bits so the order is byte-identical) to stop the states living across the match copy. --- zstd/src/decoding/seq_decoder_avx2.rs | 43 ++++++++++++++++----------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/zstd/src/decoding/seq_decoder_avx2.rs b/zstd/src/decoding/seq_decoder_avx2.rs index 38f566f31..5ceda48f9 100644 --- a/zstd/src/decoding/seq_decoder_avx2.rs +++ b/zstd/src/decoding/seq_decoder_avx2.rs @@ -50,17 +50,17 @@ macro_rules! decode_one_body { let sum_wide = u16::from(of_num_bits) + u16::from(ml_num_bits) + u16::from(ll_num_bits); let (obits, ml_add, ll_add) = if sum_wide <= 56 { - let sum = sum_wide as u8; - $br.ensure_bits(sum); - // SAFETY: enclosing fn is target_feature(bmi2,avx2); vendor - // policy cached at BitReader::new gates the PEXT-direct path. - let triple = if $br.use_pext_triple_fast() { - unsafe { $br.peek_bits_triple_bmi2(sum, of_num_bits, ml_num_bits, ll_num_bits) } - } else { - $br.peek_bits_triple(sum, of_num_bits, ml_num_bits, ll_num_bits) - }; - $br.consume(sum); - triple + // Upstream `ZSTD_decodeSequence` reads OF/ML/LL as three separate + // `BIT_readBitsFast` after one reload. The fields are independent so + // the CPU pipelines them; the old PEXT-triple folded all three into + // one serial `pext` dependency. One `ensure_bits` up front, then + // three unchecked reads (same bit order, byte-identical). + $br.ensure_bits(sum_wide as u8); + ( + $br.get_bits_unchecked(of_num_bits), + $br.get_bits_unchecked(ml_num_bits), + $br.get_bits_unchecked(ll_num_bits), + ) } else { ( $br.get_bits(of_num_bits), @@ -481,6 +481,20 @@ pub(crate) unsafe fn decode_and_execute_sequences_avx2<'fse, B: BufferBackend>( &mut br, &mut shadow_hist ); + // Advance the FSE states for the NEXT sequence before executing the + // current one, mirroring upstream `ZSTD_decodeSequence` (which + // updates the states inside decode, then calls `ZSTD_execSequence`). + // The execute reads no bits, so moving it after the state update is + // byte-identical (value bits then state-transition bits are consumed + // in the same order); it stops the three FSE states and the bit + // reader from staying live across the heavy match copy, cutting + // register pressure in the hot loop. + if i + 1 < num_sequences { + br.ensure_bits(max_update_bits); + ll_dec.update_state_fast(&mut br); + ml_dec.update_state_fast(&mut br); + of_dec.update_state_fast(&mut br); + } let r = execute_one_body!( buffer, dict, @@ -496,13 +510,6 @@ pub(crate) unsafe fn decode_and_execute_sequences_avx2<'fse, B: BufferBackend>( break; } seq_sum = seq_sum.wrapping_add(seq_ll).wrapping_add(seq_ml); - - if i + 1 < num_sequences { - br.ensure_bits(max_update_bits); - ll_dec.update_state_fast(&mut br); - ml_dec.update_state_fast(&mut br); - of_dec.update_state_fast(&mut br); - } } if let Some(e) = fallback_err { let _ = buffer.try_restore_checkpoint(buffer_checkpoint); From 2eaa22fefad9c3a4e85fae15dd56d8771d619cb6 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 27 Jun 2026 00:24:17 +0300 Subject: [PATCH 3/3] perf(decode): drop carried nbl registers from HUF burst loop The four per-stream sub-byte phases were written every reload but read only at burst writeback, where trailing_zeros of the just-reloaded register recovers the same value. Recompute it there instead of carrying four registers across the burst. Instruction-identical (the optimiser already folded it), byte-identical. --- zstd/src/decoding/literals_section_decoder.rs | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/zstd/src/decoding/literals_section_decoder.rs b/zstd/src/decoding/literals_section_decoder.rs index cb343ec36..23af0dd6a 100644 --- a/zstd/src/decoding/literals_section_decoder.rs +++ b/zstd/src/decoding/literals_section_decoder.rs @@ -719,10 +719,6 @@ unsafe fn run_4stream_burst_loop( let mut ip1 = brs[1].index; let mut ip2 = brs[2].index; let mut ip3 = brs[3].index; - let mut nbl0 = brs[0].bits_consumed - max_num_bits; - let mut nbl1 = brs[1].bits_consumed - max_num_bits; - let mut nbl2 = brs[2].bits_consumed - max_num_bits; - let mut nbl3 = brs[3].bits_consumed - max_num_bits; let mut c0 = cursors[0]; let mut c1 = cursors[1]; let mut c2 = cursors[2]; @@ -755,7 +751,7 @@ unsafe fn run_4stream_burst_loop( // `min_ip >= bytes_per_iter_upper` gate at loop entry keeps `ip -= // nb_bytes` and the 8-byte window read in-bounds (see the budget note). macro_rules! reload1 { - ($b:ident, $ip:ident, $nbl:ident, $src:expr) => {{ + ($b:ident, $ip:ident, $src:expr) => {{ let ctz = $b.trailing_zeros(); $ip -= (ctz >> 3) as usize; let nb_bits = (ctz & 7) as u8; @@ -765,7 +761,6 @@ unsafe fn run_4stream_burst_loop( .unwrap_unchecked() }); $b = (new_window | 1) << nb_bits; - $nbl = nb_bits; }}; } // One burst: `$n` symbols across all four streams. `$n` is a literal so @@ -847,10 +842,10 @@ unsafe fn run_4stream_burst_loop( // the sentinel must land at bit `nb_bits` so the next reload's `ctz` // accumulates the sub-byte phase; resetting it to bit 0 loses the // phase between reloads. - reload1!(b0, ip0, nbl0, src0); - reload1!(b1, ip1, nbl1, src1); - reload1!(b2, ip2, nbl2, src2); - reload1!(b3, ip3, nbl3, src3); + reload1!(b0, ip0, src0); + reload1!(b1, ip1, src1); + reload1!(b2, ip2, src2); + reload1!(b3, ip3, src3); } // Commit cursors to the caller's array (the drain phase reads them) @@ -871,21 +866,26 @@ unsafe fn run_4stream_burst_loop( // sub-byte phase from the last reload plus the `max_num_bits` consumed // for the state just extracted), `state = b >> table_shift`. macro_rules! writeback { - ($i:literal, $b:ident, $ip:ident, $nbl:ident, $src:expr) => {{ + ($i:literal, $b:ident, $ip:ident, $src:expr) => {{ brs[$i].index = $ip; brs[$i].bit_container = u64::from_le_bytes(unsafe { $src.get_unchecked($ip..$ip + 8) .try_into() .unwrap_unchecked() }); - brs[$i].bits_consumed = $nbl + max_num_bits; + // bits_consumed = sub-byte phase + max_num_bits. After the final + // reload b{s} = (window | 1) << nb_bits, so trailing_zeros(b{s}) is + // exactly that nb_bits (the sentinel sits at bit nb_bits). Recompute + // it here instead of carrying a per-stream `nbl` register across the + // whole burst loop — four fewer loop-live values. + brs[$i].bits_consumed = $b.trailing_zeros() as u8 + max_num_bits; decoders[$i].state = $b >> table_shift; }}; } - writeback!(0, b0, ip0, nbl0, src0); - writeback!(1, b1, ip1, nbl1, src1); - writeback!(2, b2, ip2, nbl2, src2); - writeback!(3, b3, ip3, nbl3, src3); + writeback!(0, b0, ip0, src0); + writeback!(1, b1, ip1, src1); + writeback!(2, b2, ip2, src2); + writeback!(3, b3, ip3, src3); } #[cfg(test)]