perf(decode): hot-loop register-pressure trims + release-plz concurrency guard

.github/workflows/release-plz.yml

@@ -5,6 +5,16 @@ on:
     branches:
       - main
 
+# Serialize release-plz runs. Two pushes landing within ~1 min (two PRs merged
+# back-to-back) otherwise start two overlapping runs that race on the release-PR
+# git refs: one closes the release PR, the other's recreate fails on the
+# already-taken ref, leaving the release PR closed with no replacement.
+# `cancel-in-progress: false` QUEUES the second run rather than cancelling it —
+# the `release` (publish) step must never be interrupted mid-publish.
+concurrency:
+  group: release-plz-${{ github.ref }}
+  cancel-in-progress: false
+
 jobs:
   release-plz:
     runs-on: ubuntu-latest

zstd/src/decoding/literals_section_decoder.rs

@@ -719,10 +719,6 @@ unsafe fn run_4stream_burst_loop<K: CpuKernel>(
     let mut ip1 = brs[1].index;
     let mut ip2 = brs[2].index;
     let mut ip3 = brs[3].index;
-    let mut nbl0 = brs[0].bits_consumed - max_num_bits;
-    let mut nbl1 = brs[1].bits_consumed - max_num_bits;
-    let mut nbl2 = brs[2].bits_consumed - max_num_bits;
-    let mut nbl3 = brs[3].bits_consumed - max_num_bits;
     let mut c0 = cursors[0];
     let mut c1 = cursors[1];
     let mut c2 = cursors[2];
@@ -755,7 +751,7 @@ unsafe fn run_4stream_burst_loop<K: CpuKernel>(
     // `min_ip >= bytes_per_iter_upper` gate at loop entry keeps `ip -=
     // nb_bytes` and the 8-byte window read in-bounds (see the budget note).
     macro_rules! reload1 {
-        ($b:ident, $ip:ident, $nbl:ident, $src:expr) => {{
+        ($b:ident, $ip:ident, $src:expr) => {{
             let ctz = $b.trailing_zeros();
             $ip -= (ctz >> 3) as usize;
             let nb_bits = (ctz & 7) as u8;
@@ -765,7 +761,6 @@ unsafe fn run_4stream_burst_loop<K: CpuKernel>(
                     .unwrap_unchecked()
             });
             $b = (new_window | 1) << nb_bits;
-            $nbl = nb_bits;
         }};
     }
     // One burst: `$n` symbols across all four streams. `$n` is a literal so
@@ -847,10 +842,10 @@ unsafe fn run_4stream_burst_loop<K: CpuKernel>(
         // the sentinel must land at bit `nb_bits` so the next reload's `ctz`
         // accumulates the sub-byte phase; resetting it to bit 0 loses the
         // phase between reloads.
-        reload1!(b0, ip0, nbl0, src0);
-        reload1!(b1, ip1, nbl1, src1);
-        reload1!(b2, ip2, nbl2, src2);
-        reload1!(b3, ip3, nbl3, src3);
+        reload1!(b0, ip0, src0);
+        reload1!(b1, ip1, src1);
+        reload1!(b2, ip2, src2);
+        reload1!(b3, ip3, src3);
     }
 
     // Commit cursors to the caller's array (the drain phase reads them)
@@ -871,21 +866,26 @@ unsafe fn run_4stream_burst_loop<K: CpuKernel>(
     // sub-byte phase from the last reload plus the `max_num_bits` consumed
     // for the state just extracted), `state = b >> table_shift`.
     macro_rules! writeback {
-        ($i:literal, $b:ident, $ip:ident, $nbl:ident, $src:expr) => {{
+        ($i:literal, $b:ident, $ip:ident, $src:expr) => {{
             brs[$i].index = $ip;
             brs[$i].bit_container = u64::from_le_bytes(unsafe {
                 $src.get_unchecked($ip..$ip + 8)
                     .try_into()
                     .unwrap_unchecked()
             });
-            brs[$i].bits_consumed = $nbl + max_num_bits;
+            // bits_consumed = sub-byte phase + max_num_bits. After the final
+            // reload b{s} = (window | 1) << nb_bits, so trailing_zeros(b{s}) is
+            // exactly that nb_bits (the sentinel sits at bit nb_bits). Recompute
+            // it here instead of carrying a per-stream `nbl` register across the
+            // whole burst loop — four fewer loop-live values.
+            brs[$i].bits_consumed = $b.trailing_zeros() as u8 + max_num_bits;
             decoders[$i].state = $b >> table_shift;
         }};
     }
-    writeback!(0, b0, ip0, nbl0, src0);
-    writeback!(1, b1, ip1, nbl1, src1);
-    writeback!(2, b2, ip2, nbl2, src2);
-    writeback!(3, b3, ip3, nbl3, src3);
+    writeback!(0, b0, ip0, src0);
+    writeback!(1, b1, ip1, src1);
+    writeback!(2, b2, ip2, src2);
+    writeback!(3, b3, ip3, src3);
 }
 
 #[cfg(test)]

zstd/src/decoding/seq_decoder_avx2.rs

@@ -50,17 +50,17 @@ macro_rules! decode_one_body {
 
         let sum_wide = u16::from(of_num_bits) + u16::from(ml_num_bits) + u16::from(ll_num_bits);
         let (obits, ml_add, ll_add) = if sum_wide <= 56 {
-            let sum = sum_wide as u8;
-            $br.ensure_bits(sum);
-            // SAFETY: enclosing fn is target_feature(bmi2,avx2); vendor
-            // policy cached at BitReader::new gates the PEXT-direct path.
-            let triple = if $br.use_pext_triple_fast() {
-                unsafe { $br.peek_bits_triple_bmi2(sum, of_num_bits, ml_num_bits, ll_num_bits) }
-            } else {
-                $br.peek_bits_triple(sum, of_num_bits, ml_num_bits, ll_num_bits)
-            };
-            $br.consume(sum);
-            triple
+            // Upstream `ZSTD_decodeSequence` reads OF/ML/LL as three separate
+            // `BIT_readBitsFast` after one reload. The fields are independent so
+            // the CPU pipelines them; the old PEXT-triple folded all three into
+            // one serial `pext` dependency. One `ensure_bits` up front, then
+            // three unchecked reads (same bit order, byte-identical).
+            $br.ensure_bits(sum_wide as u8);
+            (
+                $br.get_bits_unchecked(of_num_bits),
+                $br.get_bits_unchecked(ml_num_bits),
+                $br.get_bits_unchecked(ll_num_bits),
+            )
         } else {
             (
                 $br.get_bits(of_num_bits),
@@ -481,6 +481,20 @@ pub(crate) unsafe fn decode_and_execute_sequences_avx2<'fse, B: BufferBackend>(
                 &mut br,
                 &mut shadow_hist
             );
+            // Advance the FSE states for the NEXT sequence before executing the
+            // current one, mirroring upstream `ZSTD_decodeSequence` (which
+            // updates the states inside decode, then calls `ZSTD_execSequence`).
+            // The execute reads no bits, so moving it after the state update is
+            // byte-identical (value bits then state-transition bits are consumed
+            // in the same order); it stops the three FSE states and the bit
+            // reader from staying live across the heavy match copy, cutting
+            // register pressure in the hot loop.
+            if i + 1 < num_sequences {
+                br.ensure_bits(max_update_bits);
+                ll_dec.update_state_fast(&mut br);
+                ml_dec.update_state_fast(&mut br);
+                of_dec.update_state_fast(&mut br);
+            }
             let r = execute_one_body!(
                 buffer,
                 dict,
@@ -496,13 +510,6 @@ pub(crate) unsafe fn decode_and_execute_sequences_avx2<'fse, B: BufferBackend>(
                 break;
             }
             seq_sum = seq_sum.wrapping_add(seq_ll).wrapping_add(seq_ml);
-
-            if i + 1 < num_sequences {
-                br.ensure_bits(max_update_bits);
-                ll_dec.update_state_fast(&mut br);
-                ml_dec.update_state_fast(&mut br);
-                of_dec.update_state_fast(&mut br);
-            }
         }
         if let Some(e) = fallback_err {
             let _ = buffer.try_restore_checkpoint(buffer_checkpoint);