perf(decode): expand overlapping matches by doubling, not offset chunks

zstd/Cargo.toml

@@ -70,6 +70,13 @@ critical-section = ["dep:critical-section"]
 bench_internals = []
 fuzz_exports = []
 std = []
+# Diagnostic-only: atomic histograms of the match/literal copy shape on the
+# decode path (call counts + size buckets + requested-vs-overshoot byte
+# totals). Off in every shipping / bench build (zero codegen impact). Enabled
+# by the `copy_shape` example to capture the copy-call distribution, which is
+# deterministic from the compressed input and therefore architecture-
+# independent (only the per-call timing is CPU-tier specific).
+copy_shape_stats = ["std"]
 # Diagnostic tracing of the Fast kernel's inner loop — per-iteration state
 # dumps gated at compile time so production builds carry zero cost. Runtime
 # activation via `STRUCTURED_ZSTD_KERNEL_TRACE=1` env var. Used by the
@@ -95,6 +102,13 @@ lsm = []
 # stable interface of this crate.
 rustc-dep-of-std = ["dep:compiler_builtins", "dep:core", "dep:alloc"]
 
+# Diagnostic example: only buildable with the copy-shape counters compiled
+# in, so `--all-targets` on the default feature set skips it (the
+# `shape_stats` re-export is feature-gated).
+[[example]]
+name = "copy_shape"
+required-features = ["copy_shape_stats", "dict_builder"]
+
 [[bench]]
 name = "decode_all"
 harness = false

zstd/examples/copy_shape.rs

@@ -0,0 +1,153 @@
+//! Diagnostic: capture the decode-path copy-call shape (counts + size
+//! buckets + overshoot byte totals) for a chosen level on the
+//! low-entropy corpus. The distribution is deterministic from the
+//! compressed input, so this runs identically on any CPU tier — use it
+//! to reason about call shape without an idle bench host.
+//!
+//! Build/run (feature gates the atomic counters):
+//!   cargo run --release -p structured-zstd \
+//!     --features "copy_shape_stats dict_builder" \
+//!     --example copy_shape -- 18
+//!
+//! Arg 1 = compression level (default 18). Arg 2 = iters (default 1).
+
+use std::env;
+
+use structured_zstd::WILDCOPY_OVERLENGTH;
+use structured_zstd::decoding::FrameDecoder;
+use structured_zstd::decoding::shape_stats;
+use zstd::zstd_safe::zstd_sys;
+
+/// Replica of the `low_entropy_bytes` bench corpus: runs of 8..=31
+/// identical bytes, the byte value advancing by 37 (mod 256) per run,
+/// up to 1 MiB.
+fn low_entropy_bytes() -> Vec<u8> {
+    let n = 1_048_576usize;
+    let mut out = Vec::with_capacity(n + 32);
+    let mut val: u8 = 0;
+    while out.len() < n {
+        let run = 8 + (val as usize % 24); // 8..=31
+        for _ in 0..run {
+            out.push(val);
+        }
+        val = val.wrapping_add(37);
+    }
+    out.truncate(n);
+    out
+}
+
+fn main() {
+    let args: Vec<String> = env::args().collect();
+    let level: i32 = args.get(1).and_then(|s| s.parse().ok()).unwrap_or(18);
+    let iters: u32 = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(1);
+
+    let src = low_entropy_bytes();
+    let n = src.len();
+
+    let dst_cap = unsafe { zstd_sys::ZSTD_compressBound(src.len()) };
+    let mut compressed = vec![0u8; dst_cap];
+    let written = unsafe {
+        zstd_sys::ZSTD_compress(
+            compressed.as_mut_ptr().cast::<core::ffi::c_void>(),
+            dst_cap,
+            src.as_ptr().cast::<core::ffi::c_void>(),
+            src.len(),
+            level,
+        )
+    };
+    assert_eq!(
+        unsafe { zstd_sys::ZSTD_isError(written) },
+        0,
+        "encode failed"
+    );
+    compressed.truncate(written);
+    eprintln!(
+        "level {level}: {n} bytes -> {written} bytes (ratio {:.3}x)",
+        n as f64 / written as f64
+    );
+
+    let mut target = vec![0u8; n + WILDCOPY_OVERLENGTH];
+    let mut decoder = FrameDecoder::new();
+
+    // Warm decode once, then reset counters so the reported shape is for a
+    // single clean decode (multiply by iters if >1).
+    let _ = decoder
+        .decode_all(compressed.as_slice(), &mut target)
+        .expect("decode_all");
+    let _ = shape_stats::take();
+    let _ = shape_stats::take_repeat();
+
+    for _ in 0..iters {
+        let got = decoder
+            .decode_all(compressed.as_slice(), &mut target)
+            .expect("decode_all");
+        assert_eq!(got, n, "decoded size mismatch");
+    }
+
+    let repeat = shape_stats::take_repeat();
+    let [le8, b9_16, b17_32, gt32, req_gt32, written_gt32, max_len] = shape_stats::take();
+    let total_calls = le8 + b9_16 + b17_32 + gt32;
+    eprintln!("--- copy_bytes_overshooting call shape ({iters} iter(s)) ---");
+    eprintln!("  total calls      : {total_calls}");
+    eprintln!("  <=8 bytes        : {le8} ({:.1}%)", pct(le8, total_calls));
+    eprintln!(
+        "  9..=16 bytes     : {b9_16} ({:.1}%)",
+        pct(b9_16, total_calls)
+    );
+    eprintln!(
+        "  17..=32 bytes    : {b17_32} ({:.1}%)",
+        pct(b17_32, total_calls)
+    );
+    eprintln!(
+        "  >32 bytes        : {gt32} ({:.1}%)  <- the copy_avx2 chunk path",
+        pct(gt32, total_calls)
+    );
+    if gt32 > 0 {
+        eprintln!(
+            "  >32 avg req len  : {:.1} bytes",
+            req_gt32 as f64 / gt32 as f64
+        );
+        eprintln!(
+            "  >32 overshoot    : {} req -> {} written (+{:.2}% waste)",
+            req_gt32,
+            written_gt32,
+            100.0 * (written_gt32 as f64 - req_gt32 as f64) / req_gt32 as f64
+        );
+    }
+    eprintln!("  max single copy  : {max_len} bytes");
+    eprintln!(
+        "  decoded bytes/it : {n}  (>32 written/it covers {:.1}% of output)",
+        100.0 * (written_gt32 as f64 / iters as f64) / n as f64
+    );
+
+    let labels = [
+        "non-overlap     ",
+        "ovl offset <8   ",
+        "ovl offset 8-15 ",
+        "ovl offset 16-31",
+        "ovl offset 32-63",
+        "ovl offset >=64 ",
+    ];
+    let total_match_bytes: u64 = repeat.iter().map(|(_, b)| b).sum();
+    eprintln!("--- match-repeat shape by offset bucket ---");
+    for (lab, (cnt, bytes)) in labels.iter().zip(repeat.iter()) {
+        eprintln!(
+            "  {lab} : {cnt:>8} calls  {bytes:>12} bytes ({:.1}% of match bytes)",
+            pct(*bytes, total_match_bytes)
+        );
+    }
+    eprintln!("  total match bytes: {total_match_bytes}");
+    let chunked_ovl: u64 = repeat[3].1 + repeat[4].1 + repeat[5].1;
+    eprintln!(
+        "  offset>=16 overlapping (chunked-by-offset; C single-passes): {chunked_ovl} bytes ({:.1}% of output)",
+        pct(chunked_ovl, n as u64)
+    );
+}
+
+fn pct(part: u64, whole: u64) -> f64 {
+    if whole == 0 {
+        0.0
+    } else {
+        100.0 * part as f64 / whole as f64
+    }
+}

zstd/src/decoding/decode_buffer.rs

@@ -363,6 +363,18 @@ impl<B: BufferBackend> DecodeBuffer<B> {
                     capacity: o.capacity,
                 }
             })?;
+
+            // Record the copy-shape histogram only after the reserve
+            // succeeds: on `OutputBufferOverflow` the repeat never runs, so
+            // counting it here would inflate the diagnostic with match
+            // traffic that was never materialised.
+            #[cfg(feature = "copy_shape_stats")]
+            crate::decoding::simd_copy::shape_stats::record_repeat(
+                offset,
+                match_length,
+                end_idx > buf_len,
+            );
+
             if !SKIP_PREFETCH {
                 self.prefetch_match_source(start_idx, match_length);
             }
@@ -401,32 +413,65 @@ impl<B: BufferBackend> DecodeBuffer<B> {
         }
     }
 
-    #[inline(always)]
+    /// Materialise an overlapping match (`offset < match_length`) by
+    /// exponential doubling rather than fixed `offset`-sized chunks.
+    ///
+    /// The period occupies `[start_idx, start_idx + offset)` and the output
+    /// grows from the current tail. Each step copies the *entire* contiguous
+    /// run already materialised from the period base — length `buffer.len() -
+    /// start_idx` — and appends it directly after itself: src = `[start_idx,
+    /// start_idx + n)`, dst = the tail, with `n <= buffer.len() - start_idx`
+    /// so `start_idx + n <= tail`. Every copy is therefore a clean
+    /// *non-overlapping adjacent* block that satisfies
+    /// `extend_from_within_unchecked`'s contract, and `start_idx` never moves.
+    ///
+    /// Because the materialised run doubles after each step, this emits
+    /// `O(log2(match_length / offset))` copies — each as large as the run so
+    /// far — instead of `match_length / offset` offset-sized ones. Fewer call
+    /// boundaries and larger contiguous copies (better for the hardware
+    /// prefetcher / store streaming) while the bytes produced are identical:
+    /// the output is periodic with period `offset`, and copying any prefix of
+    /// that period onto its own tail preserves the periodicity.
+    ///
+    /// `#[inline]` (hint, not force): this is the overlapping-match cold-ish
+    /// arm of `repeat_inner`. Forcing it inline bloats the hot fully-inlined
+    /// sequence executor and measurably shifts codegen of the common
+    /// non-overlapping path (a small-match regression on realistic corpora);
+    /// the doubling win comes from issuing fewer/larger copies, not from
+    /// inlining, so the per-call boundary here is irrelevant next to the
+    /// copy work it dispatches.
+    #[inline]
     fn repeat_in_chunks(
         &mut self,
         offset: usize,
         match_length: usize,
         start_idx: usize,
         use_branchless_copy: bool,
     ) {
-        let mut start_idx = start_idx;
-        let mut copied_counter_left = match_length;
-        while copied_counter_left > 0 {
-            let chunksize = usize::min(offset, copied_counter_left);
-
-            // SAFETY: chunksize <= offset keeps each single copy in the currently readable
-            // source range, and repeat() reserved enough destination capacity.
+        debug_assert!(offset >= 8, "doubling path expects offset >= 8");
+        let mut remaining = match_length;
+        while remaining > 0 {
+            // Contiguous, already-correct run from the period base. Since
+            // `extend_from_within` keeps `start_idx` fixed and appends at the
+            // tail, the run length is exactly `tail - start_idx`; it starts at
+            // `offset` and doubles each iteration until capped by `remaining`.
+            let run = self.buffer.len() - start_idx;
+            let n = usize::min(run, remaining);
+
+            // SAFETY: `n <= run = buffer.len() - start_idx` gives
+            // `start_idx + n <= buffer.len()` (the tail / dst start), so the
+            // source `[start_idx, start_idx + n)` and the destination at the
+            // tail are adjacent and non-overlapping. `repeat()` reserved
+            // `match_length` destination capacity up front.
             unsafe {
                 if use_branchless_copy {
                     self.buffer
-                        .extend_from_within_unchecked_branchless(start_idx, chunksize);
+                        .extend_from_within_unchecked_branchless(start_idx, n);
                 } else {
-                    self.buffer
-                        .extend_from_within_unchecked(start_idx, chunksize);
+                    self.buffer.extend_from_within_unchecked(start_idx, n);
                 }
             };
-            copied_counter_left -= chunksize;
-            start_idx += chunksize;
+            remaining -= n;
         }
     }
 
@@ -923,6 +968,49 @@ mod tests {
         assert_eq!(buf.drain(), b"ok");
     }
 
+    #[test]
+    fn test_repeat_doubling_matches_reference_across_offsets() {
+        // Naive reference: dst[i] = dst[i - offset]. The exponential-doubling
+        // `repeat_in_chunks` must produce byte-identical output for every
+        // offset/length, including lengths that straddle the 32-byte SIMD
+        // overshoot boundary and large lengths that force many doublings.
+        fn reference_repeat(prefix: &[u8], offset: usize, match_length: usize) -> Vec<u8> {
+            let mut out = prefix.to_vec();
+            for _ in 0..match_length {
+                let src = out.len() - offset;
+                out.push(out[src]);
+            }
+            out
+        }
+
+        let prefix: Vec<u8> = (0..200u32)
+            .map(|i| i.wrapping_mul(31).wrapping_add(7) as u8)
+            .collect();
+        // Offsets cover every `repeat_overlapping` arm: <8 period-tiled,
+        // 8..15 and >=16 doubling, exact powers of two, and the
+        // non-overlapping `offset == match_length` edge.
+        let offsets = [1usize, 2, 3, 5, 7, 8, 13, 16, 31, 32, 63, 64, 100, 200];
+        let lengths = [
+            1usize, 2, 7, 8, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 200, 511, 1000, 5000,
+        ];
+        for &offset in &offsets {
+            for &match_length in &lengths {
+                let prefix_slice = &prefix[..offset.max(1)];
+                let mut buffer = DecodeBuffer::<RingBuffer>::new(usize::MAX);
+                buffer.push(prefix_slice);
+                buffer.repeat(offset, match_length).unwrap();
+                let expected = reference_repeat(prefix_slice, offset, match_length);
+                let mut got: Vec<u8> = Vec::new();
+                buffer.drain_to_writer(&mut got).unwrap();
+                assert_eq!(
+                    got.as_slice(),
+                    expected.as_slice(),
+                    "mismatch at offset={offset} match_length={match_length}",
+                );
+            }
+        }
+    }
+
     #[test]
     fn checkpoint_restore_undoes_pushes() {
         // Regression test for the fused-decode transactional contract:

zstd/src/decoding/mod.rs

@@ -87,6 +87,10 @@ pub(crate) mod seq_decoder_vbmi2;
 pub(crate) mod sequence_execution;
 pub(crate) mod sequence_section_decoder;
 pub(crate) mod simd_copy;
+/// Diagnostic-only re-export of the copy-shape histogram counters. Public
+/// only when the `copy_shape_stats` feature is on (off in shipping builds).
+#[cfg(feature = "copy_shape_stats")]
+pub use simd_copy::shape_stats;
 // `UserSliceBackend` is the compile-time-monomorphised backend that
 // writes directly into the caller's `&mut [u8]` output slice, used
 // by the `FrameDecoder::decode_all` direct-decode path. It