perf: borrowed in-place over-window scan for dfast + row + btlazy2

zstd/src/encoding/dfast/mod.rs

@@ -853,6 +853,7 @@ impl DfastMatchGenerator {
         current_len: usize,
         ip0: usize,
         literals_start: usize,
+        borrowed: bool,
     ) -> Option<MatchCandidate> {
         debug_assert!(ip0 + HASH_READ_SIZE <= current_len);
         const PRIME: u64 = 0xCF1BBCDCB7A56463_u64;
@@ -866,6 +867,19 @@ impl DfastMatchGenerator {
         let (history_base_ptr, history_start_offset, history_abs_start, position_base, concat_len) =
             self.scan_source();
 
+        // Per-position window-low bound, identical to the fast loop's `wlow0`
+        // (see `advertised_window` there). In borrowed mode `history_abs_start`
+        // is 0, so a bare `cand_pos >= history_abs_start` floor admits
+        // candidates older than the advertised window and could emit an
+        // unresolvable offset for over-window inputs; bound by `abs_ip0 -
+        // advertised_window` instead. Owned mode keeps the eviction-floor
+        // `history_abs_start`.
+        let wlow = if borrowed {
+            abs_ip0.saturating_sub(self.max_window_size)
+        } else {
+            history_abs_start
+        };
+
         let concat_idx0 = abs_ip0 - history_abs_start;
 
         // SAFETY: `concat_idx0 + 8 <= concat_len` follows from the
@@ -876,7 +890,7 @@ impl DfastMatchGenerator {
                 .read_unaligned()
         };
         // `v4_0` (low 4 bytes) is the 4-byte equality-gate key below; the short
-        // HASH keys on the donor 5-byte window (`v8_0 << 24`, ZSTD_hash5 shape).
+        // HASH keys on the upstream zstd 5-byte window (`v8_0 << 24`, ZSTD_hash5 shape).
         let v4_0 = v8_0 & 0xFFFF_FFFF;
         let hl0_idx = (v8_0.wrapping_mul(PRIME) >> long_shift) as usize;
         let hs0_idx = ((v8_0 << 24).wrapping_mul(PRIME) >> short_shift) as usize;
@@ -921,7 +935,7 @@ impl DfastMatchGenerator {
         // beats a 4-byte hit even before extension).
         if idxl0 != DFAST_EMPTY_SLOT {
             let cand_pos = position_base + (idxl0 as usize) - 1;
-            if cand_pos >= history_abs_start && cand_pos < abs_ip0 {
+            if cand_pos >= wlow && cand_pos < abs_ip0 {
                 let cand_idx = cand_pos - history_abs_start;
                 let cand_v8 = unsafe {
                     (history_base_ptr.add(history_start_offset + cand_idx) as *const u64)
@@ -961,7 +975,7 @@ impl DfastMatchGenerator {
         // comment there).
         if idxs0 != DFAST_EMPTY_SLOT {
             let cand_pos_s = position_base + (idxs0 as usize) - 1;
-            if cand_pos_s >= history_abs_start && cand_pos_s < abs_ip0 {
+            if cand_pos_s >= wlow && cand_pos_s < abs_ip0 {
                 let cand_idx_s = cand_pos_s - history_abs_start;
                 let cand4 = unsafe {
                     (history_base_ptr.add(history_start_offset + cand_idx_s) as *const u32)
@@ -1760,6 +1774,14 @@ macro_rules! start_matching_fast_loop_body {
                 (core::ptr::null(), core::ptr::null(), 0)
             };
 
+        // Advertised window cap = `1 << window_log`. Owned mode evicts, so
+        // `history_abs_start` already bounds candidates to the live window;
+        // borrowed mode keeps the whole input in place (no eviction), so an
+        // OVER-window borrowed scan must explicitly reject candidates whose
+        // offset would exceed the advertised window — otherwise it would emit
+        // an offset the decoder cannot resolve. Hoisted (loop-invariant).
+        let advertised_window = $self.max_window_size;
+
         'outer: loop {
             // Outer-iter precondition: at least `HASH_READ_SIZE = 8` bytes
             // ahead of `pos` so the unconditional 8-byte `u64` load below
@@ -1798,12 +1820,16 @@ macro_rules! start_matching_fast_loop_body {
             // `p + HASH_READ_SIZE <= iend`). Handle the boundary inline
             // before exiting: a single-cursor probe at `ip0` (rep peek
             // and `_search_next_long` retry both depend on `ip1` so
-            // they're skipped — donor accepts that exact tradeoff at
+            // they're skipped — upstream zstd accepts that exact tradeoff at
             // the iend boundary).
             if ip1 + HASH_READ_SIZE > $current_len {
-                if let Some(committed) =
-                    $self.probe_tail_ip0_only($current_abs_start, $current_len, ip0, literals_start)
-                {
+                if let Some(committed) = $self.probe_tail_ip0_only(
+                    $current_abs_start,
+                    $current_len,
+                    ip0,
+                    literals_start,
+                    $borrowed,
+                ) {
                     let start = $self.emit_candidate(
                         $current_abs_start,
                         &mut literals_start,
@@ -1908,6 +1934,21 @@ macro_rules! start_matching_fast_loop_body {
             let inner_exit: InnerExit = 'inner: loop {
                 let abs_ip0 = $current_abs_start + ip0;
                 let abs_ip1 = $current_abs_start + ip1;
+                // Per-position candidate window-low bound (see `advertised_window`
+                // above). `$borrowed` is const, so owned collapses to
+                // `history_abs_start` (byte-identical) and borrowed-in-window
+                // saturates to 0 (== history_abs_start, also byte-identical);
+                // only borrowed-over-window gains the `abs_ip - window` cap.
+                let wlow0 = if $borrowed {
+                    abs_ip0.saturating_sub(advertised_window)
+                } else {
+                    history_abs_start
+                };
+                let wlow1 = if $borrowed {
+                    abs_ip1.saturating_sub(advertised_window)
+                } else {
+                    history_abs_start
+                };
                 let lit_len_ip0 = ip0 - literals_start;
                 let lit_len_ip1 = ip1 - literals_start;
                 let packed_curr = ((abs_ip0 - position_base) as u32) + 1;
@@ -1962,7 +2003,7 @@ macro_rules! start_matching_fast_loop_body {
                 let rep1 = $self.offset_hist[0] as usize;
                 if rep1 != 0 && rep1 <= abs_ip1 {
                     let cand_pos_r = abs_ip1 - rep1;
-                    if cand_pos_r >= history_abs_start
+                    if cand_pos_r >= wlow1
                         && cand_pos_r >= $current_abs_start + literals_start
                     {
                         let cand_idx_r = cand_pos_r - history_abs_start;
@@ -2065,7 +2106,7 @@ macro_rules! start_matching_fast_loop_body {
                 // mask — a stall the predictor cannot hide.
                 if idxl0 != DFAST_EMPTY_SLOT {
                     let cand_pos = position_base + ((idxl0 as usize) - 1);
-                    if cand_pos >= history_abs_start && cand_pos < abs_ip0 {
+                    if cand_pos >= wlow0 && cand_pos < abs_ip0 {
                         let cand_idx = cand_pos - history_abs_start;
                         // SAFETY: the bounds above make `cand_idx` a valid
                         // in-window concat index, so the 8-byte load stays in
@@ -2198,7 +2239,7 @@ macro_rules! start_matching_fast_loop_body {
                 // the load address to the mask, serialising it.
                 if idxs0 != DFAST_EMPTY_SLOT {
                     let cand_pos_s = position_base + ((idxs0 as usize) - 1);
-                    if cand_pos_s >= history_abs_start && cand_pos_s < abs_ip0 {
+                    if cand_pos_s >= wlow0 && cand_pos_s < abs_ip0 {
                         let cand_idx_s = cand_pos_s - history_abs_start;
                         let cand4 = unsafe {
                             (history_base_ptr.add(history_start_offset + cand_idx_s) as *const u32)
@@ -2262,7 +2303,7 @@ macro_rules! start_matching_fast_loop_body {
                             let mut live_l1_hit = false;
                             if idxl1 != DFAST_EMPTY_SLOT {
                                 let cand_pos_l1 = position_base + (idxl1 as usize) - 1;
-                                if cand_pos_l1 >= history_abs_start && cand_pos_l1 < abs_ip1 {
+                                if cand_pos_l1 >= wlow1 && cand_pos_l1 < abs_ip1 {
                                     let cand_idx_l1 = cand_pos_l1 - history_abs_start;
                                     let cand_v8_l1 = unsafe {
                                         (history_base_ptr.add(history_start_offset + cand_idx_l1)

zstd/src/encoding/frame_compressor.rs

@@ -609,6 +609,32 @@ pub(crate) fn block_splitter_decision_for_bench(block: &[u8], split_level: usize
     }
 }
 
+/// Pull a pre-split window into cache with one bandwidth-bound sequential
+/// pass before the strided fingerprint histogram + match scan read it.
+///
+/// The borrowed (no-copy) over-window path matches in place on the caller's
+/// input, so the pre-split fingerprint is the FIRST touch of that 128 KiB
+/// region — a cache-cold read. `presplit_record_fingerprint` reads it with a
+/// `sampling_rate` stride and interleaved random writes into the 1 KiB events
+/// table, a latency-bound pattern that pays full DRAM miss latency per line
+/// (measured ~3x the cost of an ERMS streaming read of the same bytes). The
+/// owned path never hits this because its history-mirror copy already warmed
+/// the bytes; this restores that warmth without the copy's write half. One
+/// dependent load per 64-byte line (the i9 line size) streams under the
+/// hardware prefetcher, so the cold read is paid once at memory bandwidth and
+/// every subsequent strided sample lands in L1/L2. `black_box` keeps the loop
+/// from being optimized away as a dead read.
+#[inline]
+fn warm_presplit_window(window: &[u8]) {
+    let mut acc = 0u8;
+    let mut i = 0usize;
+    while i < window.len() {
+        acc ^= window[i];
+        i += 64;
+    }
+    core::hint::black_box(acc);
+}
+
 pub(crate) fn optimal_block_size(
     level: CompressionLevel,
     block: &[u8],
@@ -949,26 +975,20 @@ impl<R: Read, W: Write> FrameCompressor<R, W, MatchGeneratorDriver> {
     /// The owned/evicting path keeps the scanned window bounded (positions
     /// stay small), so >4 GiB inputs fall back to it.
     fn borrowed_eligible(&self, input_len: usize, prep: &FramePrep) -> bool {
-        use crate::encoding::strategy::StrategyTag;
         if prep.use_dictionary_state
             || matches!(self.compression_level, CompressionLevel::Uncompressed)
             || input_len > u32::MAX as usize
         {
             return false;
         }
-        match self.state.strategy_tag {
-            // Fast handles over-window inputs in the borrowed scan via an
-            // explicit `window_low = block_end - advertised_window` bound.
-            StrategyTag::Fast => true,
-            // The Dfast borrowed scan uses `history_abs_start` (== 0 in
-            // borrowed mode) as the candidate lower bound, not `window_low`,
-            // so it reproduces the owned (evicting) path only when the whole
-            // input fits the window — then neither side evicts, so neither
-            // rejects an in-window candidate and the sequence streams match.
-            // Over-window inputs fall back to the owned path.
-            StrategyTag::Dfast => input_len <= self.state.matcher.window_size() as usize,
-            _ => false,
-        }
+        // The borrowed (no-copy, in-place over-window) scan exists for the
+        // Simple (Fast), Dfast, and Row backends, and for the HashChain
+        // backend's lazy CHAIN parser; BT/optimal (BinaryTree search) stay on
+        // the owned path. Every borrowed scan applies the per-position
+        // `window_low = abs_ip - advertised_window` offset cap so over-window
+        // inputs are matched in place (no input->history copy), matching C's
+        // continuous-index + windowLow one-shot behaviour.
+        self.state.matcher.borrowed_supported()
     }
 
     /// Compress `input` as one frame's worth of blocks into `out` (appended
@@ -1167,6 +1187,22 @@ impl<R: Read, W: Write> FrameCompressor<R, W, MatchGeneratorDriver> {
             // already spans the whole input, so a smaller block is just a
             // narrower `(block_start, block_end)` range into it.
             let savings = start as i64 - (out.len() - blocks_start) as i64;
+            // Borrowed path only: warm the pre-split window before the
+            // cache-cold strided fingerprint read. Gated to exactly the
+            // conditions under which `optimal_block_size` reads `block`
+            // (a pre-split level, a full 128 KiB block remaining, the
+            // block-size cap admits a full block, and `savings >= 3` so the
+            // splitter actually runs) — so non-pre-split levels, the first
+            // block, and the trailing partial block pay nothing. See
+            // `warm_presplit_window`.
+            if savings >= 3
+                && input.len() - start >= MAX_BLOCK_SIZE as usize
+                && block_capacity >= MAX_BLOCK_SIZE as usize
+                && crate::encoding::match_generator::level_pre_split(self.compression_level)
+                    .is_some()
+            {
+                warm_presplit_window(&input[start..start + MAX_BLOCK_SIZE as usize]);
+            }
             let block_len = optimal_block_size(
                 self.compression_level,
                 &input[start..],

zstd/src/encoding/hc/mod.rs

@@ -144,9 +144,12 @@ impl HcMatcher {
             if next == cur {
                 // Self-loop: two positions share chain_idx, stop to
                 // avoid spinning on the same candidate forever.
-                if let Some(candidate_abs) =
-                    candidate_abs.filter(|&p| p >= table.history_abs_start && p < abs_pos)
-                {
+                if let Some(candidate_abs) = candidate_abs.filter(|&p| {
+                    p >= table
+                        .history_abs_start
+                        .max(abs_pos.saturating_sub(table.max_window_size))
+                        && p < abs_pos
+                }) {
                     buf[filled] = candidate_abs;
                 }
                 break;
@@ -155,7 +158,12 @@ impl HcMatcher {
             let Some(candidate_abs) = candidate_abs else {
                 continue;
             };
-            if candidate_abs < table.history_abs_start || candidate_abs >= abs_pos {
+            if candidate_abs
+                < table
+                    .history_abs_start
+                    .max(abs_pos.saturating_sub(table.max_window_size))
+                || candidate_abs >= abs_pos
+            {
                 continue;
             }
             buf[filled] = candidate_abs;
@@ -198,7 +206,11 @@ impl HcMatcher {
                 continue;
             }
             let candidate_pos = abs_pos - rep;
-            if candidate_pos < table.history_abs_start {
+            if candidate_pos
+                < table
+                    .history_abs_start
+                    .max(abs_pos.saturating_sub(table.max_window_size))
+            {
                 continue;
             }
             let candidate_idx = candidate_pos - table.history_abs_start;
@@ -323,7 +335,8 @@ impl HcMatcher {
 
             // Only process candidates in the live window [history_abs_start, abs_pos).
             if let Some(candidate_abs) = candidate_abs_opt
-                && candidate_abs >= history_abs_start
+                && candidate_abs
+                    >= history_abs_start.max(abs_pos.saturating_sub(table.max_window_size))
                 && candidate_abs < abs_pos
             {
                 let candidate_idx = candidate_abs - history_abs_start;

zstd/src/encoding/levels/fastest.rs

@@ -245,6 +245,18 @@ pub(crate) fn compress_block_encoded_borrowed(
     #[cfg(feature = "lsm")] block_decompressed_sizes: Option<&mut Vec<u32>>,
     #[cfg(all(feature = "lsm", feature = "hash"))] block_checksums: Option<&mut Vec<u32>>,
 ) -> BlockType {
+    // The borrowed one-shot path emits ONE block per staged range (no
+    // pre-split partition loop). `borrowed_supported()` is the single source
+    // of truth for which backend + search configs have a borrowed scan
+    // (Simple / Dfast / Row, and HashChain's lazy CHAIN parser + btlazy2); the
+    // optimal BT search stays on the owned path. `borrowed_eligible` gates on
+    // the same predicate, so this only ever fires on a wiring bug. Checked at
+    // entry (not per-branch) so RLE / raw-fast / compressed paths all stage
+    // their borrowed range under the same invariant.
+    debug_assert!(
+        state.matcher.borrowed_supported(),
+        "borrowed one-shot path reached for an unsupported backend/search config",
+    );
     let block_size = block.len() as u32;
     if !block.is_empty() && block.iter().all(|x| block[0].eq(x)) {
         let rle_byte = block[0];
@@ -286,13 +298,15 @@ pub(crate) fn compress_block_encoded_borrowed(
         output.extend_from_slice(block);
         BlockType::Raw
     } else {
-        assert!(
-            !matches!(compression_level, CompressionLevel::Level(16..=22)),
-            "borrowed one-shot path is gated to Fast levels; post-split is unreachable",
-        );
         // Stage the borrowed range so `compress_block`'s internal
         // `start_matching` scans it in place (no `commit_space` copy).
         state.matcher.set_borrowed_block(block_start, block_end);
+        // No post-split branch here: the optimal levels (16-22), the only
+        // strategies that post-split, are NOT borrowed-eligible
+        // (`borrowed_supported` keeps them owned because the borrowed
+        // continuous-index scan yields ratio-worse candidates for their
+        // cost-based DP). btlazy2 (L13-15) and every other borrowed backend
+        // emit a single block per staged range, handled by the path below.
         #[cfg(feature = "lsm")]
         if let Some(sink) = block_decompressed_sizes {
             sink.push(block_size);