diff --git a/zstd/src/encoding/dfast/mod.rs b/zstd/src/encoding/dfast/mod.rs index 307cbc26a..0ca66d54e 100644 --- a/zstd/src/encoding/dfast/mod.rs +++ b/zstd/src/encoding/dfast/mod.rs @@ -853,6 +853,7 @@ impl DfastMatchGenerator { current_len: usize, ip0: usize, literals_start: usize, + borrowed: bool, ) -> Option { debug_assert!(ip0 + HASH_READ_SIZE <= current_len); const PRIME: u64 = 0xCF1BBCDCB7A56463_u64; @@ -866,6 +867,19 @@ impl DfastMatchGenerator { let (history_base_ptr, history_start_offset, history_abs_start, position_base, concat_len) = self.scan_source(); + // Per-position window-low bound, identical to the fast loop's `wlow0` + // (see `advertised_window` there). In borrowed mode `history_abs_start` + // is 0, so a bare `cand_pos >= history_abs_start` floor admits + // candidates older than the advertised window and could emit an + // unresolvable offset for over-window inputs; bound by `abs_ip0 - + // advertised_window` instead. Owned mode keeps the eviction-floor + // `history_abs_start`. + let wlow = if borrowed { + abs_ip0.saturating_sub(self.max_window_size) + } else { + history_abs_start + }; + let concat_idx0 = abs_ip0 - history_abs_start; // SAFETY: `concat_idx0 + 8 <= concat_len` follows from the @@ -876,7 +890,7 @@ impl DfastMatchGenerator { .read_unaligned() }; // `v4_0` (low 4 bytes) is the 4-byte equality-gate key below; the short - // HASH keys on the donor 5-byte window (`v8_0 << 24`, ZSTD_hash5 shape). + // HASH keys on the upstream zstd 5-byte window (`v8_0 << 24`, ZSTD_hash5 shape). let v4_0 = v8_0 & 0xFFFF_FFFF; let hl0_idx = (v8_0.wrapping_mul(PRIME) >> long_shift) as usize; let hs0_idx = ((v8_0 << 24).wrapping_mul(PRIME) >> short_shift) as usize; @@ -921,7 +935,7 @@ impl DfastMatchGenerator { // beats a 4-byte hit even before extension). if idxl0 != DFAST_EMPTY_SLOT { let cand_pos = position_base + (idxl0 as usize) - 1; - if cand_pos >= history_abs_start && cand_pos < abs_ip0 { + if cand_pos >= wlow && cand_pos < abs_ip0 { let cand_idx = cand_pos - history_abs_start; let cand_v8 = unsafe { (history_base_ptr.add(history_start_offset + cand_idx) as *const u64) @@ -961,7 +975,7 @@ impl DfastMatchGenerator { // comment there). if idxs0 != DFAST_EMPTY_SLOT { let cand_pos_s = position_base + (idxs0 as usize) - 1; - if cand_pos_s >= history_abs_start && cand_pos_s < abs_ip0 { + if cand_pos_s >= wlow && cand_pos_s < abs_ip0 { let cand_idx_s = cand_pos_s - history_abs_start; let cand4 = unsafe { (history_base_ptr.add(history_start_offset + cand_idx_s) as *const u32) @@ -1760,6 +1774,14 @@ macro_rules! start_matching_fast_loop_body { (core::ptr::null(), core::ptr::null(), 0) }; + // Advertised window cap = `1 << window_log`. Owned mode evicts, so + // `history_abs_start` already bounds candidates to the live window; + // borrowed mode keeps the whole input in place (no eviction), so an + // OVER-window borrowed scan must explicitly reject candidates whose + // offset would exceed the advertised window — otherwise it would emit + // an offset the decoder cannot resolve. Hoisted (loop-invariant). + let advertised_window = $self.max_window_size; + 'outer: loop { // Outer-iter precondition: at least `HASH_READ_SIZE = 8` bytes // ahead of `pos` so the unconditional 8-byte `u64` load below @@ -1798,12 +1820,16 @@ macro_rules! start_matching_fast_loop_body { // `p + HASH_READ_SIZE <= iend`). Handle the boundary inline // before exiting: a single-cursor probe at `ip0` (rep peek // and `_search_next_long` retry both depend on `ip1` so - // they're skipped — donor accepts that exact tradeoff at + // they're skipped — upstream zstd accepts that exact tradeoff at // the iend boundary). if ip1 + HASH_READ_SIZE > $current_len { - if let Some(committed) = - $self.probe_tail_ip0_only($current_abs_start, $current_len, ip0, literals_start) - { + if let Some(committed) = $self.probe_tail_ip0_only( + $current_abs_start, + $current_len, + ip0, + literals_start, + $borrowed, + ) { let start = $self.emit_candidate( $current_abs_start, &mut literals_start, @@ -1908,6 +1934,21 @@ macro_rules! start_matching_fast_loop_body { let inner_exit: InnerExit = 'inner: loop { let abs_ip0 = $current_abs_start + ip0; let abs_ip1 = $current_abs_start + ip1; + // Per-position candidate window-low bound (see `advertised_window` + // above). `$borrowed` is const, so owned collapses to + // `history_abs_start` (byte-identical) and borrowed-in-window + // saturates to 0 (== history_abs_start, also byte-identical); + // only borrowed-over-window gains the `abs_ip - window` cap. + let wlow0 = if $borrowed { + abs_ip0.saturating_sub(advertised_window) + } else { + history_abs_start + }; + let wlow1 = if $borrowed { + abs_ip1.saturating_sub(advertised_window) + } else { + history_abs_start + }; let lit_len_ip0 = ip0 - literals_start; let lit_len_ip1 = ip1 - literals_start; let packed_curr = ((abs_ip0 - position_base) as u32) + 1; @@ -1962,7 +2003,7 @@ macro_rules! start_matching_fast_loop_body { let rep1 = $self.offset_hist[0] as usize; if rep1 != 0 && rep1 <= abs_ip1 { let cand_pos_r = abs_ip1 - rep1; - if cand_pos_r >= history_abs_start + if cand_pos_r >= wlow1 && cand_pos_r >= $current_abs_start + literals_start { let cand_idx_r = cand_pos_r - history_abs_start; @@ -2065,7 +2106,7 @@ macro_rules! start_matching_fast_loop_body { // mask — a stall the predictor cannot hide. if idxl0 != DFAST_EMPTY_SLOT { let cand_pos = position_base + ((idxl0 as usize) - 1); - if cand_pos >= history_abs_start && cand_pos < abs_ip0 { + if cand_pos >= wlow0 && cand_pos < abs_ip0 { let cand_idx = cand_pos - history_abs_start; // SAFETY: the bounds above make `cand_idx` a valid // in-window concat index, so the 8-byte load stays in @@ -2198,7 +2239,7 @@ macro_rules! start_matching_fast_loop_body { // the load address to the mask, serialising it. if idxs0 != DFAST_EMPTY_SLOT { let cand_pos_s = position_base + ((idxs0 as usize) - 1); - if cand_pos_s >= history_abs_start && cand_pos_s < abs_ip0 { + if cand_pos_s >= wlow0 && cand_pos_s < abs_ip0 { let cand_idx_s = cand_pos_s - history_abs_start; let cand4 = unsafe { (history_base_ptr.add(history_start_offset + cand_idx_s) as *const u32) @@ -2262,7 +2303,7 @@ macro_rules! start_matching_fast_loop_body { let mut live_l1_hit = false; if idxl1 != DFAST_EMPTY_SLOT { let cand_pos_l1 = position_base + (idxl1 as usize) - 1; - if cand_pos_l1 >= history_abs_start && cand_pos_l1 < abs_ip1 { + if cand_pos_l1 >= wlow1 && cand_pos_l1 < abs_ip1 { let cand_idx_l1 = cand_pos_l1 - history_abs_start; let cand_v8_l1 = unsafe { (history_base_ptr.add(history_start_offset + cand_idx_l1) diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index 0cb3de765..b03376073 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -609,6 +609,32 @@ pub(crate) fn block_splitter_decision_for_bench(block: &[u8], split_level: usize } } +/// Pull a pre-split window into cache with one bandwidth-bound sequential +/// pass before the strided fingerprint histogram + match scan read it. +/// +/// The borrowed (no-copy) over-window path matches in place on the caller's +/// input, so the pre-split fingerprint is the FIRST touch of that 128 KiB +/// region — a cache-cold read. `presplit_record_fingerprint` reads it with a +/// `sampling_rate` stride and interleaved random writes into the 1 KiB events +/// table, a latency-bound pattern that pays full DRAM miss latency per line +/// (measured ~3x the cost of an ERMS streaming read of the same bytes). The +/// owned path never hits this because its history-mirror copy already warmed +/// the bytes; this restores that warmth without the copy's write half. One +/// dependent load per 64-byte line (the i9 line size) streams under the +/// hardware prefetcher, so the cold read is paid once at memory bandwidth and +/// every subsequent strided sample lands in L1/L2. `black_box` keeps the loop +/// from being optimized away as a dead read. +#[inline] +fn warm_presplit_window(window: &[u8]) { + let mut acc = 0u8; + let mut i = 0usize; + while i < window.len() { + acc ^= window[i]; + i += 64; + } + core::hint::black_box(acc); +} + pub(crate) fn optimal_block_size( level: CompressionLevel, block: &[u8], @@ -949,26 +975,20 @@ impl FrameCompressor { /// The owned/evicting path keeps the scanned window bounded (positions /// stay small), so >4 GiB inputs fall back to it. fn borrowed_eligible(&self, input_len: usize, prep: &FramePrep) -> bool { - use crate::encoding::strategy::StrategyTag; if prep.use_dictionary_state || matches!(self.compression_level, CompressionLevel::Uncompressed) || input_len > u32::MAX as usize { return false; } - match self.state.strategy_tag { - // Fast handles over-window inputs in the borrowed scan via an - // explicit `window_low = block_end - advertised_window` bound. - StrategyTag::Fast => true, - // The Dfast borrowed scan uses `history_abs_start` (== 0 in - // borrowed mode) as the candidate lower bound, not `window_low`, - // so it reproduces the owned (evicting) path only when the whole - // input fits the window — then neither side evicts, so neither - // rejects an in-window candidate and the sequence streams match. - // Over-window inputs fall back to the owned path. - StrategyTag::Dfast => input_len <= self.state.matcher.window_size() as usize, - _ => false, - } + // The borrowed (no-copy, in-place over-window) scan exists for the + // Simple (Fast), Dfast, and Row backends, and for the HashChain + // backend's lazy CHAIN parser; BT/optimal (BinaryTree search) stay on + // the owned path. Every borrowed scan applies the per-position + // `window_low = abs_ip - advertised_window` offset cap so over-window + // inputs are matched in place (no input->history copy), matching C's + // continuous-index + windowLow one-shot behaviour. + self.state.matcher.borrowed_supported() } /// Compress `input` as one frame's worth of blocks into `out` (appended @@ -1167,6 +1187,22 @@ impl FrameCompressor { // already spans the whole input, so a smaller block is just a // narrower `(block_start, block_end)` range into it. let savings = start as i64 - (out.len() - blocks_start) as i64; + // Borrowed path only: warm the pre-split window before the + // cache-cold strided fingerprint read. Gated to exactly the + // conditions under which `optimal_block_size` reads `block` + // (a pre-split level, a full 128 KiB block remaining, the + // block-size cap admits a full block, and `savings >= 3` so the + // splitter actually runs) — so non-pre-split levels, the first + // block, and the trailing partial block pay nothing. See + // `warm_presplit_window`. + if savings >= 3 + && input.len() - start >= MAX_BLOCK_SIZE as usize + && block_capacity >= MAX_BLOCK_SIZE as usize + && crate::encoding::match_generator::level_pre_split(self.compression_level) + .is_some() + { + warm_presplit_window(&input[start..start + MAX_BLOCK_SIZE as usize]); + } let block_len = optimal_block_size( self.compression_level, &input[start..], diff --git a/zstd/src/encoding/hc/mod.rs b/zstd/src/encoding/hc/mod.rs index 6163ac2ac..e7ac1ed36 100644 --- a/zstd/src/encoding/hc/mod.rs +++ b/zstd/src/encoding/hc/mod.rs @@ -144,9 +144,12 @@ impl HcMatcher { if next == cur { // Self-loop: two positions share chain_idx, stop to // avoid spinning on the same candidate forever. - if let Some(candidate_abs) = - candidate_abs.filter(|&p| p >= table.history_abs_start && p < abs_pos) - { + if let Some(candidate_abs) = candidate_abs.filter(|&p| { + p >= table + .history_abs_start + .max(abs_pos.saturating_sub(table.max_window_size)) + && p < abs_pos + }) { buf[filled] = candidate_abs; } break; @@ -155,7 +158,12 @@ impl HcMatcher { let Some(candidate_abs) = candidate_abs else { continue; }; - if candidate_abs < table.history_abs_start || candidate_abs >= abs_pos { + if candidate_abs + < table + .history_abs_start + .max(abs_pos.saturating_sub(table.max_window_size)) + || candidate_abs >= abs_pos + { continue; } buf[filled] = candidate_abs; @@ -198,7 +206,11 @@ impl HcMatcher { continue; } let candidate_pos = abs_pos - rep; - if candidate_pos < table.history_abs_start { + if candidate_pos + < table + .history_abs_start + .max(abs_pos.saturating_sub(table.max_window_size)) + { continue; } let candidate_idx = candidate_pos - table.history_abs_start; @@ -323,7 +335,8 @@ impl HcMatcher { // Only process candidates in the live window [history_abs_start, abs_pos). if let Some(candidate_abs) = candidate_abs_opt - && candidate_abs >= history_abs_start + && candidate_abs + >= history_abs_start.max(abs_pos.saturating_sub(table.max_window_size)) && candidate_abs < abs_pos { let candidate_idx = candidate_abs - history_abs_start; diff --git a/zstd/src/encoding/levels/fastest.rs b/zstd/src/encoding/levels/fastest.rs index 5b3421612..6713ece56 100644 --- a/zstd/src/encoding/levels/fastest.rs +++ b/zstd/src/encoding/levels/fastest.rs @@ -245,6 +245,18 @@ pub(crate) fn compress_block_encoded_borrowed( #[cfg(feature = "lsm")] block_decompressed_sizes: Option<&mut Vec>, #[cfg(all(feature = "lsm", feature = "hash"))] block_checksums: Option<&mut Vec>, ) -> BlockType { + // The borrowed one-shot path emits ONE block per staged range (no + // pre-split partition loop). `borrowed_supported()` is the single source + // of truth for which backend + search configs have a borrowed scan + // (Simple / Dfast / Row, and HashChain's lazy CHAIN parser + btlazy2); the + // optimal BT search stays on the owned path. `borrowed_eligible` gates on + // the same predicate, so this only ever fires on a wiring bug. Checked at + // entry (not per-branch) so RLE / raw-fast / compressed paths all stage + // their borrowed range under the same invariant. + debug_assert!( + state.matcher.borrowed_supported(), + "borrowed one-shot path reached for an unsupported backend/search config", + ); let block_size = block.len() as u32; if !block.is_empty() && block.iter().all(|x| block[0].eq(x)) { let rle_byte = block[0]; @@ -286,13 +298,15 @@ pub(crate) fn compress_block_encoded_borrowed( output.extend_from_slice(block); BlockType::Raw } else { - assert!( - !matches!(compression_level, CompressionLevel::Level(16..=22)), - "borrowed one-shot path is gated to Fast levels; post-split is unreachable", - ); // Stage the borrowed range so `compress_block`'s internal // `start_matching` scans it in place (no `commit_space` copy). state.matcher.set_borrowed_block(block_start, block_end); + // No post-split branch here: the optimal levels (16-22), the only + // strategies that post-split, are NOT borrowed-eligible + // (`borrowed_supported` keeps them owned because the borrowed + // continuous-index scan yields ratio-worse candidates for their + // cost-based DP). btlazy2 (L13-15) and every other borrowed backend + // emit a single block per staged range, handled by the path below. #[cfg(feature = "lsm")] if let Some(sink) = block_decompressed_sizes { sink.push(block_size); diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs index 07794ef64..9d1d11315 100644 --- a/zstd/src/encoding/match_generator.rs +++ b/zstd/src/encoding/match_generator.rs @@ -1491,10 +1491,39 @@ impl MatchGeneratorDriver { /// Active backend family derived from the storage variant. Single /// source of truth — no separate runtime tag to drift against. - fn active_backend(&self) -> super::strategy::BackendTag { + pub(crate) fn active_backend(&self) -> super::strategy::BackendTag { self.storage.backend() } + /// Whether the borrowed (no-copy, in-place over-window) scan is + /// implemented for the current backend + search configuration. The + /// HashChain backend serves both the lazy CHAIN parser + /// (`SearchMethod::HashChain`) and the BT/optimal parsers + /// (`SearchMethod::BinaryTree`); only the lazy chain has a borrowed scan + /// so far, so BT/optimal stay on the owned path. + pub(crate) fn borrowed_supported(&self) -> bool { + use super::strategy::{BackendTag, SearchMethod, StrategyTag}; + match self.active_backend() { + BackendTag::Simple | BackendTag::Dfast | BackendTag::Row => true, + // The HashChain backend covers two searches: the lazy CHAIN parser + // (borrowed-capable) and the BINARY-TREE search (btlazy2 L13-15 + + // optimal BtOpt/BtUltra/BtUltra2 L16-22). btlazy2's BT-tree borrowed + // scan is byte-identical to owned (reads via live_history()), so it + // takes the in-place path. The OPTIMAL parsers stay owned: their + // cost-based DP is sensitive to candidate quality, and the borrowed + // continuous-index scan yields slightly different (ratio-worse) + // candidates than the owned evict+rehash scan — borrowed optimal + // both diverged from owned and fell outside the ffi ratio bound. + // Search-aware (not just strategy_tag) so optimal BT can never be + // staged on the borrowed path even via an internal caller. + BackendTag::HashChain => match self.search { + SearchMethod::HashChain => true, + SearchMethod::BinaryTree => matches!(self.strategy_tag, StrategyTag::Btlazy2), + _ => false, + }, + } + } + fn simple_mut(&mut self) -> &mut FastKernelMatcher { match &mut self.storage { MatcherStorage::Simple(m) => m, @@ -1550,7 +1579,12 @@ impl MatchGeneratorDriver { super::strategy::BackendTag::Dfast => unsafe { self.dfast_matcher_mut().set_borrowed_window(buffer) }, - other => unreachable!("borrowed window only for Simple/Dfast backends, got {other:?}"), + super::strategy::BackendTag::Row => unsafe { + self.row_matcher_mut().set_borrowed_window(buffer) + }, + super::strategy::BackendTag::HashChain => unsafe { + self.hc_matcher_mut().set_borrowed_window(buffer) + }, } } @@ -1560,6 +1594,9 @@ impl MatchGeneratorDriver { match self.active_backend() { super::strategy::BackendTag::Simple => self.simple_mut().clear_borrowed_window(), super::strategy::BackendTag::Dfast => self.dfast_matcher_mut().clear_borrowed_window(), + super::strategy::BackendTag::Row => self.row_matcher_mut().clear_borrowed_window(), + super::strategy::BackendTag::HashChain => self.hc_matcher_mut().clear_borrowed_window(), + #[allow(unreachable_patterns)] _ => {} } self.borrowed_pending = None; @@ -1574,11 +1611,8 @@ impl MatchGeneratorDriver { /// [`Matcher::skip_matching_with_hint`] on this type. pub(crate) fn set_borrowed_block(&mut self, block_start: usize, block_end: usize) { assert!( - matches!( - self.active_backend(), - super::strategy::BackendTag::Simple | super::strategy::BackendTag::Dfast - ), - "borrowed block staging is only valid for the Simple (Fast) / Dfast backends", + self.borrowed_supported(), + "borrowed block staging is not supported for the active backend/search config", ); assert!( block_start <= block_end, @@ -1597,7 +1631,13 @@ impl MatchGeneratorDriver { super::strategy::BackendTag::Dfast => self .dfast_matcher_mut() .stage_borrowed_block(block_start, block_end), - _ => unreachable!(), + super::strategy::BackendTag::Row => self + .row_matcher_mut() + .stage_borrowed_block(block_start, block_end), + super::strategy::BackendTag::HashChain => self + .hc_matcher_mut() + .table + .stage_borrowed_block(block_start, block_end), } } @@ -2874,7 +2914,49 @@ impl Matcher for MatchGeneratorDriver { super::strategy::BackendTag::Dfast => self .dfast_matcher_mut() .start_matching_borrowed(block_start, block_end, &mut handle_sequence), - other => unreachable!("borrowed scan only for Simple/Dfast, got {other:?}"), + super::strategy::BackendTag::Row => { + // Same greedy/lazy parse split as the owned RowHash arm. + let greedy = self.parse == super::strategy::ParseMode::Greedy; + self.row_matcher_mut().start_matching_borrowed( + block_start, + block_end, + greedy, + &mut handle_sequence, + ); + } + super::strategy::BackendTag::HashChain => match self.search { + super::strategy::SearchMethod::HashChain => self + .hc_matcher_mut() + .start_matching_lazy_borrowed(block_start, block_end, &mut handle_sequence), + super::strategy::SearchMethod::BinaryTree => { + // Run the SAME BT dispatch as the owned BinaryTree arm + // below — every BT body reads its range via + // current_block_range() and bytes via live_history() + // (borrowed-aware), so the staged block is scanned in + // place. The table was already staged by + // `set_borrowed_block` (the HashChain arm at the top of + // this file calls `table.stage_borrowed_block` with the + // same range, and `borrowed_pending` is set only there), + // so no re-stage is needed here. + // Only btlazy2 reaches the borrowed BinaryTree scan: + // `borrowed_supported()` keeps the optimal parsers + // (BtOpt/BtUltra/BtUltra2) on the owned path, and + // `set_borrowed_block` asserts that predicate before any + // range is staged, so an optimal strategy_tag can never + // arrive here. + match self.strategy_tag { + StrategyTag::Btlazy2 => self + .hc_matcher_mut() + .start_matching_btlazy2(&mut handle_sequence), + other => unreachable!( + "borrowed BinaryTree scan is only supported for Btlazy2, got {other:?}" + ), + } + } + other => { + unreachable!("HashChain backend with unexpected search {other:?}") + } + }, } return; } @@ -2953,7 +3035,14 @@ impl Matcher for MatchGeneratorDriver { super::strategy::BackendTag::Dfast => self .dfast_matcher_mut() .skip_matching_borrowed(block_start, block_end, incompressible_hint), - other => unreachable!("borrowed skip only for Simple/Dfast, got {other:?}"), + super::strategy::BackendTag::Row => self.row_matcher_mut().skip_matching_borrowed( + block_start, + block_end, + incompressible_hint, + ), + super::strategy::BackendTag::HashChain => self + .hc_matcher_mut() + .skip_matching_borrowed(block_start, block_end, incompressible_hint), } return; } @@ -3096,7 +3185,14 @@ struct HcMatchGenerator { macro_rules! bt_insert_step_no_rebase_body { ($table:expr, $search_depth:expr, $abs_pos:ident, $current_abs_end:ident, $target_abs:ident, $cmf:path) => {{ let idx = $abs_pos - $table.history_abs_start; - let concat = &$table.history[$table.history_start..]; + // Borrowed-aware live region (owned: `history[history_start..]`; + // borrowed: the in-place input `[0, block_end)`). Reborrow-then-raw-ptr + // so the slice holds NO borrow and coexists with the `&mut $table` + // binary-tree writes below. Owned is byte-identical (same bytes). + let concat: &[u8] = unsafe { + let lh = $table.live_history(); + core::slice::from_raw_parts(lh.as_ptr(), lh.len()) + }; if idx + 8 > concat.len() { return 1; } @@ -3376,7 +3472,8 @@ fn btlazy2_gain(match_len: usize, offset: usize, reps: [u32; 3], ll0: bool) -> i macro_rules! start_matching_btlazy2_body { ($self:ident, $handle_sequence:ident, $collect:ident, $cmf:path $(,)?) => {{ $self.table.ensure_tables(); - let current_len = *$self.table.chunk_lens.back().unwrap(); + // Borrowed-aware: owned → last committed chunk; borrowed → staged block. + let (current_abs_start, current_len) = $self.table.current_block_range(); if current_len == 0 { return; } @@ -3384,18 +3481,18 @@ macro_rules! start_matching_btlazy2_body { // Mutates tables but never reallocates `history`, so this tail slice // stays valid for the routine's duration (same as the other parsers). let current: &[u8] = unsafe { core::slice::from_raw_parts(current_ptr, current_len) }; - // Full contiguous history (dict + prior blocks + current block) as a - // raw slice, for the explicit repcode probe: a rep offset can point - // before the current block, which `current` can't reach. Same - // no-realloc validity contract as `current`; holds no borrow, so it - // coexists with the `&mut self` collector calls below. + // Full contiguous live region (owned: dict + prior blocks + current + // block in `history`; borrowed: `[0, block_end)` of the in-place + // input) as a raw slice, for the explicit repcode probe: a rep offset + // can point before the current block, which `current` can't reach. + // `live_history()` is borrowed-aware; reborrow-then-raw-ptr so the + // slice holds NO borrow and coexists with the `&mut self` collector + // calls below. Same no-realloc validity contract as `current`. let history_abs_start = $self.table.history_abs_start; let concat_full: &[u8] = unsafe { - let base = $self.table.history.as_ptr().add($self.table.history_start); - core::slice::from_raw_parts(base, $self.table.history.len() - $self.table.history_start) + let lh = $self.table.live_history(); + core::slice::from_raw_parts(lh.as_ptr(), lh.len()) }; - let current_abs_start = - $self.table.history_abs_start + $self.table.window_size - current_len; let current_abs_end = current_abs_start + current_len; $self .table @@ -4507,7 +4604,7 @@ macro_rules! collect_optimal_candidates_initialized_body { } else if !skip_further_match_search { $self.table.insert_position($abs_pos); let max_chain_depth = $profile.max_chain_depth.min($self.hc.search_depth); - let concat = &$self.table.history[$self.table.history_start..]; + let concat = $self.table.live_history(); // Raw `+ 9` is safe here — see `bt_insert_step_no_rebase_body!` // for the full discussion of the upstream `STREAM_ABS_HEADROOM` // cap in `MatchTable::add_data`. @@ -4525,7 +4622,9 @@ macro_rules! collect_optimal_candidates_initialized_body { if candidate_abs == usize::MAX { break; } - if candidate_abs < $self.table.history_abs_start || candidate_abs >= $abs_pos { + if candidate_abs < $self.table.window_low_abs_for_target($abs_pos) + || candidate_abs >= $abs_pos + { continue; } let candidate_idx = candidate_abs - $self.table.history_abs_start; @@ -4755,7 +4854,14 @@ macro_rules! bt_insert_and_collect_matches_body { $cmf:path $(,)? ) => {{ let idx = $abs_pos - $table.history_abs_start; - let concat = &$table.history[$table.history_start..]; + // Borrowed-aware live region (owned: `history[history_start..]`; + // borrowed: the in-place input `[0, block_end)`). Reborrow-then-raw-ptr + // so the slice holds NO borrow and coexists with the `&mut $table` + // binary-tree writes below. Owned is byte-identical (same bytes). + let concat: &[u8] = unsafe { + let lh = $table.live_history(); + core::slice::from_raw_parts(lh.as_ptr(), lh.len()) + }; if idx + 8 > concat.len() { return; } @@ -5299,19 +5405,21 @@ impl HcMatchGenerator { ) { self.table.ensure_tables(); - let current_len = *self.table.chunk_lens.back().unwrap(); + // `current_block_range()` is borrowed-aware: owned → last committed + // chunk; borrowed → the staged in-place block range. + let (current_abs_start, current_len) = self.table.current_block_range(); if current_len == 0 { return; } - // The current block is the tail of `history`. Hoist it as a raw - // slice (like `start_matching_optimal`): the routine mutates the - // hash/chain tables + `offset_hist` but never reallocates - // `history`, so the slice stays valid and we avoid re-borrowing - // `self.table` (which would conflict with the `offset_hist` write). + // The current block is the tail of `history` (owned) or the staged + // borrowed range (`get_last_space()` resolves both). Hoist it as a raw + // slice: the routine mutates the hash/chain tables + `offset_hist` but + // never reallocates `history`, so the slice stays valid and we avoid + // re-borrowing `self.table` (which would conflict with the + // `offset_hist` write). let current_ptr = self.table.get_last_space().as_ptr(); let current: &[u8] = unsafe { core::slice::from_raw_parts(current_ptr, current_len) }; - let current_abs_start = self.table.history_abs_start + self.table.window_size - current_len; let current_abs_end = current_abs_start + current_len; self.table .backfill_boundary_positions(current_abs_start, current_abs_end); @@ -5360,6 +5468,45 @@ impl HcMatchGenerator { } } + /// Register the borrowed input window for the no-copy one-shot path. + /// # Safety + /// `buffer` must outlive the borrowed scans (see `MatchTable`). + pub(crate) unsafe fn set_borrowed_window(&mut self, buffer: &[u8]) { + // SAFETY: forwarded liveness contract. + unsafe { self.table.set_borrowed_window(buffer) }; + } + + pub(crate) fn clear_borrowed_window(&mut self) { + self.table.clear_borrowed_window(); + } + + /// Borrowed (no-copy) equivalent of [`Self::start_matching_lazy`]: stage + /// the in-place block range, then run the same lazy chain parse. The + /// parse reads its range via `current_block_range()` and its bytes via + /// `get_last_space()` / `live_history()`, all borrowed-aware, so the block + /// is scanned in place with the per-position window_low offset cap. + pub(crate) fn start_matching_lazy_borrowed( + &mut self, + block_start: usize, + block_end: usize, + handle_sequence: impl for<'a> FnMut(Sequence<'a>), + ) { + self.table.stage_borrowed_block(block_start, block_end); + self.start_matching_lazy(handle_sequence); + } + + /// Borrowed (no-copy) equivalent of the lazy `skip_matching`: stage the + /// in-place block, then seed positions without an owned-history append. + pub(crate) fn skip_matching_borrowed( + &mut self, + block_start: usize, + block_end: usize, + incompressible_hint: Option, + ) { + self.table.stage_borrowed_block(block_start, block_end); + self.table.skip_matching(incompressible_hint); + } + /// Donor `ZSTD_btlazy2` (levels 13-15): binary-tree match finder with a /// greedy/lazy parse. Bare dispatcher — resolves the runtime tier ONCE /// per block via `select_kernel()` and calls the matching @@ -5460,7 +5607,9 @@ impl HcMatchGenerator { mut handle_sequence: impl for<'a> FnMut(Sequence<'a>), ) { self.table.ensure_tables(); - let current_len = *self.table.chunk_lens.back().unwrap(); + // Borrowed-aware: owned → last committed chunk; borrowed → staged + // in-place block range. + let (current_abs_start, current_len) = self.table.current_block_range(); if current_len == 0 { return; } @@ -5470,7 +5619,6 @@ impl HcMatchGenerator { // the duration of the routine and avoids cloning the full block. let current = unsafe { core::slice::from_raw_parts(current_ptr, current_len) }; - let current_abs_start = self.table.history_abs_start + self.table.window_size - current_len; let current_abs_end = current_abs_start + current_len; self.table .apply_limited_update_after_long_match(current_abs_start); diff --git a/zstd/src/encoding/match_table/storage.rs b/zstd/src/encoding/match_table/storage.rs index 83fb9d78e..f6b8b19c1 100644 --- a/zstd/src/encoding/match_table/storage.rs +++ b/zstd/src/encoding/match_table/storage.rs @@ -297,6 +297,15 @@ pub(crate) struct MatchTable { /// searched by the BT/optimal collect alongside the live tree. `Some` /// once primed from a non-empty dictionary on a BT level. pub(crate) dms: DictAttach, + /// Borrowed (no-copy) one-shot input window `(ptr, len)`. When set, the + /// scan reads candidate/cursor bytes straight from the caller's slice + /// instead of the owned `history` mirror, so an over-window one-shot + /// input is matched in place (no input->mirror memmove). Raw pointer: + /// live until `clear_borrowed_window` / `reset`. + pub(crate) borrowed_input: Option<(*const u8, usize)>, + /// Active borrowed block range `[start, end)` within `borrowed_input`, + /// staged before each borrowed scan. + pub(crate) borrowed_block: Option<(usize, usize)>, } impl MatchTable { @@ -327,6 +336,8 @@ impl MatchTable { uses_bt: false, search_mls: 4, dms: DictAttach::new(), + borrowed_input: None, + borrowed_block: None, } } @@ -812,6 +823,21 @@ impl MatchTable { /// mirror. Match finders operate on this slice rather than the raw /// `history` Vec. pub(crate) fn live_history(&self) -> &[u8] { + // Borrowed one-shot: expose `[0, block_end)` of the caller's input so + // candidate/cursor reads land in place (the owned `history` mirror is + // empty under the no-copy path). Loop-invariant branch, inlines. + if let Some((_start, end)) = self.borrowed_block { + let (ptr, total) = self + .borrowed_input + .expect("borrowed_block set without a registered borrowed window"); + debug_assert!( + end <= total, + "borrowed block end {end} exceeds window {total}" + ); + // SAFETY: `ptr` is the registered window start (live by contract); + // `end <= total` in bounds. + return unsafe { core::slice::from_raw_parts(ptr, end) }; + } &self.history[self.history_start..] } @@ -824,10 +850,65 @@ impl MatchTable { /// the most recent buffer in the rolling window — panics if no /// data has been committed yet. pub(crate) fn get_last_space(&self) -> &[u8] { + if let (Some((ptr, _total)), Some((block_start, block_end))) = + (self.borrowed_input, self.borrowed_block) + { + // SAFETY: borrowed liveness contract; range validated when staged. + return unsafe { + core::slice::from_raw_parts(ptr.add(block_start), block_end - block_start) + }; + } let last = *self.chunk_lens.back().unwrap(); &self.history[self.history.len() - last..] } + /// Register the borrowed input window. Zeroes `history_abs_start` so + /// borrowed positions are absolute input offsets (the owned floor-advance + /// reset leaves it non-zero across frames; borrowed never `add_data`s, so + /// it stays 0 for the frame). Every probed candidate is byte-verified, so + /// stale table entries from a prior frame are rejected (or, if their bytes + /// coincidentally match, are genuine in-window matches). + /// + /// # Safety + /// `buffer` must stay live and unmodified until `clear_borrowed_window` + /// or `reset`. + pub(crate) unsafe fn set_borrowed_window(&mut self, buffer: &[u8]) { + self.borrowed_input = Some((buffer.as_ptr(), buffer.len())); + self.borrowed_block = None; + self.history_abs_start = 0; + } + + pub(crate) fn clear_borrowed_window(&mut self) { + self.borrowed_input = None; + self.borrowed_block = None; + } + + /// Stage `[block_start, block_end)` as the active borrowed block. + pub(crate) fn stage_borrowed_block(&mut self, block_start: usize, block_end: usize) { + let (_ptr, total) = self + .borrowed_input + .expect("stage_borrowed_block requires a registered borrowed window"); + assert!( + block_start <= block_end && block_end <= total, + "borrowed block bounds out of range: start={block_start} end={block_end} total={total}", + ); + self.borrowed_block = Some((block_start, block_end)); + } + + /// `(current_abs_start, current_len)` for the active scan. Borrowed: the + /// staged block range. Owned: the last committed chunk in the live window. + pub(crate) fn current_block_range(&self) -> (usize, usize) { + if let Some((start, end)) = self.borrowed_block { + (start, end - start) + } else { + let current_len = *self.chunk_lens.back().unwrap(); + ( + self.history_abs_start + self.window_size - current_len, + current_len, + ) + } + } + /// Test-only: append a chunk to the live window without eviction / /// compaction, mirroring the storage side of [`add_data`]. Replaces /// the old `window.push_back(vec)` setup now that chunk bytes live @@ -961,6 +1042,10 @@ impl MatchTable { self.history_start = 0; self.offset_hist = [1, 4, 8]; self.skip_insert_until_abs = 0; + // Clear borrowed-window state so a following OWNED frame reads the + // owned mirror; a borrowed frame re-arms via `set_borrowed_window`. + self.borrowed_input = None; + self.borrowed_block = None; self.dictionary_limit_abs = None; self.dictionary_primed_for_frame = false; self.allow_zero_relative_position = false; @@ -1793,8 +1878,7 @@ impl MatchTable { pub(crate) fn skip_matching(&mut self, incompressible_hint: Option) { self.ensure_tables(); - let current_len = *self.chunk_lens.back().unwrap(); - let current_abs_start = self.history_abs_start + self.window_size - current_len; + let (current_abs_start, current_len) = self.current_block_range(); let current_abs_end = current_abs_start + current_len; self.backfill_boundary_positions(current_abs_start, current_abs_end); if self.uses_bt { @@ -1982,10 +2066,21 @@ impl MatchTable { plan: &[HcOptimalSequence], handle_sequence: &mut impl for<'a> FnMut(Sequence<'a>), ) { - // Direct field borrow of `history` (not `get_last_space()`, which - // borrows all of `self`) so the per-sequence `&mut self.offset_hist` - // write below stays a disjoint-field borrow. - let current = &self.history[self.history.len() - current_len..]; + // Current block bytes. `get_last_space()` is borrowed-aware (owned: + // last committed chunk = `history[len-current_len..]`, byte-identical; + // borrowed: the staged in-place block). Reborrow-then-raw-ptr so the + // slice holds NO borrow and the per-sequence `&mut self.offset_hist` + // write below stays valid (the old direct `&self.history[..]` slice + // underflowed on the borrowed path, where `history` is empty). + let current: &[u8] = unsafe { + let ls = self.get_last_space(); + debug_assert!( + current_len <= ls.len(), + "current_len ({current_len}) exceeds block size ({})", + ls.len() + ); + core::slice::from_raw_parts(ls.as_ptr(), current_len) + }; if plan.is_empty() { handle_sequence(Sequence::Literals { literals: current }); return; diff --git a/zstd/src/encoding/row/mod.rs b/zstd/src/encoding/row/mod.rs index 81ce1a0bf..ebbb41523 100644 --- a/zstd/src/encoding/row/mod.rs +++ b/zstd/src/encoding/row/mod.rs @@ -239,11 +239,10 @@ macro_rules! greedy_parse_body { debug_assert_eq!($rl, $m.row_log); $m.ensure_tables(); - let current_len = *$m.chunk_lens.back().unwrap(); + let (current_abs_start, current_len) = $m.current_block_range(); if current_len == 0 { break 'parse; } - let current_abs_start = $m.history_abs_start + $m.window_size - current_len; let backfill_start = $m.backfill_start(current_abs_start); if backfill_start < current_abs_start { $m.insert_positions::<$rl>(backfill_start, current_abs_start); @@ -384,7 +383,14 @@ macro_rules! greedy_parse_body { // ~+447 over `abs_pos` (537897 -> 538344), so the // narrower range is intentional. $m.insert_match_span::<$rl>(abs_pos, candidate.start + candidate.match_len); - let current = &$m.history[$m.history.len() - current_len..]; + // Trailing literals of the current block. Read via + // `live_history()` (borrowed-aware) sliced at the block's + // offset within the live region: owned → + // `live_history()[window_size - current_len..]` (byte-identical + // to the old `history[history.len()-current_len..]`); borrowed → + // `live_history()[block_start..]` (the in-place input, since the + // owned `history` mirror is empty under the no-copy path). + let current = &$m.live_history()[(current_abs_start - $m.history_abs_start)..]; let literals = ¤t[literals_start..start]; $handle(Sequence::Triple { literals, @@ -434,7 +440,14 @@ macro_rules! greedy_parse_body { } if literals_start < current_len { - let current = &$m.history[$m.history.len() - current_len..]; + // Trailing literals of the current block. Read via + // `live_history()` (borrowed-aware) sliced at the block's + // offset within the live region: owned → + // `live_history()[window_size - current_len..]` (byte-identical + // to the old `history[history.len()-current_len..]`); borrowed → + // `live_history()[block_start..]` (the in-place input, since the + // owned `history` mirror is empty under the no-copy path). + let current = &$m.live_history()[(current_abs_start - $m.history_abs_start)..]; $handle(Sequence::Literals { literals: ¤t[literals_start..], }); @@ -494,11 +507,10 @@ macro_rules! lazy_parse_body { debug_assert_eq!($rl, $m.row_log); $m.ensure_tables(); - let current_len = *$m.chunk_lens.back().unwrap(); + let (current_abs_start, current_len) = $m.current_block_range(); if current_len == 0 { break 'parse; } - let current_abs_start = $m.history_abs_start + $m.window_size - current_len; let backfill_start = $m.backfill_start(current_abs_start); if backfill_start < current_abs_start { $m.insert_positions::<$rl>(backfill_start, current_abs_start); @@ -578,7 +590,14 @@ macro_rules! lazy_parse_body { }; if let Some(candidate) = picked { $m.insert_match_span::<$rl>(abs_pos, candidate.start + candidate.match_len); - let current = &$m.history[$m.history.len() - current_len..]; + // Trailing literals of the current block. Read via + // `live_history()` (borrowed-aware) sliced at the block's + // offset within the live region: owned → + // `live_history()[window_size - current_len..]` (byte-identical + // to the old `history[history.len()-current_len..]`); borrowed → + // `live_history()[block_start..]` (the in-place input, since the + // owned `history` mirror is empty under the no-copy path). + let current = &$m.live_history()[(current_abs_start - $m.history_abs_start)..]; let start = candidate.start - current_abs_start; let literals = ¤t[literals_start..start]; $handle(Sequence::Triple { @@ -624,7 +643,14 @@ macro_rules! lazy_parse_body { } if literals_start < current_len { - let current = &$m.history[$m.history.len() - current_len..]; + // Trailing literals of the current block. Read via + // `live_history()` (borrowed-aware) sliced at the block's + // offset within the live region: owned → + // `live_history()[window_size - current_len..]` (byte-identical + // to the old `history[history.len()-current_len..]`); borrowed → + // `live_history()[block_start..]` (the in-place input, since the + // owned `history` mirror is empty under the no-copy path). + let current = &$m.live_history()[(current_abs_start - $m.history_abs_start)..]; $handle(Sequence::Literals { literals: ¤t[literals_start..], }); @@ -989,7 +1015,17 @@ macro_rules! row_probe_body { continue; } let candidate_pos = raw_pos as usize; - if candidate_pos < $m.history_abs_start || candidate_pos >= $abs_pos { + // Lower bound = window low. Owned: `history_abs_start` (eviction + // floor) is always >= `abs_pos - max_window_size` (window_size <= + // max_window_size), so the `max` picks it — byte-identical to the + // pre-window_low check. Borrowed (history_abs_start forced to 0 in + // set_borrowed_window): the `max` picks `abs_pos - max_window_size`, + // capping the offset to the advertised window so an over-window + // in-place scan never emits an unresolvable offset. + let window_low = $m + .history_abs_start + .max($abs_pos.saturating_sub($m.max_window_size)); + if candidate_pos < window_low || candidate_pos >= $abs_pos { continue; } let candidate_idx = candidate_pos - $m.history_abs_start; @@ -1278,6 +1314,17 @@ pub(crate) struct RowMatchGenerator { /// activates the bounded dict probe in `row_candidate_rl`; built once and /// cached across frames via `DictAttach`, invalidated on eviction / resize. pub(crate) dict: DictAttach, + /// Borrowed (no-copy) one-shot input window: `(ptr, len)` into the + /// caller's slice. When set, the borrowed scan reads candidate/cursor + /// bytes straight from here instead of the owned `history` mirror, so an + /// over-window one-shot input is matched in place (no input->mirror copy). + /// Raw pointer: the slice must stay live until `clear_borrowed_window` / + /// `reset` (same contract as the Dfast/Simple borrowed backends). + pub(crate) borrowed_input: Option<(*const u8, usize)>, + /// Active borrowed block range `[start, end)` within `borrowed_input`, + /// staged before each borrowed scan so `live_history()` exposes + /// `[0, end)` and the parse loop scans `[start, end)`. + pub(crate) borrowed_block: Option<(usize, usize)>, } impl RowMatchGenerator { @@ -1302,6 +1349,8 @@ impl RowMatchGenerator { row_tags: Vec::new(), tag_kernel: crate::encoding::fastpath::select_kernel(), dict: DictAttach::new(), + borrowed_input: None, + borrowed_block: None, } } @@ -1380,6 +1429,12 @@ impl RowMatchGenerator { self.history.clear(); self.history_start = 0; self.offset_hist = [1, 4, 8]; + // Clear borrowed-window state so a following OWNED frame's + // `current_block_range()` / `live_history()` read the owned mirror, + // not a stale borrowed range. A borrowed frame re-arms via + // `set_borrowed_window` after this reset. + self.borrowed_input = None; + self.borrowed_block = None; if next_floor <= REBASE_RESET_FLOOR_CEILING && !self.row_positions.is_empty() { self.history_abs_start = next_floor; } else { @@ -1398,6 +1453,18 @@ impl RowMatchGenerator { } pub(crate) fn get_last_space(&self) -> &[u8] { + if let (Some((ptr, _total)), Some((block_start, block_end))) = + (self.borrowed_input, self.borrowed_block) + { + // Borrowed window: the active block is the in-place input range + // `[block_start, block_end)`, staged before the scan so the emit + // pipeline's pre-scan `get_last_space().len()` reserve is correct. + // SAFETY: borrowed liveness contract; `block_start <= block_end <= + // buffer len` (validated when staged). + return unsafe { + core::slice::from_raw_parts(ptr.add(block_start), block_end - block_start) + }; + } let last = *self.chunk_lens.back().unwrap(); &self.history[self.history.len() - last..] } @@ -1502,8 +1569,7 @@ impl RowMatchGenerator { ) { debug_assert_eq!(ROW_LOG, self.row_log); self.ensure_tables(); - let current_len = *self.chunk_lens.back().unwrap(); - let current_abs_start = self.history_abs_start + self.window_size - current_len; + let (current_abs_start, current_len) = self.current_block_range(); let current_abs_end = current_abs_start + current_len; let backfill_start = self.backfill_start(current_abs_start); if backfill_start < current_abs_start { @@ -1676,6 +1742,23 @@ impl RowMatchGenerator { } pub(crate) fn live_history(&self) -> &[u8] { + // Borrowed one-shot: candidate/cursor bytes live in the caller's + // input slice, not the owned mirror. Expose `[0, block_end)` so the + // scan reads every prior byte in place (no input->mirror copy). The + // branch is loop-invariant for a whole scan and inlines. + if let Some((_start, end)) = self.borrowed_block { + let (ptr, total) = self + .borrowed_input + .expect("borrowed_block set without a registered borrowed window"); + debug_assert!( + end <= total, + "borrowed block end {end} exceeds window {total}" + ); + // SAFETY: `ptr` is the registered borrowed window's start (live by + // the `set_borrowed_window` contract) and `end <= total` bytes are + // in bounds. + return unsafe { core::slice::from_raw_parts(ptr, end) }; + } &self.history[self.history_start..] } @@ -1683,6 +1766,59 @@ impl RowMatchGenerator { self.history_abs_start + self.live_history().len() } + /// Register the borrowed input window (the whole caller slice). Borrowed + /// blocks staged after this read their bytes from `buffer` in place. + /// + /// Zeroes `history_abs_start`: borrowed positions are absolute input + /// offsets (0-based), so the floor-advance reset's persistent non-zero + /// floor must be cleared for the duration of the borrowed frame. The + /// owned history is unused while borrowed (no `add_data` copy), so this + /// is safe; every probed candidate is byte-verified by the prefix + /// compare, so any stale table entry left from a prior frame is rejected + /// (or, if its bytes coincidentally match, is a genuine in-window match). + /// + /// # Safety + /// `buffer` must stay live and unmodified until `clear_borrowed_window` + /// or `reset` — the matcher stores a raw pointer into it. + pub(crate) unsafe fn set_borrowed_window(&mut self, buffer: &[u8]) { + self.borrowed_input = Some((buffer.as_ptr(), buffer.len())); + self.borrowed_block = None; + self.history_abs_start = 0; + } + + pub(crate) fn clear_borrowed_window(&mut self) { + self.borrowed_input = None; + self.borrowed_block = None; + } + + /// Stage `[block_start, block_end)` as the active borrowed block before a + /// scan so `live_history()` / `current_block_range()` report it. + pub(crate) fn stage_borrowed_block(&mut self, block_start: usize, block_end: usize) { + let (_ptr, total) = self + .borrowed_input + .expect("stage_borrowed_block requires a registered borrowed window"); + assert!( + block_start <= block_end && block_end <= total, + "borrowed block bounds out of range: start={block_start} end={block_end} total={total}", + ); + self.borrowed_block = Some((block_start, block_end)); + } + + /// `(current_abs_start, current_len)` for the active scan. Borrowed: the + /// staged block range (absolute input offsets). Owned: derived from the + /// last committed chunk in the live window. + fn current_block_range(&self) -> (usize, usize) { + if let Some((start, end)) = self.borrowed_block { + (start, end - start) + } else { + let current_len = *self.chunk_lens.back().unwrap(); + ( + self.history_abs_start + self.window_size - current_len, + current_len, + ) + } + } + /// Row hash key at `idx`: `key_len` bytes (donor `mls`, 5-6 on the row /// levels) via one masked 8-byte read, degrading to the 4-byte key in /// the last <8 bytes of the window. Shared by the live hash and the @@ -1980,6 +2116,42 @@ impl RowMatchGenerator { } } + /// Borrowed (no-copy) one-shot equivalent of [`Self::start_matching`]: + /// stage `[block_start, block_end)` of the registered borrowed window, + /// then run the SAME parse dispatch. The parse body reads its block range + /// via `current_block_range()` and its bytes via `live_history()`, both + /// borrowed-aware, so the staged block is scanned in place (no + /// `add_data` copy into the owned mirror). `history_abs_start` was forced + /// to 0 in `set_borrowed_window`, so positions stay absolute input + /// offsets and the window-low candidate cap bounds offsets to the window. + pub(crate) fn start_matching_borrowed( + &mut self, + block_start: usize, + block_end: usize, + greedy: bool, + handle_sequence: impl for<'a> FnMut(Sequence<'a>), + ) { + self.stage_borrowed_block(block_start, block_end); + if greedy { + self.start_matching_greedy(handle_sequence); + } else { + self.start_matching(handle_sequence); + } + } + + /// Borrowed equivalent of [`Self::skip_matching_with_hint`]: stage the + /// block (so the RLE/Raw emit's `get_last_space` reserve reports it) and + /// seed the row tables without a copy, mirroring the owned skip. + pub(crate) fn skip_matching_borrowed( + &mut self, + block_start: usize, + block_end: usize, + incompressible_hint: Option, + ) { + self.stage_borrowed_block(block_start, block_end); + self.skip_matching_with_hint(incompressible_hint); + } + #[allow(dead_code)] pub(crate) fn best_match(&self, abs_pos: usize, lit_len: usize) -> Option { dispatch_tag_kernel!(self.best_match_k(abs_pos, lit_len)) diff --git a/zstd/src/tests/roundtrip_integrity.rs b/zstd/src/tests/roundtrip_integrity.rs index be95f36f5..7355a0884 100644 --- a/zstd/src/tests/roundtrip_integrity.rs +++ b/zstd/src/tests/roundtrip_integrity.rs @@ -912,9 +912,16 @@ fn borrowed_oneshot_matches_owned_and_roundtrips() { CompressionLevel::Level(1), CompressionLevel::Level(2), CompressionLevel::Level(-6), - // Non-Fast: pins the owned-fallback branch that - // compress_oneshot_borrowed takes for every non-Fast caller. + // Dfast (L3) borrowed over-window path. CompressionLevel::Default, + // Row backend borrowed over-window path: L5 = greedy parse, L9 = + // lazy parse. The 3 MiB case forces eviction in the owned baseline + // while the borrowed scan applies the per-position window_low cap; + // the two must still produce byte-identical frames. + CompressionLevel::Level(5), + CompressionLevel::Level(9), + // HashChain/BinaryTree btlazy2 borrowed over-window path (L13-15). + CompressionLevel::Level(15), ] { for &(seed, len) in &cases { for data in [generate_compressible(seed, len), generate_data(seed, len)] {