From 10ce4992ed3c8e34e9d1cc18889b1620714a984b Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Fri, 5 Jun 2026 23:05:33 +0300 Subject: [PATCH 01/33] perf(decode): share dictionary by handle instead of per-frame copy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The decoder copied the whole dictionary content into a per-frame `DecodeBuffer::dict_content` Vec on every `init_from_dict` (memmove ~27% of a small dict-frame decode). Hold the dictionary by shared handle (`DictionaryHandle` = `Arc`/`Rc` on the no-std fallback) and read match bytes straight out of it in `repeat_from_dict` — donor `ZSTD_refDDict` semantics. One dictionary content copy is now shared across every frame AND across decoder instances on other threads (`Arc` is Send + Sync); attaching a dictionary is a refcount bump, never a content memcpy. Registered dicts (`owned_dicts`) also move to handles so all attach paths are uniform. no-std stays intact via the existing SharedDictionary alias (Arc with atomics, Rc without). --- zstd/src/decoding/decode_buffer.rs | 66 ++++++++++++++++--- zstd/src/decoding/frame_decoder.rs | 17 +++-- zstd/src/decoding/scratch.rs | 23 ++++--- zstd/src/decoding/sequence_section_decoder.rs | 10 ++- 4 files changed, 88 insertions(+), 28 deletions(-) diff --git a/zstd/src/decoding/decode_buffer.rs b/zstd/src/decoding/decode_buffer.rs index bdbd08f27..ae7220d31 100644 --- a/zstd/src/decoding/decode_buffer.rs +++ b/zstd/src/decoding/decode_buffer.rs @@ -26,7 +26,14 @@ use crate::decoding::errors::DecodeBufferError; /// backlog item #132. pub struct DecodeBuffer { buffer: B, - pub dict_content: Vec, + /// Active dictionary, held by shared handle (`Arc`/`Rc`) rather than a + /// per-frame owned copy. `repeat_from_dict` reads match bytes straight + /// out of the handle's content (donor `ZSTD_refDDict` semantics): one + /// dictionary copy is shared across every frame AND across decoder + /// instances on other threads (`Arc` is `Send + Sync`), so + /// reusing a dictionary costs a refcount bump, never a content memcpy. + /// `None` = no dictionary (a no-dict frame can never read a stale one). + pub(crate) dict: Option, pub window_size: usize, total_output_counter: u64, @@ -83,7 +90,7 @@ impl DecodeBuffer { pub fn new(window_size: usize) -> DecodeBuffer { DecodeBuffer { buffer: B::new(), - dict_content: Vec::new(), + dict: None, window_size, total_output_counter: 0, block_output_limit: usize::MAX, @@ -108,7 +115,7 @@ impl DecodeBuffer { buffer.clear(); DecodeBuffer { buffer, - dict_content: Vec::new(), + dict: None, window_size, total_output_counter: 0, block_output_limit: usize::MAX, @@ -159,7 +166,7 @@ impl DecodeBuffer { // `DecoderScratchKind::reserve_buffer(window_size)` before any // block writes — that is the only call site that knows whether // the frame will actually hit this buffer. - self.dict_content.clear(); + self.dict = None; self.total_output_counter = 0; // Cleared per frame; re-armed per block by the sequence decoder. self.block_output_limit = usize::MAX; @@ -173,6 +180,25 @@ impl DecodeBuffer { self.buffer.len() } + /// Active dictionary content bytes, borrowed through the shared handle + /// (no copy). Empty slice when no dictionary is attached. + #[inline] + pub(crate) fn dict_content(&self) -> &[u8] { + match &self.dict { + Some(h) => &h.as_dict().dict_content, + None => &[], + } + } + + /// Attach a dictionary by shared handle. This is a refcount bump + /// (`Arc`/`Rc` clone) — the dictionary content is shared, never copied, + /// so the same dictionary is free to reuse across frames and across + /// decoder instances on other threads. + #[inline] + pub(crate) fn set_dict(&mut self, handle: crate::decoding::dictionary::DictionaryHandle) { + self.dict = Some(handle); + } + /// Return the last `n` bytes of the visible buffer as two /// contiguous slices (`(s1, s2)` matching the wrap semantics of /// the underlying backend). `n` must be `<= self.len()`. Used by @@ -775,24 +801,36 @@ impl DecodeBuffer { // at least part of that repeat is from the dictionary content let bytes_from_dict = offset - self.buffer.len(); - if bytes_from_dict > self.dict_content.len() { + // Borrow the dictionary content through the shared handle as a + // field access (`self.dict`), kept disjoint from the `self.buffer` + // mutation below so the borrow checker allows the read+extend + // without an intermediate copy. `None` → empty slice → the + // length guard rejects (matches the old empty-`dict_content` + // behaviour on the direct/no-dict path). + let dict_content: &[u8] = match &self.dict { + Some(h) => &h.as_dict().dict_content, + None => &[], + }; + let dict_len = dict_content.len(); + + if bytes_from_dict > dict_len { return Err(DecodeBufferError::NotEnoughBytesInDictionary { - got: self.dict_content.len(), + got: dict_len, need: bytes_from_dict, }); } if bytes_from_dict < match_length { - let dict_slice = &self.dict_content[self.dict_content.len() - bytes_from_dict..]; + let dict_slice = &dict_content[dict_len - bytes_from_dict..]; prefetch::prefetch_slice(dict_slice); self.buffer.extend(dict_slice); self.total_output_counter += bytes_from_dict as u64; return self.repeat(self.buffer.len(), match_length - bytes_from_dict); } else { - let low = self.dict_content.len() - bytes_from_dict; + let low = dict_len - bytes_from_dict; let high = low + match_length; - let dict_slice = &self.dict_content[low..high]; + let dict_slice = &dict_content[low..high]; prefetch::prefetch_slice(dict_slice); self.buffer.extend(dict_slice); self.total_output_counter += match_length as u64; @@ -1320,7 +1358,15 @@ mod tests { #[test] fn repeat_from_dict_full_copy_updates_total_output_counter() { let mut decode_buf = DecodeBuffer::::new(1); - decode_buf.dict_content = b"0123456789".to_vec(); + decode_buf.set_dict( + crate::decoding::dictionary::DictionaryHandle::from_dictionary( + crate::decoding::dictionary::Dictionary::from_raw_content( + 1, + b"0123456789".to_vec(), + ) + .unwrap(), + ), + ); decode_buf.repeat(10, 2).unwrap(); let err = decode_buf.repeat(10, 1).unwrap_err(); diff --git a/zstd/src/decoding/frame_decoder.rs b/zstd/src/decoding/frame_decoder.rs index e899d044e..305322796 100644 --- a/zstd/src/decoding/frame_decoder.rs +++ b/zstd/src/decoding/frame_decoder.rs @@ -145,7 +145,11 @@ fn block_body_decode_error( /// ``` pub struct FrameDecoder { state: Option, - owned_dicts: BTreeMap, + // Registered dictionaries are stored by shared handle (Arc/Rc) so a + // single content copy is referenced by every frame the decoder decodes + // (donor `ZSTD_refDDict`), rather than re-copied into the decode buffer + // per frame. `add_dict` wraps an owned `Dictionary` into a handle. + owned_dicts: BTreeMap, #[cfg(target_has_atomic = "ptr")] shared_dicts: BTreeMap, #[cfg(not(target_has_atomic = "ptr"))] @@ -257,7 +261,7 @@ impl DecoderScratchKind { } } - fn init_from_dict(&mut self, dict: &Dictionary) { + fn init_from_dict(&mut self, dict: &DictionaryHandle) { match self { Self::Ring(s) => s.init_from_dict(dict), Self::Flat(s) => s.init_from_dict(dict), @@ -774,7 +778,7 @@ impl FrameDecoder { .or_else(|| { #[cfg(target_has_atomic = "ptr")] { - shared_dicts.get(&dict_id).map(DictionaryHandle::as_dict) + shared_dicts.get(&dict_id) } #[cfg(not(target_has_atomic = "ptr"))] { @@ -855,7 +859,7 @@ impl FrameDecoder { provided: dict.id(), }); } - state.decoder_scratch.init_from_dict(dict.as_dict()); + state.decoder_scratch.init_from_dict(dict); state.using_dict = Some(dict.id()); Ok(()) } @@ -870,7 +874,8 @@ impl FrameDecoder { if self.owned_dicts.contains_key(&dict_id) || self.shared_dict_exists(dict_id) { return Err(FrameDecoderError::DictAlreadyRegistered { dict_id }); } - self.owned_dicts.insert(dict_id, dict); + self.owned_dicts + .insert(dict_id, DictionaryHandle::from_dictionary(dict)); Ok(()) } @@ -910,7 +915,7 @@ impl FrameDecoder { .or_else(|| { #[cfg(target_has_atomic = "ptr")] { - shared_dicts.get(&dict_id).map(DictionaryHandle::as_dict) + shared_dicts.get(&dict_id) } #[cfg(not(target_has_atomic = "ptr"))] { diff --git a/zstd/src/decoding/scratch.rs b/zstd/src/decoding/scratch.rs index 8a454e98d..bd8edcc9a 100644 --- a/zstd/src/decoding/scratch.rs +++ b/zstd/src/decoding/scratch.rs @@ -3,7 +3,7 @@ use super::buffer_backend::BufferBackend; use super::decode_buffer::DecodeBuffer; use super::ringbuffer::RingBuffer; -use crate::decoding::dictionary::Dictionary; +use crate::decoding::dictionary::DictionaryHandle; use crate::fse::SeqFSETable; use crate::huff0::HuffmanTable; use alloc::vec::Vec; @@ -222,14 +222,15 @@ impl DecoderScratch { self.huf.table.reset(); } - pub fn init_from_dict(&mut self, dict: &Dictionary) { - self.fse.reinit_from(&dict.fse); - self.huf.table.reinit_from(&dict.huf.table); - self.offset_hist = dict.offset_hist; - self.buffer.dict_content.clear(); - self.buffer - .dict_content - .extend_from_slice(&dict.dict_content); + pub fn init_from_dict(&mut self, dict: &DictionaryHandle) { + let d = dict.as_dict(); + self.fse.reinit_from(&d.fse); + self.huf.table.reinit_from(&d.huf.table); + self.offset_hist = d.offset_hist; + // Share the dictionary content by handle (Arc/Rc clone = refcount + // bump) instead of copying it into a per-frame buffer; the decoder + // reads match bytes straight out of the shared content. + self.buffer.set_dict(dict.clone()); // Donor parity: `ZSTD_decompressBegin_usingDDict` sets // `dctx->ddictIsCold = 1` so the first block of the frame // engages the prefetch decoder regardless of long-offset @@ -365,7 +366,9 @@ mod tests { extern crate std; let dict_raw = std::fs::read("./dict_tests/dictionary").expect("dictionary fixture should load"); - let dict = Dictionary::decode_dict(&dict_raw).expect("dictionary should parse"); + let dict = DictionaryHandle::from_dictionary( + Dictionary::decode_dict(&dict_raw).expect("dictionary should parse"), + ); let mut scratch: DecoderScratch = DecoderScratch::new(1024); assert!( !scratch.fse.ddict_is_cold, diff --git a/zstd/src/decoding/sequence_section_decoder.rs b/zstd/src/decoding/sequence_section_decoder.rs index 6856957e1..25b875b95 100644 --- a/zstd/src/decoding/sequence_section_decoder.rs +++ b/zstd/src/decoding/sequence_section_decoder.rs @@ -164,8 +164,14 @@ where let old_buffer_size = buffer.len(); let num_sequences = section.num_sequences as usize; - // `saturating_add` defuses 32-bit `usize` wrap. - let total_history = buffer.window_size.saturating_add(buffer.dict_content.len()); + // Overflow is only reachable on 32-bit `usize` (a 4 GiB-class + // window_size plus a dict). The gate below asks "does history exceed + // the prefetch threshold", so on the overflow path the clamped maximum + // is the correct answer, not a wrapped small value. + let total_history = match buffer.window_size.checked_add(buffer.dict_content().len()) { + Some(sum) => sum, + None => usize::MAX, + }; let use_long_pipeline = compute_use_long_pipeline( num_sequences, ddict_is_cold, From cadab085a44cb3ef0ce321a0b6965db316b10310 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Fri, 5 Jun 2026 23:21:39 +0300 Subject: [PATCH 02/33] perf(decode): copy only decode-essential entropy on dict reinit Per-frame dict attach (`init_from_dict`) ran `reinit_from` on the FSE and HUF tables, which copied the full build-time workspace into the decoder scratch every frame: FSE `symbol_probabilities` + spread buffer, and HUF `weights` / `bits` / `rank_indexes` plus the recursive weight-decoding `fse_table`. The decode hot path (and Repeat-mode reuse) reads only the `decode` table (FSE) and `packed_decode` + `max_num_bits` + `state_mask` (HUF); the rest is scratch any rebuilding block repopulates itself. Copy only the decode-essential state, mirroring the donor copying just the entropy decode tables per frame rather than the whole build workspace. Cuts the per-frame dict-setup memmove on small dict frames. --- zstd/src/fse/fse_decoder.rs | 11 +++++++---- zstd/src/huff0/huff0_decoder.rs | 10 ++++++---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/zstd/src/fse/fse_decoder.rs b/zstd/src/fse/fse_decoder.rs index 2fb0d6a45..e9acd8a31 100644 --- a/zstd/src/fse/fse_decoder.rs +++ b/zstd/src/fse/fse_decoder.rs @@ -288,10 +288,13 @@ impl FSETableImpl { /// but skip the bytes copy. pub fn reinit_from(&mut self, other: &Self) { self.reset(); - self.symbol_probabilities - .extend_from_slice(&other.symbol_probabilities); - self.symbol_spread_buffer - .reserve(other.symbol_spread_buffer.len()); + // Copy ONLY the decode-time state. `symbol_probabilities` and + // `symbol_spread_buffer` are build-time scratch ("used while building + // the decode Vector"): the decode hot path and Repeat-mode reuse read + // only `decode` + `accuracy_log`, and any block that rebuilds the + // table (`build_decoder`) repopulates the scratch itself. Skipping + // them mirrors the donor copying just the FSE decode table per frame + // instead of the full build workspace. self.decode.extend_from_slice(&other.decode); self.accuracy_log = other.accuracy_log; } diff --git a/zstd/src/huff0/huff0_decoder.rs b/zstd/src/huff0/huff0_decoder.rs index c3bfa0261..e52e9e8c8 100644 --- a/zstd/src/huff0/huff0_decoder.rs +++ b/zstd/src/huff0/huff0_decoder.rs @@ -356,13 +356,15 @@ impl HuffmanTable { /// of `other`. pub fn reinit_from(&mut self, other: &Self) { self.reset(); + // Copy ONLY the decode-time state. `weights` / `bits` / + // `rank_indexes` and the weight-decoding `fse_table` are build-time + // scratch (repopulated when a block carries a new HUF table); the + // literal-decode hot path reads only `packed_decode` + `max_num_bits` + // + `state_mask`. Skipping the rest mirrors the donor copying just the + // HUF decode table per frame, not the full build workspace. self.packed_decode.extend_from_slice(&other.packed_decode); - self.weights.extend_from_slice(&other.weights); self.max_num_bits = other.max_num_bits; self.state_mask = other.state_mask; - self.bits.extend_from_slice(&other.bits); - self.rank_indexes.extend_from_slice(&other.rank_indexes); - self.fse_table.reinit_from(&other.fse_table); } /// Completely empty the table of all data. From 1c830ae645a74d5baf2bdd0baabef7ed25b99c36 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Fri, 5 Jun 2026 23:21:53 +0300 Subject: [PATCH 03/33] test(bench): measure compress-dict steady-state with reused context The compress-dict arms built a fresh compressor + parsed the dictionary inside b.iter every iteration, measuring one-time setup cost rather than steady-state throughput and unfairly penalising the side with heavier per-attach setup. Build the compressor + attach the dictionary ONCE before b.iter on both sides (C CDict created once + compress per frame; Rust prepared EncoderDictionary + compress_independent_frame per frame), the real prepared-dictionary lifecycle. compress_independent_frame_into reads input in place and takes the output per call, so it needs no set_source/set_drain (sidesteps the lifetime issue that forced the old per-iter shape). --- zstd/benches/compare_ffi.rs | 97 ++++++++++++++++++------------------- 1 file changed, 48 insertions(+), 49 deletions(-) diff --git a/zstd/benches/compare_ffi.rs b/zstd/benches/compare_ffi.rs index 7a1675466..875224385 100644 --- a/zstd/benches/compare_ffi.rs +++ b/zstd/benches/compare_ffi.rs @@ -30,7 +30,7 @@ use structured_zstd::decoding::FrameDecoder; use structured_zstd::dictionary::{ FastCoverOptions, FinalizeOptions, finalize_raw_dict, train_fastcover_raw_from_slice, }; -use structured_zstd::encoding::FrameCompressor; +use structured_zstd::encoding::{EncoderDictionary, FrameCompressor}; use support::{ LevelConfig, Scenario, ScenarioClass, benchmark_scenarios, kernel_report_line, supported_levels_filtered, @@ -651,60 +651,59 @@ fn bench_dictionary(c: &mut Criterion) { }) }); - // c_ffi_with_dict + pure_rust_with_dict: both construct - // their compressor + attach dict INSIDE `b.iter` so the - // measurement covers the same shape on both sides. - // Asymmetric setup-vs-iter placement was flagged in - // PR #277 review (Copilot threads #2/#3/#4): the Rust - // FrameCompressor API stores `set_drain`'s `&mut Vec` - // inside the compressor for its full lifetime, so a - // "compressor outside b.iter, fresh Vec inside" pattern - // doesn't type-check. Rather than gymnast around that, - // match the per-iter shape on both arms — the FFI per- - // iter construction cost is small (`Compressor::with_ - // dictionary` is a single CDict reference attach into a - // freshly-allocated CCtx) and stays comparable to the - // Rust per-iter `FrameCompressor::new + - // set_dictionary_from_bytes`. + // c_ffi_with_dict + pure_rust_with_dict: STEADY-STATE measurement. + // Both build the compressor + attach the dictionary ONCE before + // `b.iter`, then compress in the loop reusing that context — the + // real prepared-dictionary lifecycle (C `CDict` created once + + // `ZSTD_compress_usingCDict` per frame; Rust prepared + // `EncoderDictionary` + `compress_independent_frame` per frame). + // The earlier per-iter shape (fresh compressor + dict parse every + // iteration) measured one-time setup cost, not steady-state + // throughput, and unfairly penalised the side with heavier + // per-attach setup. `compress_independent_frame_into` reads the + // input in place and takes the output buffer per call, so it needs + // no `set_drain`/`set_source` — sidestepping the lifetime issue + // (PR #277) that forced the old per-iter shape. group.bench_function("c_ffi_with_dict", |b| { - b.iter(|| { - let mut compressor = - zstd::bulk::Compressor::with_dictionary(level.ffi_level, &ffi_dictionary) - .unwrap(); - black_box(compressor.compress(&scenario.bytes).unwrap()) - }) + let mut compressor = + zstd::bulk::Compressor::with_dictionary(level.ffi_level, &ffi_dictionary) + .unwrap(); + b.iter(|| black_box(compressor.compress(&scenario.bytes).unwrap())) }); // Gate pure_rust_with_dict registration on the same - // `rust_dict_handle.is_some()` signal that - // decompress-dict uses below — if DictionaryHandle:: - // decode_dict failed earlier (the per-scenario parse - // above), FrameCompressor::set_dictionary_from_bytes - // routes through the same Dictionary::decode_dict and - // would fail identically. An `.expect()` panic inside - // b.iter would abort the whole bench suite. - if let Some(preallocated_capacity) = rust_with_dict_len { - // `rust_with_dict_len` above already did the one-time - // pre-compress that learns the Rust encoder's actual output - // size at this (scenario, level, dict) — reuse it as the - // timing-loop preallocation hint. Using `with_dict_bytes.len()` - // (the FFI-side size) would under-allocate when Rust emits a - // slightly larger frame, forcing `Vec` reallocations inside the - // timing loop and skewing the measurement vs FFI (which always - // allocates exact output size internally). + // `rust_dict_handle.is_some()` signal that decompress-dict uses + // below — if the per-scenario dictionary parse failed earlier, + // `EncoderDictionary::from_bytes` routes through the same parse and + // would fail identically; an `.expect()` panic before `b.iter` + // would abort the whole bench suite. + if let Some(preallocated_capacity) = rust_with_dict_len + && EncoderDictionary::from_bytes(&ffi_dictionary).is_ok() + { group.bench_function("pure_rust_with_dict", |b| { + // `compress_independent_frame_into` reads input in + // place + takes the output buffer per call, so neither + // the source `R` nor drain `W` generic is ever bound by + // a `set_source`/`set_drain` call — pin them to the + // defaults so inference has a concrete type. + let mut compressor: FrameCompressor = FrameCompressor::new(level.rust_level); + compressor + .set_encoder_dictionary( + EncoderDictionary::from_bytes(&ffi_dictionary) + .expect("dictionary parse checked above"), + ) + .expect("prepared dictionary should attach"); + // Reuse one output buffer across iterations (the + // CCtx-equivalent caller-owned `dst`). `rust_with_dict_len` + // pre-sizes it so no realloc happens inside the loop. + let mut compressed = Vec::with_capacity(preallocated_capacity); b.iter(|| { - let mut compressor = FrameCompressor::new(level.rust_level); - compressor - .set_dictionary_from_bytes(&ffi_dictionary) - .expect("dictionary should attach"); - compressor.set_source_size_hint(scenario.bytes.len() as u64); - compressor.set_source(scenario.bytes.as_slice()); - let mut compressed = Vec::with_capacity(preallocated_capacity); - compressor.set_drain(&mut compressed); - compressor.compress(); - black_box(compressed) - }) + compressor.compress_independent_frame_into( + scenario.bytes.as_slice(), + &mut compressed, + ); + black_box(&compressed); + }); }); } From 007864f0c498a34c6276c957650ea7b597d95e57 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Fri, 5 Jun 2026 23:45:41 +0300 Subject: [PATCH 04/33] perf(encode): cache primed dict matcher state (CDict-equivalent) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reusing a dictionary across frames re-hashed every dictionary position into the match tables on EVERY frame (prepare_frame -> reset + full prime_with_dictionary), the dominant cost of small dict-compress (steady-state ~10x slower than C, which copies a precomputed cdict->matchState). Snapshot the post-prime matcher state once (the backend storage hash tables + dictionary history + offset history + window, plus the driver's dictionary_retained_budget — the only state prime writes), keyed by compression level, and restore it (a table copy) on later frames instead of re-priming. Mirrors donor ZSTD_compressBegin_usingCDict. The snapshot is invalidated on dictionary attach/clear and ignored on a level mismatch, so it can never be restored against the wrong dict/level. Matcher backends gain Clone for the storage snapshot; CompressionLevel gains PartialEq for the level key. New regression test asserts frame 2 (restored) is byte-identical to frame 1 (freshly primed) and both round-trip through a dict-primed decoder. --- zstd/src/encoding/bt/mod.rs | 1 + zstd/src/encoding/dfast/mod.rs | 1 + zstd/src/encoding/frame_compressor.rs | 73 ++++++++++++++++++- zstd/src/encoding/hc/mod.rs | 1 + zstd/src/encoding/ldm/mod.rs | 1 + zstd/src/encoding/ldm/table.rs | 1 + zstd/src/encoding/match_generator.rs | 39 ++++++++++ zstd/src/encoding/match_table/storage.rs | 1 + zstd/src/encoding/mod.rs | 17 ++++- zstd/src/encoding/row/mod.rs | 1 + .../encoding/simple/fast_kernel/hash_table.rs | 1 + zstd/src/encoding/simple/fast_matcher.rs | 1 + 12 files changed, 135 insertions(+), 3 deletions(-) diff --git a/zstd/src/encoding/bt/mod.rs b/zstd/src/encoding/bt/mod.rs index 099a4130d..b15b8b75a 100644 --- a/zstd/src/encoding/bt/mod.rs +++ b/zstd/src/encoding/bt/mod.rs @@ -36,6 +36,7 @@ pub(crate) const HC3_MAX_OFFSET: usize = 1 << 18; /// `BtUltra2` parse modes. Owns the cost model and the per-frame /// scratch arenas; the actual BT pointer-pair table lives on the /// shared [`super::match_table::storage::MatchTable`]. +#[derive(Clone)] pub(crate) struct BtMatcher { /// Donor `optStatePtr_t` — Huffman / FSE-derived literal and /// sequence-symbol cost tables that drive the optimal parser. diff --git a/zstd/src/encoding/dfast/mod.rs b/zstd/src/encoding/dfast/mod.rs index 5932373c1..a9fe554fe 100644 --- a/zstd/src/encoding/dfast/mod.rs +++ b/zstd/src/encoding/dfast/mod.rs @@ -51,6 +51,7 @@ const HASH_READ_SIZE: usize = 8; /// rep emissions that upstream produces. const DFAST_REP_MIN_MATCH_LEN: usize = 4; +#[derive(Clone)] pub(crate) struct DfastMatchGenerator { pub(crate) max_window_size: usize, /// Per-block length queue. Previously held the raw input diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index 212ef091a..e18474f3a 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -924,9 +924,25 @@ impl FrameCompressor { // This state drives sequence encoding, while matcher priming below updates // the match generator's internal repeat-offset history for match finding. self.state.offset_hist = dict.inner.offset_hist; - self.state + // CDict fast path (donor `ZSTD_compressBegin_usingCDict`): restore + // the precomputed primed matcher state (a table copy) instead of + // re-hashing every dictionary position into the match tables. The + // snapshot is captured on the first prime and reused while the + // dictionary + level stay the same; `restore` returns false when + // it must be (re)built. + if !self + .state .matcher - .prime_with_dictionary(dict.inner.dict_content.as_slice(), dict.inner.offset_hist); + .restore_primed_dictionary(self.compression_level) + { + self.state.matcher.prime_with_dictionary( + dict.inner.dict_content.as_slice(), + dict.inner.offset_hist, + ); + self.state + .matcher + .capture_primed_dictionary(self.compression_level); + } } if let Some(cache) = cached_entropy { self.state.last_huff_table.clone_from(&cache.huff); @@ -1486,6 +1502,10 @@ impl FrameCompressor { /// Remove the attached dictionary, returning it as an [`EncoderDictionary`]. pub fn clear_dictionary(&mut self) -> Option { self.dictionary_entropy_cache = None; + // Drop the CDict prime snapshot — it is keyed to the dictionary + // being removed and must not be restored against a different (or no) + // dictionary on the next frame. + self.state.matcher.invalidate_primed_dictionary(); self.dictionary.take() } @@ -1526,6 +1546,10 @@ impl FrameCompressor { .to_encoder_table() .map(|table| PreviousFseTable::Custom(Box::new(table))), }); + // A previously-captured CDict prime snapshot belongs to the OLD + // dictionary; drop it so the first frame with the new dictionary + // re-primes (and re-captures) instead of restoring stale tables. + self.state.matcher.invalidate_primed_dictionary(); Ok(self.dictionary.replace(enc)) } } @@ -2362,6 +2386,51 @@ mod tests { ); } + #[test] + fn dict_primed_matcher_snapshot_reused_across_frames_is_byte_identical() { + // CDict-equivalent: a compressor reused across frames with the same + // dictionary restores the primed matcher snapshot on frames 2..N + // (a table copy) instead of re-hashing the dictionary. The restored + // state must reproduce the first-frame (freshly-primed) output + // byte-for-byte, and every frame must round-trip through a + // dict-primed decoder. + let dict_raw = include_bytes!("../../dict_tests/dictionary"); + let payload = b"tenant=demo table=orders op=put key=1 value=aaaaabbbbbcccccdddddeeeee\n\ + tenant=demo table=orders op=put key=2 value=aaaaabbbbbcccccdddddeeeee\n"; + + let prepared = + super::EncoderDictionary::from_bytes(dict_raw).expect("dict bytes should parse"); + let dict_id = prepared.id(); + let mut compressor: FrameCompressor = + FrameCompressor::new(super::CompressionLevel::Fastest); + compressor + .set_encoder_dictionary(prepared) + .expect("prepared dictionary should attach"); + + // Frame 1 primes + captures the snapshot; frame 2 restores it. + let frame1 = compressor.compress_independent_frame(payload.as_slice()); + let frame2 = compressor.compress_independent_frame(payload.as_slice()); + assert_eq!( + frame1, frame2, + "restored prime snapshot must reproduce the freshly-primed frame byte-for-byte" + ); + + // Both frames advertise the dict id and round-trip through a + // dict-primed decoder. + for frame in [&frame1, &frame2] { + let (hdr, _) = + crate::decoding::frame::read_frame_header(frame.as_slice()).expect("frame header"); + assert_eq!(hdr.dictionary_id(), Some(dict_id)); + let mut decoder = FrameDecoder::new(); + decoder + .add_dict(crate::decoding::Dictionary::decode_dict(dict_raw).unwrap()) + .unwrap(); + let mut decoded = Vec::with_capacity(payload.len()); + decoder.decode_all_to_vec(frame, &mut decoded).unwrap(); + assert_eq!(decoded.as_slice(), payload.as_slice()); + } + } + #[test] fn set_dictionary_from_bytes_matches_full_decode_byte_for_byte() { // The encoder-only dict parse (`decode_dict_for_encoding`, used by diff --git a/zstd/src/encoding/hc/mod.rs b/zstd/src/encoding/hc/mod.rs index 9eea1c351..572238d63 100644 --- a/zstd/src/encoding/hc/mod.rs +++ b/zstd/src/encoding/hc/mod.rs @@ -35,6 +35,7 @@ pub(crate) const MAX_HC_SEARCH_DEPTH: usize = 512; /// tables live on the shared /// [`super::match_table::storage::MatchTable`] that this matcher /// borrows when it runs. +#[derive(Clone)] pub(crate) struct HcMatcher { /// Lookahead depth (1 = lazy, 2 = lazy2). Donor parity: /// `params->cParams.strategy >= ZSTD_lazy2`. diff --git a/zstd/src/encoding/ldm/mod.rs b/zstd/src/encoding/ldm/mod.rs index 31fc69b53..879731767 100644 --- a/zstd/src/encoding/ldm/mod.rs +++ b/zstd/src/encoding/ldm/mod.rs @@ -95,6 +95,7 @@ const LDM_XXH64_SEED: u64 = 0; /// [`Self::clear`] only when starting a new frame (so the /// long-range history accumulated across blocks is preserved /// within a frame, mirroring donor's `ldmState_t` lifecycle). +#[derive(Clone)] pub(crate) struct LdmProducer { /// Parameter set this producer was built with. Used by the /// split walker (next commit) to honour `min_match_length` / diff --git a/zstd/src/encoding/ldm/table.rs b/zstd/src/encoding/ldm/table.rs index c94d8902b..d30babee7 100644 --- a/zstd/src/encoding/ldm/table.rs +++ b/zstd/src/encoding/ldm/table.rs @@ -91,6 +91,7 @@ const REBASE_GUARD_BAND: u32 = 1u32 << 30; /// [`LdmHashTable::insert_absolute`] / [`LdmHashTable::resolve`] /// helpers so absolute-position support extends past `u32::MAX` /// transparently — same pattern as `MatchTable`'s chain rebase. +#[derive(Clone)] pub(crate) struct LdmHashTable { entries: Vec, bucket_offsets: Vec, diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs index 72649b001..ffd986378 100644 --- a/zstd/src/encoding/match_generator.rs +++ b/zstd/src/encoding/match_generator.rs @@ -615,6 +615,7 @@ pub(crate) fn level_pre_split(level: CompressionLevel) -> Option { /// number of arms, but `storage.backend()` is now the canonical source /// of truth and dead variants are dropped when the active backend /// changes. +#[derive(Clone)] enum MatcherStorage { /// Donor `ZSTD_fast` family. Constructed by /// [`MatchGeneratorDriver::new`] as the initial variant and @@ -716,6 +717,16 @@ pub struct MatchGeneratorDriver { // committed-block path; consumed (reset to `None`) by the routed // call. Always `None` on the owned streaming path. borrowed_pending: Option<(usize, usize)>, + /// CDict-equivalent: snapshot of the post-prime matcher state taken + /// once after the first dictionary prime — the backend `storage` + /// (hash tables + dictionary history + offset history + window) plus + /// the driver-level `dictionary_retained_budget`, the only two pieces + /// `prime_with_dictionary` writes. Subsequent frames restore this + /// (a table memcpy) instead of re-hashing every dictionary position, + /// mirroring donor `ZSTD_compressBegin_usingCDict` copying the + /// precomputed `cdict->matchState`. Invalidated when the dictionary or + /// level changes (keyed by the captured `CompressionLevel`). + primed: Option<(MatcherStorage, usize, super::CompressionLevel)>, } impl MatchGeneratorDriver { @@ -798,6 +809,7 @@ impl MatchGeneratorDriver { dictionary_retained_budget: 0, source_size_hint: None, borrowed_pending: None, + primed: None, } } @@ -1453,6 +1465,31 @@ impl Matcher for MatchGeneratorDriver { } } + fn restore_primed_dictionary(&mut self, level: super::CompressionLevel) -> bool { + // Only the (storage, dictionary_retained_budget) pair is what + // `prime_with_dictionary` writes; restoring them reproduces the + // post-prime state exactly. Gated on the captured level so a driver + // reused at a different level (different backend / params) re-primes + // instead of restoring a mismatched table. + let (storage, budget) = match &self.primed { + Some((storage, budget, captured_level)) if *captured_level == level => { + (storage.clone(), *budget) + } + _ => return false, + }; + self.storage = storage; + self.dictionary_retained_budget = budget; + true + } + + fn capture_primed_dictionary(&mut self, level: super::CompressionLevel) { + self.primed = Some((self.storage.clone(), self.dictionary_retained_budget, level)); + } + + fn invalidate_primed_dictionary(&mut self) { + self.primed = None; + } + fn seed_dictionary_entropy( &mut self, huff: Option<&crate::huff0::huff0_encoder::HuffmanTable>, @@ -1757,6 +1794,7 @@ impl MatchGeneratorDriver { /// The discriminator lives next to `parse_mode` so `configure()` can /// promote between the two on a level change without touching the /// `MatchTable` storage. +#[derive(Clone)] pub(crate) enum HcBackend { /// Lazy / lazy2 modes — no per-frame backend state. Hc, @@ -1782,6 +1820,7 @@ impl HcBackend { } } +#[derive(Clone)] struct HcMatchGenerator { /// Shared match-finder storage (window, history, hash / chain / /// hash3 tables, dictionary-priming flags). Used identically by HC diff --git a/zstd/src/encoding/match_table/storage.rs b/zstd/src/encoding/match_table/storage.rs index b639087fd..6c585f438 100644 --- a/zstd/src/encoding/match_table/storage.rs +++ b/zstd/src/encoding/match_table/storage.rs @@ -214,6 +214,7 @@ pub(crate) const HC3_HASH_LOG: usize = 17; /// tables. Methods on this struct contain only logic that's identical /// between HC and BT modes — backend-specific table interpretation /// lives in the matcher modules. +#[derive(Clone)] pub(crate) struct MatchTable { pub(crate) max_window_size: usize, /// Per-chunk lengths of the live window, in add order. The bytes diff --git a/zstd/src/encoding/mod.rs b/zstd/src/encoding/mod.rs index 58bada471..2bef67acf 100644 --- a/zstd/src/encoding/mod.rs +++ b/zstd/src/encoding/mod.rs @@ -197,7 +197,7 @@ pub fn compress_slice_to_vec(source: &[u8], level: CompressionLevel) -> Vec /// The compression mode used impacts the speed of compression, /// and resulting compression ratios. Faster compression will result /// in worse compression ratios, and vice versa. -#[derive(Copy, Clone, Debug)] +#[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum CompressionLevel { /// This level does not compress the data at all, and simply wraps /// it in a Zstandard frame. @@ -344,6 +344,21 @@ pub trait Matcher { /// Prime matcher state with dictionary history before compressing the next frame. /// Default implementation is a no-op for custom matchers that do not support this. fn prime_with_dictionary(&mut self, _dict_content: &[u8], _offset_hist: [u32; 3]) {} + /// CDict-equivalent fast path for repeated frames sharing one dictionary. + /// Restore the matcher state captured by [`Self::capture_primed_dictionary`] + /// at the SAME level (a table copy) instead of re-running + /// [`Self::prime_with_dictionary`] (which re-hashes every dictionary + /// position). Returns `true` when a matching snapshot was restored; + /// `false` (the default) means the caller must prime then capture. + fn restore_primed_dictionary(&mut self, _level: CompressionLevel) -> bool { + false + } + /// Snapshot the post-prime matcher state for the given level so later + /// frames can [`Self::restore_primed_dictionary`] it. Default no-op. + fn capture_primed_dictionary(&mut self, _level: CompressionLevel) {} + /// Drop any captured prime snapshot (dictionary or level changed). + /// Default no-op. + fn invalidate_primed_dictionary(&mut self) {} /// Seed matcher cost model with dictionary entropy tables before the next frame. /// Default implementation is a no-op for custom matchers. fn seed_dictionary_entropy( diff --git a/zstd/src/encoding/row/mod.rs b/zstd/src/encoding/row/mod.rs index 459a712c5..b1aa00e27 100644 --- a/zstd/src/encoding/row/mod.rs +++ b/zstd/src/encoding/row/mod.rs @@ -335,6 +335,7 @@ unsafe fn row_tag_match_mask_neon(tags: &[u8], tag: u8) -> u64 { mask } +#[derive(Clone)] pub(crate) struct RowMatchGenerator { pub(crate) max_window_size: usize, /// Per-committed-block lengths of the live window, mirroring the diff --git a/zstd/src/encoding/simple/fast_kernel/hash_table.rs b/zstd/src/encoding/simple/fast_kernel/hash_table.rs index c3c1893bc..c94ac4fe9 100644 --- a/zstd/src/encoding/simple/fast_kernel/hash_table.rs +++ b/zstd/src/encoding/simple/fast_kernel/hash_table.rs @@ -53,6 +53,7 @@ const PRIME_8_BYTES: u64 = 0xCF1BBCDCB7A56463; /// initial prefix (where the `+= (ip0 == prefixStart)` adjustment at /// loop entry skips it) or is below `prefixStartIndex` and filtered by /// the in-range check. +#[derive(Clone)] pub(crate) struct FastHashTable { table: Vec, /// Donor `hash_log` — number of bits the hash output is reduced to. diff --git a/zstd/src/encoding/simple/fast_matcher.rs b/zstd/src/encoding/simple/fast_matcher.rs index fdc0a24e2..597d3fa02 100644 --- a/zstd/src/encoding/simple/fast_matcher.rs +++ b/zstd/src/encoding/simple/fast_matcher.rs @@ -137,6 +137,7 @@ const INITIAL_PREFIX_START_INDEX: u32 = 1; /// across blocks (cleared only on full `reset`). /// - `pending` holds the most recently `commit_space`'d block before /// `start_matching` appends it onto `history` and runs the kernel. +#[derive(Clone)] pub(crate) struct FastKernelMatcher { /// Concatenated input history: prior-block bytes followed by the /// most-recently-committed (still pending-matching) tail. From 32a4ebe87388d31692abbe75683a6b95300f3b1e Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Fri, 5 Jun 2026 23:48:12 +0300 Subject: [PATCH 05/33] Revert "perf(encode): cache primed dict matcher state (CDict-equivalent)" This reverts commit 5895680f7650eee5b786f95941beb0aa77318cb2. --- zstd/src/encoding/bt/mod.rs | 1 - zstd/src/encoding/dfast/mod.rs | 1 - zstd/src/encoding/frame_compressor.rs | 73 +------------------ zstd/src/encoding/hc/mod.rs | 1 - zstd/src/encoding/ldm/mod.rs | 1 - zstd/src/encoding/ldm/table.rs | 1 - zstd/src/encoding/match_generator.rs | 39 ---------- zstd/src/encoding/match_table/storage.rs | 1 - zstd/src/encoding/mod.rs | 17 +---- zstd/src/encoding/row/mod.rs | 1 - .../encoding/simple/fast_kernel/hash_table.rs | 1 - zstd/src/encoding/simple/fast_matcher.rs | 1 - 12 files changed, 3 insertions(+), 135 deletions(-) diff --git a/zstd/src/encoding/bt/mod.rs b/zstd/src/encoding/bt/mod.rs index b15b8b75a..099a4130d 100644 --- a/zstd/src/encoding/bt/mod.rs +++ b/zstd/src/encoding/bt/mod.rs @@ -36,7 +36,6 @@ pub(crate) const HC3_MAX_OFFSET: usize = 1 << 18; /// `BtUltra2` parse modes. Owns the cost model and the per-frame /// scratch arenas; the actual BT pointer-pair table lives on the /// shared [`super::match_table::storage::MatchTable`]. -#[derive(Clone)] pub(crate) struct BtMatcher { /// Donor `optStatePtr_t` — Huffman / FSE-derived literal and /// sequence-symbol cost tables that drive the optimal parser. diff --git a/zstd/src/encoding/dfast/mod.rs b/zstd/src/encoding/dfast/mod.rs index a9fe554fe..5932373c1 100644 --- a/zstd/src/encoding/dfast/mod.rs +++ b/zstd/src/encoding/dfast/mod.rs @@ -51,7 +51,6 @@ const HASH_READ_SIZE: usize = 8; /// rep emissions that upstream produces. const DFAST_REP_MIN_MATCH_LEN: usize = 4; -#[derive(Clone)] pub(crate) struct DfastMatchGenerator { pub(crate) max_window_size: usize, /// Per-block length queue. Previously held the raw input diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index e18474f3a..212ef091a 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -924,25 +924,9 @@ impl FrameCompressor { // This state drives sequence encoding, while matcher priming below updates // the match generator's internal repeat-offset history for match finding. self.state.offset_hist = dict.inner.offset_hist; - // CDict fast path (donor `ZSTD_compressBegin_usingCDict`): restore - // the precomputed primed matcher state (a table copy) instead of - // re-hashing every dictionary position into the match tables. The - // snapshot is captured on the first prime and reused while the - // dictionary + level stay the same; `restore` returns false when - // it must be (re)built. - if !self - .state + self.state .matcher - .restore_primed_dictionary(self.compression_level) - { - self.state.matcher.prime_with_dictionary( - dict.inner.dict_content.as_slice(), - dict.inner.offset_hist, - ); - self.state - .matcher - .capture_primed_dictionary(self.compression_level); - } + .prime_with_dictionary(dict.inner.dict_content.as_slice(), dict.inner.offset_hist); } if let Some(cache) = cached_entropy { self.state.last_huff_table.clone_from(&cache.huff); @@ -1502,10 +1486,6 @@ impl FrameCompressor { /// Remove the attached dictionary, returning it as an [`EncoderDictionary`]. pub fn clear_dictionary(&mut self) -> Option { self.dictionary_entropy_cache = None; - // Drop the CDict prime snapshot — it is keyed to the dictionary - // being removed and must not be restored against a different (or no) - // dictionary on the next frame. - self.state.matcher.invalidate_primed_dictionary(); self.dictionary.take() } @@ -1546,10 +1526,6 @@ impl FrameCompressor { .to_encoder_table() .map(|table| PreviousFseTable::Custom(Box::new(table))), }); - // A previously-captured CDict prime snapshot belongs to the OLD - // dictionary; drop it so the first frame with the new dictionary - // re-primes (and re-captures) instead of restoring stale tables. - self.state.matcher.invalidate_primed_dictionary(); Ok(self.dictionary.replace(enc)) } } @@ -2386,51 +2362,6 @@ mod tests { ); } - #[test] - fn dict_primed_matcher_snapshot_reused_across_frames_is_byte_identical() { - // CDict-equivalent: a compressor reused across frames with the same - // dictionary restores the primed matcher snapshot on frames 2..N - // (a table copy) instead of re-hashing the dictionary. The restored - // state must reproduce the first-frame (freshly-primed) output - // byte-for-byte, and every frame must round-trip through a - // dict-primed decoder. - let dict_raw = include_bytes!("../../dict_tests/dictionary"); - let payload = b"tenant=demo table=orders op=put key=1 value=aaaaabbbbbcccccdddddeeeee\n\ - tenant=demo table=orders op=put key=2 value=aaaaabbbbbcccccdddddeeeee\n"; - - let prepared = - super::EncoderDictionary::from_bytes(dict_raw).expect("dict bytes should parse"); - let dict_id = prepared.id(); - let mut compressor: FrameCompressor = - FrameCompressor::new(super::CompressionLevel::Fastest); - compressor - .set_encoder_dictionary(prepared) - .expect("prepared dictionary should attach"); - - // Frame 1 primes + captures the snapshot; frame 2 restores it. - let frame1 = compressor.compress_independent_frame(payload.as_slice()); - let frame2 = compressor.compress_independent_frame(payload.as_slice()); - assert_eq!( - frame1, frame2, - "restored prime snapshot must reproduce the freshly-primed frame byte-for-byte" - ); - - // Both frames advertise the dict id and round-trip through a - // dict-primed decoder. - for frame in [&frame1, &frame2] { - let (hdr, _) = - crate::decoding::frame::read_frame_header(frame.as_slice()).expect("frame header"); - assert_eq!(hdr.dictionary_id(), Some(dict_id)); - let mut decoder = FrameDecoder::new(); - decoder - .add_dict(crate::decoding::Dictionary::decode_dict(dict_raw).unwrap()) - .unwrap(); - let mut decoded = Vec::with_capacity(payload.len()); - decoder.decode_all_to_vec(frame, &mut decoded).unwrap(); - assert_eq!(decoded.as_slice(), payload.as_slice()); - } - } - #[test] fn set_dictionary_from_bytes_matches_full_decode_byte_for_byte() { // The encoder-only dict parse (`decode_dict_for_encoding`, used by diff --git a/zstd/src/encoding/hc/mod.rs b/zstd/src/encoding/hc/mod.rs index 572238d63..9eea1c351 100644 --- a/zstd/src/encoding/hc/mod.rs +++ b/zstd/src/encoding/hc/mod.rs @@ -35,7 +35,6 @@ pub(crate) const MAX_HC_SEARCH_DEPTH: usize = 512; /// tables live on the shared /// [`super::match_table::storage::MatchTable`] that this matcher /// borrows when it runs. -#[derive(Clone)] pub(crate) struct HcMatcher { /// Lookahead depth (1 = lazy, 2 = lazy2). Donor parity: /// `params->cParams.strategy >= ZSTD_lazy2`. diff --git a/zstd/src/encoding/ldm/mod.rs b/zstd/src/encoding/ldm/mod.rs index 879731767..31fc69b53 100644 --- a/zstd/src/encoding/ldm/mod.rs +++ b/zstd/src/encoding/ldm/mod.rs @@ -95,7 +95,6 @@ const LDM_XXH64_SEED: u64 = 0; /// [`Self::clear`] only when starting a new frame (so the /// long-range history accumulated across blocks is preserved /// within a frame, mirroring donor's `ldmState_t` lifecycle). -#[derive(Clone)] pub(crate) struct LdmProducer { /// Parameter set this producer was built with. Used by the /// split walker (next commit) to honour `min_match_length` / diff --git a/zstd/src/encoding/ldm/table.rs b/zstd/src/encoding/ldm/table.rs index d30babee7..c94d8902b 100644 --- a/zstd/src/encoding/ldm/table.rs +++ b/zstd/src/encoding/ldm/table.rs @@ -91,7 +91,6 @@ const REBASE_GUARD_BAND: u32 = 1u32 << 30; /// [`LdmHashTable::insert_absolute`] / [`LdmHashTable::resolve`] /// helpers so absolute-position support extends past `u32::MAX` /// transparently — same pattern as `MatchTable`'s chain rebase. -#[derive(Clone)] pub(crate) struct LdmHashTable { entries: Vec, bucket_offsets: Vec, diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs index ffd986378..72649b001 100644 --- a/zstd/src/encoding/match_generator.rs +++ b/zstd/src/encoding/match_generator.rs @@ -615,7 +615,6 @@ pub(crate) fn level_pre_split(level: CompressionLevel) -> Option { /// number of arms, but `storage.backend()` is now the canonical source /// of truth and dead variants are dropped when the active backend /// changes. -#[derive(Clone)] enum MatcherStorage { /// Donor `ZSTD_fast` family. Constructed by /// [`MatchGeneratorDriver::new`] as the initial variant and @@ -717,16 +716,6 @@ pub struct MatchGeneratorDriver { // committed-block path; consumed (reset to `None`) by the routed // call. Always `None` on the owned streaming path. borrowed_pending: Option<(usize, usize)>, - /// CDict-equivalent: snapshot of the post-prime matcher state taken - /// once after the first dictionary prime — the backend `storage` - /// (hash tables + dictionary history + offset history + window) plus - /// the driver-level `dictionary_retained_budget`, the only two pieces - /// `prime_with_dictionary` writes. Subsequent frames restore this - /// (a table memcpy) instead of re-hashing every dictionary position, - /// mirroring donor `ZSTD_compressBegin_usingCDict` copying the - /// precomputed `cdict->matchState`. Invalidated when the dictionary or - /// level changes (keyed by the captured `CompressionLevel`). - primed: Option<(MatcherStorage, usize, super::CompressionLevel)>, } impl MatchGeneratorDriver { @@ -809,7 +798,6 @@ impl MatchGeneratorDriver { dictionary_retained_budget: 0, source_size_hint: None, borrowed_pending: None, - primed: None, } } @@ -1465,31 +1453,6 @@ impl Matcher for MatchGeneratorDriver { } } - fn restore_primed_dictionary(&mut self, level: super::CompressionLevel) -> bool { - // Only the (storage, dictionary_retained_budget) pair is what - // `prime_with_dictionary` writes; restoring them reproduces the - // post-prime state exactly. Gated on the captured level so a driver - // reused at a different level (different backend / params) re-primes - // instead of restoring a mismatched table. - let (storage, budget) = match &self.primed { - Some((storage, budget, captured_level)) if *captured_level == level => { - (storage.clone(), *budget) - } - _ => return false, - }; - self.storage = storage; - self.dictionary_retained_budget = budget; - true - } - - fn capture_primed_dictionary(&mut self, level: super::CompressionLevel) { - self.primed = Some((self.storage.clone(), self.dictionary_retained_budget, level)); - } - - fn invalidate_primed_dictionary(&mut self) { - self.primed = None; - } - fn seed_dictionary_entropy( &mut self, huff: Option<&crate::huff0::huff0_encoder::HuffmanTable>, @@ -1794,7 +1757,6 @@ impl MatchGeneratorDriver { /// The discriminator lives next to `parse_mode` so `configure()` can /// promote between the two on a level change without touching the /// `MatchTable` storage. -#[derive(Clone)] pub(crate) enum HcBackend { /// Lazy / lazy2 modes — no per-frame backend state. Hc, @@ -1820,7 +1782,6 @@ impl HcBackend { } } -#[derive(Clone)] struct HcMatchGenerator { /// Shared match-finder storage (window, history, hash / chain / /// hash3 tables, dictionary-priming flags). Used identically by HC diff --git a/zstd/src/encoding/match_table/storage.rs b/zstd/src/encoding/match_table/storage.rs index 6c585f438..b639087fd 100644 --- a/zstd/src/encoding/match_table/storage.rs +++ b/zstd/src/encoding/match_table/storage.rs @@ -214,7 +214,6 @@ pub(crate) const HC3_HASH_LOG: usize = 17; /// tables. Methods on this struct contain only logic that's identical /// between HC and BT modes — backend-specific table interpretation /// lives in the matcher modules. -#[derive(Clone)] pub(crate) struct MatchTable { pub(crate) max_window_size: usize, /// Per-chunk lengths of the live window, in add order. The bytes diff --git a/zstd/src/encoding/mod.rs b/zstd/src/encoding/mod.rs index 2bef67acf..58bada471 100644 --- a/zstd/src/encoding/mod.rs +++ b/zstd/src/encoding/mod.rs @@ -197,7 +197,7 @@ pub fn compress_slice_to_vec(source: &[u8], level: CompressionLevel) -> Vec /// The compression mode used impacts the speed of compression, /// and resulting compression ratios. Faster compression will result /// in worse compression ratios, and vice versa. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[derive(Copy, Clone, Debug)] pub enum CompressionLevel { /// This level does not compress the data at all, and simply wraps /// it in a Zstandard frame. @@ -344,21 +344,6 @@ pub trait Matcher { /// Prime matcher state with dictionary history before compressing the next frame. /// Default implementation is a no-op for custom matchers that do not support this. fn prime_with_dictionary(&mut self, _dict_content: &[u8], _offset_hist: [u32; 3]) {} - /// CDict-equivalent fast path for repeated frames sharing one dictionary. - /// Restore the matcher state captured by [`Self::capture_primed_dictionary`] - /// at the SAME level (a table copy) instead of re-running - /// [`Self::prime_with_dictionary`] (which re-hashes every dictionary - /// position). Returns `true` when a matching snapshot was restored; - /// `false` (the default) means the caller must prime then capture. - fn restore_primed_dictionary(&mut self, _level: CompressionLevel) -> bool { - false - } - /// Snapshot the post-prime matcher state for the given level so later - /// frames can [`Self::restore_primed_dictionary`] it. Default no-op. - fn capture_primed_dictionary(&mut self, _level: CompressionLevel) {} - /// Drop any captured prime snapshot (dictionary or level changed). - /// Default no-op. - fn invalidate_primed_dictionary(&mut self) {} /// Seed matcher cost model with dictionary entropy tables before the next frame. /// Default implementation is a no-op for custom matchers. fn seed_dictionary_entropy( diff --git a/zstd/src/encoding/row/mod.rs b/zstd/src/encoding/row/mod.rs index b1aa00e27..459a712c5 100644 --- a/zstd/src/encoding/row/mod.rs +++ b/zstd/src/encoding/row/mod.rs @@ -335,7 +335,6 @@ unsafe fn row_tag_match_mask_neon(tags: &[u8], tag: u8) -> u64 { mask } -#[derive(Clone)] pub(crate) struct RowMatchGenerator { pub(crate) max_window_size: usize, /// Per-committed-block lengths of the live window, mirroring the diff --git a/zstd/src/encoding/simple/fast_kernel/hash_table.rs b/zstd/src/encoding/simple/fast_kernel/hash_table.rs index c94ac4fe9..c3c1893bc 100644 --- a/zstd/src/encoding/simple/fast_kernel/hash_table.rs +++ b/zstd/src/encoding/simple/fast_kernel/hash_table.rs @@ -53,7 +53,6 @@ const PRIME_8_BYTES: u64 = 0xCF1BBCDCB7A56463; /// initial prefix (where the `+= (ip0 == prefixStart)` adjustment at /// loop entry skips it) or is below `prefixStartIndex` and filtered by /// the in-range check. -#[derive(Clone)] pub(crate) struct FastHashTable { table: Vec, /// Donor `hash_log` — number of bits the hash output is reduced to. diff --git a/zstd/src/encoding/simple/fast_matcher.rs b/zstd/src/encoding/simple/fast_matcher.rs index 597d3fa02..fdc0a24e2 100644 --- a/zstd/src/encoding/simple/fast_matcher.rs +++ b/zstd/src/encoding/simple/fast_matcher.rs @@ -137,7 +137,6 @@ const INITIAL_PREFIX_START_INDEX: u32 = 1; /// across blocks (cleared only on full `reset`). /// - `pending` holds the most recently `commit_space`'d block before /// `start_matching` appends it onto `history` and runs the kernel. -#[derive(Clone)] pub(crate) struct FastKernelMatcher { /// Concatenated input history: prior-block bytes followed by the /// most-recently-committed (still pending-matching) tail. From 7f04de0d96e59c72134001026f7872106750d6c1 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Fri, 5 Jun 2026 23:49:54 +0300 Subject: [PATCH 06/33] Reapply "perf(encode): cache primed dict matcher state (CDict-equivalent)" This reverts commit a0943b2d588400041ccb9ae4cec5677203d92cf7. --- zstd/src/encoding/bt/mod.rs | 1 + zstd/src/encoding/dfast/mod.rs | 1 + zstd/src/encoding/frame_compressor.rs | 73 ++++++++++++++++++- zstd/src/encoding/hc/mod.rs | 1 + zstd/src/encoding/ldm/mod.rs | 1 + zstd/src/encoding/ldm/table.rs | 1 + zstd/src/encoding/match_generator.rs | 39 ++++++++++ zstd/src/encoding/match_table/storage.rs | 1 + zstd/src/encoding/mod.rs | 17 ++++- zstd/src/encoding/row/mod.rs | 1 + .../encoding/simple/fast_kernel/hash_table.rs | 1 + zstd/src/encoding/simple/fast_matcher.rs | 1 + 12 files changed, 135 insertions(+), 3 deletions(-) diff --git a/zstd/src/encoding/bt/mod.rs b/zstd/src/encoding/bt/mod.rs index 099a4130d..b15b8b75a 100644 --- a/zstd/src/encoding/bt/mod.rs +++ b/zstd/src/encoding/bt/mod.rs @@ -36,6 +36,7 @@ pub(crate) const HC3_MAX_OFFSET: usize = 1 << 18; /// `BtUltra2` parse modes. Owns the cost model and the per-frame /// scratch arenas; the actual BT pointer-pair table lives on the /// shared [`super::match_table::storage::MatchTable`]. +#[derive(Clone)] pub(crate) struct BtMatcher { /// Donor `optStatePtr_t` — Huffman / FSE-derived literal and /// sequence-symbol cost tables that drive the optimal parser. diff --git a/zstd/src/encoding/dfast/mod.rs b/zstd/src/encoding/dfast/mod.rs index 5932373c1..a9fe554fe 100644 --- a/zstd/src/encoding/dfast/mod.rs +++ b/zstd/src/encoding/dfast/mod.rs @@ -51,6 +51,7 @@ const HASH_READ_SIZE: usize = 8; /// rep emissions that upstream produces. const DFAST_REP_MIN_MATCH_LEN: usize = 4; +#[derive(Clone)] pub(crate) struct DfastMatchGenerator { pub(crate) max_window_size: usize, /// Per-block length queue. Previously held the raw input diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index 212ef091a..e18474f3a 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -924,9 +924,25 @@ impl FrameCompressor { // This state drives sequence encoding, while matcher priming below updates // the match generator's internal repeat-offset history for match finding. self.state.offset_hist = dict.inner.offset_hist; - self.state + // CDict fast path (donor `ZSTD_compressBegin_usingCDict`): restore + // the precomputed primed matcher state (a table copy) instead of + // re-hashing every dictionary position into the match tables. The + // snapshot is captured on the first prime and reused while the + // dictionary + level stay the same; `restore` returns false when + // it must be (re)built. + if !self + .state .matcher - .prime_with_dictionary(dict.inner.dict_content.as_slice(), dict.inner.offset_hist); + .restore_primed_dictionary(self.compression_level) + { + self.state.matcher.prime_with_dictionary( + dict.inner.dict_content.as_slice(), + dict.inner.offset_hist, + ); + self.state + .matcher + .capture_primed_dictionary(self.compression_level); + } } if let Some(cache) = cached_entropy { self.state.last_huff_table.clone_from(&cache.huff); @@ -1486,6 +1502,10 @@ impl FrameCompressor { /// Remove the attached dictionary, returning it as an [`EncoderDictionary`]. pub fn clear_dictionary(&mut self) -> Option { self.dictionary_entropy_cache = None; + // Drop the CDict prime snapshot — it is keyed to the dictionary + // being removed and must not be restored against a different (or no) + // dictionary on the next frame. + self.state.matcher.invalidate_primed_dictionary(); self.dictionary.take() } @@ -1526,6 +1546,10 @@ impl FrameCompressor { .to_encoder_table() .map(|table| PreviousFseTable::Custom(Box::new(table))), }); + // A previously-captured CDict prime snapshot belongs to the OLD + // dictionary; drop it so the first frame with the new dictionary + // re-primes (and re-captures) instead of restoring stale tables. + self.state.matcher.invalidate_primed_dictionary(); Ok(self.dictionary.replace(enc)) } } @@ -2362,6 +2386,51 @@ mod tests { ); } + #[test] + fn dict_primed_matcher_snapshot_reused_across_frames_is_byte_identical() { + // CDict-equivalent: a compressor reused across frames with the same + // dictionary restores the primed matcher snapshot on frames 2..N + // (a table copy) instead of re-hashing the dictionary. The restored + // state must reproduce the first-frame (freshly-primed) output + // byte-for-byte, and every frame must round-trip through a + // dict-primed decoder. + let dict_raw = include_bytes!("../../dict_tests/dictionary"); + let payload = b"tenant=demo table=orders op=put key=1 value=aaaaabbbbbcccccdddddeeeee\n\ + tenant=demo table=orders op=put key=2 value=aaaaabbbbbcccccdddddeeeee\n"; + + let prepared = + super::EncoderDictionary::from_bytes(dict_raw).expect("dict bytes should parse"); + let dict_id = prepared.id(); + let mut compressor: FrameCompressor = + FrameCompressor::new(super::CompressionLevel::Fastest); + compressor + .set_encoder_dictionary(prepared) + .expect("prepared dictionary should attach"); + + // Frame 1 primes + captures the snapshot; frame 2 restores it. + let frame1 = compressor.compress_independent_frame(payload.as_slice()); + let frame2 = compressor.compress_independent_frame(payload.as_slice()); + assert_eq!( + frame1, frame2, + "restored prime snapshot must reproduce the freshly-primed frame byte-for-byte" + ); + + // Both frames advertise the dict id and round-trip through a + // dict-primed decoder. + for frame in [&frame1, &frame2] { + let (hdr, _) = + crate::decoding::frame::read_frame_header(frame.as_slice()).expect("frame header"); + assert_eq!(hdr.dictionary_id(), Some(dict_id)); + let mut decoder = FrameDecoder::new(); + decoder + .add_dict(crate::decoding::Dictionary::decode_dict(dict_raw).unwrap()) + .unwrap(); + let mut decoded = Vec::with_capacity(payload.len()); + decoder.decode_all_to_vec(frame, &mut decoded).unwrap(); + assert_eq!(decoded.as_slice(), payload.as_slice()); + } + } + #[test] fn set_dictionary_from_bytes_matches_full_decode_byte_for_byte() { // The encoder-only dict parse (`decode_dict_for_encoding`, used by diff --git a/zstd/src/encoding/hc/mod.rs b/zstd/src/encoding/hc/mod.rs index 9eea1c351..572238d63 100644 --- a/zstd/src/encoding/hc/mod.rs +++ b/zstd/src/encoding/hc/mod.rs @@ -35,6 +35,7 @@ pub(crate) const MAX_HC_SEARCH_DEPTH: usize = 512; /// tables live on the shared /// [`super::match_table::storage::MatchTable`] that this matcher /// borrows when it runs. +#[derive(Clone)] pub(crate) struct HcMatcher { /// Lookahead depth (1 = lazy, 2 = lazy2). Donor parity: /// `params->cParams.strategy >= ZSTD_lazy2`. diff --git a/zstd/src/encoding/ldm/mod.rs b/zstd/src/encoding/ldm/mod.rs index 31fc69b53..879731767 100644 --- a/zstd/src/encoding/ldm/mod.rs +++ b/zstd/src/encoding/ldm/mod.rs @@ -95,6 +95,7 @@ const LDM_XXH64_SEED: u64 = 0; /// [`Self::clear`] only when starting a new frame (so the /// long-range history accumulated across blocks is preserved /// within a frame, mirroring donor's `ldmState_t` lifecycle). +#[derive(Clone)] pub(crate) struct LdmProducer { /// Parameter set this producer was built with. Used by the /// split walker (next commit) to honour `min_match_length` / diff --git a/zstd/src/encoding/ldm/table.rs b/zstd/src/encoding/ldm/table.rs index c94d8902b..d30babee7 100644 --- a/zstd/src/encoding/ldm/table.rs +++ b/zstd/src/encoding/ldm/table.rs @@ -91,6 +91,7 @@ const REBASE_GUARD_BAND: u32 = 1u32 << 30; /// [`LdmHashTable::insert_absolute`] / [`LdmHashTable::resolve`] /// helpers so absolute-position support extends past `u32::MAX` /// transparently — same pattern as `MatchTable`'s chain rebase. +#[derive(Clone)] pub(crate) struct LdmHashTable { entries: Vec, bucket_offsets: Vec, diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs index 72649b001..ffd986378 100644 --- a/zstd/src/encoding/match_generator.rs +++ b/zstd/src/encoding/match_generator.rs @@ -615,6 +615,7 @@ pub(crate) fn level_pre_split(level: CompressionLevel) -> Option { /// number of arms, but `storage.backend()` is now the canonical source /// of truth and dead variants are dropped when the active backend /// changes. +#[derive(Clone)] enum MatcherStorage { /// Donor `ZSTD_fast` family. Constructed by /// [`MatchGeneratorDriver::new`] as the initial variant and @@ -716,6 +717,16 @@ pub struct MatchGeneratorDriver { // committed-block path; consumed (reset to `None`) by the routed // call. Always `None` on the owned streaming path. borrowed_pending: Option<(usize, usize)>, + /// CDict-equivalent: snapshot of the post-prime matcher state taken + /// once after the first dictionary prime — the backend `storage` + /// (hash tables + dictionary history + offset history + window) plus + /// the driver-level `dictionary_retained_budget`, the only two pieces + /// `prime_with_dictionary` writes. Subsequent frames restore this + /// (a table memcpy) instead of re-hashing every dictionary position, + /// mirroring donor `ZSTD_compressBegin_usingCDict` copying the + /// precomputed `cdict->matchState`. Invalidated when the dictionary or + /// level changes (keyed by the captured `CompressionLevel`). + primed: Option<(MatcherStorage, usize, super::CompressionLevel)>, } impl MatchGeneratorDriver { @@ -798,6 +809,7 @@ impl MatchGeneratorDriver { dictionary_retained_budget: 0, source_size_hint: None, borrowed_pending: None, + primed: None, } } @@ -1453,6 +1465,31 @@ impl Matcher for MatchGeneratorDriver { } } + fn restore_primed_dictionary(&mut self, level: super::CompressionLevel) -> bool { + // Only the (storage, dictionary_retained_budget) pair is what + // `prime_with_dictionary` writes; restoring them reproduces the + // post-prime state exactly. Gated on the captured level so a driver + // reused at a different level (different backend / params) re-primes + // instead of restoring a mismatched table. + let (storage, budget) = match &self.primed { + Some((storage, budget, captured_level)) if *captured_level == level => { + (storage.clone(), *budget) + } + _ => return false, + }; + self.storage = storage; + self.dictionary_retained_budget = budget; + true + } + + fn capture_primed_dictionary(&mut self, level: super::CompressionLevel) { + self.primed = Some((self.storage.clone(), self.dictionary_retained_budget, level)); + } + + fn invalidate_primed_dictionary(&mut self) { + self.primed = None; + } + fn seed_dictionary_entropy( &mut self, huff: Option<&crate::huff0::huff0_encoder::HuffmanTable>, @@ -1757,6 +1794,7 @@ impl MatchGeneratorDriver { /// The discriminator lives next to `parse_mode` so `configure()` can /// promote between the two on a level change without touching the /// `MatchTable` storage. +#[derive(Clone)] pub(crate) enum HcBackend { /// Lazy / lazy2 modes — no per-frame backend state. Hc, @@ -1782,6 +1820,7 @@ impl HcBackend { } } +#[derive(Clone)] struct HcMatchGenerator { /// Shared match-finder storage (window, history, hash / chain / /// hash3 tables, dictionary-priming flags). Used identically by HC diff --git a/zstd/src/encoding/match_table/storage.rs b/zstd/src/encoding/match_table/storage.rs index b639087fd..6c585f438 100644 --- a/zstd/src/encoding/match_table/storage.rs +++ b/zstd/src/encoding/match_table/storage.rs @@ -214,6 +214,7 @@ pub(crate) const HC3_HASH_LOG: usize = 17; /// tables. Methods on this struct contain only logic that's identical /// between HC and BT modes — backend-specific table interpretation /// lives in the matcher modules. +#[derive(Clone)] pub(crate) struct MatchTable { pub(crate) max_window_size: usize, /// Per-chunk lengths of the live window, in add order. The bytes diff --git a/zstd/src/encoding/mod.rs b/zstd/src/encoding/mod.rs index 58bada471..2bef67acf 100644 --- a/zstd/src/encoding/mod.rs +++ b/zstd/src/encoding/mod.rs @@ -197,7 +197,7 @@ pub fn compress_slice_to_vec(source: &[u8], level: CompressionLevel) -> Vec /// The compression mode used impacts the speed of compression, /// and resulting compression ratios. Faster compression will result /// in worse compression ratios, and vice versa. -#[derive(Copy, Clone, Debug)] +#[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum CompressionLevel { /// This level does not compress the data at all, and simply wraps /// it in a Zstandard frame. @@ -344,6 +344,21 @@ pub trait Matcher { /// Prime matcher state with dictionary history before compressing the next frame. /// Default implementation is a no-op for custom matchers that do not support this. fn prime_with_dictionary(&mut self, _dict_content: &[u8], _offset_hist: [u32; 3]) {} + /// CDict-equivalent fast path for repeated frames sharing one dictionary. + /// Restore the matcher state captured by [`Self::capture_primed_dictionary`] + /// at the SAME level (a table copy) instead of re-running + /// [`Self::prime_with_dictionary`] (which re-hashes every dictionary + /// position). Returns `true` when a matching snapshot was restored; + /// `false` (the default) means the caller must prime then capture. + fn restore_primed_dictionary(&mut self, _level: CompressionLevel) -> bool { + false + } + /// Snapshot the post-prime matcher state for the given level so later + /// frames can [`Self::restore_primed_dictionary`] it. Default no-op. + fn capture_primed_dictionary(&mut self, _level: CompressionLevel) {} + /// Drop any captured prime snapshot (dictionary or level changed). + /// Default no-op. + fn invalidate_primed_dictionary(&mut self) {} /// Seed matcher cost model with dictionary entropy tables before the next frame. /// Default implementation is a no-op for custom matchers. fn seed_dictionary_entropy( diff --git a/zstd/src/encoding/row/mod.rs b/zstd/src/encoding/row/mod.rs index 459a712c5..b1aa00e27 100644 --- a/zstd/src/encoding/row/mod.rs +++ b/zstd/src/encoding/row/mod.rs @@ -335,6 +335,7 @@ unsafe fn row_tag_match_mask_neon(tags: &[u8], tag: u8) -> u64 { mask } +#[derive(Clone)] pub(crate) struct RowMatchGenerator { pub(crate) max_window_size: usize, /// Per-committed-block lengths of the live window, mirroring the diff --git a/zstd/src/encoding/simple/fast_kernel/hash_table.rs b/zstd/src/encoding/simple/fast_kernel/hash_table.rs index c3c1893bc..c94ac4fe9 100644 --- a/zstd/src/encoding/simple/fast_kernel/hash_table.rs +++ b/zstd/src/encoding/simple/fast_kernel/hash_table.rs @@ -53,6 +53,7 @@ const PRIME_8_BYTES: u64 = 0xCF1BBCDCB7A56463; /// initial prefix (where the `+= (ip0 == prefixStart)` adjustment at /// loop entry skips it) or is below `prefixStartIndex` and filtered by /// the in-range check. +#[derive(Clone)] pub(crate) struct FastHashTable { table: Vec, /// Donor `hash_log` — number of bits the hash output is reduced to. diff --git a/zstd/src/encoding/simple/fast_matcher.rs b/zstd/src/encoding/simple/fast_matcher.rs index fdc0a24e2..597d3fa02 100644 --- a/zstd/src/encoding/simple/fast_matcher.rs +++ b/zstd/src/encoding/simple/fast_matcher.rs @@ -137,6 +137,7 @@ const INITIAL_PREFIX_START_INDEX: u32 = 1; /// across blocks (cleared only on full `reset`). /// - `pending` holds the most recently `commit_space`'d block before /// `start_matching` appends it onto `history` and runs the kernel. +#[derive(Clone)] pub(crate) struct FastKernelMatcher { /// Concatenated input history: prior-block bytes followed by the /// most-recently-committed (still pending-matching) tail. From 3a2a5ebf61f20fbc99b795219efe80801b7381fb Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 6 Jun 2026 00:00:57 +0300 Subject: [PATCH 07/33] perf(encode): gate dict prime-snapshot copy to large sources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Donor ZSTD_shouldAttachDict only COPIES a precomputed dictionary table into the working context when the source exceeds a per-strategy cutoff (fast 8K, dfast 16K, greedy/lazy/btopt 32K, btultra/btultra2 8K); at or below it the donor ATTACHES the dictionary tables by reference with no per-frame table touch. The prime snapshot is a whole-table copy, so applying it unconditionally regressed small-source dict frames (the copy costs more than the sparse re-prime — which is exactly why the donor attaches by reference there). Gate restore/capture to sources above the cutoff so the snapshot is used only in the donor COPY regime; small/unknown sources keep re-priming until an attach-by-reference path lands. --- zstd/src/encoding/frame_compressor.rs | 61 ++++++++++++++++++++------- 1 file changed, 45 insertions(+), 16 deletions(-) diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index e18474f3a..4498572a6 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -924,24 +924,45 @@ impl FrameCompressor { // This state drives sequence encoding, while matcher priming below updates // the match generator's internal repeat-offset history for match finding. self.state.offset_hist = dict.inner.offset_hist; - // CDict fast path (donor `ZSTD_compressBegin_usingCDict`): restore - // the precomputed primed matcher state (a table copy) instead of - // re-hashing every dictionary position into the match tables. The - // snapshot is captured on the first prime and reused while the - // dictionary + level stay the same; `restore` returns false when - // it must be (re)built. - if !self - .state - .matcher - .restore_primed_dictionary(self.compression_level) - { + // Donor `ZSTD_shouldAttachDict` (`zstd_compress.c`): a + // precomputed-dictionary table is COPIED into the working context + // only when the source is larger than a per-strategy cutoff; at or + // below it (and for unknown size) the donor ATTACHES the dictionary + // tables by reference (no per-frame table touch at all). We don't + // have an attach-by-reference path yet, so: + // - large source (> cutoff): reuse the captured prime snapshot + // (a table copy) instead of re-hashing the dictionary — the + // donor COPY regime, where the copy is cheaper than re-priming; + // - small / unknown source: re-prime (the snapshot copy of the + // whole table would cost MORE than the sparse re-prime here, + // which is exactly why the donor attaches by reference instead). + // `attachDictSizeCutoffs` per strategy: fast 8K, dfast 16K, + // greedy/lazy/btopt 32K, btultra/btultra2 8K. + let cutoff = match self.state.strategy_tag { + crate::encoding::strategy::StrategyTag::Fast + | crate::encoding::strategy::StrategyTag::BtUltra + | crate::encoding::strategy::StrategyTag::BtUltra2 => 8 * 1024, + crate::encoding::strategy::StrategyTag::Dfast => 16 * 1024, + crate::encoding::strategy::StrategyTag::Greedy + | crate::encoding::strategy::StrategyTag::Lazy + | crate::encoding::strategy::StrategyTag::BtOpt => 32 * 1024, + }; + let prefer_copy_snapshot = initial_size_hint.is_some_and(|s| s as usize > cutoff); + let restored = prefer_copy_snapshot + && self + .state + .matcher + .restore_primed_dictionary(self.compression_level); + if !restored { self.state.matcher.prime_with_dictionary( dict.inner.dict_content.as_slice(), dict.inner.offset_hist, ); - self.state - .matcher - .capture_primed_dictionary(self.compression_level); + if prefer_copy_snapshot { + self.state + .matcher + .capture_primed_dictionary(self.compression_level); + } } } if let Some(cache) = cached_entropy { @@ -2395,8 +2416,16 @@ mod tests { // byte-for-byte, and every frame must round-trip through a // dict-primed decoder. let dict_raw = include_bytes!("../../dict_tests/dictionary"); - let payload = b"tenant=demo table=orders op=put key=1 value=aaaaabbbbbcccccdddddeeeee\n\ - tenant=demo table=orders op=put key=2 value=aaaaabbbbbcccccdddddeeeee\n"; + // Source must exceed the Fast strategy's 8 KiB attach cutoff so the + // copy-snapshot (restore) path is taken on frame 2 — at or below the + // cutoff the donor attaches by reference and we fall back to re-prime, + // which would not exercise restore. + let mut payload = Vec::new(); + while payload.len() < 16 * 1024 { + payload.extend_from_slice( + b"tenant=demo table=orders op=put key=1 value=aaaaabbbbbcccccdddddeeeee\n", + ); + } let prepared = super::EncoderDictionary::from_bytes(dict_raw).expect("dict bytes should parse"); From 677b163a36c2e0ec2900142d7112a9d0ed557a4e Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 6 Jun 2026 00:23:46 +0300 Subject: [PATCH 08/33] perf(encode): size block-read buffer to source hint, not always 128 KiB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit run_owned_block_loop zero-filled a full MAX_BLOCK_SIZE (128 KiB) buffer via resize(_, 0) every block just to read the input into it and truncate back down — a small frame paid a 128 KiB memset per block to read a few KiB (37% of steady-state small dict-compress was memset, the largest single cost). Bound the initial fill by the source-size hint (exact for the slice entry points) and grow toward block_capacity only if the hint under-counts, so inexact / unknown (None -> full capacity) hints stay correct. Helps every small-frame compress, dict or not. --- zstd/src/encoding/frame_compressor.rs | 33 ++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index 4498572a6..22eab3f21 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -1067,15 +1067,42 @@ impl FrameCompressor { uncompressed_data.extend_from_slice(&pending_input); let mut filled = pending_input.len(); pending_input.clear(); - if uncompressed_data.len() < block_capacity { - uncompressed_data.resize(block_capacity, 0); + // Size the read buffer to the bytes this block actually expects + // rather than always zero-filling a full MAX_BLOCK_SIZE: a small + // frame otherwise pays a 128 KiB `resize(_, 0)` memset per block + // just to read a few KiB (the zero-fill past `filled` is then + // truncated away). Bound the initial fill by the source-size hint + // (exact for the slice entry points), and grow toward + // `block_capacity` only if the hint under-counted, so an inexact / + // unknown (`None` → full `block_capacity`) hint stays correct. + let initial_target = initial_size_hint + .map(|h| { + let remaining = (h as usize).saturating_sub(total_uncompressed as usize); + filled + .saturating_add(remaining) + .clamp(filled.max(1), block_capacity) + }) + .unwrap_or(block_capacity); + if uncompressed_data.len() < initial_target { + uncompressed_data.resize(initial_target, 0); } 'read_loop: loop { if reached_eof || filled == block_capacity { break 'read_loop; } + if filled == uncompressed_data.len() { + // Hint under-counted the block; grow toward block_capacity + // (doubling, capped) so reading continues without paying a + // full-buffer zero up front. + let grow_to = uncompressed_data + .len() + .saturating_mul(2) + .clamp(filled + 1, block_capacity); + uncompressed_data.resize(grow_to, 0); + } + let read_end = uncompressed_data.len(); let new_bytes = source - .read(&mut uncompressed_data[filled..block_capacity]) + .read(&mut uncompressed_data[filled..read_end]) .unwrap(); if new_bytes == 0 { reached_eof = true; From 9bcb13be473d33d9022660e719e8f66c4b1afe69 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 6 Jun 2026 00:29:08 +0300 Subject: [PATCH 09/33] refactor(encode): bound block-read sizing without saturating arithmetic Replace the saturating_sub/add/mul in the block-read buffer sizing with plain checked-by-construction arithmetic: filled <= block_capacity holds on every path (the read only targets [filled..len] with len <= block_capacity, and a carried pending_input is a sub-block split_off), so block_capacity - filled never underflows; remaining is pinned to block_capacity before the usize cast and the doubling stays within usize. Behaviour identical; no saturating masking. --- zstd/src/encoding/frame_compressor.rs | 38 ++++++++++++++++----------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index 22eab3f21..c4795fb85 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -1073,16 +1073,24 @@ impl FrameCompressor { // just to read a few KiB (the zero-fill past `filled` is then // truncated away). Bound the initial fill by the source-size hint // (exact for the slice entry points), and grow toward - // `block_capacity` only if the hint under-counted, so an inexact / - // unknown (`None` → full `block_capacity`) hint stays correct. - let initial_target = initial_size_hint - .map(|h| { - let remaining = (h as usize).saturating_sub(total_uncompressed as usize); - filled - .saturating_add(remaining) - .clamp(filled.max(1), block_capacity) - }) - .unwrap_or(block_capacity); + // `block_capacity` only if the hint under-counted. + // + // Overflow-free by construction (no `saturating_*` masking): + // `filled <= block_capacity` always (the read only ever targets + // `[filled..len]` with `len <= block_capacity`, and a carried-over + // `pending_input` is a `split_off` below `block_capacity`), so + // `block_capacity - filled` never underflows; pinning `remaining` + // to `block_capacity` before the `usize` cast keeps the cast and + // the final add within `usize` on every target. + let initial_target = match initial_size_hint { + Some(hint) if hint > total_uncompressed => { + let remaining = (hint - total_uncompressed).min(block_capacity as u64) as usize; + filled + remaining.min(block_capacity - filled) + } + // Unknown hint, or an inexact hint already met by prior blocks: + // read against the full block window. + _ => block_capacity, + }; if uncompressed_data.len() < initial_target { uncompressed_data.resize(initial_target, 0); } @@ -1093,11 +1101,11 @@ impl FrameCompressor { if filled == uncompressed_data.len() { // Hint under-counted the block; grow toward block_capacity // (doubling, capped) so reading continues without paying a - // full-buffer zero up front. - let grow_to = uncompressed_data - .len() - .saturating_mul(2) - .clamp(filled + 1, block_capacity); + // full-buffer zero up front. `len <= block_capacity` so the + // double stays well within `usize`; `filled < block_capacity` + // here (the `== block_capacity` break fired otherwise), so + // `filled + 1 <= block_capacity`. + let grow_to = (uncompressed_data.len() * 2).clamp(filled + 1, block_capacity); uncompressed_data.resize(grow_to, 0); } let read_end = uncompressed_data.len(); From a9b0f38fb0771576701e836ff169f78cf8eb3828 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 6 Jun 2026 00:40:34 +0300 Subject: [PATCH 10/33] perf(encode): share dictionary entropy tables by handle, not per-frame clone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit prepare_frame deep-cloned the cached dictionary FSE encoder tables into the per-frame state every frame (PreviousFseTable::clone was ~16% of steady-state small dict-compress). The previous-table is only ever read or REPLACED wholesale, never mutated in place, so back PreviousFseTable:: Custom with a shared handle (SharedFseTable = Arc on atomic targets, Rc otherwise — keeps no_std no-atomics building) instead of Box. The per-frame entropy seed is now a refcount bump, mirroring the donor referencing cdict->cBlockState rather than rebuilding it per frame. --- zstd/src/encoding/blocks/compressed.rs | 12 +++++++----- zstd/src/encoding/frame_compressor.rs | 26 +++++++++++++++++++++----- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/zstd/src/encoding/blocks/compressed.rs b/zstd/src/encoding/blocks/compressed.rs index f9d671c9f..d61660dd6 100644 --- a/zstd/src/encoding/blocks/compressed.rs +++ b/zstd/src/encoding/blocks/compressed.rs @@ -4,7 +4,7 @@ use crate::{ bit_io::BitWriter, blocks::block::BlockType, encoding::block_header::BlockHeader, - encoding::frame_compressor::{CompressState, FseTables, PreviousFseTable}, + encoding::frame_compressor::{CompressState, FseTables, PreviousFseTable, SharedFseTable}, encoding::{Matcher, Sequence}, fse::fse_encoder::{FSETable, build_table_from_symbol_counts, fse_header_bits_for_counts}, huff0::huff0_encoder, @@ -1668,7 +1668,7 @@ fn remember_last_used_table(slot: &mut Option, next: Option) -> Option { match mode { - FseTableMode::Encoded(table) => Some(PreviousFseTable::Custom(Box::new(table))), + FseTableMode::Encoded(table) => Some(PreviousFseTable::Custom(SharedFseTable::new(table))), FseTableMode::Predefined(_) => Some(PreviousFseTable::Default), FseTableMode::Rle(symbol) => Some(PreviousFseTable::Rle(symbol)), FseTableMode::RepeatLast(_) => None, @@ -2200,7 +2200,6 @@ fn compress_literals( #[cfg(test)] mod tests { - use alloc::boxed::Box; use super::{ FseTableMode, RawSequence, choose_table, emit_single_sequence_block, encode_match_len, @@ -2693,7 +2692,8 @@ mod tests { fn fast_band_strategies_prefer_repeat_fse_table() { use crate::encoding::strategy::StrategyTag; let prev = build_table_from_symbol_counts(&[8, 1], 9, false); - let previous = PreviousFseTable::Custom(Box::new(prev)); + let previous = + PreviousFseTable::Custom(crate::encoding::frame_compressor::SharedFseTable::new(prev)); let fse_tables = FseTables::new(); // Distribution over symbols {0,1}, both covered by `previous`. let mut counts = [0usize; 256]; @@ -2724,7 +2724,9 @@ mod tests { fn remember_last_used_tables_reuses_existing_custom_slot_for_repeat() { let mut fse_tables = FseTables::new(); let custom = build_table_from_symbol_counts(&[1, 1], 5, false); - fse_tables.ll_previous = Some(PreviousFseTable::Custom(Box::new(custom))); + fse_tables.ll_previous = Some(PreviousFseTable::Custom( + crate::encoding::frame_compressor::SharedFseTable::new(custom), + )); let before = core::ptr::from_ref( previous_table(fse_tables.ll_previous.as_ref(), fse_tables.ll_default_ref()).unwrap(), diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index c4795fb85..1cbf95ce1 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -1,6 +1,6 @@ //! Utilities and interfaces for encoding an entire frame. Allows reusing resources -use alloc::{boxed::Box, vec::Vec}; +use alloc::vec::Vec; use core::convert::TryInto; #[cfg(feature = "hash")] use twox_hash::XxHash64; @@ -132,12 +132,28 @@ struct CachedDictionaryEntropy { of_previous: Option, } +/// Shared owner for a custom "previous" FSE encoder table. `Arc` on +/// atomic-pointer targets, `Rc` otherwise (keeps `no_std` no-atomics +/// builds compiling, single-thread there anyway), mirroring +/// `decoding::dictionary::SharedDictionary`. Cloning the cached +/// dictionary entropy into the per-frame state is then a refcount bump, +/// not a full `FSETable` copy — the donor references `cdict->cBlockState` +/// instead of rebuilding it per frame. +#[cfg(target_has_atomic = "ptr")] +pub(crate) type SharedFseTable = alloc::sync::Arc; +#[cfg(not(target_has_atomic = "ptr"))] +pub(crate) type SharedFseTable = alloc::rc::Rc; + #[derive(Clone)] pub(crate) enum PreviousFseTable { // Default tables are immutable and already stored alongside the state, so // repeating them only needs a lightweight marker instead of cloning FSETable. Default, - Custom(Box), + // Shared handle: cloning (per-frame dictionary entropy seed) is a refcount + // bump. The table is only ever read or REPLACED wholesale (a block that + // builds a new table swaps in a fresh `SharedFseTable`), never mutated in + // place, so sharing is sound. + Custom(SharedFseTable), Rle(u8), } @@ -1590,17 +1606,17 @@ impl FrameCompressor { .fse .literal_lengths .to_encoder_table() - .map(|table| PreviousFseTable::Custom(Box::new(table))), + .map(|table| PreviousFseTable::Custom(SharedFseTable::new(table))), ml_previous: dictionary .fse .match_lengths .to_encoder_table() - .map(|table| PreviousFseTable::Custom(Box::new(table))), + .map(|table| PreviousFseTable::Custom(SharedFseTable::new(table))), of_previous: dictionary .fse .offsets .to_encoder_table() - .map(|table| PreviousFseTable::Custom(Box::new(table))), + .map(|table| PreviousFseTable::Custom(SharedFseTable::new(table))), }); // A previously-captured CDict prime snapshot belongs to the OLD // dictionary; drop it so the first frame with the new dictionary From 4a05b149782dee55102b0cbc3971eff683e4fe73 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 6 Jun 2026 00:56:38 +0300 Subject: [PATCH 11/33] perf(encode): cache Fast dict table across attach-path frames Small-source dictionary frames (at/below the Fast 8 KiB attach cutoff) re-prime every frame instead of restoring a copy snapshot. Re-priming re-committed the dict bytes to history AND re-hashed every position into the immutable dict table each frame. Keep the built dict table across a same-parameter reset and skip the re-hash: the dictionary lands at identical absolute history positions every frame (history is cleared then the dict is committed first), so the cached hashes stay valid. The dict bytes are still re-committed (needed for match extension); only the hashing is elided. The cache is dropped on parameter change, history eviction, and dictionary attach/clear, so the kernel never probes a dict region whose bytes are no longer re-committed. Output is byte-identical: cold-cache and warm-cache frames match. Part of #316. Closes #349 --- zstd/src/encoding/frame_compressor.rs | 62 ++++++++++++++++++++++++ zstd/src/encoding/match_generator.rs | 15 ++++++ zstd/src/encoding/simple/fast_matcher.rs | 58 +++++++++++++++++++--- 3 files changed, 127 insertions(+), 8 deletions(-) diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index 1cbf95ce1..d213d2d2f 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -2511,6 +2511,68 @@ mod tests { } } + #[test] + fn dict_primed_matcher_cache_reused_across_small_attach_frames_is_byte_identical() { + // CDict-equivalent ATTACH path (small source, at/below the Fast 8 KiB + // attach cutoff): frames 2..N re-prime — re-committing the dict bytes + // to history — but reuse the already-built dict table instead of + // re-hashing it. The cached-table frame must reproduce the + // freshly-primed first frame byte-for-byte, and a fresh single-frame + // compressor (no prior dict cache) must produce the identical bytes + // too, proving the cache changes timing, not output. + let dict_raw = include_bytes!("../../dict_tests/dictionary"); + // Stay under the 8 KiB cutoff so the attach (re-prime) path is taken + // every frame rather than the copy-snapshot restore. + let mut payload = Vec::new(); + while payload.len() < 2 * 1024 { + payload.extend_from_slice(b"tenant=demo op=put key=1 value=aaaaabbbbbcccccddddd\n"); + } + + let prepared = + super::EncoderDictionary::from_bytes(dict_raw).expect("dict bytes should parse"); + let dict_id = prepared.id(); + let mut compressor: FrameCompressor = + FrameCompressor::new(super::CompressionLevel::Fastest); + compressor + .set_encoder_dictionary(prepared) + .expect("prepared dictionary should attach"); + + // Frame 1 builds + marks the dict table; frame 2 reuses it. + let frame1 = compressor.compress_independent_frame(payload.as_slice()); + let frame2 = compressor.compress_independent_frame(payload.as_slice()); + assert_eq!( + frame1, frame2, + "reused dict table (attach path) must reproduce the freshly-built frame byte-for-byte" + ); + + // A fresh compressor (cold dict cache) must emit the same bytes — the + // cache is a timing optimization, never a content change. + let fresh_prepared = + super::EncoderDictionary::from_bytes(dict_raw).expect("dict bytes should parse"); + let mut fresh: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Fastest); + fresh + .set_encoder_dictionary(fresh_prepared) + .expect("prepared dictionary should attach"); + let fresh_frame = fresh.compress_independent_frame(payload.as_slice()); + assert_eq!( + fresh_frame, frame1, + "cold-cache compressor must match the warm-cache frame byte-for-byte" + ); + + for frame in [&frame1, &frame2] { + let (hdr, _) = + crate::decoding::frame::read_frame_header(frame.as_slice()).expect("frame header"); + assert_eq!(hdr.dictionary_id(), Some(dict_id)); + let mut decoder = FrameDecoder::new(); + decoder + .add_dict(crate::decoding::Dictionary::decode_dict(dict_raw).unwrap()) + .unwrap(); + let mut decoded = Vec::with_capacity(payload.len()); + decoder.decode_all_to_vec(frame, &mut decoded).unwrap(); + assert_eq!(decoded.as_slice(), payload.as_slice()); + } + } + #[test] fn set_dictionary_from_bytes_matches_full_decode_byte_for_byte() { // The encoder-only dict parse (`decode_dict_for_encoding`, used by diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs index ffd986378..7b10c2ab1 100644 --- a/zstd/src/encoding/match_generator.rs +++ b/zstd/src/encoding/match_generator.rs @@ -1463,6 +1463,14 @@ impl Matcher for MatchGeneratorDriver { .table .set_dictionary_limit_from_primed_bytes(committed_dict_budget); } + // CDict-equivalent: now that every dict chunk is indexed, mark the + // Fast-backend dict table primed so the next frame's re-prime reuses + // it (skips the re-hash) while still re-committing the dict bytes to + // history. No-op when the attach path built no table (copy mode or a + // sub-8-byte dict) — `mark_dict_primed` self-guards on table presence. + if self.active_backend() == super::strategy::BackendTag::Simple { + self.simple_mut().mark_dict_primed(); + } } fn restore_primed_dictionary(&mut self, level: super::CompressionLevel) -> bool { @@ -1488,6 +1496,13 @@ impl Matcher for MatchGeneratorDriver { fn invalidate_primed_dictionary(&mut self) { self.primed = None; + // Drop the Fast-backend CDict-equivalent table cache too: it is keyed + // to the dictionary being removed / replaced. Left in place, the next + // same-params `reset` would retain it and the kernel would probe a + // dict region whose bytes are no longer re-committed to history. + if self.active_backend() == super::strategy::BackendTag::Simple { + self.simple_mut().invalidate_dict_cache(); + } } fn seed_dictionary_entropy( diff --git a/zstd/src/encoding/simple/fast_matcher.rs b/zstd/src/encoding/simple/fast_matcher.rs index 597d3fa02..7169e65bd 100644 --- a/zstd/src/encoding/simple/fast_matcher.rs +++ b/zstd/src/encoding/simple/fast_matcher.rs @@ -249,6 +249,15 @@ pub(crate) struct FastKernelMatcher { /// dict kernel uses to separate main-table (input) from dict-table /// matches. `0` when no dictionary is primed. dict_region_len: usize, + /// CDict-equivalent cache flag: `true` once [`Self::dict_table`] has been + /// fully built for the attached dictionary. A reused compressor keeps the + /// built `dict_table` across the per-frame `reset` (the dictionary + /// re-commits to the same absolute history positions, so the hashed + /// positions stay valid) and skips re-hashing it — the donor copies / + /// references the precomputed `cdict->matchState` rather than rebuilding + /// it per frame. Cleared on parameter change, history eviction, or + /// dictionary attach/clear (see `invalidate_dict_cache`). + dict_primed: bool, } impl FastKernelMatcher { @@ -338,6 +347,7 @@ impl FastKernelMatcher { last_borrowed_block: None, dict_table: None, dict_region_len: 0, + dict_primed: false, } } @@ -360,12 +370,20 @@ impl FastKernelMatcher { ); if self.hash_table.hash_log() != hash_log || self.hash_table.mls() != mls { // Parameters changed — rebuild the table at the new size. - // Cannot reuse the old allocation because the donor-shape - // hash table dimensions are baked in at construction. + // Cannot reuse the old allocation because the hash table + // dimensions are baked in at construction. A reshape also + // invalidates the cached dict table: its absolute positions + // index a table whose shape no longer matches. self.hash_table = FastHashTable::new(hash_log, mls); + self.dict_table = None; + self.dict_region_len = 0; + self.dict_primed = false; } else { // Same shape — keep the allocation, zero the entries via - // `memset` (donor's `ZSTD_window_clear` cadence). + // `memset` (ZSTD_window_clear cadence). A primed dict table + // is retained: the dictionary lands at the same absolute + // history positions every frame, so its hashes stay valid + // and the per-frame re-hash is skipped (CDict-equivalent). self.hash_table.clear(); } // M8: history starts empty (HISTORY_DRAIN_BASE = 0). @@ -387,11 +405,6 @@ impl FastKernelMatcher { // different allocation, so a stale (ptr, len) would dangle. self.borrowed = None; self.last_borrowed_block = None; - // Drop any primed dictionary state: the next frame re-primes (or - // not) from scratch, and stale absolute dict positions must not - // leak across frames. - self.dict_table = None; - self.dict_region_len = 0; } /// Reported decoder-side window size (bytes) — test-only. @@ -630,6 +643,7 @@ impl FastKernelMatcher { // reachable anyway. self.dict_table = None; self.dict_region_len = 0; + self.dict_primed = false; self.hash_table.clear(); self.last_block_start = self.last_block_start.saturating_sub(drop_n); // Skip position 0 — `prefix_start_index = 1` means the kernel @@ -1156,6 +1170,27 @@ impl FastKernelMatcher { self.prime_dict_table_for_range(block_start); } + /// Mark the dict table as fully built (CDict-equivalent). Called by the + /// driver after the final dictionary chunk has been primed, so the next + /// frame's [`Self::prime_dict_table_for_range`] skips the re-hash while + /// the dict bytes are still re-committed to history. Only marks when a + /// table actually exists — a sub-8-byte dict builds no table and must + /// re-run the (cheap, no-op) prime path each frame. + pub(crate) fn mark_dict_primed(&mut self) { + if self.dict_table.is_some() { + self.dict_primed = true; + } + } + + /// Drop the cached dict table and its primed flag. Called by the driver + /// when the next frame carries no dictionary, so the kernel never probes + /// a stale dict region whose bytes are no longer re-committed. + pub(crate) fn invalidate_dict_cache(&mut self) { + self.dict_table = None; + self.dict_region_len = 0; + self.dict_primed = false; + } + /// Build (or extend) [`Self::dict_table`] over `history[range_start..]`, /// the freshly-appended dictionary bytes. Lazily allocates the dict table /// at the same `(hash_log, mls)` as the main table so one hash keys both. @@ -1165,6 +1200,13 @@ impl FastKernelMatcher { // Record the dict/input boundary regardless of whether any position // is hashable (a sub-8-byte dict still bounds the input floor). self.dict_region_len = history_len; + if self.dict_primed { + // CDict-equivalent fast path: `dict_table` was built over the + // identical dictionary bytes on a prior frame, and those bytes + // sit at the same absolute history positions now (the dict is + // re-committed before this call). Skip the re-hash entirely. + return; + } if history_len < HASH_READ_SIZE { return; } From 3e8be02d4238eee0999a2a137d194e8920a4a1f0 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 6 Jun 2026 01:23:55 +0300 Subject: [PATCH 12/33] perf(decode): move decompression-bomb ceiling off the per-match hot path The per-block output ceiling added to guard against decompression-bomb OOMs was checked in repeat_inner on EVERY match copy (buffer.len() + match_length > block_output_limit). That compare plus a field load and a cold branch bloated the #[inline(always)] repeat path inlined into the pipelined sequence loop, regressing lazy-level decode on small frames (~-31% on small-4k-log-lines, level_7/8/10 lazy, stream decode through the growable RingBuffer) where matches are dense. Enforce the ceiling on the cold growth path instead: RingBuffer gains a max_capacity lowered per block via set_max_capacity, and its try_reserve rejects a reserve that would have to GROW past it. A well-formed block is fully covered by the upfront reserve(MAX_BLOCK_SIZE), so its matches never reach the check; only an over-producing match (the bomb) forces a grow and is rejected with OutputBufferOverflow. The guard now bounds the OOM on every target (not just the i686 assert path) at zero hot-path cost, replacing the per-match BlockOutputExceedsMax check (variant removed). 742 tests pass; i686 builds (the reserve_amortized debug_assert panic path is no longer reached for over-production). --- zstd/src/decoding/buffer_backend.rs | 9 ++ zstd/src/decoding/decode_buffer.rs | 140 ++++++++++++---------------- zstd/src/decoding/errors.rs | 19 ---- zstd/src/decoding/ringbuffer.rs | 65 +++++++++++++ 4 files changed, 134 insertions(+), 99 deletions(-) diff --git a/zstd/src/decoding/buffer_backend.rs b/zstd/src/decoding/buffer_backend.rs index 784f30018..504d2a623 100644 --- a/zstd/src/decoding/buffer_backend.rs +++ b/zstd/src/decoding/buffer_backend.rs @@ -226,6 +226,15 @@ pub(crate) trait BufferBackend: Sized { Ok(()) } + /// Lower the per-block growth ceiling on backends whose `try_reserve` + /// may grow without bound (the streaming `RingBuffer`). The block + /// sequence decoder sets it to `len + MAX_BLOCK_SIZE` per block so an + /// over-producing match is rejected on the cold growth path, + /// bounding decompression-bomb OOMs. Default no-op: fixed-capacity + /// backends are already bounded, and the inline-exec (FCS-capped) + /// path never grows through `try_reserve`. + fn set_max_capacity(&mut self, _max_capacity: usize) {} + /// Live byte count: bytes between the logical head and tail. fn len(&self) -> usize; diff --git a/zstd/src/decoding/decode_buffer.rs b/zstd/src/decoding/decode_buffer.rs index ae7220d31..faaaee601 100644 --- a/zstd/src/decoding/decode_buffer.rs +++ b/zstd/src/decoding/decode_buffer.rs @@ -6,7 +6,6 @@ use core::hash::Hasher; use super::buffer_backend::BufferBackend; use super::prefetch; use super::ringbuffer::RingBuffer; -use crate::common::MAX_BLOCK_SIZE; use crate::decoding::errors::DecodeBufferError; /// Generic decode-side output buffer parameterised over the storage @@ -37,14 +36,6 @@ pub struct DecodeBuffer { pub window_size: usize, total_output_counter: u64, - /// Upper bound on `buffer.len()` for the block currently being decoded. - /// Set per block to `len_at_block_start + MAX_BLOCK_SIZE` so a single - /// block cannot decompress to more than `MAX_BLOCK_SIZE` of output; a - /// `repeat` that would cross it is rejected before its `try_reserve`, - /// bounding the growable `RingBuffer` against decompression-bomb OOMs. - /// `usize::MAX` (the default) disables the check for non-block-decode - /// callers of `repeat`. - block_output_limit: usize, #[cfg(feature = "hash")] pub hash: twox_hash::XxHash64, } @@ -93,7 +84,6 @@ impl DecodeBuffer { dict: None, window_size, total_output_counter: 0, - block_output_limit: usize::MAX, #[cfg(feature = "hash")] hash: twox_hash::XxHash64::with_seed(0), } @@ -118,40 +108,24 @@ impl DecodeBuffer { dict: None, window_size, total_output_counter: 0, - block_output_limit: usize::MAX, #[cfg(feature = "hash")] hash: twox_hash::XxHash64::with_seed(0), } } /// Arm the per-block decompressed-output ceiling for the block about to - /// be decoded: a `repeat` whose match would push `buffer.len()` past - /// `current_len + MAX_BLOCK_SIZE` is rejected (see `block_output_limit`), - /// bounding the growable backend against decompression-bomb OOMs. Called - /// once per block by the sequence decoder, before the sequence loop. + /// be decoded: the growable backend is told it may grow only up to + /// `current_len + max_block_output`, so a match that would push this + /// block's output past `MAX_BLOCK_SIZE` fails its `try_reserve` on the + /// cold growth path instead of growing the ring to gigabytes (a + /// decompression-bomb OOM) before the post-block validity check runs. + /// Called once per block by the sequence decoder, before the sequence + /// loop. Backends that cannot grow unbounded (fixed-capacity, or the + /// inline-exec FCS-capped path) take the trait's no-op default. #[inline] pub(crate) fn set_block_output_ceiling(&mut self, max_block_output: usize) { - self.block_output_limit = self.buffer.len().saturating_add(max_block_output); - } - - /// Cold-path error builder for the per-block output ceiling. Outlined - /// from `repeat_inner` so the hot per-match check stays a single - /// compare; the produced-byte arithmetic only runs on the - /// malformed-input rejection path. - #[cold] - #[inline(never)] - fn block_output_exceeded(&self, match_length: usize) -> DecodeBufferError { - let block_start = self - .block_output_limit - .saturating_sub(MAX_BLOCK_SIZE as usize); - DecodeBufferError::BlockOutputExceedsMax { - produced: self - .buffer - .len() - .saturating_add(match_length) - .saturating_sub(block_start), - max: MAX_BLOCK_SIZE as usize, - } + let ceiling = self.buffer.len().saturating_add(max_block_output); + self.buffer.set_max_capacity(ceiling); } pub fn reset(&mut self, window_size: usize) { @@ -168,8 +142,9 @@ impl DecodeBuffer { // the frame will actually hit this buffer. self.dict = None; self.total_output_counter = 0; - // Cleared per frame; re-armed per block by the sequence decoder. - self.block_output_limit = usize::MAX; + // Lift the per-block growth ceiling between frames; the sequence + // decoder re-arms it per block. Non-block callers stay unbounded. + self.buffer.set_max_capacity(usize::MAX); #[cfg(feature = "hash")] { self.hash = twox_hash::XxHash64::with_seed(0); @@ -408,22 +383,14 @@ impl DecodeBuffer { return Ok(()); } - // Bound the block's cumulative decompressed output. A single zstd - // block decompresses to at most MAX_BLOCK_SIZE; reject a match that - // would push this block's output past the per-block ceiling - // (`block_output_limit`, set per block by the sequence decoder) - // before its `try_reserve`. On the growable `RingBuffer` an - // over-producing malformed / adversarial block would otherwise grow - // the buffer to gigabytes inside the decode loop — a - // decompression-bomb OOM — before the post-block validity check can - // reject the frame. The default `usize::MAX` ceiling leaves - // non-block-decode callers of `repeat` unaffected. - if self.buffer.len().saturating_add(match_length) > self.block_output_limit { - // Cold path outlined so the per-match hot check is just the - // compare above (the error-byte arithmetic must not bloat the - // sequence loop's inlined repeat path). - return Err(self.block_output_exceeded(match_length)); - } + // The per-block decompression-bomb ceiling is NOT checked here on + // every match (that compare bloated the inlined hot loop). It is + // enforced once on the cold growth path: `set_block_output_ceiling` + // lowers the backend's `max_capacity` per block, so the + // `try_reserve` below rejects an over-producing match exactly when + // it would have to grow the buffer past the ceiling — bounding the + // OOM on every target while a well-formed block (covered by the + // upfront `reserve(MAX_BLOCK_SIZE)`) never reaches the check. if offset > self.buffer.len() { self.repeat_from_dict(offset, match_length) @@ -436,12 +403,13 @@ impl DecodeBuffer { // assumes the required free capacity exists; skipping it // would turn a malformed block (match_length past the // upfront `reserve(MAX_BLOCK_SIZE)`) into release-build - // UB. Use the fallible variant so fixed-capacity backends - // (`UserSliceBackend`) surface a structured error instead - // of panicking via the per-call `assert!` inside - // `extend_from_within_unchecked`. Growable backends' - // default impl never fails (allocation succeeds or - // aborts), so the conversion is a cheap no-op there. + // UB. The fallible variant surfaces a structured error for + // both fixed-capacity backends (`UserSliceBackend`: write + // past the user's slice) and the growable `RingBuffer` (a + // grow past the per-block `max_capacity` ceiling — the + // decompression-bomb guard). On the common path the upfront + // `reserve(MAX_BLOCK_SIZE)` already covers the write, so this + // is a cheap capacity check, not an allocation. self.buffer.try_reserve(match_length).map_err(|o| { DecodeBufferError::OutputBufferOverflow { tail: o.tail, @@ -1330,29 +1298,41 @@ mod tests { #[test] fn repeat_rejects_output_past_block_ceiling() { - // A single zstd block decompresses to at most MAX_BLOCK_SIZE. With - // the per-block ceiling armed, a `repeat` whose match would push the - // block's output past it must be rejected before its `try_reserve`. - // Without this guard the over-long match drives `try_reserve` - // unbounded on the growable `RingBuffer` — the artifact `oom-66db61d9…` - // grew the ring to ~0.5–2 GiB across many over-producing sequences - // before any post-block check ran (a decompression-bomb OOM, - // reproduced via the fuzz `decode` target). Pre-guard this `repeat` - // returned `Ok` (reserved + copied); it must now reject. + // A single zstd block decompresses to at most MAX_BLOCK_SIZE. The + // per-block ceiling is enforced on the growable `RingBuffer`'s cold + // growth path: `set_block_output_ceiling` lowers `max_capacity`, so a + // `repeat` whose match would have to grow the buffer past the ceiling + // fails its `try_reserve` instead of growing the ring unbounded. + // Without this guard the over-long match drove `try_reserve` to + // ~0.5–2 GiB across many over-producing sequences (artifact + // `oom-66db61d9…`, fuzz `decode` target) before any post-block check + // ran — a decompression-bomb OOM. The match needing growth surfaces + // as `OutputBufferOverflow` (the backend's structured reject). let mut decode_buf = DecodeBuffer::::new(4 * 1024); decode_buf.push(b"abcdef"); // len = 6 - decode_buf.set_block_output_ceiling(8); // ceiling = 6 + 8 = 14 + decode_buf.set_block_output_ceiling(8); // max_capacity = 6 + 8 = 14 let err = decode_buf.repeat(4, 16).unwrap_err(); // 6 + 16 = 22 > 14 - assert!(matches!( - err, - crate::decoding::errors::DecodeBufferError::BlockOutputExceedsMax { .. } - )); - // Display carries the produced bytes for diagnostics. `produced` - // is `(len + match_length) - block_start` = `22 - 0 = 22` (the - // ceiling of 14 sits below MAX_BLOCK_SIZE so `block_start` - // saturates to 0); `max` is always MAX_BLOCK_SIZE. - let rendered = alloc::format!("{err}"); - assert!(rendered.contains("22"), "unexpected Display: {rendered}"); + assert!( + matches!( + err, + crate::decoding::errors::DecodeBufferError::OutputBufferOverflow { .. } + ), + "over-producing match must be rejected, got {err:?}" + ); + } + + #[test] + fn repeat_within_block_ceiling_still_succeeds() { + // A match that keeps the block's output at/below the armed ceiling + // must NOT be rejected — the guard fires only on growth past the + // ceiling, never on legitimate in-bounds output. + let mut decode_buf = DecodeBuffer::::new(4 * 1024); + decode_buf.push(b"abcdef"); // len = 6 + decode_buf.set_block_output_ceiling(8); // max_capacity = 14 + decode_buf + .repeat(4, 8) + .expect("6 + 8 = 14 == ceiling is allowed"); + assert_eq!(decode_buf.len(), 14); } #[test] diff --git a/zstd/src/decoding/errors.rs b/zstd/src/decoding/errors.rs index 2e5e1949d..7831721c6 100644 --- a/zstd/src/decoding/errors.rs +++ b/zstd/src/decoding/errors.rs @@ -449,19 +449,6 @@ pub enum DecodeBufferError { requested: usize, capacity: usize, }, - /// A block's cumulative decompressed output would exceed `MAX_BLOCK_SIZE`, - /// the most a single zstd block can produce. Surfaced by - /// [`super::decode_buffer::DecodeBuffer::repeat`] before the match copy - /// when the running block output plus this match would cross the - /// per-block ceiling. Without it a malformed / adversarial frame whose - /// sequences over-produce drives an unbounded `try_reserve` on a growable - /// backend (`RingBuffer`), which grows to gigabytes inside the - /// block-decode loop (a decompression-bomb OOM) before the post-block - /// validity check can reject the frame. - BlockOutputExceedsMax { - produced: usize, - max: usize, - }, } #[cfg(feature = "std")] @@ -498,12 +485,6 @@ impl core::fmt::Display for DecodeBufferError { "Match repeat would write past fixed-capacity buffer: tail={tail}, requested={requested}, capacity={capacity}" ) } - DecodeBufferError::BlockOutputExceedsMax { produced, max } => { - write!( - f, - "Block decompressed output {produced} exceeds the per-block maximum {max}" - ) - } } } } diff --git a/zstd/src/decoding/ringbuffer.rs b/zstd/src/decoding/ringbuffer.rs index 3f43f6320..f22c24bbe 100644 --- a/zstd/src/decoding/ringbuffer.rs +++ b/zstd/src/decoding/ringbuffer.rs @@ -29,6 +29,17 @@ pub struct RingBuffer { cap: usize, head: usize, tail: usize, + /// Upper bound on the live byte count (`len()`) that a *growing* + /// reserve may target. `usize::MAX` (the default) leaves growth + /// unbounded. The block sequence decoder lowers it to + /// `len_at_block_start + MAX_BLOCK_SIZE` before each block so a + /// match that would push output past the per-block ceiling fails its + /// `try_reserve` (cold growth path) instead of growing the ring to + /// gigabytes — a decompression-bomb OOM — before the post-block + /// validity check runs. Enforced only when a reserve actually has to + /// grow, so well-formed blocks (covered by the upfront + /// `reserve(MAX_BLOCK_SIZE)`) never pay for the check. + max_capacity: usize, } // SAFETY: RingBuffer does not hold any thread specific values -> it can be sent to another thread -> RingBuffer is Send @@ -46,6 +57,7 @@ impl RingBuffer { // SAFETY: Upholds invariant 2-4 head: 0, tail: 0, + max_capacity: usize::MAX, } } @@ -173,6 +185,51 @@ impl RingBuffer { self.reserve_amortized(amount - free); } + /// Lower the growth ceiling (see [`Self::max_capacity`]). `usize::MAX` + /// restores unbounded growth. + #[inline] + pub fn set_max_capacity(&mut self, max_capacity: usize) { + self.max_capacity = max_capacity; + } + + /// Fallible [`Self::reserve`]: identical fast path, but when the + /// reserve would have to *grow* the ring it first rejects any target + /// `len() + amount` past [`Self::max_capacity`]. This is where the + /// per-block decompression-bomb ceiling is enforced — on the cold + /// growth path only, so well-formed blocks never pay for it. + #[inline] + pub fn try_reserve( + &mut self, + amount: usize, + ) -> Result<(), super::buffer_backend::BackendOverflow> { + // Flat fast path (mirrors `reserve`): no wrap and the write fits + // below `cap` — capacity already present, no growth, no ceiling + // check. + if self.head <= self.tail && amount < self.cap.saturating_sub(self.tail) { + return Ok(()); + } + let free = self.free(); + if free >= amount { + return Ok(()); + } + // Growth is required. Reject if it would cross the per-block + // ceiling before allocating (bounds the bomb on every target, + // unlike a 32-bit-only assert). + if self + .len() + .checked_add(amount) + .is_none_or(|needed| needed > self.max_capacity) + { + return Err(super::buffer_backend::BackendOverflow { + tail: self.tail, + requested: amount, + capacity: self.max_capacity, + }); + } + self.reserve_amortized(amount - free); + Ok(()) + } + #[inline(never)] #[cold] fn reserve_amortized(&mut self, amount: usize) { @@ -862,6 +919,14 @@ impl super::buffer_backend::BufferBackend for RingBuffer { Self::reserve(self, n); } #[inline] + fn try_reserve(&mut self, n: usize) -> Result<(), super::buffer_backend::BackendOverflow> { + Self::try_reserve(self, n) + } + #[inline] + fn set_max_capacity(&mut self, max_capacity: usize) { + Self::set_max_capacity(self, max_capacity); + } + #[inline] fn len(&self) -> usize { Self::len(self) } From d415f54b486660210191983c524f60ad5a9b4649 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 6 Jun 2026 01:35:23 +0300 Subject: [PATCH 13/33] docs(encode): note retire_dictionary_budget saturating_sub is a real clamp The dict-budget retire shrinks max_window_size by the reclaimed bytes. reclaimed (capped at dictionary_retained_budget) can exceed the CURRENT max_window_size when a prior eviction already shrank the window, so the saturating_sub floor at 0 is the correct clamp, not a masked underflow bug. Documented at all four backend arms so the next reader (or audit) does not mistake it for overflow-masking and convert it to checked_sub, which panics on the legitimate reclaim>window case (dfast_commit_space_eviction_uses_window_size_delta exercises it). --- zstd/src/encoding/match_generator.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs index 7b10c2ab1..60c9a84e5 100644 --- a/zstd/src/encoding/match_generator.rs +++ b/zstd/src/encoding/match_generator.rs @@ -971,18 +971,32 @@ impl MatchGeneratorDriver { match self.active_backend() { super::strategy::BackendTag::Simple => { let matcher = self.simple_mut(); + // `reclaimed` can exceed the CURRENT `max_window_size`: the + // retained dict budget is tracked independently and the + // window may already have been shrunk by a prior eviction, + // so the floor at 0 is the correct clamp, not a masked bug. matcher.max_window_size = matcher.max_window_size.saturating_sub(reclaimed); } super::strategy::BackendTag::Dfast => { let matcher = self.dfast_matcher_mut(); + // `reclaimed` can exceed the CURRENT `max_window_size`: the + // retained dict budget is tracked independently and the + // window may already have been shrunk by a prior eviction, + // so the floor at 0 is the correct clamp, not a masked bug. matcher.max_window_size = matcher.max_window_size.saturating_sub(reclaimed); } super::strategy::BackendTag::Row => { let matcher = self.row_matcher_mut(); + // `reclaimed` can exceed the CURRENT `max_window_size`: the + // retained dict budget is tracked independently and the + // window may already have been shrunk by a prior eviction, + // so the floor at 0 is the correct clamp, not a masked bug. matcher.max_window_size = matcher.max_window_size.saturating_sub(reclaimed); } super::strategy::BackendTag::HashChain => { let matcher = self.hc_matcher_mut(); + // See the Simple arm: `reclaimed` may exceed the current + // window, so saturating to 0 is the correct clamp. matcher.table.max_window_size = matcher.table.max_window_size.saturating_sub(reclaimed); } From 7f3caab8563378fc40645dea733773a23bf2ae15 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 6 Jun 2026 12:01:12 +0300 Subject: [PATCH 14/33] fix(encode): key dict prime snapshot on resolved matcher shape Cache the post-prime matcher snapshot (CDict-equivalent) keyed on the fully resolved matcher shape so repeated dictionary frames restore it (a table copy) instead of re-hashing every dictionary position. The key is the resolved LevelParams (window-log cap, HC/Fast table and search geometry, parse depth/target-length baked into the restored storage), the active backend's applied Dfast/Row hash-table width, the compression level, and the Fast attach-vs-copy mode. Keying on the raw source-size hint, or its ceil-log bucket, over-keys: the hint-to-shape mapping is many-to-one (the source-size adjustment is monotone in ceil_log2(hint), and Level 22 collapses several buckets onto one donor tier via its 16/128/256 KiB thresholds), so two hints that resolve to the identical matcher would each force a full re-prime. table_window_size is excluded from the key (it varies spuriously for HC levels that ignore it). level stays in the key because some stored state is derived from the level directly, not through params (Dfast use_fast_loop is true for L3 / false for L4 at identical params). fast_attach stays because the 8 KiB attach/copy cutoff falls inside a single resolved shape (an 8192 vs 8193 byte Level 1 hint resolves to identical params/table_bits but a different dict-table shape), keeping the snapshot identity self-sufficient rather than relying on the frame compressor's copy-path gating. The fast-attach/copy cutoff is unified on the same ceil-log bucket in the driver and the frame compressor's prefer_copy_snapshot gate, comparing source_size_ceil_log(hint) on the full u64 rather than an as-usize cast that could diverge on 32-bit targets. LevelParams/HcConfig/RowConfig gain PartialEq/Eq for the comparison. Regression tests cover same-bucket reuse, Level 22 cross-bucket donor-tier reuse, the attach/copy boundary refusal, and the window-mismatch refusal. --- zstd/src/encoding/frame_compressor.rs | 22 +- zstd/src/encoding/match_generator.rs | 350 ++++++++++++++++++++++---- 2 files changed, 322 insertions(+), 50 deletions(-) diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index d213d2d2f..fddf33df5 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -953,17 +953,27 @@ impl FrameCompressor { // whole table would cost MORE than the sparse re-prime here, // which is exactly why the donor attaches by reference instead). // `attachDictSizeCutoffs` per strategy: fast 8K, dfast 16K, - // greedy/lazy/btopt 32K, btultra/btultra2 8K. - let cutoff = match self.state.strategy_tag { + // greedy/lazy/btopt 32K, btultra/btultra2 8K. Expressed as the + // ceil-log bucket (8K = 2^13, 16K = 2^14, 32K = 2^15) so the + // decision uses the SAME bucketed representation as the driver's + // attach/copy gate (`reset_size_log`) — comparing + // `source_size_ceil_log(hint)` on the full u64 avoids the `as usize` + // truncation that could diverge from the driver on 32-bit targets. + // For a power-of-two cutoff `2^k`, `ceil_log2(hint) > k` is exactly + // `hint > 2^k`, so this is identical to the raw `hint > cutoff` on + // 64-bit. + let cutoff_log = match self.state.strategy_tag { crate::encoding::strategy::StrategyTag::Fast | crate::encoding::strategy::StrategyTag::BtUltra - | crate::encoding::strategy::StrategyTag::BtUltra2 => 8 * 1024, - crate::encoding::strategy::StrategyTag::Dfast => 16 * 1024, + | crate::encoding::strategy::StrategyTag::BtUltra2 => 13, + crate::encoding::strategy::StrategyTag::Dfast => 14, crate::encoding::strategy::StrategyTag::Greedy | crate::encoding::strategy::StrategyTag::Lazy - | crate::encoding::strategy::StrategyTag::BtOpt => 32 * 1024, + | crate::encoding::strategy::StrategyTag::BtOpt => 15, }; - let prefer_copy_snapshot = initial_size_hint.is_some_and(|s| s as usize > cutoff); + let prefer_copy_snapshot = initial_size_hint.is_some_and(|s| { + crate::encoding::match_generator::source_size_ceil_log(s) > cutoff_log + }); let restored = prefer_copy_snapshot && self .state diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs index 60c9a84e5..9b585527e 100644 --- a/zstd/src/encoding/match_generator.rs +++ b/zstd/src/encoding/match_generator.rs @@ -151,7 +151,7 @@ use super::hc::MAX_HC_SEARCH_DEPTH; /// Bundled tuning knobs for the hash-chain matcher. Using a typed config /// instead of positional `usize` args eliminates parameter-order hazards. -#[derive(Copy, Clone)] +#[derive(Copy, Clone, PartialEq, Eq)] struct HcConfig { hash_log: usize, chain_log: usize, @@ -159,7 +159,7 @@ struct HcConfig { target_len: usize, } -#[derive(Copy, Clone)] +#[derive(Copy, Clone, PartialEq, Eq)] pub(crate) struct RowConfig { pub(crate) hash_bits: usize, pub(crate) row_log: usize, @@ -250,7 +250,7 @@ const ROW_L5: RowConfig = RowConfig { /// family and the compile-time strategy consts; the runtime /// [`BackendTag`] used by the driver dispatcher is derived via /// [`StrategyTag::backend`] so the two cannot drift. -#[derive(Copy, Clone)] +#[derive(Copy, Clone, PartialEq, Eq)] struct LevelParams { strategy_tag: super::strategy::StrategyTag, /// Decoupled search-method axis. Independent of `strategy_tag`'s @@ -328,6 +328,31 @@ impl LevelParams { } } +/// `ceil(log2(size))` of a source-size hint, with a zero hint floored to +/// [`MIN_WINDOW_LOG`]. This is the single quantization every hint-dependent +/// matcher parameter is derived from: the window-log cap, the HC / Fast hash +/// and chain widths, the Dfast / Row table widths, the L22 config buckets, and +/// the Fast attach-vs-copy cutoff. Two hints sharing this value resolve to the +/// identical matcher shape, which is why it (not the raw byte count) keys the +/// primed-dictionary snapshot — see [`PrimedKey`]. Operates on the full `u64` +/// so callers comparing a hint against a cutoff get the same bucketed decision +/// here and at the driver, with no `as usize` truncation on 32-bit targets. +pub(crate) fn source_size_ceil_log(size: u64) -> u8 { + if size == 0 { + MIN_WINDOW_LOG + } else { + (64 - (size - 1).leading_zeros()) as u8 + } +} + +/// Donor `ZSTD_shouldAttachDict` cutoff for the Fast strategy, as a ceil-log +/// bucket: 8 KiB = `2^13`, and `bucket <= 13` is exactly `hint <= 8192` because +/// the bucket is monotone in the hint. A hint at or below this (or unknown, +/// `None`) ATTACHES the dictionary (a separate immutable table); a larger hint +/// COPIES it into the live table. Shared by `reset` (which records the mode in +/// the primed-snapshot key) and `prime_with_dictionary` (which acts on it). +const FAST_ATTACH_DICT_CUTOFF_LOG: u8 = 13; + fn dfast_hash_bits_for_window(max_window_size: usize) -> usize { let window_log = (usize::BITS - 1 - max_window_size.leading_zeros()) as usize; window_log.clamp(MIN_WINDOW_LOG as usize, DFAST_HASH_BITS) @@ -417,11 +442,7 @@ fn adjust_params_for_source_size(mut params: LevelParams, src_size: u64) -> Leve // (a decoder-interop requirement on the wire format), but the hash / // chain table widths are internal and never appear in the frame, so they // can track the actual source size below that floor. - let raw_src_log = if src_size == 0 { - MIN_WINDOW_LOG - } else { - (64 - (src_size - 1).leading_zeros()) as u8 // ceil_log2 - }; + let raw_src_log = source_size_ceil_log(src_size); let src_log = raw_src_log.max(MIN_WINDOW_LOG).max(MIN_HINTED_WINDOW_LOG); if src_log < params.window_log { params.window_log = src_log; @@ -470,11 +491,7 @@ fn level22_btultra2_params_for_source_size(source_size: Option) -> LevelPar if let Some(size) = source_size && size > 256 * 1024 { - let src_log = if size == 0 { - MIN_WINDOW_LOG - } else { - (64 - (size - 1).leading_zeros()) as u8 - }; + let src_log = source_size_ceil_log(size); window_log = window_log.min(src_log.max(MIN_WINDOW_LOG)); let adjusted_table_log = window_log as usize + 1; hc.hash_log = hc.hash_log.min(adjusted_table_log); @@ -703,13 +720,21 @@ pub struct MatchGeneratorDriver { dictionary_retained_budget: usize, // Source size hint for next frame (set via set_source_size_hint, cleared on reset). source_size_hint: Option, - // Snapshot of the frame's source-size hint captured at `reset` (where - // `source_size_hint` is consumed). Read by `prime_with_dictionary` to pick - // the donor `ZSTD_shouldAttachDict` dict mode for the Simple/Fast backend: - // `None` (unknown) or `<= FAST_ATTACH_DICT_CUTOFF` → attach (separate dict - // table, 2-cursor `compress_block_fast_dict`); larger → copy (dictionary - // primed into the live table, 4-cursor `compress_block_fast`). - reset_source_size: Option, + // Normalized `ceil_log2` bucket of the frame's source-size hint, captured at + // `reset` (where `source_size_hint` is consumed) via [`source_size_ceil_log`]. + // `None` means the frame was unhinted. Drives `prime_with_dictionary`'s donor + // `ZSTD_shouldAttachDict` mode for the Simple/Fast backend: `None` (unknown) + // or `<= FAST_ATTACH_DICT_CUTOFF_LOG` → attach (separate dict table, 2-cursor + // `compress_block_fast_dict`); larger → copy (dictionary primed into the live + // table, 4-cursor `compress_block_fast`). The primed-snapshot key is the + // resolved shape ([`reset_shape`](Self::reset_shape)), not this bucket. + reset_size_log: Option, + // Hint-resolved matcher shape from the last `reset`: the [`LevelParams`], the + // active backend's applied Dfast/Row hash-table width (`0` for HC/Fast), and + // the Fast attach-vs-copy mode. Combined with the frame's level into the + // [`PrimedKey`] that keys the primed snapshot, so it is only restored into a + // reset that resolved the identical matcher. `None` before the first `reset`. + reset_shape: Option<(LevelParams, usize, bool)>, // One-shot borrowed block range `[start, end)` staged by the borrowed // Fast frame path (`set_borrowed_block`) for the NEXT // `start_matching` / `skip_matching_with_hint`. `Some` routes that @@ -724,9 +749,59 @@ pub struct MatchGeneratorDriver { /// `prime_with_dictionary` writes. Subsequent frames restore this /// (a table memcpy) instead of re-hashing every dictionary position, /// mirroring donor `ZSTD_compressBegin_usingCDict` copying the - /// precomputed `cdict->matchState`. Invalidated when the dictionary or - /// level changes (keyed by the captured `CompressionLevel`). - primed: Option<(MatcherStorage, usize, super::CompressionLevel)>, + /// precomputed `cdict->matchState`. Invalidated when the dictionary + /// changes; keyed by the [`PrimedKey`] resolved matcher shape so a snapshot + /// is only restored into a reset that produces the same matcher — see + /// `restore_primed_dictionary`. + primed: Option<(MatcherStorage, usize, PrimedKey)>, +} + +/// Identity of the matcher configuration a primed snapshot was captured under: +/// the FULLY RESOLVED matcher shape, not the raw source-size hint. +/// +/// `reset()` resolves the hint into a [`LevelParams`] (window_log cap, the +/// HC/Fast table and search geometry, the parse depth/target-length that get +/// baked into the restored `storage`) plus, for the Dfast/Row backends, a +/// table-width derived from the hint's ceil-log bucket. The mapping from hint +/// to resolved shape is many-to-one: the source-size adjustment is monotone in +/// `ceil_log2(hint)`, and Level 22 additionally collapses several buckets onto +/// one donor tier (its `<= 16/128/256 KiB` thresholds). Keying on the raw hint +/// (or even its ceil-log bucket) therefore over-keys — two hints that resolve +/// to the identical matcher would each force a full re-prime. Keying on the +/// resolved (`params`, `table_bits`) pair restores across them. +/// +/// `table_bits` is the hint-dependent hash-table width the ACTIVE backend +/// applied (`set_hash_bits` value for Dfast/Row; `0` for HC/Fast, whose widths +/// already live in `params`). The snapshot is only ever captured on the COPY +/// path (a hinted, above-cutoff frame), so `table_bits` is always the resolved +/// Dfast/Row value there, never the unhinted default. +/// +/// `level` is kept alongside the resolved `params` because some stored matcher +/// state is derived from the level DIRECTLY, not through `params`: e.g. Dfast's +/// `use_fast_loop` is true for L3 but false for L4, yet L3 and L4 resolve to +/// byte-identical `params`. Without `level` a snapshot captured at L3 could be +/// restored into an L4 reset, installing the wrong `use_fast_loop`. +/// +/// `fast_attach` records the Fast backend's attach-vs-copy mode +/// ([`FAST_ATTACH_DICT_CUTOFF_LOG`]) because that cutoff (8 KiB) falls INSIDE a +/// single resolved shape: an 8192- and an 8193-byte Level 1 hint both clamp to +/// window_log 14 with identical `params`/`table_bits`, yet 8192 attaches (a +/// separate dict table) while 8193 copies into the live table — two different +/// `storage` shapes. The frame compressor only captures/restores snapshots on +/// the copy path today, but keying on the mode keeps the snapshot identity +/// self-sufficient rather than relying on that external gate. +/// +/// Restoring a snapshot whose key differs would reinstate the old `storage` +/// (and its `max_window_size` / table dimensions / parse params / dict-table +/// shape) under a reset that resolved a different shape — the encoder could +/// then search past the frame header's window and emit an undecodable match. +/// All fields must match before a restore is allowed. +#[derive(Clone, Copy, PartialEq, Eq)] +struct PrimedKey { + level: super::CompressionLevel, + params: LevelParams, + table_bits: usize, + fast_attach: bool, } impl MatchGeneratorDriver { @@ -805,7 +880,8 @@ impl MatchGeneratorDriver { // the first `reset()` overwrites both sides from the // resolved LevelParams. reported_window_size: next_pow2, - reset_source_size: None, + reset_size_log: None, + reset_shape: None, dictionary_retained_budget: 0, source_size_hint: None, borrowed_pending: None, @@ -1090,11 +1166,10 @@ impl MatchGeneratorDriver { // matches against it as window history — the path that already // matches/beats the donor on large corpora). The dispatch in // `start_matching` keys off `dict_table.is_some()`, which only - // the attach path populates. - const FAST_ATTACH_DICT_CUTOFF: u64 = 8 * 1024; + // the attach path populates. See [`FAST_ATTACH_DICT_CUTOFF_LOG`]. let attach = self - .reset_source_size - .is_none_or(|n| n <= FAST_ATTACH_DICT_CUTOFF); + .reset_size_log + .is_none_or(|log| log <= FAST_ATTACH_DICT_CUTOFF_LOG); if attach { self.simple_mut().skip_matching_for_dict_prime(); } else { @@ -1124,9 +1199,12 @@ impl Matcher for MatchGeneratorDriver { fn reset(&mut self, level: CompressionLevel) { let hint = self.source_size_hint.take(); - // Snapshot for prime_with_dictionary's attach/copy mode decision (the - // hint is consumed here, but priming happens just after reset). - self.reset_source_size = hint; + // Snapshot the hint's normalized ceil-log bucket for the primed-snapshot + // key and prime_with_dictionary's attach/copy mode decision (the hint is + // consumed here, but priming happens just after reset). Storing the + // bucket rather than the raw bytes means two hints that resolve to the + // same matcher shape share one snapshot instead of each re-priming. + self.reset_size_log = hint.map(source_size_ceil_log); let hinted = hint.is_some(); #[cfg_attr(not(test), allow(unused_mut))] let mut params = Self::level_params(level, hint); @@ -1250,11 +1328,7 @@ impl Matcher for MatchGeneratorDriver { // small frame zeroes a small table; it never exceeds the real window. let table_window_size = match hint { Some(h) => { - let raw_log = if h == 0 { - MIN_WINDOW_LOG - } else { - (64 - (h - 1).leading_zeros()) as u8 - }; + let raw_log = source_size_ceil_log(h); // Clamp the shift below the pointer width before `1usize <<`: // an oversized hint (>= 2^63 + 1, and on 32-bit usize any hint // >= 2^32) drives `raw_log` to 64 / >= 32, and the shift would @@ -1266,6 +1340,11 @@ impl Matcher for MatchGeneratorDriver { } None => max_window_size, }; + // The hint-dependent hash-table width the active backend applies, for + // the primed-snapshot key. Dfast/Row compute it from `table_window_size` + // below; HC/Fast leave it `0` because their widths live in `params` + // (`hc.{hash,chain}_log` / `fast_hash_log`) — already part of the key. + let mut resolved_table_bits: usize = 0; match &mut self.storage { MatcherStorage::Simple(m) => { // Per-level Fast cParams threaded from @@ -1287,11 +1366,12 @@ impl Matcher for MatchGeneratorDriver { | CompressionLevel::Level(0) | CompressionLevel::Level(3) ); - dfast.set_hash_bits(if hinted { + resolved_table_bits = if hinted { dfast_hash_bits_for_window(table_window_size) } else { DFAST_HASH_BITS - }); + }; + dfast.set_hash_bits(resolved_table_bits); // Dfast holds no per-block input Vecs (history owns the // bytes and `add_data` returns each Vec eagerly), so // `reset` takes no `reuse_space` callback. @@ -1302,7 +1382,8 @@ impl Matcher for MatchGeneratorDriver { row.lazy_depth = params.lazy_depth; row.configure(params.row); if hinted { - row.set_hash_bits(row_hash_bits_for_window(table_window_size)); + resolved_table_bits = row_hash_bits_for_window(table_window_size); + row.set_hash_bits(resolved_table_bits); } row.reset(); } @@ -1317,6 +1398,16 @@ impl Matcher for MatchGeneratorDriver { }); } } + // Record the resolved matcher shape for the primed-snapshot key. Captured + // here (post-resolution, after the test-only param override) so the key + // reflects exactly the geometry the restored `storage` must match. The + // Fast attach-vs-copy mode is part of the shape (it decides whether a + // dict table is built), and matches the decision `prime_with_dictionary` + // makes from the same `reset_size_log`. + let fast_attach = self + .reset_size_log + .is_none_or(|log| log <= FAST_ATTACH_DICT_CUTOFF_LOG); + self.reset_shape = Some((params, resolved_table_bits, fast_attach)); } fn prime_with_dictionary(&mut self, dict_content: &[u8], offset_hist: [u32; 3]) { @@ -1490,11 +1581,25 @@ impl Matcher for MatchGeneratorDriver { fn restore_primed_dictionary(&mut self, level: super::CompressionLevel) -> bool { // Only the (storage, dictionary_retained_budget) pair is what // `prime_with_dictionary` writes; restoring them reproduces the - // post-prime state exactly. Gated on the captured level so a driver - // reused at a different level (different backend / params) re-primes - // instead of restoring a mismatched table. + // post-prime state exactly. Gated on the FULL resolved key (level + the + // resolved `LevelParams` + the active backend's table width), not just + // the level: `reset` resolves the hint into a window/table geometry, so a + // same-level snapshot taken at a hint that resolved to a different shape + // carries a `storage.max_window_size` / table dimensions that no longer + // match this reset. Restoring it would let the encoder search past the + // frame header's window (an undecodable match), so on a key mismatch we + // refuse and the caller re-primes. + let Some((params, table_bits, fast_attach)) = self.reset_shape else { + return false; + }; + let key = PrimedKey { + level, + params, + table_bits, + fast_attach, + }; let (storage, budget) = match &self.primed { - Some((storage, budget, captured_level)) if *captured_level == level => { + Some((storage, budget, captured_key)) if *captured_key == key => { (storage.clone(), *budget) } _ => return false, @@ -1505,7 +1610,18 @@ impl Matcher for MatchGeneratorDriver { } fn capture_primed_dictionary(&mut self, level: super::CompressionLevel) { - self.primed = Some((self.storage.clone(), self.dictionary_retained_budget, level)); + // No resolved shape means `reset` has not run for this frame — nothing + // valid to key a snapshot on, so skip the capture. + let Some((params, table_bits, fast_attach)) = self.reset_shape else { + return; + }; + let key = PrimedKey { + level, + params, + table_bits, + fast_attach, + }; + self.primed = Some((self.storage.clone(), self.dictionary_retained_budget, key)); } fn invalidate_primed_dictionary(&mut self) { @@ -6787,6 +6903,152 @@ fn prime_with_dictionary_does_not_inflate_reported_window_size() { ); } +#[test] +fn primed_snapshot_not_restored_when_window_hint_differs() { + // The copy-snapshot must be keyed on the resolved reset parameters, not + // just the CompressionLevel. `reset()` caps window_log by the source-size + // hint, so two same-level frames with different hints resolve to different + // windows. Restoring a snapshot captured at the larger hint into a reset + // for the smaller hint would advertise the smaller window in the frame + // header while the matcher's `max_window_size` (from the restored storage) + // still spans the larger window — the encoder could then emit a match + // (e.g. into the dictionary) past the advertised window, producing an + // undecodable frame. Restore must REFUSE when the resolved window differs. + let mut driver = MatchGeneratorDriver::new(8, 1); + let level = CompressionLevel::Best; + + // Frame A: large hint → larger resolved window. Prime + capture. + driver.set_source_size_hint(256 * 1024); + driver.reset(level); + let big_window = driver.window_size(); + driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); + driver.capture_primed_dictionary(level); + + // Frame B: smaller hint, SAME level → smaller resolved window. + driver.set_source_size_hint(48 * 1024); + driver.reset(level); + let small_window = driver.window_size(); + assert!( + small_window < big_window, + "precondition: the two hints must resolve to different windows \ + (small={small_window}, big={big_window})" + ); + + let restored = driver.restore_primed_dictionary(level); + assert!( + !restored, + "snapshot captured at window {big_window} must NOT be restored into a \ + reset advertising window {small_window} (level alone is an insufficient key)" + ); +} + +#[test] +fn primed_snapshot_restored_for_hints_in_same_window_bucket() { + // The snapshot key must normalize the source-size hint to the resolved + // matcher geometry, not the raw hinted byte count. `reset()` derives every + // hint-dependent parameter (window_log cap, HC/Fast/Dfast/Row table widths, + // the Fast attach-vs-copy cutoff) from `ceil_log2(hint)`, so two distinct + // hints that share a ceil-log bucket resolve to the *identical* matcher + // shape. Keying on the raw bytes over-keys: it forces a full re-prime on the + // second frame even though the cached snapshot is a perfect fit. Restore + // must SUCCEED across same-bucket hints. + let mut driver = MatchGeneratorDriver::new(8, 1); + let level = CompressionLevel::Best; + + // Both hints fall in ceil_log2 bucket 19 (2^18 < n <= 2^19): 300 KiB and + // 400 KiB resolve to the same window and table widths. + driver.set_source_size_hint(300 * 1024); + driver.reset(level); + let window_a = driver.window_size(); + driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); + driver.capture_primed_dictionary(level); + + driver.set_source_size_hint(400 * 1024); + driver.reset(level); + let window_b = driver.window_size(); + assert_eq!( + window_a, window_b, + "precondition: same-bucket hints must resolve to the same window \ + (a={window_a}, b={window_b})" + ); + + let restored = driver.restore_primed_dictionary(level); + assert!( + restored, + "snapshot captured at a 300 KiB hint must be restored into a 400 KiB \ + hint that resolves to the identical matcher shape (raw bytes over-key)" + ); +} + +#[test] +fn primed_snapshot_restored_across_level22_donor_tier_hints() { + // Level 22 collapses several ceil-log buckets onto one donor source-size + // tier: `resolve_level_params(Level(22), ..)` selects the HC config and + // window_log by raw `<= 16 KiB / 128 KiB / 256 KiB` thresholds, so a 20 KiB + // and a 100 KiB hint (ceil-log buckets 15 and 17) both land in the + // `<= 128 KiB` tier and resolve to the IDENTICAL matcher (same window_log, + // same HC hash/chain/search geometry). Keying on the raw ceil-log bucket + // would still reject the restore here because the buckets differ; the key + // must compare the resolved matcher shape so these share one snapshot. + let mut driver = MatchGeneratorDriver::new(8, 1); + let level = CompressionLevel::Level(22); + + driver.set_source_size_hint(20 * 1024); + driver.reset(level); + let window_a = driver.window_size(); + driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); + driver.capture_primed_dictionary(level); + + driver.set_source_size_hint(100 * 1024); + driver.reset(level); + let window_b = driver.window_size(); + assert_eq!( + window_a, window_b, + "precondition: both hints must land in the same Level 22 donor tier \ + (a={window_a}, b={window_b})" + ); + + let restored = driver.restore_primed_dictionary(level); + assert!( + restored, + "Level 22 snapshot captured at a 20 KiB hint must be restored into a \ + 100 KiB hint that resolves to the same donor tier (different ceil-log \ + buckets, identical matcher shape)" + ); +} + +#[test] +fn primed_snapshot_not_restored_across_fast_attach_copy_boundary() { + // The Fast attach-vs-copy cutoff (8 KiB) falls INSIDE a single resolved + // matcher shape: a 8192-byte and a 8193-byte hint both clamp Level 1 to + // window_log 14 and the same Fast table widths, so `LevelParams` + + // `table_bits` are identical, yet 8192 attaches (separate dict table) while + // 8193 copies (dict primed into the live table). The snapshot key must + // therefore carry the attach/copy mode itself; without it the two resets + // would share a key and a copy-mode snapshot could be restored into an + // attach-mode reset (a different `storage` shape). Restore must REFUSE + // across the boundary. + let mut driver = MatchGeneratorDriver::new(8, 1); + let level = CompressionLevel::Level(1); + + // Copy side (hint > 8 KiB): prime + capture. + driver.set_source_size_hint(8193); + driver.reset(level); + driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); + driver.capture_primed_dictionary(level); + + // Attach side (hint <= 8 KiB), same resolved window/table shape. + driver.set_source_size_hint(8192); + driver.reset(level); + let restored = driver.restore_primed_dictionary(level); + assert!( + !restored, + "a copy-mode snapshot (8193 B hint) must NOT be restored into an \ + attach-mode reset (8192 B hint) that resolves to the same params but a \ + different dict-table shape" + ); +} + #[cfg(any())] // disabled: tested SuffixStore-per-block tail-handling specific to legacy MatchGenerator #[test] fn prime_with_dictionary_does_not_reuse_tiny_suffix_store() { From 8c1e350ee8d0143c801870adcf6258337b2f665d Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 6 Jun 2026 12:01:27 +0300 Subject: [PATCH 15/33] perf(decode): single-compare sequence bounds check + per-block bomb ceiling Extract the per-sequence output-capacity guard into sequence_output_fits in buffer_backend, used by all six inline exec_sequence variants (FlatBuf and UserSliceBackend, each SSE2 / portable / AVX2). Replaces the per-variant checked_add chain with one subtraction and one compare on plain arithmetic: lit_length and match_length are each bounded by the maximum FSE LL/ML code expansion (~131 KB) so the sums cannot overflow usize even on 32-bit, and tail <= cap holds on entry (Vec len <= capacity for the flat buffer; the user-slice tail only advances past this same check) so cap - tail cannot underflow. Complete the decompression-bomb ceiling on the growable backends: - FlatBuf gains a max_capacity field and overrides set_max_capacity + try_reserve so its fallback push/repeat path (taken when the inline slack gate fails) is bounded, mirroring RingBuffer. - Both FlatBuf and RingBuffer check the ceiling BEFORE the no-growth fast path, so a write that fits existing capacity (a large pre-reserved FCS buffer, or an over-allocated ring) but exceeds the per-block ceiling is rejected rather than silently admitted. - repeat_from_dict now try_reserves before appending dictionary bytes, so a match satisfied from the dictionary is bounded too instead of bypassing the guard via a direct extend. Length sums in these growth paths use checked/plain arithmetic, not saturating_add, so a broken bound fails at its cause. Adds regression tests for the AVX2 FlatBuf overflow, FlatBuf reserve past the ceiling both with and without growth, and dict-backed (full and mixed) matches past the ceiling; the overflow tests size their literal buffer to lit_length.next_multiple_of(16). --- zstd/src/decoding/buffer_backend.rs | 50 +++++- zstd/src/decoding/decode_buffer.rs | 73 +++++++- zstd/src/decoding/flat_buf.rs | 253 +++++++++++++++++++--------- zstd/src/decoding/ringbuffer.rs | 30 ++-- zstd/src/decoding/user_slice_buf.rs | 117 ++++--------- 5 files changed, 337 insertions(+), 186 deletions(-) diff --git a/zstd/src/decoding/buffer_backend.rs b/zstd/src/decoding/buffer_backend.rs index 504d2a623..e76c27806 100644 --- a/zstd/src/decoding/buffer_backend.rs +++ b/zstd/src/decoding/buffer_backend.rs @@ -39,6 +39,44 @@ use crate::io::{Error, Read}; /// slack contract cannot drift between backends. pub(crate) const WILDCOPY_OVERLENGTH: usize = 32; +/// Single-compare output-capacity guard for the inline sequence-exec hot +/// path, shared by every [`BufferBackend::exec_sequence_inline`] / +/// `exec_sequence_inline_avx2` override so the per-sequence bounds check has +/// one implementation instead of a duplicated `checked_add` chain. +/// +/// Returns `lit_length + match_length` when the literal+match write plus +/// `overshoot` bytes of SIMD wildcopy slack fits within `cap - tail`; +/// otherwise [`ExecuteSequencesError::OutputBufferOverflow`](super::errors::ExecuteSequencesError::OutputBufferOverflow). +/// +/// # Preconditions +/// - `tail <= cap`, so `cap - tail` cannot underflow. Holds for every +/// backend: `Vec::len() <= Vec::capacity()` on the flat / growable buffers, +/// and the user-slice tail only ever advances past this same check. +/// - `lit_length` and `match_length` are each bounded by the maximum FSE +/// LL/ML code expansion (~131 KB), so `total` and `total + overshoot` +/// cannot overflow `usize` even on 32-bit. +/// +/// Each caller documents why the first precondition holds at its site; the +/// arithmetic safety of the second is the same FSE bound everywhere. +#[inline(always)] +pub(crate) fn sequence_output_fits( + lit_length: usize, + match_length: usize, + tail: usize, + cap: usize, + overshoot: usize, +) -> Result { + let total = lit_length + match_length; + if total + overshoot > cap - tail { + return Err(super::errors::ExecuteSequencesError::OutputBufferOverflow { + tail, + requested: total, + capacity: cap, + }); + } + Ok(total) +} + /// Storage operations the decoder needs from its output buffer. /// /// The trait surface mirrors the historical `RingBuffer` API the @@ -227,12 +265,12 @@ pub(crate) trait BufferBackend: Sized { } /// Lower the per-block growth ceiling on backends whose `try_reserve` - /// may grow without bound (the streaming `RingBuffer`). The block - /// sequence decoder sets it to `len + MAX_BLOCK_SIZE` per block so an - /// over-producing match is rejected on the cold growth path, - /// bounding decompression-bomb OOMs. Default no-op: fixed-capacity - /// backends are already bounded, and the inline-exec (FCS-capped) - /// path never grows through `try_reserve`. + /// may grow without bound. The block sequence decoder sets it to + /// `len + MAX_BLOCK_SIZE` per block so an over-producing match is rejected + /// on the cold growth path, bounding decompression-bomb OOMs. Overridden by + /// the streaming `RingBuffer` and the growable `FlatBuf` (whose fallback + /// `push`/`repeat` path grows through `try_reserve`). Default no-op for + /// fixed-capacity backends (`UserSliceBackend`), which are already bounded. fn set_max_capacity(&mut self, _max_capacity: usize) {} /// Live byte count: bytes between the logical head and tail. diff --git a/zstd/src/decoding/decode_buffer.rs b/zstd/src/decoding/decode_buffer.rs index faaaee601..2860e6b4d 100644 --- a/zstd/src/decoding/decode_buffer.rs +++ b/zstd/src/decoding/decode_buffer.rs @@ -120,11 +120,18 @@ impl DecodeBuffer { /// cold growth path instead of growing the ring to gigabytes (a /// decompression-bomb OOM) before the post-block validity check runs. /// Called once per block by the sequence decoder, before the sequence - /// loop. Backends that cannot grow unbounded (fixed-capacity, or the - /// inline-exec FCS-capped path) take the trait's no-op default. + /// loop. The growable backends (`RingBuffer`, `FlatBuf`) enforce it in + /// `try_reserve`; fixed-capacity backends (`UserSliceBackend`) are already + /// bounded and take the trait's no-op default. #[inline] pub(crate) fn set_block_output_ceiling(&mut self, max_block_output: usize) { - let ceiling = self.buffer.len().saturating_add(max_block_output); + // Plain add (not saturating): `len()` is the bytes decoded so far and + // `max_block_output` is one block's `MAX_BLOCK_SIZE` (128 KiB), so the + // sum is nowhere near `usize::MAX` — overflow is unreachable. Saturating + // would silently turn the ceiling into `usize::MAX` (no guard at all) + // if that invariant were ever broken, masking the bug instead of + // tripping the debug overflow check at its cause. + let ceiling = self.buffer.len() + max_block_output; self.buffer.set_max_capacity(ceiling); } @@ -788,6 +795,21 @@ impl DecodeBuffer { }); } + // Enforce the per-block bomb ceiling on the dictionary-backed + // output too: the `extend` below appends `dict_slice` directly + // (the inline `push`/`repeat` guard never runs for it), so without + // this an over-producing match satisfied from the dictionary could + // grow past the armed ceiling. Reserving the full `match_length` + // covers both the dict portion here and the buffer-history + // remainder the recursive `repeat` appends. + self.buffer.try_reserve(match_length).map_err(|o| { + DecodeBufferError::OutputBufferOverflow { + tail: o.tail, + requested: o.requested, + capacity: o.capacity, + } + })?; + if bytes_from_dict < match_length { let dict_slice = &dict_content[dict_len - bytes_from_dict..]; prefetch::prefetch_slice(dict_slice); @@ -1335,6 +1357,51 @@ mod tests { assert_eq!(decode_buf.len(), 14); } + #[test] + fn repeat_from_dict_rejects_output_past_block_ceiling() { + // A match satisfied (fully or partially) from the dictionary appends + // `dict_slice` directly via `buffer.extend`, bypassing the inline + // push/repeat guard. The per-block bomb ceiling must still bound it, so + // a dict-backed over-producing match returns `OutputBufferOverflow` + // instead of growing the buffer toward OOM. + let dict = || { + crate::decoding::dictionary::DictionaryHandle::from_dictionary( + crate::decoding::dictionary::Dictionary::from_raw_content( + 1, + alloc::vec![0xABu8; 256], + ) + .unwrap(), + ) + }; + + // Fully-dictionary match: empty buffer, offset reaches into the dict. + let mut full = DecodeBuffer::::new(4 * 1024); + full.set_dict(dict()); + full.set_block_output_ceiling(8); // max_capacity = 0 + 8 = 8 + let err = full.repeat(200, 100).unwrap_err(); // 0 + 100 > 8, all from dict + assert!( + matches!( + err, + crate::decoding::errors::DecodeBufferError::OutputBufferOverflow { .. } + ), + "fully-dictionary over-producing match must be rejected, got {err:?}" + ); + + // Mixed match: part from dict, remainder from buffer history. + let mut mixed = DecodeBuffer::::new(4 * 1024); + mixed.set_dict(dict()); + mixed.push(b"abcd"); // len = 4 + mixed.set_block_output_ceiling(8); // max_capacity = 4 + 8 = 12 + let err = mixed.repeat(10, 100).unwrap_err(); // 6 from dict + rest, 4 + 100 > 12 + assert!( + matches!( + err, + crate::decoding::errors::DecodeBufferError::OutputBufferOverflow { .. } + ), + "mixed dict+buffer over-producing match must be rejected, got {err:?}" + ); + } + #[test] fn repeat_from_dict_full_copy_updates_total_output_counter() { let mut decode_buf = DecodeBuffer::::new(1); diff --git a/zstd/src/decoding/flat_buf.rs b/zstd/src/decoding/flat_buf.rs index 0d981d3fc..e97cc3659 100644 --- a/zstd/src/decoding/flat_buf.rs +++ b/zstd/src/decoding/flat_buf.rs @@ -17,7 +17,7 @@ use crate::io::{Error, Read}; use alloc::vec::Vec; -use super::buffer_backend::{BufferBackend, WILDCOPY_OVERLENGTH}; +use super::buffer_backend::{BackendOverflow, BufferBackend, WILDCOPY_OVERLENGTH}; pub(crate) struct FlatBuf { buf: Vec, @@ -42,6 +42,14 @@ pub(crate) struct FlatBuf { /// path can't observe a streaming-drain scenario where the /// distinction would matter. head: usize, + /// Per-block decompression-bomb growth ceiling, mirroring + /// [`super::ringbuffer::RingBuffer`]. `usize::MAX` is unbounded; the + /// sequence decoder lowers it to `len + MAX_BLOCK_SIZE` per block via + /// [`BufferBackend::set_max_capacity`]. `FlatBuf` is growable (the inline + /// exec path is capacity-bounded, but the fallback `push`/`repeat` route + /// grows through `try_reserve`), so it must honour this ceiling to bound a + /// malformed single-segment block's output instead of growing toward OOM. + max_capacity: usize, } impl FlatBuf { @@ -59,6 +67,7 @@ impl FlatBuf { Self { buf: Vec::with_capacity(cap + WILDCOPY_OVERLENGTH), head: 0, + max_capacity: usize::MAX, } } } @@ -86,7 +95,7 @@ impl BufferBackend for FlatBuf { offset: usize, match_length: usize, ) -> Result<(), super::errors::ExecuteSequencesError> { - use super::errors::ExecuteSequencesError; + use super::buffer_backend::sequence_output_fits; use super::exec_sequence_inline::x86::{ copy16, overlap_copy8, wildcopy_no_overlap, wildcopy_overlap_8byte_stride, }; @@ -98,35 +107,19 @@ impl BufferBackend for FlatBuf { // headroom. Surface that as `OutputBufferOverflow` (mirrors // `UserSliceBackend::exec_sequence_inline`) so the safe // public decode APIs see a structured error instead of UB - // from writing past `Vec::capacity()`. All sums use - // `checked_*` against adversarial input that could wrap - // `usize`. + // from writing past `Vec::capacity()`. const MAX_WILDCOPY_OVERSHOOT: usize = 15; let cap = self.buf.capacity(); let buf_len = self.buf.len(); - let total = match lit_length.checked_add(match_length) { - Some(v) => v, - None => { - return Err(ExecuteSequencesError::OutputBufferOverflow { - tail: buf_len, - requested: usize::MAX, - capacity: cap, - }); - } - }; - let cap_required = buf_len - .checked_add(total) - .and_then(|new_tail| new_tail.checked_add(MAX_WILDCOPY_OVERSHOOT)); - match cap_required { - Some(v) if v <= cap => {} - _ => { - return Err(ExecuteSequencesError::OutputBufferOverflow { - tail: buf_len, - requested: total, - capacity: cap, - }); - } - } + // `buf.len() <= buf.capacity()` (a `Vec` invariant) satisfies the + // `tail <= cap` precondition; see `sequence_output_fits`. + let total = sequence_output_fits( + lit_length, + match_length, + buf_len, + cap, + MAX_WILDCOPY_OVERSHOOT, + )?; debug_assert!(offset >= 1); debug_assert!(match_length >= 1); let live_len = buf_len - self.head; @@ -181,7 +174,7 @@ impl BufferBackend for FlatBuf { offset: usize, match_length: usize, ) -> Result<(), super::errors::ExecuteSequencesError> { - use super::errors::ExecuteSequencesError; + use super::buffer_backend::sequence_output_fits; use super::exec_sequence_inline::portable::{ copy16, overlap_copy8, wildcopy_no_overlap, wildcopy_overlap_8byte_stride, }; @@ -193,29 +186,15 @@ impl BufferBackend for FlatBuf { const MAX_WILDCOPY_OVERSHOOT: usize = 15; let cap = self.buf.capacity(); let buf_len = self.buf.len(); - let total = match lit_length.checked_add(match_length) { - Some(v) => v, - None => { - return Err(ExecuteSequencesError::OutputBufferOverflow { - tail: buf_len, - requested: usize::MAX, - capacity: cap, - }); - } - }; - let cap_required = buf_len - .checked_add(total) - .and_then(|new_tail| new_tail.checked_add(MAX_WILDCOPY_OVERSHOOT)); - match cap_required { - Some(v) if v <= cap => {} - _ => { - return Err(ExecuteSequencesError::OutputBufferOverflow { - tail: buf_len, - requested: total, - capacity: cap, - }); - } - } + // `buf.len() <= buf.capacity()` (a `Vec` invariant) satisfies the + // `tail <= cap` precondition; see `sequence_output_fits`. + let total = sequence_output_fits( + lit_length, + match_length, + buf_len, + cap, + MAX_WILDCOPY_OVERSHOOT, + )?; debug_assert!(offset >= 1); debug_assert!(match_length >= 1); let live_len = buf_len - self.head; @@ -271,7 +250,7 @@ impl BufferBackend for FlatBuf { offset: usize, match_length: usize, ) -> Result<(), super::errors::ExecuteSequencesError> { - use super::errors::ExecuteSequencesError; + use super::buffer_backend::sequence_output_fits; use super::exec_sequence_inline::x86::{ copy16, overlap_copy8, wildcopy_no_overlap, wildcopy_no_overlap_avx2, wildcopy_overlap_8byte_stride, @@ -285,29 +264,15 @@ impl BufferBackend for FlatBuf { const MAX_WILDCOPY_OVERSHOOT: usize = 31; let cap = self.buf.capacity(); let buf_len = self.buf.len(); - let total = match lit_length.checked_add(match_length) { - Some(v) => v, - None => { - return Err(ExecuteSequencesError::OutputBufferOverflow { - tail: buf_len, - requested: usize::MAX, - capacity: cap, - }); - } - }; - let cap_required = buf_len - .checked_add(total) - .and_then(|new_tail| new_tail.checked_add(MAX_WILDCOPY_OVERSHOOT)); - match cap_required { - Some(v) if v <= cap => {} - _ => { - return Err(ExecuteSequencesError::OutputBufferOverflow { - tail: buf_len, - requested: total, - capacity: cap, - }); - } - } + // `buf.len() <= buf.capacity()` (a `Vec` invariant) satisfies the + // `tail <= cap` precondition; see `sequence_output_fits`. + let total = sequence_output_fits( + lit_length, + match_length, + buf_len, + cap, + MAX_WILDCOPY_OVERSHOOT, + )?; debug_assert!(offset >= 1); debug_assert!(match_length >= 1); let live_len = buf_len - self.head; @@ -351,6 +316,7 @@ impl BufferBackend for FlatBuf { Self { buf: Vec::new(), head: 0, + max_capacity: usize::MAX, } } @@ -380,7 +346,58 @@ impl BufferBackend for FlatBuf { // `dst_off + len <= capacity` debug assert. // libFuzzer artifact crash-e33ba082… exercises exactly that // shape. - self.buf.reserve(n.saturating_add(WILDCOPY_OVERLENGTH)); + // + // `checked_add` (not `saturating_add`): callers reserve a bounded + // amount (a block's `MAX_BLOCK_SIZE`, or a `try_reserve` amount already + // gated by the per-block ceiling), so `n + WILDCOPY_OVERLENGTH` cannot + // overflow in practice. If it ever did, saturating to `usize::MAX` + // would silently mask the bad input before `Vec::reserve` panicked on + // it anyway — fail loudly at the cause instead. + let additional = n + .checked_add(WILDCOPY_OVERLENGTH) + .expect("FlatBuf::reserve amount + wildcopy slack overflows usize"); + self.buf.reserve(additional); + } + + #[inline] + fn set_max_capacity(&mut self, max_capacity: usize) { + self.max_capacity = max_capacity; + } + + #[inline] + fn try_reserve(&mut self, n: usize) -> Result<(), BackendOverflow> { + // Enforce the per-block ceiling FIRST, before the no-growth fast path: + // the ceiling bounds this block's OUTPUT, not just allocation, so a + // write that would push live output past it must be rejected even when + // it fits the current capacity (e.g. a large pre-reserved FCS buffer + // whose spare exceeds `MAX_BLOCK_SIZE`). This is where the + // decompression bomb is bounded on FlatBuf's fallback push/repeat path + // (the inline exec path is already capacity-bounded by + // `sequence_output_fits`). `max_capacity = usize::MAX` between blocks + // makes this a no-op for unbounded callers. + if self + .len() + .checked_add(n) + .is_none_or(|needed| needed > self.max_capacity) + { + return Err(BackendOverflow { + tail: self.buf.len(), + requested: n, + capacity: self.max_capacity, + }); + } + // Within the ceiling: if the live region plus this write (and the + // wildcopy slack `reserve` adds) already fits the current allocation, + // no growth is needed. `capacity >= len` is a `Vec` invariant, so the + // subtraction cannot underflow. + let free = self.buf.capacity() - self.buf.len(); + if n.checked_add(WILDCOPY_OVERLENGTH) + .is_some_and(|need| free >= need) + { + return Ok(()); + } + self.reserve(n); + Ok(()) } #[inline] @@ -697,8 +714,11 @@ mod tests { let mut f = FlatBuf::with_capacity(32); f.extend(&[0u8; 16]); // Request `lit_length + match_length + 15 = 17 + 100 + 15 = 132` - // bytes past tail; well over the 64-byte allocation. - let lits = [0xAAu8; 16]; + // bytes past tail; well over the 64-byte allocation. The literal + // buffer is `lit_length.next_multiple_of(16) = 32` bytes so the call + // satisfies the inline read-slack precondition even if the capacity + // guard later moves past the first literal read. + let lits = [0xAAu8; 32]; // SAFETY: error-returning path; no writes performed. let result = unsafe { f.exec_sequence_inline(lits.as_ptr(), 17, 8, 100) }; assert!( @@ -709,4 +729,79 @@ mod tests { "expected OutputBufferOverflow, got {result:?}" ); } + + /// AVX2 analogue of the capacity guard: the 32-byte-stride variant MUST + /// also return `OutputBufferOverflow` (not write past `Vec::capacity()`) + /// when the requested write plus the 31-byte overshoot exceeds the + /// remaining headroom. Guards the single-compare bounds check on the AVX2 + /// hot path. + #[cfg(target_arch = "x86_64")] + #[test] + fn exec_sequence_inline_avx2_capacity_overflow_returns_err() { + if !std::arch::is_x86_feature_detected!("avx2") { + return; + } + let mut f = FlatBuf::with_capacity(32); + f.extend(&[0u8; 16]); + // 32-byte literal buffer = `lit_length.next_multiple_of(16)`, so the + // call satisfies the inline read-slack precondition even if the guard + // later moves past the first literal read. + let lits = [0xAAu8; 32]; + // SAFETY: AVX2 detected above; error-returning path performs no writes. + let result = unsafe { f.exec_sequence_inline_avx2(lits.as_ptr(), 17, 8, 100) }; + assert!( + matches!( + result, + Err(super::super::errors::ExecuteSequencesError::OutputBufferOverflow { .. }) + ), + "expected OutputBufferOverflow, got {result:?}" + ); + } + + /// `FlatBuf` is growable, so the per-block decompression-bomb ceiling + /// (`set_max_capacity`) MUST be honoured on its `try_reserve` growth path — + /// the fallback `push`/`repeat` route a malformed single-segment block can + /// take when the inline slack gate fails. A reserve that would grow live + /// output past the ceiling must return `BackendOverflow` instead of growing + /// the `Vec` toward a decompression-bomb OOM. + #[test] + fn try_reserve_rejects_growth_past_block_ceiling() { + let mut f = FlatBuf::with_capacity(64); + f.extend(&[0u8; 32]); + let ceiling = f.len() + 100; // 132 + f.set_max_capacity(ceiling); + // Within the ceiling: succeeds (32 + 50 = 82 <= 132). + assert!(f.try_reserve(50).is_ok()); + // Past the ceiling (32 + 200 = 232 > 132): rejected, no growth. + assert!( + matches!( + f.try_reserve(200), + Err(super::super::buffer_backend::BackendOverflow { .. }) + ), + "reserve past the per-block ceiling must be rejected, not grown" + ); + } + + /// The ceiling bounds this block's OUTPUT, so a reserve past it must be + /// rejected even when it FITS the existing allocation (no growth needed). + /// A large pre-reserved buffer (e.g. a known-FCS single-segment frame) has + /// spare capacity beyond `MAX_BLOCK_SIZE`; without checking the ceiling + /// ahead of the no-growth fast path, an over-producing block could write + /// into that spare and bypass the decompression-bomb guard. + #[test] + fn try_reserve_rejects_within_capacity_but_past_ceiling() { + let mut f = FlatBuf::with_capacity(4096); + f.extend(&[0u8; 32]); + let ceiling = f.len() + 100; // 132, far below the 4 KiB allocation + f.set_max_capacity(ceiling); + // 32 + 500 = 532 <= 4096 capacity (no growth) but > 132 ceiling. + assert!( + matches!( + f.try_reserve(500), + Err(super::super::buffer_backend::BackendOverflow { .. }) + ), + "a reserve past the ceiling must be rejected even when it fits the \ + current capacity without growth" + ); + } } diff --git a/zstd/src/decoding/ringbuffer.rs b/zstd/src/decoding/ringbuffer.rs index f22c24bbe..db6b1ad1d 100644 --- a/zstd/src/decoding/ringbuffer.rs +++ b/zstd/src/decoding/ringbuffer.rs @@ -202,19 +202,13 @@ impl RingBuffer { &mut self, amount: usize, ) -> Result<(), super::buffer_backend::BackendOverflow> { - // Flat fast path (mirrors `reserve`): no wrap and the write fits - // below `cap` — capacity already present, no growth, no ceiling - // check. - if self.head <= self.tail && amount < self.cap.saturating_sub(self.tail) { - return Ok(()); - } - let free = self.free(); - if free >= amount { - return Ok(()); - } - // Growth is required. Reject if it would cross the per-block - // ceiling before allocating (bounds the bomb on every target, - // unlike a 32-bit-only assert). + // Enforce the per-block ceiling FIRST, before the no-growth fast + // paths: the ceiling bounds this block's OUTPUT, not just allocation, + // so a write past it must be rejected even when it fits the ring's + // current capacity (a large window or an over-allocated ring can have + // more than `MAX_BLOCK_SIZE` of slack). Bounds the bomb on every target + // unlike a 32-bit-only assert; `max_capacity = usize::MAX` between + // blocks makes this a no-op for unbounded callers. if self .len() .checked_add(amount) @@ -226,6 +220,16 @@ impl RingBuffer { capacity: self.max_capacity, }); } + // Within the ceiling: fast paths when capacity is already present. + // Flat fast path (mirrors `reserve`): no wrap and the write fits below + // `cap` — capacity already present, no growth. + if self.head <= self.tail && amount < self.cap.saturating_sub(self.tail) { + return Ok(()); + } + let free = self.free(); + if free >= amount { + return Ok(()); + } self.reserve_amortized(amount - free); Ok(()) } diff --git a/zstd/src/decoding/user_slice_buf.rs b/zstd/src/decoding/user_slice_buf.rs index 50e14aee5..37d843687 100644 --- a/zstd/src/decoding/user_slice_buf.rs +++ b/zstd/src/decoding/user_slice_buf.rs @@ -169,7 +169,7 @@ impl<'a> BufferBackend for UserSliceBackend<'a> { offset: usize, match_length: usize, ) -> Result<(), super::errors::ExecuteSequencesError> { - use super::errors::ExecuteSequencesError; + use super::buffer_backend::sequence_output_fits; use super::exec_sequence_inline::x86::{ copy16, overlap_copy8, wildcopy_no_overlap, wildcopy_overlap_8byte_stride, }; @@ -190,39 +190,16 @@ impl<'a> BufferBackend for UserSliceBackend<'a> { // unsafe pointer math go out of bounds. const MAX_WILDCOPY_OVERSHOOT: usize = 15; let cap = self.slice.len(); - // `requested` reports the LOGICAL write length - // (`lit_length + match_length`) to stay consistent with - // `BackendOverflow.requested` on the `try_*` paths. The - // capacity check itself uses `tail + total + overshoot` - // because the unconditional 16-byte `copy16` over-reaches - // `tail + total` by up to 15 bytes — but that overshoot is - // an artefact of the SIMD copy shape, NOT a value the - // caller can act on, so it doesn't belong in the diagnostic. - let total = match lit_length.checked_add(match_length) { - Some(v) => v, - None => { - return Err(ExecuteSequencesError::OutputBufferOverflow { - tail: self.tail, - requested: usize::MAX, - capacity: cap, - }); - } - }; - let cap_required = self - .tail - .checked_add(total) - .and_then(|new_tail| new_tail.checked_add(MAX_WILDCOPY_OVERSHOOT)); - let cap_required = match cap_required { - Some(v) if v <= cap => v, - _ => { - return Err(ExecuteSequencesError::OutputBufferOverflow { - tail: self.tail, - requested: total, - capacity: cap, - }); - } - }; - let _ = cap_required; + // `self.tail <= cap` holds on entry (`from_slice` starts at 0 and + // every prior sequence advanced `tail` only after this same check), + // satisfying the `tail <= cap` precondition; see `sequence_output_fits`. + let total = sequence_output_fits( + lit_length, + match_length, + self.tail, + cap, + MAX_WILDCOPY_OVERSHOOT, + )?; let new_tail = self.tail + total; debug_assert!(offset >= 1); debug_assert!(match_length >= 1); @@ -303,36 +280,21 @@ impl<'a> BufferBackend for UserSliceBackend<'a> { offset: usize, match_length: usize, ) -> Result<(), super::errors::ExecuteSequencesError> { - use super::errors::ExecuteSequencesError; + use super::buffer_backend::sequence_output_fits; use super::exec_sequence_inline::portable::{ copy16, overlap_copy8, wildcopy_no_overlap, wildcopy_overlap_8byte_stride, }; const MAX_WILDCOPY_OVERSHOOT: usize = 15; let cap = self.slice.len(); - let total = match lit_length.checked_add(match_length) { - Some(v) => v, - None => { - return Err(ExecuteSequencesError::OutputBufferOverflow { - tail: self.tail, - requested: usize::MAX, - capacity: cap, - }); - } - }; - let cap_required = self - .tail - .checked_add(total) - .and_then(|new_tail| new_tail.checked_add(MAX_WILDCOPY_OVERSHOOT)); - match cap_required { - Some(v) if v <= cap => {} - _ => { - return Err(ExecuteSequencesError::OutputBufferOverflow { - tail: self.tail, - requested: total, - capacity: cap, - }); - } - } + // `self.tail <= cap` precondition holds as in the SSE2 arm; see + // `sequence_output_fits`. + let total = sequence_output_fits( + lit_length, + match_length, + self.tail, + cap, + MAX_WILDCOPY_OVERSHOOT, + )?; let new_tail = self.tail + total; debug_assert!(offset >= 1); debug_assert!(match_length >= 1); @@ -413,7 +375,7 @@ impl<'a> BufferBackend for UserSliceBackend<'a> { offset: usize, match_length: usize, ) -> Result<(), super::errors::ExecuteSequencesError> { - use super::errors::ExecuteSequencesError; + use super::buffer_backend::sequence_output_fits; use super::exec_sequence_inline::x86::{ copy16, overlap_copy8, wildcopy_no_overlap, wildcopy_no_overlap_avx2, wildcopy_overlap_8byte_stride, @@ -425,31 +387,16 @@ impl<'a> BufferBackend for UserSliceBackend<'a> { // this overshoot for well-formed frames. const MAX_WILDCOPY_OVERSHOOT: usize = 31; let cap = self.slice.len(); - let total = match lit_length.checked_add(match_length) { - Some(v) => v, - None => { - return Err(ExecuteSequencesError::OutputBufferOverflow { - tail: self.tail, - requested: usize::MAX, - capacity: cap, - }); - } - }; - let cap_required = self - .tail - .checked_add(total) - .and_then(|new_tail| new_tail.checked_add(MAX_WILDCOPY_OVERSHOOT)); - let cap_required = match cap_required { - Some(v) if v <= cap => v, - _ => { - return Err(ExecuteSequencesError::OutputBufferOverflow { - tail: self.tail, - requested: total, - capacity: cap, - }); - } - }; - let _ = cap_required; + // `self.tail <= cap` holds on entry (`from_slice` starts at 0 and every + // prior sequence advanced `tail` only after this same check), satisfying + // the `tail <= cap` precondition; see `sequence_output_fits`. + let total = sequence_output_fits( + lit_length, + match_length, + self.tail, + cap, + MAX_WILDCOPY_OVERSHOOT, + )?; let new_tail = self.tail + total; debug_assert!(offset >= 1); debug_assert!(match_length >= 1); From f2fdb979cf78ba030c2af67bec2cf7fbe821698e Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 6 Jun 2026 05:35:09 +0300 Subject: [PATCH 16/33] perf(encode): donor block-split levels for fast/dfast/greedy/lazy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cheap fingerprint pre-splitter was wired only for greedy (1) and btopt+ (4); fast/dfast/lazy kept whole 128 KiB blocks. The C reference splits all of them (splitLevels[] = fast 0, dfast 1, greedy/lazy 2, btopt+ 4 in ZSTD_optimalBlockSize), cutting each full block at a statistical boundary so the per-block Huffman tree fits local literal statistics. On decodecorpus-z000033 that block-count gap (we emitted the minimum 8 blocks, the reference 14) cost ~1.6% on the literals section at the fast levels. - pre_split() now returns the donor splitLevels[] by strategy. - run_borrowed_block_loop (the Fast one-shot path) calls optimal_block_size per block instead of always emitting fixed 128 KiB blocks, so fast levels split too. savings = consumed - produced mirrors the donor gate (first block and incompressible input stay whole). Result (i9, z000033 compress, rust vs ffi bytes): L1 571077 vs 571525, L2 550576 vs 550910, L3 489251 vs 498911, L4 488705 vs 498591 — we now beat the reference at every level (was +1.6%/+1.3% behind at L1/L2). The pre-splitter, FSE table-mode selection, and HUF reuse decision are all already byte-faithful to the reference, so split sub-blocks reuse entropy tables via the same cost comparison. A stale unit assertion (better < default on a redundant 5 MiB fixture) was corrected: with block-splitting the reference itself produces better(L7) > default(L3) there (829 vs 525; ours 795 vs 495), so the size-ordering proxy was invalid; the test now asserts both levels reach <0.1% plus the round-trip. Adds examples/section_split.rs: splits a frame into literals/sequences section bytes per block for rust vs ffi, the diagnostic that localized the gap. Part of #316 --- zstd/examples/section_split.rs | 215 ++++++++++++++++++++++++++ zstd/src/encoding/frame_compressor.rs | 46 ++++-- zstd/src/encoding/match_generator.rs | 31 ++-- zstd/src/tests/roundtrip_integrity.rs | 18 ++- 4 files changed, 272 insertions(+), 38 deletions(-) create mode 100644 zstd/examples/section_split.rs diff --git a/zstd/examples/section_split.rs b/zstd/examples/section_split.rs new file mode 100644 index 000000000..6bbb4276a --- /dev/null +++ b/zstd/examples/section_split.rs @@ -0,0 +1,215 @@ +//! Diagnostic: split each compressed block into its literals section +//! (Huffman) and sequences section (FSE) byte counts, for the pure-Rust +//! encoder vs the C FFI encoder, on a fixed (corpus, level). When the +//! sequence streams are byte-identical (see compare_ffi_sequences) but the +//! final size differs, this localizes the gap to literals vs sequences. +//! +//! Build: cargo build --release -p structured-zstd --example section_split --features dict_builder +//! Run: ./target/release/examples/section_split [corpus] [level] + +use std::env; +use std::fs; + +use structured_zstd::encoding::{CompressionLevel, compress_slice_to_vec}; +use zstd::zstd_safe::zstd_sys; + +const MAGIC: u32 = 0xFD2F_B528; + +/// Parse the literals-section header at `body[0..]`; return +/// `(lit_section_total_bytes, lit_type)`. lit_type: 0=Raw 1=RLE +/// 2=Compressed 3=Treeless. +fn lit_section_len(body: &[u8]) -> (usize, u8) { + let b0 = body[0] as usize; + let lit_type = (b0 & 0x3) as u8; + let sf = (b0 >> 2) & 0x3; + match lit_type { + 0 | 1 => { + // Raw / RLE: 1/2/3-byte header carrying Regenerated_Size. + let (hdr, regen) = match sf { + 0 | 2 => (1usize, b0 >> 3), + 1 => (2, (b0 >> 4) | ((body[1] as usize) << 4)), + _ => ( + 3, + (b0 >> 4) | ((body[1] as usize) << 4) | ((body[2] as usize) << 12), + ), + }; + // Raw payload = regen bytes; RLE payload = 1 byte. + let payload = if lit_type == 0 { regen } else { 1 }; + (hdr + payload, lit_type) + } + _ => { + // Compressed / Treeless: size fields start at bit 4. + let (hdr, compressed) = match sf { + 0 | 1 => { + let v = (b0 >> 4) | ((body[1] as usize) << 4) | ((body[2] as usize) << 12); + (3, (v >> 10) & 0x3FF) + } + 2 => { + let v = (b0 >> 4) + | ((body[1] as usize) << 4) + | ((body[2] as usize) << 12) + | ((body[3] as usize) << 20); + (4, (v >> 14) & 0x3FFF) + } + _ => { + let v = (b0 >> 4) + | ((body[1] as usize) << 4) + | ((body[2] as usize) << 12) + | ((body[3] as usize) << 20) + | ((body[4] as usize) << 28); + (5, (v >> 18) & 0x3FFFF) + } + }; + (hdr + compressed, lit_type) + } + } +} + +/// Skip magic + frame header, returning the offset of the first block. +fn frame_header_len(frame: &[u8]) -> usize { + assert_eq!( + u32::from_le_bytes([frame[0], frame[1], frame[2], frame[3]]), + MAGIC, + "not a zstd frame" + ); + let fhd = frame[4]; + let single_segment = (fhd >> 5) & 1; + let checksum = (fhd >> 2) & 1; + let _ = checksum; + let dict_id_flag = fhd & 0x3; + let fcs_flag = (fhd >> 6) & 0x3; + let mut pos = 5usize; // magic(4) + FHD(1) + if single_segment == 0 { + pos += 1; // Window_Descriptor + } + pos += match dict_id_flag { + 0 => 0, + 1 => 1, + 2 => 2, + _ => 4, + }; + let fcs_bytes = match fcs_flag { + 0 => { + if single_segment == 1 { + 1 + } else { + 0 + } + } + 1 => 2, + 2 => 4, + _ => 8, + }; + pos + fcs_bytes +} + +struct Split { + blocks: usize, + raw_blocks: usize, + rle_blocks: usize, + comp_blocks: usize, + lit_bytes: usize, + seq_bytes: usize, + lit_type_counts: [usize; 4], +} + +fn analyze(frame: &[u8]) -> Split { + let mut pos = frame_header_len(frame); + let mut s = Split { + blocks: 0, + raw_blocks: 0, + rle_blocks: 0, + comp_blocks: 0, + lit_bytes: 0, + seq_bytes: 0, + lit_type_counts: [0; 4], + }; + loop { + let bh = + frame[pos] as u32 | ((frame[pos + 1] as u32) << 8) | ((frame[pos + 2] as u32) << 16); + let last = bh & 1; + let btype = (bh >> 1) & 0x3; + let bsize = (bh >> 3) as usize; + pos += 3; + s.blocks += 1; + match btype { + 0 => { + s.raw_blocks += 1; + pos += bsize; + } + 1 => { + s.rle_blocks += 1; + pos += 1; // physical RLE body is one byte + } + _ => { + s.comp_blocks += 1; + let body = &frame[pos..pos + bsize]; + let (lit_total, lit_type) = lit_section_len(body); + s.lit_type_counts[lit_type as usize] += 1; + s.lit_bytes += lit_total; + s.seq_bytes += bsize - lit_total; + pos += bsize; + } + } + if last == 1 { + break; + } + } + s +} + +fn print_split(label: &str, total: usize, s: &Split) { + println!( + "{label}: total={total} blocks={} (raw={} rle={} comp={}) lit_section={} seq_section={} lit_types[raw/rle/comp/treeless]={:?}", + s.blocks, + s.raw_blocks, + s.rle_blocks, + s.comp_blocks, + s.lit_bytes, + s.seq_bytes, + s.lit_type_counts + ); +} + +fn main() { + let corpus = env::args() + .nth(1) + .unwrap_or_else(|| "zstd/decodecorpus_files/z000033".to_string()); + let level: i32 = env::args().nth(2).and_then(|s| s.parse().ok()).unwrap_or(1); + let bytes = fs::read(&corpus).expect("read corpus"); + + let rust = compress_slice_to_vec(&bytes, CompressionLevel::Level(level)); + + let cap = unsafe { zstd_sys::ZSTD_compressBound(bytes.len()) }; + let mut cbuf = vec![0u8; cap]; + let rc = unsafe { + zstd_sys::ZSTD_compress( + cbuf.as_mut_ptr() as *mut core::ffi::c_void, + cap, + bytes.as_ptr() as *const core::ffi::c_void, + bytes.len(), + level, + ) + }; + assert_eq!( + unsafe { zstd_sys::ZSTD_isError(rc) }, + 0, + "ZSTD_compress failed" + ); + let ffi = &cbuf[..rc]; + + println!( + "=== section_split corpus={corpus} input={} level={level} ===", + bytes.len() + ); + let rs = analyze(&rust); + let fs_ = analyze(ffi); + print_split("rust", rust.len(), &rs); + print_split("ffi ", ffi.len(), &fs_); + println!( + "DELTA: total={:+} lit_section={:+} seq_section={:+}", + rust.len() as i64 - ffi.len() as i64, + rs.lit_bytes as i64 - fs_.lit_bytes as i64, + rs.seq_bytes as i64 - fs_.seq_bytes as i64, + ); +} diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index fddf33df5..5a103480b 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -772,7 +772,22 @@ impl FrameCompressor { let block_capacity = MAX_BLOCK_SIZE as usize; let mut start = 0usize; while start < input.len() { - let end = (start + block_capacity).min(input.len()); + // Donor `ZSTD_compress_frameChunk`: size each block via the cheap + // fingerprint pre-splitter so a full 128 KiB block is cut at a + // statistical boundary when it pays. `savings = consumed - + // produced` mirrors the donor gate (the first block and + // incompressible input keep the full 128 KiB). The borrowed window + // already spans the whole input, so a smaller block is just a + // narrower `(block_start, block_end)` range into it. + let savings = start as i64 - all_blocks.len() as i64; + let block_len = optimal_block_size( + self.compression_level, + &input[start..], + input.len() - start, + block_capacity, + savings, + ); + let end = (start + block_len).min(input.len()); let block = &input[start..end]; let last_block = end == input.len(); #[cfg(feature = "hash")] @@ -3208,28 +3223,31 @@ mod tests { } /// `level_pre_split` resolves the per-level split knob through the - /// `LevelParams` table, with named presets as pure numeric aliases: - /// greedy (level 5) → 1, btopt/btultra/btultra2 (16..=22) → 4. Fast, - /// dfast and the lazy band stay unsplit (lazy split is deferred until - /// the per-block entropy path reuses tables like the reference). + /// `LevelParams` table, mirroring the donor `splitLevels[]` by strategy + /// (`ZSTD_optimalBlockSize`): fast → 0 (from-borders), dfast → 1, + /// greedy/lazy → 2, btopt/btultra/btultra2 → 4. `Uncompressed` has no + /// numeric level so it stays `None`. #[test] fn pre_split_level_dispatches_by_compression_level() { use crate::encoding::CompressionLevel; use crate::encoding::match_generator::level_pre_split; assert_eq!(level_pre_split(CompressionLevel::Uncompressed), None); - assert_eq!(level_pre_split(CompressionLevel::Fastest), None); - assert_eq!(level_pre_split(CompressionLevel::Default), None); - // Better is a pure alias for level 7 (lazy): unsplit, same as Level(7). + // Fastest = level 1 (fast) → 0 (from-borders). + assert_eq!(level_pre_split(CompressionLevel::Fastest), Some(0)); + // Default = level 3 (dfast) → 1. + assert_eq!(level_pre_split(CompressionLevel::Default), Some(1)); + // Better is a pure alias for level 7 (lazy): same as Level(7). assert_eq!( level_pre_split(CompressionLevel::Better), level_pre_split(CompressionLevel::Level(7)), ); - assert_eq!(level_pre_split(CompressionLevel::Level(4)), None); - assert_eq!(level_pre_split(CompressionLevel::Level(5)), Some(1)); - assert_eq!(level_pre_split(CompressionLevel::Level(7)), None); - assert_eq!(level_pre_split(CompressionLevel::Level(15)), None); - assert_eq!(level_pre_split(CompressionLevel::Level(16)), Some(4)); - assert_eq!(level_pre_split(CompressionLevel::Level(22)), Some(4)); + assert_eq!(level_pre_split(CompressionLevel::Level(2)), Some(0)); // fast + assert_eq!(level_pre_split(CompressionLevel::Level(4)), Some(1)); // dfast + assert_eq!(level_pre_split(CompressionLevel::Level(5)), Some(2)); // greedy + assert_eq!(level_pre_split(CompressionLevel::Level(7)), Some(2)); // lazy + assert_eq!(level_pre_split(CompressionLevel::Level(15)), Some(2)); // lazy + assert_eq!(level_pre_split(CompressionLevel::Level(16)), Some(4)); // btopt + assert_eq!(level_pre_split(CompressionLevel::Level(22)), Some(4)); // btultra2 } /// End-to-end: a 256 KB payload whose SECOND 128 KB donor block carries diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs index 9b585527e..f1b2344ea 100644 --- a/zstd/src/encoding/match_generator.rs +++ b/zstd/src/encoding/match_generator.rs @@ -299,28 +299,19 @@ impl LevelParams { } } - /// Cheap fingerprint pre-splitter level (donor `splitLevels[]` by - /// strategy), or `None` to keep the whole 128 KiB block. `Fast`/`Dfast` - /// stay un-split: their match-finding is cheap, so the splitter's - /// per-block fingerprint plus extra per-sub-block entropy builds cost - /// more throughput than the ratio they buy. The btopt/btultra/btultra2 - /// band keeps level 4 (matching the pre-existing high-level splitter). - /// This is the C-like `blockSplitterLevel` knob, regulated per level - /// here rather than scattered across the frame loop. + /// Cheap fingerprint pre-splitter level, the C-like `blockSplitterLevel` + /// knob (donor `splitLevels[]` by strategy in `ZSTD_optimalBlockSize`): + /// fast=0, dfast=1, greedy=2, lazy=2, lazy2/btlazy2=3, + /// btopt/btultra/btultra2=4. `split_level == 0` routes to the cheap + /// from-borders heuristic; `1..=4` to byChunks with internal sampling + /// level `split_level - 1`. The `savings >= 3` gate in + /// `optimal_block_size` keeps incompressible data and the first full + /// block whole, so homogeneous frames are not over-split. fn pre_split(&self) -> Option { match self.strategy_tag { - // Fast/Dfast: cheap match-finding, the splitter costs more than - // it buys. Lazy: its ratio already tracks the reference, and - // splitting it regresses on highly-compressible frames until the - // per-block entropy path reuses tables as aggressively as the - // reference (tracked separately); keep whole blocks for now. - super::strategy::StrategyTag::Fast - | super::strategy::StrategyTag::Dfast - | super::strategy::StrategyTag::Lazy => None, - // Greedy is the band whose single-block literal section loses to - // the reference; the cheap fingerprint pre-split fits per-sub- - // block entropy and recovers it. - super::strategy::StrategyTag::Greedy => Some(1), + super::strategy::StrategyTag::Fast => Some(0), + super::strategy::StrategyTag::Dfast => Some(1), + super::strategy::StrategyTag::Greedy | super::strategy::StrategyTag::Lazy => Some(2), super::strategy::StrategyTag::BtOpt | super::strategy::StrategyTag::BtUltra | super::strategy::StrategyTag::BtUltra2 => Some(4), diff --git a/zstd/src/tests/roundtrip_integrity.rs b/zstd/src/tests/roundtrip_integrity.rs index 8c1e7aab5..8f71f8fea 100644 --- a/zstd/src/tests/roundtrip_integrity.rs +++ b/zstd/src/tests/roundtrip_integrity.rs @@ -482,15 +482,25 @@ fn roundtrip_better_level_large_window() { assert_eq!(roundtrip_better(&data), data); - // Better should compress the duplicated region; Default cannot reach it. + // Both levels must compress this 5 MiB fixture to a tiny frame: the two + // identical 256 KiB regions and the patterned gap are all independently + // highly compressible, so the large window only has to round-trip + // correctly (asserted above) — it need not beat the smaller window on + // total size. The former `better < default` size assertion was a stale + // proxy: with donor-faithful block-splitting enabled, the C reference + // itself produces `better(L7) > default(L3)` on this fixture + // (829 vs 525 bytes; ours 795 vs 495), because Default's split + entropy + // fit compresses the duplicated regions without needing the cross-gap + // match. Assert strong compression on both instead of their ordering. let compressed_better = compress_to_vec(&data[..], CompressionLevel::Better); let compressed_default = compress_to_vec(&data[..], CompressionLevel::Default); assert!( - compressed_better.len() < compressed_default.len(), - "Better (8 MiB window) should beat Default (4 MiB) across 4.5 MiB gap. \ - better={} default={}", + compressed_better.len() < data.len() / 1000 && compressed_default.len() < data.len() / 1000, + "both levels should compress this highly-redundant 5 MiB fixture to <0.1% \ + (better={}, default={}, input={})", compressed_better.len(), compressed_default.len(), + data.len(), ); } From c719286a411149cb6db96ebff6797b28a68b031c Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 6 Jun 2026 05:49:11 +0300 Subject: [PATCH 17/33] test(bench): add C-decoder loop for reference decode profiling --- zstd/examples/c_decode_loop_z000033.rs | 67 ++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 zstd/examples/c_decode_loop_z000033.rs diff --git a/zstd/examples/c_decode_loop_z000033.rs b/zstd/examples/c_decode_loop_z000033.rs new file mode 100644 index 000000000..e0a38a3bb --- /dev/null +++ b/zstd/examples/c_decode_loop_z000033.rs @@ -0,0 +1,67 @@ +//! Standalone C-decoder loop for a clean perf-record profile of the +//! reference decoder, to compare its hot-path breakdown against our +//! `decode_loop_z000033`. Encodes the in-tree z000033 once at the given +//! level via FFI, then decodes it N times through a reused `ZSTD_DCtx` +//! (steady state, no per-iter context alloc). +//! +//! Build: cargo build --profile flamegraph -p structured-zstd \ +//! --example c_decode_loop_z000033 --features dict_builder +//! Run: perf record -F 999 -g --call-graph dwarf,16384 -- \ +//! target/flamegraph/examples/c_decode_loop_z000033 3 20000 + +use std::env; +use std::fs; + +use zstd::zstd_safe::zstd_sys; + +fn main() { + let args: Vec = env::args().collect(); + let level: i32 = args.get(1).and_then(|s| s.parse().ok()).unwrap_or(3); + let iters: u32 = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(20_000); + let corpus = args + .get(3) + .cloned() + .unwrap_or_else(|| "zstd/decodecorpus_files/z000033".to_string()); + let src = fs::read(&corpus).expect("read corpus"); + let n = src.len(); + + let dst_cap = unsafe { zstd_sys::ZSTD_compressBound(n) }; + let mut compressed = vec![0u8; dst_cap]; + let csize = unsafe { + zstd_sys::ZSTD_compress( + compressed.as_mut_ptr() as *mut core::ffi::c_void, + dst_cap, + src.as_ptr() as *const core::ffi::c_void, + n, + level, + ) + }; + assert_eq!( + unsafe { zstd_sys::ZSTD_isError(csize) }, + 0, + "compress failed" + ); + + let dctx = unsafe { zstd_sys::ZSTD_createDCtx() }; + assert!(!dctx.is_null(), "createDCtx failed"); + let mut out = vec![0u8; n]; + let mut total: u64 = 0; + for _ in 0..iters { + let w = unsafe { + zstd_sys::ZSTD_decompressDCtx( + dctx, + out.as_mut_ptr() as *mut core::ffi::c_void, + n, + compressed.as_ptr() as *const core::ffi::c_void, + csize, + ) + }; + assert_eq!(unsafe { zstd_sys::ZSTD_isError(w) }, 0, "decompress failed"); + total = total.wrapping_add(out[0] as u64); + } + unsafe { zstd_sys::ZSTD_freeDCtx(dctx) }; + eprintln!( + "c_decode_loop: level={level} iters={iters} csize={csize} out={n} sink={total} ({} blocks-ish)", + csize + ); +} From ee46aa3228d175052de10aa237be1f24d59468d5 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 6 Jun 2026 06:07:21 +0300 Subject: [PATCH 18/33] perf(decode): inline AVX2 exec body into seq monolith via macro The AVX2 sequence loop called BufferBackend::exec_sequence_inline_avx2 as a trait method on every sequence (14% decode self-time, a real call boundary). A #[target_feature] fn cannot be #[inline(always)] (rust#145574), so the call could not be collapsed the way the reference fuses ZSTD_decodeSequence + ZSTD_execSequence into one inlined bmi2 monolith. Expand the exec body at the call site via macro_rules! (exec_sequence_avx2_inline!) instead. Backend access goes through the inlinable trait accessors cap/tail + two new ones (inline_exec_base_ptr, inline_exec_commit), so the macro stays generic over B while only the linear inline backends (UserSliceBackend, FlatBuf) reach it (gated on SUPPORTS_INLINE_SEQUENCE_EXEC). AVX2 tier only for now; other tiers keep the trait method. Part of #316 --- zstd/src/decoding/buffer_backend.rs | 32 ++++++++++ zstd/src/decoding/flat_buf.rs | 15 +++++ zstd/src/decoding/seq_decoder_avx2.rs | 91 +++++++++++++++++++++++---- zstd/src/decoding/user_slice_buf.rs | 12 ++++ 4 files changed, 139 insertions(+), 11 deletions(-) diff --git a/zstd/src/decoding/buffer_backend.rs b/zstd/src/decoding/buffer_backend.rs index e76c27806..130e04eed 100644 --- a/zstd/src/decoding/buffer_backend.rs +++ b/zstd/src/decoding/buffer_backend.rs @@ -237,6 +237,38 @@ pub(crate) trait BufferBackend: Sized { ); } + /// Base pointer of the contiguous output region, for the inline + /// match-copy macro `exec_sequence_avx2_inline!` (which expands the + /// AVX2 `ZSTD_execSequence` body textually at the sequence-loop call + /// site so it fuses into the per-tier monolith — `#[target_feature]` + /// functions cannot be `#[inline(always)]`, rust#145574). Only valid + /// when [`Self::SUPPORTS_INLINE_SEQUENCE_EXEC`]; the linear backends + /// (`UserSliceBackend`, `FlatBuf`) override, `RingBuffer` never reaches + /// it (gated, wrap-aware fallback). + /// + /// # Safety + /// Caller must hold the macro's preconditions (inline path gated on + /// `SUPPORTS_INLINE_SEQUENCE_EXEC` + capacity validated by + /// `sequence_output_fits`). + #[inline(always)] + unsafe fn inline_exec_base_ptr(&mut self) -> *mut u8 { + unreachable!("inline_exec_base_ptr on a backend without inline-sequence support") + } + + /// Commit the post-exec write cursor (grow the live region) after the + /// inline match-copy macro has written `[tail, new_tail)`. + /// `UserSliceBackend` advances its cursor; `FlatBuf` `set_len`s the Vec. + /// Distinct from [`Self::set_tail`], which is a shrink-only rollback + /// primitive (`new_tail <= len`). + /// + /// # Safety + /// `new_tail` bytes `[0, new_tail)` must be initialised (the macro just + /// wrote `[tail, new_tail)`); `new_tail <= capacity`. + #[inline(always)] + unsafe fn inline_exec_commit(&mut self, _new_tail: usize) { + unreachable!("inline_exec_commit on a backend without inline-sequence support") + } + /// Construct an empty backend. Backend-specific sizing is done /// via `with_capacity` constructors on the concrete types (see /// [`super::flat_buf::FlatBuf::with_capacity`]). diff --git a/zstd/src/decoding/flat_buf.rs b/zstd/src/decoding/flat_buf.rs index e97cc3659..3d819003b 100644 --- a/zstd/src/decoding/flat_buf.rs +++ b/zstd/src/decoding/flat_buf.rs @@ -312,6 +312,21 @@ impl BufferBackend for FlatBuf { Ok(()) } + #[cfg(target_arch = "x86_64")] + #[inline(always)] + unsafe fn inline_exec_base_ptr(&mut self) -> *mut u8 { + self.buf.as_mut_ptr() + } + + #[cfg(target_arch = "x86_64")] + #[inline(always)] + unsafe fn inline_exec_commit(&mut self, new_tail: usize) { + // The macro wrote `[buf.len(), new_tail)`; grow the Vec to expose it. + // SAFETY: new_tail <= capacity (sequence_output_fits validated it) and + // `[0, new_tail)` is now initialised. + unsafe { self.buf.set_len(new_tail) }; + } + fn new() -> Self { Self { buf: Vec::new(), diff --git a/zstd/src/decoding/seq_decoder_avx2.rs b/zstd/src/decoding/seq_decoder_avx2.rs index d86bb09f2..1cd900b9f 100644 --- a/zstd/src/decoding/seq_decoder_avx2.rs +++ b/zstd/src/decoding/seq_decoder_avx2.rs @@ -78,8 +78,77 @@ macro_rules! decode_one_body { }}; } -/// Textual expansion of per-sequence execute. Fast path: -/// `exec_sequence_inline_avx2` (32-byte ymm wildcopy). Cold path: legacy +/// Textual expansion of the AVX2 `ZSTD_execSequence` body at the call +/// site, fusing the match-copy into the per-tier sequence monolith. A +/// `#[target_feature(avx2)]` function cannot be `#[inline(always)]` +/// (rust#145574), so the trait method `exec_sequence_inline_avx2` stays a +/// real CALL on the hot path; expanding the body via a macro removes that +/// boundary (the donor `ZSTD_decompressSequences_bmi2` is one inlined +/// monolith). Backend access goes through the inlinable trait accessors +/// `cap` / `tail` / `inline_exec_base_ptr` / `inline_exec_commit`, so the +/// macro stays generic over `B` while only the linear inline backends +/// (`UserSliceBackend`, `FlatBuf`) ever reach it (gated on +/// `SUPPORTS_INLINE_SEQUENCE_EXEC`). Returns `Result<(), +/// ExecuteSequencesError>`. +macro_rules! exec_sequence_avx2_inline { + ($buffer:expr, $lit_src:expr, $lit_length:expr, $offset:expr, $match_length:expr) => {{ + use crate::decoding::buffer_backend::sequence_output_fits; + use crate::decoding::exec_sequence_inline::x86::{ + copy16, overlap_copy8, wildcopy_no_overlap, wildcopy_no_overlap_avx2, + wildcopy_overlap_8byte_stride, + }; + const MAX_WILDCOPY_OVERSHOOT: usize = 31; + let lit_length_v: usize = $lit_length; + let offset_v: usize = $offset; + let match_length_v: usize = $match_length; + let lit_src_v: *const u8 = $lit_src; + let backend = $buffer.buffer_mut(); + let cap = backend.cap(); + let tail = backend.tail(); + match sequence_output_fits( + lit_length_v, + match_length_v, + tail, + cap, + MAX_WILDCOPY_OVERSHOOT, + ) { + Err(e) => Err(e), + Ok(total) => { + // SAFETY: the enclosing fn carries + // `#[target_feature(enable = "bmi2,avx2")]`; the inline path + // is gated on `B::SUPPORTS_INLINE_SEQUENCE_EXEC`, so the + // backend is linear and overrides `inline_exec_base_ptr` / + // `inline_exec_commit`. `sequence_output_fits` validated + // `tail + total + overshoot <= cap`. + unsafe { + let base = backend.inline_exec_base_ptr(); + let op_lit = base.add(tail); + copy16(op_lit, lit_src_v); + if lit_length_v > 16 { + wildcopy_no_overlap(op_lit.add(16), lit_src_v.add(16), lit_length_v - 16); + } + let op_match = base.add(tail + lit_length_v); + let match_src = base.cast_const().add(tail + lit_length_v - offset_v); + if offset_v >= 32 { + wildcopy_no_overlap_avx2(op_match, match_src, match_length_v); + } else if offset_v >= 16 { + wildcopy_no_overlap(op_match, match_src, match_length_v); + } else { + let (op2, ip2) = overlap_copy8(op_match, match_src, offset_v); + if match_length_v > 8 { + wildcopy_overlap_8byte_stride(op2, ip2, match_length_v - 8); + } + } + backend.inline_exec_commit(tail + total); + } + Ok(()) + } + } + }}; +} + +/// Textual expansion of per-sequence execute. Fast path: the inlined AVX2 +/// match-copy macro [`exec_sequence_avx2_inline`]. Cold path: legacy /// try_push + repeat_lookahead_prefetched. Expands as a statement-block /// returning `Result<(), DecompressBlockError>` so the caller can `?` /// or branch on it as needed. @@ -143,15 +212,15 @@ macro_rules! execute_one_body { if prefix_end_ok { // SAFETY: parent-slice provenance; offset prefix-resident. let lit_src = unsafe { $literals_buffer.as_ptr().add(lit_cur_before) }; - // SAFETY: enclosing fn carries target_feature(bmi2,avx2). - let r = unsafe { - $buffer.buffer_mut().exec_sequence_inline_avx2( - lit_src, - seq_ll_v as usize, - offset, - seq_ml_v as usize, - ) - }; + // Inline the AVX2 exec body at the call site (no trait-method + // call boundary; see `exec_sequence_avx2_inline`). + let r = exec_sequence_avx2_inline!( + $buffer, + lit_src, + seq_ll_v as usize, + offset, + seq_ml_v as usize + ); break 'exec_inner r.map_err(DecompressBlockError::ExecuteSequencesError); } } diff --git a/zstd/src/decoding/user_slice_buf.rs b/zstd/src/decoding/user_slice_buf.rs index 37d843687..68c2cf467 100644 --- a/zstd/src/decoding/user_slice_buf.rs +++ b/zstd/src/decoding/user_slice_buf.rs @@ -472,6 +472,18 @@ impl<'a> BufferBackend for UserSliceBackend<'a> { Ok(()) } + #[cfg(target_arch = "x86_64")] + #[inline(always)] + unsafe fn inline_exec_base_ptr(&mut self) -> *mut u8 { + self.slice.as_mut_ptr() + } + + #[cfg(target_arch = "x86_64")] + #[inline(always)] + unsafe fn inline_exec_commit(&mut self, new_tail: usize) { + self.tail = new_tail; + } + /// `new()` exists for trait conformance but is not used on the /// direct-decode path — the slice is always provided up-front via /// [`Self::from_slice`]. Returns an empty backend wrapping an From 5a66be18bbee9b1074fef48406700915a7bf060d Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 6 Jun 2026 06:09:51 +0300 Subject: [PATCH 19/33] fix(decode): silence dead_code on inline-exec trait accessors for non-x86_64 inline_exec_base_ptr / inline_exec_commit are only reached from the x86_64 AVX2 exec macro; on i686/aarch64 (scalar/NEON exec paths) the defaults are dead, matching the existing exec_sequence_inline_avx2 #[allow(dead_code)]. --- zstd/src/decoding/buffer_backend.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zstd/src/decoding/buffer_backend.rs b/zstd/src/decoding/buffer_backend.rs index 130e04eed..a844b139d 100644 --- a/zstd/src/decoding/buffer_backend.rs +++ b/zstd/src/decoding/buffer_backend.rs @@ -250,6 +250,10 @@ pub(crate) trait BufferBackend: Sized { /// Caller must hold the macro's preconditions (inline path gated on /// `SUPPORTS_INLINE_SEQUENCE_EXEC` + capacity validated by /// `sequence_output_fits`). + // Only reached from the x86_64 AVX2 macro; dead on other targets + // (i686/aarch64 use the scalar/NEON exec paths), same as + // `exec_sequence_inline_avx2` above. + #[allow(dead_code)] #[inline(always)] unsafe fn inline_exec_base_ptr(&mut self) -> *mut u8 { unreachable!("inline_exec_base_ptr on a backend without inline-sequence support") @@ -264,6 +268,7 @@ pub(crate) trait BufferBackend: Sized { /// # Safety /// `new_tail` bytes `[0, new_tail)` must be initialised (the macro just /// wrote `[tail, new_tail)`); `new_tail <= capacity`. + #[allow(dead_code)] #[inline(always)] unsafe fn inline_exec_commit(&mut self, _new_tail: usize) { unreachable!("inline_exec_commit on a backend without inline-sequence support") From a0c1ec929e58cc0a70672788ef5a8b9b055ea416 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 6 Jun 2026 11:59:14 +0300 Subject: [PATCH 20/33] perf(decode): switch-on-length HUF DTable fill (donor HUF_fillDTableX1) build_table_from_weights filled each symbol's run with slice::fill on a RUNTIME length, lowering to a runtime-length memset call per symbol. The short lengths dominate the symbol count (high-nbBits symbols span 1/2/4 table entries). Specialise those into compile-time-fixed stores via match dst.len(), matching the reference's switch-on-length fill, and keep slice::fill only for the longer runs. One sub-slice keeps a single bounds check. Table output unchanged (212 huf/decode/roundtrip/cross tests pass). Part of #316 --- zstd/src/huff0/huff0_decoder.rs | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/zstd/src/huff0/huff0_decoder.rs b/zstd/src/huff0/huff0_decoder.rs index e52e9e8c8..112537499 100644 --- a/zstd/src/huff0/huff0_decoder.rs +++ b/zstd/src/huff0/huff0_decoder.rs @@ -682,7 +682,28 @@ impl HuffmanTable { let len = 1 << (max_bits - bits_for_symbol); self.rank_indexes[bits_for_symbol as usize] += len; let packed = u16::from(symbol as u8) | (u16::from(bits_for_symbol) << 8); - self.packed_decode[base_idx..base_idx + len].fill(packed); + // Donor `HUF_fillDTableX1` switch-on-length: `len` here is a + // RUNTIME value (per symbol), so `slice::fill` lowers to a + // runtime-length memset call every symbol. The short lengths + // dominate the symbol count (high-`nbBits` symbols span 1/2/4 + // table entries), so specialise them into compile-time-fixed + // stores — the reference hoists the same switch out of its + // per-weight loop. A single sub-slice keeps one bounds check. + let dst = &mut self.packed_decode[base_idx..base_idx + len]; + match dst.len() { + 1 => dst[0] = packed, + 2 => { + dst[0] = packed; + dst[1] = packed; + } + 4 => { + dst[0] = packed; + dst[1] = packed; + dst[2] = packed; + dst[3] = packed; + } + _ => dst.fill(packed), + } } } From cdb6a4c33b70ffc3ccf5f30492a36492e2ae1104 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 6 Jun 2026 12:02:32 +0300 Subject: [PATCH 21/33] Revert "perf(decode): switch-on-length HUF DTable fill (donor HUF_fillDTableX1)" This reverts commit 9064d3a9f90ae9eacf2fb3e7cc3bb623105b9958. --- zstd/src/huff0/huff0_decoder.rs | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/zstd/src/huff0/huff0_decoder.rs b/zstd/src/huff0/huff0_decoder.rs index 112537499..e52e9e8c8 100644 --- a/zstd/src/huff0/huff0_decoder.rs +++ b/zstd/src/huff0/huff0_decoder.rs @@ -682,28 +682,7 @@ impl HuffmanTable { let len = 1 << (max_bits - bits_for_symbol); self.rank_indexes[bits_for_symbol as usize] += len; let packed = u16::from(symbol as u8) | (u16::from(bits_for_symbol) << 8); - // Donor `HUF_fillDTableX1` switch-on-length: `len` here is a - // RUNTIME value (per symbol), so `slice::fill` lowers to a - // runtime-length memset call every symbol. The short lengths - // dominate the symbol count (high-`nbBits` symbols span 1/2/4 - // table entries), so specialise them into compile-time-fixed - // stores — the reference hoists the same switch out of its - // per-weight loop. A single sub-slice keeps one bounds check. - let dst = &mut self.packed_decode[base_idx..base_idx + len]; - match dst.len() { - 1 => dst[0] = packed, - 2 => { - dst[0] = packed; - dst[1] = packed; - } - 4 => { - dst[0] = packed; - dst[1] = packed; - dst[2] = packed; - dst[3] = packed; - } - _ => dst.fill(packed), - } + self.packed_decode[base_idx..base_idx + len].fill(packed); } } From 3255616023b407dc810f3f0f0c8086eea9304db3 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 6 Jun 2026 12:18:01 +0300 Subject: [PATCH 22/33] fix(encode): scope fast_attach to Simple backend in prime snapshot key The attach-vs-copy split is a Simple/Fast-backend concept (the 8 KiB dict-table cutoff). Recording it in PrimedKey for HashChain/Dfast/Row over-keyed those snapshots: an unhinted capture (fast_attach = true) and a hinted reset that resolved to the identical LevelParams keyed differently and forced a needless re-prime. Gate the bit on the Simple backend so non-Simple snapshots ignore it. Regression test captures a HashChain (Best) snapshot with no hint and restores it under a large hint that resolves to the same matcher, asserting the restore succeeds. --- zstd/src/encoding/match_generator.rs | 53 ++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 6 deletions(-) diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs index 9b585527e..10cbccc4d 100644 --- a/zstd/src/encoding/match_generator.rs +++ b/zstd/src/encoding/match_generator.rs @@ -1401,12 +1401,16 @@ impl Matcher for MatchGeneratorDriver { // Record the resolved matcher shape for the primed-snapshot key. Captured // here (post-resolution, after the test-only param override) so the key // reflects exactly the geometry the restored `storage` must match. The - // Fast attach-vs-copy mode is part of the shape (it decides whether a - // dict table is built), and matches the decision `prime_with_dictionary` - // makes from the same `reset_size_log`. - let fast_attach = self - .reset_size_log - .is_none_or(|log| log <= FAST_ATTACH_DICT_CUTOFF_LOG); + // Fast attach-vs-copy mode is part of the shape ONLY for the Simple + // backend (it decides whether a separate dict table is built); the other + // backends prime the dictionary the same way regardless, so including + // the bit there would over-key identical resolved shapes. When it + // applies it matches the decision `prime_with_dictionary` makes from the + // same `reset_size_log`. + let fast_attach = matches!(next_backend, super::strategy::BackendTag::Simple) + && self + .reset_size_log + .is_none_or(|log| log <= FAST_ATTACH_DICT_CUTOFF_LOG); self.reset_shape = Some((params, resolved_table_bits, fast_attach)); } @@ -7049,6 +7053,43 @@ fn primed_snapshot_not_restored_across_fast_attach_copy_boundary() { ); } +#[test] +fn primed_snapshot_fast_attach_does_not_over_key_non_simple_backends() { + // `fast_attach` is a Simple/Fast-backend concept (the 8 KiB attach-vs-copy + // table split). On the HashChain/Dfast/Row backends the dictionary is + // always primed the same way, so the bit must NOT enter their snapshot key + // — otherwise an unhinted capture (which would record `fast_attach = true`) + // and a hinted reset that resolves to the IDENTICAL `LevelParams` would key + // differently and force a needless re-prime. `Best` is a HashChain level. + let mut driver = MatchGeneratorDriver::new(8, 1); + let level = CompressionLevel::Best; + + // Capture with no hint. + driver.reset(level); + let window_a = driver.window_size(); + driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); + driver.capture_primed_dictionary(level); + + // Reset with a hint large enough to resolve to the same window/params as + // the unhinted level (>= 2^window_log, so the source-size cap is a no-op). + driver.set_source_size_hint(64 * 1024 * 1024); + driver.reset(level); + let window_b = driver.window_size(); + assert_eq!( + window_a, window_b, + "precondition: the large hint must resolve to the same window as the \ + unhinted level (a={window_a}, b={window_b})" + ); + + let restored = driver.restore_primed_dictionary(level); + assert!( + restored, + "a HashChain snapshot must restore across an unhinted vs large-hinted \ + reset that resolves to the identical matcher — `fast_attach` is a Fast \ + backend concept and must not over-key non-Simple shapes" + ); +} + #[cfg(any())] // disabled: tested SuffixStore-per-block tail-handling specific to legacy MatchGenerator #[test] fn prime_with_dictionary_does_not_reuse_tiny_suffix_store() { From 5612adb4d56f9e6745a07361885b55b9126251e6 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 6 Jun 2026 12:29:27 +0300 Subject: [PATCH 23/33] perf(decode): fuse match-copy into VBMI2 + BMI2 seq monoliths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The AVX2-tier exec fuse (call-boundary removal via macro) only covered the AVX2 kernel; VBMI2 and BMI2 tiers still called the exec trait method per sequence. Share the exec body as two macros in exec_sequence_inline: - exec_sequence_avx2_inline (32-byte ymm, offset>=32 path) — AVX2 + VBMI2 (VBMI2 always implies AVX2+BMI2) - exec_sequence_sse2_inline (16-byte xmm, overshoot 15) — BMI2 (no AVX2) Both expand textually at the sequence-loop call site so the match-copy fuses into each per-tier monolith (target_feature fns can't be inline(always), rust#145574). The exec_sequence_inline / _avx2 trait methods remain as the unit-tested reference spec the macros mirror. VBMI2/BMI2 paths are not reachable on the available test hardware (i9-9900K = AVX2 tier, no AVX-512; M1 = aarch64; i686 = scalar): the VBMI2 macro is identical to the i9-verified AVX2 expansion, and the BMI2 SSE2 macro mirrors the directly unit-tested exec_sequence_inline body. --- zstd/src/decoding/exec_sequence_inline.rs | 136 ++++++++++++++++++++++ zstd/src/decoding/seq_decoder_avx2.rs | 70 +---------- zstd/src/decoding/seq_decoder_bmi2.rs | 20 ++-- zstd/src/decoding/seq_decoder_vbmi2.rs | 20 ++-- 4 files changed, 159 insertions(+), 87 deletions(-) diff --git a/zstd/src/decoding/exec_sequence_inline.rs b/zstd/src/decoding/exec_sequence_inline.rs index c0085ae34..8183bb779 100644 --- a/zstd/src/decoding/exec_sequence_inline.rs +++ b/zstd/src/decoding/exec_sequence_inline.rs @@ -26,6 +26,142 @@ //! See the [`portable`] module doc for how the inline path is reached //! per target. +/// Textual expansion of the AVX2 `ZSTD_execSequence` body at the call +/// site, fusing the match-copy into a per-tier sequence monolith. A +/// `#[target_feature(avx2)]` function cannot be `#[inline(always)]` +/// (rust#145574), so the [`BufferBackend::exec_sequence_inline_avx2`] +/// trait method stays a real CALL on the hot path; expanding the body via +/// a macro removes that boundary (the reference `decompressSequences_bmi2` +/// is one inlined monolith). Backend access goes through the inlinable +/// accessors `cap` / `tail` / `inline_exec_base_ptr` / `inline_exec_commit`, +/// so the macro stays generic over `B` while only the linear inline +/// backends (`UserSliceBackend`, `FlatBuf`) ever reach it (gated on +/// `SUPPORTS_INLINE_SEQUENCE_EXEC`). 32-byte ymm match-copy for +/// `offset >= 32`; usable from any tier whose enclosing fn carries +/// `target_feature(avx2,bmi2)` (AVX2 and VBMI2). The trait method +/// `exec_sequence_inline_avx2` remains the unit-tested reference spec for +/// this body. Returns `Result<(), ExecuteSequencesError>`. +#[cfg(target_arch = "x86_64")] +macro_rules! exec_sequence_avx2_inline { + ($buffer:expr, $lit_src:expr, $lit_length:expr, $offset:expr, $match_length:expr) => {{ + use crate::decoding::buffer_backend::sequence_output_fits; + use crate::decoding::exec_sequence_inline::x86::{ + copy16, overlap_copy8, wildcopy_no_overlap, wildcopy_no_overlap_avx2, + wildcopy_overlap_8byte_stride, + }; + const MAX_WILDCOPY_OVERSHOOT: usize = 31; + let lit_length_v: usize = $lit_length; + let offset_v: usize = $offset; + let match_length_v: usize = $match_length; + let lit_src_v: *const u8 = $lit_src; + let backend = $buffer.buffer_mut(); + let cap = backend.cap(); + let tail = backend.tail(); + match sequence_output_fits( + lit_length_v, + match_length_v, + tail, + cap, + MAX_WILDCOPY_OVERSHOOT, + ) { + Err(e) => Err(e), + Ok(total) => { + // SAFETY: the enclosing fn carries + // `#[target_feature(enable = "...,bmi2,avx2")]`; the inline + // path is gated on `B::SUPPORTS_INLINE_SEQUENCE_EXEC`, so the + // backend is linear and overrides `inline_exec_base_ptr` / + // `inline_exec_commit`. `sequence_output_fits` validated + // `tail + total + overshoot <= cap`. + unsafe { + let base = backend.inline_exec_base_ptr(); + let op_lit = base.add(tail); + copy16(op_lit, lit_src_v); + if lit_length_v > 16 { + wildcopy_no_overlap(op_lit.add(16), lit_src_v.add(16), lit_length_v - 16); + } + let op_match = base.add(tail + lit_length_v); + let match_src = base.cast_const().add(tail + lit_length_v - offset_v); + if offset_v >= 32 { + wildcopy_no_overlap_avx2(op_match, match_src, match_length_v); + } else if offset_v >= 16 { + wildcopy_no_overlap(op_match, match_src, match_length_v); + } else { + let (op2, ip2) = overlap_copy8(op_match, match_src, offset_v); + if match_length_v > 8 { + wildcopy_overlap_8byte_stride(op2, ip2, match_length_v - 8); + } + } + backend.inline_exec_commit(tail + total); + } + Ok(()) + } + } + }}; +} +#[cfg(target_arch = "x86_64")] +pub(crate) use exec_sequence_avx2_inline; + +/// SSE2 twin of [`exec_sequence_avx2_inline`] for the BMI2 tier (which has +/// no AVX2): 16-byte xmm match-copy only (`offset >= 16`), so the WILDCOPY +/// destination overshoot stays 15 bytes (vs 31 for the ymm path). Mirrors +/// the [`BufferBackend::exec_sequence_inline`] trait method body, which +/// remains the unit-tested reference spec. Usable from any fn carrying +/// `target_feature(bmi2)`; baseline SSE2 needs no feature gate on x86_64. +#[cfg(target_arch = "x86_64")] +macro_rules! exec_sequence_sse2_inline { + ($buffer:expr, $lit_src:expr, $lit_length:expr, $offset:expr, $match_length:expr) => {{ + use crate::decoding::buffer_backend::sequence_output_fits; + use crate::decoding::exec_sequence_inline::x86::{ + copy16, overlap_copy8, wildcopy_no_overlap, wildcopy_overlap_8byte_stride, + }; + const MAX_WILDCOPY_OVERSHOOT: usize = 15; + let lit_length_v: usize = $lit_length; + let offset_v: usize = $offset; + let match_length_v: usize = $match_length; + let lit_src_v: *const u8 = $lit_src; + let backend = $buffer.buffer_mut(); + let cap = backend.cap(); + let tail = backend.tail(); + match sequence_output_fits( + lit_length_v, + match_length_v, + tail, + cap, + MAX_WILDCOPY_OVERSHOOT, + ) { + Err(e) => Err(e), + Ok(total) => { + // SAFETY: inline path gated on `B::SUPPORTS_INLINE_SEQUENCE_EXEC` + // (linear backend, overrides the accessors); + // `sequence_output_fits` validated `tail + total + 15 <= cap`. + // All copy primitives are SSE2 baseline (no target_feature). + unsafe { + let base = backend.inline_exec_base_ptr(); + let op_lit = base.add(tail); + copy16(op_lit, lit_src_v); + if lit_length_v > 16 { + wildcopy_no_overlap(op_lit.add(16), lit_src_v.add(16), lit_length_v - 16); + } + let op_match = base.add(tail + lit_length_v); + let match_src = base.cast_const().add(tail + lit_length_v - offset_v); + if offset_v >= 16 { + wildcopy_no_overlap(op_match, match_src, match_length_v); + } else { + let (op2, ip2) = overlap_copy8(op_match, match_src, offset_v); + if match_length_v > 8 { + wildcopy_overlap_8byte_stride(op2, ip2, match_length_v - 8); + } + } + backend.inline_exec_commit(tail + total); + } + Ok(()) + } + } + }}; +} +#[cfg(target_arch = "x86_64")] +pub(crate) use exec_sequence_sse2_inline; + // x86_64 only: SSE2 is the architectural baseline there (every x86_64 // CPU has SSE2 by definition). 32-bit `x86` is excluded because the // SSE2 intrinsics here are emitted without a `#[target_feature]` diff --git a/zstd/src/decoding/seq_decoder_avx2.rs b/zstd/src/decoding/seq_decoder_avx2.rs index 1cd900b9f..c90114f88 100644 --- a/zstd/src/decoding/seq_decoder_avx2.rs +++ b/zstd/src/decoding/seq_decoder_avx2.rs @@ -18,6 +18,7 @@ use super::buffer_backend::BufferBackend; use super::decode_buffer::DecodeBuffer; +use super::exec_sequence_inline::exec_sequence_avx2_inline; use super::scratch::FSEScratch; use super::sequence_section_decoder::{ ADVANCE, ADVANCE_MASK, ExecSeq, SeqStreamSetup, init_sequence_stream, @@ -78,75 +79,6 @@ macro_rules! decode_one_body { }}; } -/// Textual expansion of the AVX2 `ZSTD_execSequence` body at the call -/// site, fusing the match-copy into the per-tier sequence monolith. A -/// `#[target_feature(avx2)]` function cannot be `#[inline(always)]` -/// (rust#145574), so the trait method `exec_sequence_inline_avx2` stays a -/// real CALL on the hot path; expanding the body via a macro removes that -/// boundary (the donor `ZSTD_decompressSequences_bmi2` is one inlined -/// monolith). Backend access goes through the inlinable trait accessors -/// `cap` / `tail` / `inline_exec_base_ptr` / `inline_exec_commit`, so the -/// macro stays generic over `B` while only the linear inline backends -/// (`UserSliceBackend`, `FlatBuf`) ever reach it (gated on -/// `SUPPORTS_INLINE_SEQUENCE_EXEC`). Returns `Result<(), -/// ExecuteSequencesError>`. -macro_rules! exec_sequence_avx2_inline { - ($buffer:expr, $lit_src:expr, $lit_length:expr, $offset:expr, $match_length:expr) => {{ - use crate::decoding::buffer_backend::sequence_output_fits; - use crate::decoding::exec_sequence_inline::x86::{ - copy16, overlap_copy8, wildcopy_no_overlap, wildcopy_no_overlap_avx2, - wildcopy_overlap_8byte_stride, - }; - const MAX_WILDCOPY_OVERSHOOT: usize = 31; - let lit_length_v: usize = $lit_length; - let offset_v: usize = $offset; - let match_length_v: usize = $match_length; - let lit_src_v: *const u8 = $lit_src; - let backend = $buffer.buffer_mut(); - let cap = backend.cap(); - let tail = backend.tail(); - match sequence_output_fits( - lit_length_v, - match_length_v, - tail, - cap, - MAX_WILDCOPY_OVERSHOOT, - ) { - Err(e) => Err(e), - Ok(total) => { - // SAFETY: the enclosing fn carries - // `#[target_feature(enable = "bmi2,avx2")]`; the inline path - // is gated on `B::SUPPORTS_INLINE_SEQUENCE_EXEC`, so the - // backend is linear and overrides `inline_exec_base_ptr` / - // `inline_exec_commit`. `sequence_output_fits` validated - // `tail + total + overshoot <= cap`. - unsafe { - let base = backend.inline_exec_base_ptr(); - let op_lit = base.add(tail); - copy16(op_lit, lit_src_v); - if lit_length_v > 16 { - wildcopy_no_overlap(op_lit.add(16), lit_src_v.add(16), lit_length_v - 16); - } - let op_match = base.add(tail + lit_length_v); - let match_src = base.cast_const().add(tail + lit_length_v - offset_v); - if offset_v >= 32 { - wildcopy_no_overlap_avx2(op_match, match_src, match_length_v); - } else if offset_v >= 16 { - wildcopy_no_overlap(op_match, match_src, match_length_v); - } else { - let (op2, ip2) = overlap_copy8(op_match, match_src, offset_v); - if match_length_v > 8 { - wildcopy_overlap_8byte_stride(op2, ip2, match_length_v - 8); - } - } - backend.inline_exec_commit(tail + total); - } - Ok(()) - } - } - }}; -} - /// Textual expansion of per-sequence execute. Fast path: the inlined AVX2 /// match-copy macro [`exec_sequence_avx2_inline`]. Cold path: legacy /// try_push + repeat_lookahead_prefetched. Expands as a statement-block diff --git a/zstd/src/decoding/seq_decoder_bmi2.rs b/zstd/src/decoding/seq_decoder_bmi2.rs index 1b7bf2f65..1416191c2 100644 --- a/zstd/src/decoding/seq_decoder_bmi2.rs +++ b/zstd/src/decoding/seq_decoder_bmi2.rs @@ -9,6 +9,7 @@ use super::buffer_backend::BufferBackend; use super::decode_buffer::DecodeBuffer; +use super::exec_sequence_inline::exec_sequence_sse2_inline; use super::scratch::FSEScratch; use super::sequence_section_decoder::{ ADVANCE, ADVANCE_MASK, ExecSeq, SeqStreamSetup, init_sequence_stream, @@ -122,15 +123,16 @@ macro_rules! execute_one_body { if prefix_end_ok { // SAFETY: parent-slice provenance; offset prefix-resident. let lit_src = unsafe { $literals_buffer.as_ptr().add(lit_cur_before) }; - // SAFETY: enclosing fn carries target_feature(bmi2). - let r = unsafe { - $buffer.buffer_mut().exec_sequence_inline( - lit_src, - seq_ll_v as usize, - offset, - seq_ml_v as usize, - ) - }; + // Inline the SSE2 exec body at the call site (no trait-method + // call boundary; BMI2 tier has no AVX2, so the 16-byte xmm + // wildcopy variant is used — see `exec_sequence_sse2_inline`). + let r = exec_sequence_sse2_inline!( + $buffer, + lit_src, + seq_ll_v as usize, + offset, + seq_ml_v as usize + ); break 'exec_inner r.map_err(DecompressBlockError::ExecuteSequencesError); } } diff --git a/zstd/src/decoding/seq_decoder_vbmi2.rs b/zstd/src/decoding/seq_decoder_vbmi2.rs index 094c66ab0..d7e89278f 100644 --- a/zstd/src/decoding/seq_decoder_vbmi2.rs +++ b/zstd/src/decoding/seq_decoder_vbmi2.rs @@ -9,6 +9,7 @@ use super::buffer_backend::BufferBackend; use super::decode_buffer::DecodeBuffer; +use super::exec_sequence_inline::exec_sequence_avx2_inline; use super::scratch::FSEScratch; use super::sequence_section_decoder::{ ADVANCE, ADVANCE_MASK, ExecSeq, SeqStreamSetup, init_sequence_stream, @@ -118,15 +119,16 @@ macro_rules! execute_one_body { if prefix_end_ok { // SAFETY: parent-slice provenance; offset prefix-resident. let lit_src = unsafe { $literals_buffer.as_ptr().add(lit_cur_before) }; - // SAFETY: enclosing fn carries full VBMI2 scope. - let r = unsafe { - $buffer.buffer_mut().exec_sequence_inline_avx2( - lit_src, - seq_ll_v as usize, - offset, - seq_ml_v as usize, - ) - }; + // Inline the AVX2 exec body at the call site (no trait-method + // call boundary; VBMI2 always implies AVX2+BMI2 so the ymm + // wildcopy is in scope — see `exec_sequence_avx2_inline`). + let r = exec_sequence_avx2_inline!( + $buffer, + lit_src, + seq_ll_v as usize, + offset, + seq_ml_v as usize + ); break 'exec_inner r.map_err(DecompressBlockError::ExecuteSequencesError); } } From fda3ce488c4710a6688555335068c523c3d188c4 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 6 Jun 2026 12:38:58 +0300 Subject: [PATCH 24/33] test(codec): harden window test + diagnostic examples - roundtrip_better_level_large_window: switch the repeated region to incompressible random bytes so the cross-gap match is the SOLE way to compress the second copy, then assert Better (8 MiB window) beats Default (4 MiB window). The prior <0.1% size floor passed even if Better lost the cross-gap match (patterned fixture compressed tiny regardless of window). - pre_split_level_dispatches: pin Best == Level(11) so the named alias can't drift from the numeric pre-split route. - section_split example: match block type 2 explicitly, panic on unknown type, assert literals do not exceed block size. - c_decode_loop example: assert decode size == expected and guard the sink read against empty output. --- zstd/examples/c_decode_loop_z000033.rs | 3 +- zstd/examples/section_split.rs | 8 +++++- zstd/src/encoding/frame_compressor.rs | 6 ++++ zstd/src/tests/roundtrip_integrity.rs | 39 +++++++++++++------------- 4 files changed, 35 insertions(+), 21 deletions(-) diff --git a/zstd/examples/c_decode_loop_z000033.rs b/zstd/examples/c_decode_loop_z000033.rs index e0a38a3bb..0436fecfb 100644 --- a/zstd/examples/c_decode_loop_z000033.rs +++ b/zstd/examples/c_decode_loop_z000033.rs @@ -57,7 +57,8 @@ fn main() { ) }; assert_eq!(unsafe { zstd_sys::ZSTD_isError(w) }, 0, "decompress failed"); - total = total.wrapping_add(out[0] as u64); + assert_eq!(w, n, "decompress produced unexpected output size"); + total = total.wrapping_add(out.first().copied().unwrap_or(0) as u64); } unsafe { zstd_sys::ZSTD_freeDCtx(dctx) }; eprintln!( diff --git a/zstd/examples/section_split.rs b/zstd/examples/section_split.rs index 6bbb4276a..2c2c68949 100644 --- a/zstd/examples/section_split.rs +++ b/zstd/examples/section_split.rs @@ -141,15 +141,21 @@ fn analyze(frame: &[u8]) -> Split { s.rle_blocks += 1; pos += 1; // physical RLE body is one byte } - _ => { + 2 => { s.comp_blocks += 1; let body = &frame[pos..pos + bsize]; let (lit_total, lit_type) = lit_section_len(body); + assert!( + lit_total <= bsize, + "invalid block split: literals exceed block size \ + (lit_total={lit_total}, bsize={bsize})" + ); s.lit_type_counts[lit_type as usize] += 1; s.lit_bytes += lit_total; s.seq_bytes += bsize - lit_total; pos += bsize; } + _ => panic!("unexpected block type {btype}"), } if last == 1 { break; diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index 5a103480b..b83c43d72 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -3241,6 +3241,12 @@ mod tests { level_pre_split(CompressionLevel::Better), level_pre_split(CompressionLevel::Level(7)), ); + // Best is a pure alias for level 11 (lazy): pin it to the numeric + // route so the named path can't drift from the pre-split table. + assert_eq!( + level_pre_split(CompressionLevel::Best), + level_pre_split(CompressionLevel::Level(11)), + ); assert_eq!(level_pre_split(CompressionLevel::Level(2)), Some(0)); // fast assert_eq!(level_pre_split(CompressionLevel::Level(4)), Some(1)); // dfast assert_eq!(level_pre_split(CompressionLevel::Level(5)), Some(2)); // greedy diff --git a/zstd/src/tests/roundtrip_integrity.rs b/zstd/src/tests/roundtrip_integrity.rs index 8f71f8fea..790a61526 100644 --- a/zstd/src/tests/roundtrip_integrity.rs +++ b/zstd/src/tests/roundtrip_integrity.rs @@ -468,12 +468,15 @@ fn better_level_compresses_close_to_default() { /// 4 MiB window so only Better (8 MiB) can match it. #[test] fn roundtrip_better_level_large_window() { - // Two identical 256 KiB regions separated by a 4.5 MiB compressible gap. - // The gap uses a different seed so it doesn't share patterns with the - // regions, but being compressible means hash chains aren't fully - // destroyed by random noise. Better's 8 MiB window can still reach the - // first region; Default's 4 MiB window cannot. - let region = generate_compressible(42, 256 * 1024); + // Two identical 256 KiB INCOMPRESSIBLE regions separated by a 4.5 MiB + // compressible gap. The region is random (not pattern-repeating), so the + // only way to compress the second copy is a back-reference to the first, + // which sits ~4.8 MiB earlier. Better's 8 MiB window reaches it (one + // cross-gap match, the second region collapses to a few bytes); Default's + // 4 MiB window cannot, so the second region stays raw. This makes the + // window behaviour the SOLE differentiator: entropy/splitting cannot + // rescue Default on random bytes. + let region = generate_data(42, 256 * 1024); let gap = generate_compressible(9999, 4 * 1024 * 1024 + 512 * 1024); let mut data = Vec::with_capacity(region.len() + gap.len() + region.len()); data.extend_from_slice(®ion); @@ -482,22 +485,20 @@ fn roundtrip_better_level_large_window() { assert_eq!(roundtrip_better(&data), data); - // Both levels must compress this 5 MiB fixture to a tiny frame: the two - // identical 256 KiB regions and the patterned gap are all independently - // highly compressible, so the large window only has to round-trip - // correctly (asserted above) — it need not beat the smaller window on - // total size. The former `better < default` size assertion was a stale - // proxy: with donor-faithful block-splitting enabled, the C reference - // itself produces `better(L7) > default(L3)` on this fixture - // (829 vs 525 bytes; ours 795 vs 495), because Default's split + entropy - // fit compresses the duplicated regions without needing the cross-gap - // match. Assert strong compression on both instead of their ordering. + // Window-load-bearing assertion: Better (8 MiB window) must beat Default + // (4 MiB window) because only Better can match the second random region + // across the 4.8 MiB gap. With an incompressible region the size delta is + // ~256 KiB (the whole second region), so a real large-window regression + // — Better losing the cross-gap match — flips this ordering and fails the + // test. (The earlier `< input/1000` floor was not window-specific: it + // passed even if Better lost the match, since the patterned fixture + // compressed tiny regardless of window.) let compressed_better = compress_to_vec(&data[..], CompressionLevel::Better); let compressed_default = compress_to_vec(&data[..], CompressionLevel::Default); assert!( - compressed_better.len() < data.len() / 1000 && compressed_default.len() < data.len() / 1000, - "both levels should compress this highly-redundant 5 MiB fixture to <0.1% \ - (better={}, default={}, input={})", + compressed_better.len() < compressed_default.len(), + "Better (8 MiB window) should beat Default (4 MiB window) across the \ + 4.5 MiB gap via a cross-gap match (better={}, default={}, input={})", compressed_better.len(), compressed_default.len(), data.len(), From b0c649ecdaa34f0892749da74c4d09be297165ce Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 6 Jun 2026 12:47:21 +0300 Subject: [PATCH 25/33] fix(decode): gate shared exec macros on their kernel features The exec_sequence_avx2_inline / _sse2_inline macros were gated only on target_arch=x86_64, but their consumers (seq_decoder_avx2/vbmi2 behind kernel_avx2, seq_decoder_bmi2 behind kernel_bmi2) are feature-gated. On an x86_64 host with --no-default-features the macros were defined but unused, tripping the no-std clippy -D warnings gate. Gate each macro + its pub(crate) use on the matching kernel feature (kernel_avx2 covers vbmi2 via the feature implication; kernel_bmi2 for the sse2 variant). --- zstd/src/decoding/exec_sequence_inline.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/zstd/src/decoding/exec_sequence_inline.rs b/zstd/src/decoding/exec_sequence_inline.rs index 8183bb779..cace4c684 100644 --- a/zstd/src/decoding/exec_sequence_inline.rs +++ b/zstd/src/decoding/exec_sequence_inline.rs @@ -41,7 +41,12 @@ /// `target_feature(avx2,bmi2)` (AVX2 and VBMI2). The trait method /// `exec_sequence_inline_avx2` remains the unit-tested reference spec for /// this body. Returns `Result<(), ExecuteSequencesError>`. -#[cfg(target_arch = "x86_64")] +// +// Gated on `kernel_avx2` (implied by `kernel_vbmi2`) so the macro is absent +// when its only consumers (`seq_decoder_avx2` / `seq_decoder_vbmi2`) are +// compiled out — otherwise the `--no-default-features` build sees an unused +// macro and trips `-D warnings`. +#[cfg(all(target_arch = "x86_64", feature = "kernel_avx2"))] macro_rules! exec_sequence_avx2_inline { ($buffer:expr, $lit_src:expr, $lit_length:expr, $offset:expr, $match_length:expr) => {{ use crate::decoding::buffer_backend::sequence_output_fits; @@ -98,7 +103,7 @@ macro_rules! exec_sequence_avx2_inline { } }}; } -#[cfg(target_arch = "x86_64")] +#[cfg(all(target_arch = "x86_64", feature = "kernel_avx2"))] pub(crate) use exec_sequence_avx2_inline; /// SSE2 twin of [`exec_sequence_avx2_inline`] for the BMI2 tier (which has @@ -107,7 +112,11 @@ pub(crate) use exec_sequence_avx2_inline; /// the [`BufferBackend::exec_sequence_inline`] trait method body, which /// remains the unit-tested reference spec. Usable from any fn carrying /// `target_feature(bmi2)`; baseline SSE2 needs no feature gate on x86_64. -#[cfg(target_arch = "x86_64")] +// +// Gated on `kernel_bmi2` so the macro is absent when its only consumer +// (`seq_decoder_bmi2`) is compiled out, keeping `--no-default-features` +// (`-D warnings`) free of an unused-macro error. +#[cfg(all(target_arch = "x86_64", feature = "kernel_bmi2"))] macro_rules! exec_sequence_sse2_inline { ($buffer:expr, $lit_src:expr, $lit_length:expr, $offset:expr, $match_length:expr) => {{ use crate::decoding::buffer_backend::sequence_output_fits; @@ -159,7 +168,7 @@ macro_rules! exec_sequence_sse2_inline { } }}; } -#[cfg(target_arch = "x86_64")] +#[cfg(all(target_arch = "x86_64", feature = "kernel_bmi2"))] pub(crate) use exec_sequence_sse2_inline; // x86_64 only: SSE2 is the architectural baseline there (every x86_64 From 5473f8f455e044138273b9250e729d8d74923fbc Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 6 Jun 2026 13:56:33 +0300 Subject: [PATCH 26/33] fix(decode): gate AVX2 boundary tests on std feature The exec_sequence_inline_avx2 tests call std::arch::is_x86_feature_detected!, which is std-only (runtime CPU detection). The crate is #![no_std] by default, so under --no-default-features the macro fails to resolve. Gate the three AVX2 tests (user_slice_buf + flat_buf) on cfg(all(target_arch=x86_64, feature=std)). --- zstd/src/decoding/flat_buf.rs | 9 ++++++--- zstd/src/decoding/user_slice_buf.rs | 4 +++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/zstd/src/decoding/flat_buf.rs b/zstd/src/decoding/flat_buf.rs index 3d819003b..529d6917c 100644 --- a/zstd/src/decoding/flat_buf.rs +++ b/zstd/src/decoding/flat_buf.rs @@ -677,8 +677,10 @@ mod tests { /// offsets across the SSE2/AVX2 threshold boundary /// (offset 20 routes to SSE2 16-byte path, offset 32 to AVX2 /// 32-byte ymm path, offset 64 to deep AVX2 path). - // AVX2 override is x86_64-only; this test calls it directly. - #[cfg(target_arch = "x86_64")] + // AVX2 override is x86_64-only; this test calls it directly. The `std` + // feature gate is required: `is_x86_feature_detected!` is `std`-only, + // unavailable in the crate's `#![no_std]` build. + #[cfg(all(target_arch = "x86_64", feature = "std"))] #[test] fn exec_sequence_inline_avx2_offset_boundary_correctness() { if !std::arch::is_x86_feature_detected!("avx2") { @@ -750,7 +752,8 @@ mod tests { /// when the requested write plus the 31-byte overshoot exceeds the /// remaining headroom. Guards the single-compare bounds check on the AVX2 /// hot path. - #[cfg(target_arch = "x86_64")] + // `std` feature gate required: `is_x86_feature_detected!` is `std`-only. + #[cfg(all(target_arch = "x86_64", feature = "std"))] #[test] fn exec_sequence_inline_avx2_capacity_overflow_returns_err() { if !std::arch::is_x86_feature_detected!("avx2") { diff --git a/zstd/src/decoding/user_slice_buf.rs b/zstd/src/decoding/user_slice_buf.rs index 68c2cf467..fd055fedc 100644 --- a/zstd/src/decoding/user_slice_buf.rs +++ b/zstd/src/decoding/user_slice_buf.rs @@ -1273,7 +1273,9 @@ mod tests { /// - offset 64 (deep AVX2 path) /// /// against a byte-by-byte reference of the same repeat semantics. - #[cfg(target_arch = "x86_64")] + // `std` feature gates the test: `is_x86_feature_detected!` is `std`-only + // (runtime CPU detection), unavailable in the crate's `#![no_std]` build. + #[cfg(all(target_arch = "x86_64", feature = "std"))] #[test] fn exec_sequence_inline_avx2_offset_boundary_correctness() { if !std::arch::is_x86_feature_detected!("avx2") { From 73b2b950af667abff1d88402ad8e7a292d6b9384 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 6 Jun 2026 13:57:01 +0300 Subject: [PATCH 27/33] fix(encode): donor-correct pre-split level for lazy2/btlazy2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pre_split matched only the coarse StrategyTag, returning split level 2 for the whole Lazy band. The donor splitLevels[] table (ZSTD_optimalBlockSize, indexed by ZSTD_strategy: {0,0,1,2,2,3,3,4,4,4}) assigns lazy=2 but lazy2=3 and btlazy2=3. Our level table runs both lazy2 (L8-12) and btlazy2 (L13-15) on the hash-chain Lazy tag at lazy_depth 2, so they were under-split by one level versus the donor. Distinguish them by lazy_depth: Lazy at depth >= 2 now returns split level 3. optimal_block_size already supports level 3 (byChunks sampling index 2). Ratio gate (arch-independent, compare_ffi REPORT) on the 1 MB z000033 fixture at level 11 (lazy2): rust_bytes=465531 vs ffi_bytes=509326 — 8.6% smaller than the C reference. Test pre_split_level_dispatches updated: Level(11)/Level(15) now Some(3). --- zstd/src/encoding/frame_compressor.rs | 10 +++++---- zstd/src/encoding/match_generator.rs | 29 +++++++++++++++++++-------- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index b83c43d72..b4b3e3615 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -3225,8 +3225,9 @@ mod tests { /// `level_pre_split` resolves the per-level split knob through the /// `LevelParams` table, mirroring the donor `splitLevels[]` by strategy /// (`ZSTD_optimalBlockSize`): fast → 0 (from-borders), dfast → 1, - /// greedy/lazy → 2, btopt/btultra/btultra2 → 4. `Uncompressed` has no - /// numeric level so it stays `None`. + /// greedy/lazy → 2, lazy2/btlazy2 (Lazy tag at depth 2) → 3, + /// btopt/btultra/btultra2 → 4. `Uncompressed` has no numeric level so it + /// stays `None`. #[test] fn pre_split_level_dispatches_by_compression_level() { use crate::encoding::CompressionLevel; @@ -3250,8 +3251,9 @@ mod tests { assert_eq!(level_pre_split(CompressionLevel::Level(2)), Some(0)); // fast assert_eq!(level_pre_split(CompressionLevel::Level(4)), Some(1)); // dfast assert_eq!(level_pre_split(CompressionLevel::Level(5)), Some(2)); // greedy - assert_eq!(level_pre_split(CompressionLevel::Level(7)), Some(2)); // lazy - assert_eq!(level_pre_split(CompressionLevel::Level(15)), Some(2)); // lazy + assert_eq!(level_pre_split(CompressionLevel::Level(7)), Some(2)); // lazy (depth 1) + assert_eq!(level_pre_split(CompressionLevel::Level(11)), Some(3)); // lazy2 (depth 2) + assert_eq!(level_pre_split(CompressionLevel::Level(15)), Some(3)); // btlazy2 (depth 2) assert_eq!(level_pre_split(CompressionLevel::Level(16)), Some(4)); // btopt assert_eq!(level_pre_split(CompressionLevel::Level(22)), Some(4)); // btultra2 } diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs index 9258f5baa..07d196275 100644 --- a/zstd/src/encoding/match_generator.rs +++ b/zstd/src/encoding/match_generator.rs @@ -300,18 +300,31 @@ impl LevelParams { } /// Cheap fingerprint pre-splitter level, the C-like `blockSplitterLevel` - /// knob (donor `splitLevels[]` by strategy in `ZSTD_optimalBlockSize`): - /// fast=0, dfast=1, greedy=2, lazy=2, lazy2/btlazy2=3, - /// btopt/btultra/btultra2=4. `split_level == 0` routes to the cheap - /// from-borders heuristic; `1..=4` to byChunks with internal sampling - /// level `split_level - 1`. The `savings >= 3` gate in - /// `optimal_block_size` keeps incompressible data and the first full - /// block whole, so homogeneous frames are not over-split. + /// knob. Mirrors the donor `splitLevels[]` table indexed by strategy in + /// `ZSTD_optimalBlockSize` (`{0,0,1,2,2,3,3,4,4,4}` over fast..btultra2): + /// fast=0, dfast=1, greedy=2, lazy=2, lazy2=3, btlazy2=3, + /// btopt/btultra/btultra2=4. We collapse the donor `lazy2` and `btlazy2` + /// strategies into the hash-chain `Lazy` tag, distinguished here by + /// `lazy_depth` (the level table runs both at depth 2), so depth 2 routes + /// to split level 3 to match the donor. `split_level == 0` routes to the + /// cheap from-borders heuristic; `1..=4` to byChunks with internal + /// sampling level `split_level - 1`. The `savings >= 3` gate in + /// `optimal_block_size` keeps incompressible data and the first full block + /// whole, so homogeneous frames are not over-split. fn pre_split(&self) -> Option { match self.strategy_tag { super::strategy::StrategyTag::Fast => Some(0), super::strategy::StrategyTag::Dfast => Some(1), - super::strategy::StrategyTag::Greedy | super::strategy::StrategyTag::Lazy => Some(2), + super::strategy::StrategyTag::Greedy => Some(2), + // lazy=2, lazy2/btlazy2=3; both lazy2 and btlazy2 ride the Lazy + // tag at lazy_depth 2 in the level table. + super::strategy::StrategyTag::Lazy => { + if self.lazy_depth >= 2 { + Some(3) + } else { + Some(2) + } + } super::strategy::StrategyTag::BtOpt | super::strategy::StrategyTag::BtUltra | super::strategy::StrategyTag::BtUltra2 => Some(4), From f7a57a6ff538958265332f760b63dcb5ffa1d82a Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 6 Jun 2026 14:36:15 +0300 Subject: [PATCH 28/33] test(codec): cover fast borrowed split path + harden header parse - Add fast_oneshot_borrowed_split_emits_subblock: compress_slice_to_vec on a Fast level routes through run_borrowed_block_loop, which the existing greedy test (owned loop) did not exercise. Asserts >= 3 decoded blocks on a 256 KiB fixture (two 128 KiB blocks unsplit) so the borrowed split path cannot silently regress to fixed 128 KiB blocks. - Pin the lazy2/btlazy2 split range: add Level(8)/Level(12)/Level(13) -> Some(3) boundary assertions. - section_split example: pack the compressed/treeless literals header with u64 arithmetic; the 5-byte (sf 3) header reaches bit 35 (body[4] << 28), which truncates in a 32-bit usize. --- zstd/examples/section_split.rs | 28 ++++++++-------- zstd/src/encoding/frame_compressor.rs | 48 +++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 13 deletions(-) diff --git a/zstd/examples/section_split.rs b/zstd/examples/section_split.rs index 2c2c68949..e4ff00d9a 100644 --- a/zstd/examples/section_split.rs +++ b/zstd/examples/section_split.rs @@ -39,25 +39,27 @@ fn lit_section_len(body: &[u8]) -> (usize, u8) { } _ => { // Compressed / Treeless: size fields start at bit 4. + // Fixed-width u64: the 5-byte (sf 3) header packs up to bit 35 + // (`body[4] << 28`), which truncates in a 32-bit `usize`. let (hdr, compressed) = match sf { 0 | 1 => { - let v = (b0 >> 4) | ((body[1] as usize) << 4) | ((body[2] as usize) << 12); - (3, (v >> 10) & 0x3FF) + let v = ((b0 as u64) >> 4) | ((body[1] as u64) << 4) | ((body[2] as u64) << 12); + (3, ((v >> 10) & 0x3FF) as usize) } 2 => { - let v = (b0 >> 4) - | ((body[1] as usize) << 4) - | ((body[2] as usize) << 12) - | ((body[3] as usize) << 20); - (4, (v >> 14) & 0x3FFF) + let v = ((b0 as u64) >> 4) + | ((body[1] as u64) << 4) + | ((body[2] as u64) << 12) + | ((body[3] as u64) << 20); + (4, ((v >> 14) & 0x3FFF) as usize) } _ => { - let v = (b0 >> 4) - | ((body[1] as usize) << 4) - | ((body[2] as usize) << 12) - | ((body[3] as usize) << 20) - | ((body[4] as usize) << 28); - (5, (v >> 18) & 0x3FFFF) + let v = ((b0 as u64) >> 4) + | ((body[1] as u64) << 4) + | ((body[2] as u64) << 12) + | ((body[3] as u64) << 20) + | ((body[4] as u64) << 28); + (5, ((v >> 18) & 0x3FFFF) as usize) } }; (hdr + compressed, lit_type) diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index b4b3e3615..5aeac3eb2 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -3252,7 +3252,10 @@ mod tests { assert_eq!(level_pre_split(CompressionLevel::Level(4)), Some(1)); // dfast assert_eq!(level_pre_split(CompressionLevel::Level(5)), Some(2)); // greedy assert_eq!(level_pre_split(CompressionLevel::Level(7)), Some(2)); // lazy (depth 1) + assert_eq!(level_pre_split(CompressionLevel::Level(8)), Some(3)); // lazy2 lower bound assert_eq!(level_pre_split(CompressionLevel::Level(11)), Some(3)); // lazy2 (depth 2) + assert_eq!(level_pre_split(CompressionLevel::Level(12)), Some(3)); // lazy2 upper bound + assert_eq!(level_pre_split(CompressionLevel::Level(13)), Some(3)); // btlazy2 lower bound assert_eq!(level_pre_split(CompressionLevel::Level(15)), Some(3)); // btlazy2 (depth 2) assert_eq!(level_pre_split(CompressionLevel::Level(16)), Some(4)); // btopt assert_eq!(level_pre_split(CompressionLevel::Level(22)), Some(4)); // btultra2 @@ -3326,6 +3329,51 @@ mod tests { assert_eq!(decoded, data, "roundtrip must reproduce the input verbatim"); } + /// Outside-diff coverage for the FAST one-shot path. + /// `compress_slice_to_vec` / `compress_independent_frame` on a Fast level + /// routes through `run_borrowed_block_loop` (not the owned loop the test + /// above covers), which must honour `optimal_block_size` and emit a + /// sub-`MAX_BLOCK_SIZE` boundary rather than fixed 128 KiB blocks. A + /// 256 KiB input is two 128 KiB blocks when unsplit; a chunk boundary in + /// the second block yields >= 3 decoded blocks, asserted on the round-trip. + #[test] + fn fast_oneshot_borrowed_split_emits_subblock() { + use crate::encoding::{CompressionLevel, compress_slice_to_vec}; + // First 192 KiB: homogeneous zero run (banks the savings the split + // gate needs). The second 128 KiB block flips to a counter sequence + // at its 64 KiB midpoint (the 192 KiB mark) — a fingerprint + // transition the Fast from-borders splitter (split level 0) resolves + // into a sub-block boundary. + let mut data = vec![0u8; 256 * 1024]; + for (i, byte) in data.iter_mut().enumerate() { + if i >= 192 * 1024 { + *byte = (i % 251 + 1) as u8; + } + } + + let frame = compress_slice_to_vec(&data, CompressionLevel::Fastest); + + let mut decoder = FrameDecoder::new(); + let mut source = frame.as_slice(); + decoder + .reset(&mut source) + .expect("frame header should parse"); + while !decoder.is_finished() { + decoder + .decode_blocks(&mut source, crate::decoding::BlockDecodingStrategy::All) + .expect("decode should succeed"); + } + let mut decoded = Vec::with_capacity(data.len()); + decoder.collect_to_writer(&mut decoded).unwrap(); + assert_eq!(decoded, data, "roundtrip must reproduce the input verbatim"); + assert!( + decoder.blocks_decoded() >= 3, + "fast one-shot borrowed path must split the second donor block \ + (256 KiB unsplit = 2 blocks), got {} blocks", + decoder.blocks_decoded(), + ); + } + /// Regression: `set_compression_level` followed by `compress()` must /// refresh `state.strategy_tag` through the reset-time sync so the /// literal-compression gates (`min_literals_to_compress`, From adfe3afe0272ccbd25cba81b534c79dc3fab5392 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 6 Jun 2026 15:24:39 +0300 Subject: [PATCH 29/33] perf(encode): align lazy band to donor clevels.h, drop oversized preset windows Set L6-L12 lazy params to the donor default-row clevels.h values: window_log 21/22 (was 23/24/25), hash/chain/search_depth per donor (search_depth = 1<donor preset-window reach that no longer exists. --- zstd/src/encoding/match_generator.rs | 23 ++++---- zstd/src/tests/roundtrip_integrity.rs | 79 --------------------------- 2 files changed, 11 insertions(+), 91 deletions(-) diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs index 07d196275..dd01dad29 100644 --- a/zstd/src/encoding/match_generator.rs +++ b/zstd/src/encoding/match_generator.rs @@ -9,7 +9,6 @@ use alloc::vec::Vec; // SIMD/CRC intrinsics now live in `crate::encoding::fastpath::*` where they // sit under per-CPU `#[target_feature]` umbrellas; no architecture-specific // intrinsic imports remain in this file. -use super::BETTER_WINDOW_LOG; use super::CompressionLevel; use super::Matcher; use super::Sequence; @@ -391,13 +390,13 @@ const LEVEL_TABLE: [LevelParams; 22] = [ // long-enough match — the dominant cost in the L5..=L15 speed // regression vs FFI (see lazy_band_target_len_matches_donor_default_table). /* 5 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Greedy, search: super::strategy::SearchMethod::RowHash, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 0, hc: HcConfig { hash_log: 18, chain_log: 17, search_depth: 4, target_len: 2 }, row: ROW_L5 }, - /* 6 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: BETTER_WINDOW_LOG, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 1, hc: HcConfig { hash_log: 19, chain_log: 18, search_depth: 8, target_len: 4 }, row: ROW_CONFIG }, - /* 7 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: BETTER_WINDOW_LOG, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 1, hc: HcConfig { hash_log: 20, chain_log: 19, search_depth: 16, target_len: 8 }, row: ROW_CONFIG }, - /* 8 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: BETTER_WINDOW_LOG, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 20, chain_log: 19, search_depth: 24, target_len: 16 }, row: ROW_CONFIG }, - /* 9 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: BETTER_WINDOW_LOG, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 21, chain_log: 20, search_depth: 24, target_len: 16 }, row: ROW_CONFIG }, - /*10 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 24, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 21, chain_log: 20, search_depth: 28, target_len: 16 }, row: ROW_CONFIG }, - /*11 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 24, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 21, chain_log: 20, search_depth: 32, target_len: 16 }, row: ROW_CONFIG }, - /*12 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 25, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 22, chain_log: 21, search_depth: 32, target_len: 32 }, row: ROW_CONFIG }, + /* 6 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 21, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 1, hc: HcConfig { hash_log: 19, chain_log: 18, search_depth: 8, target_len: 4 }, row: ROW_CONFIG }, + /* 7 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 21, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 1, hc: HcConfig { hash_log: 20, chain_log: 19, search_depth: 16, target_len: 8 }, row: ROW_CONFIG }, + /* 8 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 21, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 20, chain_log: 19, search_depth: 16, target_len: 16 }, row: ROW_CONFIG }, + /* 9 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 21, chain_log: 20, search_depth: 16, target_len: 16 }, row: ROW_CONFIG }, + /*10 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 22, chain_log: 21, search_depth: 32, target_len: 16 }, row: ROW_CONFIG }, + /*11 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 22, chain_log: 21, search_depth: 64, target_len: 16 }, row: ROW_CONFIG }, + /*12 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 23, chain_log: 22, search_depth: 64, target_len: 32 }, row: ROW_CONFIG }, // L13-15: reference uses btlazy2 (binary-tree finder) with searchLog 4/5/6 // (search_depth 16/32/64) and targetLength 32. We run the hash-chain Lazy // parser here, so we mirror the reference search budget rather than inflate @@ -6674,9 +6673,9 @@ fn pooled_space_keeps_capacity_when_slice_size_shrinks() { fn driver_best_to_fastest_releases_oversized_hc_tables() { let mut driver = MatchGeneratorDriver::new(32, 2); - // Initialize at Best — allocates large HC tables (2M hash, 1M chain). + // Initialize at Best — allocates large HC tables (4M hash, 2M chain). driver.reset(CompressionLevel::Best); - assert_eq!(driver.window_size(), (1u64 << 24)); + assert_eq!(driver.window_size(), (1u64 << 22)); // Feed data so tables are actually allocated via ensure_tables(). let mut space = driver.get_next_space(); @@ -6710,7 +6709,7 @@ fn driver_better_to_best_resizes_hc_tables() { // Initialize at Better — allocates small HC tables (1M hash, 512K chain). driver.reset(CompressionLevel::Better); - assert_eq!(driver.window_size(), (1u64 << 23)); + assert_eq!(driver.window_size(), (1u64 << 21)); let mut space = driver.get_next_space(); space[..12].copy_from_slice(b"abcabcabcabc"); @@ -6724,7 +6723,7 @@ fn driver_better_to_best_resizes_hc_tables() { // Switch to Best — must resize to larger tables. driver.reset(CompressionLevel::Best); - assert_eq!(driver.window_size(), (1u64 << 24)); + assert_eq!(driver.window_size(), (1u64 << 22)); // Feed data to trigger ensure_tables with new sizes. let mut space = driver.get_next_space(); diff --git a/zstd/src/tests/roundtrip_integrity.rs b/zstd/src/tests/roundtrip_integrity.rs index 790a61526..99204f60e 100644 --- a/zstd/src/tests/roundtrip_integrity.rs +++ b/zstd/src/tests/roundtrip_integrity.rs @@ -80,18 +80,10 @@ fn roundtrip_default(data: &[u8]) -> Vec { roundtrip_at_level(data, CompressionLevel::Default) } -fn roundtrip_better(data: &[u8]) -> Vec { - roundtrip_at_level(data, CompressionLevel::Better) -} - fn roundtrip_better_streaming(data: &[u8]) -> Vec { roundtrip_streaming_at_level(data, CompressionLevel::Better) } -fn roundtrip_best(data: &[u8]) -> Vec { - roundtrip_at_level(data, CompressionLevel::Best) -} - fn roundtrip_best_streaming(data: &[u8]) -> Vec { roundtrip_streaming_at_level(data, CompressionLevel::Best) } @@ -464,47 +456,6 @@ fn better_level_compresses_close_to_default() { ); } -/// Exercise the 8 MiB window: place a repeated pattern beyond Default's -/// 4 MiB window so only Better (8 MiB) can match it. -#[test] -fn roundtrip_better_level_large_window() { - // Two identical 256 KiB INCOMPRESSIBLE regions separated by a 4.5 MiB - // compressible gap. The region is random (not pattern-repeating), so the - // only way to compress the second copy is a back-reference to the first, - // which sits ~4.8 MiB earlier. Better's 8 MiB window reaches it (one - // cross-gap match, the second region collapses to a few bytes); Default's - // 4 MiB window cannot, so the second region stays raw. This makes the - // window behaviour the SOLE differentiator: entropy/splitting cannot - // rescue Default on random bytes. - let region = generate_data(42, 256 * 1024); - let gap = generate_compressible(9999, 4 * 1024 * 1024 + 512 * 1024); - let mut data = Vec::with_capacity(region.len() + gap.len() + region.len()); - data.extend_from_slice(®ion); - data.extend_from_slice(&gap); - data.extend_from_slice(®ion); - - assert_eq!(roundtrip_better(&data), data); - - // Window-load-bearing assertion: Better (8 MiB window) must beat Default - // (4 MiB window) because only Better can match the second random region - // across the 4.8 MiB gap. With an incompressible region the size delta is - // ~256 KiB (the whole second region), so a real large-window regression - // — Better losing the cross-gap match — flips this ordering and fails the - // test. (The earlier `< input/1000` floor was not window-specific: it - // passed even if Better lost the match, since the patterned fixture - // compressed tiny regardless of window.) - let compressed_better = compress_to_vec(&data[..], CompressionLevel::Better); - let compressed_default = compress_to_vec(&data[..], CompressionLevel::Default); - assert!( - compressed_better.len() < compressed_default.len(), - "Better (8 MiB window) should beat Default (4 MiB window) across the \ - 4.5 MiB gap via a cross-gap match (better={}, default={}, input={})", - compressed_better.len(), - compressed_default.len(), - data.len(), - ); -} - /// Best must not regress vs Better on this repetitive fixture. Equal /// output is expected here (HC finds identical matches at any depth); /// the strict Best < Better check lives in cross_validation.rs on the @@ -522,36 +473,6 @@ fn best_level_does_not_regress_vs_better() { ); } -/// Exercise the 16 MiB window: place a repeated pattern beyond Better's -/// 8 MiB window so only Best (16 MiB) can match it. -#[test] -fn roundtrip_best_level_large_window() { - // Two identical 256 KiB high-entropy regions separated by a 9 MiB - // compressible gap. The region is random so the only way to compress - // the second copy is via long-distance matching (window reach). - // Best's 16 MiB window can still reach the first region; - // Better's 8 MiB window cannot. - let region = generate_data(42, 256 * 1024); - let gap = generate_compressible(7777, 9 * 1024 * 1024); - let mut data = Vec::with_capacity(region.len() + gap.len() + region.len()); - data.extend_from_slice(®ion); - data.extend_from_slice(&gap); - data.extend_from_slice(®ion); - - assert_eq!(roundtrip_best(&data), data); - - // Best should compress the duplicated region; Better cannot reach it. - let compressed_best = compress_to_vec(&data[..], CompressionLevel::Best); - let compressed_better = compress_to_vec(&data[..], CompressionLevel::Better); - assert!( - compressed_best.len() < compressed_better.len(), - "Best (16 MiB window) should beat Better (8 MiB) across 9 MiB gap. \ - best={} better={}", - compressed_best.len(), - compressed_better.len(), - ); -} - /// Best level streaming should produce identical decompressed output. #[test] fn roundtrip_best_level_streaming_multi_block() { From d8ed7d04c73f4a1d4920fafeef952161398ea235 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 6 Jun 2026 15:40:23 +0300 Subject: [PATCH 30/33] perf(encode): config-drive dfast hash sizing from donor clevels.h Add DfastConfig { long_hash_log, short_hash_log } carried per level as LevelParams.dfast (Some only on Dfast rows, None elsewhere so the table self-documents which levels the Dfast backend configures). L3 = {17,16}, L4 = {18,18} per the donor default-row clevels.h columns; window_log 21. The Dfast backend now sets its long/short hash table bits from this config instead of deriving the long hash from the window clamped to a hardcoded DFAST_HASH_BITS=17 ceiling, which previously capped L4 below its donor hashLog of 18. set_hash_bits takes (long, short) explicitly; the hinted small-input path caps each by the source-size window. minMatch stays the donor-fixed 5 (used in const contexts). z000033 ratio still beats C (L3 489251 vs 527148, L4 489092 vs 526163). --- zstd/src/encoding/dfast/mod.rs | 18 +++-- zstd/src/encoding/match_generator.rs | 104 +++++++++++++++++++-------- 2 files changed, 83 insertions(+), 39 deletions(-) diff --git a/zstd/src/encoding/dfast/mod.rs b/zstd/src/encoding/dfast/mod.rs index a9fe554fe..f0705ec0c 100644 --- a/zstd/src/encoding/dfast/mod.rs +++ b/zstd/src/encoding/dfast/mod.rs @@ -139,17 +139,15 @@ impl DfastMatchGenerator { } } - /// Set both hash table sizes. `bits` is the long-hash bit count - /// (donor `cParams.hashLog`); the short hash is derived as - /// `bits - DFAST_SHORT_HASH_BITS_DELTA`, donor-correct for dfast - /// levels. Both clamps stay above `MIN_WINDOW_LOG` so very small - /// windows don't underflow. - pub(crate) fn set_hash_bits(&mut self, bits: usize) { + /// Set both hash table sizes from the per-level [`DfastConfig`]: + /// `long_bits` = donor `cParams.hashLog`, `short_bits` = donor + /// `cParams.chainLog`. Both clamps stay above `MIN_WINDOW_LOG` so very + /// small windows don't underflow. The caller already caps `long_bits` by + /// the source-size window when hinted, so no upper clamp is applied here. + pub(crate) fn set_hash_bits(&mut self, long_bits: usize, short_bits: usize) { let min_bits = MIN_WINDOW_LOG as usize; - let long_clamped = bits.clamp(min_bits, DFAST_HASH_BITS); - let short_clamped = long_clamped - .saturating_sub(DFAST_SHORT_HASH_BITS_DELTA) - .max(min_bits); + let long_clamped = long_bits.max(min_bits); + let short_clamped = short_bits.max(min_bits); if self.long_hash_bits != long_clamped { self.long_hash_bits = long_clamped; self.long_hash = Vec::new(); diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs index dd01dad29..ec4b2285c 100644 --- a/zstd/src/encoding/match_generator.rs +++ b/zstd/src/encoding/match_generator.rs @@ -244,6 +244,30 @@ const ROW_L5: RowConfig = RowConfig { mls: ROW_MIN_MATCH_LEN, }; +/// Per-level Double-Fast hash sizing, mirroring the donor `clevels.h` columns +/// (config-driven, not a hardcoded constant): `long_hash_log` = +/// `cParams.hashLog` (the long 8-byte hash table), `short_hash_log` = +/// `cParams.chainLog` (the short hash table dfast repurposes as its +/// secondary index). Only the Dfast backend reads it, so non-dfast level +/// rows carry `dfast: None`. `minMatch` stays the donor-fixed `5` +/// (`DFAST_MIN_MATCH_LEN`, used in const contexts). +#[derive(Copy, Clone, PartialEq, Eq)] +struct DfastConfig { + long_hash_log: u8, + short_hash_log: u8, +} + +// Donor clevels.h default row (srcSize > 256 KB): L3 {hashLog 17, chainLog 16}, +// L4 {hashLog 18, chainLog 18}. +const DFAST_L3: DfastConfig = DfastConfig { + long_hash_log: 17, + short_hash_log: 16, +}; +const DFAST_L4: DfastConfig = DfastConfig { + long_hash_log: 18, + short_hash_log: 18, +}; + /// Resolved tuning parameters for a compression level. The /// [`StrategyTag`] is the single source of truth for the backend /// family and the compile-time strategy consts; the runtime @@ -273,6 +297,10 @@ struct LevelParams { /// skip schedule. fast_step_size: usize, lazy_depth: u8, + /// Donor dfast tuning (`clevels.h` columns). `Some` only on Dfast level + /// rows; `None` elsewhere so the table self-documents which levels the + /// Dfast backend configures. + dfast: Option, hc: HcConfig, row: RowConfig, } @@ -356,9 +384,12 @@ pub(crate) fn source_size_ceil_log(size: u64) -> u8 { /// the primed-snapshot key) and `prime_with_dictionary` (which acts on it). const FAST_ATTACH_DICT_CUTOFF_LOG: u8 = 13; +// Source-size cap for the dfast hash bits when a size hint is present: a tiny +// input needs no larger hash than its window. The donor `cParams.hashLog` / +// `chainLog` (from `DfastConfig`) caps it from above at the call site. fn dfast_hash_bits_for_window(max_window_size: usize) -> usize { let window_log = (usize::BITS - 1 - max_window_size.leading_zeros()) as usize; - window_log.clamp(MIN_WINDOW_LOG as usize, DFAST_HASH_BITS) + window_log.max(MIN_WINDOW_LOG as usize) } fn row_hash_bits_for_window(max_window_size: usize) -> usize { @@ -378,10 +409,10 @@ fn row_hash_bits_for_window(max_window_size: usize) -> usize { const LEVEL_TABLE: [LevelParams; 22] = [ // Lvl Strategy wlog fast_hlog fast_mls fast_step lazy HC config row config // --- -------------- ---- --------- -------- --------- ---- ------------------------------------------ ---------- - /* 1 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Fast, search: super::strategy::SearchMethod::Fast, window_log: 19, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 0, hc: HC_CONFIG, row: ROW_CONFIG }, - /* 2 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Fast, search: super::strategy::SearchMethod::Fast, window_log: 20, fast_hash_log: 16, fast_mls: 6, fast_step_size: 2, lazy_depth: 0, hc: HC_CONFIG, row: ROW_CONFIG }, - /* 3 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Dfast, search: super::strategy::SearchMethod::DoubleFast, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 1, hc: HC_CONFIG, row: ROW_CONFIG }, - /* 4 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Dfast, search: super::strategy::SearchMethod::DoubleFast, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 1, hc: HC_CONFIG, row: ROW_CONFIG }, + /* 1 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Fast, search: super::strategy::SearchMethod::Fast, window_log: 19, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 0, dfast: None, hc: HC_CONFIG, row: ROW_CONFIG }, + /* 2 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Fast, search: super::strategy::SearchMethod::Fast, window_log: 20, fast_hash_log: 16, fast_mls: 6, fast_step_size: 2, lazy_depth: 0, dfast: None, hc: HC_CONFIG, row: ROW_CONFIG }, + /* 3 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Dfast, search: super::strategy::SearchMethod::DoubleFast, window_log: 21, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 1, dfast: Some(DFAST_L3), hc: HC_CONFIG, row: ROW_CONFIG }, + /* 4 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Dfast, search: super::strategy::SearchMethod::DoubleFast, window_log: 21, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 1, dfast: Some(DFAST_L4), hc: HC_CONFIG, row: ROW_CONFIG }, // target_len column for L5..=L15 matches donor cParams.targetLength // from clevels.h table[0] (default — srcSize > 256 KB). Donor uses // it as the lazy outer loop's `sufficient_len` (nice-match) threshold. @@ -389,14 +420,14 @@ const LEVEL_TABLE: [LevelParams; 22] = [ // search_depth iterations instead of breaking on the first // long-enough match — the dominant cost in the L5..=L15 speed // regression vs FFI (see lazy_band_target_len_matches_donor_default_table). - /* 5 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Greedy, search: super::strategy::SearchMethod::RowHash, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 0, hc: HcConfig { hash_log: 18, chain_log: 17, search_depth: 4, target_len: 2 }, row: ROW_L5 }, - /* 6 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 21, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 1, hc: HcConfig { hash_log: 19, chain_log: 18, search_depth: 8, target_len: 4 }, row: ROW_CONFIG }, - /* 7 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 21, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 1, hc: HcConfig { hash_log: 20, chain_log: 19, search_depth: 16, target_len: 8 }, row: ROW_CONFIG }, - /* 8 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 21, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 20, chain_log: 19, search_depth: 16, target_len: 16 }, row: ROW_CONFIG }, - /* 9 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 21, chain_log: 20, search_depth: 16, target_len: 16 }, row: ROW_CONFIG }, - /*10 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 22, chain_log: 21, search_depth: 32, target_len: 16 }, row: ROW_CONFIG }, - /*11 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 22, chain_log: 21, search_depth: 64, target_len: 16 }, row: ROW_CONFIG }, - /*12 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 23, chain_log: 22, search_depth: 64, target_len: 32 }, row: ROW_CONFIG }, + /* 5 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Greedy, search: super::strategy::SearchMethod::RowHash, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 0, dfast: None, hc: HcConfig { hash_log: 18, chain_log: 17, search_depth: 4, target_len: 2 }, row: ROW_L5 }, + /* 6 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 21, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 1, dfast: None, hc: HcConfig { hash_log: 19, chain_log: 18, search_depth: 8, target_len: 4 }, row: ROW_CONFIG }, + /* 7 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 21, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 1, dfast: None, hc: HcConfig { hash_log: 20, chain_log: 19, search_depth: 16, target_len: 8 }, row: ROW_CONFIG }, + /* 8 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 21, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: HcConfig { hash_log: 20, chain_log: 19, search_depth: 16, target_len: 16 }, row: ROW_CONFIG }, + /* 9 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: HcConfig { hash_log: 21, chain_log: 20, search_depth: 16, target_len: 16 }, row: ROW_CONFIG }, + /*10 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: HcConfig { hash_log: 22, chain_log: 21, search_depth: 32, target_len: 16 }, row: ROW_CONFIG }, + /*11 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: HcConfig { hash_log: 22, chain_log: 21, search_depth: 64, target_len: 16 }, row: ROW_CONFIG }, + /*12 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: HcConfig { hash_log: 23, chain_log: 22, search_depth: 64, target_len: 32 }, row: ROW_CONFIG }, // L13-15: reference uses btlazy2 (binary-tree finder) with searchLog 4/5/6 // (search_depth 16/32/64) and targetLength 32. We run the hash-chain Lazy // parser here, so we mirror the reference search budget rather than inflate @@ -405,16 +436,16 @@ const LEVEL_TABLE: [LevelParams; 22] = [ // smaller searchLog find longer matches (and re-establish a strict ratio // ladder above L12) is tracked separately; until it lands these levels sit // close to L12 on hash-chain inputs by design. - /*13 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 22, chain_log: 22, search_depth: 16, target_len: 32 }, row: ROW_CONFIG }, - /*14 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 23, chain_log: 22, search_depth: 32, target_len: 32 }, row: ROW_CONFIG }, - /*15 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 23, chain_log: 23, search_depth: 64, target_len: 32 }, row: ROW_CONFIG }, - /*16 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtOpt, search: super::strategy::SearchMethod::BinaryTree, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 22, chain_log: 22, search_depth: 32, target_len: 48 }, row: ROW_CONFIG }, - /*17 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtOpt, search: super::strategy::SearchMethod::BinaryTree, window_log: 23, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 22, chain_log: 23, search_depth: 32, target_len: 64 }, row: ROW_CONFIG }, - /*18 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra, search: super::strategy::SearchMethod::BinaryTree, window_log: 23, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 22, chain_log: 23, search_depth: 64, target_len: 64 }, row: ROW_CONFIG }, - /*19 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log: 23, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 22, chain_log: 24, search_depth: 128, target_len: 256 }, row: ROW_CONFIG }, - /*20 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log: 25, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 23, chain_log: 25, search_depth: 128, target_len: 256 }, row: ROW_CONFIG }, - /*21 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log: 26, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: BTULTRA2_HC_CONFIG, row: ROW_CONFIG }, - /*22 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log: 27, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: BTULTRA2_HC_CONFIG_L22, row: ROW_CONFIG }, + /*13 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: HcConfig { hash_log: 22, chain_log: 22, search_depth: 16, target_len: 32 }, row: ROW_CONFIG }, + /*14 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: HcConfig { hash_log: 23, chain_log: 22, search_depth: 32, target_len: 32 }, row: ROW_CONFIG }, + /*15 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: HcConfig { hash_log: 23, chain_log: 23, search_depth: 64, target_len: 32 }, row: ROW_CONFIG }, + /*16 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtOpt, search: super::strategy::SearchMethod::BinaryTree, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: HcConfig { hash_log: 22, chain_log: 22, search_depth: 32, target_len: 48 }, row: ROW_CONFIG }, + /*17 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtOpt, search: super::strategy::SearchMethod::BinaryTree, window_log: 23, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: HcConfig { hash_log: 22, chain_log: 23, search_depth: 32, target_len: 64 }, row: ROW_CONFIG }, + /*18 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra, search: super::strategy::SearchMethod::BinaryTree, window_log: 23, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: HcConfig { hash_log: 22, chain_log: 23, search_depth: 64, target_len: 64 }, row: ROW_CONFIG }, + /*19 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log: 23, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: HcConfig { hash_log: 22, chain_log: 24, search_depth: 128, target_len: 256 }, row: ROW_CONFIG }, + /*20 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log: 25, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: HcConfig { hash_log: 23, chain_log: 25, search_depth: 128, target_len: 256 }, row: ROW_CONFIG }, + /*21 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log: 26, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: BTULTRA2_HC_CONFIG, row: ROW_CONFIG }, + /*22 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log: 27, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: BTULTRA2_HC_CONFIG_L22, row: ROW_CONFIG }, ]; /// Smallest window_log the encoder will use regardless of source size. @@ -508,6 +539,7 @@ fn level22_btultra2_params_for_source_size(source_size: Option) -> LevelPar fast_mls: 7, fast_step_size: 2, lazy_depth: 2, + dfast: None, hc, row: ROW_CONFIG, } @@ -535,6 +567,7 @@ fn resolve_level_params(level: CompressionLevel, source_size: Option) -> Le // which gives step_size = 1 + 0 + 1 = 2. fast_step_size: 2, lazy_depth: 0, + dfast: None, hc: HC_CONFIG, row: ROW_CONFIG, }, @@ -590,6 +623,7 @@ fn resolve_level_params(level: CompressionLevel, source_size: Option) -> Le fast_mls: 7, fast_step_size: step_size, lazy_depth: 0, + dfast: None, hc: HC_CONFIG, row: ROW_CONFIG, } @@ -1369,12 +1403,24 @@ impl Matcher for MatchGeneratorDriver { | CompressionLevel::Level(0) | CompressionLevel::Level(3) ); - resolved_table_bits = if hinted { - dfast_hash_bits_for_window(table_window_size) + let dcfg = params + .dfast + .expect("Dfast level row must carry a DfastConfig"); + // Donor `cParams.hashLog`/`chainLog`, capped by the + // source-size window when hinted so tiny inputs don't + // over-allocate. + let long_bits = if hinted { + dfast_hash_bits_for_window(table_window_size).min(dcfg.long_hash_log as usize) + } else { + dcfg.long_hash_log as usize + }; + let short_bits = if hinted { + dfast_hash_bits_for_window(table_window_size).min(dcfg.short_hash_log as usize) } else { - DFAST_HASH_BITS + dcfg.short_hash_log as usize }; - dfast.set_hash_bits(resolved_table_bits); + resolved_table_bits = long_bits; + dfast.set_hash_bits(long_bits, short_bits); // Dfast holds no per-block input Vecs (history owns the // bytes and `add_data` returns each Vec eagerly), so // `reset` takes no `reuse_space` callback. @@ -4646,7 +4692,7 @@ fn driver_switches_backends_and_initializes_dfast_via_reset() { driver.reset(CompressionLevel::Default); assert_eq!(driver.active_backend(), super::strategy::BackendTag::Dfast); - assert_eq!(driver.window_size(), (1u64 << 22)); + assert_eq!(driver.window_size(), (1u64 << 21)); let mut first = driver.get_next_space(); first[..12].copy_from_slice(b"abcabcabcabc"); @@ -9232,7 +9278,7 @@ fn dfast_seed_remaining_hashable_starts_handles_pos_at_block_end() { #[test] fn dfast_ensure_room_for_rebases_above_guard_band() { let mut dfast = DfastMatchGenerator::new(1 << 22); - dfast.set_hash_bits(10); + dfast.set_hash_bits(10, 10); dfast.ensure_hash_tables(); // Seed an early insert near the current base in BOTH tables. From 82cf40cc1897cab6931352ad35a9e3b8c6f22648 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 6 Jun 2026 16:08:12 +0300 Subject: [PATCH 31/33] refactor(encode): per-strategy Option configs in LevelParams + L5 donor window MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Group the Fast knobs into FastConfig and make every per-strategy config an Option on LevelParams: fast / dfast / hc / row. Exactly one is Some per level row, matching the strategy backend; the rest are None — so the table self-documents which knobs each level consumes instead of carrying dead placeholder values (the old fast_hash_log:14 / fast_mls:7 on lazy/bt rows). Backend configure/reset arms read their config via expect() (the native backend's config is always Some). The test-only parse×search override synthesizes a default config for a cross-backend pairing via get_or_insert. HC_CONFIG / ROW_CONFIG (and the HC_HASH_LOG/HC_CHAIN_LOG imports) are now test-only and cfg(test)-gated. L5 greedy window_log 22 -> 21 (donor clevels.h). Behaviour-neutral refactor: 749 tests pass incl. the FFI donor-parity checks. --- zstd/src/encoding/match_generator.rs | 342 ++++++++++++++++----------- 1 file changed, 204 insertions(+), 138 deletions(-) diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs index ec4b2285c..fbb8abc18 100644 --- a/zstd/src/encoding/match_generator.rs +++ b/zstd/src/encoding/match_generator.rs @@ -128,7 +128,10 @@ use super::match_table::storage::{HC_PRIME3BYTES, HC_PRIME4BYTES}; // macros / configs / tests keep their unqualified names. #[cfg(test)] use super::match_table::storage::HC_EMPTY; -use super::match_table::storage::{HC_CHAIN_LOG, HC_HASH_LOG, HC3_HASH_LOG}; +use super::match_table::storage::HC3_HASH_LOG; +// HC_HASH_LOG / HC_CHAIN_LOG feed the test-only `HC_CONFIG` default. +#[cfg(test)] +use super::match_table::storage::{HC_CHAIN_LOG, HC_HASH_LOG}; // HC3_MAX_OFFSET moved to encoding::bt alongside the hash3 candidate // probe macro that consumes it; the macro references it via the // fully-qualified `$crate::encoding::bt::HC3_MAX_OFFSET` path so this @@ -173,6 +176,9 @@ pub(crate) struct RowConfig { pub(crate) mls: usize, } +// Only used as the default HashChain config when the test-only parse×search +// override pairs a level with a backend its native row doesn't populate. +#[cfg(test)] const HC_CONFIG: HcConfig = HcConfig { hash_log: HC_HASH_LOG, chain_log: HC_CHAIN_LOG, @@ -215,6 +221,9 @@ const BTULTRA2_HC_CONFIG_L22_16K: HcConfig = HcConfig { target_len: 999, }; +// Default Row config: only used by tests and the test-only parse×search +// override (production greedy L5 carries its own `ROW_L5`). +#[cfg(test)] const ROW_CONFIG: RowConfig = RowConfig { hash_bits: ROW_HASH_BITS, row_log: ROW_LOG, @@ -268,6 +277,28 @@ const DFAST_L4: DfastConfig = DfastConfig { short_hash_log: 18, }; +/// Per-level Fast-strategy tuning, only consumed by the `FastKernelMatcher` +/// (Simple backend): `hash_log` = donor `cParams.hashLog`, `mls` = donor +/// `cParams.minMatch` (4..=8), `step_size` = donor `stepSize`. Carried as +/// `LevelParams.fast` (`Some` only on Fast level rows; `None` elsewhere). +#[derive(Copy, Clone, PartialEq, Eq)] +struct FastConfig { + hash_log: u32, + mls: u32, + step_size: usize, +} + +const FAST_L1: FastConfig = FastConfig { + hash_log: 14, + mls: 7, + step_size: 2, +}; +const FAST_L2: FastConfig = FastConfig { + hash_log: 16, + mls: 6, + step_size: 2, +}; + /// Resolved tuning parameters for a compression level. The /// [`StrategyTag`] is the single source of truth for the backend /// family and the compile-time strategy consts; the runtime @@ -283,26 +314,17 @@ struct LevelParams { /// per level so the parse×search matrix can be swept and tuned. search: super::strategy::SearchMethod, window_log: u8, - /// Donor `cParams.hashLog` — only consumed by the Fast strategy - /// backend (`FastKernelMatcher`). Other backends ignore. - fast_hash_log: u32, - /// Donor `cParams.minMatch` (mls) — only consumed by the Fast - /// strategy backend. Range 4..=8 per donor's mml dispatch. - fast_mls: u32, - /// Donor's `stepSize = targetLength + !(targetLength) + 1` - /// (min 2). For Fast strategy, negative levels use - /// `targetLength = -level` (1..7), giving step_size 2..8. - /// L1 / L2 / Uncompressed use targetLength=0 → step_size=2. - /// Drives the kernel's initial `step` for the 4-cursor body's - /// skip schedule. - fast_step_size: usize, lazy_depth: u8, - /// Donor dfast tuning (`clevels.h` columns). `Some` only on Dfast level - /// rows; `None` elsewhere so the table self-documents which levels the - /// Dfast backend configures. + /// Per-strategy tuning. Exactly one is `Some` on each level row, matching + /// `strategy_tag`'s backend, so the table self-documents which knobs a + /// level actually consumes (the others are `None`, not dead placeholders): + /// `fast` for the Fast/Simple backend, `dfast` for Double-Fast, `hc` for + /// the HashChain (lazy / btopt / btultra*) backend, `row` for the Row + /// (greedy L5) backend. + fast: Option, dfast: Option, - hc: HcConfig, - row: RowConfig, + hc: Option, + row: Option, } impl LevelParams { @@ -407,12 +429,14 @@ fn row_hash_bits_for_window(max_window_size: usize) -> usize { /// Index 0 = level 1, index 21 = level 22. #[rustfmt::skip] const LEVEL_TABLE: [LevelParams; 22] = [ - // Lvl Strategy wlog fast_hlog fast_mls fast_step lazy HC config row config - // --- -------------- ---- --------- -------- --------- ---- ------------------------------------------ ---------- - /* 1 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Fast, search: super::strategy::SearchMethod::Fast, window_log: 19, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 0, dfast: None, hc: HC_CONFIG, row: ROW_CONFIG }, - /* 2 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Fast, search: super::strategy::SearchMethod::Fast, window_log: 20, fast_hash_log: 16, fast_mls: 6, fast_step_size: 2, lazy_depth: 0, dfast: None, hc: HC_CONFIG, row: ROW_CONFIG }, - /* 3 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Dfast, search: super::strategy::SearchMethod::DoubleFast, window_log: 21, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 1, dfast: Some(DFAST_L3), hc: HC_CONFIG, row: ROW_CONFIG }, - /* 4 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Dfast, search: super::strategy::SearchMethod::DoubleFast, window_log: 21, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 1, dfast: Some(DFAST_L4), hc: HC_CONFIG, row: ROW_CONFIG }, + // Exactly one of fast/dfast/hc/row is Some per row, matching the strategy + // backend; the rest are None (not dead placeholders). + // Lvl Strategy wlog lazy per-strategy config + // --- -------------- ---- ---- ------------------- + /* 1 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Fast, search: super::strategy::SearchMethod::Fast, window_log: 19, lazy_depth: 0, fast: Some(FAST_L1), dfast: None, hc: None, row: None }, + /* 2 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Fast, search: super::strategy::SearchMethod::Fast, window_log: 20, lazy_depth: 0, fast: Some(FAST_L2), dfast: None, hc: None, row: None }, + /* 3 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Dfast, search: super::strategy::SearchMethod::DoubleFast, window_log: 21, lazy_depth: 1, fast: None, dfast: Some(DFAST_L3), hc: None, row: None }, + /* 4 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Dfast, search: super::strategy::SearchMethod::DoubleFast, window_log: 21, lazy_depth: 1, fast: None, dfast: Some(DFAST_L4), hc: None, row: None }, // target_len column for L5..=L15 matches donor cParams.targetLength // from clevels.h table[0] (default — srcSize > 256 KB). Donor uses // it as the lazy outer loop's `sufficient_len` (nice-match) threshold. @@ -420,14 +444,14 @@ const LEVEL_TABLE: [LevelParams; 22] = [ // search_depth iterations instead of breaking on the first // long-enough match — the dominant cost in the L5..=L15 speed // regression vs FFI (see lazy_band_target_len_matches_donor_default_table). - /* 5 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Greedy, search: super::strategy::SearchMethod::RowHash, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 0, dfast: None, hc: HcConfig { hash_log: 18, chain_log: 17, search_depth: 4, target_len: 2 }, row: ROW_L5 }, - /* 6 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 21, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 1, dfast: None, hc: HcConfig { hash_log: 19, chain_log: 18, search_depth: 8, target_len: 4 }, row: ROW_CONFIG }, - /* 7 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 21, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 1, dfast: None, hc: HcConfig { hash_log: 20, chain_log: 19, search_depth: 16, target_len: 8 }, row: ROW_CONFIG }, - /* 8 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 21, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: HcConfig { hash_log: 20, chain_log: 19, search_depth: 16, target_len: 16 }, row: ROW_CONFIG }, - /* 9 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: HcConfig { hash_log: 21, chain_log: 20, search_depth: 16, target_len: 16 }, row: ROW_CONFIG }, - /*10 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: HcConfig { hash_log: 22, chain_log: 21, search_depth: 32, target_len: 16 }, row: ROW_CONFIG }, - /*11 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: HcConfig { hash_log: 22, chain_log: 21, search_depth: 64, target_len: 16 }, row: ROW_CONFIG }, - /*12 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: HcConfig { hash_log: 23, chain_log: 22, search_depth: 64, target_len: 32 }, row: ROW_CONFIG }, + /* 5 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Greedy, search: super::strategy::SearchMethod::RowHash, window_log: 21, lazy_depth: 0, fast: None, dfast: None, hc: None, row: Some(ROW_L5) }, + /* 6 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 21, lazy_depth: 1, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 19, chain_log: 18, search_depth: 8, target_len: 4 }), row: None }, + /* 7 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 21, lazy_depth: 1, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 20, chain_log: 19, search_depth: 16, target_len: 8 }), row: None }, + /* 8 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 21, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 20, chain_log: 19, search_depth: 16, target_len: 16 }), row: None }, + /* 9 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 21, chain_log: 20, search_depth: 16, target_len: 16 }), row: None }, + /*10 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 22, chain_log: 21, search_depth: 32, target_len: 16 }), row: None }, + /*11 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 22, chain_log: 21, search_depth: 64, target_len: 16 }), row: None }, + /*12 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 23, chain_log: 22, search_depth: 64, target_len: 32 }), row: None }, // L13-15: reference uses btlazy2 (binary-tree finder) with searchLog 4/5/6 // (search_depth 16/32/64) and targetLength 32. We run the hash-chain Lazy // parser here, so we mirror the reference search budget rather than inflate @@ -436,16 +460,16 @@ const LEVEL_TABLE: [LevelParams; 22] = [ // smaller searchLog find longer matches (and re-establish a strict ratio // ladder above L12) is tracked separately; until it lands these levels sit // close to L12 on hash-chain inputs by design. - /*13 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: HcConfig { hash_log: 22, chain_log: 22, search_depth: 16, target_len: 32 }, row: ROW_CONFIG }, - /*14 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: HcConfig { hash_log: 23, chain_log: 22, search_depth: 32, target_len: 32 }, row: ROW_CONFIG }, - /*15 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: HcConfig { hash_log: 23, chain_log: 23, search_depth: 64, target_len: 32 }, row: ROW_CONFIG }, - /*16 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtOpt, search: super::strategy::SearchMethod::BinaryTree, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: HcConfig { hash_log: 22, chain_log: 22, search_depth: 32, target_len: 48 }, row: ROW_CONFIG }, - /*17 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtOpt, search: super::strategy::SearchMethod::BinaryTree, window_log: 23, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: HcConfig { hash_log: 22, chain_log: 23, search_depth: 32, target_len: 64 }, row: ROW_CONFIG }, - /*18 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra, search: super::strategy::SearchMethod::BinaryTree, window_log: 23, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: HcConfig { hash_log: 22, chain_log: 23, search_depth: 64, target_len: 64 }, row: ROW_CONFIG }, - /*19 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log: 23, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: HcConfig { hash_log: 22, chain_log: 24, search_depth: 128, target_len: 256 }, row: ROW_CONFIG }, - /*20 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log: 25, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: HcConfig { hash_log: 23, chain_log: 25, search_depth: 128, target_len: 256 }, row: ROW_CONFIG }, - /*21 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log: 26, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: BTULTRA2_HC_CONFIG, row: ROW_CONFIG }, - /*22 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log: 27, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, dfast: None, hc: BTULTRA2_HC_CONFIG_L22, row: ROW_CONFIG }, + /*13 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 22, chain_log: 22, search_depth: 16, target_len: 32 }), row: None }, + /*14 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 23, chain_log: 22, search_depth: 32, target_len: 32 }), row: None }, + /*15 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 23, chain_log: 23, search_depth: 64, target_len: 32 }), row: None }, + /*16 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtOpt, search: super::strategy::SearchMethod::BinaryTree, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 22, chain_log: 22, search_depth: 32, target_len: 48 }), row: None }, + /*17 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtOpt, search: super::strategy::SearchMethod::BinaryTree, window_log: 23, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 22, chain_log: 23, search_depth: 32, target_len: 64 }), row: None }, + /*18 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra, search: super::strategy::SearchMethod::BinaryTree, window_log: 23, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 22, chain_log: 23, search_depth: 64, target_len: 64 }), row: None }, + /*19 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log: 23, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 22, chain_log: 24, search_depth: 128, target_len: 256 }), row: None }, + /*20 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log: 25, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 23, chain_log: 25, search_depth: 128, target_len: 256 }), row: None }, + /*21 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log: 26, lazy_depth: 2, fast: None, dfast: None, hc: Some(BTULTRA2_HC_CONFIG), row: None }, + /*22 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log: 27, lazy_depth: 2, fast: None, dfast: None, hc: Some(BTULTRA2_HC_CONFIG_L22), row: None }, ]; /// Smallest window_log the encoder will use regardless of source size. @@ -494,16 +518,24 @@ fn adjust_params_for_source_size(mut params: LevelParams, src_size: u64) -> Leve let table_log = raw_src_log.max(MIN_WINDOW_LOG); let backend = params.backend(); if backend == super::strategy::BackendTag::HashChain { - if (table_log + 2) < params.hc.hash_log as u8 { - params.hc.hash_log = (table_log + 2) as usize; + let hc = params + .hc + .as_mut() + .expect("HashChain level row carries an HcConfig"); + if (table_log + 2) < hc.hash_log as u8 { + hc.hash_log = (table_log + 2) as usize; } - if (table_log + 1) < params.hc.chain_log as u8 { - params.hc.chain_log = (table_log + 1) as usize; + if (table_log + 1) < hc.chain_log as u8 { + hc.chain_log = (table_log + 1) as usize; } } else if backend == super::strategy::BackendTag::Simple { + let fast = params + .fast + .as_mut() + .expect("Fast level row carries a FastConfig"); let fast_cap = (table_log + 1) as u32; - if fast_cap < params.fast_hash_log { - params.fast_hash_log = fast_cap; + if fast_cap < fast.hash_log { + fast.hash_log = fast_cap; } } params @@ -535,13 +567,11 @@ fn level22_btultra2_params_for_source_size(source_size: Option) -> LevelPar strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log, - fast_hash_log: 14, - fast_mls: 7, - fast_step_size: 2, lazy_depth: 2, + fast: None, dfast: None, - hc, - row: ROW_CONFIG, + hc: Some(hc), + row: None, } } @@ -559,17 +589,18 @@ fn resolve_level_params(level: CompressionLevel, source_size: Option) -> Le // history; advertising a larger window only inflates // decoder-side buffer reservation. Stay at 17 (128 KiB). window_log: 17, - // Beyond-donor: hash_log=14 (vs donor's 13) for 2× fewer - // collisions on structured corpora. - fast_hash_log: 14, - fast_mls: 6, - // Donor's "base for negative" row has targetLength=1, - // which gives step_size = 1 + 0 + 1 = 2. - fast_step_size: 2, lazy_depth: 0, + // Beyond-donor: hash_log=14 (vs donor's 13) for 2× fewer + // collisions on structured corpora. Donor's "base for negative" + // row has targetLength=1 → step_size = 1 + 0 + 1 = 2. + fast: Some(FastConfig { + hash_log: 14, + mls: 6, + step_size: 2, + }), dfast: None, - hc: HC_CONFIG, - row: ROW_CONFIG, + hc: None, + row: None, }, CompressionLevel::Fastest => { // Only the Fast-specific cParams @@ -579,9 +610,11 @@ fn resolve_level_params(level: CompressionLevel, source_size: Option) -> Le // does real compression on a full window, unlike // Uncompressed which clamps to 17. let mut p = LEVEL_TABLE[0]; - p.fast_hash_log = 14; - p.fast_mls = 6; - p.fast_step_size = 2; + p.fast = Some(FastConfig { + hash_log: 14, + mls: 6, + step_size: 2, + }); p } CompressionLevel::Default => LEVEL_TABLE[2], @@ -619,13 +652,15 @@ fn resolve_level_params(level: CompressionLevel, source_size: Option) -> Le strategy_tag: super::strategy::StrategyTag::Fast, search: super::strategy::SearchMethod::Fast, window_log: 19, - fast_hash_log: 13, - fast_mls: 7, - fast_step_size: step_size, lazy_depth: 0, + fast: Some(FastConfig { + hash_log: 13, + mls: 7, + step_size, + }), dfast: None, - hc: HC_CONFIG, - row: ROW_CONFIG, + hc: None, + row: None, } } } @@ -1256,6 +1291,25 @@ impl Matcher for MatchGeneratorDriver { if let Some((search, parse)) = self.config_override.take() { params.search = search; params.lazy_depth = parse.lazy_depth(); + // The matrix sweep can pair a level with a backend its native + // row doesn't populate (e.g. greedy L5, which carries only `row`, + // run on HashChain). Synthesize a default config for the + // overridden backend so its `configure` arm has something to read. + use super::strategy::SearchMethod; + match search { + SearchMethod::Fast => { + params.fast.get_or_insert(FAST_L1); + } + SearchMethod::DoubleFast => { + params.dfast.get_or_insert(DFAST_L3); + } + SearchMethod::RowHash => { + params.row.get_or_insert(ROW_CONFIG); + } + SearchMethod::HashChain | SearchMethod::BinaryTree => { + params.hc.get_or_insert(HC_CONFIG); + } + } } let next_backend = params.backend(); let max_window_size = 1usize << params.window_log; @@ -1328,11 +1382,12 @@ impl Matcher for MatchGeneratorDriver { // get donor row-0 (hash_log=13, mls=7); Fastest / // Uncompressed keep (hash_log=14, mls=6). See // resolve_level_params for rationale. + let fast = params.fast.expect("Fast level row carries a FastConfig"); MatcherStorage::Simple(FastKernelMatcher::with_params( params.window_log, - params.fast_hash_log, - params.fast_mls, - params.fast_step_size, + fast.hash_log, + fast.mls, + fast.step_size, )) } super::strategy::BackendTag::Dfast => { @@ -1387,12 +1442,8 @@ impl Matcher for MatchGeneratorDriver { // Per-level Fast cParams threaded from // resolve_level_params (see Simple-backend swap // arm above for the (level → params) mapping). - m.reset( - params.window_log, - params.fast_hash_log, - params.fast_mls, - params.fast_step_size, - ); + let fast = params.fast.expect("Fast level row carries a FastConfig"); + m.reset(params.window_log, fast.hash_log, fast.mls, fast.step_size); } MatcherStorage::Dfast(dfast) => { dfast.max_window_size = max_window_size; @@ -1429,7 +1480,7 @@ impl Matcher for MatchGeneratorDriver { MatcherStorage::Row(row) => { row.max_window_size = max_window_size; row.lazy_depth = params.lazy_depth; - row.configure(params.row); + row.configure(params.row.expect("Row level row carries a RowConfig")); if hinted { resolved_table_bits = row_hash_bits_for_window(table_window_size); row.set_hash_bits(resolved_table_bits); @@ -1439,7 +1490,11 @@ impl Matcher for MatchGeneratorDriver { MatcherStorage::HashChain(hc) => { hc.table.max_window_size = max_window_size; hc.hc.lazy_depth = params.lazy_depth; - hc.configure(params.hc, strategy_tag, params.window_log); + hc.configure( + params.hc.expect("HashChain level row carries an HcConfig"), + strategy_tag, + params.window_log, + ); let vec_pool = &mut self.vec_pool; hc.reset(|mut data| { data.resize(data.capacity(), 0); @@ -5375,10 +5430,11 @@ fn level_20_22_map_to_btultra2_strategy() { fn level22_uses_donor_target_length_and_large_input_tables() { let params = resolve_level_params(CompressionLevel::Level(22), None); assert_eq!(params.window_log, 27); - assert_eq!(params.hc.hash_log, 25); - assert_eq!(params.hc.chain_log, 27); - assert_eq!(params.hc.search_depth, 1 << 9); - assert_eq!(params.hc.target_len, 999); + let hc = params.hc.unwrap(); + assert_eq!(hc.hash_log, 25); + assert_eq!(hc.chain_log, 27); + assert_eq!(hc.search_depth, 1 << 9); + assert_eq!(hc.target_len, 999); } #[test] @@ -5401,10 +5457,11 @@ fn bt_levels_16_to_21_pin_clevels_params() { for (level, wlog, hlog, clog, sd, tl) in expected { let p = resolve_level_params(CompressionLevel::Level(level as i32), None); assert_eq!(p.window_log, wlog, "level {level} window_log"); - assert_eq!(p.hc.hash_log, hlog, "level {level} hash_log"); - assert_eq!(p.hc.chain_log, clog, "level {level} chain_log"); - assert_eq!(p.hc.search_depth, sd, "level {level} search_depth"); - assert_eq!(p.hc.target_len, tl, "level {level} target_len"); + let hc = p.hc.unwrap(); + assert_eq!(hc.hash_log, hlog, "level {level} hash_log"); + assert_eq!(hc.chain_log, clog, "level {level} chain_log"); + assert_eq!(hc.search_depth, sd, "level {level} search_depth"); + assert_eq!(hc.target_len, tl, "level {level} target_len"); } } @@ -5412,24 +5469,27 @@ fn bt_levels_16_to_21_pin_clevels_params() { fn level22_source_size_hint_uses_donor_btultra2_tiers() { let p16k = resolve_level_params(CompressionLevel::Level(22), Some(16 * 1024)); assert_eq!(p16k.window_log, 14); - assert_eq!(p16k.hc.hash_log, 15); - assert_eq!(p16k.hc.chain_log, 15); - assert_eq!(p16k.hc.search_depth, 1 << 10); - assert_eq!(p16k.hc.target_len, 999); + let hc16k = p16k.hc.unwrap(); + assert_eq!(hc16k.hash_log, 15); + assert_eq!(hc16k.chain_log, 15); + assert_eq!(hc16k.search_depth, 1 << 10); + assert_eq!(hc16k.target_len, 999); let p128k = resolve_level_params(CompressionLevel::Level(22), Some(128 * 1024)); assert_eq!(p128k.window_log, 17); - assert_eq!(p128k.hc.hash_log, 17); - assert_eq!(p128k.hc.chain_log, 18); - assert_eq!(p128k.hc.search_depth, 1 << 11); - assert_eq!(p128k.hc.target_len, 999); + let hc128k = p128k.hc.unwrap(); + assert_eq!(hc128k.hash_log, 17); + assert_eq!(hc128k.chain_log, 18); + assert_eq!(hc128k.search_depth, 1 << 11); + assert_eq!(hc128k.target_len, 999); let p256k = resolve_level_params(CompressionLevel::Level(22), Some(256 * 1024)); assert_eq!(p256k.window_log, 18); - assert_eq!(p256k.hc.hash_log, 19); - assert_eq!(p256k.hc.chain_log, 19); - assert_eq!(p256k.hc.search_depth, 1 << 13); - assert_eq!(p256k.hc.target_len, 999); + let hc256k = p256k.hc.unwrap(); + assert_eq!(hc256k.hash_log, 19); + assert_eq!(hc256k.chain_log, 19); + assert_eq!(hc256k.search_depth, 1 << 13); + assert_eq!(hc256k.target_len, 999); } #[test] @@ -5440,12 +5500,13 @@ fn level22_small_source_size_hint_matches_donor_cparams() { let donor = unsafe { zstd_sys::ZSTD_getCParams(22, source_size, 0) }; let params = resolve_level_params(CompressionLevel::Level(22), Some(source_size)); + let hc = params.hc.unwrap(); assert_eq!(params.window_log as u32, donor.windowLog); - assert_eq!(params.hc.chain_log as u32, donor.chainLog); - assert_eq!(params.hc.hash_log as u32, donor.hashLog); - assert_eq!(params.hc.search_depth as u32, 1u32 << donor.searchLog); + assert_eq!(hc.chain_log as u32, donor.chainLog); + assert_eq!(hc.hash_log as u32, donor.hashLog); + assert_eq!(hc.search_depth as u32, 1u32 << donor.searchLog); assert_eq!(HC_OPT_MIN_MATCH_LEN as u32, donor.minMatch); - assert_eq!(params.hc.target_len as u32, donor.targetLength); + assert_eq!(hc.target_len as u32, donor.targetLength); } #[test] @@ -9460,10 +9521,12 @@ fn fastest_hint_iteration_23_sequences_reconstruct_source() { fn fast_levels_dispatch_per_level_hash_log_and_mls() { // Level 1 — donor `{ 19, 13, 14, 1, 7, 0, ZSTD_fast }` row: // window_log=19, hash_log=14, mls=7. - let p1 = resolve_level_params(CompressionLevel::Level(1), None); - assert_eq!(p1.fast_hash_log, 14); - assert_eq!(p1.fast_mls, 7); - assert_eq!(p1.fast_step_size, 2); + let f1 = resolve_level_params(CompressionLevel::Level(1), None) + .fast + .unwrap(); + assert_eq!(f1.hash_log, 14); + assert_eq!(f1.mls, 7); + assert_eq!(f1.step_size, 2); // Negative levels — donor row-0 ("base for negative"): // hash_log=13, mls=7. The 32 KiB table is L1d-resident (every @@ -9473,35 +9536,29 @@ fn fast_levels_dispatch_per_level_hash_log_and_mls() { // step_size follows donor's formula: targetLength = -level, // step_size = (-level) + 1, giving 2..8 for L-1..L-7. for n in -7..=-1 { - let p = resolve_level_params(CompressionLevel::Level(n), None); - assert_eq!(p.fast_hash_log, 13, "Level({n}) fast_hash_log"); - assert_eq!(p.fast_mls, 7, "Level({n}) fast_mls"); + let f = resolve_level_params(CompressionLevel::Level(n), None) + .fast + .unwrap(); + assert_eq!(f.hash_log, 13, "Level({n}) fast_hash_log"); + assert_eq!(f.mls, 7, "Level({n}) fast_mls"); let expected_step = ((-n) as usize) + 1; - assert_eq!(p.fast_step_size, expected_step, "Level({n}) fast_step_size"); + assert_eq!(f.step_size, expected_step, "Level({n}) fast_step_size"); } // Fastest + Uncompressed keep hash_log=14 / mls=6 (their own // tuning; not part of the negative-level donor ladder). let pf = resolve_level_params(CompressionLevel::Fastest, None); + let ff = pf.fast.unwrap(); assert_eq!( - ( - pf.window_log, - pf.fast_hash_log, - pf.fast_mls, - pf.fast_step_size - ), + (pf.window_log, ff.hash_log, ff.mls, ff.step_size), (19, 14, 6, 2), ); // Uncompressed keeps window_log=17 (no history references, smaller // decoder reservation); fast cParams same as negative-base row. let pu = resolve_level_params(CompressionLevel::Uncompressed, None); + let fu = pu.fast.unwrap(); assert_eq!( - ( - pu.window_log, - pu.fast_hash_log, - pu.fast_mls, - pu.fast_step_size - ), + (pu.window_log, fu.hash_log, fu.mls, fu.step_size), (17, 14, 6, 2), ); } @@ -9556,20 +9613,21 @@ fn fast_levels_driver_wiring_threads_cparams_into_inner_matcher() { // by FrameCompressor / StreamingEncoder). crate::encoding::Matcher::reset(&mut driver, level); + let f = p.fast.unwrap(); let m = driver.simple_mut(); assert_eq!( m.hash_log(), - p.fast_hash_log, + f.hash_log, "{level:?}: inner matcher hash_log mismatch — argument swap?", ); assert_eq!( m.mls(), - p.fast_mls, + f.mls, "{level:?}: inner matcher mls mismatch — argument swap?", ); assert_eq!( m.step_size(), - p.fast_step_size, + f.step_size, "{level:?}: inner matcher step_size mismatch — stale value carried from prior reset?", ); } @@ -9596,10 +9654,17 @@ fn lazy_band_target_len_matches_donor_default_table() { // call with any (level, srcSize, dictSize) combination. let reference = unsafe { zstd_sys::ZSTD_getCParams(level, 0, 0) }; let params = resolve_level_params(CompressionLevel::Level(level), None); + // L5 = greedy (Row backend → `row`); L6-15 = lazy (HashChain → `hc`). + // Both surface the donor `targetLength` as their nice-match threshold. + let target_len = params + .hc + .map(|hc| hc.target_len) + .or_else(|| params.row.map(|row| row.target_len)) + .expect("lazy/greedy level carries hc or row config"); assert_eq!( - params.hc.target_len as u32, reference.targetLength, - "L{level}: hc.target_len ({}) must match reference cParams.targetLength ({})", - params.hc.target_len, reference.targetLength + target_len as u32, reference.targetLength, + "L{level}: target_len ({target_len}) must match reference cParams.targetLength ({})", + reference.targetLength ); } } @@ -9621,11 +9686,12 @@ fn upper_lazy_band_params_match_donor_default_table() { // call with any (level, srcSize, dictSize) combination. let reference = unsafe { zstd_sys::ZSTD_getCParams(level, 0, 0) }; let params = resolve_level_params(CompressionLevel::Level(level), None); + let hc = params.hc.unwrap(); assert_eq!( - params.hc.search_depth as u32, + hc.search_depth as u32, 1u32 << reference.searchLog, "L{level}: hc.search_depth ({}) must equal 1< Date: Sat, 6 Jun 2026 16:12:39 +0300 Subject: [PATCH 32/33] refactor(encode): name raw-fast-path window cutoff for its purpose The incompressible raw-fast-path window ceiling reused BETTER_WINDOW_LOG, which no longer matches the Better preset window after the donor alignment. Give it a self-describing name (RAW_FAST_PATH_MAX_WINDOW_LOG = 23, 8 MiB) in incompressible.rs and drop the now-unused BETTER_WINDOW_LOG constant. Value unchanged. Also corrects a stale fast-matcher test doc reference. --- zstd/src/encoding/incompressible.rs | 19 ++++++++++++------- zstd/src/encoding/mod.rs | 2 -- zstd/src/encoding/simple/fast_matcher.rs | 3 +-- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/zstd/src/encoding/incompressible.rs b/zstd/src/encoding/incompressible.rs index 1513b75c2..722141162 100644 --- a/zstd/src/encoding/incompressible.rs +++ b/zstd/src/encoding/incompressible.rs @@ -1,9 +1,14 @@ -use super::{BETTER_WINDOW_LOG, CompressionLevel}; +use super::CompressionLevel; pub(crate) const RAW_FAST_PATH_MIN_BLOCK_LEN: usize = 512; pub(crate) const RAW_FAST_PATH_MAX_SAMPLE_LEN: usize = 4096; pub(crate) const RAW_FAST_PATH_MIN_SAMPLE_LEN: usize = 32; -const BETTER_WINDOW_SIZE_BYTES: u64 = 1u64 << BETTER_WINDOW_LOG; +/// Window-size ceiling (8 MiB) above which the incompressible raw-fast-path is +/// disabled for `Best` / numeric levels: the largest-window levels (L20-22) +/// do full match-finding even on apparently-incompressible blocks rather than +/// risk emitting raw blocks where a far back-reference might still pay off. +const RAW_FAST_PATH_MAX_WINDOW_LOG: u8 = 23; +const RAW_FAST_PATH_MAX_WINDOW_SIZE_BYTES: u64 = 1u64 << RAW_FAST_PATH_MAX_WINDOW_LOG; // Keep classifier scratch modest for no_std/small-stack targets: 1024 slots // cuts per-call stack for repeat tracking from ~8 KiB to ~4 KiB. @@ -77,8 +82,8 @@ pub(crate) fn compression_level_allows_raw_fast_path( ) -> bool { match level { CompressionLevel::Fastest | CompressionLevel::Default | CompressionLevel::Better => true, - CompressionLevel::Best => window_size <= BETTER_WINDOW_SIZE_BYTES, - CompressionLevel::Level(_) => window_size <= BETTER_WINDOW_SIZE_BYTES, + CompressionLevel::Best => window_size <= RAW_FAST_PATH_MAX_WINDOW_SIZE_BYTES, + CompressionLevel::Level(_) => window_size <= RAW_FAST_PATH_MAX_WINDOW_SIZE_BYTES, CompressionLevel::Uncompressed => false, } } @@ -324,11 +329,11 @@ mod tests { fn best_raw_fast_path_requires_better_sized_window() { assert!(compression_level_allows_raw_fast_path( CompressionLevel::Best, - BETTER_WINDOW_SIZE_BYTES + RAW_FAST_PATH_MAX_WINDOW_SIZE_BYTES )); assert!(!compression_level_allows_raw_fast_path( CompressionLevel::Best, - BETTER_WINDOW_SIZE_BYTES + 1 + RAW_FAST_PATH_MAX_WINDOW_SIZE_BYTES + 1 )); } @@ -336,7 +341,7 @@ mod tests { fn level4_row_raw_fast_path_allowed_with_better_window_reach() { assert!(compression_level_allows_raw_fast_path( CompressionLevel::Level(4), - BETTER_WINDOW_SIZE_BYTES + RAW_FAST_PATH_MAX_WINDOW_SIZE_BYTES )); } diff --git a/zstd/src/encoding/mod.rs b/zstd/src/encoding/mod.rs index 2bef67acf..5a9031372 100644 --- a/zstd/src/encoding/mod.rs +++ b/zstd/src/encoding/mod.rs @@ -96,8 +96,6 @@ pub use streaming_encoder::StreamingEncoder; use crate::io::{Read, Write}; use alloc::vec::Vec; -pub(crate) const BETTER_WINDOW_LOG: u8 = 23; - /// Convenience function to compress some source into a target without reusing any resources of the compressor /// ```rust /// use structured_zstd::encoding::{compress, CompressionLevel}; diff --git a/zstd/src/encoding/simple/fast_matcher.rs b/zstd/src/encoding/simple/fast_matcher.rs index 7169e65bd..c3b6ae8fa 100644 --- a/zstd/src/encoding/simple/fast_matcher.rs +++ b/zstd/src/encoding/simple/fast_matcher.rs @@ -1636,8 +1636,7 @@ mod tests { let m = FastKernelMatcher::with_params(16, 12, 5, 2); assert_eq!(m.window_size(), 1u64 << 16); // Larger window_log → larger reported window. window_log = 22 - // (4 MiB, donor's BETTER_WINDOW_LOG) confirms the shift width - // (`u64` head room). + // (4 MiB) confirms the shift width (`u64` head room). let m = FastKernelMatcher::with_params(22, 14, 7, 2); assert_eq!(m.window_size(), 1u64 << 22); } From 143ec1105ab375e167bcbc9c441be99d481ef9dc Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 6 Jun 2026 17:04:26 +0300 Subject: [PATCH 33/33] test(encode): pin fast borrowed split decision + raw-fast-path over-cap - fast_oneshot_borrowed_split_emits_subblock: assert optimal_block_size for Fastest resolves the second donor block below MAX_BLOCK_SIZE before encoding, and drive the borrowed route explicitly via compress_independent_frame, so the block-count check cannot pass through the owned loop. - level4 raw-fast-path test: add the over-cap reject case (RAW_FAST_PATH_MAX_WINDOW_SIZE_BYTES + 1) so numeric levels and Best stay locked to the same window boundary. --- zstd/src/encoding/frame_compressor.rs | 22 ++++++++++++++++++++-- zstd/src/encoding/incompressible.rs | 6 ++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index 5aeac3eb2..a927ecff5 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -3338,7 +3338,7 @@ mod tests { /// the second block yields >= 3 decoded blocks, asserted on the round-trip. #[test] fn fast_oneshot_borrowed_split_emits_subblock() { - use crate::encoding::{CompressionLevel, compress_slice_to_vec}; + use crate::encoding::CompressionLevel; // First 192 KiB: homogeneous zero run (banks the savings the split // gate needs). The second 128 KiB block flips to a counter sequence // at its 64 KiB midpoint (the 192 KiB mark) — a fingerprint @@ -3351,7 +3351,25 @@ mod tests { } } - let frame = compress_slice_to_vec(&data, CompressionLevel::Fastest); + // Pin the splitter decision for the Fast path directly (mirrors the + // greedy test): the second donor block must resolve to a sub-block + // boundary, so the >= 3 block count below cannot pass vacuously. + let second_block = &data[128 * 1024..]; + assert!( + super::optimal_block_size( + CompressionLevel::Fastest, + second_block, + second_block.len(), + MAX_BLOCK_SIZE as usize, + 100, + ) < MAX_BLOCK_SIZE as usize, + "fixture must resolve to a sub-block split in the second donor block", + ); + + // Drive the borrowed one-shot route explicitly (Fast level -> + // run_borrowed_block_loop via compress_independent_frame). + let mut compressor: FrameCompressor = FrameCompressor::new(CompressionLevel::Fastest); + let frame = compressor.compress_independent_frame(&data); let mut decoder = FrameDecoder::new(); let mut source = frame.as_slice(); diff --git a/zstd/src/encoding/incompressible.rs b/zstd/src/encoding/incompressible.rs index 722141162..7344336e8 100644 --- a/zstd/src/encoding/incompressible.rs +++ b/zstd/src/encoding/incompressible.rs @@ -343,6 +343,12 @@ mod tests { CompressionLevel::Level(4), RAW_FAST_PATH_MAX_WINDOW_SIZE_BYTES )); + // Over-cap numeric level is rejected, same boundary as `Best`, so the + // two branches can't drift apart. + assert!(!compression_level_allows_raw_fast_path( + CompressionLevel::Level(4), + RAW_FAST_PATH_MAX_WINDOW_SIZE_BYTES + 1 + )); } #[test]