From d77a0bd60ee00c0e8b9bd6230dc5013cd8123e6e Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Wed, 10 Jun 2026 11:37:41 +0300 Subject: [PATCH 01/22] perf(encode): reuse rollback entropy snapshot buffers per block - manual HuffmanTable::clone_from reuses destination Vec buffers (derived version degraded to a full clone: 2 mallocs + 2 frees per per-frame dict entropy seed and per-block rollback snapshot) - compress_block_encoded (owned + borrowed) snapshots the Huffman table into a persistent scratch slot via clone_from instead of a fresh clone every block Profiled on small-4k-log-lines level_2_fast compress-dict: per-block clone churn (clone memmove + malloc/free pairs) was ~8% of frame time. --- zstd/src/encoding/blocks/compressed.rs | 5 +++++ zstd/src/encoding/levels/fastest.rs | 29 ++++++++++++++++++++++---- zstd/src/huff0/huff0_encoder.rs | 28 ++++++++++++++++++++++++- 3 files changed, 57 insertions(+), 5 deletions(-) diff --git a/zstd/src/encoding/blocks/compressed.rs b/zstd/src/encoding/blocks/compressed.rs index c82b9e13a..9f83ed550 100644 --- a/zstd/src/encoding/blocks/compressed.rs +++ b/zstd/src/encoding/blocks/compressed.rs @@ -150,6 +150,11 @@ pub(crate) struct CompressedBlockScratch { /// first block-split). The estimator uses `EntropyOnlyMatcher` and /// never re-splits, so this nesting is one level deep. estimator_inner: Option>, + /// Persistent slot for `compress_block_encoded`'s pre-block entropy + /// rollback snapshot. `clone_from` into this slot reuses its `Vec` + /// buffers across blocks; a fresh `.clone()` per block paid a + /// malloc + free pair on both Huffman code containers every block. + pub(crate) huff_rollback: Option, } impl CompressedBlockScratch { diff --git a/zstd/src/encoding/levels/fastest.rs b/zstd/src/encoding/levels/fastest.rs index 9e74fb4ac..9219f0f90 100644 --- a/zstd/src/encoding/levels/fastest.rs +++ b/zstd/src/encoding/levels/fastest.rs @@ -155,7 +155,13 @@ pub(crate) fn compress_block_encoded( // `compress_block` can mutate entropy/history state before we know // whether the compressed payload fits `MAX_BLOCK_SIZE`. let saved_offset_hist = state.offset_hist; - let saved_huff_table = state.last_huff_table.clone(); + // Snapshot the Huffman table into the scratch's persistent rollback + // slot: `clone_from` reuses the slot's buffers across blocks (a + // fresh `.clone()` paid a malloc + free pair on both code containers + // every block). FSE previous tables are `SharedFseTable` handles — + // their clone is a refcount bump, no slot needed. + let mut saved_huff_table = core::mem::take(&mut state.block_scratch.huff_rollback); + saved_huff_table.clone_from(&state.last_huff_table); let saved_ll_previous = state.fse_tables.ll_previous.clone(); let saved_ml_previous = state.fse_tables.ml_previous.clone(); let saved_of_previous = state.fse_tables.of_previous.clone(); @@ -182,10 +188,13 @@ pub(crate) fn compress_block_encoded( // Roll back the payload + reserved header and the entropy state. output.truncate(hdr_off); state.offset_hist = saved_offset_hist; - state.last_huff_table = saved_huff_table; + // Swap (not move) so the slot keeps owning a reusable table + // allocation for the next block's snapshot. + core::mem::swap(&mut state.last_huff_table, &mut saved_huff_table); state.fse_tables.ll_previous = saved_ll_previous; state.fse_tables.ml_previous = saved_ml_previous; state.fse_tables.of_previous = saved_of_previous; + state.block_scratch.huff_rollback = saved_huff_table; let header = BlockHeader { last_block, block_type: BlockType::Raw, @@ -196,6 +205,9 @@ pub(crate) fn compress_block_encoded( output.extend_from_slice(state.matcher.get_last_space()); BlockType::Raw } else { + // Return the snapshot to its slot so the next block's + // `clone_from` reuses the allocation. + state.block_scratch.huff_rollback = saved_huff_table; let header = BlockHeader { last_block, block_type: BlockType::Compressed, @@ -293,7 +305,10 @@ pub(crate) fn compress_block_encoded_borrowed( sink.push(crate::encoding::frame_compressor::xxh64_block_low32(block)); } let saved_offset_hist = state.offset_hist; - let saved_huff_table = state.last_huff_table.clone(); + // Persistent rollback slot — same allocation-reuse rationale as the + // owned `compress_block_encoded` snapshot above. + let mut saved_huff_table = core::mem::take(&mut state.block_scratch.huff_rollback); + saved_huff_table.clone_from(&state.last_huff_table); let saved_ll_previous = state.fse_tables.ll_previous.clone(); let saved_ml_previous = state.fse_tables.ml_previous.clone(); let saved_of_previous = state.fse_tables.of_previous.clone(); @@ -312,10 +327,13 @@ pub(crate) fn compress_block_encoded_borrowed( // entropy state, then emit a stored Raw block. output.truncate(hdr_off); state.offset_hist = saved_offset_hist; - state.last_huff_table = saved_huff_table; + // Swap (not move) so the slot keeps owning a reusable table + // allocation for the next block's snapshot. + core::mem::swap(&mut state.last_huff_table, &mut saved_huff_table); state.fse_tables.ll_previous = saved_ll_previous; state.fse_tables.ml_previous = saved_ml_previous; state.fse_tables.of_previous = saved_of_previous; + state.block_scratch.huff_rollback = saved_huff_table; let header = BlockHeader { last_block, block_type: BlockType::Raw, @@ -325,6 +343,9 @@ pub(crate) fn compress_block_encoded_borrowed( output.extend_from_slice(block); BlockType::Raw } else { + // Return the snapshot to its slot so the next block's + // `clone_from` reuses the allocation. + state.block_scratch.huff_rollback = saved_huff_table; let header = BlockHeader { last_block, block_type: BlockType::Compressed, diff --git a/zstd/src/huff0/huff0_encoder.rs b/zstd/src/huff0/huff0_encoder.rs index 1bbaf1f7e..38f17af45 100644 --- a/zstd/src/huff0/huff0_encoder.rs +++ b/zstd/src/huff0/huff0_encoder.rs @@ -448,7 +448,6 @@ impl>> HuffmanEncoder<'_, '_, V> { } } -#[derive(Clone)] pub struct HuffmanTable { /// Index is the symbol, values are the bitstring in the lower bits of the u32 and the amount of bits in the u8 codes: Vec<(u32, u8)>, @@ -475,6 +474,33 @@ pub struct HuffmanTable { cached_encoded_weight_description: CachedDescription, } +/// Manual impl so `clone_from` reuses the destination's existing `Vec` +/// buffers; the derived version falls back to `*self = source.clone()`, +/// re-allocating both code containers on every per-frame entropy seed and +/// per-block rollback snapshot. +impl Clone for HuffmanTable { + fn clone(&self) -> Self { + Self { + codes: self.codes.clone(), + packed_codes: self.packed_codes.clone(), + table_log: self.table_log, + #[cfg(feature = "std")] + cached_encoded_weight_description: self.cached_encoded_weight_description.clone(), + } + } + + fn clone_from(&mut self, source: &Self) { + self.codes.clone_from(&source.codes); + self.packed_codes.clone_from(&source.packed_codes); + self.table_log = source.table_log; + #[cfg(feature = "std")] + { + self.cached_encoded_weight_description = + source.cached_encoded_weight_description.clone(); + } + } +} + impl HuffmanTable { /// Heap bytes this table holds: the per-symbol code table and the packed /// dual-container codes. The lazily-built weight-description cache is a From 1287a275427b9b6f5d28e5ca3548de3df8e8cec7 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Wed, 10 Jun 2026 11:37:55 +0300 Subject: [PATCH 02/22] perf(encode): feed slice inputs to owned block loop without zero-fill MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OwnedBlockSource trait splits per-block buffer filling: the &[u8] impl appends with one extend_from_slice; the generic reader keeps the historical initialized-resize + Read loop (ReaderBlockSource adapter). The slice entry points (compress_independent_frame_into and the owned fallbacks) previously paid resize(_, 0) zero-fill of the whole block + a doubling re-grow before every read — ~10% of dict-compress frame time on small-4k-log-lines level_2_fast. EOF semantics preserved exactly (eof iff the block could not be filled), so exact-multiple inputs still emit the trailing empty last block. --- zstd/src/encoding/frame_compressor.rs | 179 ++++++++++++++++++-------- 1 file changed, 123 insertions(+), 56 deletions(-) diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index ba11104db..cdc6b8f6f 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -626,6 +626,113 @@ fn initial_all_blocks_cap(initial_size_hint: Option) -> usize { } } +/// Per-block feeder for `run_owned_block_loop`. +/// +/// `fill_block` appends source bytes to `buf` (which already holds any +/// carried pre-split suffix) until `buf.len() == block_capacity` or the +/// source is exhausted, returning `(bytes_appended, reached_eof)`. +/// `reached_eof` is true iff the block could NOT be filled to +/// `block_capacity` — the boundary the historical `Read`-loop produced (an +/// input that is an exact multiple of the block size still yields a +/// trailing empty last block on the next iteration). +/// +/// The slice impl exists so the slice entry points +/// (`compress_independent_frame_into`, `compress_oneshot_*` fallbacks) +/// append with one `extend_from_slice` — the generic reader impl must +/// `resize` an initialized target region before `Read::read` can fill it, +/// which costs a zero-fill memset of the whole block on every frame. +pub(crate) trait OwnedBlockSource { + fn fill_block( + &mut self, + buf: &mut Vec, + block_capacity: usize, + size_hint_remaining: Option, + ) -> (usize, bool); +} + +impl OwnedBlockSource for &[u8] { + fn fill_block( + &mut self, + buf: &mut Vec, + block_capacity: usize, + _size_hint_remaining: Option, + ) -> (usize, bool) { + let want = block_capacity - buf.len(); + let take = want.min(self.len()); + buf.extend_from_slice(&self[..take]); + *self = &self[take..]; + (take, take < want) + } +} + +/// Adapter routing a generic [`Read`] source through [`OwnedBlockSource`]: +/// preserves the historical sizing behaviour — an initialized target region +/// bounded by the source-size hint, grown (doubling, capped) only when the +/// hint under-counted. +pub(crate) struct ReaderBlockSource(pub(crate) Rd); + +impl OwnedBlockSource for ReaderBlockSource { + fn fill_block( + &mut self, + buf: &mut Vec, + block_capacity: usize, + size_hint_remaining: Option, + ) -> (usize, bool) { + let start = buf.len(); + let mut filled = start; + let mut reached_eof = false; + // Size the read buffer to the bytes this block actually expects + // rather than always zero-filling a full MAX_BLOCK_SIZE: a small + // frame otherwise pays a 128 KiB `resize(_, 0)` memset per block + // just to read a few KiB (the zero-fill past `filled` is then + // truncated away). + // + // Overflow-free by construction (no `saturating_*` masking): + // `filled <= block_capacity` always (the read only ever targets + // `[filled..len]` with `len <= block_capacity`, and a carried-over + // pre-split suffix is a `split_off` below `block_capacity`), so + // `block_capacity - filled` never underflows; pinning `remaining` + // to `block_capacity` before the `usize` cast keeps the cast and + // the final add within `usize` on every target. + let initial_target = match size_hint_remaining { + Some(remaining) => { + let remaining = remaining.min(block_capacity as u64) as usize; + filled + remaining.min(block_capacity - filled) + } + // Unknown hint, or an inexact hint already met by prior blocks: + // read against the full block window. + None => block_capacity, + }; + if buf.len() < initial_target { + buf.resize(initial_target, 0); + } + loop { + if reached_eof || filled == block_capacity { + break; + } + if filled == buf.len() { + // Hint under-counted the block; grow toward block_capacity + // (doubling, capped) so reading continues without paying a + // full-buffer zero up front. `len <= block_capacity` so the + // double stays well within `usize`; `filled < block_capacity` + // here (the `== block_capacity` break fired otherwise), so + // `filled + 1 <= block_capacity`. + let grow_to = (buf.len() * 2).clamp(filled + 1, block_capacity); + buf.resize(grow_to, 0); + } + let read_end = buf.len(); + let new_bytes = self.0.read(&mut buf[filled..read_end]).unwrap(); + if new_bytes == 0 { + reached_eof = true; + break; + } + filled += new_bytes; + } + buf.truncate(filled); + (filled - start, reached_eof) + } +} + impl FrameCompressor { /// Create a new `FrameCompressor` pub fn new(compression_level: CompressionLevel) -> Self { @@ -1106,8 +1213,9 @@ impl FrameCompressor { // buffer and let `finish_frame` write header + blocks to the drain. let mut all_blocks: Vec = Vec::with_capacity(initial_all_blocks_cap(prep.initial_size_hint)); + let mut block_source = ReaderBlockSource(&mut source); let total_uncompressed = - self.run_owned_block_loop(&mut source, prep.initial_size_hint, &mut all_blocks); + self.run_owned_block_loop(&mut block_source, prep.initial_size_hint, &mut all_blocks); self.uncompressed_data = Some(source); self.finish_frame(all_blocks, total_uncompressed, &prep); } @@ -1298,9 +1406,9 @@ impl FrameCompressor { /// (`compress_oneshot_borrowed`, `compress_independent_frame`) feed an /// in-place `&[u8]` cursor without baking its lifetime into the /// compressor type. - fn run_owned_block_loop( + fn run_owned_block_loop( &mut self, - source: &mut Rd, + source: &mut S, initial_size_hint: Option, out: &mut Vec, ) -> u64 { @@ -1332,61 +1440,20 @@ impl FrameCompressor { let mut uncompressed_data = self.state.matcher.get_next_space(); uncompressed_data.clear(); uncompressed_data.extend_from_slice(&pending_input); - let mut filled = pending_input.len(); pending_input.clear(); - // Size the read buffer to the bytes this block actually expects - // rather than always zero-filling a full MAX_BLOCK_SIZE: a small - // frame otherwise pays a 128 KiB `resize(_, 0)` memset per block - // just to read a few KiB (the zero-fill past `filled` is then - // truncated away). Bound the initial fill by the source-size hint - // (exact for the slice entry points), and grow toward - // `block_capacity` only if the hint under-counted. - // - // Overflow-free by construction (no `saturating_*` masking): - // `filled <= block_capacity` always (the read only ever targets - // `[filled..len]` with `len <= block_capacity`, and a carried-over - // `pending_input` is a `split_off` below `block_capacity`), so - // `block_capacity - filled` never underflows; pinning `remaining` - // to `block_capacity` before the `usize` cast keeps the cast and - // the final add within `usize` on every target. - let initial_target = match initial_size_hint { - Some(hint) if hint > total_uncompressed => { - let remaining = (hint - total_uncompressed).min(block_capacity as u64) as usize; - filled + remaining.min(block_capacity - filled) - } - // Unknown hint, or an inexact hint already met by prior blocks: - // read against the full block window. - _ => block_capacity, - }; - if uncompressed_data.len() < initial_target { - uncompressed_data.resize(initial_target, 0); - } - 'read_loop: loop { - if reached_eof || filled == block_capacity { - break 'read_loop; - } - if filled == uncompressed_data.len() { - // Hint under-counted the block; grow toward block_capacity - // (doubling, capped) so reading continues without paying a - // full-buffer zero up front. `len <= block_capacity` so the - // double stays well within `usize`; `filled < block_capacity` - // here (the `== block_capacity` break fired otherwise), so - // `filled + 1 <= block_capacity`. - let grow_to = (uncompressed_data.len() * 2).clamp(filled + 1, block_capacity); - uncompressed_data.resize(grow_to, 0); - } - let read_end = uncompressed_data.len(); - let new_bytes = source - .read(&mut uncompressed_data[filled..read_end]) - .unwrap(); - if new_bytes == 0 { - reached_eof = true; - break 'read_loop; - } - filled += new_bytes; - total_uncompressed += new_bytes as u64; + if !reached_eof { + // Remaining-bytes expectation for the reader source's sizing + // (`None` = unknown, or an inexact hint already met by prior + // blocks). The slice source appends directly and ignores it. + let size_hint_remaining = match initial_size_hint { + Some(hint) if hint > total_uncompressed => Some(hint - total_uncompressed), + _ => None, + }; + let (appended, eof) = + source.fill_block(&mut uncompressed_data, block_capacity, size_hint_remaining); + total_uncompressed += appended as u64; + reached_eof = eof; } - uncompressed_data.truncate(filled); let mut last_block = reached_eof; let remaining_for_split = if reached_eof { uncompressed_data.len() From 1b5279c03f6b19c373fc9de5b15a804ed6cd52ff Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Wed, 10 Jun 2026 11:51:18 +0300 Subject: [PATCH 03/22] perf(encode): epoch-bias Fast table reset for dict-attach frames Donor ZSTD_continueCCtx cadence: instead of memsetting the whole Fast hash table on every dict-attach frame reset, keep the entries and advance an epoch bias past the previous frame's position high-water mark. FastHashTable::get maps any pre-advance entry to the empty sentinel (saturating_sub, sub+cmov); put stores position + bias. - bias applies only on the dict-attach main table; no-dict kernels go through hot_state's raw slice and are guarded by a bias==0 debug assert (copy-mode and no-dict resets keep the historical clear) - overflow falls back to a real clear (position ceiling 2^31, bias capped at u32::MAX - 2^31) - regression test: 32 reused attach frames + attach/copy cutoff alternation, all byte-identical to fresh-compressor reference Per-frame table memset was 13% of compress-dict frame time on small-4k-log-lines level_2_fast. --- zstd/src/encoding/frame_compressor.rs | 65 +++++++++++++++++++ zstd/src/encoding/match_generator.rs | 17 ++++- .../encoding/simple/fast_kernel/hash_table.rs | 52 ++++++++++++++- zstd/src/encoding/simple/fast_matcher.rs | 60 +++++++++++++++-- 4 files changed, 186 insertions(+), 8 deletions(-) diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index cdc6b8f6f..5eb42e925 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -2930,6 +2930,71 @@ mod tests { } } + #[test] + fn dict_fast_epoch_reset_many_frames_and_attach_copy_alternation_byte_identical() { + // The Fast attach path invalidates the main hash table between + // frames with an epoch-bias advance instead of a memset. Two things + // need proving against a fresh-compressor reference: + // 1. the bias accumulates across MANY reused frames without ever + // letting a stale entry through (every frame byte-identical); + // 2. crossing the 8 KiB attach/copy cutoff in both directions + // (attach → copy clears the bias for the raw-slice kernel, + // copy → attach re-enters epoch mode) stays byte-identical. + let dict_raw = include_bytes!("../../dict_tests/dictionary"); + let mut small = Vec::new(); + while small.len() < 2 * 1024 { + small.extend_from_slice(b"tenant=demo op=put key=1 value=aaaaabbbbbcccccddddd\n"); + } + // Over the Fast 8 KiB attach cutoff → copy-mode frame. + let mut large = Vec::new(); + while large.len() < 64 * 1024 { + large.extend_from_slice(b"tenant=demo op=scan range=[k0,k9) limit=500 order=asc\n"); + } + + let mut reused: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Fastest); + reused + .set_encoder_dictionary( + super::EncoderDictionary::from_bytes(dict_raw).expect("dict bytes should parse"), + ) + .expect("prepared dictionary should attach"); + + let reference = |payload: &[u8]| -> alloc::vec::Vec { + let mut fresh: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Fastest); + fresh + .set_encoder_dictionary( + super::EncoderDictionary::from_bytes(dict_raw) + .expect("dict bytes should parse"), + ) + .expect("prepared dictionary should attach"); + fresh.compress_independent_frame(payload) + }; + + let small_expected = reference(&small); + let large_expected = reference(&large); + + // 1. Long attach-only run: every frame advances the epoch bias. + for i in 0..32 { + let frame = reused.compress_independent_frame(small.as_slice()); + assert_eq!( + frame, small_expected, + "attach frame {i} diverged from the fresh-compressor reference" + ); + } + // 2. Cutoff alternation: attach → copy → attach → copy. + for i in 0..4 { + let frame = reused.compress_independent_frame(large.as_slice()); + assert_eq!( + frame, large_expected, + "copy frame {i} diverged from the fresh-compressor reference" + ); + let frame = reused.compress_independent_frame(small.as_slice()); + assert_eq!( + frame, small_expected, + "attach frame after copy {i} diverged from the fresh-compressor reference" + ); + } + } + #[test] fn dict_primed_btlazy2_reused_across_attach_and_copy_boundary_is_byte_identical() { // Btlazy2 (Level 15) uses the 32 KiB dict attach/copy cutoff in diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs index af9726962..c5e9f8101 100644 --- a/zstd/src/encoding/match_generator.rs +++ b/zstd/src/encoding/match_generator.rs @@ -1953,7 +1953,22 @@ impl Matcher for MatchGeneratorDriver { // resolve_level_params (see Simple-backend swap // arm above for the (level → params) mapping). let fast = params.fast.expect("Fast level row carries a FastConfig"); - m.reset(params.window_log, fast.hash_log, fast.mls, fast.step_size); + // Same attach/copy split the dict-prime dispatch applies + // below (`prime_with_dictionary`): only attach-mode dict + // frames may keep the main table across the reset via an + // epoch advance — copy-mode and no-dict frames must memset + // it back to bias 0 for the raw-slice kernels. + let dict_attach_epoch = dict_hint.is_some() + && self + .reset_size_log + .is_none_or(|log| log <= FAST_ATTACH_DICT_CUTOFF_LOG); + m.reset( + params.window_log, + fast.hash_log, + fast.mls, + fast.step_size, + dict_attach_epoch, + ); } MatcherStorage::Dfast(dfast) => { dfast.max_window_size = max_window_size; diff --git a/zstd/src/encoding/simple/fast_kernel/hash_table.rs b/zstd/src/encoding/simple/fast_kernel/hash_table.rs index 956181950..aeb1a9053 100644 --- a/zstd/src/encoding/simple/fast_kernel/hash_table.rs +++ b/zstd/src/encoding/simple/fast_kernel/hash_table.rs @@ -62,6 +62,17 @@ pub(crate) struct FastHashTable { /// Valid range `4..=8`; the kernel monomorphises over this so it /// compiles to a constant inside each instantiation. mls: u32, + /// Epoch bias for continue-mode frame resets (donor `ZSTD_continueCCtx` + /// cadence): stored values are `position + bias`, and [`Self::get`] + /// reads any entry below the current bias as the empty sentinel `0`. + /// Advancing the bias past every previously-stored value + /// ([`Self::advance_epoch`]) therefore invalidates the whole table + /// without the per-frame full-table memset of [`Self::clear`]. + /// + /// Always `0` on paths that access the table storage directly + /// ([`Self::hot_state`] — the no-dict kernels) and on cached dict + /// tables; only the dict-attach main table advances it. + bias: u32, } impl FastHashTable { @@ -144,6 +155,7 @@ impl FastHashTable { table: vec![0u32; entries], hash_log, mls, + bias: 0, } } @@ -169,6 +181,28 @@ impl FastHashTable { // `fill(0)` lowers to a single `memset` and is significantly // faster than re-allocating; the table can be hundreds of KiB. self.table.fill(0); + self.bias = 0; + } + + /// Continue-mode frame reset (donor `ZSTD_continueCCtx` cadence): keep + /// the table contents and advance the epoch bias past every entry the + /// previous frame stored, so all of them read back as the empty + /// sentinel via [`Self::get`]'s epoch filter — no full-table memset. + /// + /// `span` must be strictly greater than the largest unbiased position + /// stored since the last clear/advance (the caller passes its history + /// high-water mark). Falls back to a real [`Self::clear`] when the + /// biased position space would no longer fit `u32`. + pub(crate) fn advance_epoch(&mut self, span: u32) { + // Stored positions are bounded by the eviction band (2 * max + // window = 2^31, see the matcher's `window_log <= 30` ceiling), + // so a bias at or below `u32::MAX - 2^31` can never overflow in + // `put`. + const POSITION_CEILING: u32 = 1 << 31; + match self.bias.checked_add(span) { + Some(new_bias) if new_bias <= u32::MAX - POSITION_CEILING => self.bias = new_bias, + _ => self.clear(), + } } /// Donor-parity `ZSTD_hashPtr` — multiply-shift hash over the first @@ -222,6 +256,11 @@ impl FastHashTable { /// never touches `self` and stays reload-free. #[inline(always)] pub(crate) fn hot_state(&mut self) -> (&mut [u32], u32) { + // The raw-slice consumers (no-dict kernels) store and read + // UNBIASED positions; they may only run on a bias-0 table (the + // matcher clears — rather than epoch-advances — whenever the next + // frame is not a dict-attach frame). + debug_assert_eq!(self.bias, 0, "hot_state requires an unbiased table"); (self.table.as_mut_slice(), self.hash_log) } @@ -240,7 +279,13 @@ impl FastHashTable { debug_assert!((hash as usize) < self.table.len()); // SAFETY: see method-level doc — `hash` is bounded by the // table-size invariant from `hash_ptr`. - unsafe { *self.table.get_unchecked(hash as usize) } + let raw = unsafe { *self.table.get_unchecked(hash as usize) }; + // Epoch filter: entries stored before the last `advance_epoch` + // (raw < bias, including the all-zero sentinel) must read as the + // empty sentinel 0 — the saturation floor IS the semantics here. + // Compiles to sub + cmov; on bias == 0 tables (no-dict, cached + // dict) it is the identity. + raw.saturating_sub(self.bias) } /// Direct table write — `table[hash] = pos`. Same bounds reasoning @@ -252,9 +297,12 @@ impl FastHashTable { #[inline(always)] pub(crate) unsafe fn put(&mut self, hash: u32, pos: u32) { debug_assert!((hash as usize) < self.table.len()); + // Cannot overflow: `advance_epoch` caps the bias at + // `u32::MAX - 2^31` and `pos` is bounded by the eviction band. + let biased = pos + self.bias; // SAFETY: see method-level doc. unsafe { - *self.table.get_unchecked_mut(hash as usize) = pos; + *self.table.get_unchecked_mut(hash as usize) = biased; } } } diff --git a/zstd/src/encoding/simple/fast_matcher.rs b/zstd/src/encoding/simple/fast_matcher.rs index fc24108b1..7ec20b21d 100644 --- a/zstd/src/encoding/simple/fast_matcher.rs +++ b/zstd/src/encoding/simple/fast_matcher.rs @@ -246,6 +246,13 @@ pub(crate) struct FastKernelMatcher { /// large enough to slide the dictionary out of the window. `region_len()` /// is the dict/input boundary (`dict_end`). dict: DictAttach, + /// High-water mark of any position storable into [`Self::hash_table`] + /// since the last table clear / epoch advance: the largest history + /// length seen by [`Self::extend_history_with_pending`] and the largest + /// borrowed `block_end` scanned. `reset` feeds it to + /// [`FastHashTable::advance_epoch`] as the span that makes every + /// previously-stored entry stale, then rearms it at 0. + table_pos_high_water: usize, } impl FastKernelMatcher { @@ -334,6 +341,7 @@ impl FastKernelMatcher { borrowed: None, last_borrowed_block: None, dict: DictAttach::new(), + table_pos_high_water: 0, } } @@ -343,7 +351,23 @@ impl FastKernelMatcher { /// either clears the existing hash table (if `(hash_log, mls)` are /// unchanged) or reallocates it. The window_log update redirects /// the soft-eviction bound and the decoder-side reported window. - pub(crate) fn reset(&mut self, window_log: u8, hash_log: u32, mls: u32, step_size: usize) { + /// + /// `dict_attach_epoch`: the upcoming frame re-primes the SAME + /// dictionary in attach mode (separate cached dict table, dual-probe + /// kernel). When the cached dict table is still primed, the main + /// table is then invalidated via an epoch advance (donor + /// `ZSTD_continueCCtx` cadence — stale entries filtered by the bias, + /// no full-table memset); every other shape keeps the historical + /// `clear()` so the raw-slice no-dict kernels always see a bias-0 + /// table. + pub(crate) fn reset( + &mut self, + window_log: u8, + hash_log: u32, + mls: u32, + step_size: usize, + dict_attach_epoch: bool, + ) { assert!( step_size >= 2, "FastKernelMatcher requires step_size >= 2 (got {step_size})" @@ -362,14 +386,27 @@ impl FastKernelMatcher { // index a table whose shape no longer matches. self.hash_table = FastHashTable::new(hash_log, mls); self.dict.invalidate(); + } else if dict_attach_epoch && self.dict.is_primed() { + // Dict-attach frame over the same primed dictionary: advance + // the epoch bias past every position the previous frames could + // have stored instead of memsetting the whole table (donor + // `ZSTD_continueCCtx`). The dual-probe dict kernel reads the + // main table only through `FastHashTable::get`, which maps + // pre-advance entries to the empty sentinel. The cached dict + // table is untouched (its own instance, bias 0): the dictionary + // lands at the same absolute history positions every frame, so + // its hashes stay valid and the per-frame re-hash is skipped + // (CDict-equivalent). + let span = u32::try_from(self.table_pos_high_water).unwrap_or(u32::MAX); + self.hash_table.advance_epoch(span); } else { // Same shape — keep the allocation, zero the entries via // `memset` (ZSTD_window_clear cadence). A primed dict table - // is retained: the dictionary lands at the same absolute - // history positions every frame, so its hashes stay valid - // and the per-frame re-hash is skipped (CDict-equivalent). + // is retained (see the epoch branch above for why that is + // sound). self.hash_table.clear(); } + self.table_pos_high_water = 0; // M8: history starts empty (HISTORY_DRAIN_BASE = 0). self.history.clear(); self.history.resize(HISTORY_DRAIN_BASE, 0); @@ -672,6 +709,10 @@ impl FastKernelMatcher { // max_window_size` already holds — just append. let block_start = self.history.len(); self.history.extend_from_slice(&space); + // Track the largest position any kernel scan over this history + // could store into the hash table (consumed by `reset`'s epoch + // advance). + self.table_pos_high_water = self.table_pos_high_water.max(self.history.len()); // Record where this newly-appended block starts so // `last_committed_space` can return its bytes AFTER the // kernel call consumes pending. @@ -862,6 +903,13 @@ impl FastKernelMatcher { block_start <= block_end && block_end <= total_len, "borrowed block bounds out of range: start={block_start} end={block_end} total={total_len}", ); + // Borrowed scans store raw (bias-0) positions up to `block_end` + // through the table's hot-state slice; record them for `reset`'s + // epoch-advance span. (A borrowed window never coexists with a + // primed dict, so the table bias is 0 here — see `hot_state` — + // but the high-water must still cover these positions in case a + // dictionary is attached on a later frame.) + self.table_pos_high_water = self.table_pos_high_water.max(block_end); // Same window math as the owned path, but against the absolute // block end in the borrowed buffer rather than the accumulated // history length. @@ -1510,6 +1558,7 @@ mod tests { FAST_LEVEL_1_HASH_LOG, FAST_LEVEL_1_MLS, 2, + false, ); // After reset the borrowed window is dropped — back to the // (now empty) owned buffer, never the dangling external range. @@ -1652,6 +1701,7 @@ mod tests { FAST_LEVEL_1_HASH_LOG, FAST_LEVEL_1_MLS, 2, + false, ); // Post-reset: history empty (HISTORY_DRAIN_BASE=0; no @@ -1674,7 +1724,7 @@ mod tests { let mut m = FastKernelMatcher::new(); // Force a parameter change — every Vec we hand the new // FastHashTable will be a fresh allocation. - m.reset(16, 10, 4, 2); + m.reset(16, 10, 4, 2, false); assert_eq!(m.hash_table.hash_log(), 10); assert_eq!(m.hash_table.mls(), 4); assert_eq!(m.window_log, 16); From f2cd65b553d537f57b6383e8734d468cbcf2f156 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Wed, 10 Jun 2026 11:59:10 +0300 Subject: [PATCH 04/22] perf(encode): fold optimal-plan profile dispatch on strategy consts build_optimal_plan dispatched its 4 (accurate, favor_small_offsets) arms on the RUNTIME profile while debug-asserting it equal to the strategy's associated consts. With build_optimal_plan_impl marked inline(always), every BT strategy monomorphisation carried all four ~16 KB DP bodies. Matching on (S::ACCURATE_PRICE, S::FAVOR_SMALL_OFFSETS) folds the three dead arms at monomorphisation. simd128 wasm payload: 777006 -> 633482 bytes (-18.5%), back under the 768 KiB CI budget with ~150 KB headroom. Native: same reachable code, smaller icache footprint. --- zstd/src/encoding/match_generator.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs index c5e9f8101..9911f9f46 100644 --- a/zstd/src/encoding/match_generator.rs +++ b/zstd/src/encoding/match_generator.rs @@ -5202,12 +5202,14 @@ impl HcMatchGenerator { ); // `S::ACCURATE_PRICE` / `S::FAVOR_SMALL_OFFSETS` cannot appear // as const-generic arguments yet (`generic_const_exprs` is - // still unstable), so we keep the 4-arm runtime dispatch here. - // Each S monomorphisation only reaches one arm in practice - // (BtOpt → false/true, BtUltra/BtUltra2 → true/false), so the - // optimiser folds away the others. - let profile = initial_state.profile; - match (profile.accurate, profile.favor_small_offsets) { + // still unstable), so dispatch over a 4-arm match — but on the + // strategy's ASSOCIATED CONSTS, not the runtime profile (the + // `debug_assert_eq`s above pin the runtime profile to those + // consts). A const scrutinee folds the three dead arms at + // monomorphisation; matching the runtime profile instead kept + // all four `#[inline(always)]` DP bodies (~16 KB each) alive in + // EVERY `S` instantiation — ~360 KB of the wasm payload. + match (S::ACCURATE_PRICE, S::FAVOR_SMALL_OFFSETS) { (true, false) => self.build_optimal_plan_impl::( current, current_abs_start, From 53e83059ae7293de04b212958c5a759ae7033904 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Wed, 10 Jun 2026 12:16:40 +0300 Subject: [PATCH 05/22] test(bench): decode_loop_dict profiling example + dict payload dump - decode_loop_dict example mirrors the decompress-dict bench arm (DictionaryHandle parsed once, reused FrameDecoder, decode_all_with_dict_handle per iteration) for clean perf profiles - compare_ffi dumps the FFI dict-encoded payload per (scenario, level) under the existing STRUCTURED_ZSTD_DUMP_DICT_DIR gate so the example decodes the exact bytes the bench measures --- zstd/benches/compare_ffi.rs | 9 +++++ zstd/examples/decode_loop_dict.rs | 57 +++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 zstd/examples/decode_loop_dict.rs diff --git a/zstd/benches/compare_ffi.rs b/zstd/benches/compare_ffi.rs index 21991104f..71b827917 100644 --- a/zstd/benches/compare_ffi.rs +++ b/zstd/benches/compare_ffi.rs @@ -666,6 +666,15 @@ fn bench_dictionary(c: &mut Criterion) { let no_dict_bytes = no_dict.compress(&scenario.bytes).unwrap(); let with_dict_bytes = with_dict.compress(&scenario.bytes).unwrap(); + // Diagnostic: dump the FFI dict-encoded payload next to the + // trained dict (same env gate) so standalone profiling binaries + // (`decode_loop_dict`) can decode the EXACT bytes the + // `decompress-dict/...` bench arm measures. + if let Ok(dir) = std::env::var("STRUCTURED_ZSTD_DUMP_DICT_DIR") { + let path = format!("{dir}/{}.{}.zst", scenario.id, level.name); + std::fs::write(&path, &with_dict_bytes).expect("dump dict payload"); + } + // Rust dict-compressed output size, for the compress-dict // compression-ratio report (rust vs FFI). Only computable when the // dictionary parsed into a Rust handle; mirrors the gate the diff --git a/zstd/examples/decode_loop_dict.rs b/zstd/examples/decode_loop_dict.rs new file mode 100644 index 000000000..166fd9a85 --- /dev/null +++ b/zstd/examples/decode_loop_dict.rs @@ -0,0 +1,57 @@ +//! Standalone decode-loop binary for CLEAN perf-record profiles of the +//! dictionary DECODE hot path. Mirrors the `decompress-dict/...` bench arm +//! exactly: parse the dictionary into a [`DictionaryHandle`] ONCE, build one +//! reused [`FrameDecoder`], then loop `decode_all_with_dict_handle` over a +//! fixed `.zst` payload into a preallocated output buffer — no criterion, no +//! FFI, no training; samples land purely on the per-frame decode path. +//! +//! The payload should be the FFI dict-encoded frame the bench measures; dump +//! both files from a bench run via +//! `STRUCTURED_ZSTD_DUMP_DICT_DIR= cargo bench --bench compare_ffi ...` +//! (`.dict` + `..zst`). +//! +//! Build: `cargo build --profile flamegraph -p structured-zstd +//! --example decode_loop_dict` +//! Run: `decode_loop_dict ` + +use std::env; + +use structured_zstd::decoding::{DictionaryHandle, FrameDecoder}; + +fn main() { + let args: Vec = env::args().collect(); + let iters: u32 = args.get(1).and_then(|s| s.parse().ok()).unwrap_or(500_000); + let payload_path = args + .get(2) + .map(String::as_str) + .unwrap_or("/tmp/szstd-dicts/small-4k-log-lines.level_2_fast.zst"); + let dict_path = args + .get(3) + .map(String::as_str) + .unwrap_or("/tmp/szstd-dicts/small-4k-log-lines.dict"); + let expected_len: usize = args.get(4).and_then(|s| s.parse().ok()).unwrap_or(4096); + + let payload = std::fs::read(payload_path).expect("read payload file"); + let dict = std::fs::read(dict_path).expect("read dict file"); + let handle = DictionaryHandle::decode_dict(dict.as_slice()).expect("parse dictionary"); + + let mut decoder = FrameDecoder::new(); + let mut output = vec![0u8; expected_len]; + let mut sink: usize = 0; + for _ in 0..iters { + let n = decoder + .decode_all_with_dict_handle(payload.as_slice(), output.as_mut_slice(), &handle) + .expect("dict decode should succeed"); + sink = sink.wrapping_add(n); + core::hint::black_box(&output); + } + + eprintln!( + "decoded {} bytes x {} iters (payload {} bytes, dict {} bytes); out-sum={}", + expected_len, + iters, + payload.len(), + dict.len(), + sink + ); +} From 01aa801f26af896aadbf6be822ab0c588c8b582e Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Wed, 10 Jun 2026 12:18:52 +0300 Subject: [PATCH 06/22] test(bench): dump scenario input bytes alongside the trained dict --- zstd/benches/compare_ffi.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zstd/benches/compare_ffi.rs b/zstd/benches/compare_ffi.rs index 71b827917..e0c33ee3d 100644 --- a/zstd/benches/compare_ffi.rs +++ b/zstd/benches/compare_ffi.rs @@ -573,6 +573,11 @@ fn bench_dictionary(c: &mut Criterion) { if let Ok(dir) = std::env::var("STRUCTURED_ZSTD_DUMP_DICT_DIR") { let path = format!("{dir}/{}.dict", scenario.id); std::fs::write(&path, &ffi_dictionary).expect("dump dict"); + // Scenario input bytes too, so standalone profiling binaries + // (`encode_loop_dict` / `decode_loop_dict`) can replay the + // exact (input, dict) pair this scenario benches. + let path = format!("{dir}/{}.bin", scenario.id); + std::fs::write(&path, scenario.bytes.as_slice()).expect("dump scenario bytes"); } if emit_reports { From 737909e89170fb35954650caf0963bfd8d9b8e97 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Wed, 10 Jun 2026 12:25:10 +0300 Subject: [PATCH 07/22] perf(encode): donor pre-build incompressibility gate for literals Port HUF_compress_internal's flat-histogram bail (largest <= (srcSize >> 7) + 4): emit the raw literals section BEFORE building the Huffman tree. Previously near-random literals paid histogram + leaf sort + tree build + a FULL encode4x pass per block, only for the post-hoc use_raw_literal_fallback gate to discard it; ~65% of frame time on small-10k-random compress-dict level_-7_fast. The surviving build path reuses the gate's histogram via build_from_counts (drops build_from_data's second counting pass). Estimator mirrors the gate so splitter probe costs stay byte-equal to emitted sections. --- zstd/src/encoding/blocks/compressed.rs | 30 +++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/zstd/src/encoding/blocks/compressed.rs b/zstd/src/encoding/blocks/compressed.rs index 9f83ed550..876c0ba7b 100644 --- a/zstd/src/encoding/blocks/compressed.rs +++ b/zstd/src/encoding/blocks/compressed.rs @@ -751,11 +751,14 @@ fn estimate_literals_section_bytes( return total; } - counts.fill(0); - for &b in literals { - counts[b as usize] += 1; + let (max_sym, largest_count) = crate::histogram::count_bytes(literals, counts); + // Mirror `compress_literals`' donor pre-build incompressibility gate + // byte-for-byte (flat histogram → raw section, no tree build) so + // splitter probe costs match what the emitter writes. + if largest_count <= (literals.len() >> 7) + 4 { + *last_huff = None; + return uncompressed_literals_header_bytes(literals.len()) + literals.len(); } - let max_sym = counts.iter().rposition(|&c| c > 0).unwrap_or_default(); let new_table = huff0_encoder::HuffmanTable::build_from_counts(&counts[..=max_sym]); let Some(new_desc) = new_table.writeable_table_description_size() else { @@ -2149,7 +2152,24 @@ fn compress_literals( return emit_reuse_literals(literals, prev, writer, reset_idx, strategy); } - let new_encoder_table = huff0_encoder::HuffmanTable::build_from_data(literals); + let mut counts = [0usize; 256]; + let (max_symbol, largest_count) = crate::histogram::count_bytes(literals, &mut counts); + // Donor pre-build incompressibility gate (`huf_compress.c`, + // `HUF_compress_internal`): a histogram this flat + // (`largest <= (srcSize >> 7) + 4`) is heuristically not worth + // compressing — bail to raw BEFORE the tree build and the full + // `encode4x` pass. Without it, near-random literals paid histogram + + // sort + tree + a full encode of the section only for the post-hoc + // `use_raw_literal_fallback` below to throw it all away (~65% of + // frame time on the random-payload dict scenarios). The single-symbol + // case (`largest == srcSize`) never reaches here: the block emitter + // routes all-identical sections to RLE first. + if largest_count <= (literals.len() >> 7) + 4 { + raw_literals(literals, writer); + return HuffmanTableUpdate::Cleared; + } + + let new_encoder_table = huff0_encoder::HuffmanTable::build_from_counts(&counts[..=max_symbol]); let Some(new_table_description_size) = new_encoder_table.writeable_table_description_size() else { From 4b58210cdfb781eb8ccf73caef22042f563d64a0 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Wed, 10 Jun 2026 12:38:34 +0300 Subject: [PATCH 08/22] docs(encode): document why compress() source is not restored on unwind A partially-consumed Read source handed back after a panic would let a catch_unwind caller silently compress the remaining tail from an arbitrary midpoint; the loud expect on the emptied slot is the safe post-panic behavior (context reset required after errors, matching the reference implementation's contract). --- zstd/src/encoding/frame_compressor.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index 5eb42e925..1bb3c4932 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -1204,6 +1204,15 @@ impl FrameCompressor { // mutably alongside `&mut self` (the rest of the loop touches // `self.state` / `self.hasher`, disjoint from the reader). Restored // before the frame tail so a reused compressor keeps its source. + // + // Deliberately NOT restored on unwind: if the block loop panics the + // source has been partially consumed, so handing it back would let a + // `catch_unwind` caller "successfully" compress the remaining tail + // from an arbitrary midpoint — silent data corruption. Leaving the + // slot empty makes any post-panic reuse fail loudly at the `expect` + // below (matcher/entropy state is equally unre-usable after an + // unwind; the reference implementation likewise requires a context + // reset after an error). let mut source = self .uncompressed_data .take() From 6233d0f6fdc13fc86eb96e01ea6016e106dc3dd4 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Wed, 10 Jun 2026 13:00:36 +0300 Subject: [PATCH 09/22] perf(encode): run the lazy band (L6-12) on the row match finder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The donor defaults greedy..lazy2 to the row-based finder whenever SIMD is available (ZSTD_resolveRowMatchFinderMode); our lazy levels still walked hash chains, and the dependent chain-table loads were ~75% of L10 wall time on the 1 MiB corpus. - LEVEL_TABLE L6-12 -> SearchMethod::RowHash with per-level RowConfig derived from donor clevels.h (verified via ZSTD_getCParams): rowLog = clamp(searchLog, 4, 6), depth = 1 << min(searchLog, rowLog), donor hashLog/targetLength per level - StrategyTag bridge + Lazy::BACKEND follow (Greedy | Lazy -> Row) - row_hash_bits_for_window drops its constant 20-bit ceiling (predates the lazy band on Row; L9-12 carry donor hashLog 21-23) — window- derived cap only - Row configure records the RESOLVED hash width in the primed-snapshot key on the unhinted path too (a 0 default keyed unhinted-vs-hinted captures apart, forcing needless dict re-primes) - HC-specific unit tests keep their coverage through the test-only reset_on_hc_lazy override helper; the HC resize test moves to the BT levels that still own per-level HcConfig widths --- zstd/examples/donor_cparams_range.rs | 27 +++++ zstd/src/encoding/match_generator.rs | 149 ++++++++++++++++++++++----- zstd/src/encoding/strategy.rs | 22 ++-- 3 files changed, 164 insertions(+), 34 deletions(-) create mode 100644 zstd/examples/donor_cparams_range.rs diff --git a/zstd/examples/donor_cparams_range.rs b/zstd/examples/donor_cparams_range.rs new file mode 100644 index 000000000..0dd52f2ad --- /dev/null +++ b/zstd/examples/donor_cparams_range.rs @@ -0,0 +1,27 @@ +//! Print donor cParams for levels 1..=22 at a given source size. +//! Companion to `donor_cparams_check` for sweeping a whole level band. +//! +//! Run: `cargo run --release -p structured-zstd --example donor_cparams_range +//! --features dict_builder -- [src_size]` (0 / omitted = unbounded). +use zstd::zstd_safe::zstd_sys; + +fn main() { + let src_size: u64 = std::env::args() + .nth(1) + .and_then(|s| s.parse().ok()) + .unwrap_or(0); + for level in 1..=22i32 { + // SAFETY: standard libzstd query. + let cp = unsafe { zstd_sys::ZSTD_getCParams(level, src_size, 0) }; + println!( + "L{level}: wlog={} clog={} hlog={} slog={} mml={} tlen={} strat={}", + cp.windowLog, + cp.chainLog, + cp.hashLog, + cp.searchLog, + cp.minMatch, + cp.targetLength, + cp.strategy as u32 + ); + } +} diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs index 9911f9f46..6eda376d8 100644 --- a/zstd/src/encoding/match_generator.rs +++ b/zstd/src/encoding/match_generator.rs @@ -280,6 +280,69 @@ const ROW_L5: RowConfig = RowConfig { mls: ROW_MIN_MATCH_LEN, }; +// Donor `clevels.h` unbounded defaults for the lazy band, verified via +// `ZSTD_getCParams(level, 0, 0)`: +// L6 { w21 c18 h19 s3 mml5 t4 lazy } → rowLog 4, depth 1<<3 = 8 +// L7 { w21 c19 h20 s4 mml5 t8 lazy } → rowLog 4, depth 16 +// L8 { w21 c19 h20 s4 mml5 t16 lazy2 } → rowLog 4, depth 16 +// L9 { w22 c20 h21 s4 mml5 t16 lazy2 } → rowLog 4, depth 16 +// L10 { w22 c21 h22 s5 mml5 t16 lazy2 } → rowLog 5, depth 32 +// L11 { w22 c21 h22 s6 mml5 t16 lazy2 } → rowLog 6, depth 64 +// L12 { w22 c22 h23 s6 mml5 t32 lazy2 } → rowLog 6, depth 64 +// `rowLog = clamp(searchLog, 4, 6)`, `depth = 1 << min(searchLog, rowLog)` +// (same derivation as `ROW_L5` above). `hash_bits` carries the donor +// `hashLog`; the hinted-source clamp in `configure` caps it by the window +// exactly like the donor `ZSTD_adjustCParams` path. +const ROW_L6: RowConfig = RowConfig { + hash_bits: 19, + row_log: 4, + search_depth: 8, + target_len: 4, + mls: ROW_MIN_MATCH_LEN, +}; +const ROW_L7: RowConfig = RowConfig { + hash_bits: 20, + row_log: 4, + search_depth: 16, + target_len: 8, + mls: ROW_MIN_MATCH_LEN, +}; +const ROW_L8: RowConfig = RowConfig { + hash_bits: 20, + row_log: 4, + search_depth: 16, + target_len: 16, + mls: ROW_MIN_MATCH_LEN, +}; +const ROW_L9: RowConfig = RowConfig { + hash_bits: 21, + row_log: 4, + search_depth: 16, + target_len: 16, + mls: ROW_MIN_MATCH_LEN, +}; +const ROW_L10: RowConfig = RowConfig { + hash_bits: 22, + row_log: 5, + search_depth: 32, + target_len: 16, + mls: ROW_MIN_MATCH_LEN, +}; +const ROW_L11: RowConfig = RowConfig { + hash_bits: 22, + row_log: 6, + search_depth: 64, + target_len: 16, + mls: ROW_MIN_MATCH_LEN, +}; +const ROW_L12: RowConfig = RowConfig { + hash_bits: 23, + row_log: 6, + search_depth: 64, + target_len: 32, + mls: ROW_MIN_MATCH_LEN, +}; + /// Per-level Double-Fast hash sizing, mirroring the donor `clevels.h` columns /// (config-driven, not a hardcoded constant): `long_hash_log` = /// `cParams.hashLog` (the long 8-byte hash table), `short_hash_log` = @@ -628,8 +691,14 @@ fn dfast_hash_bits_for_window(max_window_size: usize) -> usize { } fn row_hash_bits_for_window(max_window_size: usize) -> usize { + // Window-derived cap on the row hash width (stricter than the donor + // `ZSTD_adjustCParams_internal` `hashLog <= windowLog + 1` bound, so a + // tiny hinted source keeps the tiny table). No constant upper clamp: + // the old `ROW_HASH_BITS` (20) ceiling predates the lazy band moving + // onto Row — L9-12 carry donor hashLog 21-23 and must keep it when the + // hinted window is large enough. let window_log = (usize::BITS - 1 - max_window_size.leading_zeros()) as usize; - window_log.clamp(MIN_WINDOW_LOG as usize, ROW_HASH_BITS) + window_log.max(MIN_WINDOW_LOG as usize) } /// `floor(log2(window))` for the HashChain table-log cap (donor @@ -667,13 +736,21 @@ const LEVEL_TABLE: [LevelParams; 22] = [ // long-enough match — the dominant cost in the L5..=L15 speed // regression vs FFI (see lazy_band_target_len_matches_donor_default_table). /* 5 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Greedy, search: super::strategy::SearchMethod::RowHash, window_log: 21, lazy_depth: 0, fast: None, dfast: None, hc: None, row: Some(ROW_L5) }, - /* 6 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 21, lazy_depth: 1, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 19, chain_log: 18, search_depth: 8, target_len: 4, search_mls: 4 }), row: None }, - /* 7 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 21, lazy_depth: 1, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 20, chain_log: 19, search_depth: 16, target_len: 8, search_mls: 4 }), row: None }, - /* 8 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 21, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 20, chain_log: 19, search_depth: 16, target_len: 16, search_mls: 4 }), row: None }, - /* 9 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 21, chain_log: 20, search_depth: 16, target_len: 16, search_mls: 4 }), row: None }, - /*10 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 22, chain_log: 21, search_depth: 32, target_len: 16, search_mls: 4 }), row: None }, - /*11 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 22, chain_log: 21, search_depth: 64, target_len: 16, search_mls: 4 }), row: None }, - /*12 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 23, chain_log: 22, search_depth: 64, target_len: 32, search_mls: 4 }), row: None }, + // L6-12: the donor runs the lazy/lazy2 strategies on the ROW-based + // match finder by default (`ZSTD_resolveRowMatchFinderMode`: row mode + // is on for greedy..lazy2 whenever SIMD is available) — a bounded + // SIMD tag scan per row instead of a pointer-chasing hash-chain walk. + // Our HashChain walk on these levels was ~75% of L10 wall time on the + // 1 MiB corpus (dependent chain-table loads). Same `RowConfig` + // derivation as `ROW_L5` above, donor values per level in the + // `ROW_L6..ROW_L12` comment block. + /* 6 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::RowHash, window_log: 21, lazy_depth: 1, fast: None, dfast: None, hc: None, row: Some(ROW_L6) }, + /* 7 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::RowHash, window_log: 21, lazy_depth: 1, fast: None, dfast: None, hc: None, row: Some(ROW_L7) }, + /* 8 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::RowHash, window_log: 21, lazy_depth: 2, fast: None, dfast: None, hc: None, row: Some(ROW_L8) }, + /* 9 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::RowHash, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: None, row: Some(ROW_L9) }, + /*10 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::RowHash, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: None, row: Some(ROW_L10) }, + /*11 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::RowHash, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: None, row: Some(ROW_L11) }, + /*12 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::RowHash, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: None, row: Some(ROW_L12) }, // L13-15: reference uses btlazy2 (binary-tree finder) with searchLog 4/5/6 // (search_depth 16/32/64) and targetLength 32. We run the hash-chain Lazy // parser here, so we mirror the reference search budget rather than inflate @@ -2000,17 +2077,25 @@ impl Matcher for MatchGeneratorDriver { row.lazy_depth = params.lazy_depth; let row_cfg = params.row.expect("Row level row carries a RowConfig"); row.configure(row_cfg); - if hinted { + // Record the RESOLVED hash width in the primed-snapshot key on + // both paths. Leaving the unhinted path at the 0 default keyed + // an unhinted capture differently from a hinted reset that + // resolves to the IDENTICAL tables (hint >= window makes the + // clamp a no-op), forcing a needless dictionary re-prime. + resolved_table_bits = if hinted { // Clamp the configured hash width by the hinted window // (donor `ZSTD_adjustCParams` caps hashLog by windowLog) — // `min`, not replace, so an explicit `hash_log` param // override (`row_cfg.hash_bits`) survives the hinted path // instead of being overwritten by the window value. - resolved_table_bits = row_cfg + let bits = row_cfg .hash_bits .min(row_hash_bits_for_window(table_window_size)); - row.set_hash_bits(resolved_table_bits); - } + row.set_hash_bits(bits); + bits + } else { + row_cfg.hash_bits + }; row.reset(); } MatcherStorage::HashChain(hc) => { @@ -6150,6 +6235,18 @@ impl MatchGeneratorDriver { ) { self.config_override = Some((search, parse)); } + + /// Test-only: reset `level` routed onto the lazy HashChain pairing. + /// The lazy band runs on the Row backend in production, so HC-specific + /// behaviour (live-chain dict prime, eviction budget accounting, seed + /// pass gates) is exercised through this override-backed reset. + pub(crate) fn reset_on_hc_lazy(&mut self, level: CompressionLevel) { + self.set_config_override( + super::strategy::SearchMethod::HashChain, + super::strategy::ParseMode::Lazy2, + ); + self.reset(level); + } } /// Drive a full compress parse for `data` at `level` (optionally with a @@ -8025,9 +8122,11 @@ fn driver_best_to_fastest_releases_oversized_hc_tables() { fn driver_better_to_best_resizes_hc_tables() { let mut driver = MatchGeneratorDriver::new(32, 2); - // Initialize at Better — allocates small HC tables (1M hash, 512K chain). - driver.reset(CompressionLevel::Better); - assert_eq!(driver.window_size(), (1u64 << 21)); + // The lazy band runs on the Row backend now, so the HC resize path is + // exercised across two BT levels whose native `HcConfig` widths differ: + // L13 (hash_log 22, chain_log 22) -> L15 (hash_log 23, chain_log 23). + driver.reset(CompressionLevel::Level(13)); + assert_eq!(driver.window_size(), (1u64 << 22)); let mut space = driver.get_next_space(); space[..12].copy_from_slice(b"abcabcabcabc"); @@ -8039,8 +8138,8 @@ fn driver_better_to_best_resizes_hc_tables() { let better_hash_len = hc.table.hash_table.len(); let better_chain_len = hc.table.chain_table.len(); - // Switch to Best — must resize to larger tables. - driver.reset(CompressionLevel::Best); + // Switch to L15 — must resize to larger tables. + driver.reset(CompressionLevel::Level(15)); assert_eq!(driver.window_size(), (1u64 << 22)); // Feed data to trigger ensure_tables with new sizes. @@ -8053,13 +8152,13 @@ fn driver_better_to_best_resizes_hc_tables() { let hc = driver.hc_matcher(); assert!( hc.table.hash_table.len() > better_hash_len, - "Best hash_table ({}) should be larger than Better ({})", + "L15 hash_table ({}) should be larger than L13 ({})", hc.table.hash_table.len(), better_hash_len ); assert!( hc.table.chain_table.len() > better_chain_len, - "Best chain_table ({}) should be larger than Better ({})", + "L15 chain_table ({}) should be larger than L13 ({})", hc.table.chain_table.len(), better_chain_len ); @@ -8148,7 +8247,7 @@ fn prime_with_dictionary_applies_offset_history_even_when_content_is_empty() { #[test] fn hc_prime_with_empty_dictionary_disables_btultra2_seed_pass() { let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset(CompressionLevel::Better); + driver.reset_on_hc_lazy(CompressionLevel::Better); driver.prime_with_dictionary(&[], [11, 7, 3]); @@ -8213,7 +8312,7 @@ fn primed_snapshot_not_restored_across_ldm_config_change() { #[test] fn hc_prime_with_dictionary_disables_btultra2_seed_pass() { let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset(CompressionLevel::Better); + driver.reset_on_hc_lazy(CompressionLevel::Better); driver.prime_with_dictionary(b"abcdefgh", [1, 4, 8]); @@ -8432,7 +8531,9 @@ fn primed_snapshot_fast_attach_does_not_over_key_non_simple_backends() { // always primed the same way, so the bit must NOT enter their snapshot key // — otherwise an unhinted capture (which would record `fast_attach = true`) // and a hinted reset that resolves to the IDENTICAL `LevelParams` would key - // differently and force a needless re-prime. `Best` is a HashChain level. + // differently and force a needless re-prime. `Best` is a Row-backend lazy + // level; this also pins the Row arm recording its RESOLVED hash width on + // the unhinted path (a 0 default there keyed unhinted-vs-hinted apart). let mut driver = MatchGeneratorDriver::new(8, 1); let level = CompressionLevel::Best; @@ -9569,7 +9670,7 @@ fn hc_prime_with_dictionary_preserves_history_for_first_full_block() { #[test] fn prime_with_dictionary_budget_shrinks_after_hc_eviction() { let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset(CompressionLevel::Better); + driver.reset_on_hc_lazy(CompressionLevel::Better); // Use a small live window so dictionary-primed slices are evicted quickly. driver.hc_matcher_mut().table.max_window_size = 8; driver.reported_window_size = 8; @@ -9607,7 +9708,7 @@ fn hc_commit_without_eviction_retires_no_dictionary_budget() { // as "evicted" and prematurely retire dictionary budget even when the // window is nowhere near full. let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset(CompressionLevel::Better); + driver.reset_on_hc_lazy(CompressionLevel::Better); // A large live window so a small committed block evicts nothing. driver.hc_matcher_mut().table.max_window_size = 1 << 20; driver.reported_window_size = 1 << 20; diff --git a/zstd/src/encoding/strategy.rs b/zstd/src/encoding/strategy.rs index b422c2294..c3dbcb278 100644 --- a/zstd/src/encoding/strategy.rs +++ b/zstd/src/encoding/strategy.rs @@ -245,15 +245,16 @@ impl Strategy for Greedy { const SUFFICIENT_MATCH_LEN: usize = 32; } -/// Levels 5-15 — donor `ZSTD_lazy2` on a hash chain. Levels inside -/// the band differ only by runtime `HcConfig` fields (`search_depth`, -/// `hash_log`, `chain_log`, `target_len`, `lazy_depth`), not by +/// Levels 6-12 — donor `ZSTD_lazy`/`ZSTD_lazy2` on the row finder +/// (donor row mode is the greedy..lazy2 default). Levels inside the +/// band differ only by runtime `RowConfig` fields (`search_depth`, +/// `hash_bits`, `row_log`, `target_len`, `lazy_depth`), not by /// compile-time `Strategy` consts, so they share a single type. #[derive(Copy, Clone, Debug, Default)] pub(crate) struct Lazy; impl Strategy for Lazy { - const BACKEND: BackendTag = BackendTag::HashChain; + const BACKEND: BackendTag = BackendTag::Row; const MIN_MATCH: usize = 4; const ACCURATE_PRICE: bool = false; const FAVOR_SMALL_OFFSETS: bool = true; @@ -436,14 +437,16 @@ impl StrategyTag { } /// Bridge to [`BackendTag`] for the dispatcher entry point. + /// Greedy AND lazy run on the Row finder (donor + /// `ZSTD_resolveRowMatchFinderMode`: row mode is the default for + /// greedy..lazy2); the BT strategies keep the HashChain storage + /// (their tree scratch lives inside it). pub(crate) const fn backend(self) -> BackendTag { match self { Self::Fast => BackendTag::Simple, Self::Dfast => BackendTag::Dfast, - Self::Greedy => BackendTag::Row, - Self::Lazy | Self::Btlazy2 | Self::BtOpt | Self::BtUltra | Self::BtUltra2 => { - BackendTag::HashChain - } + Self::Greedy | Self::Lazy => BackendTag::Row, + Self::Btlazy2 | Self::BtOpt | Self::BtUltra | Self::BtUltra2 => BackendTag::HashChain, } } @@ -455,8 +458,7 @@ impl StrategyTag { match self { Self::Fast => SearchMethod::Fast, Self::Dfast => SearchMethod::DoubleFast, - Self::Greedy => SearchMethod::RowHash, - Self::Lazy => SearchMethod::HashChain, + Self::Greedy | Self::Lazy => SearchMethod::RowHash, Self::Btlazy2 | Self::BtOpt | Self::BtUltra | Self::BtUltra2 => { SearchMethod::BinaryTree } From 773b09a289a5958304fb92787ecf1251e45c65da Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Wed, 10 Jun 2026 13:07:12 +0300 Subject: [PATCH 10/22] perf(encode): iterate row tag-mask hits instead of scanning every slot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Donor ZSTD_RowFindBestMatch iteration: rotate the tag mask into head (newest-first) order once, then visit only the set bits via tzcnt + clear-lowest. The former loop burned slot arithmetic + a bit test on every entry (rows are 16-64 wide, typical tag hits 0-2) — ~14% of L10 wall time after the lazy band moved onto Row. max_walk now bounds ATTEMPTED candidates (donor nbAttempts decrements per mask hit, not per scanned slot): a search depth below the row width searches up to depth hits across the whole row. Full-row walks (L7-L12) keep byte-identical output (same candidates, same head order); L5/L6 (depth 8 < 16 entries) gain donor candidate coverage. The scalar tier advances to the next on-the-fly tag hit so its visit order and attempt accounting stay bit-identical to the mask tiers. Applies to both the live row and the dict dual-probe row. --- zstd/src/encoding/row/mod.rs | 108 +++++++++++++++++++++++++++++------ 1 file changed, 90 insertions(+), 18 deletions(-) diff --git a/zstd/src/encoding/row/mod.rs b/zstd/src/encoding/row/mod.rs index 5c07d0995..89efc33b5 100644 --- a/zstd/src/encoding/row/mod.rs +++ b/zstd/src/encoding/row/mod.rs @@ -434,17 +434,61 @@ macro_rules! gen_row_probe { }; let mut best: Option = None; - for i in 0..max_walk { - let slot = (head + i) & row_mask; - let idx = row_base + slot; - let matched = if $use_mask { - (tag_match >> slot) & 1 != 0 + // Donor `ZSTD_RowFindBestMatch` mask iteration: rotate the tag + // mask into head (newest-first) order once, then visit ONLY the + // set bits via tzcnt + clear-lowest. The former per-slot loop + // burned slot arithmetic + a bit test on EVERY entry (rows are + // 16-64 wide, typical tag hits 0-2) — ~14% of L10 wall time. + // `max_walk` bounds ATTEMPTED candidates (donor `nbAttempts` + // decrements per mask hit, not per scanned slot), so a depth + // below the row width searches up to `depth` hits across the + // WHOLE row — donor semantics on both the SIMD and scalar tiers + // (the scalar arm advances to the next on-the-fly tag hit so + // its visit order and attempt accounting stay bit-identical to + // the mask tiers). + let entries_bits: u64 = if row_entries >= 64 { + u64::MAX + } else { + (1u64 << row_entries) - 1 + }; + #[allow(unused_mut)] + let mut pending: u64 = if $use_mask { + let m = tag_match & entries_bits; + if head == 0 { + m } else { - self.row_tags[idx] == tag - }; - if !matched { - continue; + ((m >> head) | (m << (row_entries - head))) & entries_bits } + } else { + 0 + }; + #[allow(unused_mut)] + let mut scan = 0usize; + let mut attempts = 0usize; + while attempts < max_walk { + let slot_opt = if $use_mask { + if pending == 0 { + None + } else { + let i = pending.trailing_zeros() as usize; + pending &= pending - 1; + Some((head + i) & row_mask) + } + } else { + let mut found = None; + while scan < row_entries { + let s = (head + scan) & row_mask; + scan += 1; + if self.row_tags[row_base + s] == tag { + found = Some(s); + break; + } + } + found + }; + let Some(slot) = slot_opt else { break }; + attempts += 1; + let idx = row_base + slot; let raw_pos = self.row_positions[idx]; if raw_pos == ROW_EMPTY_SLOT { continue; @@ -507,17 +551,45 @@ macro_rules! gen_row_probe { } else { 0 }; - for i in 0..max_walk { - let slot = (dhead + i) & row_mask; - let didx = drow_base + slot; - let matched = if $use_mask { - (dtag_match >> slot) & 1 != 0 + // Same donor mask iteration as the live row above. + #[allow(unused_mut)] + let mut dpending: u64 = if $use_mask { + let m = dtag_match & entries_bits; + if dhead == 0 { + m } else { - dict.tags[didx] == tag - }; - if !matched { - continue; + ((m >> dhead) | (m << (row_entries - dhead))) & entries_bits } + } else { + 0 + }; + #[allow(unused_mut)] + let mut dscan = 0usize; + let mut dattempts = 0usize; + while dattempts < max_walk { + let slot_opt = if $use_mask { + if dpending == 0 { + None + } else { + let i = dpending.trailing_zeros() as usize; + dpending &= dpending - 1; + Some((dhead + i) & row_mask) + } + } else { + let mut found = None; + while dscan < row_entries { + let s = (dhead + dscan) & row_mask; + dscan += 1; + if dict.tags[drow_base + s] == tag { + found = Some(s); + break; + } + } + found + }; + let Some(slot) = slot_opt else { break }; + dattempts += 1; + let didx = drow_base + slot; let dp = dict.positions[didx]; if dp == ROW_EMPTY_SLOT { continue; From 6992e67b59c8a0715753e3fa1f2ca449c4e7e11d Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Wed, 10 Jun 2026 13:15:49 +0300 Subject: [PATCH 11/22] fix(bench): non-fatal dump writes; validate decode length in decode_loop_dict MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - STRUCTURED_ZSTD_DUMP_DICT_DIR dump writes warn (BENCH_WARN) on I/O failure instead of aborting the whole bench run — the artifacts are optional diagnostics - decode_loop_dict asserts the decoded length matches expected_len and black_boxes only the written prefix, so a mismatched (payload, dict, expected_len) triple fails loudly instead of profiling a wrong workload --- zstd/benches/compare_ffi.rs | 15 ++++++++++++--- zstd/examples/decode_loop_dict.rs | 5 ++++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/zstd/benches/compare_ffi.rs b/zstd/benches/compare_ffi.rs index e0c33ee3d..be02acdc1 100644 --- a/zstd/benches/compare_ffi.rs +++ b/zstd/benches/compare_ffi.rs @@ -571,13 +571,19 @@ fn bench_dictionary(c: &mut Criterion) { // compress-dict bench/REPORT use. Gated on an env var so normal // bench runs are unaffected. if let Ok(dir) = std::env::var("STRUCTURED_ZSTD_DUMP_DICT_DIR") { + // Diagnostic artifacts are non-critical: warn and keep benching on + // I/O failure instead of aborting the whole run. let path = format!("{dir}/{}.dict", scenario.id); - std::fs::write(&path, &ffi_dictionary).expect("dump dict"); + if let Err(err) = std::fs::write(&path, &ffi_dictionary) { + eprintln!("BENCH_WARN failed to dump dict {path}: {err}"); + } // Scenario input bytes too, so standalone profiling binaries // (`encode_loop_dict` / `decode_loop_dict`) can replay the // exact (input, dict) pair this scenario benches. let path = format!("{dir}/{}.bin", scenario.id); - std::fs::write(&path, scenario.bytes.as_slice()).expect("dump scenario bytes"); + if let Err(err) = std::fs::write(&path, scenario.bytes.as_slice()) { + eprintln!("BENCH_WARN failed to dump scenario bytes {path}: {err}"); + } } if emit_reports { @@ -676,8 +682,11 @@ fn bench_dictionary(c: &mut Criterion) { // (`decode_loop_dict`) can decode the EXACT bytes the // `decompress-dict/...` bench arm measures. if let Ok(dir) = std::env::var("STRUCTURED_ZSTD_DUMP_DICT_DIR") { + // Non-critical diagnostic: warn, do not abort the bench. let path = format!("{dir}/{}.{}.zst", scenario.id, level.name); - std::fs::write(&path, &with_dict_bytes).expect("dump dict payload"); + if let Err(err) = std::fs::write(&path, &with_dict_bytes) { + eprintln!("BENCH_WARN failed to dump dict payload {path}: {err}"); + } } // Rust dict-compressed output size, for the compress-dict diff --git a/zstd/examples/decode_loop_dict.rs b/zstd/examples/decode_loop_dict.rs index 166fd9a85..2bc011381 100644 --- a/zstd/examples/decode_loop_dict.rs +++ b/zstd/examples/decode_loop_dict.rs @@ -42,8 +42,11 @@ fn main() { let n = decoder .decode_all_with_dict_handle(payload.as_slice(), output.as_mut_slice(), &handle) .expect("dict decode should succeed"); + // A short decode means the (payload, dict, expected_len) triple is + // mismatched — fail loudly instead of profiling the wrong workload. + assert_eq!(n, expected_len, "decoded length mismatch"); sink = sink.wrapping_add(n); - core::hint::black_box(&output); + core::hint::black_box(&output[..n]); } eprintln!( From bfd49505c74b63e239e5098c7853472e58d9a2d7 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Wed, 10 Jun 2026 13:23:06 +0300 Subject: [PATCH 12/22] perf(encode): 4-byte gate before repcode common_prefix_len Donor MEM_read32 rep gate from the zstd_lazy.c lazy loop: a first-4-byte mismatch can never reach the >= min_match_len floor (asserted >= 4), so non-matching rep offsets are rejected on one scalar compare instead of a SIMD count call. Same shape as the HC chain gate; byte-identical output. repcode_candidate_shared runs once per encoded byte on lazy/Dfast/Row (10.2% self on the L10 row profile). --- zstd/src/encoding/match_table/helpers.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/zstd/src/encoding/match_table/helpers.rs b/zstd/src/encoding/match_table/helpers.rs index 648af328d..1c2b34f4a 100644 --- a/zstd/src/encoding/match_table/helpers.rs +++ b/zstd/src/encoding/match_table/helpers.rs @@ -144,6 +144,9 @@ pub(crate) fn repcode_candidate_shared( if current_idx + min_match_len > concat.len() { return None; } + // The 4-byte gate below relies on a first-4-byte mismatch implying the + // match can never reach the acceptance floor. + debug_assert!(min_match_len >= 4, "repcode gate requires min_match >= 4"); // Called once per input byte (10% exclusive on default-level profile). // The previous form built an `[Option; 3]` and walked it via @@ -173,7 +176,17 @@ pub(crate) fn repcode_candidate_shared( let rep = $rep; if rep != 0 && rep <= abs_pos { let candidate_pos = abs_pos - rep; - if candidate_pos >= history_abs_start { + // Donor `MEM_read32` rep gate (`zstd_lazy.c` lazy loop): a + // first-4-byte mismatch can never reach the + // `>= min_match_len` floor (asserted >= 4 above), so reject + // on one scalar compare instead of paying the SIMD count + // call on every non-matching rep. In-bounds: `current_idx + + // 4 <= concat.len()` from the entry guard, and + // `candidate_idx < current_idx`. + if candidate_pos >= history_abs_start && { + let candidate_idx = candidate_pos - history_abs_start; + concat[candidate_idx..candidate_idx + 4] == concat[current_idx..current_idx + 4] + } { let candidate_idx = candidate_pos - history_abs_start; let match_len = common_prefix_len_with_kernel( kernel, From 6b5773c5112e6c86acae2cf2d048183cf5be34e5 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Wed, 10 Jun 2026 13:30:43 +0300 Subject: [PATCH 13/22] perf(encode): prefetch the upcoming row in the lazy scan loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Donor ZSTD_row_fillHashCache / ZSTD_row_prefetch cadence: each lazy-loop iteration prefetches the tag bytes + position words of the row 8 positions ahead (ZSTD_ROW_HASH_CACHE_SIZE). The row address depends only on input bytes, so the load is not on a serial dependency chain and the prefetch genuinely overlaps the current position's work — row-tag load latency was 16.7% self in the L10 probe. Also refreshes the stale start_matching_rl doc (it is the lazy band's hot path now). --- zstd/src/encoding/row/mod.rs | 61 ++++++++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/zstd/src/encoding/row/mod.rs b/zstd/src/encoding/row/mod.rs index 89efc33b5..b64149045 100644 --- a/zstd/src/encoding/row/mod.rs +++ b/zstd/src/encoding/row/mod.rs @@ -219,6 +219,12 @@ macro_rules! dispatch_tag_kernel { /// Bind the runtime `row_log` (clamped 4..=6) to the const `ROW_LOG` of a /// `*_rl::` hot loop. Mirrors the donor's per-rowLog variant /// table; the branch is cold (once per block / call). +/// Lookahead distance (positions) for the lazy-loop row prefetch — +/// donor `ZSTD_ROW_HASH_CACHE_SIZE` (8): far enough to cover an L2 +/// round-trip at ~1 position/iteration, close enough that the row is +/// still resident when the scan arrives. +const ROW_PREFETCH_DISTANCE: usize = 8; + macro_rules! dispatch_row_log { ($self:ident . $rl_method:ident :: <$k:ty> ( $($arg:expr),* )) => { match $self.row_log { @@ -987,19 +993,9 @@ impl RowMatchGenerator { } } - /// Lazy-style parse (depth >= 1) for the Row backend. - /// - /// Currently unused: the only strategy mapped to `BackendTag::Row` - /// is `StrategyTag::Greedy` (level 4), which dispatches to - /// [`Self::start_matching_greedy`] via the `debug_assert_eq!` in - /// the `BackendTag::Row` arm of - /// `MatchGeneratorDriver::compress_block`. This method is kept as - /// scaffolding for the case where a future level routes a lazy - /// strategy through the Row backend — extracting `pick_lazy_match` - /// behavior to a fresh module then would mean re-deriving the - /// row-hash machinery, which is wasteful. The `dead_code` allow is - /// scoped to this method and its private helpers so any new - /// caller will pick them up unmodified. + /// Lazy-style parse (depth >= 1) for the Row backend — the hot path + /// for the lazy band (L6-12), which runs on the Row finder per the + /// donor's default row-matchfinder mode. pub(crate) fn start_matching_rl( &mut self, mut handle_sequence: impl for<'a> FnMut(Sequence<'a>), @@ -1024,6 +1020,10 @@ impl RowMatchGenerator { let abs_pos = current_abs_start + pos; let lit_len = pos - literals_start; + // Donor `ZSTD_row_fillHashCache` cadence: prefetch the row a + // few positions ahead so the probe's tag/position loads are + // L1-resident by the time the scan reaches them. + self.prefetch_row(abs_pos + ROW_PREFETCH_DISTANCE); let best = self.best_match_rl::(abs_pos, lit_len); if let Some(candidate) = self.pick_lazy_match_rl::(abs_pos, lit_len, best) { self.insert_match_span::(abs_pos, candidate.start + candidate.match_len); @@ -1327,6 +1327,41 @@ impl RowMatchGenerator { self.history_abs_start + self.live_history().len() } + /// Donor `ZSTD_row_prefetch`: pull the row's tag bytes and position + /// words toward L1 ahead of the probe that will touch them. The row + /// address depends only on input bytes (not on table contents), so + /// unlike a chain-walk candidate the load is NOT on a serial + /// dependency chain and prefetching genuinely overlaps it with the + /// current position's work. x86-only (the bench-relevant targets); + /// other arches compile it out. + #[inline(always)] + fn prefetch_row(&self, abs_pos: usize) { + #[cfg(target_arch = "x86_64")] + if let Some((row, _tag)) = self.hash_and_row(abs_pos) { + let row_base = row << self.row_log; + let entries = 1usize << self.row_log; + // SAFETY: `row_base` indexes a full row inside the tag / + // position tables (`row < 1 << row_hash_log`, tables sized + // `rows * entries`); prefetch is non-faulting regardless. + unsafe { + use core::arch::x86_64::{_MM_HINT_T0, _mm_prefetch}; + _mm_prefetch( + self.row_tags.as_ptr().add(row_base).cast::(), + _MM_HINT_T0, + ); + let positions = self.row_positions.as_ptr().add(row_base).cast::(); + _mm_prefetch(positions, _MM_HINT_T0); + // Position words span `entries * 4` bytes — touch the + // second cache line too once the row exceeds one line. + if entries * core::mem::size_of::() > 64 { + _mm_prefetch(positions.add(64), _MM_HINT_T0); + } + } + } + #[cfg(not(target_arch = "x86_64"))] + let _ = abs_pos; + } + pub(crate) fn hash_and_row(&self, abs_pos: usize) -> Option<(usize, u8)> { let idx = abs_pos - self.history_abs_start; let concat = self.live_history(); From ba30e7840244dddb7c6d96c30144696782dd6328 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Wed, 10 Jun 2026 13:35:46 +0300 Subject: [PATCH 14/22] perf(encode): row hash ring cache feeding probe and post-miss insert MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Donor ZSTD_row_fillHashCache: the lazy loop hashes the position 8 ahead ONCE, remembers (row, tag) in an 8-slot ring keyed by exact abs_pos, and prefetches that row. The probe and the post-miss insert then reuse the cached hash instead of recomputing (hash_and_row was 8% self; the prefetch alone was net-neutral because it paid a second hash per position). Ring is poisoned per block scan — add_data may rebase the absolute coordinate origin between blocks. Cache misses (lookahead past the horizon, match-span inserts after jumps) fall back to the plain hash; same value either way, byte-identical output. --- zstd/src/encoding/row/mod.rs | 91 +++++++++++++++++++++++++++++------- 1 file changed, 74 insertions(+), 17 deletions(-) diff --git a/zstd/src/encoding/row/mod.rs b/zstd/src/encoding/row/mod.rs index b64149045..4347f2af4 100644 --- a/zstd/src/encoding/row/mod.rs +++ b/zstd/src/encoding/row/mod.rs @@ -422,7 +422,7 @@ macro_rules! gen_row_probe { return None; } - let (row, tag) = self.hash_and_row(abs_pos)?; + let (row, tag) = self.hash_and_row_cached(abs_pos)?; let row_entries = 1usize << ROW_LOG; let row_mask = row_entries - 1; let row_base = row << ROW_LOG; @@ -742,6 +742,31 @@ pub(crate) struct RowMatchGenerator { /// activates the bounded dict probe in `row_candidate_rl`; built once and /// cached across frames via `DictAttach`, invalidated on eviction / resize. pub(crate) dict: DictAttach, + /// Donor `ZSTD_row_fillHashCache` ring: `(row, tag)` precomputed for the + /// position `ROW_PREFETCH_DISTANCE` ahead of the lazy scan, filled + /// together with the row prefetch so the probe (and the post-miss + /// insert) reuse the hash instead of recomputing it. Entries are keyed + /// by exact `abs_pos` (`usize::MAX` = empty) and the ring is poisoned at + /// the top of every block scan — `add_data` may rebase the absolute + /// coordinate origin between blocks, which would otherwise let a new + /// position collide with a stale entry's key. + hash_cache: [RowHashCacheEntry; ROW_PREFETCH_DISTANCE], +} + +/// One [`RowMatchGenerator::hash_cache`] slot. +#[derive(Copy, Clone)] +struct RowHashCacheEntry { + pos: usize, + row: u32, + tag: u8, +} + +impl RowHashCacheEntry { + const EMPTY: Self = Self { + pos: usize::MAX, + row: 0, + tag: 0, + }; } impl RowMatchGenerator { @@ -766,6 +791,7 @@ impl RowMatchGenerator { row_tags: Vec::new(), tag_kernel: crate::encoding::fastpath::select_kernel(), dict: DictAttach::new(), + hash_cache: [RowHashCacheEntry::EMPTY; ROW_PREFETCH_DISTANCE], } } @@ -1013,6 +1039,11 @@ impl RowMatchGenerator { self.insert_positions::(backfill_start, current_abs_start); } + // Poison the hash ring at block-scan entry: `add_data` can rebase + // the absolute coordinate origin between blocks, so a stale entry's + // key could collide with a new position over different bytes. + self.hash_cache = [RowHashCacheEntry::EMPTY; ROW_PREFETCH_DISTANCE]; + let mls = self.mls; let mut pos = 0usize; let mut literals_start = 0usize; @@ -1020,10 +1051,11 @@ impl RowMatchGenerator { let abs_pos = current_abs_start + pos; let lit_len = pos - literals_start; - // Donor `ZSTD_row_fillHashCache` cadence: prefetch the row a - // few positions ahead so the probe's tag/position loads are - // L1-resident by the time the scan reaches them. - self.prefetch_row(abs_pos + ROW_PREFETCH_DISTANCE); + // Donor `ZSTD_row_fillHashCache` cadence: hash + prefetch the + // row a few positions ahead so the probe's tag/position loads + // are L1-resident by the time the scan reaches them, and the + // probe / post-miss insert reuse the cached hash. + self.fill_hash_cache_ahead(abs_pos); let best = self.best_match_rl::(abs_pos, lit_len); if let Some(candidate) = self.pick_lazy_match_rl::(abs_pos, lit_len, best) { self.insert_match_span::(abs_pos, candidate.start + candidate.match_len); @@ -1327,17 +1359,27 @@ impl RowMatchGenerator { self.history_abs_start + self.live_history().len() } - /// Donor `ZSTD_row_prefetch`: pull the row's tag bytes and position - /// words toward L1 ahead of the probe that will touch them. The row - /// address depends only on input bytes (not on table contents), so - /// unlike a chain-walk candidate the load is NOT on a serial - /// dependency chain and prefetching genuinely overlaps it with the - /// current position's work. x86-only (the bench-relevant targets); - /// other arches compile it out. + /// Donor `ZSTD_row_fillHashCache` step: hash the position + /// `ROW_PREFETCH_DISTANCE` ahead of the scan, remember its `(row, + /// tag)` in the ring so the probe / post-miss insert reuse it, and + /// prefetch the row's tag bytes + position words toward L1 (donor + /// `ZSTD_row_prefetch`). The row address depends only on input bytes + /// (not on table contents), so unlike a chain-walk candidate the + /// load is NOT on a serial dependency chain and the prefetch + /// genuinely overlaps the current position's work. #[inline(always)] - fn prefetch_row(&self, abs_pos: usize) { + fn fill_hash_cache_ahead(&mut self, abs_pos: usize) { + let ahead = abs_pos + ROW_PREFETCH_DISTANCE; + let Some((row, tag)) = self.hash_and_row(ahead) else { + return; + }; + self.hash_cache[ahead % ROW_PREFETCH_DISTANCE] = RowHashCacheEntry { + pos: ahead, + row: row as u32, + tag, + }; #[cfg(target_arch = "x86_64")] - if let Some((row, _tag)) = self.hash_and_row(abs_pos) { + { let row_base = row << self.row_log; let entries = 1usize << self.row_log; // SAFETY: `row_base` indexes a full row inside the tag / @@ -1358,8 +1400,21 @@ impl RowMatchGenerator { } } } - #[cfg(not(target_arch = "x86_64"))] - let _ = abs_pos; + } + + /// [`Self::hash_and_row`] with a ring-cache fast path: an exact + /// `abs_pos` hit returns the precomputed `(row, tag)` from + /// [`Self::fill_hash_cache_ahead`]; anything else (lookahead probes + /// past the fill horizon, match-span inserts after a jump) falls back + /// to the plain hash. Same value either way — the cache is purely a + /// recompute saver. + #[inline(always)] + fn hash_and_row_cached(&self, abs_pos: usize) -> Option<(usize, u8)> { + let entry = self.hash_cache[abs_pos % ROW_PREFETCH_DISTANCE]; + if entry.pos == abs_pos { + return Some((entry.row as usize, entry.tag)); + } + self.hash_and_row(abs_pos) } pub(crate) fn hash_and_row(&self, abs_pos: usize) -> Option<(usize, u8)> { @@ -1641,7 +1696,9 @@ impl RowMatchGenerator { #[inline] fn insert_position(&mut self, abs_pos: usize) { - let Some((row, tag)) = self.hash_and_row(abs_pos) else { + // Cached lookup: a post-miss insert at the scan position reuses the + // hash the probe just consumed for the same `abs_pos`. + let Some((row, tag)) = self.hash_and_row_cached(abs_pos) else { return; }; // `ROW_LOG` is the compile-time row width for this monomorphisation; From d83ba6167eb38f0400300a11d6b2508e28cb9992 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Wed, 10 Jun 2026 14:01:48 +0300 Subject: [PATCH 15/22] fix(encode): keep btlazy2 off the Row finder; donor hashLog cap for row tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-ups to the lazy-band Row switch: - public Strategy::Btlazy2 no longer collapses onto the runtime Lazy tag (Lazy resolves to the Row finder now; btlazy2 is a binary-tree search and must stay on the HashChain/BT storage), and StrategyTag::for_level maps 13-15 to Btlazy2, matching LEVEL_TABLE and the donor clevels.h strategy column its doc already cited - row_hash_bits_for_window applies the donor ZSTD_adjustCParams cap (hashLog <= windowLog + 1): the +1 is load-bearing for L12 (donor hashLog 23 > windowLog 22) — a plain windowLog cap shrank the L12 table on every hinted reset and split primed snapshots between hinted/unhinted frames resolving to identical geometry - the remaining HC-pinned tests that reset via the Better/Best aliases (which resolve to Row now) route through the HashChain override so they keep covering the paths their assertions claim --- zstd/src/encoding/match_generator.rs | 32 +++++++++++++++++----------- zstd/src/encoding/parameters.rs | 13 ++++++----- zstd/src/encoding/strategy.rs | 13 +++++++++-- 3 files changed, 38 insertions(+), 20 deletions(-) diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs index 6eda376d8..3ca125a5c 100644 --- a/zstd/src/encoding/match_generator.rs +++ b/zstd/src/encoding/match_generator.rs @@ -691,14 +691,16 @@ fn dfast_hash_bits_for_window(max_window_size: usize) -> usize { } fn row_hash_bits_for_window(max_window_size: usize) -> usize { - // Window-derived cap on the row hash width (stricter than the donor - // `ZSTD_adjustCParams_internal` `hashLog <= windowLog + 1` bound, so a - // tiny hinted source keeps the tiny table). No constant upper clamp: - // the old `ROW_HASH_BITS` (20) ceiling predates the lazy band moving - // onto Row — L9-12 carry donor hashLog 21-23 and must keep it when the - // hinted window is large enough. + // Donor `ZSTD_adjustCParams_internal` cap: `hashLog <= windowLog + 1`. + // The `+ 1` is load-bearing for L12, whose donor hashLog (23) exceeds + // its windowLog (22) — a plain `windowLog` cap would shrink the L12 + // table on EVERY hinted reset and split primed snapshots between + // hinted and unhinted frames that resolve to the identical geometry. + // No constant upper clamp: the old `ROW_HASH_BITS` (20) ceiling + // predates the lazy band moving onto Row (L9-12 carry donor hashLog + // 21-23). let window_log = (usize::BITS - 1 - max_window_size.leading_zeros()) as usize; - window_log.max(MIN_WINDOW_LOG as usize) + (window_log + 1).max(MIN_WINDOW_LOG as usize) } /// `floor(log2(window))` for the HashChain table-log cap (donor @@ -7737,12 +7739,12 @@ fn driver_small_source_hint_shrinks_row_hash_tables() { // Wire `window_log` stays floored, but the row hash table is sized from // the RAW 1 KiB source: `table_window = 1 << 10`, so - // `row_hash_bits_for_window(1 << 10) = MIN_WINDOW_LOG` and the row count - // is `1 << (MIN_WINDOW_LOG - ROW_L5.row_log)`. + // `row_hash_bits_for_window(1 << 10) = 11` (donor `hashLog <= + // windowLog + 1`) and the row count is `1 << (11 - ROW_L5.row_log)`. assert_eq!(driver.window_size(), 1 << MIN_HINTED_WINDOW_LOG); assert_eq!( hinted_rows, - 1 << ((MIN_WINDOW_LOG as usize) - ROW_L5.row_log) + 1 << ((MIN_WINDOW_LOG as usize) + 1 - ROW_L5.row_log) ); assert!( hinted_rows < full_rows, @@ -8088,8 +8090,10 @@ fn pooled_space_keeps_capacity_when_slice_size_shrinks() { fn driver_best_to_fastest_releases_oversized_hc_tables() { let mut driver = MatchGeneratorDriver::new(32, 2); - // Initialize at Best — allocates large HC tables (4M hash, 2M chain). - driver.reset(CompressionLevel::Best); + // Initialize at Best routed onto HashChain (the production `Best` + // resolves to Row now) — allocates large HC tables (4M hash, 2M chain) + // so the swap below exercises the HC drain path this test pins. + driver.reset_on_hc_lazy(CompressionLevel::Best); assert_eq!(driver.window_size(), (1u64 << 22)); // Feed data so tables are actually allocated via ensure_tables(). @@ -9635,7 +9639,9 @@ fn prime_with_dictionary_budget_shrinks_after_dfast_eviction() { #[test] fn hc_prime_with_dictionary_preserves_history_for_first_full_block() { let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset(CompressionLevel::Better); + // Route onto HashChain explicitly — `Better` resolves to the Row + // backend in production, and this test pins HC dict-prime behaviour. + driver.reset_on_hc_lazy(CompressionLevel::Better); driver.prime_with_dictionary(b"abcdefgh", [1, 4, 8]); diff --git a/zstd/src/encoding/parameters.rs b/zstd/src/encoding/parameters.rs index 076fbf8f4..6356ab627 100644 --- a/zstd/src/encoding/parameters.rs +++ b/zstd/src/encoding/parameters.rs @@ -107,17 +107,20 @@ impl Strategy { }) } - /// Internal runtime strategy tag (collapses `btlazy2` onto the lazy - /// hash-chain tag at depth 2, matching `LEVEL_TABLE`). + /// Internal runtime strategy tag. pub(crate) const fn tag(self) -> crate::encoding::strategy::StrategyTag { use crate::encoding::strategy::StrategyTag; match self { Self::Fast => StrategyTag::Fast, Self::Dfast => StrategyTag::Dfast, Self::Greedy => StrategyTag::Greedy, - // Lazy / Lazy2 / Btlazy2 all ride the runtime `Lazy` tag; the - // lazy lookahead depth carries the variance (see `lazy_depth`). - Self::Lazy | Self::Lazy2 | Self::Btlazy2 => StrategyTag::Lazy, + // Lazy / Lazy2 ride the runtime `Lazy` tag (the lazy lookahead + // depth carries the variance, see `lazy_depth`). `Btlazy2` + // keeps its own tag: `Lazy` resolves to the Row finder, while + // btlazy2 is a binary-tree search and must stay on the + // HashChain/BT storage. + Self::Lazy | Self::Lazy2 => StrategyTag::Lazy, + Self::Btlazy2 => StrategyTag::Btlazy2, Self::Btopt => StrategyTag::BtOpt, Self::Btultra => StrategyTag::BtUltra, Self::Btultra2 => StrategyTag::BtUltra2, diff --git a/zstd/src/encoding/strategy.rs b/zstd/src/encoding/strategy.rs index c3dbcb278..a10024b50 100644 --- a/zstd/src/encoding/strategy.rs +++ b/zstd/src/encoding/strategy.rs @@ -400,7 +400,12 @@ impl StrategyTag { 1 | 2 => Self::Fast, 3 | 4 => Self::Dfast, 5 => Self::Greedy, - 6..=15 => Self::Lazy, + 6..=12 => Self::Lazy, + // 13-15 are btlazy2 in `LEVEL_TABLE` (BinaryTree finder on the + // HashChain storage). Folding them onto `Lazy` was harmless + // while `Lazy` also meant HashChain; with `Lazy` resolving to + // the Row finder the distinction is load-bearing. + 13..=15 => Self::Btlazy2, 16 | 17 => Self::BtOpt, 18 => Self::BtUltra, 19 => Self::BtUltra2, @@ -548,7 +553,11 @@ mod tests { assert_eq!(StrategyTag::for_level(4), StrategyTag::Dfast); assert_eq!(StrategyTag::for_level(5), StrategyTag::Greedy); assert_eq!(StrategyTag::for_level(9), StrategyTag::Lazy); - assert_eq!(StrategyTag::for_level(15), StrategyTag::Lazy); + assert_eq!(StrategyTag::for_level(12), StrategyTag::Lazy); + // Donor `clevels.h` 13-15 are `ZSTD_btlazy2` — distinct from the + // Row-backed `Lazy` band. + assert_eq!(StrategyTag::for_level(13), StrategyTag::Btlazy2); + assert_eq!(StrategyTag::for_level(15), StrategyTag::Btlazy2); assert_eq!(StrategyTag::for_level(16), StrategyTag::BtOpt); assert_eq!(StrategyTag::for_level(17), StrategyTag::BtOpt); assert_eq!(StrategyTag::for_level(18), StrategyTag::BtUltra); From bda80104985ebceea2e4f07f9e917fbc900e353c Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Wed, 10 Jun 2026 14:19:21 +0300 Subject: [PATCH 16/22] Revert "perf(encode): row hash ring cache feeding probe and post-miss insert" This reverts commit ba30e7840244dddb7c6d96c30144696782dd6328. --- zstd/src/encoding/row/mod.rs | 91 +++++++----------------------------- 1 file changed, 17 insertions(+), 74 deletions(-) diff --git a/zstd/src/encoding/row/mod.rs b/zstd/src/encoding/row/mod.rs index 4347f2af4..b64149045 100644 --- a/zstd/src/encoding/row/mod.rs +++ b/zstd/src/encoding/row/mod.rs @@ -422,7 +422,7 @@ macro_rules! gen_row_probe { return None; } - let (row, tag) = self.hash_and_row_cached(abs_pos)?; + let (row, tag) = self.hash_and_row(abs_pos)?; let row_entries = 1usize << ROW_LOG; let row_mask = row_entries - 1; let row_base = row << ROW_LOG; @@ -742,31 +742,6 @@ pub(crate) struct RowMatchGenerator { /// activates the bounded dict probe in `row_candidate_rl`; built once and /// cached across frames via `DictAttach`, invalidated on eviction / resize. pub(crate) dict: DictAttach, - /// Donor `ZSTD_row_fillHashCache` ring: `(row, tag)` precomputed for the - /// position `ROW_PREFETCH_DISTANCE` ahead of the lazy scan, filled - /// together with the row prefetch so the probe (and the post-miss - /// insert) reuse the hash instead of recomputing it. Entries are keyed - /// by exact `abs_pos` (`usize::MAX` = empty) and the ring is poisoned at - /// the top of every block scan — `add_data` may rebase the absolute - /// coordinate origin between blocks, which would otherwise let a new - /// position collide with a stale entry's key. - hash_cache: [RowHashCacheEntry; ROW_PREFETCH_DISTANCE], -} - -/// One [`RowMatchGenerator::hash_cache`] slot. -#[derive(Copy, Clone)] -struct RowHashCacheEntry { - pos: usize, - row: u32, - tag: u8, -} - -impl RowHashCacheEntry { - const EMPTY: Self = Self { - pos: usize::MAX, - row: 0, - tag: 0, - }; } impl RowMatchGenerator { @@ -791,7 +766,6 @@ impl RowMatchGenerator { row_tags: Vec::new(), tag_kernel: crate::encoding::fastpath::select_kernel(), dict: DictAttach::new(), - hash_cache: [RowHashCacheEntry::EMPTY; ROW_PREFETCH_DISTANCE], } } @@ -1039,11 +1013,6 @@ impl RowMatchGenerator { self.insert_positions::(backfill_start, current_abs_start); } - // Poison the hash ring at block-scan entry: `add_data` can rebase - // the absolute coordinate origin between blocks, so a stale entry's - // key could collide with a new position over different bytes. - self.hash_cache = [RowHashCacheEntry::EMPTY; ROW_PREFETCH_DISTANCE]; - let mls = self.mls; let mut pos = 0usize; let mut literals_start = 0usize; @@ -1051,11 +1020,10 @@ impl RowMatchGenerator { let abs_pos = current_abs_start + pos; let lit_len = pos - literals_start; - // Donor `ZSTD_row_fillHashCache` cadence: hash + prefetch the - // row a few positions ahead so the probe's tag/position loads - // are L1-resident by the time the scan reaches them, and the - // probe / post-miss insert reuse the cached hash. - self.fill_hash_cache_ahead(abs_pos); + // Donor `ZSTD_row_fillHashCache` cadence: prefetch the row a + // few positions ahead so the probe's tag/position loads are + // L1-resident by the time the scan reaches them. + self.prefetch_row(abs_pos + ROW_PREFETCH_DISTANCE); let best = self.best_match_rl::(abs_pos, lit_len); if let Some(candidate) = self.pick_lazy_match_rl::(abs_pos, lit_len, best) { self.insert_match_span::(abs_pos, candidate.start + candidate.match_len); @@ -1359,27 +1327,17 @@ impl RowMatchGenerator { self.history_abs_start + self.live_history().len() } - /// Donor `ZSTD_row_fillHashCache` step: hash the position - /// `ROW_PREFETCH_DISTANCE` ahead of the scan, remember its `(row, - /// tag)` in the ring so the probe / post-miss insert reuse it, and - /// prefetch the row's tag bytes + position words toward L1 (donor - /// `ZSTD_row_prefetch`). The row address depends only on input bytes - /// (not on table contents), so unlike a chain-walk candidate the - /// load is NOT on a serial dependency chain and the prefetch - /// genuinely overlaps the current position's work. + /// Donor `ZSTD_row_prefetch`: pull the row's tag bytes and position + /// words toward L1 ahead of the probe that will touch them. The row + /// address depends only on input bytes (not on table contents), so + /// unlike a chain-walk candidate the load is NOT on a serial + /// dependency chain and prefetching genuinely overlaps it with the + /// current position's work. x86-only (the bench-relevant targets); + /// other arches compile it out. #[inline(always)] - fn fill_hash_cache_ahead(&mut self, abs_pos: usize) { - let ahead = abs_pos + ROW_PREFETCH_DISTANCE; - let Some((row, tag)) = self.hash_and_row(ahead) else { - return; - }; - self.hash_cache[ahead % ROW_PREFETCH_DISTANCE] = RowHashCacheEntry { - pos: ahead, - row: row as u32, - tag, - }; + fn prefetch_row(&self, abs_pos: usize) { #[cfg(target_arch = "x86_64")] - { + if let Some((row, _tag)) = self.hash_and_row(abs_pos) { let row_base = row << self.row_log; let entries = 1usize << self.row_log; // SAFETY: `row_base` indexes a full row inside the tag / @@ -1400,21 +1358,8 @@ impl RowMatchGenerator { } } } - } - - /// [`Self::hash_and_row`] with a ring-cache fast path: an exact - /// `abs_pos` hit returns the precomputed `(row, tag)` from - /// [`Self::fill_hash_cache_ahead`]; anything else (lookahead probes - /// past the fill horizon, match-span inserts after a jump) falls back - /// to the plain hash. Same value either way — the cache is purely a - /// recompute saver. - #[inline(always)] - fn hash_and_row_cached(&self, abs_pos: usize) -> Option<(usize, u8)> { - let entry = self.hash_cache[abs_pos % ROW_PREFETCH_DISTANCE]; - if entry.pos == abs_pos { - return Some((entry.row as usize, entry.tag)); - } - self.hash_and_row(abs_pos) + #[cfg(not(target_arch = "x86_64"))] + let _ = abs_pos; } pub(crate) fn hash_and_row(&self, abs_pos: usize) -> Option<(usize, u8)> { @@ -1696,9 +1641,7 @@ impl RowMatchGenerator { #[inline] fn insert_position(&mut self, abs_pos: usize) { - // Cached lookup: a post-miss insert at the scan position reuses the - // hash the probe just consumed for the same `abs_pos`. - let Some((row, tag)) = self.hash_and_row_cached(abs_pos) else { + let Some((row, tag)) = self.hash_and_row(abs_pos) else { return; }; // `ROW_LOG` is the compile-time row width for this monomorphisation; From fc1cff36a64c8f29f3f3984be6b945df34e0c6bd Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Wed, 10 Jun 2026 14:19:21 +0300 Subject: [PATCH 17/22] Revert "perf(encode): prefetch the upcoming row in the lazy scan loop" This reverts commit 6b5773c5112e6c86acae2cf2d048183cf5be34e5. --- zstd/src/encoding/row/mod.rs | 61 ++++++++---------------------------- 1 file changed, 13 insertions(+), 48 deletions(-) diff --git a/zstd/src/encoding/row/mod.rs b/zstd/src/encoding/row/mod.rs index b64149045..89efc33b5 100644 --- a/zstd/src/encoding/row/mod.rs +++ b/zstd/src/encoding/row/mod.rs @@ -219,12 +219,6 @@ macro_rules! dispatch_tag_kernel { /// Bind the runtime `row_log` (clamped 4..=6) to the const `ROW_LOG` of a /// `*_rl::` hot loop. Mirrors the donor's per-rowLog variant /// table; the branch is cold (once per block / call). -/// Lookahead distance (positions) for the lazy-loop row prefetch — -/// donor `ZSTD_ROW_HASH_CACHE_SIZE` (8): far enough to cover an L2 -/// round-trip at ~1 position/iteration, close enough that the row is -/// still resident when the scan arrives. -const ROW_PREFETCH_DISTANCE: usize = 8; - macro_rules! dispatch_row_log { ($self:ident . $rl_method:ident :: <$k:ty> ( $($arg:expr),* )) => { match $self.row_log { @@ -993,9 +987,19 @@ impl RowMatchGenerator { } } - /// Lazy-style parse (depth >= 1) for the Row backend — the hot path - /// for the lazy band (L6-12), which runs on the Row finder per the - /// donor's default row-matchfinder mode. + /// Lazy-style parse (depth >= 1) for the Row backend. + /// + /// Currently unused: the only strategy mapped to `BackendTag::Row` + /// is `StrategyTag::Greedy` (level 4), which dispatches to + /// [`Self::start_matching_greedy`] via the `debug_assert_eq!` in + /// the `BackendTag::Row` arm of + /// `MatchGeneratorDriver::compress_block`. This method is kept as + /// scaffolding for the case where a future level routes a lazy + /// strategy through the Row backend — extracting `pick_lazy_match` + /// behavior to a fresh module then would mean re-deriving the + /// row-hash machinery, which is wasteful. The `dead_code` allow is + /// scoped to this method and its private helpers so any new + /// caller will pick them up unmodified. pub(crate) fn start_matching_rl( &mut self, mut handle_sequence: impl for<'a> FnMut(Sequence<'a>), @@ -1020,10 +1024,6 @@ impl RowMatchGenerator { let abs_pos = current_abs_start + pos; let lit_len = pos - literals_start; - // Donor `ZSTD_row_fillHashCache` cadence: prefetch the row a - // few positions ahead so the probe's tag/position loads are - // L1-resident by the time the scan reaches them. - self.prefetch_row(abs_pos + ROW_PREFETCH_DISTANCE); let best = self.best_match_rl::(abs_pos, lit_len); if let Some(candidate) = self.pick_lazy_match_rl::(abs_pos, lit_len, best) { self.insert_match_span::(abs_pos, candidate.start + candidate.match_len); @@ -1327,41 +1327,6 @@ impl RowMatchGenerator { self.history_abs_start + self.live_history().len() } - /// Donor `ZSTD_row_prefetch`: pull the row's tag bytes and position - /// words toward L1 ahead of the probe that will touch them. The row - /// address depends only on input bytes (not on table contents), so - /// unlike a chain-walk candidate the load is NOT on a serial - /// dependency chain and prefetching genuinely overlaps it with the - /// current position's work. x86-only (the bench-relevant targets); - /// other arches compile it out. - #[inline(always)] - fn prefetch_row(&self, abs_pos: usize) { - #[cfg(target_arch = "x86_64")] - if let Some((row, _tag)) = self.hash_and_row(abs_pos) { - let row_base = row << self.row_log; - let entries = 1usize << self.row_log; - // SAFETY: `row_base` indexes a full row inside the tag / - // position tables (`row < 1 << row_hash_log`, tables sized - // `rows * entries`); prefetch is non-faulting regardless. - unsafe { - use core::arch::x86_64::{_MM_HINT_T0, _mm_prefetch}; - _mm_prefetch( - self.row_tags.as_ptr().add(row_base).cast::(), - _MM_HINT_T0, - ); - let positions = self.row_positions.as_ptr().add(row_base).cast::(); - _mm_prefetch(positions, _MM_HINT_T0); - // Position words span `entries * 4` bytes — touch the - // second cache line too once the row exceeds one line. - if entries * core::mem::size_of::() > 64 { - _mm_prefetch(positions.add(64), _MM_HINT_T0); - } - } - } - #[cfg(not(target_arch = "x86_64"))] - let _ = abs_pos; - } - pub(crate) fn hash_and_row(&self, abs_pos: usize) -> Option<(usize, u8)> { let idx = abs_pos - self.history_abs_start; let concat = self.live_history(); From 4874f0badbff978ae05d19d5a922fde1932bb3db Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Wed, 10 Jun 2026 15:04:03 +0300 Subject: [PATCH 18/22] fix(encode): let donor row hash widths reach the backend; btlazy2 parse depth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups on the lazy-band Row switch: - RowMatchGenerator::set_hash_bits upper clamp moves from the legacy 20-bit ROW_HASH_BITS ceiling to the donor ZSTD_HASHLOG_MAX bound: the driver resolves donor widths 21-23 for L9-12, and the backend silently shrinking them left the primed-snapshot key recording a geometry the tables did not have - dict_attach_epoch treats Some(0) as "no dictionary", matching the dict-sizing filter above it — an empty dict primes nothing, so an epoch-advance reset would preserve stale attach state - StrategyTag::Btlazy2 parse_mode maps to Lazy2 (donor btlazy2 = BT finder + depth-2 lazy parse; lazy_depth() already reported 2) - repcode probe hoists candidate_idx (computed once for the gate and the count) --- zstd/src/encoding/match_generator.rs | 6 +++- zstd/src/encoding/match_table/helpers.rs | 37 ++++++++++++------------ zstd/src/encoding/row/mod.rs | 8 ++++- zstd/src/encoding/strategy.rs | 7 +++-- 4 files changed, 36 insertions(+), 22 deletions(-) diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs index 3ca125a5c..21d3b8ebc 100644 --- a/zstd/src/encoding/match_generator.rs +++ b/zstd/src/encoding/match_generator.rs @@ -2037,7 +2037,11 @@ impl Matcher for MatchGeneratorDriver { // frames may keep the main table across the reset via an // epoch advance — copy-mode and no-dict frames must memset // it back to bias 0 for the raw-slice kernels. - let dict_attach_epoch = dict_hint.is_some() + // `Some(0)` is "no dictionary" (the dict-sizing path above + // filters it the same way): an empty dict primes nothing, so + // an epoch-advance reset would preserve stale attach state + // instead of clearing it. + let dict_attach_epoch = matches!(dict_hint, Some(size) if size > 0) && self .reset_size_log .is_none_or(|log| log <= FAST_ATTACH_DICT_CUTOFF_LOG); diff --git a/zstd/src/encoding/match_table/helpers.rs b/zstd/src/encoding/match_table/helpers.rs index 1c2b34f4a..c412df634 100644 --- a/zstd/src/encoding/match_table/helpers.rs +++ b/zstd/src/encoding/match_table/helpers.rs @@ -183,26 +183,27 @@ pub(crate) fn repcode_candidate_shared( // call on every non-matching rep. In-bounds: `current_idx + // 4 <= concat.len()` from the entry guard, and // `candidate_idx < current_idx`. - if candidate_pos >= history_abs_start && { + if candidate_pos >= history_abs_start { let candidate_idx = candidate_pos - history_abs_start; - concat[candidate_idx..candidate_idx + 4] == concat[current_idx..current_idx + 4] - } { - let candidate_idx = candidate_pos - history_abs_start; - let match_len = common_prefix_len_with_kernel( - kernel, - &concat[candidate_idx..], - &concat[current_idx..], - ); - if match_len >= min_match_len { - let candidate = extend_backwards_shared( - concat, - history_abs_start, - candidate_pos, - abs_pos, - match_len, - lit_len, + if concat[candidate_idx..candidate_idx + 4] + == concat[current_idx..current_idx + 4] + { + let match_len = common_prefix_len_with_kernel( + kernel, + &concat[candidate_idx..], + &concat[current_idx..], ); - best = best_len_offset_candidate(best, Some(candidate)); + if match_len >= min_match_len { + let candidate = extend_backwards_shared( + concat, + history_abs_start, + candidate_pos, + abs_pos, + match_len, + lit_len, + ); + best = best_len_offset_candidate(best, Some(candidate)); + } } } } diff --git a/zstd/src/encoding/row/mod.rs b/zstd/src/encoding/row/mod.rs index 89efc33b5..4ac055a04 100644 --- a/zstd/src/encoding/row/mod.rs +++ b/zstd/src/encoding/row/mod.rs @@ -778,7 +778,13 @@ impl RowMatchGenerator { } pub(crate) fn set_hash_bits(&mut self, bits: usize) { - let clamped = bits.clamp(self.row_log + 1, ROW_HASH_BITS); + // Upper bound mirrors the donor `ZSTD_HASHLOG_MAX` (27) rather than + // the old `ROW_HASH_BITS` (20) ceiling: the lazy band's donor + // configs carry hashLog 21-23 (L9-12), and clamping below the + // driver's resolved width would silently shrink the tables while + // the primed-snapshot key still recorded the wider geometry. + const ROW_HASH_BITS_MAX: usize = 27; + let clamped = bits.clamp(self.row_log + 1, ROW_HASH_BITS_MAX); let row_hash_log = clamped.saturating_sub(self.row_log); if self.row_hash_log != row_hash_log { self.row_hash_log = row_hash_log; diff --git a/zstd/src/encoding/strategy.rs b/zstd/src/encoding/strategy.rs index a10024b50..8019a66eb 100644 --- a/zstd/src/encoding/strategy.rs +++ b/zstd/src/encoding/strategy.rs @@ -476,7 +476,10 @@ impl StrategyTag { pub(crate) const fn parse_mode(self) -> ParseMode { match self { Self::Fast | Self::Dfast | Self::Greedy => ParseMode::Greedy, - Self::Lazy | Self::Btlazy2 => ParseMode::Lazy, + Self::Lazy => ParseMode::Lazy, + // Donor btlazy2 = BinaryTree finder + depth-2 lazy parse + // (`Strategy::lazy_depth()` reports 2 for it as well). + Self::Btlazy2 => ParseMode::Lazy2, Self::BtOpt | Self::BtUltra | Self::BtUltra2 => ParseMode::Optimal, } } @@ -508,7 +511,7 @@ mod tests { fn btlazy2_tag_bridge_contract() { assert_eq!(StrategyTag::Btlazy2.backend(), BackendTag::HashChain); assert_eq!(StrategyTag::Btlazy2.search(), SearchMethod::BinaryTree); - assert_eq!(StrategyTag::Btlazy2.parse_mode(), ParseMode::Lazy); + assert_eq!(StrategyTag::Btlazy2.parse_mode(), ParseMode::Lazy2); // The BT walk cap must let L15's search_depth = 64 govern (BtOpt's // 32 would silently halve it); full find, no early bail. assert_eq!(Btlazy2::MAX_CHAIN_DEPTH, 64); From ee9bc0905980d20c4b29f3a6117b7385490de271 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Wed, 10 Jun 2026 15:12:11 +0300 Subject: [PATCH 19/22] fix(encode): key row snapshots on the applied hash width, keep 20-bit cap Measured follow-up to the donor-width attempt: the honest 21-bit L10 row table cost +26.8% wall (flat control) for a 19-byte output delta, so the 20-bit ROW_HASH_BITS cap stays as a deliberate deviation (our lazy-band ratio already beats the donor with it). The REAL bug was the primed-snapshot key recording the requested width while set_hash_bits clamped the tables: the key now reads RowMatchGenerator::hash_bits() (the applied width) on both hinted and unhinted paths, so identical geometries key identically. --- zstd/src/encoding/match_generator.rs | 27 +++++++++++++-------------- zstd/src/encoding/row/mod.rs | 22 +++++++++++++++------- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs index 21d3b8ebc..ac8b0c0c4 100644 --- a/zstd/src/encoding/match_generator.rs +++ b/zstd/src/encoding/match_generator.rs @@ -2083,25 +2083,24 @@ impl Matcher for MatchGeneratorDriver { row.lazy_depth = params.lazy_depth; let row_cfg = params.row.expect("Row level row carries a RowConfig"); row.configure(row_cfg); - // Record the RESOLVED hash width in the primed-snapshot key on - // both paths. Leaving the unhinted path at the 0 default keyed - // an unhinted capture differently from a hinted reset that - // resolves to the IDENTICAL tables (hint >= window makes the - // clamp a no-op), forcing a needless dictionary re-prime. - resolved_table_bits = if hinted { + if hinted { // Clamp the configured hash width by the hinted window // (donor `ZSTD_adjustCParams` caps hashLog by windowLog) — // `min`, not replace, so an explicit `hash_log` param // override (`row_cfg.hash_bits`) survives the hinted path // instead of being overwritten by the window value. - let bits = row_cfg - .hash_bits - .min(row_hash_bits_for_window(table_window_size)); - row.set_hash_bits(bits); - bits - } else { - row_cfg.hash_bits - }; + row.set_hash_bits( + row_cfg + .hash_bits + .min(row_hash_bits_for_window(table_window_size)), + ); + } + // Key the primed snapshot on the width the backend ACTUALLY + // applied (`set_hash_bits` clamps the request): recording the + // request — or the 0 default on the unhinted path — keys + // identical table geometries apart and forces needless + // dictionary re-primes. + resolved_table_bits = row.hash_bits(); row.reset(); } MatcherStorage::HashChain(hc) => { diff --git a/zstd/src/encoding/row/mod.rs b/zstd/src/encoding/row/mod.rs index 4ac055a04..0d3724a71 100644 --- a/zstd/src/encoding/row/mod.rs +++ b/zstd/src/encoding/row/mod.rs @@ -777,14 +777,22 @@ impl RowMatchGenerator { }) } + /// Effective row hash width currently configured (`row_hash_log + + /// row_log`). The primed-snapshot key records THIS value — the + /// configured request may exceed the [`ROW_HASH_BITS`] cap below, and + /// keying on the request while the tables use the clamped width forces + /// needless dictionary re-primes. + pub(crate) fn hash_bits(&self) -> usize { + self.row_hash_log + self.row_log + } + pub(crate) fn set_hash_bits(&mut self, bits: usize) { - // Upper bound mirrors the donor `ZSTD_HASHLOG_MAX` (27) rather than - // the old `ROW_HASH_BITS` (20) ceiling: the lazy band's donor - // configs carry hashLog 21-23 (L9-12), and clamping below the - // driver's resolved width would silently shrink the tables while - // the primed-snapshot key still recorded the wider geometry. - const ROW_HASH_BITS_MAX: usize = 27; - let clamped = bits.clamp(self.row_log + 1, ROW_HASH_BITS_MAX); + // Deliberate deviation from donor hashLog 21-23 on L9-12: the + // 20-bit cap keeps the row table L2/L3-resident. Measured on the + // 1 MiB corpus at L10 (tight pair, flat control): the honest + // 21-bit table cost +26.8% wall for a 19-byte output delta — our + // lazy-band ratio already beats the donor with the capped width. + let clamped = bits.clamp(self.row_log + 1, ROW_HASH_BITS); let row_hash_log = clamped.saturating_sub(self.row_log); if self.row_hash_log != row_hash_log { self.row_hash_log = row_hash_log; From 985ca2408f7c27a502bc1d162192781cd20b348a Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Wed, 10 Jun 2026 15:30:18 +0300 Subject: [PATCH 20/22] fix(bench): checksum parity for the FFI dictionary arms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dict-group FFI compressors (bulk::Compressor) never set ZSTD_c_checksumFlag, so they skipped the XXH64 pass the Rust side pays by default (content checksum on) and the no-dict groups' raw-CCtx helper enables under the hash feature. ~6% of frame time on small dict frames was charged to the Rust arm only. configure_ffi_bulk_compressor (nee the LDM-only helper) now sets the flag under the same feature gate; dict-encoded payloads consequently carry a checksum, so the decompress-dict arms verify on BOTH sides too — consistent with the plain decompress groups. --- zstd/benches/compare_ffi.rs | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/zstd/benches/compare_ffi.rs b/zstd/benches/compare_ffi.rs index be02acdc1..79116bf16 100644 --- a/zstd/benches/compare_ffi.rs +++ b/zstd/benches/compare_ffi.rs @@ -38,9 +38,10 @@ use support::{ static BENCHMARK_SCENARIOS: OnceLock> = OnceLock::new(); -/// Enable `ZSTD_c_enableLongDistanceMatching` on an FFI bulk compressor for -/// the LDM variants; a no-op for the plain numeric levels. -fn apply_ffi_ldm(compressor: &mut zstd::bulk::Compressor<'_>, level: &LevelConfig) { +/// Apply the matrix variant's options to an FFI bulk compressor: LDM for +/// the `*_ldm*` variants and the checksum flag (parity with both +/// `ffi_encode_to_vec` and the Rust encoder's default). +fn configure_ffi_bulk_compressor(compressor: &mut zstd::bulk::Compressor<'_>, level: &LevelConfig) { if level.ldm { compressor .set_parameter(zstd::zstd_safe::CParameter::EnableLongDistanceMatching( @@ -48,6 +49,16 @@ fn apply_ffi_ldm(compressor: &mut zstd::bulk::Compressor<'_>, level: &LevelConfi )) .expect("FFI bulk compressor accepts EnableLongDistanceMatching"); } + // Checksum parity with the no-dict groups: `ffi_encode_to_vec` sets + // `ZSTD_c_checksumFlag` under the `hash` feature, but the bulk + // compressors used by the dictionary arms default it OFF — leaving the + // Rust side (content checksum on by default) paying the XXH64 pass the + // FFI side skipped (~6% of frame time on small dict frames). + if cfg!(feature = "hash") { + compressor + .set_parameter(zstd::zstd_safe::CParameter::ChecksumFlag(true)) + .expect("FFI bulk compressor accepts ChecksumFlag"); + } } /// Build the matching Rust-encoder bytes for a matrix variant. The plain @@ -664,16 +675,16 @@ fn bench_dictionary(c: &mut Criterion) { // the plain compress/decompress groups, not the dictionary group. // Everything else runs here: the numeric levels (unchanged // behaviour) and the `*_ldm_dict` variants (`dict = true`), the - // latter with LDM enabled on both sides via `apply_ffi_ldm` / + // latter with LDM enabled on both sides via `configure_ffi_bulk_compressor` / // `ldm_parameters` below. if level.ldm && !level.dict { continue; } let mut no_dict = zstd::bulk::Compressor::new(level.ffi_level).unwrap(); - apply_ffi_ldm(&mut no_dict, &level); + configure_ffi_bulk_compressor(&mut no_dict, &level); let mut with_dict = zstd::bulk::Compressor::with_dictionary(level.ffi_level, &ffi_dictionary).unwrap(); - apply_ffi_ldm(&mut with_dict, &level); + configure_ffi_bulk_compressor(&mut with_dict, &level); let no_dict_bytes = no_dict.compress(&scenario.bytes).unwrap(); let with_dict_bytes = with_dict.compress(&scenario.bytes).unwrap(); @@ -745,7 +756,7 @@ fn bench_dictionary(c: &mut Criterion) { group.bench_function("c_ffi_without_dict", |b| { b.iter(|| { let mut compressor = zstd::bulk::Compressor::new(level.ffi_level).unwrap(); - apply_ffi_ldm(&mut compressor, &level); + configure_ffi_bulk_compressor(&mut compressor, &level); black_box(compressor.compress(&scenario.bytes).unwrap()) }) }); @@ -767,7 +778,7 @@ fn bench_dictionary(c: &mut Criterion) { let mut compressor = zstd::bulk::Compressor::with_dictionary(level.ffi_level, &ffi_dictionary) .unwrap(); - apply_ffi_ldm(&mut compressor, &level); + configure_ffi_bulk_compressor(&mut compressor, &level); b.iter(|| black_box(compressor.compress(&scenario.bytes).unwrap())) }); From d044bc1a2306f1ff3a6f5db15e89dbed73a3ac48 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Wed, 10 Jun 2026 15:44:20 +0300 Subject: [PATCH 21/22] fix(encode)!: default content checksum off (upstream library parity) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Out-of-the-box comparisons run library defaults against library defaults, and the upstream library default is ZSTD_c_checksumFlag = 0. Our encoders defaulted the XXH64 content checksum ON, paying a hashing pass (and 4 trailing bytes) the reference implementation does not — a standing handicap in any external default-vs-default benchmark. - FrameCompressor and StreamingEncoder default content_checksum = false; set_content_checksum docs note the upstream parity - the CLI explicitly enables the checksum (the reference zstd COMMAND defaults it ON, unlike the library API) - c-api and wasm already pinned their own explicit defaults and are unaffected - benches measure the FULL feature gate on both sides: every Rust arm enables the checksum under the same hash feature that turns on ZSTD_c_checksumFlag for the FFI arms (compare_ffi + memory bench) - checksum-intent tests construct their frames with the flag explicit BREAKING CHANGE: frames produced through encoder defaults no longer carry a trailing XXH64 content checksum; call set_content_checksum(true) to restore the previous layout. --- cli/src/main.rs | 6 ++++++ zstd/benches/compare_ffi.rs | 23 +++++++++++++++-------- zstd/benches/compare_ffi_memory.rs | 18 ++++++++++-------- zstd/src/decoding/frame_decoder.rs | 7 +++++++ zstd/src/decoding/streaming_decoder.rs | 3 +++ zstd/src/encoding/frame_compressor.rs | 16 ++++++++++------ zstd/src/encoding/streaming_encoder.rs | 8 ++++++-- 7 files changed, 57 insertions(+), 24 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index e4e7fdda4..3110e538c 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -868,6 +868,12 @@ fn compress_stream( CompressionLevel::from_level(level) }; let mut encoder = structured_zstd::encoding::StreamingEncoder::new(writer, compression_level); + // The reference `zstd` COMMAND defaults the content checksum ON (unlike + // the library API, whose default is off and which our encoder mirrors) — + // set it explicitly so CLI output matches `zstd ` byte layout. + encoder + .set_content_checksum(true) + .wrap_err("failed to enable content checksum")?; // Long-distance matching (`--long`) is a per-knob override applied via the // compression-parameters API; skip it for `--store` (raw frames don't match). if long && !store { diff --git a/zstd/benches/compare_ffi.rs b/zstd/benches/compare_ffi.rs index 79116bf16..d6cbca688 100644 --- a/zstd/benches/compare_ffi.rs +++ b/zstd/benches/compare_ffi.rs @@ -61,16 +61,19 @@ fn configure_ffi_bulk_compressor(compressor: &mut zstd::bulk::Compressor<'_>, le } } -/// Build the matching Rust-encoder bytes for a matrix variant. The plain -/// numeric levels keep the historical `compress_slice_to_vec` path so their -/// output stays byte-for-byte identical to pre-#362 runs; the LDM variants -/// route through `compress_with_parameters` with -/// `enable_long_distance_matching(true)` on the variant's base level. +/// Build the matching Rust-encoder bytes for a matrix variant. The bench +/// matrix measures the FULL feature gate on both sides: the content +/// checksum is enabled explicitly here (the encoder's default mirrors the +/// upstream library default, OFF) just as `ffi_encode_to_vec` and +/// `configure_ffi_bulk_compressor` enable `ZSTD_c_checksumFlag`, all under +/// the same `hash` feature. fn rust_encode_to_vec(input: &[u8], level: &LevelConfig) -> Vec { - match ldm_parameters(level) { - Some(params) => structured_zstd::encoding::compress_with_parameters(input, ¶ms), - None => structured_zstd::encoding::compress_slice_to_vec(input, level.rust_level), + let mut enc: FrameCompressor = FrameCompressor::new(level.rust_level); + if let Some(params) = ldm_parameters(level) { + enc.set_parameters(¶ms); } + enc.set_content_checksum(cfg!(feature = "hash")); + enc.compress_independent_frame(input) } /// FFI encode helper used by criterion's timing loop. Uses @@ -713,6 +716,8 @@ fn bench_dictionary(c: &mut Criterion) { if let Some(params) = ldm_parameters(&level) { warmup_compressor.set_parameters(¶ms); } + // Full feature gate: checksum on, matching the FFI arms. + warmup_compressor.set_content_checksum(cfg!(feature = "hash")); warmup_compressor .set_dictionary_from_bytes(&ffi_dictionary) .expect("dictionary should attach"); @@ -798,6 +803,8 @@ fn bench_dictionary(c: &mut Criterion) { // a `set_source`/`set_drain` call — pin them to the // defaults so inference has a concrete type. let mut compressor: FrameCompressor = FrameCompressor::new(level.rust_level); + // Full feature gate: checksum on, matching the FFI arms. + compressor.set_content_checksum(cfg!(feature = "hash")); // Enable LDM before attaching the dictionary (see the // warmup compressor above for why the order is safe). if let Some(params) = ldm_parameters(&level) { diff --git a/zstd/benches/compare_ffi_memory.rs b/zstd/benches/compare_ffi_memory.rs index 7d1ce94db..cd632ce41 100644 --- a/zstd/benches/compare_ffi_memory.rs +++ b/zstd/benches/compare_ffi_memory.rs @@ -504,6 +504,8 @@ fn main() { let (rust_compressed, rust_peak) = measure_peak(|| { let mut compressor: FrameCompressor = FrameCompressor::new(level.rust_level); + // Full feature gate: checksum on, matching the FFI arm. + compressor.set_content_checksum(cfg!(feature = "hash")); if let Some(params) = &ldm_params { compressor.set_parameters(params); } @@ -556,15 +558,15 @@ fn main() { continue; } - // Compress (no dictionary; LDM wired on both sides when set) - let (rust_compressed, rust_peak) = measure_peak(|| match &ldm_params { - Some(params) => { - structured_zstd::encoding::compress_with_parameters(&scenario.bytes[..], params) + // Compress (no dictionary; LDM wired on both sides when set). + // Full feature gate: checksum on, matching the FFI arm. + let (rust_compressed, rust_peak) = measure_peak(|| { + let mut compressor: FrameCompressor = FrameCompressor::new(level.rust_level); + compressor.set_content_checksum(cfg!(feature = "hash")); + if let Some(params) = &ldm_params { + compressor.set_parameters(params); } - None => structured_zstd::encoding::compress_slice_to_vec( - &scenario.bytes[..], - level.rust_level, - ), + compressor.compress_independent_frame(&scenario.bytes[..]) }); let (ffi_compressed, ffi_peak) = measure_peak(|| ffi_encode(&scenario.bytes[..], level.ffi_level, level.ldm, None)); diff --git a/zstd/src/decoding/frame_decoder.rs b/zstd/src/decoding/frame_decoder.rs index 7ee702222..902bb68e0 100644 --- a/zstd/src/decoding/frame_decoder.rs +++ b/zstd/src/decoding/frame_decoder.rs @@ -3022,6 +3022,7 @@ mod tests { // into the persistent scratch's hasher. let payload: Vec = (0..8192u32).map(|i| (i & 0xFF) as u8).collect(); let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_content_checksum(true); compressor.set_source(payload.as_slice()); let mut compressed = Vec::new(); compressor.set_drain(&mut compressed); @@ -3058,6 +3059,7 @@ mod tests { use crate::decoding::ContentChecksum; let payload: Vec = (0..8192u32).map(|i| (i & 0xFF) as u8).collect(); let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_content_checksum(true); compressor.set_source(payload.as_slice()); let mut compressed = Vec::new(); compressor.set_drain(&mut compressed); @@ -3080,6 +3082,7 @@ mod tests { use crate::decoding::errors::FrameDecoderError; let payload: Vec = (0..8192u32).map(|i| (i & 0xFF) as u8).collect(); let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_content_checksum(true); compressor.set_source(payload.as_slice()); let mut compressed = Vec::new(); compressor.set_drain(&mut compressed); @@ -3114,6 +3117,7 @@ mod tests { use crate::decoding::errors::FrameDecoderError; let payload: Vec = (0..8192u32).map(|i| (i & 0xFF) as u8).collect(); let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_content_checksum(true); compressor.set_source(payload.as_slice()); let mut compressed = Vec::new(); compressor.set_drain(&mut compressed); @@ -3150,6 +3154,7 @@ mod tests { // the buffered tail (it used to early-return Ok((4,0)) and lose it). let payload: Vec = (0..8192u32).map(|i| (i & 0xFF) as u8).collect(); let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_content_checksum(true); compressor.set_source(payload.as_slice()); let mut compressed = Vec::new(); compressor.set_drain(&mut compressed); @@ -3179,6 +3184,7 @@ mod tests { use crate::decoding::ContentChecksum; let payload: Vec = (0..8192u32).map(|i| (i & 0xFF) as u8).collect(); let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_content_checksum(true); compressor.set_source(payload.as_slice()); let mut compressed = Vec::new(); compressor.set_drain(&mut compressed); @@ -3204,6 +3210,7 @@ mod tests { let mut with = Vec::new(); let mut c_with = FrameCompressor::new(CompressionLevel::Default); + c_with.set_content_checksum(true); c_with.set_source(payload.as_slice()); c_with.set_drain(&mut with); c_with.compress(); diff --git a/zstd/src/decoding/streaming_decoder.rs b/zstd/src/decoding/streaming_decoder.rs index 11366cf7f..d608838cc 100644 --- a/zstd/src/decoding/streaming_decoder.rs +++ b/zstd/src/decoding/streaming_decoder.rs @@ -235,6 +235,9 @@ mod tests { let payload: Vec = (0..8192u32).map(|i| (i & 0xFF) as u8).collect(); let mut compressor = FrameCompressor::new(CompressionLevel::Default); + // Checksum is the subject under test; the encoder default is off + // (upstream library parity). + compressor.set_content_checksum(true); compressor.set_source(payload.as_slice()); let mut compressed = Vec::new(); compressor.set_drain(&mut compressed); diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index 1bb3c4932..04f9ff1ff 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -104,9 +104,10 @@ pub struct FrameCompressor< magicless: bool, /// Whether to emit a trailing XXH64 content checksum and set the frame /// header's `Content_Checksum_flag` (semantics of upstream - /// `ZSTD_c_checksumFlag`). Default `true`; combined with the `hash` - /// feature at frame-build time, so without `hash` no checksum is emitted - /// regardless. Set via [`Self::set_content_checksum`]. + /// `ZSTD_c_checksumFlag`). Default `false`, matching the upstream + /// library default; combined with the `hash` feature at frame-build + /// time, so without `hash` no checksum is emitted regardless. Set via + /// [`Self::set_content_checksum`]. content_checksum: bool, #[cfg(feature = "hash")] hasher: XxHash64, @@ -754,7 +755,7 @@ impl FrameCompressor { ), }, magicless: false, - content_checksum: true, + content_checksum: false, #[cfg(feature = "hash")] hasher: XxHash64::with_seed(0), #[cfg(feature = "lsm")] @@ -1090,7 +1091,7 @@ impl FrameCompressor { }, compression_level, magicless: false, - content_checksum: true, + content_checksum: false, #[cfg(feature = "hash")] hasher: XxHash64::with_seed(0), #[cfg(feature = "lsm")] @@ -1116,7 +1117,10 @@ impl FrameCompressor { } /// Enable or disable the trailing XXH64 content checksum - /// (semantics of upstream `ZSTD_c_checksumFlag`). Default `true`. + /// (semantics of upstream `ZSTD_c_checksumFlag`). Default `false`, + /// matching the upstream library default (`ZSTD_c_checksumFlag = 0`) + /// so out-of-the-box frames carry the same layout and pay the same + /// costs as the reference implementation. /// /// When `false`, emitted frames set `Content_Checksum_flag = 0` and carry /// no trailing digest; such frames are valid (RFC 8878) and decode diff --git a/zstd/src/encoding/streaming_encoder.rs b/zstd/src/encoding/streaming_encoder.rs index d61023220..ac8634f36 100644 --- a/zstd/src/encoding/streaming_encoder.rs +++ b/zstd/src/encoding/streaming_encoder.rs @@ -131,7 +131,7 @@ impl StreamingEncoder { bytes_consumed: 0, strategy_override: None, magicless: false, - content_checksum: true, + content_checksum: false, dictionary: None, dictionary_entropy_cache: None, #[cfg(feature = "hash")] @@ -140,7 +140,8 @@ impl StreamingEncoder { } /// Enable or disable the trailing XXH64 content checksum - /// (upstream `ZSTD_c_checksumFlag`). Default `true`. Must be called + /// (upstream `ZSTD_c_checksumFlag`). Default `false`, matching the + /// upstream library default (`ZSTD_c_checksumFlag = 0`). Must be called /// before the first [`write`](Write::write); once the frame header is /// emitted the flag is fixed, so a late change returns an error rather /// than producing a header/trailer mismatch. Without the `hash` feature @@ -1576,6 +1577,9 @@ mod tests { let payload = b"streaming-checksum-trailer-".repeat(64); let mut with = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); + // Explicit: the encoder default is off (upstream library parity). + with.set_content_checksum(true) + .expect("set_content_checksum pre-write"); with.write_all(&payload).unwrap(); let with_checksum = with.finish().unwrap(); From 5bf5a03d4bf87ef2b90100963cab46b3b0b0bcb8 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Wed, 10 Jun 2026 19:05:23 +0300 Subject: [PATCH 22/22] docs(encode): align checksum-default docs and bench config ordering - StreamingEncoder field doc now states the false default (the method doc was updated with the flip, the field doc was missed) - Lazy strategy const comment no longer claims HashChain backing - every bench arm configures params before the checksum flag, matching rust_encode_to_vec (set_parameters does not touch the flag today; identical ordering prevents drift if it ever grows a full reset) --- zstd/benches/compare_ffi.rs | 4 ++-- zstd/benches/compare_ffi_memory.rs | 11 ++++++++--- zstd/src/encoding/strategy.rs | 10 +++++----- zstd/src/encoding/streaming_encoder.rs | 5 +++-- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/zstd/benches/compare_ffi.rs b/zstd/benches/compare_ffi.rs index d6cbca688..048066221 100644 --- a/zstd/benches/compare_ffi.rs +++ b/zstd/benches/compare_ffi.rs @@ -803,13 +803,13 @@ fn bench_dictionary(c: &mut Criterion) { // a `set_source`/`set_drain` call — pin them to the // defaults so inference has a concrete type. let mut compressor: FrameCompressor = FrameCompressor::new(level.rust_level); - // Full feature gate: checksum on, matching the FFI arms. - compressor.set_content_checksum(cfg!(feature = "hash")); // Enable LDM before attaching the dictionary (see the // warmup compressor above for why the order is safe). if let Some(params) = ldm_parameters(&level) { compressor.set_parameters(¶ms); } + // Full feature gate: checksum on, matching the FFI arms. + compressor.set_content_checksum(cfg!(feature = "hash")); compressor .set_encoder_dictionary( EncoderDictionary::from_bytes(&ffi_dictionary) diff --git a/zstd/benches/compare_ffi_memory.rs b/zstd/benches/compare_ffi_memory.rs index cd632ce41..856098ff8 100644 --- a/zstd/benches/compare_ffi_memory.rs +++ b/zstd/benches/compare_ffi_memory.rs @@ -504,11 +504,14 @@ fn main() { let (rust_compressed, rust_peak) = measure_peak(|| { let mut compressor: FrameCompressor = FrameCompressor::new(level.rust_level); - // Full feature gate: checksum on, matching the FFI arm. - compressor.set_content_checksum(cfg!(feature = "hash")); + // Params before the checksum flag — same ordering as + // `rust_encode_to_vec` so every bench arm configures the + // compressor identically. if let Some(params) = &ldm_params { compressor.set_parameters(params); } + // Full feature gate: checksum on, matching the FFI arm. + compressor.set_content_checksum(cfg!(feature = "hash")); compressor .set_dictionary_from_bytes(&dict) .expect("dictionary should attach"); @@ -562,10 +565,12 @@ fn main() { // Full feature gate: checksum on, matching the FFI arm. let (rust_compressed, rust_peak) = measure_peak(|| { let mut compressor: FrameCompressor = FrameCompressor::new(level.rust_level); - compressor.set_content_checksum(cfg!(feature = "hash")); + // Params before the checksum flag — same ordering as + // `rust_encode_to_vec`. if let Some(params) = &ldm_params { compressor.set_parameters(params); } + compressor.set_content_checksum(cfg!(feature = "hash")); compressor.compress_independent_frame(&scenario.bytes[..]) }); let (ffi_compressed, ffi_peak) = diff --git a/zstd/src/encoding/strategy.rs b/zstd/src/encoding/strategy.rs index 8019a66eb..9217ade6b 100644 --- a/zstd/src/encoding/strategy.rs +++ b/zstd/src/encoding/strategy.rs @@ -261,11 +261,11 @@ impl Strategy for Lazy { const USE_HASH3: bool = false; const USE_BT: bool = false; const OPT_LEVEL: u8 = 0; - // Lazy is HashChain-backed but `USE_BT == false`, so the optimal - // parser entry point is unreachable for this strategy. These - // values mirror the donor `lazy2` cost profile (would be the - // right defaults if a future caller did build a profile for the - // lazy/hc path), but with no current reader the same + // Lazy runs on the Row backend with `USE_BT == false`, so the + // optimal parser entry point is unreachable for this strategy. + // These values mirror the donor `lazy2` cost profile (would be + // the right defaults if a future caller did build a profile for + // the lazy path), but with no current reader the same // unreachable-by-design contract from `Fast` applies. const MAX_CHAIN_DEPTH: usize = 8; const SUFFICIENT_MATCH_LEN: usize = 32; diff --git a/zstd/src/encoding/streaming_encoder.rs b/zstd/src/encoding/streaming_encoder.rs index ac8634f36..49c9954e1 100644 --- a/zstd/src/encoding/streaming_encoder.rs +++ b/zstd/src/encoding/streaming_encoder.rs @@ -45,8 +45,9 @@ pub struct StreamingEncoder { magicless: bool, /// Whether to emit a trailing XXH64 content checksum and set the frame /// header's `Content_Checksum_flag` (upstream `ZSTD_c_checksumFlag`). - /// Default `true`; combined with the `hash` feature, so without `hash` - /// no checksum is emitted regardless. See [`Self::set_content_checksum`]. + /// Default `false`, matching the upstream library default; combined with + /// the `hash` feature, so without `hash` no checksum is emitted + /// regardless. See [`Self::set_content_checksum`]. content_checksum: bool, /// Dictionary applied to the frame (donor `ZSTD_CCtx_loadDictionary` on a /// streaming context). `None` = no dictionary. Set before the first write.