diff --git a/zstd/examples/c_decode_loop_z000033.rs b/zstd/examples/c_decode_loop_z000033.rs new file mode 100644 index 000000000..0436fecfb --- /dev/null +++ b/zstd/examples/c_decode_loop_z000033.rs @@ -0,0 +1,68 @@ +//! Standalone C-decoder loop for a clean perf-record profile of the +//! reference decoder, to compare its hot-path breakdown against our +//! `decode_loop_z000033`. Encodes the in-tree z000033 once at the given +//! level via FFI, then decodes it N times through a reused `ZSTD_DCtx` +//! (steady state, no per-iter context alloc). +//! +//! Build: cargo build --profile flamegraph -p structured-zstd \ +//! --example c_decode_loop_z000033 --features dict_builder +//! Run: perf record -F 999 -g --call-graph dwarf,16384 -- \ +//! target/flamegraph/examples/c_decode_loop_z000033 3 20000 + +use std::env; +use std::fs; + +use zstd::zstd_safe::zstd_sys; + +fn main() { + let args: Vec = env::args().collect(); + let level: i32 = args.get(1).and_then(|s| s.parse().ok()).unwrap_or(3); + let iters: u32 = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(20_000); + let corpus = args + .get(3) + .cloned() + .unwrap_or_else(|| "zstd/decodecorpus_files/z000033".to_string()); + let src = fs::read(&corpus).expect("read corpus"); + let n = src.len(); + + let dst_cap = unsafe { zstd_sys::ZSTD_compressBound(n) }; + let mut compressed = vec![0u8; dst_cap]; + let csize = unsafe { + zstd_sys::ZSTD_compress( + compressed.as_mut_ptr() as *mut core::ffi::c_void, + dst_cap, + src.as_ptr() as *const core::ffi::c_void, + n, + level, + ) + }; + assert_eq!( + unsafe { zstd_sys::ZSTD_isError(csize) }, + 0, + "compress failed" + ); + + let dctx = unsafe { zstd_sys::ZSTD_createDCtx() }; + assert!(!dctx.is_null(), "createDCtx failed"); + let mut out = vec![0u8; n]; + let mut total: u64 = 0; + for _ in 0..iters { + let w = unsafe { + zstd_sys::ZSTD_decompressDCtx( + dctx, + out.as_mut_ptr() as *mut core::ffi::c_void, + n, + compressed.as_ptr() as *const core::ffi::c_void, + csize, + ) + }; + assert_eq!(unsafe { zstd_sys::ZSTD_isError(w) }, 0, "decompress failed"); + assert_eq!(w, n, "decompress produced unexpected output size"); + total = total.wrapping_add(out.first().copied().unwrap_or(0) as u64); + } + unsafe { zstd_sys::ZSTD_freeDCtx(dctx) }; + eprintln!( + "c_decode_loop: level={level} iters={iters} csize={csize} out={n} sink={total} ({} blocks-ish)", + csize + ); +} diff --git a/zstd/examples/section_split.rs b/zstd/examples/section_split.rs new file mode 100644 index 000000000..e4ff00d9a --- /dev/null +++ b/zstd/examples/section_split.rs @@ -0,0 +1,223 @@ +//! Diagnostic: split each compressed block into its literals section +//! (Huffman) and sequences section (FSE) byte counts, for the pure-Rust +//! encoder vs the C FFI encoder, on a fixed (corpus, level). When the +//! sequence streams are byte-identical (see compare_ffi_sequences) but the +//! final size differs, this localizes the gap to literals vs sequences. +//! +//! Build: cargo build --release -p structured-zstd --example section_split --features dict_builder +//! Run: ./target/release/examples/section_split [corpus] [level] + +use std::env; +use std::fs; + +use structured_zstd::encoding::{CompressionLevel, compress_slice_to_vec}; +use zstd::zstd_safe::zstd_sys; + +const MAGIC: u32 = 0xFD2F_B528; + +/// Parse the literals-section header at `body[0..]`; return +/// `(lit_section_total_bytes, lit_type)`. lit_type: 0=Raw 1=RLE +/// 2=Compressed 3=Treeless. +fn lit_section_len(body: &[u8]) -> (usize, u8) { + let b0 = body[0] as usize; + let lit_type = (b0 & 0x3) as u8; + let sf = (b0 >> 2) & 0x3; + match lit_type { + 0 | 1 => { + // Raw / RLE: 1/2/3-byte header carrying Regenerated_Size. + let (hdr, regen) = match sf { + 0 | 2 => (1usize, b0 >> 3), + 1 => (2, (b0 >> 4) | ((body[1] as usize) << 4)), + _ => ( + 3, + (b0 >> 4) | ((body[1] as usize) << 4) | ((body[2] as usize) << 12), + ), + }; + // Raw payload = regen bytes; RLE payload = 1 byte. + let payload = if lit_type == 0 { regen } else { 1 }; + (hdr + payload, lit_type) + } + _ => { + // Compressed / Treeless: size fields start at bit 4. + // Fixed-width u64: the 5-byte (sf 3) header packs up to bit 35 + // (`body[4] << 28`), which truncates in a 32-bit `usize`. + let (hdr, compressed) = match sf { + 0 | 1 => { + let v = ((b0 as u64) >> 4) | ((body[1] as u64) << 4) | ((body[2] as u64) << 12); + (3, ((v >> 10) & 0x3FF) as usize) + } + 2 => { + let v = ((b0 as u64) >> 4) + | ((body[1] as u64) << 4) + | ((body[2] as u64) << 12) + | ((body[3] as u64) << 20); + (4, ((v >> 14) & 0x3FFF) as usize) + } + _ => { + let v = ((b0 as u64) >> 4) + | ((body[1] as u64) << 4) + | ((body[2] as u64) << 12) + | ((body[3] as u64) << 20) + | ((body[4] as u64) << 28); + (5, ((v >> 18) & 0x3FFFF) as usize) + } + }; + (hdr + compressed, lit_type) + } + } +} + +/// Skip magic + frame header, returning the offset of the first block. +fn frame_header_len(frame: &[u8]) -> usize { + assert_eq!( + u32::from_le_bytes([frame[0], frame[1], frame[2], frame[3]]), + MAGIC, + "not a zstd frame" + ); + let fhd = frame[4]; + let single_segment = (fhd >> 5) & 1; + let checksum = (fhd >> 2) & 1; + let _ = checksum; + let dict_id_flag = fhd & 0x3; + let fcs_flag = (fhd >> 6) & 0x3; + let mut pos = 5usize; // magic(4) + FHD(1) + if single_segment == 0 { + pos += 1; // Window_Descriptor + } + pos += match dict_id_flag { + 0 => 0, + 1 => 1, + 2 => 2, + _ => 4, + }; + let fcs_bytes = match fcs_flag { + 0 => { + if single_segment == 1 { + 1 + } else { + 0 + } + } + 1 => 2, + 2 => 4, + _ => 8, + }; + pos + fcs_bytes +} + +struct Split { + blocks: usize, + raw_blocks: usize, + rle_blocks: usize, + comp_blocks: usize, + lit_bytes: usize, + seq_bytes: usize, + lit_type_counts: [usize; 4], +} + +fn analyze(frame: &[u8]) -> Split { + let mut pos = frame_header_len(frame); + let mut s = Split { + blocks: 0, + raw_blocks: 0, + rle_blocks: 0, + comp_blocks: 0, + lit_bytes: 0, + seq_bytes: 0, + lit_type_counts: [0; 4], + }; + loop { + let bh = + frame[pos] as u32 | ((frame[pos + 1] as u32) << 8) | ((frame[pos + 2] as u32) << 16); + let last = bh & 1; + let btype = (bh >> 1) & 0x3; + let bsize = (bh >> 3) as usize; + pos += 3; + s.blocks += 1; + match btype { + 0 => { + s.raw_blocks += 1; + pos += bsize; + } + 1 => { + s.rle_blocks += 1; + pos += 1; // physical RLE body is one byte + } + 2 => { + s.comp_blocks += 1; + let body = &frame[pos..pos + bsize]; + let (lit_total, lit_type) = lit_section_len(body); + assert!( + lit_total <= bsize, + "invalid block split: literals exceed block size \ + (lit_total={lit_total}, bsize={bsize})" + ); + s.lit_type_counts[lit_type as usize] += 1; + s.lit_bytes += lit_total; + s.seq_bytes += bsize - lit_total; + pos += bsize; + } + _ => panic!("unexpected block type {btype}"), + } + if last == 1 { + break; + } + } + s +} + +fn print_split(label: &str, total: usize, s: &Split) { + println!( + "{label}: total={total} blocks={} (raw={} rle={} comp={}) lit_section={} seq_section={} lit_types[raw/rle/comp/treeless]={:?}", + s.blocks, + s.raw_blocks, + s.rle_blocks, + s.comp_blocks, + s.lit_bytes, + s.seq_bytes, + s.lit_type_counts + ); +} + +fn main() { + let corpus = env::args() + .nth(1) + .unwrap_or_else(|| "zstd/decodecorpus_files/z000033".to_string()); + let level: i32 = env::args().nth(2).and_then(|s| s.parse().ok()).unwrap_or(1); + let bytes = fs::read(&corpus).expect("read corpus"); + + let rust = compress_slice_to_vec(&bytes, CompressionLevel::Level(level)); + + let cap = unsafe { zstd_sys::ZSTD_compressBound(bytes.len()) }; + let mut cbuf = vec![0u8; cap]; + let rc = unsafe { + zstd_sys::ZSTD_compress( + cbuf.as_mut_ptr() as *mut core::ffi::c_void, + cap, + bytes.as_ptr() as *const core::ffi::c_void, + bytes.len(), + level, + ) + }; + assert_eq!( + unsafe { zstd_sys::ZSTD_isError(rc) }, + 0, + "ZSTD_compress failed" + ); + let ffi = &cbuf[..rc]; + + println!( + "=== section_split corpus={corpus} input={} level={level} ===", + bytes.len() + ); + let rs = analyze(&rust); + let fs_ = analyze(ffi); + print_split("rust", rust.len(), &rs); + print_split("ffi ", ffi.len(), &fs_); + println!( + "DELTA: total={:+} lit_section={:+} seq_section={:+}", + rust.len() as i64 - ffi.len() as i64, + rs.lit_bytes as i64 - fs_.lit_bytes as i64, + rs.seq_bytes as i64 - fs_.seq_bytes as i64, + ); +} diff --git a/zstd/src/decoding/buffer_backend.rs b/zstd/src/decoding/buffer_backend.rs index e76c27806..a844b139d 100644 --- a/zstd/src/decoding/buffer_backend.rs +++ b/zstd/src/decoding/buffer_backend.rs @@ -237,6 +237,43 @@ pub(crate) trait BufferBackend: Sized { ); } + /// Base pointer of the contiguous output region, for the inline + /// match-copy macro `exec_sequence_avx2_inline!` (which expands the + /// AVX2 `ZSTD_execSequence` body textually at the sequence-loop call + /// site so it fuses into the per-tier monolith — `#[target_feature]` + /// functions cannot be `#[inline(always)]`, rust#145574). Only valid + /// when [`Self::SUPPORTS_INLINE_SEQUENCE_EXEC`]; the linear backends + /// (`UserSliceBackend`, `FlatBuf`) override, `RingBuffer` never reaches + /// it (gated, wrap-aware fallback). + /// + /// # Safety + /// Caller must hold the macro's preconditions (inline path gated on + /// `SUPPORTS_INLINE_SEQUENCE_EXEC` + capacity validated by + /// `sequence_output_fits`). + // Only reached from the x86_64 AVX2 macro; dead on other targets + // (i686/aarch64 use the scalar/NEON exec paths), same as + // `exec_sequence_inline_avx2` above. + #[allow(dead_code)] + #[inline(always)] + unsafe fn inline_exec_base_ptr(&mut self) -> *mut u8 { + unreachable!("inline_exec_base_ptr on a backend without inline-sequence support") + } + + /// Commit the post-exec write cursor (grow the live region) after the + /// inline match-copy macro has written `[tail, new_tail)`. + /// `UserSliceBackend` advances its cursor; `FlatBuf` `set_len`s the Vec. + /// Distinct from [`Self::set_tail`], which is a shrink-only rollback + /// primitive (`new_tail <= len`). + /// + /// # Safety + /// `new_tail` bytes `[0, new_tail)` must be initialised (the macro just + /// wrote `[tail, new_tail)`); `new_tail <= capacity`. + #[allow(dead_code)] + #[inline(always)] + unsafe fn inline_exec_commit(&mut self, _new_tail: usize) { + unreachable!("inline_exec_commit on a backend without inline-sequence support") + } + /// Construct an empty backend. Backend-specific sizing is done /// via `with_capacity` constructors on the concrete types (see /// [`super::flat_buf::FlatBuf::with_capacity`]). diff --git a/zstd/src/decoding/exec_sequence_inline.rs b/zstd/src/decoding/exec_sequence_inline.rs index c0085ae34..cace4c684 100644 --- a/zstd/src/decoding/exec_sequence_inline.rs +++ b/zstd/src/decoding/exec_sequence_inline.rs @@ -26,6 +26,151 @@ //! See the [`portable`] module doc for how the inline path is reached //! per target. +/// Textual expansion of the AVX2 `ZSTD_execSequence` body at the call +/// site, fusing the match-copy into a per-tier sequence monolith. A +/// `#[target_feature(avx2)]` function cannot be `#[inline(always)]` +/// (rust#145574), so the [`BufferBackend::exec_sequence_inline_avx2`] +/// trait method stays a real CALL on the hot path; expanding the body via +/// a macro removes that boundary (the reference `decompressSequences_bmi2` +/// is one inlined monolith). Backend access goes through the inlinable +/// accessors `cap` / `tail` / `inline_exec_base_ptr` / `inline_exec_commit`, +/// so the macro stays generic over `B` while only the linear inline +/// backends (`UserSliceBackend`, `FlatBuf`) ever reach it (gated on +/// `SUPPORTS_INLINE_SEQUENCE_EXEC`). 32-byte ymm match-copy for +/// `offset >= 32`; usable from any tier whose enclosing fn carries +/// `target_feature(avx2,bmi2)` (AVX2 and VBMI2). The trait method +/// `exec_sequence_inline_avx2` remains the unit-tested reference spec for +/// this body. Returns `Result<(), ExecuteSequencesError>`. +// +// Gated on `kernel_avx2` (implied by `kernel_vbmi2`) so the macro is absent +// when its only consumers (`seq_decoder_avx2` / `seq_decoder_vbmi2`) are +// compiled out — otherwise the `--no-default-features` build sees an unused +// macro and trips `-D warnings`. +#[cfg(all(target_arch = "x86_64", feature = "kernel_avx2"))] +macro_rules! exec_sequence_avx2_inline { + ($buffer:expr, $lit_src:expr, $lit_length:expr, $offset:expr, $match_length:expr) => {{ + use crate::decoding::buffer_backend::sequence_output_fits; + use crate::decoding::exec_sequence_inline::x86::{ + copy16, overlap_copy8, wildcopy_no_overlap, wildcopy_no_overlap_avx2, + wildcopy_overlap_8byte_stride, + }; + const MAX_WILDCOPY_OVERSHOOT: usize = 31; + let lit_length_v: usize = $lit_length; + let offset_v: usize = $offset; + let match_length_v: usize = $match_length; + let lit_src_v: *const u8 = $lit_src; + let backend = $buffer.buffer_mut(); + let cap = backend.cap(); + let tail = backend.tail(); + match sequence_output_fits( + lit_length_v, + match_length_v, + tail, + cap, + MAX_WILDCOPY_OVERSHOOT, + ) { + Err(e) => Err(e), + Ok(total) => { + // SAFETY: the enclosing fn carries + // `#[target_feature(enable = "...,bmi2,avx2")]`; the inline + // path is gated on `B::SUPPORTS_INLINE_SEQUENCE_EXEC`, so the + // backend is linear and overrides `inline_exec_base_ptr` / + // `inline_exec_commit`. `sequence_output_fits` validated + // `tail + total + overshoot <= cap`. + unsafe { + let base = backend.inline_exec_base_ptr(); + let op_lit = base.add(tail); + copy16(op_lit, lit_src_v); + if lit_length_v > 16 { + wildcopy_no_overlap(op_lit.add(16), lit_src_v.add(16), lit_length_v - 16); + } + let op_match = base.add(tail + lit_length_v); + let match_src = base.cast_const().add(tail + lit_length_v - offset_v); + if offset_v >= 32 { + wildcopy_no_overlap_avx2(op_match, match_src, match_length_v); + } else if offset_v >= 16 { + wildcopy_no_overlap(op_match, match_src, match_length_v); + } else { + let (op2, ip2) = overlap_copy8(op_match, match_src, offset_v); + if match_length_v > 8 { + wildcopy_overlap_8byte_stride(op2, ip2, match_length_v - 8); + } + } + backend.inline_exec_commit(tail + total); + } + Ok(()) + } + } + }}; +} +#[cfg(all(target_arch = "x86_64", feature = "kernel_avx2"))] +pub(crate) use exec_sequence_avx2_inline; + +/// SSE2 twin of [`exec_sequence_avx2_inline`] for the BMI2 tier (which has +/// no AVX2): 16-byte xmm match-copy only (`offset >= 16`), so the WILDCOPY +/// destination overshoot stays 15 bytes (vs 31 for the ymm path). Mirrors +/// the [`BufferBackend::exec_sequence_inline`] trait method body, which +/// remains the unit-tested reference spec. Usable from any fn carrying +/// `target_feature(bmi2)`; baseline SSE2 needs no feature gate on x86_64. +// +// Gated on `kernel_bmi2` so the macro is absent when its only consumer +// (`seq_decoder_bmi2`) is compiled out, keeping `--no-default-features` +// (`-D warnings`) free of an unused-macro error. +#[cfg(all(target_arch = "x86_64", feature = "kernel_bmi2"))] +macro_rules! exec_sequence_sse2_inline { + ($buffer:expr, $lit_src:expr, $lit_length:expr, $offset:expr, $match_length:expr) => {{ + use crate::decoding::buffer_backend::sequence_output_fits; + use crate::decoding::exec_sequence_inline::x86::{ + copy16, overlap_copy8, wildcopy_no_overlap, wildcopy_overlap_8byte_stride, + }; + const MAX_WILDCOPY_OVERSHOOT: usize = 15; + let lit_length_v: usize = $lit_length; + let offset_v: usize = $offset; + let match_length_v: usize = $match_length; + let lit_src_v: *const u8 = $lit_src; + let backend = $buffer.buffer_mut(); + let cap = backend.cap(); + let tail = backend.tail(); + match sequence_output_fits( + lit_length_v, + match_length_v, + tail, + cap, + MAX_WILDCOPY_OVERSHOOT, + ) { + Err(e) => Err(e), + Ok(total) => { + // SAFETY: inline path gated on `B::SUPPORTS_INLINE_SEQUENCE_EXEC` + // (linear backend, overrides the accessors); + // `sequence_output_fits` validated `tail + total + 15 <= cap`. + // All copy primitives are SSE2 baseline (no target_feature). + unsafe { + let base = backend.inline_exec_base_ptr(); + let op_lit = base.add(tail); + copy16(op_lit, lit_src_v); + if lit_length_v > 16 { + wildcopy_no_overlap(op_lit.add(16), lit_src_v.add(16), lit_length_v - 16); + } + let op_match = base.add(tail + lit_length_v); + let match_src = base.cast_const().add(tail + lit_length_v - offset_v); + if offset_v >= 16 { + wildcopy_no_overlap(op_match, match_src, match_length_v); + } else { + let (op2, ip2) = overlap_copy8(op_match, match_src, offset_v); + if match_length_v > 8 { + wildcopy_overlap_8byte_stride(op2, ip2, match_length_v - 8); + } + } + backend.inline_exec_commit(tail + total); + } + Ok(()) + } + } + }}; +} +#[cfg(all(target_arch = "x86_64", feature = "kernel_bmi2"))] +pub(crate) use exec_sequence_sse2_inline; + // x86_64 only: SSE2 is the architectural baseline there (every x86_64 // CPU has SSE2 by definition). 32-bit `x86` is excluded because the // SSE2 intrinsics here are emitted without a `#[target_feature]` diff --git a/zstd/src/decoding/flat_buf.rs b/zstd/src/decoding/flat_buf.rs index e97cc3659..529d6917c 100644 --- a/zstd/src/decoding/flat_buf.rs +++ b/zstd/src/decoding/flat_buf.rs @@ -312,6 +312,21 @@ impl BufferBackend for FlatBuf { Ok(()) } + #[cfg(target_arch = "x86_64")] + #[inline(always)] + unsafe fn inline_exec_base_ptr(&mut self) -> *mut u8 { + self.buf.as_mut_ptr() + } + + #[cfg(target_arch = "x86_64")] + #[inline(always)] + unsafe fn inline_exec_commit(&mut self, new_tail: usize) { + // The macro wrote `[buf.len(), new_tail)`; grow the Vec to expose it. + // SAFETY: new_tail <= capacity (sequence_output_fits validated it) and + // `[0, new_tail)` is now initialised. + unsafe { self.buf.set_len(new_tail) }; + } + fn new() -> Self { Self { buf: Vec::new(), @@ -662,8 +677,10 @@ mod tests { /// offsets across the SSE2/AVX2 threshold boundary /// (offset 20 routes to SSE2 16-byte path, offset 32 to AVX2 /// 32-byte ymm path, offset 64 to deep AVX2 path). - // AVX2 override is x86_64-only; this test calls it directly. - #[cfg(target_arch = "x86_64")] + // AVX2 override is x86_64-only; this test calls it directly. The `std` + // feature gate is required: `is_x86_feature_detected!` is `std`-only, + // unavailable in the crate's `#![no_std]` build. + #[cfg(all(target_arch = "x86_64", feature = "std"))] #[test] fn exec_sequence_inline_avx2_offset_boundary_correctness() { if !std::arch::is_x86_feature_detected!("avx2") { @@ -735,7 +752,8 @@ mod tests { /// when the requested write plus the 31-byte overshoot exceeds the /// remaining headroom. Guards the single-compare bounds check on the AVX2 /// hot path. - #[cfg(target_arch = "x86_64")] + // `std` feature gate required: `is_x86_feature_detected!` is `std`-only. + #[cfg(all(target_arch = "x86_64", feature = "std"))] #[test] fn exec_sequence_inline_avx2_capacity_overflow_returns_err() { if !std::arch::is_x86_feature_detected!("avx2") { diff --git a/zstd/src/decoding/seq_decoder_avx2.rs b/zstd/src/decoding/seq_decoder_avx2.rs index d86bb09f2..c90114f88 100644 --- a/zstd/src/decoding/seq_decoder_avx2.rs +++ b/zstd/src/decoding/seq_decoder_avx2.rs @@ -18,6 +18,7 @@ use super::buffer_backend::BufferBackend; use super::decode_buffer::DecodeBuffer; +use super::exec_sequence_inline::exec_sequence_avx2_inline; use super::scratch::FSEScratch; use super::sequence_section_decoder::{ ADVANCE, ADVANCE_MASK, ExecSeq, SeqStreamSetup, init_sequence_stream, @@ -78,8 +79,8 @@ macro_rules! decode_one_body { }}; } -/// Textual expansion of per-sequence execute. Fast path: -/// `exec_sequence_inline_avx2` (32-byte ymm wildcopy). Cold path: legacy +/// Textual expansion of per-sequence execute. Fast path: the inlined AVX2 +/// match-copy macro [`exec_sequence_avx2_inline`]. Cold path: legacy /// try_push + repeat_lookahead_prefetched. Expands as a statement-block /// returning `Result<(), DecompressBlockError>` so the caller can `?` /// or branch on it as needed. @@ -143,15 +144,15 @@ macro_rules! execute_one_body { if prefix_end_ok { // SAFETY: parent-slice provenance; offset prefix-resident. let lit_src = unsafe { $literals_buffer.as_ptr().add(lit_cur_before) }; - // SAFETY: enclosing fn carries target_feature(bmi2,avx2). - let r = unsafe { - $buffer.buffer_mut().exec_sequence_inline_avx2( - lit_src, - seq_ll_v as usize, - offset, - seq_ml_v as usize, - ) - }; + // Inline the AVX2 exec body at the call site (no trait-method + // call boundary; see `exec_sequence_avx2_inline`). + let r = exec_sequence_avx2_inline!( + $buffer, + lit_src, + seq_ll_v as usize, + offset, + seq_ml_v as usize + ); break 'exec_inner r.map_err(DecompressBlockError::ExecuteSequencesError); } } diff --git a/zstd/src/decoding/seq_decoder_bmi2.rs b/zstd/src/decoding/seq_decoder_bmi2.rs index 1b7bf2f65..1416191c2 100644 --- a/zstd/src/decoding/seq_decoder_bmi2.rs +++ b/zstd/src/decoding/seq_decoder_bmi2.rs @@ -9,6 +9,7 @@ use super::buffer_backend::BufferBackend; use super::decode_buffer::DecodeBuffer; +use super::exec_sequence_inline::exec_sequence_sse2_inline; use super::scratch::FSEScratch; use super::sequence_section_decoder::{ ADVANCE, ADVANCE_MASK, ExecSeq, SeqStreamSetup, init_sequence_stream, @@ -122,15 +123,16 @@ macro_rules! execute_one_body { if prefix_end_ok { // SAFETY: parent-slice provenance; offset prefix-resident. let lit_src = unsafe { $literals_buffer.as_ptr().add(lit_cur_before) }; - // SAFETY: enclosing fn carries target_feature(bmi2). - let r = unsafe { - $buffer.buffer_mut().exec_sequence_inline( - lit_src, - seq_ll_v as usize, - offset, - seq_ml_v as usize, - ) - }; + // Inline the SSE2 exec body at the call site (no trait-method + // call boundary; BMI2 tier has no AVX2, so the 16-byte xmm + // wildcopy variant is used — see `exec_sequence_sse2_inline`). + let r = exec_sequence_sse2_inline!( + $buffer, + lit_src, + seq_ll_v as usize, + offset, + seq_ml_v as usize + ); break 'exec_inner r.map_err(DecompressBlockError::ExecuteSequencesError); } } diff --git a/zstd/src/decoding/seq_decoder_vbmi2.rs b/zstd/src/decoding/seq_decoder_vbmi2.rs index 094c66ab0..d7e89278f 100644 --- a/zstd/src/decoding/seq_decoder_vbmi2.rs +++ b/zstd/src/decoding/seq_decoder_vbmi2.rs @@ -9,6 +9,7 @@ use super::buffer_backend::BufferBackend; use super::decode_buffer::DecodeBuffer; +use super::exec_sequence_inline::exec_sequence_avx2_inline; use super::scratch::FSEScratch; use super::sequence_section_decoder::{ ADVANCE, ADVANCE_MASK, ExecSeq, SeqStreamSetup, init_sequence_stream, @@ -118,15 +119,16 @@ macro_rules! execute_one_body { if prefix_end_ok { // SAFETY: parent-slice provenance; offset prefix-resident. let lit_src = unsafe { $literals_buffer.as_ptr().add(lit_cur_before) }; - // SAFETY: enclosing fn carries full VBMI2 scope. - let r = unsafe { - $buffer.buffer_mut().exec_sequence_inline_avx2( - lit_src, - seq_ll_v as usize, - offset, - seq_ml_v as usize, - ) - }; + // Inline the AVX2 exec body at the call site (no trait-method + // call boundary; VBMI2 always implies AVX2+BMI2 so the ymm + // wildcopy is in scope — see `exec_sequence_avx2_inline`). + let r = exec_sequence_avx2_inline!( + $buffer, + lit_src, + seq_ll_v as usize, + offset, + seq_ml_v as usize + ); break 'exec_inner r.map_err(DecompressBlockError::ExecuteSequencesError); } } diff --git a/zstd/src/decoding/user_slice_buf.rs b/zstd/src/decoding/user_slice_buf.rs index 37d843687..fd055fedc 100644 --- a/zstd/src/decoding/user_slice_buf.rs +++ b/zstd/src/decoding/user_slice_buf.rs @@ -472,6 +472,18 @@ impl<'a> BufferBackend for UserSliceBackend<'a> { Ok(()) } + #[cfg(target_arch = "x86_64")] + #[inline(always)] + unsafe fn inline_exec_base_ptr(&mut self) -> *mut u8 { + self.slice.as_mut_ptr() + } + + #[cfg(target_arch = "x86_64")] + #[inline(always)] + unsafe fn inline_exec_commit(&mut self, new_tail: usize) { + self.tail = new_tail; + } + /// `new()` exists for trait conformance but is not used on the /// direct-decode path — the slice is always provided up-front via /// [`Self::from_slice`]. Returns an empty backend wrapping an @@ -1261,7 +1273,9 @@ mod tests { /// - offset 64 (deep AVX2 path) /// /// against a byte-by-byte reference of the same repeat semantics. - #[cfg(target_arch = "x86_64")] + // `std` feature gates the test: `is_x86_feature_detected!` is `std`-only + // (runtime CPU detection), unavailable in the crate's `#![no_std]` build. + #[cfg(all(target_arch = "x86_64", feature = "std"))] #[test] fn exec_sequence_inline_avx2_offset_boundary_correctness() { if !std::arch::is_x86_feature_detected!("avx2") { diff --git a/zstd/src/encoding/dfast/mod.rs b/zstd/src/encoding/dfast/mod.rs index a9fe554fe..f0705ec0c 100644 --- a/zstd/src/encoding/dfast/mod.rs +++ b/zstd/src/encoding/dfast/mod.rs @@ -139,17 +139,15 @@ impl DfastMatchGenerator { } } - /// Set both hash table sizes. `bits` is the long-hash bit count - /// (donor `cParams.hashLog`); the short hash is derived as - /// `bits - DFAST_SHORT_HASH_BITS_DELTA`, donor-correct for dfast - /// levels. Both clamps stay above `MIN_WINDOW_LOG` so very small - /// windows don't underflow. - pub(crate) fn set_hash_bits(&mut self, bits: usize) { + /// Set both hash table sizes from the per-level [`DfastConfig`]: + /// `long_bits` = donor `cParams.hashLog`, `short_bits` = donor + /// `cParams.chainLog`. Both clamps stay above `MIN_WINDOW_LOG` so very + /// small windows don't underflow. The caller already caps `long_bits` by + /// the source-size window when hinted, so no upper clamp is applied here. + pub(crate) fn set_hash_bits(&mut self, long_bits: usize, short_bits: usize) { let min_bits = MIN_WINDOW_LOG as usize; - let long_clamped = bits.clamp(min_bits, DFAST_HASH_BITS); - let short_clamped = long_clamped - .saturating_sub(DFAST_SHORT_HASH_BITS_DELTA) - .max(min_bits); + let long_clamped = long_bits.max(min_bits); + let short_clamped = short_bits.max(min_bits); if self.long_hash_bits != long_clamped { self.long_hash_bits = long_clamped; self.long_hash = Vec::new(); diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index fddf33df5..a927ecff5 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -772,7 +772,22 @@ impl FrameCompressor { let block_capacity = MAX_BLOCK_SIZE as usize; let mut start = 0usize; while start < input.len() { - let end = (start + block_capacity).min(input.len()); + // Donor `ZSTD_compress_frameChunk`: size each block via the cheap + // fingerprint pre-splitter so a full 128 KiB block is cut at a + // statistical boundary when it pays. `savings = consumed - + // produced` mirrors the donor gate (the first block and + // incompressible input keep the full 128 KiB). The borrowed window + // already spans the whole input, so a smaller block is just a + // narrower `(block_start, block_end)` range into it. + let savings = start as i64 - all_blocks.len() as i64; + let block_len = optimal_block_size( + self.compression_level, + &input[start..], + input.len() - start, + block_capacity, + savings, + ); + let end = (start + block_len).min(input.len()); let block = &input[start..end]; let last_block = end == input.len(); #[cfg(feature = "hash")] @@ -3208,28 +3223,42 @@ mod tests { } /// `level_pre_split` resolves the per-level split knob through the - /// `LevelParams` table, with named presets as pure numeric aliases: - /// greedy (level 5) → 1, btopt/btultra/btultra2 (16..=22) → 4. Fast, - /// dfast and the lazy band stay unsplit (lazy split is deferred until - /// the per-block entropy path reuses tables like the reference). + /// `LevelParams` table, mirroring the donor `splitLevels[]` by strategy + /// (`ZSTD_optimalBlockSize`): fast → 0 (from-borders), dfast → 1, + /// greedy/lazy → 2, lazy2/btlazy2 (Lazy tag at depth 2) → 3, + /// btopt/btultra/btultra2 → 4. `Uncompressed` has no numeric level so it + /// stays `None`. #[test] fn pre_split_level_dispatches_by_compression_level() { use crate::encoding::CompressionLevel; use crate::encoding::match_generator::level_pre_split; assert_eq!(level_pre_split(CompressionLevel::Uncompressed), None); - assert_eq!(level_pre_split(CompressionLevel::Fastest), None); - assert_eq!(level_pre_split(CompressionLevel::Default), None); - // Better is a pure alias for level 7 (lazy): unsplit, same as Level(7). + // Fastest = level 1 (fast) → 0 (from-borders). + assert_eq!(level_pre_split(CompressionLevel::Fastest), Some(0)); + // Default = level 3 (dfast) → 1. + assert_eq!(level_pre_split(CompressionLevel::Default), Some(1)); + // Better is a pure alias for level 7 (lazy): same as Level(7). assert_eq!( level_pre_split(CompressionLevel::Better), level_pre_split(CompressionLevel::Level(7)), ); - assert_eq!(level_pre_split(CompressionLevel::Level(4)), None); - assert_eq!(level_pre_split(CompressionLevel::Level(5)), Some(1)); - assert_eq!(level_pre_split(CompressionLevel::Level(7)), None); - assert_eq!(level_pre_split(CompressionLevel::Level(15)), None); - assert_eq!(level_pre_split(CompressionLevel::Level(16)), Some(4)); - assert_eq!(level_pre_split(CompressionLevel::Level(22)), Some(4)); + // Best is a pure alias for level 11 (lazy): pin it to the numeric + // route so the named path can't drift from the pre-split table. + assert_eq!( + level_pre_split(CompressionLevel::Best), + level_pre_split(CompressionLevel::Level(11)), + ); + assert_eq!(level_pre_split(CompressionLevel::Level(2)), Some(0)); // fast + assert_eq!(level_pre_split(CompressionLevel::Level(4)), Some(1)); // dfast + assert_eq!(level_pre_split(CompressionLevel::Level(5)), Some(2)); // greedy + assert_eq!(level_pre_split(CompressionLevel::Level(7)), Some(2)); // lazy (depth 1) + assert_eq!(level_pre_split(CompressionLevel::Level(8)), Some(3)); // lazy2 lower bound + assert_eq!(level_pre_split(CompressionLevel::Level(11)), Some(3)); // lazy2 (depth 2) + assert_eq!(level_pre_split(CompressionLevel::Level(12)), Some(3)); // lazy2 upper bound + assert_eq!(level_pre_split(CompressionLevel::Level(13)), Some(3)); // btlazy2 lower bound + assert_eq!(level_pre_split(CompressionLevel::Level(15)), Some(3)); // btlazy2 (depth 2) + assert_eq!(level_pre_split(CompressionLevel::Level(16)), Some(4)); // btopt + assert_eq!(level_pre_split(CompressionLevel::Level(22)), Some(4)); // btultra2 } /// End-to-end: a 256 KB payload whose SECOND 128 KB donor block carries @@ -3300,6 +3329,69 @@ mod tests { assert_eq!(decoded, data, "roundtrip must reproduce the input verbatim"); } + /// Outside-diff coverage for the FAST one-shot path. + /// `compress_slice_to_vec` / `compress_independent_frame` on a Fast level + /// routes through `run_borrowed_block_loop` (not the owned loop the test + /// above covers), which must honour `optimal_block_size` and emit a + /// sub-`MAX_BLOCK_SIZE` boundary rather than fixed 128 KiB blocks. A + /// 256 KiB input is two 128 KiB blocks when unsplit; a chunk boundary in + /// the second block yields >= 3 decoded blocks, asserted on the round-trip. + #[test] + fn fast_oneshot_borrowed_split_emits_subblock() { + use crate::encoding::CompressionLevel; + // First 192 KiB: homogeneous zero run (banks the savings the split + // gate needs). The second 128 KiB block flips to a counter sequence + // at its 64 KiB midpoint (the 192 KiB mark) — a fingerprint + // transition the Fast from-borders splitter (split level 0) resolves + // into a sub-block boundary. + let mut data = vec![0u8; 256 * 1024]; + for (i, byte) in data.iter_mut().enumerate() { + if i >= 192 * 1024 { + *byte = (i % 251 + 1) as u8; + } + } + + // Pin the splitter decision for the Fast path directly (mirrors the + // greedy test): the second donor block must resolve to a sub-block + // boundary, so the >= 3 block count below cannot pass vacuously. + let second_block = &data[128 * 1024..]; + assert!( + super::optimal_block_size( + CompressionLevel::Fastest, + second_block, + second_block.len(), + MAX_BLOCK_SIZE as usize, + 100, + ) < MAX_BLOCK_SIZE as usize, + "fixture must resolve to a sub-block split in the second donor block", + ); + + // Drive the borrowed one-shot route explicitly (Fast level -> + // run_borrowed_block_loop via compress_independent_frame). + let mut compressor: FrameCompressor = FrameCompressor::new(CompressionLevel::Fastest); + let frame = compressor.compress_independent_frame(&data); + + let mut decoder = FrameDecoder::new(); + let mut source = frame.as_slice(); + decoder + .reset(&mut source) + .expect("frame header should parse"); + while !decoder.is_finished() { + decoder + .decode_blocks(&mut source, crate::decoding::BlockDecodingStrategy::All) + .expect("decode should succeed"); + } + let mut decoded = Vec::with_capacity(data.len()); + decoder.collect_to_writer(&mut decoded).unwrap(); + assert_eq!(decoded, data, "roundtrip must reproduce the input verbatim"); + assert!( + decoder.blocks_decoded() >= 3, + "fast one-shot borrowed path must split the second donor block \ + (256 KiB unsplit = 2 blocks), got {} blocks", + decoder.blocks_decoded(), + ); + } + /// Regression: `set_compression_level` followed by `compress()` must /// refresh `state.strategy_tag` through the reset-time sync so the /// literal-compression gates (`min_literals_to_compress`, diff --git a/zstd/src/encoding/incompressible.rs b/zstd/src/encoding/incompressible.rs index 1513b75c2..7344336e8 100644 --- a/zstd/src/encoding/incompressible.rs +++ b/zstd/src/encoding/incompressible.rs @@ -1,9 +1,14 @@ -use super::{BETTER_WINDOW_LOG, CompressionLevel}; +use super::CompressionLevel; pub(crate) const RAW_FAST_PATH_MIN_BLOCK_LEN: usize = 512; pub(crate) const RAW_FAST_PATH_MAX_SAMPLE_LEN: usize = 4096; pub(crate) const RAW_FAST_PATH_MIN_SAMPLE_LEN: usize = 32; -const BETTER_WINDOW_SIZE_BYTES: u64 = 1u64 << BETTER_WINDOW_LOG; +/// Window-size ceiling (8 MiB) above which the incompressible raw-fast-path is +/// disabled for `Best` / numeric levels: the largest-window levels (L20-22) +/// do full match-finding even on apparently-incompressible blocks rather than +/// risk emitting raw blocks where a far back-reference might still pay off. +const RAW_FAST_PATH_MAX_WINDOW_LOG: u8 = 23; +const RAW_FAST_PATH_MAX_WINDOW_SIZE_BYTES: u64 = 1u64 << RAW_FAST_PATH_MAX_WINDOW_LOG; // Keep classifier scratch modest for no_std/small-stack targets: 1024 slots // cuts per-call stack for repeat tracking from ~8 KiB to ~4 KiB. @@ -77,8 +82,8 @@ pub(crate) fn compression_level_allows_raw_fast_path( ) -> bool { match level { CompressionLevel::Fastest | CompressionLevel::Default | CompressionLevel::Better => true, - CompressionLevel::Best => window_size <= BETTER_WINDOW_SIZE_BYTES, - CompressionLevel::Level(_) => window_size <= BETTER_WINDOW_SIZE_BYTES, + CompressionLevel::Best => window_size <= RAW_FAST_PATH_MAX_WINDOW_SIZE_BYTES, + CompressionLevel::Level(_) => window_size <= RAW_FAST_PATH_MAX_WINDOW_SIZE_BYTES, CompressionLevel::Uncompressed => false, } } @@ -324,11 +329,11 @@ mod tests { fn best_raw_fast_path_requires_better_sized_window() { assert!(compression_level_allows_raw_fast_path( CompressionLevel::Best, - BETTER_WINDOW_SIZE_BYTES + RAW_FAST_PATH_MAX_WINDOW_SIZE_BYTES )); assert!(!compression_level_allows_raw_fast_path( CompressionLevel::Best, - BETTER_WINDOW_SIZE_BYTES + 1 + RAW_FAST_PATH_MAX_WINDOW_SIZE_BYTES + 1 )); } @@ -336,7 +341,13 @@ mod tests { fn level4_row_raw_fast_path_allowed_with_better_window_reach() { assert!(compression_level_allows_raw_fast_path( CompressionLevel::Level(4), - BETTER_WINDOW_SIZE_BYTES + RAW_FAST_PATH_MAX_WINDOW_SIZE_BYTES + )); + // Over-cap numeric level is rejected, same boundary as `Best`, so the + // two branches can't drift apart. + assert!(!compression_level_allows_raw_fast_path( + CompressionLevel::Level(4), + RAW_FAST_PATH_MAX_WINDOW_SIZE_BYTES + 1 )); } diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs index 10cbccc4d..fbb8abc18 100644 --- a/zstd/src/encoding/match_generator.rs +++ b/zstd/src/encoding/match_generator.rs @@ -9,7 +9,6 @@ use alloc::vec::Vec; // SIMD/CRC intrinsics now live in `crate::encoding::fastpath::*` where they // sit under per-CPU `#[target_feature]` umbrellas; no architecture-specific // intrinsic imports remain in this file. -use super::BETTER_WINDOW_LOG; use super::CompressionLevel; use super::Matcher; use super::Sequence; @@ -129,7 +128,10 @@ use super::match_table::storage::{HC_PRIME3BYTES, HC_PRIME4BYTES}; // macros / configs / tests keep their unqualified names. #[cfg(test)] use super::match_table::storage::HC_EMPTY; -use super::match_table::storage::{HC_CHAIN_LOG, HC_HASH_LOG, HC3_HASH_LOG}; +use super::match_table::storage::HC3_HASH_LOG; +// HC_HASH_LOG / HC_CHAIN_LOG feed the test-only `HC_CONFIG` default. +#[cfg(test)] +use super::match_table::storage::{HC_CHAIN_LOG, HC_HASH_LOG}; // HC3_MAX_OFFSET moved to encoding::bt alongside the hash3 candidate // probe macro that consumes it; the macro references it via the // fully-qualified `$crate::encoding::bt::HC3_MAX_OFFSET` path so this @@ -174,6 +176,9 @@ pub(crate) struct RowConfig { pub(crate) mls: usize, } +// Only used as the default HashChain config when the test-only parse×search +// override pairs a level with a backend its native row doesn't populate. +#[cfg(test)] const HC_CONFIG: HcConfig = HcConfig { hash_log: HC_HASH_LOG, chain_log: HC_CHAIN_LOG, @@ -216,6 +221,9 @@ const BTULTRA2_HC_CONFIG_L22_16K: HcConfig = HcConfig { target_len: 999, }; +// Default Row config: only used by tests and the test-only parse×search +// override (production greedy L5 carries its own `ROW_L5`). +#[cfg(test)] const ROW_CONFIG: RowConfig = RowConfig { hash_bits: ROW_HASH_BITS, row_log: ROW_LOG, @@ -245,6 +253,52 @@ const ROW_L5: RowConfig = RowConfig { mls: ROW_MIN_MATCH_LEN, }; +/// Per-level Double-Fast hash sizing, mirroring the donor `clevels.h` columns +/// (config-driven, not a hardcoded constant): `long_hash_log` = +/// `cParams.hashLog` (the long 8-byte hash table), `short_hash_log` = +/// `cParams.chainLog` (the short hash table dfast repurposes as its +/// secondary index). Only the Dfast backend reads it, so non-dfast level +/// rows carry `dfast: None`. `minMatch` stays the donor-fixed `5` +/// (`DFAST_MIN_MATCH_LEN`, used in const contexts). +#[derive(Copy, Clone, PartialEq, Eq)] +struct DfastConfig { + long_hash_log: u8, + short_hash_log: u8, +} + +// Donor clevels.h default row (srcSize > 256 KB): L3 {hashLog 17, chainLog 16}, +// L4 {hashLog 18, chainLog 18}. +const DFAST_L3: DfastConfig = DfastConfig { + long_hash_log: 17, + short_hash_log: 16, +}; +const DFAST_L4: DfastConfig = DfastConfig { + long_hash_log: 18, + short_hash_log: 18, +}; + +/// Per-level Fast-strategy tuning, only consumed by the `FastKernelMatcher` +/// (Simple backend): `hash_log` = donor `cParams.hashLog`, `mls` = donor +/// `cParams.minMatch` (4..=8), `step_size` = donor `stepSize`. Carried as +/// `LevelParams.fast` (`Some` only on Fast level rows; `None` elsewhere). +#[derive(Copy, Clone, PartialEq, Eq)] +struct FastConfig { + hash_log: u32, + mls: u32, + step_size: usize, +} + +const FAST_L1: FastConfig = FastConfig { + hash_log: 14, + mls: 7, + step_size: 2, +}; +const FAST_L2: FastConfig = FastConfig { + hash_log: 16, + mls: 6, + step_size: 2, +}; + /// Resolved tuning parameters for a compression level. The /// [`StrategyTag`] is the single source of truth for the backend /// family and the compile-time strategy consts; the runtime @@ -260,22 +314,17 @@ struct LevelParams { /// per level so the parse×search matrix can be swept and tuned. search: super::strategy::SearchMethod, window_log: u8, - /// Donor `cParams.hashLog` — only consumed by the Fast strategy - /// backend (`FastKernelMatcher`). Other backends ignore. - fast_hash_log: u32, - /// Donor `cParams.minMatch` (mls) — only consumed by the Fast - /// strategy backend. Range 4..=8 per donor's mml dispatch. - fast_mls: u32, - /// Donor's `stepSize = targetLength + !(targetLength) + 1` - /// (min 2). For Fast strategy, negative levels use - /// `targetLength = -level` (1..7), giving step_size 2..8. - /// L1 / L2 / Uncompressed use targetLength=0 → step_size=2. - /// Drives the kernel's initial `step` for the 4-cursor body's - /// skip schedule. - fast_step_size: usize, lazy_depth: u8, - hc: HcConfig, - row: RowConfig, + /// Per-strategy tuning. Exactly one is `Some` on each level row, matching + /// `strategy_tag`'s backend, so the table self-documents which knobs a + /// level actually consumes (the others are `None`, not dead placeholders): + /// `fast` for the Fast/Simple backend, `dfast` for Double-Fast, `hc` for + /// the HashChain (lazy / btopt / btultra*) backend, `row` for the Row + /// (greedy L5) backend. + fast: Option, + dfast: Option, + hc: Option, + row: Option, } impl LevelParams { @@ -299,28 +348,32 @@ impl LevelParams { } } - /// Cheap fingerprint pre-splitter level (donor `splitLevels[]` by - /// strategy), or `None` to keep the whole 128 KiB block. `Fast`/`Dfast` - /// stay un-split: their match-finding is cheap, so the splitter's - /// per-block fingerprint plus extra per-sub-block entropy builds cost - /// more throughput than the ratio they buy. The btopt/btultra/btultra2 - /// band keeps level 4 (matching the pre-existing high-level splitter). - /// This is the C-like `blockSplitterLevel` knob, regulated per level - /// here rather than scattered across the frame loop. + /// Cheap fingerprint pre-splitter level, the C-like `blockSplitterLevel` + /// knob. Mirrors the donor `splitLevels[]` table indexed by strategy in + /// `ZSTD_optimalBlockSize` (`{0,0,1,2,2,3,3,4,4,4}` over fast..btultra2): + /// fast=0, dfast=1, greedy=2, lazy=2, lazy2=3, btlazy2=3, + /// btopt/btultra/btultra2=4. We collapse the donor `lazy2` and `btlazy2` + /// strategies into the hash-chain `Lazy` tag, distinguished here by + /// `lazy_depth` (the level table runs both at depth 2), so depth 2 routes + /// to split level 3 to match the donor. `split_level == 0` routes to the + /// cheap from-borders heuristic; `1..=4` to byChunks with internal + /// sampling level `split_level - 1`. The `savings >= 3` gate in + /// `optimal_block_size` keeps incompressible data and the first full block + /// whole, so homogeneous frames are not over-split. fn pre_split(&self) -> Option { match self.strategy_tag { - // Fast/Dfast: cheap match-finding, the splitter costs more than - // it buys. Lazy: its ratio already tracks the reference, and - // splitting it regresses on highly-compressible frames until the - // per-block entropy path reuses tables as aggressively as the - // reference (tracked separately); keep whole blocks for now. - super::strategy::StrategyTag::Fast - | super::strategy::StrategyTag::Dfast - | super::strategy::StrategyTag::Lazy => None, - // Greedy is the band whose single-block literal section loses to - // the reference; the cheap fingerprint pre-split fits per-sub- - // block entropy and recovers it. - super::strategy::StrategyTag::Greedy => Some(1), + super::strategy::StrategyTag::Fast => Some(0), + super::strategy::StrategyTag::Dfast => Some(1), + super::strategy::StrategyTag::Greedy => Some(2), + // lazy=2, lazy2/btlazy2=3; both lazy2 and btlazy2 ride the Lazy + // tag at lazy_depth 2 in the level table. + super::strategy::StrategyTag::Lazy => { + if self.lazy_depth >= 2 { + Some(3) + } else { + Some(2) + } + } super::strategy::StrategyTag::BtOpt | super::strategy::StrategyTag::BtUltra | super::strategy::StrategyTag::BtUltra2 => Some(4), @@ -353,9 +406,12 @@ pub(crate) fn source_size_ceil_log(size: u64) -> u8 { /// the primed-snapshot key) and `prime_with_dictionary` (which acts on it). const FAST_ATTACH_DICT_CUTOFF_LOG: u8 = 13; +// Source-size cap for the dfast hash bits when a size hint is present: a tiny +// input needs no larger hash than its window. The donor `cParams.hashLog` / +// `chainLog` (from `DfastConfig`) caps it from above at the call site. fn dfast_hash_bits_for_window(max_window_size: usize) -> usize { let window_log = (usize::BITS - 1 - max_window_size.leading_zeros()) as usize; - window_log.clamp(MIN_WINDOW_LOG as usize, DFAST_HASH_BITS) + window_log.max(MIN_WINDOW_LOG as usize) } fn row_hash_bits_for_window(max_window_size: usize) -> usize { @@ -373,12 +429,14 @@ fn row_hash_bits_for_window(max_window_size: usize) -> usize { /// Index 0 = level 1, index 21 = level 22. #[rustfmt::skip] const LEVEL_TABLE: [LevelParams; 22] = [ - // Lvl Strategy wlog fast_hlog fast_mls fast_step lazy HC config row config - // --- -------------- ---- --------- -------- --------- ---- ------------------------------------------ ---------- - /* 1 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Fast, search: super::strategy::SearchMethod::Fast, window_log: 19, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 0, hc: HC_CONFIG, row: ROW_CONFIG }, - /* 2 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Fast, search: super::strategy::SearchMethod::Fast, window_log: 20, fast_hash_log: 16, fast_mls: 6, fast_step_size: 2, lazy_depth: 0, hc: HC_CONFIG, row: ROW_CONFIG }, - /* 3 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Dfast, search: super::strategy::SearchMethod::DoubleFast, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 1, hc: HC_CONFIG, row: ROW_CONFIG }, - /* 4 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Dfast, search: super::strategy::SearchMethod::DoubleFast, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 1, hc: HC_CONFIG, row: ROW_CONFIG }, + // Exactly one of fast/dfast/hc/row is Some per row, matching the strategy + // backend; the rest are None (not dead placeholders). + // Lvl Strategy wlog lazy per-strategy config + // --- -------------- ---- ---- ------------------- + /* 1 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Fast, search: super::strategy::SearchMethod::Fast, window_log: 19, lazy_depth: 0, fast: Some(FAST_L1), dfast: None, hc: None, row: None }, + /* 2 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Fast, search: super::strategy::SearchMethod::Fast, window_log: 20, lazy_depth: 0, fast: Some(FAST_L2), dfast: None, hc: None, row: None }, + /* 3 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Dfast, search: super::strategy::SearchMethod::DoubleFast, window_log: 21, lazy_depth: 1, fast: None, dfast: Some(DFAST_L3), hc: None, row: None }, + /* 4 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Dfast, search: super::strategy::SearchMethod::DoubleFast, window_log: 21, lazy_depth: 1, fast: None, dfast: Some(DFAST_L4), hc: None, row: None }, // target_len column for L5..=L15 matches donor cParams.targetLength // from clevels.h table[0] (default — srcSize > 256 KB). Donor uses // it as the lazy outer loop's `sufficient_len` (nice-match) threshold. @@ -386,14 +444,14 @@ const LEVEL_TABLE: [LevelParams; 22] = [ // search_depth iterations instead of breaking on the first // long-enough match — the dominant cost in the L5..=L15 speed // regression vs FFI (see lazy_band_target_len_matches_donor_default_table). - /* 5 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Greedy, search: super::strategy::SearchMethod::RowHash, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 0, hc: HcConfig { hash_log: 18, chain_log: 17, search_depth: 4, target_len: 2 }, row: ROW_L5 }, - /* 6 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: BETTER_WINDOW_LOG, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 1, hc: HcConfig { hash_log: 19, chain_log: 18, search_depth: 8, target_len: 4 }, row: ROW_CONFIG }, - /* 7 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: BETTER_WINDOW_LOG, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 1, hc: HcConfig { hash_log: 20, chain_log: 19, search_depth: 16, target_len: 8 }, row: ROW_CONFIG }, - /* 8 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: BETTER_WINDOW_LOG, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 20, chain_log: 19, search_depth: 24, target_len: 16 }, row: ROW_CONFIG }, - /* 9 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: BETTER_WINDOW_LOG, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 21, chain_log: 20, search_depth: 24, target_len: 16 }, row: ROW_CONFIG }, - /*10 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 24, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 21, chain_log: 20, search_depth: 28, target_len: 16 }, row: ROW_CONFIG }, - /*11 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 24, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 21, chain_log: 20, search_depth: 32, target_len: 16 }, row: ROW_CONFIG }, - /*12 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 25, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 22, chain_log: 21, search_depth: 32, target_len: 32 }, row: ROW_CONFIG }, + /* 5 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Greedy, search: super::strategy::SearchMethod::RowHash, window_log: 21, lazy_depth: 0, fast: None, dfast: None, hc: None, row: Some(ROW_L5) }, + /* 6 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 21, lazy_depth: 1, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 19, chain_log: 18, search_depth: 8, target_len: 4 }), row: None }, + /* 7 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 21, lazy_depth: 1, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 20, chain_log: 19, search_depth: 16, target_len: 8 }), row: None }, + /* 8 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 21, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 20, chain_log: 19, search_depth: 16, target_len: 16 }), row: None }, + /* 9 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 21, chain_log: 20, search_depth: 16, target_len: 16 }), row: None }, + /*10 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 22, chain_log: 21, search_depth: 32, target_len: 16 }), row: None }, + /*11 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 22, chain_log: 21, search_depth: 64, target_len: 16 }), row: None }, + /*12 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 23, chain_log: 22, search_depth: 64, target_len: 32 }), row: None }, // L13-15: reference uses btlazy2 (binary-tree finder) with searchLog 4/5/6 // (search_depth 16/32/64) and targetLength 32. We run the hash-chain Lazy // parser here, so we mirror the reference search budget rather than inflate @@ -402,16 +460,16 @@ const LEVEL_TABLE: [LevelParams; 22] = [ // smaller searchLog find longer matches (and re-establish a strict ratio // ladder above L12) is tracked separately; until it lands these levels sit // close to L12 on hash-chain inputs by design. - /*13 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 22, chain_log: 22, search_depth: 16, target_len: 32 }, row: ROW_CONFIG }, - /*14 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 23, chain_log: 22, search_depth: 32, target_len: 32 }, row: ROW_CONFIG }, - /*15 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 23, chain_log: 23, search_depth: 64, target_len: 32 }, row: ROW_CONFIG }, - /*16 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtOpt, search: super::strategy::SearchMethod::BinaryTree, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 22, chain_log: 22, search_depth: 32, target_len: 48 }, row: ROW_CONFIG }, - /*17 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtOpt, search: super::strategy::SearchMethod::BinaryTree, window_log: 23, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 22, chain_log: 23, search_depth: 32, target_len: 64 }, row: ROW_CONFIG }, - /*18 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra, search: super::strategy::SearchMethod::BinaryTree, window_log: 23, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 22, chain_log: 23, search_depth: 64, target_len: 64 }, row: ROW_CONFIG }, - /*19 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log: 23, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 22, chain_log: 24, search_depth: 128, target_len: 256 }, row: ROW_CONFIG }, - /*20 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log: 25, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 23, chain_log: 25, search_depth: 128, target_len: 256 }, row: ROW_CONFIG }, - /*21 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log: 26, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: BTULTRA2_HC_CONFIG, row: ROW_CONFIG }, - /*22 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log: 27, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: BTULTRA2_HC_CONFIG_L22, row: ROW_CONFIG }, + /*13 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 22, chain_log: 22, search_depth: 16, target_len: 32 }), row: None }, + /*14 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 23, chain_log: 22, search_depth: 32, target_len: 32 }), row: None }, + /*15 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::HashChain, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 23, chain_log: 23, search_depth: 64, target_len: 32 }), row: None }, + /*16 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtOpt, search: super::strategy::SearchMethod::BinaryTree, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 22, chain_log: 22, search_depth: 32, target_len: 48 }), row: None }, + /*17 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtOpt, search: super::strategy::SearchMethod::BinaryTree, window_log: 23, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 22, chain_log: 23, search_depth: 32, target_len: 64 }), row: None }, + /*18 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra, search: super::strategy::SearchMethod::BinaryTree, window_log: 23, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 22, chain_log: 23, search_depth: 64, target_len: 64 }), row: None }, + /*19 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log: 23, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 22, chain_log: 24, search_depth: 128, target_len: 256 }), row: None }, + /*20 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log: 25, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 23, chain_log: 25, search_depth: 128, target_len: 256 }), row: None }, + /*21 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log: 26, lazy_depth: 2, fast: None, dfast: None, hc: Some(BTULTRA2_HC_CONFIG), row: None }, + /*22 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log: 27, lazy_depth: 2, fast: None, dfast: None, hc: Some(BTULTRA2_HC_CONFIG_L22), row: None }, ]; /// Smallest window_log the encoder will use regardless of source size. @@ -460,16 +518,24 @@ fn adjust_params_for_source_size(mut params: LevelParams, src_size: u64) -> Leve let table_log = raw_src_log.max(MIN_WINDOW_LOG); let backend = params.backend(); if backend == super::strategy::BackendTag::HashChain { - if (table_log + 2) < params.hc.hash_log as u8 { - params.hc.hash_log = (table_log + 2) as usize; + let hc = params + .hc + .as_mut() + .expect("HashChain level row carries an HcConfig"); + if (table_log + 2) < hc.hash_log as u8 { + hc.hash_log = (table_log + 2) as usize; } - if (table_log + 1) < params.hc.chain_log as u8 { - params.hc.chain_log = (table_log + 1) as usize; + if (table_log + 1) < hc.chain_log as u8 { + hc.chain_log = (table_log + 1) as usize; } } else if backend == super::strategy::BackendTag::Simple { + let fast = params + .fast + .as_mut() + .expect("Fast level row carries a FastConfig"); let fast_cap = (table_log + 1) as u32; - if fast_cap < params.fast_hash_log { - params.fast_hash_log = fast_cap; + if fast_cap < fast.hash_log { + fast.hash_log = fast_cap; } } params @@ -501,12 +567,11 @@ fn level22_btultra2_params_for_source_size(source_size: Option) -> LevelPar strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log, - fast_hash_log: 14, - fast_mls: 7, - fast_step_size: 2, lazy_depth: 2, - hc, - row: ROW_CONFIG, + fast: None, + dfast: None, + hc: Some(hc), + row: None, } } @@ -524,16 +589,18 @@ fn resolve_level_params(level: CompressionLevel, source_size: Option) -> Le // history; advertising a larger window only inflates // decoder-side buffer reservation. Stay at 17 (128 KiB). window_log: 17, - // Beyond-donor: hash_log=14 (vs donor's 13) for 2× fewer - // collisions on structured corpora. - fast_hash_log: 14, - fast_mls: 6, - // Donor's "base for negative" row has targetLength=1, - // which gives step_size = 1 + 0 + 1 = 2. - fast_step_size: 2, lazy_depth: 0, - hc: HC_CONFIG, - row: ROW_CONFIG, + // Beyond-donor: hash_log=14 (vs donor's 13) for 2× fewer + // collisions on structured corpora. Donor's "base for negative" + // row has targetLength=1 → step_size = 1 + 0 + 1 = 2. + fast: Some(FastConfig { + hash_log: 14, + mls: 6, + step_size: 2, + }), + dfast: None, + hc: None, + row: None, }, CompressionLevel::Fastest => { // Only the Fast-specific cParams @@ -543,9 +610,11 @@ fn resolve_level_params(level: CompressionLevel, source_size: Option) -> Le // does real compression on a full window, unlike // Uncompressed which clamps to 17. let mut p = LEVEL_TABLE[0]; - p.fast_hash_log = 14; - p.fast_mls = 6; - p.fast_step_size = 2; + p.fast = Some(FastConfig { + hash_log: 14, + mls: 6, + step_size: 2, + }); p } CompressionLevel::Default => LEVEL_TABLE[2], @@ -583,12 +652,15 @@ fn resolve_level_params(level: CompressionLevel, source_size: Option) -> Le strategy_tag: super::strategy::StrategyTag::Fast, search: super::strategy::SearchMethod::Fast, window_log: 19, - fast_hash_log: 13, - fast_mls: 7, - fast_step_size: step_size, lazy_depth: 0, - hc: HC_CONFIG, - row: ROW_CONFIG, + fast: Some(FastConfig { + hash_log: 13, + mls: 7, + step_size, + }), + dfast: None, + hc: None, + row: None, } } } @@ -1219,6 +1291,25 @@ impl Matcher for MatchGeneratorDriver { if let Some((search, parse)) = self.config_override.take() { params.search = search; params.lazy_depth = parse.lazy_depth(); + // The matrix sweep can pair a level with a backend its native + // row doesn't populate (e.g. greedy L5, which carries only `row`, + // run on HashChain). Synthesize a default config for the + // overridden backend so its `configure` arm has something to read. + use super::strategy::SearchMethod; + match search { + SearchMethod::Fast => { + params.fast.get_or_insert(FAST_L1); + } + SearchMethod::DoubleFast => { + params.dfast.get_or_insert(DFAST_L3); + } + SearchMethod::RowHash => { + params.row.get_or_insert(ROW_CONFIG); + } + SearchMethod::HashChain | SearchMethod::BinaryTree => { + params.hc.get_or_insert(HC_CONFIG); + } + } } let next_backend = params.backend(); let max_window_size = 1usize << params.window_log; @@ -1291,11 +1382,12 @@ impl Matcher for MatchGeneratorDriver { // get donor row-0 (hash_log=13, mls=7); Fastest / // Uncompressed keep (hash_log=14, mls=6). See // resolve_level_params for rationale. + let fast = params.fast.expect("Fast level row carries a FastConfig"); MatcherStorage::Simple(FastKernelMatcher::with_params( params.window_log, - params.fast_hash_log, - params.fast_mls, - params.fast_step_size, + fast.hash_log, + fast.mls, + fast.step_size, )) } super::strategy::BackendTag::Dfast => { @@ -1350,12 +1442,8 @@ impl Matcher for MatchGeneratorDriver { // Per-level Fast cParams threaded from // resolve_level_params (see Simple-backend swap // arm above for the (level → params) mapping). - m.reset( - params.window_log, - params.fast_hash_log, - params.fast_mls, - params.fast_step_size, - ); + let fast = params.fast.expect("Fast level row carries a FastConfig"); + m.reset(params.window_log, fast.hash_log, fast.mls, fast.step_size); } MatcherStorage::Dfast(dfast) => { dfast.max_window_size = max_window_size; @@ -1366,12 +1454,24 @@ impl Matcher for MatchGeneratorDriver { | CompressionLevel::Level(0) | CompressionLevel::Level(3) ); - resolved_table_bits = if hinted { - dfast_hash_bits_for_window(table_window_size) + let dcfg = params + .dfast + .expect("Dfast level row must carry a DfastConfig"); + // Donor `cParams.hashLog`/`chainLog`, capped by the + // source-size window when hinted so tiny inputs don't + // over-allocate. + let long_bits = if hinted { + dfast_hash_bits_for_window(table_window_size).min(dcfg.long_hash_log as usize) } else { - DFAST_HASH_BITS + dcfg.long_hash_log as usize }; - dfast.set_hash_bits(resolved_table_bits); + let short_bits = if hinted { + dfast_hash_bits_for_window(table_window_size).min(dcfg.short_hash_log as usize) + } else { + dcfg.short_hash_log as usize + }; + resolved_table_bits = long_bits; + dfast.set_hash_bits(long_bits, short_bits); // Dfast holds no per-block input Vecs (history owns the // bytes and `add_data` returns each Vec eagerly), so // `reset` takes no `reuse_space` callback. @@ -1380,7 +1480,7 @@ impl Matcher for MatchGeneratorDriver { MatcherStorage::Row(row) => { row.max_window_size = max_window_size; row.lazy_depth = params.lazy_depth; - row.configure(params.row); + row.configure(params.row.expect("Row level row carries a RowConfig")); if hinted { resolved_table_bits = row_hash_bits_for_window(table_window_size); row.set_hash_bits(resolved_table_bits); @@ -1390,7 +1490,11 @@ impl Matcher for MatchGeneratorDriver { MatcherStorage::HashChain(hc) => { hc.table.max_window_size = max_window_size; hc.hc.lazy_depth = params.lazy_depth; - hc.configure(params.hc, strategy_tag, params.window_log); + hc.configure( + params.hc.expect("HashChain level row carries an HcConfig"), + strategy_tag, + params.window_log, + ); let vec_pool = &mut self.vec_pool; hc.reset(|mut data| { data.resize(data.capacity(), 0); @@ -4643,7 +4747,7 @@ fn driver_switches_backends_and_initializes_dfast_via_reset() { driver.reset(CompressionLevel::Default); assert_eq!(driver.active_backend(), super::strategy::BackendTag::Dfast); - assert_eq!(driver.window_size(), (1u64 << 22)); + assert_eq!(driver.window_size(), (1u64 << 21)); let mut first = driver.get_next_space(); first[..12].copy_from_slice(b"abcabcabcabc"); @@ -5326,10 +5430,11 @@ fn level_20_22_map_to_btultra2_strategy() { fn level22_uses_donor_target_length_and_large_input_tables() { let params = resolve_level_params(CompressionLevel::Level(22), None); assert_eq!(params.window_log, 27); - assert_eq!(params.hc.hash_log, 25); - assert_eq!(params.hc.chain_log, 27); - assert_eq!(params.hc.search_depth, 1 << 9); - assert_eq!(params.hc.target_len, 999); + let hc = params.hc.unwrap(); + assert_eq!(hc.hash_log, 25); + assert_eq!(hc.chain_log, 27); + assert_eq!(hc.search_depth, 1 << 9); + assert_eq!(hc.target_len, 999); } #[test] @@ -5352,10 +5457,11 @@ fn bt_levels_16_to_21_pin_clevels_params() { for (level, wlog, hlog, clog, sd, tl) in expected { let p = resolve_level_params(CompressionLevel::Level(level as i32), None); assert_eq!(p.window_log, wlog, "level {level} window_log"); - assert_eq!(p.hc.hash_log, hlog, "level {level} hash_log"); - assert_eq!(p.hc.chain_log, clog, "level {level} chain_log"); - assert_eq!(p.hc.search_depth, sd, "level {level} search_depth"); - assert_eq!(p.hc.target_len, tl, "level {level} target_len"); + let hc = p.hc.unwrap(); + assert_eq!(hc.hash_log, hlog, "level {level} hash_log"); + assert_eq!(hc.chain_log, clog, "level {level} chain_log"); + assert_eq!(hc.search_depth, sd, "level {level} search_depth"); + assert_eq!(hc.target_len, tl, "level {level} target_len"); } } @@ -5363,24 +5469,27 @@ fn bt_levels_16_to_21_pin_clevels_params() { fn level22_source_size_hint_uses_donor_btultra2_tiers() { let p16k = resolve_level_params(CompressionLevel::Level(22), Some(16 * 1024)); assert_eq!(p16k.window_log, 14); - assert_eq!(p16k.hc.hash_log, 15); - assert_eq!(p16k.hc.chain_log, 15); - assert_eq!(p16k.hc.search_depth, 1 << 10); - assert_eq!(p16k.hc.target_len, 999); + let hc16k = p16k.hc.unwrap(); + assert_eq!(hc16k.hash_log, 15); + assert_eq!(hc16k.chain_log, 15); + assert_eq!(hc16k.search_depth, 1 << 10); + assert_eq!(hc16k.target_len, 999); let p128k = resolve_level_params(CompressionLevel::Level(22), Some(128 * 1024)); assert_eq!(p128k.window_log, 17); - assert_eq!(p128k.hc.hash_log, 17); - assert_eq!(p128k.hc.chain_log, 18); - assert_eq!(p128k.hc.search_depth, 1 << 11); - assert_eq!(p128k.hc.target_len, 999); + let hc128k = p128k.hc.unwrap(); + assert_eq!(hc128k.hash_log, 17); + assert_eq!(hc128k.chain_log, 18); + assert_eq!(hc128k.search_depth, 1 << 11); + assert_eq!(hc128k.target_len, 999); let p256k = resolve_level_params(CompressionLevel::Level(22), Some(256 * 1024)); assert_eq!(p256k.window_log, 18); - assert_eq!(p256k.hc.hash_log, 19); - assert_eq!(p256k.hc.chain_log, 19); - assert_eq!(p256k.hc.search_depth, 1 << 13); - assert_eq!(p256k.hc.target_len, 999); + let hc256k = p256k.hc.unwrap(); + assert_eq!(hc256k.hash_log, 19); + assert_eq!(hc256k.chain_log, 19); + assert_eq!(hc256k.search_depth, 1 << 13); + assert_eq!(hc256k.target_len, 999); } #[test] @@ -5391,12 +5500,13 @@ fn level22_small_source_size_hint_matches_donor_cparams() { let donor = unsafe { zstd_sys::ZSTD_getCParams(22, source_size, 0) }; let params = resolve_level_params(CompressionLevel::Level(22), Some(source_size)); + let hc = params.hc.unwrap(); assert_eq!(params.window_log as u32, donor.windowLog); - assert_eq!(params.hc.chain_log as u32, donor.chainLog); - assert_eq!(params.hc.hash_log as u32, donor.hashLog); - assert_eq!(params.hc.search_depth as u32, 1u32 << donor.searchLog); + assert_eq!(hc.chain_log as u32, donor.chainLog); + assert_eq!(hc.hash_log as u32, donor.hashLog); + assert_eq!(hc.search_depth as u32, 1u32 << donor.searchLog); assert_eq!(HC_OPT_MIN_MATCH_LEN as u32, donor.minMatch); - assert_eq!(params.hc.target_len as u32, donor.targetLength); + assert_eq!(hc.target_len as u32, donor.targetLength); } #[test] @@ -6670,9 +6780,9 @@ fn pooled_space_keeps_capacity_when_slice_size_shrinks() { fn driver_best_to_fastest_releases_oversized_hc_tables() { let mut driver = MatchGeneratorDriver::new(32, 2); - // Initialize at Best — allocates large HC tables (2M hash, 1M chain). + // Initialize at Best — allocates large HC tables (4M hash, 2M chain). driver.reset(CompressionLevel::Best); - assert_eq!(driver.window_size(), (1u64 << 24)); + assert_eq!(driver.window_size(), (1u64 << 22)); // Feed data so tables are actually allocated via ensure_tables(). let mut space = driver.get_next_space(); @@ -6706,7 +6816,7 @@ fn driver_better_to_best_resizes_hc_tables() { // Initialize at Better — allocates small HC tables (1M hash, 512K chain). driver.reset(CompressionLevel::Better); - assert_eq!(driver.window_size(), (1u64 << 23)); + assert_eq!(driver.window_size(), (1u64 << 21)); let mut space = driver.get_next_space(); space[..12].copy_from_slice(b"abcabcabcabc"); @@ -6720,7 +6830,7 @@ fn driver_better_to_best_resizes_hc_tables() { // Switch to Best — must resize to larger tables. driver.reset(CompressionLevel::Best); - assert_eq!(driver.window_size(), (1u64 << 24)); + assert_eq!(driver.window_size(), (1u64 << 22)); // Feed data to trigger ensure_tables with new sizes. let mut space = driver.get_next_space(); @@ -9229,7 +9339,7 @@ fn dfast_seed_remaining_hashable_starts_handles_pos_at_block_end() { #[test] fn dfast_ensure_room_for_rebases_above_guard_band() { let mut dfast = DfastMatchGenerator::new(1 << 22); - dfast.set_hash_bits(10); + dfast.set_hash_bits(10, 10); dfast.ensure_hash_tables(); // Seed an early insert near the current base in BOTH tables. @@ -9411,10 +9521,12 @@ fn fastest_hint_iteration_23_sequences_reconstruct_source() { fn fast_levels_dispatch_per_level_hash_log_and_mls() { // Level 1 — donor `{ 19, 13, 14, 1, 7, 0, ZSTD_fast }` row: // window_log=19, hash_log=14, mls=7. - let p1 = resolve_level_params(CompressionLevel::Level(1), None); - assert_eq!(p1.fast_hash_log, 14); - assert_eq!(p1.fast_mls, 7); - assert_eq!(p1.fast_step_size, 2); + let f1 = resolve_level_params(CompressionLevel::Level(1), None) + .fast + .unwrap(); + assert_eq!(f1.hash_log, 14); + assert_eq!(f1.mls, 7); + assert_eq!(f1.step_size, 2); // Negative levels — donor row-0 ("base for negative"): // hash_log=13, mls=7. The 32 KiB table is L1d-resident (every @@ -9424,35 +9536,29 @@ fn fast_levels_dispatch_per_level_hash_log_and_mls() { // step_size follows donor's formula: targetLength = -level, // step_size = (-level) + 1, giving 2..8 for L-1..L-7. for n in -7..=-1 { - let p = resolve_level_params(CompressionLevel::Level(n), None); - assert_eq!(p.fast_hash_log, 13, "Level({n}) fast_hash_log"); - assert_eq!(p.fast_mls, 7, "Level({n}) fast_mls"); + let f = resolve_level_params(CompressionLevel::Level(n), None) + .fast + .unwrap(); + assert_eq!(f.hash_log, 13, "Level({n}) fast_hash_log"); + assert_eq!(f.mls, 7, "Level({n}) fast_mls"); let expected_step = ((-n) as usize) + 1; - assert_eq!(p.fast_step_size, expected_step, "Level({n}) fast_step_size"); + assert_eq!(f.step_size, expected_step, "Level({n}) fast_step_size"); } // Fastest + Uncompressed keep hash_log=14 / mls=6 (their own // tuning; not part of the negative-level donor ladder). let pf = resolve_level_params(CompressionLevel::Fastest, None); + let ff = pf.fast.unwrap(); assert_eq!( - ( - pf.window_log, - pf.fast_hash_log, - pf.fast_mls, - pf.fast_step_size - ), + (pf.window_log, ff.hash_log, ff.mls, ff.step_size), (19, 14, 6, 2), ); // Uncompressed keeps window_log=17 (no history references, smaller // decoder reservation); fast cParams same as negative-base row. let pu = resolve_level_params(CompressionLevel::Uncompressed, None); + let fu = pu.fast.unwrap(); assert_eq!( - ( - pu.window_log, - pu.fast_hash_log, - pu.fast_mls, - pu.fast_step_size - ), + (pu.window_log, fu.hash_log, fu.mls, fu.step_size), (17, 14, 6, 2), ); } @@ -9507,20 +9613,21 @@ fn fast_levels_driver_wiring_threads_cparams_into_inner_matcher() { // by FrameCompressor / StreamingEncoder). crate::encoding::Matcher::reset(&mut driver, level); + let f = p.fast.unwrap(); let m = driver.simple_mut(); assert_eq!( m.hash_log(), - p.fast_hash_log, + f.hash_log, "{level:?}: inner matcher hash_log mismatch — argument swap?", ); assert_eq!( m.mls(), - p.fast_mls, + f.mls, "{level:?}: inner matcher mls mismatch — argument swap?", ); assert_eq!( m.step_size(), - p.fast_step_size, + f.step_size, "{level:?}: inner matcher step_size mismatch — stale value carried from prior reset?", ); } @@ -9547,10 +9654,17 @@ fn lazy_band_target_len_matches_donor_default_table() { // call with any (level, srcSize, dictSize) combination. let reference = unsafe { zstd_sys::ZSTD_getCParams(level, 0, 0) }; let params = resolve_level_params(CompressionLevel::Level(level), None); + // L5 = greedy (Row backend → `row`); L6-15 = lazy (HashChain → `hc`). + // Both surface the donor `targetLength` as their nice-match threshold. + let target_len = params + .hc + .map(|hc| hc.target_len) + .or_else(|| params.row.map(|row| row.target_len)) + .expect("lazy/greedy level carries hc or row config"); assert_eq!( - params.hc.target_len as u32, reference.targetLength, - "L{level}: hc.target_len ({}) must match reference cParams.targetLength ({})", - params.hc.target_len, reference.targetLength + target_len as u32, reference.targetLength, + "L{level}: target_len ({target_len}) must match reference cParams.targetLength ({})", + reference.targetLength ); } } @@ -9572,11 +9686,12 @@ fn upper_lazy_band_params_match_donor_default_table() { // call with any (level, srcSize, dictSize) combination. let reference = unsafe { zstd_sys::ZSTD_getCParams(level, 0, 0) }; let params = resolve_level_params(CompressionLevel::Level(level), None); + let hc = params.hc.unwrap(); assert_eq!( - params.hc.search_depth as u32, + hc.search_depth as u32, 1u32 << reference.searchLog, "L{level}: hc.search_depth ({}) must equal 1< Vec { roundtrip_at_level(data, CompressionLevel::Default) } -fn roundtrip_better(data: &[u8]) -> Vec { - roundtrip_at_level(data, CompressionLevel::Better) -} - fn roundtrip_better_streaming(data: &[u8]) -> Vec { roundtrip_streaming_at_level(data, CompressionLevel::Better) } -fn roundtrip_best(data: &[u8]) -> Vec { - roundtrip_at_level(data, CompressionLevel::Best) -} - fn roundtrip_best_streaming(data: &[u8]) -> Vec { roundtrip_streaming_at_level(data, CompressionLevel::Best) } @@ -464,36 +456,6 @@ fn better_level_compresses_close_to_default() { ); } -/// Exercise the 8 MiB window: place a repeated pattern beyond Default's -/// 4 MiB window so only Better (8 MiB) can match it. -#[test] -fn roundtrip_better_level_large_window() { - // Two identical 256 KiB regions separated by a 4.5 MiB compressible gap. - // The gap uses a different seed so it doesn't share patterns with the - // regions, but being compressible means hash chains aren't fully - // destroyed by random noise. Better's 8 MiB window can still reach the - // first region; Default's 4 MiB window cannot. - let region = generate_compressible(42, 256 * 1024); - let gap = generate_compressible(9999, 4 * 1024 * 1024 + 512 * 1024); - let mut data = Vec::with_capacity(region.len() + gap.len() + region.len()); - data.extend_from_slice(®ion); - data.extend_from_slice(&gap); - data.extend_from_slice(®ion); - - assert_eq!(roundtrip_better(&data), data); - - // Better should compress the duplicated region; Default cannot reach it. - let compressed_better = compress_to_vec(&data[..], CompressionLevel::Better); - let compressed_default = compress_to_vec(&data[..], CompressionLevel::Default); - assert!( - compressed_better.len() < compressed_default.len(), - "Better (8 MiB window) should beat Default (4 MiB) across 4.5 MiB gap. \ - better={} default={}", - compressed_better.len(), - compressed_default.len(), - ); -} - /// Best must not regress vs Better on this repetitive fixture. Equal /// output is expected here (HC finds identical matches at any depth); /// the strict Best < Better check lives in cross_validation.rs on the @@ -511,36 +473,6 @@ fn best_level_does_not_regress_vs_better() { ); } -/// Exercise the 16 MiB window: place a repeated pattern beyond Better's -/// 8 MiB window so only Best (16 MiB) can match it. -#[test] -fn roundtrip_best_level_large_window() { - // Two identical 256 KiB high-entropy regions separated by a 9 MiB - // compressible gap. The region is random so the only way to compress - // the second copy is via long-distance matching (window reach). - // Best's 16 MiB window can still reach the first region; - // Better's 8 MiB window cannot. - let region = generate_data(42, 256 * 1024); - let gap = generate_compressible(7777, 9 * 1024 * 1024); - let mut data = Vec::with_capacity(region.len() + gap.len() + region.len()); - data.extend_from_slice(®ion); - data.extend_from_slice(&gap); - data.extend_from_slice(®ion); - - assert_eq!(roundtrip_best(&data), data); - - // Best should compress the duplicated region; Better cannot reach it. - let compressed_best = compress_to_vec(&data[..], CompressionLevel::Best); - let compressed_better = compress_to_vec(&data[..], CompressionLevel::Better); - assert!( - compressed_best.len() < compressed_better.len(), - "Best (16 MiB window) should beat Better (8 MiB) across 9 MiB gap. \ - best={} better={}", - compressed_best.len(), - compressed_better.len(), - ); -} - /// Best level streaming should produce identical decompressed output. #[test] fn roundtrip_best_level_streaming_multi_block() {