diff --git a/cli/src/main.rs b/cli/src/main.rs index 3110e538c..29bad4bba 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -1111,223 +1111,4 @@ fn add_extension>(path: &Path, extension: P) -> PathBuf { } #[cfg(test)] -mod tests { - use super::*; - - fn parse(args: &[&str]) -> color_eyre::Result { - let owned: Vec = args.iter().map(|s| s.to_string()).collect(); - match parse_args(&owned, Mode::Compress, false)? { - Parsed::Run(opts) => Ok(opts), - Parsed::Handled => bail!("parse handled (help/version) unexpectedly"), - } - } - - #[test] - fn extension_added() { - assert_eq!( - add_extension(Path::new("README.md"), ".zst"), - PathBuf::from("README.md.zst") - ); - } - - #[test] - fn list_file_walks_multi_frame_archive_by_seeking() { - use structured_zstd::encoding::{CompressionLevel, compress_slice_to_vec}; - // Two concatenated frames: the seek-based frame walk must land exactly on - // the second frame's start, or parsing it would fail. A wrong frame - // length (the old fs::read path computed it differently) would surface as - // an error here. - let mut archive = compress_slice_to_vec(&[7u8; 4096], CompressionLevel::Default); - archive.extend_from_slice(&compress_slice_to_vec( - b"second frame payload, distinct content", - CompressionLevel::Default, - )); - - let dir = std::env::temp_dir(); - let path = dir.join(format!("szstd-list-test-{}.zst", std::process::id())); - fs::write(&path, &archive).unwrap(); - let result = list_file(&path); - let _ = fs::remove_file(&path); - result.expect("list_file must walk both frames without error"); - } - - #[test] - fn argv0_unzstd_defaults_to_decompress() { - assert_eq!(program_mode("unzstd"), (Mode::Decompress, false)); - assert_eq!(program_mode("/usr/bin/unzstd"), (Mode::Decompress, false)); - } - - #[test] - fn argv0_zstdcat_decompresses_to_stdout() { - assert_eq!(program_mode("zstdcat"), (Mode::Decompress, true)); - assert_eq!(program_mode("zstd"), (Mode::Compress, false)); - } - - #[test] - fn bare_numeric_flag_is_a_level() { - let opts = parse(&["-19", "in.txt"]).unwrap(); - assert_eq!(opts.level, 19); - assert_eq!(opts.mode, Mode::Compress); - assert_eq!(opts.inputs, vec!["in.txt".to_string()]); - } - - #[test] - fn levels_above_19_require_ultra() { - assert!(parse(&["-22", "in.txt"]).is_err()); - let opts = parse(&["--ultra", "-22", "in.txt"]).unwrap(); - assert_eq!(opts.level, 22); - } - - #[test] - fn fast_flag_maps_to_negative_level() { - assert_eq!(parse(&["--fast"]).unwrap().level, -1); - assert_eq!(parse(&["--fast=5"]).unwrap().level, -5); - } - - #[test] - fn clustered_short_flags() { - // -d (decompress) + -c (stdout) + -k (keep) in one token. - let opts = parse(&["-dck", "a.zst"]).unwrap(); - assert_eq!(opts.mode, Mode::Decompress); - assert!(opts.to_stdout); - assert!(opts.keep); - } - - #[test] - fn dict_and_output_take_values() { - let opts = parse(&["-D", "dict.bin", "-o", "out.zst", "in.txt"]).unwrap(); - assert_eq!(opts.dict, Some(PathBuf::from("dict.bin"))); - assert_eq!(opts.output, Some(PathBuf::from("out.zst"))); - // Attached value form: -Ddict.bin - let opts = parse(&["-Ddict.bin", "in.txt"]).unwrap(); - assert_eq!(opts.dict, Some(PathBuf::from("dict.bin"))); - } - - #[test] - fn output_rejects_multiple_inputs() { - assert!(parse(&["-o", "out.zst", "a.txt", "b.txt"]).is_err()); - } - - #[test] - fn train_flags_parse_and_allow_many_samples_with_output() { - let opts = parse(&[ - "--train", - "--maxdict=4096", - "--dictID=42", - "-o", - "dict.bin", - "s1.txt", - "s2.txt", - "s3.txt", - ]) - .unwrap(); - assert_eq!(opts.mode, Mode::Train); - assert_eq!(opts.max_dict, 4096); - assert_eq!(opts.dict_id, Some(42)); - assert_eq!(opts.output, Some(PathBuf::from("dict.bin"))); - // --train legitimately fans many samples into one -o dictionary. - assert_eq!(opts.inputs.len(), 3); - } - - #[test] - fn list_mode_parses() { - assert_eq!(parse(&["-l", "a.zst"]).unwrap().mode, Mode::List); - assert_eq!(parse(&["--list", "a.zst"]).unwrap().mode, Mode::List); - } - - #[test] - fn long_flag_enables_ldm() { - assert!(parse(&["-19", "--long", "in.txt"]).unwrap().long); - assert!(parse(&["--long=27", "in.txt"]).unwrap().long); - assert!(!parse(&["-19", "in.txt"]).unwrap().long); - } - - #[test] - fn benchmark_flags_parse_level_range() { - let opts = parse(&["-b3", "-e7", "in.txt"]).unwrap(); - assert!(opts.bench); - assert_eq!(opts.bench_start, 3); - assert_eq!(opts.bench_end, 7); - // Bare `-b` benchmarks the default level (single-level range). - let opts = parse(&["-b", "in.txt"]).unwrap(); - assert!(opts.bench); - assert_eq!(opts.bench_end, opts.bench_start); - } - - #[test] - fn dash_is_a_stdin_input() { - let opts = parse(&["-d", "-"]).unwrap(); - assert_eq!(opts.inputs, vec!["-".to_string()]); - } - - #[test] - fn double_dash_forces_positional() { - let opts = parse(&["--", "-weird-name.txt"]).unwrap(); - assert_eq!(opts.inputs, vec!["-weird-name.txt".to_string()]); - } - - #[test] - fn unknown_flag_errors() { - assert!(parse(&["--definitely-not-a-flag"]).is_err()); - assert!(parse(&["-Z"]).is_err()); - } - - #[test] - fn fast_and_long_match_exactly_not_by_prefix() { - // Exact options succeed. - assert_eq!(parse(&["--fast"]).unwrap().level, -1); - assert_eq!(parse(&["--fast=5"]).unwrap().level, -5); - assert!(parse(&["--long"]).unwrap().long); - // Typos must NOT be silently accepted as `--fast`/`--long`; they fall - // through to the unknown-option path. - assert!(parse(&["--faster"]).is_err()); - assert!(parse(&["--longer"]).is_err()); - // Invalid payloads are rejected, not silently reinterpreted: - // `--fast=-5` must not flip into a positive level, and `--long=` / - // `--long=abc` must not be accepted as a no-op. - assert!(parse(&["--fast=-5"]).is_err()); - assert!(parse(&["--long="]).is_err()); - assert!(parse(&["--long=abc"]).is_err()); - assert!(parse(&["--long=27"]).unwrap().long); - } - - #[test] - fn unsupported_format_flags_are_rejected_not_ignored() { - // These change the wire format but are not wired through yet — accepting - // them silently would hand the caller the wrong frame layout. - assert!(parse(&["--no-check"]).is_err()); - assert!(parse(&["--no-content-size"]).is_err()); - assert!(parse(&["--no-dictID"]).is_err()); - // Verbosity aliases stay honest no-ops. - assert!(parse(&["--quiet"]).is_ok()); - assert!(parse(&["--verbose"]).is_ok()); - } - - #[test] - fn decompress_suffix_stripping() { - let opts = Options { - mode: Mode::Decompress, - level: 3, - store: false, - dict: None, - to_stdout: false, - output: None, - force: false, - keep: false, - remove_source: false, - inputs: vec!["archive.tar.zst".to_string()], - max_dict: DEFAULT_MAX_DICT, - dict_id: None, - bench: false, - bench_start: 3, - bench_end: 0, - bench_secs: 1.0, - long: false, - }; - assert_eq!( - derive_output_path(&opts, Path::new("archive.tar.zst")).unwrap(), - PathBuf::from("archive.tar") - ); - assert!(derive_output_path(&opts, Path::new("noext")).is_err()); - } -} +mod tests; diff --git a/cli/src/progress.rs b/cli/src/progress.rs index 7830b27df..c04f5e7de 100644 --- a/cli/src/progress.rs +++ b/cli/src/progress.rs @@ -118,34 +118,4 @@ fn fmt_duration(duration: Duration) -> String { } #[cfg(test)] -mod tests { - use std::time::Duration; - - use super::{fmt_duration, fmt_size}; - - #[test] - fn human_readable_filesize() { - // Bytes - assert_eq!(&fmt_size(100.0), "100B"); - // Kibibytes - assert_eq!(&fmt_size(12.0 * 2.0_f64.powi(10)), "12.00KiB"); - // Mebibytes - assert_eq!(&fmt_size(7.0 * 2.0_f64.powi(20)), "7.00MiB"); - // Gibibytes - assert_eq!(&fmt_size(123.0 * 2.0_f64.powi(30)), "123.00GiB"); - } - - #[test] - fn human_readable_duration() { - assert_eq!(&fmt_duration(Duration::from_millis(7)), "7.00ms"); - assert_eq!(&fmt_duration(Duration::from_millis(1500)), "1.50s"); - assert_eq!(&fmt_duration(Duration::from_secs(30)), "30.0s"); - assert_eq!(&fmt_duration(Duration::from_secs(90)), "1m 30s"); - assert_eq!(&fmt_duration(Duration::from_secs(5 * 60)), "5m"); - assert_eq!(&fmt_duration(Duration::from_secs(3 * 60 * 60)), "3h"); - assert_eq!( - &fmt_duration(Duration::from_secs(60 * 60 + 20 * 60 + 30)), - "1h 20m 30s" - ); - } -} +mod tests; diff --git a/cli/src/progress/tests.rs b/cli/src/progress/tests.rs new file mode 100644 index 000000000..a4bdd730c --- /dev/null +++ b/cli/src/progress/tests.rs @@ -0,0 +1,29 @@ +use std::time::Duration; + +use super::{fmt_duration, fmt_size}; + +#[test] +fn human_readable_filesize() { + // Bytes + assert_eq!(&fmt_size(100.0), "100B"); + // Kibibytes + assert_eq!(&fmt_size(12.0 * 2.0_f64.powi(10)), "12.00KiB"); + // Mebibytes + assert_eq!(&fmt_size(7.0 * 2.0_f64.powi(20)), "7.00MiB"); + // Gibibytes + assert_eq!(&fmt_size(123.0 * 2.0_f64.powi(30)), "123.00GiB"); +} + +#[test] +fn human_readable_duration() { + assert_eq!(&fmt_duration(Duration::from_millis(7)), "7.00ms"); + assert_eq!(&fmt_duration(Duration::from_millis(1500)), "1.50s"); + assert_eq!(&fmt_duration(Duration::from_secs(30)), "30.0s"); + assert_eq!(&fmt_duration(Duration::from_secs(90)), "1m 30s"); + assert_eq!(&fmt_duration(Duration::from_secs(5 * 60)), "5m"); + assert_eq!(&fmt_duration(Duration::from_secs(3 * 60 * 60)), "3h"); + assert_eq!( + &fmt_duration(Duration::from_secs(60 * 60 + 20 * 60 + 30)), + "1h 20m 30s" + ); +} diff --git a/cli/src/tests.rs b/cli/src/tests.rs new file mode 100644 index 000000000..76dbde192 --- /dev/null +++ b/cli/src/tests.rs @@ -0,0 +1,218 @@ +use super::*; + +fn parse(args: &[&str]) -> color_eyre::Result { + let owned: Vec = args.iter().map(|s| s.to_string()).collect(); + match parse_args(&owned, Mode::Compress, false)? { + Parsed::Run(opts) => Ok(opts), + Parsed::Handled => bail!("parse handled (help/version) unexpectedly"), + } +} + +#[test] +fn extension_added() { + assert_eq!( + add_extension(Path::new("README.md"), ".zst"), + PathBuf::from("README.md.zst") + ); +} + +#[test] +fn list_file_walks_multi_frame_archive_by_seeking() { + use structured_zstd::encoding::{CompressionLevel, compress_slice_to_vec}; + // Two concatenated frames: the seek-based frame walk must land exactly on + // the second frame's start, or parsing it would fail. A wrong frame + // length (the old fs::read path computed it differently) would surface as + // an error here. + let mut archive = compress_slice_to_vec(&[7u8; 4096], CompressionLevel::Default); + archive.extend_from_slice(&compress_slice_to_vec( + b"second frame payload, distinct content", + CompressionLevel::Default, + )); + + let dir = std::env::temp_dir(); + let path = dir.join(format!("szstd-list-test-{}.zst", std::process::id())); + fs::write(&path, &archive).unwrap(); + let result = list_file(&path); + let _ = fs::remove_file(&path); + result.expect("list_file must walk both frames without error"); +} + +#[test] +fn argv0_unzstd_defaults_to_decompress() { + assert_eq!(program_mode("unzstd"), (Mode::Decompress, false)); + assert_eq!(program_mode("/usr/bin/unzstd"), (Mode::Decompress, false)); +} + +#[test] +fn argv0_zstdcat_decompresses_to_stdout() { + assert_eq!(program_mode("zstdcat"), (Mode::Decompress, true)); + assert_eq!(program_mode("zstd"), (Mode::Compress, false)); +} + +#[test] +fn bare_numeric_flag_is_a_level() { + let opts = parse(&["-19", "in.txt"]).unwrap(); + assert_eq!(opts.level, 19); + assert_eq!(opts.mode, Mode::Compress); + assert_eq!(opts.inputs, vec!["in.txt".to_string()]); +} + +#[test] +fn levels_above_19_require_ultra() { + assert!(parse(&["-22", "in.txt"]).is_err()); + let opts = parse(&["--ultra", "-22", "in.txt"]).unwrap(); + assert_eq!(opts.level, 22); +} + +#[test] +fn fast_flag_maps_to_negative_level() { + assert_eq!(parse(&["--fast"]).unwrap().level, -1); + assert_eq!(parse(&["--fast=5"]).unwrap().level, -5); +} + +#[test] +fn clustered_short_flags() { + // -d (decompress) + -c (stdout) + -k (keep) in one token. + let opts = parse(&["-dck", "a.zst"]).unwrap(); + assert_eq!(opts.mode, Mode::Decompress); + assert!(opts.to_stdout); + assert!(opts.keep); +} + +#[test] +fn dict_and_output_take_values() { + let opts = parse(&["-D", "dict.bin", "-o", "out.zst", "in.txt"]).unwrap(); + assert_eq!(opts.dict, Some(PathBuf::from("dict.bin"))); + assert_eq!(opts.output, Some(PathBuf::from("out.zst"))); + // Attached value form: -Ddict.bin + let opts = parse(&["-Ddict.bin", "in.txt"]).unwrap(); + assert_eq!(opts.dict, Some(PathBuf::from("dict.bin"))); +} + +#[test] +fn output_rejects_multiple_inputs() { + assert!(parse(&["-o", "out.zst", "a.txt", "b.txt"]).is_err()); +} + +#[test] +fn train_flags_parse_and_allow_many_samples_with_output() { + let opts = parse(&[ + "--train", + "--maxdict=4096", + "--dictID=42", + "-o", + "dict.bin", + "s1.txt", + "s2.txt", + "s3.txt", + ]) + .unwrap(); + assert_eq!(opts.mode, Mode::Train); + assert_eq!(opts.max_dict, 4096); + assert_eq!(opts.dict_id, Some(42)); + assert_eq!(opts.output, Some(PathBuf::from("dict.bin"))); + // --train legitimately fans many samples into one -o dictionary. + assert_eq!(opts.inputs.len(), 3); +} + +#[test] +fn list_mode_parses() { + assert_eq!(parse(&["-l", "a.zst"]).unwrap().mode, Mode::List); + assert_eq!(parse(&["--list", "a.zst"]).unwrap().mode, Mode::List); +} + +#[test] +fn long_flag_enables_ldm() { + assert!(parse(&["-19", "--long", "in.txt"]).unwrap().long); + assert!(parse(&["--long=27", "in.txt"]).unwrap().long); + assert!(!parse(&["-19", "in.txt"]).unwrap().long); +} + +#[test] +fn benchmark_flags_parse_level_range() { + let opts = parse(&["-b3", "-e7", "in.txt"]).unwrap(); + assert!(opts.bench); + assert_eq!(opts.bench_start, 3); + assert_eq!(opts.bench_end, 7); + // Bare `-b` benchmarks the default level (single-level range). + let opts = parse(&["-b", "in.txt"]).unwrap(); + assert!(opts.bench); + assert_eq!(opts.bench_end, opts.bench_start); +} + +#[test] +fn dash_is_a_stdin_input() { + let opts = parse(&["-d", "-"]).unwrap(); + assert_eq!(opts.inputs, vec!["-".to_string()]); +} + +#[test] +fn double_dash_forces_positional() { + let opts = parse(&["--", "-weird-name.txt"]).unwrap(); + assert_eq!(opts.inputs, vec!["-weird-name.txt".to_string()]); +} + +#[test] +fn unknown_flag_errors() { + assert!(parse(&["--definitely-not-a-flag"]).is_err()); + assert!(parse(&["-Z"]).is_err()); +} + +#[test] +fn fast_and_long_match_exactly_not_by_prefix() { + // Exact options succeed. + assert_eq!(parse(&["--fast"]).unwrap().level, -1); + assert_eq!(parse(&["--fast=5"]).unwrap().level, -5); + assert!(parse(&["--long"]).unwrap().long); + // Typos must NOT be silently accepted as `--fast`/`--long`; they fall + // through to the unknown-option path. + assert!(parse(&["--faster"]).is_err()); + assert!(parse(&["--longer"]).is_err()); + // Invalid payloads are rejected, not silently reinterpreted: + // `--fast=-5` must not flip into a positive level, and `--long=` / + // `--long=abc` must not be accepted as a no-op. + assert!(parse(&["--fast=-5"]).is_err()); + assert!(parse(&["--long="]).is_err()); + assert!(parse(&["--long=abc"]).is_err()); + assert!(parse(&["--long=27"]).unwrap().long); +} + +#[test] +fn unsupported_format_flags_are_rejected_not_ignored() { + // These change the wire format but are not wired through yet — accepting + // them silently would hand the caller the wrong frame layout. + assert!(parse(&["--no-check"]).is_err()); + assert!(parse(&["--no-content-size"]).is_err()); + assert!(parse(&["--no-dictID"]).is_err()); + // Verbosity aliases stay honest no-ops. + assert!(parse(&["--quiet"]).is_ok()); + assert!(parse(&["--verbose"]).is_ok()); +} + +#[test] +fn decompress_suffix_stripping() { + let opts = Options { + mode: Mode::Decompress, + level: 3, + store: false, + dict: None, + to_stdout: false, + output: None, + force: false, + keep: false, + remove_source: false, + inputs: vec!["archive.tar.zst".to_string()], + max_dict: DEFAULT_MAX_DICT, + dict_id: None, + bench: false, + bench_start: 3, + bench_end: 0, + bench_secs: 1.0, + long: false, + }; + assert_eq!( + derive_output_path(&opts, Path::new("archive.tar.zst")).unwrap(), + PathBuf::from("archive.tar") + ); + assert!(derive_output_path(&opts, Path::new("noext")).is_err()); +} diff --git a/zstd/src/bit_io/bit_reader_reverse.rs b/zstd/src/bit_io/bit_reader_reverse.rs index 2499bb3d4..4449a0b9e 100644 --- a/zstd/src/bit_io/bit_reader_reverse.rs +++ b/zstd/src/bit_io/bit_reader_reverse.rs @@ -24,62 +24,6 @@ const BIT_MASK: [u64; 65] = { table }; -/// Return the lowest `n` bits of `value` (zero the rest). -/// -/// On x86-64 with BMI2 this compiles to a single `bzhi` instruction. -/// Everywhere else it computes the mask via `u64::MAX >> (64 - n)` -/// (replaced the previous `BIT_MASK[n]` table load — one shift + one -/// predicted cmov vs one 3–5 cycle L1 load on the hot FSE path). -/// -/// This function supports `n <= 64`; zstd callers normally guarantee -/// `n <= 56` (the maximum single-symbol width in zstd). The -/// `debug_assert!(n <= 64)` on the FIRST line of the function body -/// (not just on `get_bits` callers) is the input-validation gate that -/// the fuzz suite relies on — invalid `n > 64` (e.g. from a malformed -/// FSE table or `accuracy_log`) trips it instead of silently returning -/// 0 from the release path. On the BMI2 path `_bzhi_u64` would -/// silently truncate without it; on the fallback path `checked_shr` -/// returns `None` for the wrapping-underflow shift and the -/// `unwrap_or(0)` would otherwise hide the upstream bug. -// Used only by the in-file mask_lower_bits unit tests after the -// hot-path migration to `K::mask_lower_bits` via the `CpuKernel` -// trait. Gating with `#[cfg(test)]` keeps the helper available for -// the regression tests below without triggering a `dead_code` -// warning under `-D warnings` in normal builds. -#[cfg(test)] -#[inline(always)] -fn mask_lower_bits(value: u64, n: u8) -> u64 { - // Input-validation gate documented in the rustdoc above — keep this - // as the first statement; removing it lets malformed inputs (`n > - // 64`) silently decode to 0 in release builds instead of being - // caught by the fuzz suite. - debug_assert!(n <= 64, "mask_lower_bits: n must be <= 64, got {}", n); - #[cfg(all(target_arch = "x86_64", target_feature = "bmi2"))] - { - // SAFETY: `_bzhi_u64` is always safe to call when the target supports BMI2. - unsafe { core::arch::x86_64::_bzhi_u64(value, n as u32) } - } - #[cfg(not(all(target_arch = "x86_64", target_feature = "bmi2")))] - { - // Compute the mask via `u64::MAX >> (64 - n)` instead of a - // `BIT_MASK[n]` table load. One shift + one (predicted) cmov - // vs one L1 load (3-5 cycle latency). For the hot FSE bitstream - // decode path this fires 3x per sequence; saving the load - // latency per call compounds over thousands of sequences. - // - // `checked_shr` returns `None` when the shift count is ≥ 64, - // which happens exactly when `n == 0` (`64 - 0 = 64`) or when - // the debug_assert above would have fired (`n > 64`, underflow - // wraps to a huge value). Mapping both to `0` gives the - // mathematically-correct empty mask for n=0 and a safe-ish - // fallback for the invalid range. - let mask = u64::MAX - .checked_shr(64u32.wrapping_sub(n as u32)) - .unwrap_or(0); - value & mask - } -} - #[cfg(all(feature = "std", target_arch = "x86_64", feature = "kernel_bmi2"))] #[derive(Copy, Clone)] struct TripleExtractDispatch { @@ -131,22 +75,6 @@ fn detect_triple_extract_dispatch() -> TripleExtractDispatch { } } -// Used only by the in-file extract_triple correctness tests after -// `peek_bits_triple` switched to the per-reader `use_pext_triple` -// cached flag (commit 8805122f) — production now calls -// `extract_triple_pext` directly via that path. Gating with -// `#[cfg(test)]` keeps the helper available for the tests while -// avoiding a `dead_code` warning under `-D warnings`. -#[cfg(all(test, feature = "std", target_arch = "x86_64", feature = "kernel_bmi2"))] -#[inline(always)] -fn try_extract_triple_with_pext(all_three: u64, n1: u8, n2: u8, n3: u8) -> Option<(u64, u64, u64)> { - if !triple_extract_dispatch().use_pext { - return None; - } - - Some(unsafe { extract_triple_pext(all_three, n1, n2, n3) }) -} - #[cfg(all(target_arch = "x86_64", feature = "kernel_bmi2"))] #[target_feature(enable = "bmi2")] unsafe fn extract_triple_pext(all_three: u64, n1: u8, n2: u8, n3: u8) -> (u64, u64, u64) { @@ -560,343 +488,4 @@ impl<'s, K: CpuKernel> BitReaderReversed<'s, K> { } #[cfg(test)] -mod test { - #[cfg(all(feature = "std", target_arch = "x86_64", feature = "kernel_bmi2"))] - use std::arch::is_x86_feature_detected; - - #[cfg(all(feature = "std", target_arch = "x86_64", feature = "kernel_bmi2"))] - #[inline] - fn scalar_extract_triple(all_three: u64, n1: u8, n2: u8, n3: u8) -> (u64, u64, u64) { - let val3 = all_three & super::BIT_MASK[n3 as usize]; - let val2 = all_three.wrapping_shr(u32::from(n3)) & super::BIT_MASK[n2 as usize]; - let val1 = - all_three.wrapping_shr(u32::from(n2) + u32::from(n3)) & super::BIT_MASK[n1 as usize]; - (val1, val2, val3) - } - - #[cfg(all(feature = "std", target_arch = "x86_64", feature = "kernel_bmi2"))] - #[inline] - fn next_test_value(state: &mut u64) -> u64 { - let mut x = *state; - x ^= x << 13; - x ^= x >> 7; - x ^= x << 17; - *state = x; - x - } - - #[test] - fn it_works() { - let data = [0b10101010, 0b01010101]; - let mut br = super::BitReaderReversed::::new(&data); - assert_eq!(br.get_bits(1), 0); - assert_eq!(br.get_bits(1), 1); - assert_eq!(br.get_bits(1), 0); - assert_eq!(br.get_bits(4), 0b1010); - assert_eq!(br.get_bits(4), 0b1101); - assert_eq!(br.get_bits(4), 0b0101); - // Last 0 from source, three zeroes filled in - assert_eq!(br.get_bits(4), 0b0000); - // All zeroes filled in - assert_eq!(br.get_bits(4), 0b0000); - assert_eq!(br.bits_remaining(), -7); - } - - /// Verify that `ensure_bits(n)` + `get_bits_unchecked(..)` returns the same - /// values as plain `get_bits(..)`, including across refill boundaries and - /// for edge cases like n=0. - #[test] - fn ensure_and_unchecked_match_get_bits() { - // 10 bytes = 80 bits — enough to force multiple refills - let data: [u8; 10] = [0xDE, 0xAD, 0xBE, 0xEF, 0x42, 0x13, 0x37, 0xCA, 0xFE, 0x01]; - - // Reference: read with get_bits - let mut ref_br = super::BitReaderReversed::::new(&data); - let r1 = ref_br.get_bits(0); - let r2 = ref_br.get_bits(7); - let r3 = ref_br.get_bits(13); - let r4 = ref_br.get_bits(9); - let r5 = ref_br.get_bits(8); - let r5b = ref_br.get_bits(2); - // After 39 bits consumed, ensure_bits(26) triggers a real refill - // because 39 + 26 = 65 > 64. - let r6 = ref_br.get_bits(9); - let r7 = ref_br.get_bits(9); - let r8 = ref_br.get_bits(8); - - // Unchecked path: same reads via ensure_bits + get_bits_unchecked - let mut fast_br = super::BitReaderReversed::::new(&data); - - // n=0 edge case - fast_br.ensure_bits(0); - assert_eq!(fast_br.get_bits_unchecked(0), r1); - - // Single reads - fast_br.ensure_bits(7); - assert_eq!(fast_br.get_bits_unchecked(7), r2); - - fast_br.ensure_bits(13); - assert_eq!(fast_br.get_bits_unchecked(13), r3); - - fast_br.ensure_bits(9); - assert_eq!(fast_br.get_bits_unchecked(9), r4); - - fast_br.ensure_bits(8); - assert_eq!(fast_br.get_bits_unchecked(8), r5); - - fast_br.ensure_bits(2); - assert_eq!(fast_br.get_bits_unchecked(2), r5b); - - // Batched: one ensure covering 9+9+8 = 26 bits. - // At 39 bits consumed, this forces a real refill (39+26=65 > 64). - fast_br.ensure_bits(26); - assert_eq!(fast_br.get_bits_unchecked(9), r6); - assert_eq!(fast_br.get_bits_unchecked(9), r7); - assert_eq!(fast_br.get_bits_unchecked(8), r8); - - assert_eq!(ref_br.bits_remaining(), fast_br.bits_remaining()); - } - - /// Verify that the pre-computed BIT_MASK table produces correct values. - #[test] - fn mask_table_correctness() { - assert_eq!(super::BIT_MASK[0], 0); - assert_eq!(super::BIT_MASK[1], 1); - assert_eq!(super::BIT_MASK[8], 0xFF); - assert_eq!(super::BIT_MASK[16], 0xFFFF); - assert_eq!(super::BIT_MASK[32], 0xFFFF_FFFF); - assert_eq!(super::BIT_MASK[63], (1u64 << 63) - 1); - assert_eq!(super::BIT_MASK[64], u64::MAX); - for n in 0..64u32 { - assert_eq!( - super::BIT_MASK[n as usize], - (1u64 << n) - 1, - "BIT_MASK[{n}] mismatch" - ); - } - } - - /// Verify mask_lower_bits matches manual computation for edge values. - #[test] - fn mask_lower_bits_edge_cases() { - assert_eq!(super::mask_lower_bits(u64::MAX, 0), 0); - assert_eq!(super::mask_lower_bits(u64::MAX, 1), 1); - assert_eq!( - super::mask_lower_bits(0xABCD_1234_5678_9ABC, 64), - 0xABCD_1234_5678_9ABC - ); - assert_eq!(super::mask_lower_bits(0xABCD_1234_5678_9ABC, 8), 0xBC); - assert_eq!(super::mask_lower_bits(0xABCD_1234_5678_9ABC, 16), 0x9ABC); - } - - /// peek_bits(0) must return 0 in all states, including when - /// bits_consumed is 0 (post-exhaustion refill). - #[test] - fn peek_bits_zero_is_always_zero() { - let data = [0xFF; 8]; - let mut br = super::BitReaderReversed::::new(&data); - - // Initial state: bits_consumed = 64 - assert_eq!(br.peek_bits(0), 0); - - // After reading some bits: bits_consumed < 64 - br.get_bits(7); - assert_eq!(br.peek_bits(0), 0); - - // Force bits_consumed == 0 to exercise the shift-by-64 edge case - // in peek_bits. This state occurs naturally during refill() when the - // source is exhausted. We set it directly because get_bits always - // calls consume(n) after refill, making bits_consumed > 0 by the - // time it returns. - br.bits_consumed = 0; - assert_eq!(br.peek_bits(0), 0); - } - - /// get_bits_triple must produce the same values as three individual - /// get_bits calls, both with and without a refill in between. - #[test] - fn get_bits_triple_matches_individual() { - let data: [u8; 16] = [ - 0xDE, 0xAD, 0xBE, 0xEF, 0x42, 0x13, 0x37, 0xCA, 0xFE, 0x01, 0x99, 0x88, 0x77, 0x66, - 0x55, 0x44, - ]; - - // Reference: individual reads - let mut ref_br = super::BitReaderReversed::::new(&data); - let r1 = ref_br.get_bits(8); - let r2 = ref_br.get_bits(9); - let r3 = ref_br.get_bits(9); - - // Triple read - let mut triple_br = super::BitReaderReversed::::new(&data); - let (t1, t2, t3) = triple_br.get_bits_triple(8, 9, 9); - - assert_eq!((r1, r2, r3), (t1, t2, t3)); - assert_eq!(ref_br.bits_remaining(), triple_br.bits_remaining()); - - // No-refill fast path: 8 bits already consumed, so the next 26 bits - // still fit in the current container and `ensure_bits(26)` should - // skip `refill()`. - let mut ref_br = super::BitReaderReversed::::new(&data); - let mut triple_br = super::BitReaderReversed::::new(&data); - let _ = ref_br.get_bits(8); - let _ = triple_br.get_bits(8); - - let r1 = ref_br.get_bits(8); - let r2 = ref_br.get_bits(9); - let r3 = ref_br.get_bits(9); - let (t1, t2, t3) = triple_br.get_bits_triple(8, 9, 9); - - assert_eq!((r1, r2, r3), (t1, t2, t3)); - assert_eq!(ref_br.bits_remaining(), triple_br.bits_remaining()); - - // Mixed zero-widths: individual sequence extra-bit fields can be zero. - let mut ref_br = super::BitReaderReversed::::new(&data); - let mut triple_br = super::BitReaderReversed::::new(&data); - - let r1 = ref_br.get_bits(5); - let r2 = ref_br.get_bits(0); - let r3 = ref_br.get_bits(4); - let (t1, t2, t3) = triple_br.get_bits_triple(5, 0, 4); - - assert_eq!((r1, r2, r3), (t1, t2, t3)); - assert_eq!(ref_br.bits_remaining(), triple_br.bits_remaining()); - } - - /// `peek_bits_bmi2` MUST produce the same value as scalar `peek_bits` - /// on every BMI2-capable CPU. Without parity the bmi2 fast-path - /// chain (when wired) would silently corrupt FSE state. - #[cfg(all(feature = "std", target_arch = "x86_64", feature = "kernel_bmi2"))] - #[test] - fn peek_bits_bmi2_matches_scalar() { - if !is_x86_feature_detected!("bmi2") { - return; - } - let data: [u8; 16] = [ - 0xDE, 0xAD, 0xBE, 0xEF, 0x42, 0x13, 0x37, 0xCA, 0xFE, 0x01, 0x99, 0x88, 0x77, 0x66, - 0x55, 0x44, - ]; - - for n in [0u8, 1, 5, 8, 13, 24, 32, 48, 56] { - let mut scalar = - super::BitReaderReversed::::new(&data); - let mut bmi2 = super::BitReaderReversed::::new(&data); - scalar.ensure_bits(n); - bmi2.ensure_bits(n); - let s = scalar.peek_bits(n); - // SAFETY: gated on `is_x86_feature_detected!("bmi2")` above. - let b = unsafe { bmi2.peek_bits_bmi2(n) }; - assert_eq!(s, b, "mismatch at n={}", n); - } - } - - /// `peek_bits_triple_bmi2` MUST produce the same triple as the - /// scalar variant for every width combination the FSE/HUF decoders - /// can reach. - #[cfg(all(feature = "std", target_arch = "x86_64", feature = "kernel_bmi2"))] - #[test] - fn peek_bits_triple_bmi2_matches_scalar() { - if !is_x86_feature_detected!("bmi2") { - return; - } - let data: [u8; 16] = [ - 0xDE, 0xAD, 0xBE, 0xEF, 0x42, 0x13, 0x37, 0xCA, 0xFE, 0x01, 0x99, 0x88, 0x77, 0x66, - 0x55, 0x44, - ]; - - let widths = [ - (0, 0, 0), - (1, 1, 1), - (3, 5, 7), - (8, 8, 8), - (15, 16, 17), - (5, 0, 4), - ]; - for &(n1, n2, n3) in &widths { - let sum = n1 + n2 + n3; - let mut scalar = - super::BitReaderReversed::::new(&data); - let mut bmi2 = super::BitReaderReversed::::new(&data); - scalar.ensure_bits(sum); - bmi2.ensure_bits(sum); - let s = scalar.peek_bits_triple(sum, n1, n2, n3); - // SAFETY: gated on `is_x86_feature_detected!("bmi2")` above. - let b = unsafe { bmi2.peek_bits_triple_bmi2(sum, n1, n2, n3) }; - assert_eq!(s, b, "mismatch at widths=({},{},{})", n1, n2, n3); - } - } - - #[cfg(all(feature = "std", target_arch = "x86_64", feature = "kernel_bmi2"))] - #[test] - fn should_use_pext_policy_table() { - let cases = [ - (*b"AuthenticAMD", 0x17, false), - (*b"AuthenticAMD", 0x19, true), - (*b"GenuineIntel", 0x06, true), - ]; - - for (vendor, family, expected) in cases { - assert_eq!(super::should_use_pext(vendor, family), expected); - } - } - - #[cfg(all(feature = "std", target_arch = "x86_64", feature = "kernel_bmi2"))] - #[test] - fn bmi2_triple_extract_matches_scalar_reference() { - if !is_x86_feature_detected!("bmi2") { - return; - } - - let widths = [ - (0, 0, 0), - (1, 1, 1), - (3, 5, 7), - (8, 8, 8), - (15, 16, 17), - (21, 21, 21), - (0, 13, 27), - (31, 0, 1), - (1, 31, 0), - (20, 20, 24), - ]; - let fixed_values = [ - 0, - 1, - u64::MAX, - 0x0123_4567_89AB_CDEF, - 0xFEDC_BA98_7654_3210, - 0xAAAA_AAAA_AAAA_AAAA, - 0x5555_5555_5555_5555, - 1u64 << 63, - (1u64 << 32) - 1, - ]; - - for &(n1, n2, n3) in &widths { - for &all_three in &fixed_values { - let expected = scalar_extract_triple(all_three, n1, n2, n3); - let pext = unsafe { super::extract_triple_pext(all_three, n1, n2, n3) }; - assert_eq!(pext, expected); - - if let Some(dispatched) = super::try_extract_triple_with_pext(all_three, n1, n2, n3) - { - assert_eq!(dispatched, expected); - } - } - } - - let mut state = 0xD6E8_FD9D_5A2C_19B7u64; - for &(n1, n2, n3) in &widths { - for _ in 0..64 { - let all_three = next_test_value(&mut state); - let expected = scalar_extract_triple(all_three, n1, n2, n3); - let pext = unsafe { super::extract_triple_pext(all_three, n1, n2, n3) }; - assert_eq!(pext, expected); - - if let Some(dispatched) = super::try_extract_triple_with_pext(all_three, n1, n2, n3) - { - assert_eq!(dispatched, expected); - } - } - } - } -} +mod tests; diff --git a/zstd/src/bit_io/bit_reader_reverse/tests.rs b/zstd/src/bit_io/bit_reader_reverse/tests.rs new file mode 100644 index 000000000..891127ce5 --- /dev/null +++ b/zstd/src/bit_io/bit_reader_reverse/tests.rs @@ -0,0 +1,409 @@ +// `try_extract_triple_with_pext` (x86_64 + BMI2 only) reaches the production +// `triple_extract_dispatch` / `extract_triple_pext` through this glob; on other +// targets that helper compiles out, leaving the import unused. +#[allow(unused_imports)] +use super::*; + +/// Return the lowest `n` bits of `value` (zero the rest). +/// +/// On x86-64 with BMI2 this compiles to a single `bzhi` instruction. +/// Everywhere else it computes the mask via `u64::MAX >> (64 - n)` +/// (replaced the previous `BIT_MASK[n]` table load — one shift + one +/// predicted cmov vs one 3–5 cycle L1 load on the hot FSE path). +/// +/// This function supports `n <= 64`; zstd callers normally guarantee +/// `n <= 56` (the maximum single-symbol width in zstd). The +/// `debug_assert!(n <= 64)` on the FIRST line of the function body +/// (not just on `get_bits` callers) is the input-validation gate that +/// the fuzz suite relies on — invalid `n > 64` (e.g. from a malformed +/// FSE table or `accuracy_log`) trips it instead of silently returning +/// 0 from the release path. On the BMI2 path `_bzhi_u64` would +/// silently truncate without it; on the fallback path `checked_shr` +/// returns `None` for the wrapping-underflow shift and the +/// `unwrap_or(0)` would otherwise hide the upstream bug. +// Used only by the in-file mask_lower_bits unit tests after the +// hot-path migration to `K::mask_lower_bits` via the `CpuKernel` +// trait. Gating with `#[cfg(test)]` keeps the helper available for +// the regression tests below without triggering a `dead_code` +// warning under `-D warnings` in normal builds. +#[cfg(test)] +#[inline(always)] +fn mask_lower_bits(value: u64, n: u8) -> u64 { + // Input-validation gate documented in the rustdoc above — keep this + // as the first statement; removing it lets malformed inputs (`n > + // 64`) silently decode to 0 in release builds instead of being + // caught by the fuzz suite. + debug_assert!(n <= 64, "mask_lower_bits: n must be <= 64, got {}", n); + #[cfg(all(target_arch = "x86_64", target_feature = "bmi2"))] + { + // SAFETY: `_bzhi_u64` is always safe to call when the target supports BMI2. + unsafe { core::arch::x86_64::_bzhi_u64(value, n as u32) } + } + #[cfg(not(all(target_arch = "x86_64", target_feature = "bmi2")))] + { + // Compute the mask via `u64::MAX >> (64 - n)` instead of a + // `BIT_MASK[n]` table load. One shift + one (predicted) cmov + // vs one L1 load (3-5 cycle latency). For the hot FSE bitstream + // decode path this fires 3x per sequence; saving the load + // latency per call compounds over thousands of sequences. + // + // `checked_shr` returns `None` when the shift count is ≥ 64, + // which happens exactly when `n == 0` (`64 - 0 = 64`) or when + // the debug_assert above would have fired (`n > 64`, underflow + // wraps to a huge value). Mapping both to `0` gives the + // mathematically-correct empty mask for n=0 and a safe-ish + // fallback for the invalid range. + let mask = u64::MAX + .checked_shr(64u32.wrapping_sub(n as u32)) + .unwrap_or(0); + value & mask + } +} +// Used only by the in-file extract_triple correctness tests after +// `peek_bits_triple` switched to the per-reader `use_pext_triple` +// cached flag (commit 8805122f) — production now calls +// `extract_triple_pext` directly via that path. Gating with +// `#[cfg(test)]` keeps the helper available for the tests while +// avoiding a `dead_code` warning under `-D warnings`. +#[cfg(all(test, feature = "std", target_arch = "x86_64", feature = "kernel_bmi2"))] +#[inline(always)] +fn try_extract_triple_with_pext(all_three: u64, n1: u8, n2: u8, n3: u8) -> Option<(u64, u64, u64)> { + if !triple_extract_dispatch().use_pext { + return None; + } + + Some(unsafe { extract_triple_pext(all_three, n1, n2, n3) }) +} +#[cfg(all(feature = "std", target_arch = "x86_64", feature = "kernel_bmi2"))] +use std::arch::is_x86_feature_detected; + +#[cfg(all(feature = "std", target_arch = "x86_64", feature = "kernel_bmi2"))] +#[inline] +fn scalar_extract_triple(all_three: u64, n1: u8, n2: u8, n3: u8) -> (u64, u64, u64) { + let val3 = all_three & super::BIT_MASK[n3 as usize]; + let val2 = all_three.wrapping_shr(u32::from(n3)) & super::BIT_MASK[n2 as usize]; + let val1 = all_three.wrapping_shr(u32::from(n2) + u32::from(n3)) & super::BIT_MASK[n1 as usize]; + (val1, val2, val3) +} + +#[cfg(all(feature = "std", target_arch = "x86_64", feature = "kernel_bmi2"))] +#[inline] +fn next_test_value(state: &mut u64) -> u64 { + let mut x = *state; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + *state = x; + x +} + +#[test] +fn it_works() { + let data = [0b10101010, 0b01010101]; + let mut br = super::BitReaderReversed::::new(&data); + assert_eq!(br.get_bits(1), 0); + assert_eq!(br.get_bits(1), 1); + assert_eq!(br.get_bits(1), 0); + assert_eq!(br.get_bits(4), 0b1010); + assert_eq!(br.get_bits(4), 0b1101); + assert_eq!(br.get_bits(4), 0b0101); + // Last 0 from source, three zeroes filled in + assert_eq!(br.get_bits(4), 0b0000); + // All zeroes filled in + assert_eq!(br.get_bits(4), 0b0000); + assert_eq!(br.bits_remaining(), -7); +} + +/// Verify that `ensure_bits(n)` + `get_bits_unchecked(..)` returns the same +/// values as plain `get_bits(..)`, including across refill boundaries and +/// for edge cases like n=0. +#[test] +fn ensure_and_unchecked_match_get_bits() { + // 10 bytes = 80 bits — enough to force multiple refills + let data: [u8; 10] = [0xDE, 0xAD, 0xBE, 0xEF, 0x42, 0x13, 0x37, 0xCA, 0xFE, 0x01]; + + // Reference: read with get_bits + let mut ref_br = super::BitReaderReversed::::new(&data); + let r1 = ref_br.get_bits(0); + let r2 = ref_br.get_bits(7); + let r3 = ref_br.get_bits(13); + let r4 = ref_br.get_bits(9); + let r5 = ref_br.get_bits(8); + let r5b = ref_br.get_bits(2); + // After 39 bits consumed, ensure_bits(26) triggers a real refill + // because 39 + 26 = 65 > 64. + let r6 = ref_br.get_bits(9); + let r7 = ref_br.get_bits(9); + let r8 = ref_br.get_bits(8); + + // Unchecked path: same reads via ensure_bits + get_bits_unchecked + let mut fast_br = super::BitReaderReversed::::new(&data); + + // n=0 edge case + fast_br.ensure_bits(0); + assert_eq!(fast_br.get_bits_unchecked(0), r1); + + // Single reads + fast_br.ensure_bits(7); + assert_eq!(fast_br.get_bits_unchecked(7), r2); + + fast_br.ensure_bits(13); + assert_eq!(fast_br.get_bits_unchecked(13), r3); + + fast_br.ensure_bits(9); + assert_eq!(fast_br.get_bits_unchecked(9), r4); + + fast_br.ensure_bits(8); + assert_eq!(fast_br.get_bits_unchecked(8), r5); + + fast_br.ensure_bits(2); + assert_eq!(fast_br.get_bits_unchecked(2), r5b); + + // Batched: one ensure covering 9+9+8 = 26 bits. + // At 39 bits consumed, this forces a real refill (39+26=65 > 64). + fast_br.ensure_bits(26); + assert_eq!(fast_br.get_bits_unchecked(9), r6); + assert_eq!(fast_br.get_bits_unchecked(9), r7); + assert_eq!(fast_br.get_bits_unchecked(8), r8); + + assert_eq!(ref_br.bits_remaining(), fast_br.bits_remaining()); +} + +/// Verify that the pre-computed BIT_MASK table produces correct values. +#[test] +fn mask_table_correctness() { + assert_eq!(super::BIT_MASK[0], 0); + assert_eq!(super::BIT_MASK[1], 1); + assert_eq!(super::BIT_MASK[8], 0xFF); + assert_eq!(super::BIT_MASK[16], 0xFFFF); + assert_eq!(super::BIT_MASK[32], 0xFFFF_FFFF); + assert_eq!(super::BIT_MASK[63], (1u64 << 63) - 1); + assert_eq!(super::BIT_MASK[64], u64::MAX); + for n in 0..64u32 { + assert_eq!( + super::BIT_MASK[n as usize], + (1u64 << n) - 1, + "BIT_MASK[{n}] mismatch" + ); + } +} + +/// Verify mask_lower_bits matches manual computation for edge values. +#[test] +fn mask_lower_bits_edge_cases() { + assert_eq!(mask_lower_bits(u64::MAX, 0), 0); + assert_eq!(mask_lower_bits(u64::MAX, 1), 1); + assert_eq!( + mask_lower_bits(0xABCD_1234_5678_9ABC, 64), + 0xABCD_1234_5678_9ABC + ); + assert_eq!(mask_lower_bits(0xABCD_1234_5678_9ABC, 8), 0xBC); + assert_eq!(mask_lower_bits(0xABCD_1234_5678_9ABC, 16), 0x9ABC); +} + +/// peek_bits(0) must return 0 in all states, including when +/// bits_consumed is 0 (post-exhaustion refill). +#[test] +fn peek_bits_zero_is_always_zero() { + let data = [0xFF; 8]; + let mut br = super::BitReaderReversed::::new(&data); + + // Initial state: bits_consumed = 64 + assert_eq!(br.peek_bits(0), 0); + + // After reading some bits: bits_consumed < 64 + br.get_bits(7); + assert_eq!(br.peek_bits(0), 0); + + // Force bits_consumed == 0 to exercise the shift-by-64 edge case + // in peek_bits. This state occurs naturally during refill() when the + // source is exhausted. We set it directly because get_bits always + // calls consume(n) after refill, making bits_consumed > 0 by the + // time it returns. + br.bits_consumed = 0; + assert_eq!(br.peek_bits(0), 0); +} + +/// get_bits_triple must produce the same values as three individual +/// get_bits calls, both with and without a refill in between. +#[test] +fn get_bits_triple_matches_individual() { + let data: [u8; 16] = [ + 0xDE, 0xAD, 0xBE, 0xEF, 0x42, 0x13, 0x37, 0xCA, 0xFE, 0x01, 0x99, 0x88, 0x77, 0x66, 0x55, + 0x44, + ]; + + // Reference: individual reads + let mut ref_br = super::BitReaderReversed::::new(&data); + let r1 = ref_br.get_bits(8); + let r2 = ref_br.get_bits(9); + let r3 = ref_br.get_bits(9); + + // Triple read + let mut triple_br = super::BitReaderReversed::::new(&data); + let (t1, t2, t3) = triple_br.get_bits_triple(8, 9, 9); + + assert_eq!((r1, r2, r3), (t1, t2, t3)); + assert_eq!(ref_br.bits_remaining(), triple_br.bits_remaining()); + + // No-refill fast path: 8 bits already consumed, so the next 26 bits + // still fit in the current container and `ensure_bits(26)` should + // skip `refill()`. + let mut ref_br = super::BitReaderReversed::::new(&data); + let mut triple_br = super::BitReaderReversed::::new(&data); + let _ = ref_br.get_bits(8); + let _ = triple_br.get_bits(8); + + let r1 = ref_br.get_bits(8); + let r2 = ref_br.get_bits(9); + let r3 = ref_br.get_bits(9); + let (t1, t2, t3) = triple_br.get_bits_triple(8, 9, 9); + + assert_eq!((r1, r2, r3), (t1, t2, t3)); + assert_eq!(ref_br.bits_remaining(), triple_br.bits_remaining()); + + // Mixed zero-widths: individual sequence extra-bit fields can be zero. + let mut ref_br = super::BitReaderReversed::::new(&data); + let mut triple_br = super::BitReaderReversed::::new(&data); + + let r1 = ref_br.get_bits(5); + let r2 = ref_br.get_bits(0); + let r3 = ref_br.get_bits(4); + let (t1, t2, t3) = triple_br.get_bits_triple(5, 0, 4); + + assert_eq!((r1, r2, r3), (t1, t2, t3)); + assert_eq!(ref_br.bits_remaining(), triple_br.bits_remaining()); +} + +/// `peek_bits_bmi2` MUST produce the same value as scalar `peek_bits` +/// on every BMI2-capable CPU. Without parity the bmi2 fast-path +/// chain (when wired) would silently corrupt FSE state. +#[cfg(all(feature = "std", target_arch = "x86_64", feature = "kernel_bmi2"))] +#[test] +fn peek_bits_bmi2_matches_scalar() { + if !is_x86_feature_detected!("bmi2") { + return; + } + let data: [u8; 16] = [ + 0xDE, 0xAD, 0xBE, 0xEF, 0x42, 0x13, 0x37, 0xCA, 0xFE, 0x01, 0x99, 0x88, 0x77, 0x66, 0x55, + 0x44, + ]; + + for n in [0u8, 1, 5, 8, 13, 24, 32, 48, 56] { + let mut scalar = super::BitReaderReversed::::new(&data); + let mut bmi2 = super::BitReaderReversed::::new(&data); + scalar.ensure_bits(n); + bmi2.ensure_bits(n); + let s = scalar.peek_bits(n); + // SAFETY: gated on `is_x86_feature_detected!("bmi2")` above. + let b = unsafe { bmi2.peek_bits_bmi2(n) }; + assert_eq!(s, b, "mismatch at n={}", n); + } +} + +/// `peek_bits_triple_bmi2` MUST produce the same triple as the +/// scalar variant for every width combination the FSE/HUF decoders +/// can reach. +#[cfg(all(feature = "std", target_arch = "x86_64", feature = "kernel_bmi2"))] +#[test] +fn peek_bits_triple_bmi2_matches_scalar() { + if !is_x86_feature_detected!("bmi2") { + return; + } + let data: [u8; 16] = [ + 0xDE, 0xAD, 0xBE, 0xEF, 0x42, 0x13, 0x37, 0xCA, 0xFE, 0x01, 0x99, 0x88, 0x77, 0x66, 0x55, + 0x44, + ]; + + let widths = [ + (0, 0, 0), + (1, 1, 1), + (3, 5, 7), + (8, 8, 8), + (15, 16, 17), + (5, 0, 4), + ]; + for &(n1, n2, n3) in &widths { + let sum = n1 + n2 + n3; + let mut scalar = super::BitReaderReversed::::new(&data); + let mut bmi2 = super::BitReaderReversed::::new(&data); + scalar.ensure_bits(sum); + bmi2.ensure_bits(sum); + let s = scalar.peek_bits_triple(sum, n1, n2, n3); + // SAFETY: gated on `is_x86_feature_detected!("bmi2")` above. + let b = unsafe { bmi2.peek_bits_triple_bmi2(sum, n1, n2, n3) }; + assert_eq!(s, b, "mismatch at widths=({},{},{})", n1, n2, n3); + } +} + +#[cfg(all(feature = "std", target_arch = "x86_64", feature = "kernel_bmi2"))] +#[test] +fn should_use_pext_policy_table() { + let cases = [ + (*b"AuthenticAMD", 0x17, false), + (*b"AuthenticAMD", 0x19, true), + (*b"GenuineIntel", 0x06, true), + ]; + + for (vendor, family, expected) in cases { + assert_eq!(super::should_use_pext(vendor, family), expected); + } +} + +#[cfg(all(feature = "std", target_arch = "x86_64", feature = "kernel_bmi2"))] +#[test] +fn bmi2_triple_extract_matches_scalar_reference() { + if !is_x86_feature_detected!("bmi2") { + return; + } + + let widths = [ + (0, 0, 0), + (1, 1, 1), + (3, 5, 7), + (8, 8, 8), + (15, 16, 17), + (21, 21, 21), + (0, 13, 27), + (31, 0, 1), + (1, 31, 0), + (20, 20, 24), + ]; + let fixed_values = [ + 0, + 1, + u64::MAX, + 0x0123_4567_89AB_CDEF, + 0xFEDC_BA98_7654_3210, + 0xAAAA_AAAA_AAAA_AAAA, + 0x5555_5555_5555_5555, + 1u64 << 63, + (1u64 << 32) - 1, + ]; + + for &(n1, n2, n3) in &widths { + for &all_three in &fixed_values { + let expected = scalar_extract_triple(all_three, n1, n2, n3); + let pext = unsafe { super::extract_triple_pext(all_three, n1, n2, n3) }; + assert_eq!(pext, expected); + + if let Some(dispatched) = try_extract_triple_with_pext(all_three, n1, n2, n3) { + assert_eq!(dispatched, expected); + } + } + } + + let mut state = 0xD6E8_FD9D_5A2C_19B7u64; + for &(n1, n2, n3) in &widths { + for _ in 0..64 { + let all_three = next_test_value(&mut state); + let expected = scalar_extract_triple(all_three, n1, n2, n3); + let pext = unsafe { super::extract_triple_pext(all_three, n1, n2, n3) }; + assert_eq!(pext, expected); + + if let Some(dispatched) = try_extract_triple_with_pext(all_three, n1, n2, n3) { + assert_eq!(dispatched, expected); + } + } + } +} diff --git a/zstd/src/bit_io/bit_writer.rs b/zstd/src/bit_io/bit_writer.rs index 4f8a23f6b..3cd9a5bdb 100644 --- a/zstd/src/bit_io/bit_writer.rs +++ b/zstd/src/bit_io/bit_writer.rs @@ -441,171 +441,4 @@ impl>> BitWriter { } #[cfg(test)] -mod tests { - use super::BitWriter; - use alloc::vec; - - #[test] - fn from_existing() { - // Define an existing vec, write some bits into it - let mut existing_vec = vec![255_u8]; - let mut bw = BitWriter::from(&mut existing_vec); - bw.write_bits(0u8, 8); - bw.flush(); - assert_eq!(vec![255, 0], existing_vec); - } - - #[test] - fn change_bits() { - let mut writer = BitWriter::new(); - writer.write_bits(0u32, 24); - writer.change_bits(8, 0xFFu8, 8); - assert_eq!(vec![0, 0xFF, 0], writer.dump()); - - let mut writer = BitWriter::new(); - writer.write_bits(0u32, 24); - writer.change_bits(6, 0x0FFFu16, 12); - assert_eq!(vec![0b11000000, 0xFF, 0b00000011], writer.dump()); - } - - #[test] - fn single_byte_written_4_4() { - // Write the first 4 bits as 1s and the last 4 bits as 0s - // 1010 is used where values should never be read from. - let mut bw = BitWriter::new(); - bw.write_bits(0b1111u8, 4); - bw.write_bits(0b0000u8, 4); - let output = bw.dump(); - assert!( - output.len() == 1, - "Single byte written into writer returned a vec that wasn't one byte, vec was {} elements long", - output.len() - ); - assert_eq!( - 0b0000_1111, output[0], - "4 bits and 4 bits written into buffer" - ); - } - - #[test] - fn single_byte_written_3_5() { - // Write the first 3 bits as 1s and the last 5 bits as 0s - let mut bw = BitWriter::new(); - bw.write_bits(0b111u8, 3); - bw.write_bits(0b0_0000u8, 5); - let output = bw.dump(); - assert!( - output.len() == 1, - "Single byte written into writer return a vec that wasn't one byte, vec was {} elements long", - output.len() - ); - assert_eq!(0b0000_0111, output[0], "3 and 5 bits written into buffer"); - } - - #[test] - fn single_byte_written_1_7() { - // Write the first bit as a 1 and the last 7 bits as 0s - let mut bw = BitWriter::new(); - bw.write_bits(0b1u8, 1); - bw.write_bits(0u8, 7); - let output = bw.dump(); - assert!( - output.len() == 1, - "Single byte written into writer return a vec that wasn't one byte, vec was {} elements long", - output.len() - ); - assert_eq!(0b0000_0001, output[0], "1 and 7 bits written into buffer"); - } - - #[test] - fn single_byte_written_8() { - // Write an entire byte - let mut bw = BitWriter::new(); - bw.write_bits(1u8, 8); - let output = bw.dump(); - assert!( - output.len() == 1, - "Single byte written into writer return a vec that wasn't one byte, vec was {} elements long", - output.len() - ); - assert_eq!(1, output[0], "1 and 7 bits written into buffer"); - } - - #[test] - fn multi_byte_clean_boundary_4_4_4_4() { - // Writing 4 bits at a time for 2 bytes - let mut bw = BitWriter::new(); - bw.write_bits(0u8, 4); - bw.write_bits(0b1111u8, 4); - bw.write_bits(0b1111u8, 4); - bw.write_bits(0u8, 4); - assert_eq!(vec![0b1111_0000, 0b0000_1111], bw.dump()); - } - - #[test] - fn multi_byte_clean_boundary_16_8() { - // Writing 16 bits at once - let mut bw = BitWriter::new(); - bw.write_bits(0x0100u16, 16); - bw.write_bits(69u8, 8); - assert_eq!(vec![0, 1, 69], bw.dump()) - } - - #[test] - fn multi_byte_boundary_crossed_4_12() { - // Writing 4 1s and then 12 zeros - let mut bw = BitWriter::new(); - bw.write_bits(0b1111u8, 4); - bw.write_bits(0b0000_0011_0100_0010u16, 12); - assert_eq!(vec![0b0010_1111, 0b0011_0100], bw.dump()); - } - - #[test] - fn multi_byte_boundary_crossed_4_5_7() { - // Writing 4 1s and then 5 zeros then 7 1s - let mut bw = BitWriter::new(); - bw.write_bits(0b1111u8, 4); - bw.write_bits(0b0_0000u8, 5); - bw.write_bits(0b111_1111u8, 7); - assert_eq!(vec![0b0000_1111, 0b1111_1110], bw.dump()); - } - - #[test] - fn multi_byte_boundary_crossed_1_9_6() { - // Writing 1 1 and then 9 zeros then 6 1s - let mut bw = BitWriter::new(); - bw.write_bits(0b1u8, 1); - bw.write_bits(0b0_0000_0000u16, 9); - bw.write_bits(0b11_1111u8, 6); - assert_eq!(vec![0b0000_0001, 0b1111_1100], bw.dump()); - } - - #[test] - #[should_panic] - fn catches_unaligned_dump() { - // Write a single bit in then dump it, making sure - // the correct error is returned - let mut bw = BitWriter::new(); - bw.write_bits(0u8, 1); - bw.dump(); - } - - #[test] - #[should_panic] - fn catches_dirty_upper_bits() { - let mut bw = BitWriter::new(); - bw.write_bits(10u8, 1); - } - - #[test] - fn add_multiple_aligned() { - let mut bw = BitWriter::new(); - bw.write_bits(0x00_0F_F0_FFu32, 32); - assert_eq!(vec![0xFF, 0xF0, 0x0F, 0x00], bw.dump()); - } - - // #[test] - // fn catches_more_than_in_buf() { - // todo!(); - // } -} +mod tests; diff --git a/zstd/src/bit_io/bit_writer/tests.rs b/zstd/src/bit_io/bit_writer/tests.rs new file mode 100644 index 000000000..5f1faeb50 --- /dev/null +++ b/zstd/src/bit_io/bit_writer/tests.rs @@ -0,0 +1,166 @@ +use super::BitWriter; +use alloc::vec; + +#[test] +fn from_existing() { + // Define an existing vec, write some bits into it + let mut existing_vec = vec![255_u8]; + let mut bw = BitWriter::from(&mut existing_vec); + bw.write_bits(0u8, 8); + bw.flush(); + assert_eq!(vec![255, 0], existing_vec); +} + +#[test] +fn change_bits() { + let mut writer = BitWriter::new(); + writer.write_bits(0u32, 24); + writer.change_bits(8, 0xFFu8, 8); + assert_eq!(vec![0, 0xFF, 0], writer.dump()); + + let mut writer = BitWriter::new(); + writer.write_bits(0u32, 24); + writer.change_bits(6, 0x0FFFu16, 12); + assert_eq!(vec![0b11000000, 0xFF, 0b00000011], writer.dump()); +} + +#[test] +fn single_byte_written_4_4() { + // Write the first 4 bits as 1s and the last 4 bits as 0s + // 1010 is used where values should never be read from. + let mut bw = BitWriter::new(); + bw.write_bits(0b1111u8, 4); + bw.write_bits(0b0000u8, 4); + let output = bw.dump(); + assert!( + output.len() == 1, + "Single byte written into writer returned a vec that wasn't one byte, vec was {} elements long", + output.len() + ); + assert_eq!( + 0b0000_1111, output[0], + "4 bits and 4 bits written into buffer" + ); +} + +#[test] +fn single_byte_written_3_5() { + // Write the first 3 bits as 1s and the last 5 bits as 0s + let mut bw = BitWriter::new(); + bw.write_bits(0b111u8, 3); + bw.write_bits(0b0_0000u8, 5); + let output = bw.dump(); + assert!( + output.len() == 1, + "Single byte written into writer return a vec that wasn't one byte, vec was {} elements long", + output.len() + ); + assert_eq!(0b0000_0111, output[0], "3 and 5 bits written into buffer"); +} + +#[test] +fn single_byte_written_1_7() { + // Write the first bit as a 1 and the last 7 bits as 0s + let mut bw = BitWriter::new(); + bw.write_bits(0b1u8, 1); + bw.write_bits(0u8, 7); + let output = bw.dump(); + assert!( + output.len() == 1, + "Single byte written into writer return a vec that wasn't one byte, vec was {} elements long", + output.len() + ); + assert_eq!(0b0000_0001, output[0], "1 and 7 bits written into buffer"); +} + +#[test] +fn single_byte_written_8() { + // Write an entire byte + let mut bw = BitWriter::new(); + bw.write_bits(1u8, 8); + let output = bw.dump(); + assert!( + output.len() == 1, + "Single byte written into writer return a vec that wasn't one byte, vec was {} elements long", + output.len() + ); + assert_eq!(1, output[0], "1 and 7 bits written into buffer"); +} + +#[test] +fn multi_byte_clean_boundary_4_4_4_4() { + // Writing 4 bits at a time for 2 bytes + let mut bw = BitWriter::new(); + bw.write_bits(0u8, 4); + bw.write_bits(0b1111u8, 4); + bw.write_bits(0b1111u8, 4); + bw.write_bits(0u8, 4); + assert_eq!(vec![0b1111_0000, 0b0000_1111], bw.dump()); +} + +#[test] +fn multi_byte_clean_boundary_16_8() { + // Writing 16 bits at once + let mut bw = BitWriter::new(); + bw.write_bits(0x0100u16, 16); + bw.write_bits(69u8, 8); + assert_eq!(vec![0, 1, 69], bw.dump()) +} + +#[test] +fn multi_byte_boundary_crossed_4_12() { + // Writing 4 1s and then 12 zeros + let mut bw = BitWriter::new(); + bw.write_bits(0b1111u8, 4); + bw.write_bits(0b0000_0011_0100_0010u16, 12); + assert_eq!(vec![0b0010_1111, 0b0011_0100], bw.dump()); +} + +#[test] +fn multi_byte_boundary_crossed_4_5_7() { + // Writing 4 1s and then 5 zeros then 7 1s + let mut bw = BitWriter::new(); + bw.write_bits(0b1111u8, 4); + bw.write_bits(0b0_0000u8, 5); + bw.write_bits(0b111_1111u8, 7); + assert_eq!(vec![0b0000_1111, 0b1111_1110], bw.dump()); +} + +#[test] +fn multi_byte_boundary_crossed_1_9_6() { + // Writing 1 1 and then 9 zeros then 6 1s + let mut bw = BitWriter::new(); + bw.write_bits(0b1u8, 1); + bw.write_bits(0b0_0000_0000u16, 9); + bw.write_bits(0b11_1111u8, 6); + assert_eq!(vec![0b0000_0001, 0b1111_1100], bw.dump()); +} + +#[test] +#[should_panic] +fn catches_unaligned_dump() { + // Write a single bit in then dump it, making sure + // the correct error is returned + let mut bw = BitWriter::new(); + bw.write_bits(0u8, 1); + bw.dump(); +} + +#[test] +#[should_panic] +fn catches_dirty_upper_bits() { + let mut bw = BitWriter::new(); + bw.write_bits(10u8, 1); +} + +#[test] +fn add_multiple_aligned() { + let mut bw = BitWriter::new(); + bw.write_bits(0x00_0F_F0_FFu32, 32); + assert_eq!(vec![0xFF, 0xF0, 0x0F, 0x00], bw.dump()); +} + +// #[test] +// fn catches_more_than_in_buf() { +// todo!(); +// } diff --git a/zstd/src/cpu_kernel.rs b/zstd/src/cpu_kernel.rs index 95babb3e4..8a46b989f 100644 --- a/zstd/src/cpu_kernel.rs +++ b/zstd/src/cpu_kernel.rs @@ -412,167 +412,4 @@ pub fn active_cpu_kernel_name() -> &'static str { } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn scalar_mask_lower_bits_zero_n_returns_zero() { - assert_eq!(ScalarKernel::mask_lower_bits(0xDEADBEEF, 0), 0); - } - - #[test] - fn scalar_mask_lower_bits_full_64_returns_full_value() { - assert_eq!( - ScalarKernel::mask_lower_bits(0xFFFF_FFFF_FFFF_FFFF, 64), - 0xFFFF_FFFF_FFFF_FFFF - ); - } - - #[test] - fn scalar_mask_lower_bits_mid_keeps_low_n_bits() { - // n=8: keep low 8 bits, zero the rest - assert_eq!(ScalarKernel::mask_lower_bits(0xDEAD_BEEF, 8), 0xEF); - assert_eq!( - ScalarKernel::mask_lower_bits(0x0102_0304_0506_0708, 16), - 0x0708 - ); - } - - // Gated on `std` AND `kernel_avx2`: the `is_x86_feature_detected!` - // guard below is a no-op under `--no-default-features` (no std, - // no runtime feature detection), so the test body would call - // `Avx2Kernel::mask_lower_bits` unconditionally and SIGILL on any - // non-BMI2 CPU — hence `feature = "std"`. `Avx2Kernel` itself is - // `#[cfg(feature = "kernel_avx2")]`, so the test must also require - // that feature or a `std`-only trimmed build (`kernel_avx2` off) - // fails to compile against the undefined type. - #[cfg(all(target_arch = "x86_64", feature = "std", feature = "kernel_avx2"))] - #[test] - fn avx2_mask_lower_bits_matches_scalar_on_bmi2_hw() { - // Only run when BMI2 actually available — otherwise constructing - // Avx2Kernel via dispatch wouldn't happen. - if !std::arch::is_x86_feature_detected!("bmi2") { - return; - } - for n in 0..=64u8 { - let v = 0x1234_5678_9ABC_DEF0u64; - assert_eq!( - Avx2Kernel::mask_lower_bits(v, n), - ScalarKernel::mask_lower_bits(v, n), - "mismatch at n={}", - n - ); - } - } - - /// Regression: a CPU advertising AVX-512 VBMI2 but NOT AVX2 (the - /// AMD64 baseline allows this combination at the spec level) was - /// previously selected as `Vbmi2`, which would SIGILL on the - /// first AVX2-mixed VBMI2 kernel invocation. The selection must - /// fall through to Scalar (or a non-AVX tier) in that case. - #[cfg(all(target_arch = "x86_64", feature = "kernel_vbmi2"))] - #[test] - fn select_x86_kernel_vbmi2_without_avx2_does_not_pick_vbmi2() { - let tag = select_x86_kernel( - /* avx512vbmi2 */ true, /* avx512f */ true, /* avx512vl */ true, - /* avx512bw */ true, /* bmi2 */ true, /* avx2 */ false, - /* sse2 */ true, - ); - assert_ne!( - tag, - CpuKernelTag::Vbmi2, - "selecting Vbmi2 without AVX2 would call AVX2 instructions and SIGILL" - ); - } - - /// Sanity: when every flag is present the selector returns Vbmi2. - #[cfg(all(target_arch = "x86_64", feature = "kernel_vbmi2"))] - #[test] - fn select_x86_kernel_full_x86_v4_picks_vbmi2() { - let tag = select_x86_kernel(true, true, true, true, true, true, true); - assert_eq!(tag, CpuKernelTag::Vbmi2); - } - - /// Sanity: AVX2 + BMI2 without AVX-512 → Avx2. - #[cfg(all(target_arch = "x86_64", feature = "kernel_avx2"))] - #[test] - fn select_x86_kernel_avx2_baseline_picks_avx2() { - let tag = select_x86_kernel(false, false, false, false, true, true, true); - assert_eq!(tag, CpuKernelTag::Avx2); - } - - /// SSE2-only (no BMI2/AVX2) → Sse2, the x86_64 floor above Scalar. - #[cfg(all(target_arch = "x86_64", feature = "kernel_sse2"))] - #[test] - fn select_x86_kernel_sse2_only_picks_sse2() { - let tag = select_x86_kernel(false, false, false, false, false, false, true); - assert_eq!(tag, CpuKernelTag::Sse2); - } - - /// No SIMD flags at all → Scalar (off-x86_64 / pre-SSE2 x86). - #[cfg(target_arch = "x86_64")] - #[test] - fn select_x86_kernel_no_features_picks_scalar() { - let tag = select_x86_kernel(false, false, false, false, false, false, false); - assert_eq!(tag, CpuKernelTag::Scalar); - } - - #[test] - fn detect_returns_consistent_tag() { - let first = detect_cpu_kernel(); - let second = detect_cpu_kernel(); - assert_eq!( - first, second, - "cached detect must return same tag on repeated calls" - ); - } - - #[test] - fn active_kernel_name_is_known_lowercase_tier() { - // The diagnostic name must be one of the stable lowercase tier - // strings the dashboard parses, and must match whatever tier - // detection resolves to on this host (no `unknown` / empty leak). - const KNOWN: &[&str] = &["scalar", "sse2", "bmi2", "avx2", "vbmi2", "neon", "sve"]; - let name = active_cpu_kernel_name(); - assert!( - KNOWN.contains(&name), - "active kernel name {name:?} is not a recognised tier" - ); - assert_eq!( - name, - name.to_ascii_lowercase(), - "tier name must be lowercase for stable dashboard parsing" - ); - } - - #[test] - fn every_kernel_tag_maps_to_its_lowercase_name() { - // `active_cpu_kernel_name` only exercises whichever arm the running - // CPU resolves to, so map each constructible tag directly to cover - // every branch on this build's feature set. - assert_eq!(CpuKernelTag::Scalar.name(), "scalar"); - #[cfg(all(target_arch = "x86_64", feature = "kernel_sse2"))] - assert_eq!(CpuKernelTag::Sse2.name(), "sse2"); - #[cfg(all(target_arch = "x86_64", feature = "kernel_bmi2"))] - assert_eq!(CpuKernelTag::Bmi2.name(), "bmi2"); - #[cfg(all(target_arch = "x86_64", feature = "kernel_avx2"))] - assert_eq!(CpuKernelTag::Avx2.name(), "avx2"); - #[cfg(all(target_arch = "x86_64", feature = "kernel_vbmi2"))] - assert_eq!(CpuKernelTag::Vbmi2.name(), "vbmi2"); - #[cfg(all(target_arch = "aarch64", feature = "kernel_neon"))] - assert_eq!(CpuKernelTag::Neon.name(), "neon"); - #[cfg(all( - target_arch = "aarch64", - feature = "kernel_sve", - any(feature = "std", target_feature = "sve"), - ))] - assert_eq!(CpuKernelTag::Sve.name(), "sve"); - } - - #[test] - fn active_kernel_name_is_stable_across_calls() { - // Backed by the cached `detect_cpu_kernel`, so repeated calls must - // return the identical static string. - assert_eq!(active_cpu_kernel_name(), active_cpu_kernel_name()); - } -} +mod tests; diff --git a/zstd/src/cpu_kernel/tests.rs b/zstd/src/cpu_kernel/tests.rs new file mode 100644 index 000000000..131da5c5d --- /dev/null +++ b/zstd/src/cpu_kernel/tests.rs @@ -0,0 +1,162 @@ +use super::*; + +#[test] +fn scalar_mask_lower_bits_zero_n_returns_zero() { + assert_eq!(ScalarKernel::mask_lower_bits(0xDEADBEEF, 0), 0); +} + +#[test] +fn scalar_mask_lower_bits_full_64_returns_full_value() { + assert_eq!( + ScalarKernel::mask_lower_bits(0xFFFF_FFFF_FFFF_FFFF, 64), + 0xFFFF_FFFF_FFFF_FFFF + ); +} + +#[test] +fn scalar_mask_lower_bits_mid_keeps_low_n_bits() { + // n=8: keep low 8 bits, zero the rest + assert_eq!(ScalarKernel::mask_lower_bits(0xDEAD_BEEF, 8), 0xEF); + assert_eq!( + ScalarKernel::mask_lower_bits(0x0102_0304_0506_0708, 16), + 0x0708 + ); +} + +// Gated on `std` AND `kernel_avx2`: the `is_x86_feature_detected!` +// guard below is a no-op under `--no-default-features` (no std, +// no runtime feature detection), so the test body would call +// `Avx2Kernel::mask_lower_bits` unconditionally and SIGILL on any +// non-BMI2 CPU — hence `feature = "std"`. `Avx2Kernel` itself is +// `#[cfg(feature = "kernel_avx2")]`, so the test must also require +// that feature or a `std`-only trimmed build (`kernel_avx2` off) +// fails to compile against the undefined type. +#[cfg(all(target_arch = "x86_64", feature = "std", feature = "kernel_avx2"))] +#[test] +fn avx2_mask_lower_bits_matches_scalar_on_bmi2_hw() { + // Only run when BMI2 actually available — otherwise constructing + // Avx2Kernel via dispatch wouldn't happen. + if !std::arch::is_x86_feature_detected!("bmi2") { + return; + } + for n in 0..=64u8 { + let v = 0x1234_5678_9ABC_DEF0u64; + assert_eq!( + Avx2Kernel::mask_lower_bits(v, n), + ScalarKernel::mask_lower_bits(v, n), + "mismatch at n={}", + n + ); + } +} + +/// Regression: a CPU advertising AVX-512 VBMI2 but NOT AVX2 (the +/// AMD64 baseline allows this combination at the spec level) was +/// previously selected as `Vbmi2`, which would SIGILL on the +/// first AVX2-mixed VBMI2 kernel invocation. The selection must +/// fall through to Scalar (or a non-AVX tier) in that case. +#[cfg(all(target_arch = "x86_64", feature = "kernel_vbmi2"))] +#[test] +fn select_x86_kernel_vbmi2_without_avx2_does_not_pick_vbmi2() { + let tag = select_x86_kernel( + /* avx512vbmi2 */ true, /* avx512f */ true, /* avx512vl */ true, + /* avx512bw */ true, /* bmi2 */ true, /* avx2 */ false, + /* sse2 */ true, + ); + assert_ne!( + tag, + CpuKernelTag::Vbmi2, + "selecting Vbmi2 without AVX2 would call AVX2 instructions and SIGILL" + ); +} + +/// Sanity: when every flag is present the selector returns Vbmi2. +#[cfg(all(target_arch = "x86_64", feature = "kernel_vbmi2"))] +#[test] +fn select_x86_kernel_full_x86_v4_picks_vbmi2() { + let tag = select_x86_kernel(true, true, true, true, true, true, true); + assert_eq!(tag, CpuKernelTag::Vbmi2); +} + +/// Sanity: AVX2 + BMI2 without AVX-512 → Avx2. +#[cfg(all(target_arch = "x86_64", feature = "kernel_avx2"))] +#[test] +fn select_x86_kernel_avx2_baseline_picks_avx2() { + let tag = select_x86_kernel(false, false, false, false, true, true, true); + assert_eq!(tag, CpuKernelTag::Avx2); +} + +/// SSE2-only (no BMI2/AVX2) → Sse2, the x86_64 floor above Scalar. +#[cfg(all(target_arch = "x86_64", feature = "kernel_sse2"))] +#[test] +fn select_x86_kernel_sse2_only_picks_sse2() { + let tag = select_x86_kernel(false, false, false, false, false, false, true); + assert_eq!(tag, CpuKernelTag::Sse2); +} + +/// No SIMD flags at all → Scalar (off-x86_64 / pre-SSE2 x86). +#[cfg(target_arch = "x86_64")] +#[test] +fn select_x86_kernel_no_features_picks_scalar() { + let tag = select_x86_kernel(false, false, false, false, false, false, false); + assert_eq!(tag, CpuKernelTag::Scalar); +} + +#[test] +fn detect_returns_consistent_tag() { + let first = detect_cpu_kernel(); + let second = detect_cpu_kernel(); + assert_eq!( + first, second, + "cached detect must return same tag on repeated calls" + ); +} + +#[test] +fn active_kernel_name_is_known_lowercase_tier() { + // The diagnostic name must be one of the stable lowercase tier + // strings the dashboard parses, and must match whatever tier + // detection resolves to on this host (no `unknown` / empty leak). + const KNOWN: &[&str] = &["scalar", "sse2", "bmi2", "avx2", "vbmi2", "neon", "sve"]; + let name = active_cpu_kernel_name(); + assert!( + KNOWN.contains(&name), + "active kernel name {name:?} is not a recognised tier" + ); + assert_eq!( + name, + name.to_ascii_lowercase(), + "tier name must be lowercase for stable dashboard parsing" + ); +} + +#[test] +fn every_kernel_tag_maps_to_its_lowercase_name() { + // `active_cpu_kernel_name` only exercises whichever arm the running + // CPU resolves to, so map each constructible tag directly to cover + // every branch on this build's feature set. + assert_eq!(CpuKernelTag::Scalar.name(), "scalar"); + #[cfg(all(target_arch = "x86_64", feature = "kernel_sse2"))] + assert_eq!(CpuKernelTag::Sse2.name(), "sse2"); + #[cfg(all(target_arch = "x86_64", feature = "kernel_bmi2"))] + assert_eq!(CpuKernelTag::Bmi2.name(), "bmi2"); + #[cfg(all(target_arch = "x86_64", feature = "kernel_avx2"))] + assert_eq!(CpuKernelTag::Avx2.name(), "avx2"); + #[cfg(all(target_arch = "x86_64", feature = "kernel_vbmi2"))] + assert_eq!(CpuKernelTag::Vbmi2.name(), "vbmi2"); + #[cfg(all(target_arch = "aarch64", feature = "kernel_neon"))] + assert_eq!(CpuKernelTag::Neon.name(), "neon"); + #[cfg(all( + target_arch = "aarch64", + feature = "kernel_sve", + any(feature = "std", target_feature = "sve"), + ))] + assert_eq!(CpuKernelTag::Sve.name(), "sve"); +} + +#[test] +fn active_kernel_name_is_stable_across_calls() { + // Backed by the cached `detect_cpu_kernel`, so repeated calls must + // return the identical static string. + assert_eq!(active_cpu_kernel_name(), active_cpu_kernel_name()); +} diff --git a/zstd/src/decoding/block_decoder.rs b/zstd/src/decoding/block_decoder.rs index 9684f98b4..9d6ba9a06 100644 --- a/zstd/src/decoding/block_decoder.rs +++ b/zstd/src/decoding/block_decoder.rs @@ -488,241 +488,4 @@ impl BlockDecoder { } #[cfg(test)] -mod tests { - //! Coverage for `decode_block_content_from_slice` error branches — - //! truncated / empty source on each block type, plus the - //! `DecoderState::Failed` / `ReadyToDecodeNextHeader` entry-state - //! guards. The happy path is exercised indirectly via the - //! roundtrip tests on `decode_all`; these tests pin the - //! fail-fast behaviour for malformed input. - use super::*; - use crate::blocks::block::{BlockHeader, BlockType}; - use crate::decoding::ringbuffer::RingBuffer; - use crate::decoding::scratch::DecoderScratch; - - fn header(block_type: BlockType, decompressed_size: u32, content_size: u32) -> BlockHeader { - BlockHeader { - last_block: true, - block_type, - decompressed_size, - content_size, - } - } - - fn fresh_workspace() -> DecoderScratch { - DecoderScratch::::new(1 << 20) - } - - fn primed_decoder() -> BlockDecoder { - let mut d = new(); - d.internal_state = DecoderState::ReadyToDecodeNextBody; - d - } - - #[test] - fn rejects_when_internal_state_expects_header() { - // Default state is ReadyToDecodeNextHeader -> calling - // decode_block_content_from_slice on a body must error, - // not silently decode garbage. - let mut d = new(); - let mut ws = fresh_workspace(); - let mut src: &[u8] = &[]; - let h = header(BlockType::RLE, 4, 1); - let err = d - .decode_block_content_from_slice(&h, &mut ws, &mut src) - .expect_err("must err on body before header"); - assert!(matches!( - err, - DecodeBlockContentError::ExpectedHeaderOfPreviousBlock - )); - } - - #[test] - fn rejects_when_internal_state_failed() { - let mut d = new(); - d.internal_state = DecoderState::Failed; - let mut ws = fresh_workspace(); - let mut src: &[u8] = &[0x42]; - let h = header(BlockType::RLE, 4, 1); - let err = d - .decode_block_content_from_slice(&h, &mut ws, &mut src) - .expect_err("must err on Failed state"); - assert!(matches!(err, DecodeBlockContentError::DecoderStateIsFailed)); - } - - #[test] - fn rle_empty_source_errors_not_panics() { - // RLE block needs at least 1 fill byte in source. Empty - // source must return ReadError, not panic on source[0]. - let mut d = primed_decoder(); - let mut ws = fresh_workspace(); - let mut src: &[u8] = &[]; - let h = header(BlockType::RLE, 4, 1); - let err = d - .decode_block_content_from_slice(&h, &mut ws, &mut src) - .expect_err("must err on empty RLE source"); - match &err { - DecodeBlockContentError::ReadError { step, source } => { - assert_eq!(*step, BlockType::RLE); - assert_eq!( - source.kind(), - crate::io::ErrorKind::UnexpectedEof, - "slice-source truncation must report UnexpectedEof to match the streaming path's Read::read_exact behaviour" - ); - } - other => panic!("expected ReadError, got {other:?}"), - } - } - - #[test] - fn raw_truncated_source_errors_not_panics() { - // Raw block header claims 10 decompressed bytes but only - // 3 are available -> ReadError. The pre-split bounds check - // catches this before split_at would panic. - let mut d = primed_decoder(); - let mut ws = fresh_workspace(); - let mut src: &[u8] = &[1, 2, 3]; - let h = header(BlockType::Raw, 10, 10); - let err = d - .decode_block_content_from_slice(&h, &mut ws, &mut src) - .expect_err("must err on truncated raw source"); - match &err { - DecodeBlockContentError::ReadError { step, source } => { - assert_eq!(*step, BlockType::Raw); - assert_eq!(source.kind(), crate::io::ErrorKind::UnexpectedEof); - } - other => panic!("expected ReadError, got {other:?}"), - } - } - - #[test] - fn compressed_truncated_source_errors_not_panics() { - // Compressed block header claims 100 compressed bytes but - // only 8 are available -> ReadError. Pre-split bound check. - let mut d = primed_decoder(); - let mut ws = fresh_workspace(); - let mut src: &[u8] = &[0u8; 8]; - let h = header(BlockType::Compressed, 0, 100); - let err = d - .decode_block_content_from_slice(&h, &mut ws, &mut src) - .expect_err("must err on truncated compressed source"); - match &err { - DecodeBlockContentError::ReadError { step, source } => { - assert_eq!(*step, BlockType::Compressed); - assert_eq!(source.kind(), crate::io::ErrorKind::UnexpectedEof); - } - other => panic!("expected ReadError, got {other:?}"), - } - } - - /// Exercise the BackendOverflow -> DecodeBlockContentError mapping - /// on the direct-decode path. Constructs a fixed-capacity - /// `UserSliceBackend` over a 4-byte slice and feeds it an RLE - /// block whose `decompressed_size` (10) exceeds the slice; the - /// `try_extend_and_fill` failure must surface as - /// `BackendOverflow { step: RLE }`, never panic. - #[test] - fn rle_oversized_against_user_slice_backend_returns_backend_overflow() { - use crate::decoding::decode_buffer::DecodeBuffer; - use crate::decoding::scratch::{DirectScratch, FSEScratch, HuffmanScratch}; - use crate::decoding::user_slice_buf::UserSliceBackend; - - let mut output = [0u8; 4]; - let backend = UserSliceBackend::from_slice(&mut output); - let buffer = DecodeBuffer::from_backend(backend, 1 << 20); - let mut huf = HuffmanScratch::new(); - let mut fse = FSEScratch::new(); - let mut offset_hist = [1u32, 4, 8]; - let mut literals_buffer = alloc::vec::Vec::new(); - let mut block_content_buffer = alloc::vec::Vec::new(); - let mut direct = DirectScratch { - huf: &mut huf, - fse: &mut fse, - offset_hist: &mut offset_hist, - literals_buffer: &mut literals_buffer, - block_content_buffer: &mut block_content_buffer, - buffer, - }; - - let mut d = primed_decoder(); - let payload = [0xCDu8]; - let mut src: &[u8] = &payload; - let h = header(BlockType::RLE, 10, 1); - let err = d - .decode_block_content_from_slice(&h, &mut direct, &mut src) - .expect_err("RLE 10 bytes into 4-byte slice must error"); - match err { - DecodeBlockContentError::BackendOverflow { step } => { - assert_eq!(step, BlockType::RLE); - } - other => panic!("expected BackendOverflow, got {other:?}"), - } - assert_eq!(direct.buffer.len(), 0, "no bytes written on overflow"); - } - - /// Regression test: on BackendOverflow error from the RLE - /// fallible write, the input `*source` must NOT have been - /// advanced. Otherwise `FrameDecoder::bytes_read_counter` - /// accounting is off by one byte on the error path: the caller - /// exits early and the 1-byte advance never gets reflected in - /// the read counter, but the next call would skip past the RLE - /// byte. - #[test] - fn rle_overflow_leaves_source_unadvanced() { - use crate::decoding::decode_buffer::DecodeBuffer; - use crate::decoding::scratch::{DirectScratch, FSEScratch, HuffmanScratch}; - use crate::decoding::user_slice_buf::UserSliceBackend; - - let mut output = [0u8; 4]; - let backend = UserSliceBackend::from_slice(&mut output); - let buffer = DecodeBuffer::from_backend(backend, 1 << 20); - let mut huf = HuffmanScratch::new(); - let mut fse = FSEScratch::new(); - let mut offset_hist = [1u32, 4, 8]; - let mut literals_buffer = alloc::vec::Vec::new(); - let mut block_content_buffer = alloc::vec::Vec::new(); - let mut direct = DirectScratch { - huf: &mut huf, - fse: &mut fse, - offset_hist: &mut offset_hist, - literals_buffer: &mut literals_buffer, - block_content_buffer: &mut block_content_buffer, - buffer, - }; - - let mut d = primed_decoder(); - let payload = [0xCDu8, 0xEE, 0xFF]; - let mut src: &[u8] = &payload; - let h = header(BlockType::RLE, 10, 1); - let _ = d - .decode_block_content_from_slice(&h, &mut direct, &mut src) - .expect_err("RLE 10 bytes into 4-byte slice must error"); - assert_eq!( - src.as_ptr(), - payload.as_ptr(), - "source advanced despite write failure" - ); - assert_eq!( - src.len(), - payload.len(), - "source length changed on error path" - ); - } - - #[test] - fn rle_advances_source_by_one_byte_and_extends_buffer() { - // Happy path on a freshly primed decoder: 1 byte consumed - // from source, N bytes filled into buffer. - let mut d = primed_decoder(); - let mut ws = fresh_workspace(); - let payload = [0xCD, 0xFF, 0xAA]; - let mut src: &[u8] = &payload; - let h = header(BlockType::RLE, 7, 1); - let consumed = d - .decode_block_content_from_slice(&h, &mut ws, &mut src) - .expect("RLE happy path"); - assert_eq!(consumed, 1); - assert_eq!(src, &payload[1..], "1 byte consumed from source"); - assert_eq!(ws.buffer.len(), 7, "buffer extended by decompressed_size"); - } -} +mod tests; diff --git a/zstd/src/decoding/block_decoder/tests.rs b/zstd/src/decoding/block_decoder/tests.rs new file mode 100644 index 000000000..54dbafb29 --- /dev/null +++ b/zstd/src/decoding/block_decoder/tests.rs @@ -0,0 +1,236 @@ +//! Coverage for `decode_block_content_from_slice` error branches — +//! truncated / empty source on each block type, plus the +//! `DecoderState::Failed` / `ReadyToDecodeNextHeader` entry-state +//! guards. The happy path is exercised indirectly via the +//! roundtrip tests on `decode_all`; these tests pin the +//! fail-fast behaviour for malformed input. +use super::*; +use crate::blocks::block::{BlockHeader, BlockType}; +use crate::decoding::ringbuffer::RingBuffer; +use crate::decoding::scratch::DecoderScratch; + +fn header(block_type: BlockType, decompressed_size: u32, content_size: u32) -> BlockHeader { + BlockHeader { + last_block: true, + block_type, + decompressed_size, + content_size, + } +} + +fn fresh_workspace() -> DecoderScratch { + DecoderScratch::::new(1 << 20) +} + +fn primed_decoder() -> BlockDecoder { + let mut d = new(); + d.internal_state = DecoderState::ReadyToDecodeNextBody; + d +} + +#[test] +fn rejects_when_internal_state_expects_header() { + // Default state is ReadyToDecodeNextHeader -> calling + // decode_block_content_from_slice on a body must error, + // not silently decode garbage. + let mut d = new(); + let mut ws = fresh_workspace(); + let mut src: &[u8] = &[]; + let h = header(BlockType::RLE, 4, 1); + let err = d + .decode_block_content_from_slice(&h, &mut ws, &mut src) + .expect_err("must err on body before header"); + assert!(matches!( + err, + DecodeBlockContentError::ExpectedHeaderOfPreviousBlock + )); +} + +#[test] +fn rejects_when_internal_state_failed() { + let mut d = new(); + d.internal_state = DecoderState::Failed; + let mut ws = fresh_workspace(); + let mut src: &[u8] = &[0x42]; + let h = header(BlockType::RLE, 4, 1); + let err = d + .decode_block_content_from_slice(&h, &mut ws, &mut src) + .expect_err("must err on Failed state"); + assert!(matches!(err, DecodeBlockContentError::DecoderStateIsFailed)); +} + +#[test] +fn rle_empty_source_errors_not_panics() { + // RLE block needs at least 1 fill byte in source. Empty + // source must return ReadError, not panic on source[0]. + let mut d = primed_decoder(); + let mut ws = fresh_workspace(); + let mut src: &[u8] = &[]; + let h = header(BlockType::RLE, 4, 1); + let err = d + .decode_block_content_from_slice(&h, &mut ws, &mut src) + .expect_err("must err on empty RLE source"); + match &err { + DecodeBlockContentError::ReadError { step, source } => { + assert_eq!(*step, BlockType::RLE); + assert_eq!( + source.kind(), + crate::io::ErrorKind::UnexpectedEof, + "slice-source truncation must report UnexpectedEof to match the streaming path's Read::read_exact behaviour" + ); + } + other => panic!("expected ReadError, got {other:?}"), + } +} + +#[test] +fn raw_truncated_source_errors_not_panics() { + // Raw block header claims 10 decompressed bytes but only + // 3 are available -> ReadError. The pre-split bounds check + // catches this before split_at would panic. + let mut d = primed_decoder(); + let mut ws = fresh_workspace(); + let mut src: &[u8] = &[1, 2, 3]; + let h = header(BlockType::Raw, 10, 10); + let err = d + .decode_block_content_from_slice(&h, &mut ws, &mut src) + .expect_err("must err on truncated raw source"); + match &err { + DecodeBlockContentError::ReadError { step, source } => { + assert_eq!(*step, BlockType::Raw); + assert_eq!(source.kind(), crate::io::ErrorKind::UnexpectedEof); + } + other => panic!("expected ReadError, got {other:?}"), + } +} + +#[test] +fn compressed_truncated_source_errors_not_panics() { + // Compressed block header claims 100 compressed bytes but + // only 8 are available -> ReadError. Pre-split bound check. + let mut d = primed_decoder(); + let mut ws = fresh_workspace(); + let mut src: &[u8] = &[0u8; 8]; + let h = header(BlockType::Compressed, 0, 100); + let err = d + .decode_block_content_from_slice(&h, &mut ws, &mut src) + .expect_err("must err on truncated compressed source"); + match &err { + DecodeBlockContentError::ReadError { step, source } => { + assert_eq!(*step, BlockType::Compressed); + assert_eq!(source.kind(), crate::io::ErrorKind::UnexpectedEof); + } + other => panic!("expected ReadError, got {other:?}"), + } +} + +/// Exercise the BackendOverflow -> DecodeBlockContentError mapping +/// on the direct-decode path. Constructs a fixed-capacity +/// `UserSliceBackend` over a 4-byte slice and feeds it an RLE +/// block whose `decompressed_size` (10) exceeds the slice; the +/// `try_extend_and_fill` failure must surface as +/// `BackendOverflow { step: RLE }`, never panic. +#[test] +fn rle_oversized_against_user_slice_backend_returns_backend_overflow() { + use crate::decoding::decode_buffer::DecodeBuffer; + use crate::decoding::scratch::{DirectScratch, FSEScratch, HuffmanScratch}; + use crate::decoding::user_slice_buf::UserSliceBackend; + + let mut output = [0u8; 4]; + let backend = UserSliceBackend::from_slice(&mut output); + let buffer = DecodeBuffer::from_backend(backend, 1 << 20); + let mut huf = HuffmanScratch::new(); + let mut fse = FSEScratch::new(); + let mut offset_hist = [1u32, 4, 8]; + let mut literals_buffer = alloc::vec::Vec::new(); + let mut block_content_buffer = alloc::vec::Vec::new(); + let mut direct = DirectScratch { + huf: &mut huf, + fse: &mut fse, + offset_hist: &mut offset_hist, + literals_buffer: &mut literals_buffer, + block_content_buffer: &mut block_content_buffer, + buffer, + }; + + let mut d = primed_decoder(); + let payload = [0xCDu8]; + let mut src: &[u8] = &payload; + let h = header(BlockType::RLE, 10, 1); + let err = d + .decode_block_content_from_slice(&h, &mut direct, &mut src) + .expect_err("RLE 10 bytes into 4-byte slice must error"); + match err { + DecodeBlockContentError::BackendOverflow { step } => { + assert_eq!(step, BlockType::RLE); + } + other => panic!("expected BackendOverflow, got {other:?}"), + } + assert_eq!(direct.buffer.len(), 0, "no bytes written on overflow"); +} + +/// Regression test: on BackendOverflow error from the RLE +/// fallible write, the input `*source` must NOT have been +/// advanced. Otherwise `FrameDecoder::bytes_read_counter` +/// accounting is off by one byte on the error path: the caller +/// exits early and the 1-byte advance never gets reflected in +/// the read counter, but the next call would skip past the RLE +/// byte. +#[test] +fn rle_overflow_leaves_source_unadvanced() { + use crate::decoding::decode_buffer::DecodeBuffer; + use crate::decoding::scratch::{DirectScratch, FSEScratch, HuffmanScratch}; + use crate::decoding::user_slice_buf::UserSliceBackend; + + let mut output = [0u8; 4]; + let backend = UserSliceBackend::from_slice(&mut output); + let buffer = DecodeBuffer::from_backend(backend, 1 << 20); + let mut huf = HuffmanScratch::new(); + let mut fse = FSEScratch::new(); + let mut offset_hist = [1u32, 4, 8]; + let mut literals_buffer = alloc::vec::Vec::new(); + let mut block_content_buffer = alloc::vec::Vec::new(); + let mut direct = DirectScratch { + huf: &mut huf, + fse: &mut fse, + offset_hist: &mut offset_hist, + literals_buffer: &mut literals_buffer, + block_content_buffer: &mut block_content_buffer, + buffer, + }; + + let mut d = primed_decoder(); + let payload = [0xCDu8, 0xEE, 0xFF]; + let mut src: &[u8] = &payload; + let h = header(BlockType::RLE, 10, 1); + let _ = d + .decode_block_content_from_slice(&h, &mut direct, &mut src) + .expect_err("RLE 10 bytes into 4-byte slice must error"); + assert_eq!( + src.as_ptr(), + payload.as_ptr(), + "source advanced despite write failure" + ); + assert_eq!( + src.len(), + payload.len(), + "source length changed on error path" + ); +} + +#[test] +fn rle_advances_source_by_one_byte_and_extends_buffer() { + // Happy path on a freshly primed decoder: 1 byte consumed + // from source, N bytes filled into buffer. + let mut d = primed_decoder(); + let mut ws = fresh_workspace(); + let payload = [0xCD, 0xFF, 0xAA]; + let mut src: &[u8] = &payload; + let h = header(BlockType::RLE, 7, 1); + let consumed = d + .decode_block_content_from_slice(&h, &mut ws, &mut src) + .expect("RLE happy path"); + assert_eq!(consumed, 1); + assert_eq!(src, &payload[1..], "1 byte consumed from source"); + assert_eq!(ws.buffer.len(), 7, "buffer extended by decompressed_size"); +} diff --git a/zstd/src/decoding/buffer_backend.rs b/zstd/src/decoding/buffer_backend.rs index b01247e01..d083a2c12 100644 --- a/zstd/src/decoding/buffer_backend.rs +++ b/zstd/src/decoding/buffer_backend.rs @@ -627,71 +627,4 @@ impl core::fmt::Display for BackendOverflow { } #[cfg(test)] -mod tests { - //! Coverage for the default `try_extend_from_within` impl on - //! growable backends (`FlatBuf` / `RingBuffer` use it unchanged; - //! only `UserSliceBackend` overrides it). Tests exercise the - //! three reachable arms: success, `start + len` arithmetic - //! overflow, and source-range violation. Plus the `Display` impl - //! that the decoder formats `BackendOverflow` through. - use super::*; - use crate::decoding::flat_buf::FlatBuf; - - #[test] - fn default_try_extend_from_within_happy_path_copies_from_live_region() { - // FlatBuf uses the default impl — grow on demand, no - // capacity overshoot path on a growable backend. - let mut b = FlatBuf::with_capacity(32); - b.extend(&[1u8, 2, 3, 4, 5]); - assert_eq!(b.len(), 5); - // Copy `[1, 2, 3]` from the head into the tail. - b.try_extend_from_within(0, 3).expect("happy path"); - assert_eq!(b.len(), 8); - let (s, t) = b.as_slices(); - assert_eq!(s, &[1u8, 2, 3, 4, 5, 1, 2, 3]); - assert!(t.is_empty(), "FlatBuf does not wrap"); - } - - #[test] - fn default_try_extend_from_within_arithmetic_overflow_returns_err() { - // `start.checked_add(len)` wraps `usize` only on adversarial - // inputs (`usize::MAX`-ish values). The default impl must - // surface that as `Err(BackendOverflow)` without touching the - // backend. - let mut b = FlatBuf::with_capacity(32); - b.extend(&[1u8, 2, 3, 4]); - let live_before = b.len(); - let err = b - .try_extend_from_within(usize::MAX, 1) - .expect_err("usize wrap must Err"); - assert_eq!(err.requested, 1); - assert_eq!(b.len(), live_before, "backend untouched on Err"); - } - - #[test] - fn default_try_extend_from_within_source_past_live_region_returns_err() { - // `start + len > self.len()` reads from outside the live - // region. The default impl must Err without growing or - // writing. - let mut b = FlatBuf::with_capacity(32); - b.extend(&[10u8, 20, 30]); - let err = b - .try_extend_from_within(2, 10) - .expect_err("start+len past live region must Err"); - assert_eq!(err.requested, 10); - assert_eq!(b.len(), 3, "backend untouched on Err"); - } - - #[test] - fn backend_overflow_display_renders_diagnostic_fields() { - let err = BackendOverflow { - tail: 5, - requested: 7, - capacity: 10, - }; - let rendered = alloc::format!("{}", err); - assert!(rendered.contains("tail=5"), "tail field rendered"); - assert!(rendered.contains("requested=7"), "requested field rendered"); - assert!(rendered.contains("capacity=10"), "capacity field rendered"); - } -} +mod tests; diff --git a/zstd/src/decoding/buffer_backend/tests.rs b/zstd/src/decoding/buffer_backend/tests.rs new file mode 100644 index 000000000..5e7516f60 --- /dev/null +++ b/zstd/src/decoding/buffer_backend/tests.rs @@ -0,0 +1,66 @@ +//! Coverage for the default `try_extend_from_within` impl on +//! growable backends (`FlatBuf` / `RingBuffer` use it unchanged; +//! only `UserSliceBackend` overrides it). Tests exercise the +//! three reachable arms: success, `start + len` arithmetic +//! overflow, and source-range violation. Plus the `Display` impl +//! that the decoder formats `BackendOverflow` through. +use super::*; +use crate::decoding::flat_buf::FlatBuf; + +#[test] +fn default_try_extend_from_within_happy_path_copies_from_live_region() { + // FlatBuf uses the default impl — grow on demand, no + // capacity overshoot path on a growable backend. + let mut b = FlatBuf::with_capacity(32); + b.extend(&[1u8, 2, 3, 4, 5]); + assert_eq!(b.len(), 5); + // Copy `[1, 2, 3]` from the head into the tail. + b.try_extend_from_within(0, 3).expect("happy path"); + assert_eq!(b.len(), 8); + let (s, t) = b.as_slices(); + assert_eq!(s, &[1u8, 2, 3, 4, 5, 1, 2, 3]); + assert!(t.is_empty(), "FlatBuf does not wrap"); +} + +#[test] +fn default_try_extend_from_within_arithmetic_overflow_returns_err() { + // `start.checked_add(len)` wraps `usize` only on adversarial + // inputs (`usize::MAX`-ish values). The default impl must + // surface that as `Err(BackendOverflow)` without touching the + // backend. + let mut b = FlatBuf::with_capacity(32); + b.extend(&[1u8, 2, 3, 4]); + let live_before = b.len(); + let err = b + .try_extend_from_within(usize::MAX, 1) + .expect_err("usize wrap must Err"); + assert_eq!(err.requested, 1); + assert_eq!(b.len(), live_before, "backend untouched on Err"); +} + +#[test] +fn default_try_extend_from_within_source_past_live_region_returns_err() { + // `start + len > self.len()` reads from outside the live + // region. The default impl must Err without growing or + // writing. + let mut b = FlatBuf::with_capacity(32); + b.extend(&[10u8, 20, 30]); + let err = b + .try_extend_from_within(2, 10) + .expect_err("start+len past live region must Err"); + assert_eq!(err.requested, 10); + assert_eq!(b.len(), 3, "backend untouched on Err"); +} + +#[test] +fn backend_overflow_display_renders_diagnostic_fields() { + let err = BackendOverflow { + tail: 5, + requested: 7, + capacity: 10, + }; + let rendered = alloc::format!("{}", err); + assert!(rendered.contains("tail=5"), "tail field rendered"); + assert!(rendered.contains("requested=7"), "requested field rendered"); + assert!(rendered.contains("capacity=10"), "capacity field rendered"); +} diff --git a/zstd/src/decoding/decode_buffer.rs b/zstd/src/decoding/decode_buffer.rs index c9ed9277f..af5124497 100644 --- a/zstd/src/decoding/decode_buffer.rs +++ b/zstd/src/decoding/decode_buffer.rs @@ -1124,593 +1124,4 @@ fn use_branchless_wildcopy() -> bool { } #[cfg(test)] -mod tests { - use super::{DecodeBuffer, RingBuffer}; - use crate::decoding::buffer_backend::BufferBackend; - use crate::io::{Error, ErrorKind, Write}; - - extern crate std; - use alloc::vec; - use alloc::vec::Vec; - - #[test] - fn dict_offsets_rejected_after_direct_path_window_drop() { - // The direct decode path writes through the inline executors - // (which skip `total_output_counter`) and bounds the visible - // buffer with `drop_to_window_size()` between blocks. Once - // cumulative output exceeds the window, dictionary-backed - // offsets are out of reach per the spec; the reachability gate - // must not reopen just because the visible length was capped - // back to `window_size`. - use crate::decoding::dictionary::Dictionary; - use crate::decoding::user_slice_buf::UserSliceBackend; - - let mut out = vec![0u8; 300]; - let backend = UserSliceBackend::from_slice(out.as_mut_slice()); - let mut buf = DecodeBuffer::from_backend(backend, 100); - let dict = Dictionary::from_raw_content(7, vec![0xAB; 64]).expect("raw-content dictionary"); - buf.set_dict(dict.into_handle()); - - // Mimic the inline executor: produce 250 bytes without touching - // `total_output_counter`, exceeding the 100-byte window. - BufferBackend::extend(&mut buf.buffer, &[1u8; 250]); - buf.drop_to_window_size(); - assert_eq!(buf.len(), 100, "visible buffer capped to the window"); - - // offset 110 > len 100 reaches 10 bytes into the dictionary; - // cumulative output (250) already exceeds the window (100), so - // this must be rejected, not served from the dictionary. - let result = buf.repeat_from_dict(110, 5); - assert!( - result.is_err(), - "dict-backed offset must be unreachable once output exceeded the window, got {result:?}" - ); - } - - #[test] - fn from_backend_clears_prepopulated_backend() { - // Regression for the round-8 review fix: `from_backend` must - // normalise a caller-supplied backend so the logical counters - // (total_output_counter=0, dict_content=empty) stay consistent - // with the physical buffer contents. A future caller that - // wires up a non-fresh backend should not silently leak stale - // bytes into the new decode. - let mut backend = RingBuffer::new(); - BufferBackend::extend(&mut backend, b"stale"); - assert!(BufferBackend::len(&backend) > 0); - - let mut buf = DecodeBuffer::::from_backend(backend, 1024); - assert_eq!(buf.len(), 0, "from_backend must clear pre-populated bytes"); - - buf.push(b"ok"); - assert_eq!(buf.drain(), b"ok"); - } - - #[test] - fn test_repeat_doubling_matches_reference_across_offsets() { - // Naive reference: dst[i] = dst[i - offset]. The exponential-doubling - // `repeat_in_chunks` must produce byte-identical output for every - // offset/length, including lengths that straddle the 32-byte SIMD - // overshoot boundary and large lengths that force many doublings. - fn reference_repeat(prefix: &[u8], offset: usize, match_length: usize) -> Vec { - let mut out = prefix.to_vec(); - for _ in 0..match_length { - let src = out.len() - offset; - out.push(out[src]); - } - out - } - - let prefix: Vec = (0..200u32) - .map(|i| i.wrapping_mul(31).wrapping_add(7) as u8) - .collect(); - // Offsets cover every `repeat_overlapping` arm: <8 period-tiled, - // 8..15 and >=16 doubling, exact powers of two, and the - // non-overlapping `offset == match_length` edge. - let offsets = [1usize, 2, 3, 5, 7, 8, 13, 16, 31, 32, 63, 64, 100, 200]; - let lengths = [ - 1usize, 2, 7, 8, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 200, 511, 1000, 5000, - ]; - for &offset in &offsets { - for &match_length in &lengths { - let prefix_slice = &prefix[..offset.max(1)]; - let mut buffer = DecodeBuffer::::new(usize::MAX); - buffer.push(prefix_slice); - buffer.repeat(offset, match_length).unwrap(); - let expected = reference_repeat(prefix_slice, offset, match_length); - let mut got: Vec = Vec::new(); - buffer.drain_to_writer(&mut got).unwrap(); - assert_eq!( - got.as_slice(), - expected.as_slice(), - "mismatch at offset={offset} match_length={match_length}", - ); - } - } - } - - #[test] - fn checkpoint_restore_undoes_pushes() { - // Regression test for the fused-decode transactional contract: - // when the post-loop bitstream validation fails, the fused - // sequence executor must restore buffer state to the moment - // before the first per-iter side-effect. This exercises the - // primitive that supports that rollback. - let mut buf = DecodeBuffer::::new(1024); - // Mirror the fused sequence executor: reserve upfront so no - // RingBuffer reallocation happens between checkpoint and restore - // (restore_checkpoint requires a stable underlying allocation). - buf.reserve_exact(64); - buf.push(&[1, 2, 3]); - let cp = buf.checkpoint(); - buf.push(&[4, 5, 6, 7]); - assert_eq!(buf.len(), 7); - assert!( - buf.try_restore_checkpoint(cp), - "no realloc → restore must succeed" - ); - assert_eq!(buf.len(), 3, "len must reflect the checkpoint"); - - // After restore, fresh writes must land contiguously where the - // first push left off (no stale tail bytes leaking through). - buf.push(&[0xAA, 0xBB]); - assert_eq!(buf.len(), 5); - // Drain & verify content. - let mut drained: Vec = Vec::new(); - buf.drain_to_writer(&mut drained).unwrap(); - assert_eq!(drained, alloc::vec![1, 2, 3, 0xAA, 0xBB]); - } - - #[test] - fn restore_checkpoint_after_realloc_returns_false() { - // Regression test: try_restore_checkpoint() must detect an - // intervening RingBuffer reallocation (which compacts the data - // layout and invalidates the captured tail) and refuse to - // restore, returning false instead of corrupting state or - // panicking. Triggered by a malformed zstd block whose sequence - // section decodes past MAX_BLOCK_SIZE; surfacing the failure to - // the caller as a normal decode Err is required behaviour — - // both silent wrong output AND an unconditional panic on - // untrusted input are unacceptable. libFuzzer artifact - // crash-bfb3bc55... originally exercised this branch via the - // panic guard added in the previous round. - let mut buf = DecodeBuffer::::new(64); - buf.push(&[0; 16]); - let cp = buf.checkpoint(); - // Force a reallocation. RingBuffer grows by powers of two and - // 4 MiB is well above the initial 64-byte starting capacity, so - // reserve() must hit reserve_amortized(). - buf.reserve_exact(4 * 1024 * 1024); - buf.push(&[0; 16]); - assert!( - !buf.try_restore_checkpoint(cp), - "realloc happened → rollback must be refused" - ); - // No state mutation when the restore is refused. - assert_eq!(buf.len(), 32); - } - - #[test] - fn short_writer() { - struct ShortWriter { - buf: Vec, - write_len: usize, - } - - impl Write for ShortWriter { - fn write(&mut self, buf: &[u8]) -> std::result::Result { - if buf.len() > self.write_len { - self.buf.extend_from_slice(&buf[..self.write_len]); - Ok(self.write_len) - } else { - self.buf.extend_from_slice(buf); - Ok(buf.len()) - } - } - - fn flush(&mut self) -> std::result::Result<(), Error> { - Ok(()) - } - } - - let mut short_writer = ShortWriter { - buf: vec![], - write_len: 10, - }; - - let mut decode_buf = DecodeBuffer::::new(100); - decode_buf.push(b"0123456789"); - decode_buf.repeat(10, 90).unwrap(); - let repeats = 1000; - for _ in 0..repeats { - assert_eq!(decode_buf.len(), 100); - decode_buf.repeat(10, 50).unwrap(); - assert_eq!(decode_buf.len(), 150); - decode_buf - .drain_to_window_size_writer(&mut short_writer) - .unwrap(); - assert_eq!(decode_buf.len(), 100); - } - - assert_eq!(short_writer.buf.len(), repeats * 50); - decode_buf.drain_to_writer(&mut short_writer).unwrap(); - assert_eq!(short_writer.buf.len(), repeats * 50 + 100); - } - - #[test] - fn wouldblock_writer() { - struct WouldblockWriter { - buf: Vec, - last_blocked: usize, - block_every: usize, - } - - impl Write for WouldblockWriter { - fn write(&mut self, buf: &[u8]) -> std::result::Result { - if self.last_blocked < self.block_every { - self.buf.extend_from_slice(buf); - self.last_blocked += 1; - Ok(buf.len()) - } else { - self.last_blocked = 0; - Err(Error::from(ErrorKind::WouldBlock)) - } - } - - fn flush(&mut self) -> std::result::Result<(), Error> { - Ok(()) - } - } - - let mut short_writer = WouldblockWriter { - buf: vec![], - last_blocked: 0, - block_every: 5, - }; - - let mut decode_buf = DecodeBuffer::::new(100); - decode_buf.push(b"0123456789"); - decode_buf.repeat(10, 90).unwrap(); - let repeats = 1000; - for _ in 0..repeats { - assert_eq!(decode_buf.len(), 100); - decode_buf.repeat(10, 50).unwrap(); - assert_eq!(decode_buf.len(), 150); - loop { - match decode_buf.drain_to_window_size_writer(&mut short_writer) { - Ok(written) => { - if written == 0 { - break; - } - } - Err(e) => { - if e.kind() == ErrorKind::WouldBlock { - continue; - } else { - panic!("Unexpected error {:?}", e); - } - } - } - } - assert_eq!(decode_buf.len(), 100); - } - - assert_eq!(short_writer.buf.len(), repeats * 50); - loop { - match decode_buf.drain_to_writer(&mut short_writer) { - Ok(written) => { - if written == 0 { - break; - } - } - Err(e) => { - if e.kind() == ErrorKind::WouldBlock { - continue; - } else { - panic!("Unexpected error {:?}", e); - } - } - } - } - assert_eq!(short_writer.buf.len(), repeats * 50 + 100); - } - - #[test] - fn repeat_overlap_fast_paths_match_reference_behavior() { - let seed = b"0123456789abcdef0123456789abcdef"; - let cases = [ - (16usize, 16usize), // non-overlapping boundary - (16usize, 211usize), - (8usize, 173usize), - (7usize, 149usize), - (3usize, 160usize), - (1usize, 255usize), - ]; - - for (offset, match_len) in cases { - let mut decode_buf = DecodeBuffer::::new(4 * 1024); - decode_buf.push(seed); - decode_buf.repeat(offset, match_len).unwrap(); - let got = decode_buf.drain(); - let expected = expected_match_expansion(seed, offset, match_len); - assert_eq!(got, expected, "offset={offset}, match_len={match_len}"); - } - } - - #[test] - fn repeat_zero_offset_returns_error() { - let mut decode_buf = DecodeBuffer::::new(1024); - decode_buf.push(b"abcdef"); - let err = decode_buf.repeat(0, 5).unwrap_err(); - assert!(matches!( - err, - crate::decoding::errors::DecodeBufferError::ZeroOffset - )); - } - - #[test] - fn repeat_rejects_output_past_block_ceiling() { - // A single zstd block decompresses to at most MAX_BLOCK_SIZE. The - // per-block ceiling is enforced on the growable `RingBuffer`'s cold - // growth path: `set_block_output_ceiling` lowers `max_capacity`, so a - // `repeat` whose match would have to grow the buffer past the ceiling - // fails its `try_reserve` instead of growing the ring unbounded. - // Without this guard the over-long match drove `try_reserve` to - // ~0.5–2 GiB across many over-producing sequences (artifact - // `oom-66db61d9…`, fuzz `decode` target) before any post-block check - // ran — a decompression-bomb OOM. The match needing growth surfaces - // as `OutputBufferOverflow` (the backend's structured reject). - let mut decode_buf = DecodeBuffer::::new(4 * 1024); - decode_buf.push(b"abcdef"); // len = 6 - decode_buf.set_block_output_ceiling(8); // max_capacity = 6 + 8 = 14 - let err = decode_buf.repeat(4, 16).unwrap_err(); // 6 + 16 = 22 > 14 - assert!( - matches!( - err, - crate::decoding::errors::DecodeBufferError::OutputBufferOverflow { .. } - ), - "over-producing match must be rejected, got {err:?}" - ); - } - - #[test] - fn repeat_within_block_ceiling_still_succeeds() { - // A match that keeps the block's output at/below the armed ceiling - // must NOT be rejected — the guard fires only on growth past the - // ceiling, never on legitimate in-bounds output. - let mut decode_buf = DecodeBuffer::::new(4 * 1024); - decode_buf.push(b"abcdef"); // len = 6 - decode_buf.set_block_output_ceiling(8); // max_capacity = 14 - decode_buf - .repeat(4, 8) - .expect("6 + 8 = 14 == ceiling is allowed"); - assert_eq!(decode_buf.len(), 14); - } - - #[test] - fn repeat_from_dict_rejects_output_past_block_ceiling() { - // A match satisfied (fully or partially) from the dictionary appends - // `dict_slice` directly via `buffer.extend`, bypassing the inline - // push/repeat guard. The per-block bomb ceiling must still bound it, so - // a dict-backed over-producing match returns `OutputBufferOverflow` - // instead of growing the buffer toward OOM. - let dict = || { - crate::decoding::dictionary::DictionaryHandle::from_dictionary( - crate::decoding::dictionary::Dictionary::from_raw_content( - 1, - alloc::vec![0xABu8; 256], - ) - .unwrap(), - ) - }; - - // Fully-dictionary match: empty buffer, offset reaches into the dict. - let mut full = DecodeBuffer::::new(4 * 1024); - full.set_dict(dict()); - full.set_block_output_ceiling(8); // max_capacity = 0 + 8 = 8 - let err = full.repeat(200, 100).unwrap_err(); // 0 + 100 > 8, all from dict - assert!( - matches!( - err, - crate::decoding::errors::DecodeBufferError::OutputBufferOverflow { .. } - ), - "fully-dictionary over-producing match must be rejected, got {err:?}" - ); - - // Mixed match: part from dict, remainder from buffer history. - let mut mixed = DecodeBuffer::::new(4 * 1024); - mixed.set_dict(dict()); - mixed.push(b"abcd"); // len = 4 - mixed.set_block_output_ceiling(8); // max_capacity = 4 + 8 = 12 - let err = mixed.repeat(10, 100).unwrap_err(); // 6 from dict + rest, 4 + 100 > 12 - assert!( - matches!( - err, - crate::decoding::errors::DecodeBufferError::OutputBufferOverflow { .. } - ), - "mixed dict+buffer over-producing match must be rejected, got {err:?}" - ); - } - - #[test] - fn repeat_from_dict_full_copy_updates_total_output_counter() { - let mut decode_buf = DecodeBuffer::::new(1); - decode_buf.set_dict( - crate::decoding::dictionary::DictionaryHandle::from_dictionary( - crate::decoding::dictionary::Dictionary::from_raw_content( - 1, - b"0123456789".to_vec(), - ) - .unwrap(), - ), - ); - - decode_buf.repeat(10, 2).unwrap(); - let err = decode_buf.repeat(10, 1).unwrap_err(); - assert!(matches!( - err, - crate::decoding::errors::DecodeBufferError::OffsetTooBig { .. } - )); - } - - #[test] - fn repeat_overlap_fast_paths_match_reference_behavior_with_wrapped_ringbuffer() { - let window = 32usize; - let seed = b"0123456789abcdef0123456789abcdef"; - let mut decode_buf = DecodeBuffer::::new(window); - let mut model = Vec::new(); - - decode_buf.push(seed); - model_push(&mut model, seed); - decode_buf.repeat(16, 16).unwrap(); - model_repeat(&mut model, 16, 16); - - let drained = decode_buf.drain_to_window_size().unwrap(); - let model_drained = model_drain_to_window(&mut model, window); - assert_eq!(drained, model_drained); - - let cases = [(3usize, 97usize), (16usize, 64usize), (7usize, 73usize)]; - for (offset, match_len) in cases { - decode_buf.repeat(offset, match_len).unwrap(); - model_repeat(&mut model, offset, match_len); - - if let Some(got) = decode_buf.drain_to_window_size() { - let expected = model_drain_to_window(&mut model, window); - assert_eq!(got, expected, "offset={offset}, match_len={match_len}"); - } - } - - assert_eq!(decode_buf.drain(), model); - } - - fn expected_match_expansion(seed: &[u8], offset: usize, match_len: usize) -> Vec { - let mut out = seed.to_vec(); - let start = out.len() - offset; - for i in 0..match_len { - let byte = out[start + i]; - out.push(byte); - } - out - } - - fn model_push(model: &mut Vec, bytes: &[u8]) { - model.extend_from_slice(bytes); - } - - fn model_repeat(model: &mut Vec, offset: usize, match_len: usize) { - let start = model.len() - offset; - for i in 0..match_len { - let byte = model[start + i]; - model.push(byte); - } - } - - fn model_drain_to_window(model: &mut Vec, window: usize) -> Vec { - if model.len() <= window { - return Vec::new(); - } - let drain_len = model.len() - window; - model.drain(0..drain_len).collect() - } - - /// Drive `DecodeBuffer::repeat` through the short-offset path and - /// compare against the canonical `output[i] = base[i % offset]` - /// reference, covering offsets that hit both the SIMD-16 fast path - /// (1, 2, 4) and the 8-byte phase-pattern path (3, 5, 6, 7). - /// - /// Regression guard for the SIMD-16 specialisation: when `period - /// divides 16` (offset ∈ {1,2,4}), the inner loop emits 16-byte - /// chunks via a pre-built `[u8; 16]` instead of 8-byte phase - /// patterns. Tail lengths span both `match_length % 16 == 0` and - /// non-zero remainders so the tail-extend codepath is also - /// exercised. - #[test] - fn repeat_short_offset_matches_canonical_for_all_offsets_and_lengths() { - for offset in 1usize..=7 { - let mut base = [0u8; 7]; - for (i, slot) in base.iter_mut().enumerate().take(offset) { - *slot = b'A' + (i as u8); - } - for &match_length in &[ - 1usize, 2, 3, 4, 5, 6, 7, 8, 9, 15, 16, 17, 23, 24, 25, 31, 32, 33, 47, 48, 49, 64, - 127, 128, 4096, - ] { - let mut buf = DecodeBuffer::::new(8192); - buf.push(&base[..offset]); - buf.repeat(offset, match_length).unwrap_or_else(|e| { - panic!("repeat failed for offset={offset} match_length={match_length}: {e:?}") - }); - - let actual = buf.drain(); - let mut expected = Vec::with_capacity(offset + match_length); - expected.extend_from_slice(&base[..offset]); - for i in 0..match_length { - expected.push(base[i % offset]); - } - assert_eq!( - actual, expected, - "mismatch at offset={offset} match_length={match_length}", - ); - } - } - } - - #[test] - fn prefetch_lookahead_in_range_does_not_panic() { - // Plain in-range lookup: start_idx well within `buffer.len()`. - // The helper should issue prefetch hints and return cleanly. - // Prefetch hints are unobservable from Rust — the assertion is - // simply that the call completes without panic / UB. - let mut buf = DecodeBuffer::::new(1024); - buf.reserve_exact(512); - buf.push(&[0xAA; 256]); - buf.prefetch_lookahead_match_source(0); - buf.prefetch_lookahead_match_source(128); - buf.prefetch_lookahead_match_source(buf.len() - 1); - } - - #[test] - fn prefetch_lookahead_out_of_range_returns_without_panic() { - // Wrap-derived garbage / dictionary-sourced match / intra-block - // self-overlap all produce `start_idx >= buffer.len()` here. - // The helper must early-return (bound check) and never touch a - // slice past the live region. - let mut buf = DecodeBuffer::::new(1024); - buf.reserve_exact(64); - buf.push(&[0x55; 32]); - buf.prefetch_lookahead_match_source(buf.len()); - buf.prefetch_lookahead_match_source(buf.len() + 1); - buf.prefetch_lookahead_match_source(usize::MAX); - // Empty buffer — every start_idx is out-of-range. - let empty: DecodeBuffer = DecodeBuffer::new(1024); - empty.prefetch_lookahead_match_source(0); - empty.prefetch_lookahead_match_source(7); - } - - #[test] - fn prefetch_lookahead_at_wrap_boundary() { - // Force the RingBuffer into a wrapped layout where - // `as_slices()` returns two non-empty halves: push, drain past - // window, push again so the write cursor wraps. Then exercise - // start_idx values at the boundary (last byte of s1, first - // byte of s2, short s1 tail < CACHE_LINE) so the - // `prefetch_first_line_l1` fallback path is touched too. - let mut buf = DecodeBuffer::::new(256); - // Fill with two passes so the underlying ringbuffer wraps. - let payload = [0xCD_u8; 320]; - buf.push(&payload); - // Drain to free read cursor capacity (write side can then wrap). - let _ = buf.drain_to_window_size(); - buf.push(&payload); - // Probe a handful of indices inside and across the wrap. - let n = buf.len(); - if n > 0 { - buf.prefetch_lookahead_match_source(0); - buf.prefetch_lookahead_match_source(n / 2); - buf.prefetch_lookahead_match_source(n - 1); - // Out-of-range probe to exercise the early-return path on - // a wrapped buffer. - buf.prefetch_lookahead_match_source(n); - } - } -} +mod tests; diff --git a/zstd/src/decoding/decode_buffer/tests.rs b/zstd/src/decoding/decode_buffer/tests.rs new file mode 100644 index 000000000..255c670d9 --- /dev/null +++ b/zstd/src/decoding/decode_buffer/tests.rs @@ -0,0 +1,582 @@ +use super::{DecodeBuffer, RingBuffer}; +use crate::decoding::buffer_backend::BufferBackend; +use crate::io::{Error, ErrorKind, Write}; + +extern crate std; +use alloc::vec; +use alloc::vec::Vec; + +#[test] +fn dict_offsets_rejected_after_direct_path_window_drop() { + // The direct decode path writes through the inline executors + // (which skip `total_output_counter`) and bounds the visible + // buffer with `drop_to_window_size()` between blocks. Once + // cumulative output exceeds the window, dictionary-backed + // offsets are out of reach per the spec; the reachability gate + // must not reopen just because the visible length was capped + // back to `window_size`. + use crate::decoding::dictionary::Dictionary; + use crate::decoding::user_slice_buf::UserSliceBackend; + + let mut out = vec![0u8; 300]; + let backend = UserSliceBackend::from_slice(out.as_mut_slice()); + let mut buf = DecodeBuffer::from_backend(backend, 100); + let dict = Dictionary::from_raw_content(7, vec![0xAB; 64]).expect("raw-content dictionary"); + buf.set_dict(dict.into_handle()); + + // Mimic the inline executor: produce 250 bytes without touching + // `total_output_counter`, exceeding the 100-byte window. + BufferBackend::extend(&mut buf.buffer, &[1u8; 250]); + buf.drop_to_window_size(); + assert_eq!(buf.len(), 100, "visible buffer capped to the window"); + + // offset 110 > len 100 reaches 10 bytes into the dictionary; + // cumulative output (250) already exceeds the window (100), so + // this must be rejected, not served from the dictionary. + let result = buf.repeat_from_dict(110, 5); + assert!( + result.is_err(), + "dict-backed offset must be unreachable once output exceeded the window, got {result:?}" + ); +} + +#[test] +fn from_backend_clears_prepopulated_backend() { + // Regression for the round-8 review fix: `from_backend` must + // normalise a caller-supplied backend so the logical counters + // (total_output_counter=0, dict_content=empty) stay consistent + // with the physical buffer contents. A future caller that + // wires up a non-fresh backend should not silently leak stale + // bytes into the new decode. + let mut backend = RingBuffer::new(); + BufferBackend::extend(&mut backend, b"stale"); + assert!(BufferBackend::len(&backend) > 0); + + let mut buf = DecodeBuffer::::from_backend(backend, 1024); + assert_eq!(buf.len(), 0, "from_backend must clear pre-populated bytes"); + + buf.push(b"ok"); + assert_eq!(buf.drain(), b"ok"); +} + +#[test] +fn test_repeat_doubling_matches_reference_across_offsets() { + // Naive reference: dst[i] = dst[i - offset]. The exponential-doubling + // `repeat_in_chunks` must produce byte-identical output for every + // offset/length, including lengths that straddle the 32-byte SIMD + // overshoot boundary and large lengths that force many doublings. + fn reference_repeat(prefix: &[u8], offset: usize, match_length: usize) -> Vec { + let mut out = prefix.to_vec(); + for _ in 0..match_length { + let src = out.len() - offset; + out.push(out[src]); + } + out + } + + let prefix: Vec = (0..200u32) + .map(|i| i.wrapping_mul(31).wrapping_add(7) as u8) + .collect(); + // Offsets cover every `repeat_overlapping` arm: <8 period-tiled, + // 8..15 and >=16 doubling, exact powers of two, and the + // non-overlapping `offset == match_length` edge. + let offsets = [1usize, 2, 3, 5, 7, 8, 13, 16, 31, 32, 63, 64, 100, 200]; + let lengths = [ + 1usize, 2, 7, 8, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 200, 511, 1000, 5000, + ]; + for &offset in &offsets { + for &match_length in &lengths { + let prefix_slice = &prefix[..offset.max(1)]; + let mut buffer = DecodeBuffer::::new(usize::MAX); + buffer.push(prefix_slice); + buffer.repeat(offset, match_length).unwrap(); + let expected = reference_repeat(prefix_slice, offset, match_length); + let mut got: Vec = Vec::new(); + buffer.drain_to_writer(&mut got).unwrap(); + assert_eq!( + got.as_slice(), + expected.as_slice(), + "mismatch at offset={offset} match_length={match_length}", + ); + } + } +} + +#[test] +fn checkpoint_restore_undoes_pushes() { + // Regression test for the fused-decode transactional contract: + // when the post-loop bitstream validation fails, the fused + // sequence executor must restore buffer state to the moment + // before the first per-iter side-effect. This exercises the + // primitive that supports that rollback. + let mut buf = DecodeBuffer::::new(1024); + // Mirror the fused sequence executor: reserve upfront so no + // RingBuffer reallocation happens between checkpoint and restore + // (restore_checkpoint requires a stable underlying allocation). + buf.reserve_exact(64); + buf.push(&[1, 2, 3]); + let cp = buf.checkpoint(); + buf.push(&[4, 5, 6, 7]); + assert_eq!(buf.len(), 7); + assert!( + buf.try_restore_checkpoint(cp), + "no realloc → restore must succeed" + ); + assert_eq!(buf.len(), 3, "len must reflect the checkpoint"); + + // After restore, fresh writes must land contiguously where the + // first push left off (no stale tail bytes leaking through). + buf.push(&[0xAA, 0xBB]); + assert_eq!(buf.len(), 5); + // Drain & verify content. + let mut drained: Vec = Vec::new(); + buf.drain_to_writer(&mut drained).unwrap(); + assert_eq!(drained, alloc::vec![1, 2, 3, 0xAA, 0xBB]); +} + +#[test] +fn restore_checkpoint_after_realloc_returns_false() { + // Regression test: try_restore_checkpoint() must detect an + // intervening RingBuffer reallocation (which compacts the data + // layout and invalidates the captured tail) and refuse to + // restore, returning false instead of corrupting state or + // panicking. Triggered by a malformed zstd block whose sequence + // section decodes past MAX_BLOCK_SIZE; surfacing the failure to + // the caller as a normal decode Err is required behaviour — + // both silent wrong output AND an unconditional panic on + // untrusted input are unacceptable. libFuzzer artifact + // crash-bfb3bc55... originally exercised this branch via the + // panic guard added in the previous round. + let mut buf = DecodeBuffer::::new(64); + buf.push(&[0; 16]); + let cp = buf.checkpoint(); + // Force a reallocation. RingBuffer grows by powers of two and + // 4 MiB is well above the initial 64-byte starting capacity, so + // reserve() must hit reserve_amortized(). + buf.reserve_exact(4 * 1024 * 1024); + buf.push(&[0; 16]); + assert!( + !buf.try_restore_checkpoint(cp), + "realloc happened → rollback must be refused" + ); + // No state mutation when the restore is refused. + assert_eq!(buf.len(), 32); +} + +#[test] +fn short_writer() { + struct ShortWriter { + buf: Vec, + write_len: usize, + } + + impl Write for ShortWriter { + fn write(&mut self, buf: &[u8]) -> std::result::Result { + if buf.len() > self.write_len { + self.buf.extend_from_slice(&buf[..self.write_len]); + Ok(self.write_len) + } else { + self.buf.extend_from_slice(buf); + Ok(buf.len()) + } + } + + fn flush(&mut self) -> std::result::Result<(), Error> { + Ok(()) + } + } + + let mut short_writer = ShortWriter { + buf: vec![], + write_len: 10, + }; + + let mut decode_buf = DecodeBuffer::::new(100); + decode_buf.push(b"0123456789"); + decode_buf.repeat(10, 90).unwrap(); + let repeats = 1000; + for _ in 0..repeats { + assert_eq!(decode_buf.len(), 100); + decode_buf.repeat(10, 50).unwrap(); + assert_eq!(decode_buf.len(), 150); + decode_buf + .drain_to_window_size_writer(&mut short_writer) + .unwrap(); + assert_eq!(decode_buf.len(), 100); + } + + assert_eq!(short_writer.buf.len(), repeats * 50); + decode_buf.drain_to_writer(&mut short_writer).unwrap(); + assert_eq!(short_writer.buf.len(), repeats * 50 + 100); +} + +#[test] +fn wouldblock_writer() { + struct WouldblockWriter { + buf: Vec, + last_blocked: usize, + block_every: usize, + } + + impl Write for WouldblockWriter { + fn write(&mut self, buf: &[u8]) -> std::result::Result { + if self.last_blocked < self.block_every { + self.buf.extend_from_slice(buf); + self.last_blocked += 1; + Ok(buf.len()) + } else { + self.last_blocked = 0; + Err(Error::from(ErrorKind::WouldBlock)) + } + } + + fn flush(&mut self) -> std::result::Result<(), Error> { + Ok(()) + } + } + + let mut short_writer = WouldblockWriter { + buf: vec![], + last_blocked: 0, + block_every: 5, + }; + + let mut decode_buf = DecodeBuffer::::new(100); + decode_buf.push(b"0123456789"); + decode_buf.repeat(10, 90).unwrap(); + let repeats = 1000; + for _ in 0..repeats { + assert_eq!(decode_buf.len(), 100); + decode_buf.repeat(10, 50).unwrap(); + assert_eq!(decode_buf.len(), 150); + loop { + match decode_buf.drain_to_window_size_writer(&mut short_writer) { + Ok(written) => { + if written == 0 { + break; + } + } + Err(e) => { + if e.kind() == ErrorKind::WouldBlock { + continue; + } else { + panic!("Unexpected error {:?}", e); + } + } + } + } + assert_eq!(decode_buf.len(), 100); + } + + assert_eq!(short_writer.buf.len(), repeats * 50); + loop { + match decode_buf.drain_to_writer(&mut short_writer) { + Ok(written) => { + if written == 0 { + break; + } + } + Err(e) => { + if e.kind() == ErrorKind::WouldBlock { + continue; + } else { + panic!("Unexpected error {:?}", e); + } + } + } + } + assert_eq!(short_writer.buf.len(), repeats * 50 + 100); +} + +#[test] +fn repeat_overlap_fast_paths_match_reference_behavior() { + let seed = b"0123456789abcdef0123456789abcdef"; + let cases = [ + (16usize, 16usize), // non-overlapping boundary + (16usize, 211usize), + (8usize, 173usize), + (7usize, 149usize), + (3usize, 160usize), + (1usize, 255usize), + ]; + + for (offset, match_len) in cases { + let mut decode_buf = DecodeBuffer::::new(4 * 1024); + decode_buf.push(seed); + decode_buf.repeat(offset, match_len).unwrap(); + let got = decode_buf.drain(); + let expected = expected_match_expansion(seed, offset, match_len); + assert_eq!(got, expected, "offset={offset}, match_len={match_len}"); + } +} + +#[test] +fn repeat_zero_offset_returns_error() { + let mut decode_buf = DecodeBuffer::::new(1024); + decode_buf.push(b"abcdef"); + let err = decode_buf.repeat(0, 5).unwrap_err(); + assert!(matches!( + err, + crate::decoding::errors::DecodeBufferError::ZeroOffset + )); +} + +#[test] +fn repeat_rejects_output_past_block_ceiling() { + // A single zstd block decompresses to at most MAX_BLOCK_SIZE. The + // per-block ceiling is enforced on the growable `RingBuffer`'s cold + // growth path: `set_block_output_ceiling` lowers `max_capacity`, so a + // `repeat` whose match would have to grow the buffer past the ceiling + // fails its `try_reserve` instead of growing the ring unbounded. + // Without this guard the over-long match drove `try_reserve` to + // ~0.5–2 GiB across many over-producing sequences (artifact + // `oom-66db61d9…`, fuzz `decode` target) before any post-block check + // ran — a decompression-bomb OOM. The match needing growth surfaces + // as `OutputBufferOverflow` (the backend's structured reject). + let mut decode_buf = DecodeBuffer::::new(4 * 1024); + decode_buf.push(b"abcdef"); // len = 6 + decode_buf.set_block_output_ceiling(8); // max_capacity = 6 + 8 = 14 + let err = decode_buf.repeat(4, 16).unwrap_err(); // 6 + 16 = 22 > 14 + assert!( + matches!( + err, + crate::decoding::errors::DecodeBufferError::OutputBufferOverflow { .. } + ), + "over-producing match must be rejected, got {err:?}" + ); +} + +#[test] +fn repeat_within_block_ceiling_still_succeeds() { + // A match that keeps the block's output at/below the armed ceiling + // must NOT be rejected — the guard fires only on growth past the + // ceiling, never on legitimate in-bounds output. + let mut decode_buf = DecodeBuffer::::new(4 * 1024); + decode_buf.push(b"abcdef"); // len = 6 + decode_buf.set_block_output_ceiling(8); // max_capacity = 14 + decode_buf + .repeat(4, 8) + .expect("6 + 8 = 14 == ceiling is allowed"); + assert_eq!(decode_buf.len(), 14); +} + +#[test] +fn repeat_from_dict_rejects_output_past_block_ceiling() { + // A match satisfied (fully or partially) from the dictionary appends + // `dict_slice` directly via `buffer.extend`, bypassing the inline + // push/repeat guard. The per-block bomb ceiling must still bound it, so + // a dict-backed over-producing match returns `OutputBufferOverflow` + // instead of growing the buffer toward OOM. + let dict = || { + crate::decoding::dictionary::DictionaryHandle::from_dictionary( + crate::decoding::dictionary::Dictionary::from_raw_content(1, alloc::vec![0xABu8; 256]) + .unwrap(), + ) + }; + + // Fully-dictionary match: empty buffer, offset reaches into the dict. + let mut full = DecodeBuffer::::new(4 * 1024); + full.set_dict(dict()); + full.set_block_output_ceiling(8); // max_capacity = 0 + 8 = 8 + let err = full.repeat(200, 100).unwrap_err(); // 0 + 100 > 8, all from dict + assert!( + matches!( + err, + crate::decoding::errors::DecodeBufferError::OutputBufferOverflow { .. } + ), + "fully-dictionary over-producing match must be rejected, got {err:?}" + ); + + // Mixed match: part from dict, remainder from buffer history. + let mut mixed = DecodeBuffer::::new(4 * 1024); + mixed.set_dict(dict()); + mixed.push(b"abcd"); // len = 4 + mixed.set_block_output_ceiling(8); // max_capacity = 4 + 8 = 12 + let err = mixed.repeat(10, 100).unwrap_err(); // 6 from dict + rest, 4 + 100 > 12 + assert!( + matches!( + err, + crate::decoding::errors::DecodeBufferError::OutputBufferOverflow { .. } + ), + "mixed dict+buffer over-producing match must be rejected, got {err:?}" + ); +} + +#[test] +fn repeat_from_dict_full_copy_updates_total_output_counter() { + let mut decode_buf = DecodeBuffer::::new(1); + decode_buf.set_dict( + crate::decoding::dictionary::DictionaryHandle::from_dictionary( + crate::decoding::dictionary::Dictionary::from_raw_content(1, b"0123456789".to_vec()) + .unwrap(), + ), + ); + + decode_buf.repeat(10, 2).unwrap(); + let err = decode_buf.repeat(10, 1).unwrap_err(); + assert!(matches!( + err, + crate::decoding::errors::DecodeBufferError::OffsetTooBig { .. } + )); +} + +#[test] +fn repeat_overlap_fast_paths_match_reference_behavior_with_wrapped_ringbuffer() { + let window = 32usize; + let seed = b"0123456789abcdef0123456789abcdef"; + let mut decode_buf = DecodeBuffer::::new(window); + let mut model = Vec::new(); + + decode_buf.push(seed); + model_push(&mut model, seed); + decode_buf.repeat(16, 16).unwrap(); + model_repeat(&mut model, 16, 16); + + let drained = decode_buf.drain_to_window_size().unwrap(); + let model_drained = model_drain_to_window(&mut model, window); + assert_eq!(drained, model_drained); + + let cases = [(3usize, 97usize), (16usize, 64usize), (7usize, 73usize)]; + for (offset, match_len) in cases { + decode_buf.repeat(offset, match_len).unwrap(); + model_repeat(&mut model, offset, match_len); + + if let Some(got) = decode_buf.drain_to_window_size() { + let expected = model_drain_to_window(&mut model, window); + assert_eq!(got, expected, "offset={offset}, match_len={match_len}"); + } + } + + assert_eq!(decode_buf.drain(), model); +} + +fn expected_match_expansion(seed: &[u8], offset: usize, match_len: usize) -> Vec { + let mut out = seed.to_vec(); + let start = out.len() - offset; + for i in 0..match_len { + let byte = out[start + i]; + out.push(byte); + } + out +} + +fn model_push(model: &mut Vec, bytes: &[u8]) { + model.extend_from_slice(bytes); +} + +fn model_repeat(model: &mut Vec, offset: usize, match_len: usize) { + let start = model.len() - offset; + for i in 0..match_len { + let byte = model[start + i]; + model.push(byte); + } +} + +fn model_drain_to_window(model: &mut Vec, window: usize) -> Vec { + if model.len() <= window { + return Vec::new(); + } + let drain_len = model.len() - window; + model.drain(0..drain_len).collect() +} + +/// Drive `DecodeBuffer::repeat` through the short-offset path and +/// compare against the canonical `output[i] = base[i % offset]` +/// reference, covering offsets that hit both the SIMD-16 fast path +/// (1, 2, 4) and the 8-byte phase-pattern path (3, 5, 6, 7). +/// +/// Regression guard for the SIMD-16 specialisation: when `period +/// divides 16` (offset ∈ {1,2,4}), the inner loop emits 16-byte +/// chunks via a pre-built `[u8; 16]` instead of 8-byte phase +/// patterns. Tail lengths span both `match_length % 16 == 0` and +/// non-zero remainders so the tail-extend codepath is also +/// exercised. +#[test] +fn repeat_short_offset_matches_canonical_for_all_offsets_and_lengths() { + for offset in 1usize..=7 { + let mut base = [0u8; 7]; + for (i, slot) in base.iter_mut().enumerate().take(offset) { + *slot = b'A' + (i as u8); + } + for &match_length in &[ + 1usize, 2, 3, 4, 5, 6, 7, 8, 9, 15, 16, 17, 23, 24, 25, 31, 32, 33, 47, 48, 49, 64, + 127, 128, 4096, + ] { + let mut buf = DecodeBuffer::::new(8192); + buf.push(&base[..offset]); + buf.repeat(offset, match_length).unwrap_or_else(|e| { + panic!("repeat failed for offset={offset} match_length={match_length}: {e:?}") + }); + + let actual = buf.drain(); + let mut expected = Vec::with_capacity(offset + match_length); + expected.extend_from_slice(&base[..offset]); + for i in 0..match_length { + expected.push(base[i % offset]); + } + assert_eq!( + actual, expected, + "mismatch at offset={offset} match_length={match_length}", + ); + } + } +} + +#[test] +fn prefetch_lookahead_in_range_does_not_panic() { + // Plain in-range lookup: start_idx well within `buffer.len()`. + // The helper should issue prefetch hints and return cleanly. + // Prefetch hints are unobservable from Rust — the assertion is + // simply that the call completes without panic / UB. + let mut buf = DecodeBuffer::::new(1024); + buf.reserve_exact(512); + buf.push(&[0xAA; 256]); + buf.prefetch_lookahead_match_source(0); + buf.prefetch_lookahead_match_source(128); + buf.prefetch_lookahead_match_source(buf.len() - 1); +} + +#[test] +fn prefetch_lookahead_out_of_range_returns_without_panic() { + // Wrap-derived garbage / dictionary-sourced match / intra-block + // self-overlap all produce `start_idx >= buffer.len()` here. + // The helper must early-return (bound check) and never touch a + // slice past the live region. + let mut buf = DecodeBuffer::::new(1024); + buf.reserve_exact(64); + buf.push(&[0x55; 32]); + buf.prefetch_lookahead_match_source(buf.len()); + buf.prefetch_lookahead_match_source(buf.len() + 1); + buf.prefetch_lookahead_match_source(usize::MAX); + // Empty buffer — every start_idx is out-of-range. + let empty: DecodeBuffer = DecodeBuffer::new(1024); + empty.prefetch_lookahead_match_source(0); + empty.prefetch_lookahead_match_source(7); +} + +#[test] +fn prefetch_lookahead_at_wrap_boundary() { + // Force the RingBuffer into a wrapped layout where + // `as_slices()` returns two non-empty halves: push, drain past + // window, push again so the write cursor wraps. Then exercise + // start_idx values at the boundary (last byte of s1, first + // byte of s2, short s1 tail < CACHE_LINE) so the + // `prefetch_first_line_l1` fallback path is touched too. + let mut buf = DecodeBuffer::::new(256); + // Fill with two passes so the underlying ringbuffer wraps. + let payload = [0xCD_u8; 320]; + buf.push(&payload); + // Drain to free read cursor capacity (write side can then wrap). + let _ = buf.drain_to_window_size(); + buf.push(&payload); + // Probe a handful of indices inside and across the wrap. + let n = buf.len(); + if n > 0 { + buf.prefetch_lookahead_match_source(0); + buf.prefetch_lookahead_match_source(n / 2); + buf.prefetch_lookahead_match_source(n - 1); + // Out-of-range probe to exercise the early-return path on + // a wrapped buffer. + buf.prefetch_lookahead_match_source(n); + } +} diff --git a/zstd/src/decoding/dictionary.rs b/zstd/src/decoding/dictionary.rs index f3a710905..9ad4d6dbf 100644 --- a/zstd/src/decoding/dictionary.rs +++ b/zstd/src/decoding/dictionary.rs @@ -305,121 +305,4 @@ impl From for DictionaryHandle { } #[cfg(test)] -mod tests { - use super::*; - use alloc::vec; - - fn offset_history_start(raw: &[u8]) -> usize { - let mut huf = crate::decoding::scratch::HuffmanScratch::new(); - let mut fse = crate::decoding::scratch::FSEScratch::new(); - let mut cursor = 8usize; - - let huf_size = huf - .table - .build_decoder(&raw[cursor..]) - .expect("reference dictionary huffman table should decode"); - cursor += huf_size as usize; - - let of_size = fse - .offsets - .build_decoder( - &raw[cursor..], - crate::decoding::sequence_section_decoder::OF_MAX_LOG, - ) - .expect("reference dictionary OF table should decode"); - cursor += of_size; - - let ml_size = fse - .match_lengths - .build_decoder( - &raw[cursor..], - crate::decoding::sequence_section_decoder::ML_MAX_LOG, - ) - .expect("reference dictionary ML table should decode"); - cursor += ml_size; - - let ll_size = fse - .literal_lengths - .build_decoder( - &raw[cursor..], - crate::decoding::sequence_section_decoder::LL_MAX_LOG, - ) - .expect("reference dictionary LL table should decode"); - cursor += ll_size; - - cursor - } - - #[test] - fn decode_dict_rejects_short_buffer_before_magic_and_id() { - let err = match Dictionary::decode_dict(&[]) { - Ok(_) => panic!("expected short dictionary to fail"), - Err(err) => err, - }; - assert!(matches!( - err, - DictionaryDecodeError::DictionaryTooSmall { got: 0, need: 8 } - )); - } - - #[test] - fn decode_dict_malformed_input_returns_error_instead_of_panicking() { - let mut raw = Vec::new(); - raw.extend_from_slice(&MAGIC_NUM); - raw.extend_from_slice(&1u32.to_le_bytes()); - raw.extend_from_slice(&[0u8; 7]); - - let result = std::panic::catch_unwind(|| Dictionary::decode_dict(&raw)); - assert!( - result.is_ok(), - "decode_dict must not panic on malformed input" - ); - assert!( - result.unwrap().is_err(), - "malformed dictionary must return error" - ); - } - - #[test] - fn decode_dict_rejects_zero_repeat_offsets() { - let mut raw = include_bytes!("../../dict_tests/dictionary").to_vec(); - let offset_start = offset_history_start(&raw); - - // Corrupt rep0 to zero. - raw[offset_start..offset_start + 4].copy_from_slice(&0u32.to_le_bytes()); - let decoded = Dictionary::decode_dict(&raw); - assert!(matches!( - decoded, - Err(DictionaryDecodeError::ZeroRepeatOffsetInDictionary { index: 0 }) - )); - } - - #[test] - fn from_raw_content_rejects_empty_dictionary_content() { - let result = Dictionary::from_raw_content(1, Vec::new()); - assert!(matches!( - result, - Err(DictionaryDecodeError::DictionaryTooSmall { got: 0, need: 1 }) - )); - } - - #[test] - fn dictionary_handle_from_raw_content_supports_as_ref() { - let dict = Dictionary::from_raw_content(7, vec![42]).expect("raw dict should build"); - let handle = dict.into_handle(); - let dict_ref: &Dictionary = handle.as_ref(); - - assert_eq!(dict_ref.id, 7); - assert_eq!(dict_ref.dict_content.as_slice(), &[42]); - } - - #[test] - fn dictionary_handle_clones_share_inner() { - let raw = include_bytes!("../../dict_tests/dictionary"); - let handle = DictionaryHandle::decode_dict(raw).expect("dictionary should parse"); - let clone = handle.clone(); - - assert_eq!(handle.id(), clone.id()); - assert!(SharedDictionary::ptr_eq(&handle.inner, &clone.inner)); - } -} +mod tests; diff --git a/zstd/src/decoding/dictionary/tests.rs b/zstd/src/decoding/dictionary/tests.rs new file mode 100644 index 000000000..3754542af --- /dev/null +++ b/zstd/src/decoding/dictionary/tests.rs @@ -0,0 +1,116 @@ +use super::*; +use alloc::vec; + +fn offset_history_start(raw: &[u8]) -> usize { + let mut huf = crate::decoding::scratch::HuffmanScratch::new(); + let mut fse = crate::decoding::scratch::FSEScratch::new(); + let mut cursor = 8usize; + + let huf_size = huf + .table + .build_decoder(&raw[cursor..]) + .expect("reference dictionary huffman table should decode"); + cursor += huf_size as usize; + + let of_size = fse + .offsets + .build_decoder( + &raw[cursor..], + crate::decoding::sequence_section_decoder::OF_MAX_LOG, + ) + .expect("reference dictionary OF table should decode"); + cursor += of_size; + + let ml_size = fse + .match_lengths + .build_decoder( + &raw[cursor..], + crate::decoding::sequence_section_decoder::ML_MAX_LOG, + ) + .expect("reference dictionary ML table should decode"); + cursor += ml_size; + + let ll_size = fse + .literal_lengths + .build_decoder( + &raw[cursor..], + crate::decoding::sequence_section_decoder::LL_MAX_LOG, + ) + .expect("reference dictionary LL table should decode"); + cursor += ll_size; + + cursor +} + +#[test] +fn decode_dict_rejects_short_buffer_before_magic_and_id() { + let err = match Dictionary::decode_dict(&[]) { + Ok(_) => panic!("expected short dictionary to fail"), + Err(err) => err, + }; + assert!(matches!( + err, + DictionaryDecodeError::DictionaryTooSmall { got: 0, need: 8 } + )); +} + +#[test] +fn decode_dict_malformed_input_returns_error_instead_of_panicking() { + let mut raw = Vec::new(); + raw.extend_from_slice(&MAGIC_NUM); + raw.extend_from_slice(&1u32.to_le_bytes()); + raw.extend_from_slice(&[0u8; 7]); + + let result = std::panic::catch_unwind(|| Dictionary::decode_dict(&raw)); + assert!( + result.is_ok(), + "decode_dict must not panic on malformed input" + ); + assert!( + result.unwrap().is_err(), + "malformed dictionary must return error" + ); +} + +#[test] +fn decode_dict_rejects_zero_repeat_offsets() { + let mut raw = include_bytes!("../../../dict_tests/dictionary").to_vec(); + let offset_start = offset_history_start(&raw); + + // Corrupt rep0 to zero. + raw[offset_start..offset_start + 4].copy_from_slice(&0u32.to_le_bytes()); + let decoded = Dictionary::decode_dict(&raw); + assert!(matches!( + decoded, + Err(DictionaryDecodeError::ZeroRepeatOffsetInDictionary { index: 0 }) + )); +} + +#[test] +fn from_raw_content_rejects_empty_dictionary_content() { + let result = Dictionary::from_raw_content(1, Vec::new()); + assert!(matches!( + result, + Err(DictionaryDecodeError::DictionaryTooSmall { got: 0, need: 1 }) + )); +} + +#[test] +fn dictionary_handle_from_raw_content_supports_as_ref() { + let dict = Dictionary::from_raw_content(7, vec![42]).expect("raw dict should build"); + let handle = dict.into_handle(); + let dict_ref: &Dictionary = handle.as_ref(); + + assert_eq!(dict_ref.id, 7); + assert_eq!(dict_ref.dict_content.as_slice(), &[42]); +} + +#[test] +fn dictionary_handle_clones_share_inner() { + let raw = include_bytes!("../../../dict_tests/dictionary"); + let handle = DictionaryHandle::decode_dict(raw).expect("dictionary should parse"); + let clone = handle.clone(); + + assert_eq!(handle.id(), clone.id()); + assert!(SharedDictionary::ptr_eq(&handle.inner, &clone.inner)); +} diff --git a/zstd/src/decoding/errors.rs b/zstd/src/decoding/errors.rs index 317441c9d..cad50f520 100644 --- a/zstd/src/decoding/errors.rs +++ b/zstd/src/decoding/errors.rs @@ -1628,180 +1628,4 @@ impl From for HuffmanDecoderError { } #[cfg(test)] -mod tests { - use alloc::{string::ToString, vec}; - - use super::{ - BlockTypeError, DecodeBlockContentError, DecodeBufferError, DecodeSequenceError, - DecompressBlockError, DecompressLiteralsError, ExecuteSequencesError, FSETableError, - FrameDecoderError, HuffmanTableError, - }; - - #[test] - fn execute_sequences_output_overflow_requested_covers_all_arms() { - // #246: `run_direct_decode` folds a Compressed-block overshoot into - // `FrameContentSizeMismatch` by reading `requested` from whichever - // overflow shape the executor produced. Cover all three arms: - // 1. inline-sequence path -> `OutputBufferOverflow` directly, - // 2. match-repeat path -> `DecodebufferError(OutputBufferOverflow)`, - // 3. any other variant -> None (no fold). - let inline = ExecuteSequencesError::OutputBufferOverflow { - tail: 10, - requested: 7, - capacity: 12, - }; - assert_eq!(inline.output_overflow_requested(), Some(7)); - - let repeat = - ExecuteSequencesError::DecodebufferError(DecodeBufferError::OutputBufferOverflow { - tail: 3, - requested: 99, - capacity: 4, - }); - assert_eq!(repeat.output_overflow_requested(), Some(99)); - - // Non-overflow variants (and non-overflow DecodeBufferError) -> None. - assert_eq!( - ExecuteSequencesError::ZeroOffset.output_overflow_requested(), - None - ); - assert_eq!( - ExecuteSequencesError::DecodebufferError(DecodeBufferError::ZeroOffset) - .output_overflow_requested(), - None - ); - } - - #[test] - fn block_and_sequence_display_messages_are_specific() { - assert_eq!( - BlockTypeError::InvalidBlocktypeNumber { num: 7 }.to_string(), - "Invalid Blocktype number. Is: 7. Should be one of: 0, 1, 2, 3 (3 is reserved)." - ); - assert_eq!( - DecompressBlockError::MalformedSectionHeader { - expected_len: 12, - remaining_bytes: 3, - } - .to_string(), - "Malformed section header. Says literals would be this long: 12 but there are only 3 bytes left" - ); - assert_eq!( - DecodeBlockContentError::ExpectedHeaderOfPreviousBlock.to_string(), - "Can't decode next block body, while expecting to decode the header of the previous block. Results will be nonsense" - ); - assert_eq!( - DecodeSequenceError::ExtraPadding { skipped_bits: 11 }.to_string(), - "Padding at the end of the sequence_section was more than a byte long: 11 bits. Probably caused by data corruption" - ); - } - - #[test] - fn frame_decoder_display_messages_are_specific() { - assert_eq!( - FrameDecoderError::TargetTooSmall.to_string(), - "Target must have at least as many bytes as the content size reported by the frame" - ); - assert_eq!( - FrameDecoderError::DictNotProvided { dict_id: 0xABCD }.to_string(), - "Frame header specified dictionary id 0xABCD that wasn't provided via add_dict()/add_dict_from_bytes() (or add_dict_handle() on atomic targets) or reset_with_dict_handle()/decode_all_with_dict_handle()/decode_all_with_dict_bytes()" - ); - assert_eq!( - FrameDecoderError::DictIdMismatch { - expected: 0xABCD, - provided: 0x1234 - } - .to_string(), - "Frame header dictionary id 0xABCD does not match provided dictionary id 0x1234" - ); - assert_eq!( - FrameDecoderError::DictAlreadyRegistered { dict_id: 0xABCD }.to_string(), - "Dictionary id 0xABCD already registered in decoder" - ); - assert_eq!( - FrameDecoderError::FrameContentSizeMismatch { - declared: 100, - produced: 87, - } - .to_string(), - "Frame content size mismatch (corrupt frame): declared 100 bytes, blocks summed to 87 bytes" - ); - // Locks the wasm-exposed checksum-mismatch contract (exact string). - assert_eq!( - FrameDecoderError::ChecksumMismatch { - expected: 0xDEAD_BEEF, - calculated: 0x0BAD_F00D, - } - .to_string(), - "Content checksum mismatch (corrupt frame): frame stored 0xDEADBEEF, decoder calculated 0x0BADF00D" - ); - } - - #[test] - fn decode_block_content_backend_overflow_display_names_the_step() { - use crate::blocks::block::BlockType; - assert_eq!( - DecodeBlockContentError::BackendOverflow { - step: BlockType::RLE - } - .to_string(), - "RLE block's decompressed payload exceeds the caller-provided output buffer" - ); - assert_eq!( - DecodeBlockContentError::BackendOverflow { - step: BlockType::Raw - } - .to_string(), - "Raw block's decompressed payload exceeds the caller-provided output buffer" - ); - } - - #[test] - fn literal_display_messages_are_specific() { - assert_eq!( - DecompressLiteralsError::MissingCompressedSize.to_string(), - "compressed size was none even though it must be set to something for compressed literals" - ); - assert_eq!( - DecompressLiteralsError::MissingNumStreams.to_string(), - "num_streams was none even though it must be set to something (1 or 4) for compressed literals" - ); - assert_eq!( - DecompressLiteralsError::ExtraPadding { skipped_bits: 9 }.to_string(), - "Padding at the end of the sequence_section was more than a byte long: 9 bits. Probably caused by data corruption" - ); - } - - #[test] - fn fse_and_huffman_display_messages_are_specific() { - assert_eq!( - FSETableError::ProbabilityCounterMismatch { - got: 4, - expected_sum: 3, - symbol_probabilities: vec![1, -1], - } - .to_string(), - "FSE probability sum mismatch: got 4, expected 3. Indicates corrupted data or an invalid distribution\n [1, -1]" - ); - assert_eq!( - HuffmanTableError::NotEnoughBytesForWeights { - got_bytes: 2, - expected_bytes: 5, - } - .to_string(), - "Header says there should be 5 bytes for the weights but there are only 2 bytes in the stream" - ); - assert_eq!( - HuffmanTableError::ExtraPadding { skipped_bits: 13 }.to_string(), - "Padding at the end of the sequence_section was more than a byte long: 13 bits. Probably caused by data corruption" - ); - assert_eq!( - HuffmanTableError::FSETableUsedTooManyBytes { - used: 7, - available_bytes: 6, - } - .to_string(), - "FSE table used more bytes: 7 than were meant to be used for the whole stream of huffman weights (6)" - ); - } -} +mod tests; diff --git a/zstd/src/decoding/errors/tests.rs b/zstd/src/decoding/errors/tests.rs new file mode 100644 index 000000000..60cf406a3 --- /dev/null +++ b/zstd/src/decoding/errors/tests.rs @@ -0,0 +1,175 @@ +use alloc::{string::ToString, vec}; + +use super::{ + BlockTypeError, DecodeBlockContentError, DecodeBufferError, DecodeSequenceError, + DecompressBlockError, DecompressLiteralsError, ExecuteSequencesError, FSETableError, + FrameDecoderError, HuffmanTableError, +}; + +#[test] +fn execute_sequences_output_overflow_requested_covers_all_arms() { + // #246: `run_direct_decode` folds a Compressed-block overshoot into + // `FrameContentSizeMismatch` by reading `requested` from whichever + // overflow shape the executor produced. Cover all three arms: + // 1. inline-sequence path -> `OutputBufferOverflow` directly, + // 2. match-repeat path -> `DecodebufferError(OutputBufferOverflow)`, + // 3. any other variant -> None (no fold). + let inline = ExecuteSequencesError::OutputBufferOverflow { + tail: 10, + requested: 7, + capacity: 12, + }; + assert_eq!(inline.output_overflow_requested(), Some(7)); + + let repeat = + ExecuteSequencesError::DecodebufferError(DecodeBufferError::OutputBufferOverflow { + tail: 3, + requested: 99, + capacity: 4, + }); + assert_eq!(repeat.output_overflow_requested(), Some(99)); + + // Non-overflow variants (and non-overflow DecodeBufferError) -> None. + assert_eq!( + ExecuteSequencesError::ZeroOffset.output_overflow_requested(), + None + ); + assert_eq!( + ExecuteSequencesError::DecodebufferError(DecodeBufferError::ZeroOffset) + .output_overflow_requested(), + None + ); +} + +#[test] +fn block_and_sequence_display_messages_are_specific() { + assert_eq!( + BlockTypeError::InvalidBlocktypeNumber { num: 7 }.to_string(), + "Invalid Blocktype number. Is: 7. Should be one of: 0, 1, 2, 3 (3 is reserved)." + ); + assert_eq!( + DecompressBlockError::MalformedSectionHeader { + expected_len: 12, + remaining_bytes: 3, + } + .to_string(), + "Malformed section header. Says literals would be this long: 12 but there are only 3 bytes left" + ); + assert_eq!( + DecodeBlockContentError::ExpectedHeaderOfPreviousBlock.to_string(), + "Can't decode next block body, while expecting to decode the header of the previous block. Results will be nonsense" + ); + assert_eq!( + DecodeSequenceError::ExtraPadding { skipped_bits: 11 }.to_string(), + "Padding at the end of the sequence_section was more than a byte long: 11 bits. Probably caused by data corruption" + ); +} + +#[test] +fn frame_decoder_display_messages_are_specific() { + assert_eq!( + FrameDecoderError::TargetTooSmall.to_string(), + "Target must have at least as many bytes as the content size reported by the frame" + ); + assert_eq!( + FrameDecoderError::DictNotProvided { dict_id: 0xABCD }.to_string(), + "Frame header specified dictionary id 0xABCD that wasn't provided via add_dict()/add_dict_from_bytes() (or add_dict_handle() on atomic targets) or reset_with_dict_handle()/decode_all_with_dict_handle()/decode_all_with_dict_bytes()" + ); + assert_eq!( + FrameDecoderError::DictIdMismatch { + expected: 0xABCD, + provided: 0x1234 + } + .to_string(), + "Frame header dictionary id 0xABCD does not match provided dictionary id 0x1234" + ); + assert_eq!( + FrameDecoderError::DictAlreadyRegistered { dict_id: 0xABCD }.to_string(), + "Dictionary id 0xABCD already registered in decoder" + ); + assert_eq!( + FrameDecoderError::FrameContentSizeMismatch { + declared: 100, + produced: 87, + } + .to_string(), + "Frame content size mismatch (corrupt frame): declared 100 bytes, blocks summed to 87 bytes" + ); + // Locks the wasm-exposed checksum-mismatch contract (exact string). + assert_eq!( + FrameDecoderError::ChecksumMismatch { + expected: 0xDEAD_BEEF, + calculated: 0x0BAD_F00D, + } + .to_string(), + "Content checksum mismatch (corrupt frame): frame stored 0xDEADBEEF, decoder calculated 0x0BADF00D" + ); +} + +#[test] +fn decode_block_content_backend_overflow_display_names_the_step() { + use crate::blocks::block::BlockType; + assert_eq!( + DecodeBlockContentError::BackendOverflow { + step: BlockType::RLE + } + .to_string(), + "RLE block's decompressed payload exceeds the caller-provided output buffer" + ); + assert_eq!( + DecodeBlockContentError::BackendOverflow { + step: BlockType::Raw + } + .to_string(), + "Raw block's decompressed payload exceeds the caller-provided output buffer" + ); +} + +#[test] +fn literal_display_messages_are_specific() { + assert_eq!( + DecompressLiteralsError::MissingCompressedSize.to_string(), + "compressed size was none even though it must be set to something for compressed literals" + ); + assert_eq!( + DecompressLiteralsError::MissingNumStreams.to_string(), + "num_streams was none even though it must be set to something (1 or 4) for compressed literals" + ); + assert_eq!( + DecompressLiteralsError::ExtraPadding { skipped_bits: 9 }.to_string(), + "Padding at the end of the sequence_section was more than a byte long: 9 bits. Probably caused by data corruption" + ); +} + +#[test] +fn fse_and_huffman_display_messages_are_specific() { + assert_eq!( + FSETableError::ProbabilityCounterMismatch { + got: 4, + expected_sum: 3, + symbol_probabilities: vec![1, -1], + } + .to_string(), + "FSE probability sum mismatch: got 4, expected 3. Indicates corrupted data or an invalid distribution\n [1, -1]" + ); + assert_eq!( + HuffmanTableError::NotEnoughBytesForWeights { + got_bytes: 2, + expected_bytes: 5, + } + .to_string(), + "Header says there should be 5 bytes for the weights but there are only 2 bytes in the stream" + ); + assert_eq!( + HuffmanTableError::ExtraPadding { skipped_bits: 13 }.to_string(), + "Padding at the end of the sequence_section was more than a byte long: 13 bits. Probably caused by data corruption" + ); + assert_eq!( + HuffmanTableError::FSETableUsedTooManyBytes { + used: 7, + available_bytes: 6, + } + .to_string(), + "FSE table used more bytes: 7 than were meant to be used for the whole stream of huffman weights (6)" + ); +} diff --git a/zstd/src/decoding/exec_sequence_inline.rs b/zstd/src/decoding/exec_sequence_inline.rs index dc3673206..aa6b8356e 100644 --- a/zstd/src/decoding/exec_sequence_inline.rs +++ b/zstd/src/decoding/exec_sequence_inline.rs @@ -547,93 +547,7 @@ pub(crate) mod portable { } #[cfg(all(test, target_arch = "x86_64"))] -mod inline_helper_tests { - use super::x86::{copy16, overlap_copy8, wildcopy_no_overlap, wildcopy_overlap_8byte_stride}; - - #[test] - fn copy16_copies_exactly_16_bytes() { - let src: [u8; 16] = [ - 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, - 0xAE, 0xAF, - ]; - let mut dst = [0u8; 16]; - unsafe { copy16(dst.as_mut_ptr(), src.as_ptr()) }; - assert_eq!(dst, src); - } - - #[test] - fn wildcopy_no_overlap_short_length_overshoots() { - // Length 1 still triggers the unconditional first 16-byte - // store — the wildcopy overshoots up to 15 bytes past the - // declared end, which is the upstream zstd contract. - let src: [u8; 32] = core::array::from_fn(|i| (i + 1) as u8); - let mut dst = [0u8; 32]; - unsafe { wildcopy_no_overlap(dst.as_mut_ptr(), src.as_ptr(), 1) }; - // First 16 bytes copied from src; remaining untouched. - assert_eq!(&dst[..16], &src[..16]); - assert!(dst[16..].iter().all(|&b| b == 0)); - } - - #[test] - fn wildcopy_no_overlap_length_above_16_uses_multiple_iters() { - // Length 24 → first 16-byte store, then one more iter that - // overshoots 8 bytes past the declared end. - let src: [u8; 32] = core::array::from_fn(|i| (i + 1) as u8); - let mut dst = [0u8; 32]; - unsafe { wildcopy_no_overlap(dst.as_mut_ptr(), src.as_ptr(), 24) }; - // 32 bytes get written (two 16-byte stores). - assert_eq!(&dst[..32], &src[..32]); - } - - #[test] - fn wildcopy_overlap_8byte_stride_rle_expansion_offset_8() { - // Offset = 8 means caller has set up src = dst - 8. Each - // 8-byte read picks up bytes the previous iter just wrote, - // expanding the seed pattern across the destination region. - let mut buf = [0u8; 32]; - buf[..8].copy_from_slice(&[1, 2, 3, 4, 5, 6, 7, 8]); - unsafe { - wildcopy_overlap_8byte_stride(buf.as_mut_ptr().add(8), buf.as_ptr(), 16); - } - // Bytes 8..16 = seed; bytes 16..24 = seed again (RLE expansion). - assert_eq!(&buf[8..16], &[1, 2, 3, 4, 5, 6, 7, 8]); - assert_eq!(&buf[16..24], &[1, 2, 3, 4, 5, 6, 7, 8]); - } - - #[test] - fn overlap_copy8_offset_ge_8_does_plain_copy() { - // offset >= 8 path: straight ZSTD_copy8 (8-byte read+write). - let mut buf = [0u8; 32]; - buf[..8].copy_from_slice(&[0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88]); - let (op2, ip2) = unsafe { overlap_copy8(buf.as_mut_ptr().add(8), buf.as_ptr(), 8) }; - // dst advances by 8 bytes, src advances by 8 bytes. - assert_eq!(op2, unsafe { buf.as_mut_ptr().add(16) }); - assert_eq!(ip2, unsafe { buf.as_ptr().add(8) }); - // bytes 8..16 = seed. - assert_eq!( - &buf[8..16], - &[0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88] - ); - } - - #[test] - fn overlap_copy8_offset_lt_8_spreads_source() { - // offset < 8 path: uses dec32table / dec64table to spread - // the source-destination distance so subsequent wildcopy can - // use the ≥ 8 stride. Test offset = 3 (a common short-offset - // RLE pattern). - let mut buf = [0u8; 32]; - buf[..3].copy_from_slice(&[0xAA, 0xBB, 0xCC]); - let (op2, _ip2) = unsafe { overlap_copy8(buf.as_mut_ptr().add(3), buf.as_ptr(), 3) }; - // dst advanced 8 bytes. - assert_eq!(op2, unsafe { buf.as_mut_ptr().add(11) }); - // First 8 bytes of the destination region are the 3-byte - // seed expanded — verify they're non-zero (exact spread - // pattern depends on the lookup tables; upstream zstd parity is the - // contract). - assert!(buf[3..11].iter().any(|&b| b != 0)); - } -} +mod inline_helper_tests; // Parallel coverage for the portable helpers (non-x86 targets). Mirrors // `inline_helper_tests` exactly: the portable module is the unsafe @@ -645,95 +559,4 @@ mod inline_helper_tests { // `cfg(test)` on x86_64 too), so the architecture-independent helpers are // covered on the main x86 CI lane as well as the i686 job. #[cfg(test)] -mod portable_helper_tests { - use super::portable::{ - copy16, overlap_copy8, wildcopy_no_overlap, wildcopy_overlap_8byte_stride, - }; - - #[test] - fn copy16_copies_exactly_16_bytes() { - let src: [u8; 16] = [ - 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, - 0xAE, 0xAF, - ]; - let mut dst = [0u8; 16]; - unsafe { copy16(dst.as_mut_ptr(), src.as_ptr()) }; - assert_eq!(dst, src); - } - - #[test] - fn wildcopy_no_overlap_short_length_overshoots() { - // Length 1 still triggers the unconditional first 16-byte store - // — the wildcopy overshoots up to 15 bytes past the declared - // end, the upstream zstd contract. - let src: [u8; 32] = core::array::from_fn(|i| (i + 1) as u8); - let mut dst = [0u8; 32]; - unsafe { wildcopy_no_overlap(dst.as_mut_ptr(), src.as_ptr(), 1) }; - assert_eq!(&dst[..16], &src[..16]); - assert!(dst[16..].iter().all(|&b| b == 0)); - } - - #[test] - fn wildcopy_no_overlap_length_above_16_uses_multiple_iters() { - // Length 24 → first 16-byte store, then one more iter - // overshooting 8 bytes past the declared end. - let src: [u8; 32] = core::array::from_fn(|i| (i + 1) as u8); - let mut dst = [0u8; 32]; - unsafe { wildcopy_no_overlap(dst.as_mut_ptr(), src.as_ptr(), 24) }; - assert_eq!(&dst[..32], &src[..32]); - } - - #[test] - fn wildcopy_overlap_8byte_stride_rle_expansion_offset_8() { - // Offset = 8: src = dst - 8. Each 8-byte read picks up bytes the - // previous iter just wrote, expanding the seed (RLE). - let mut buf = [0u8; 32]; - buf[..8].copy_from_slice(&[1, 2, 3, 4, 5, 6, 7, 8]); - unsafe { - wildcopy_overlap_8byte_stride(buf.as_mut_ptr().add(8), buf.as_ptr(), 16); - } - assert_eq!(&buf[8..16], &[1, 2, 3, 4, 5, 6, 7, 8]); - assert_eq!(&buf[16..24], &[1, 2, 3, 4, 5, 6, 7, 8]); - } - - #[test] - fn overlap_copy8_offset_ge_8_does_plain_copy() { - // offset >= 8: straight ZSTD_copy8 (8-byte read+write). - let mut buf = [0u8; 32]; - buf[..8].copy_from_slice(&[0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88]); - let (op2, ip2) = unsafe { overlap_copy8(buf.as_mut_ptr().add(8), buf.as_ptr(), 8) }; - assert_eq!(op2, unsafe { buf.as_mut_ptr().add(16) }); - assert_eq!(ip2, unsafe { buf.as_ptr().add(8) }); - assert_eq!( - &buf[8..16], - &[0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88] - ); - } - - #[test] - fn overlap_copy8_offset_lt_8_spreads_source() { - // offset < 8: dec32table / dec64table spread step. offset = 3. - let mut buf = [0u8; 32]; - buf[..3].copy_from_slice(&[0xAA, 0xBB, 0xCC]); - let (op2, _ip2) = unsafe { overlap_copy8(buf.as_mut_ptr().add(3), buf.as_ptr(), 3) }; - assert_eq!(op2, unsafe { buf.as_mut_ptr().add(11) }); - assert!(buf[3..11].iter().any(|&b| b != 0)); - } - - /// Cross-check: the portable helpers must produce byte-identical - /// output to a straightforward scalar reference for the no-overlap - /// copy across a range of lengths. Guards against a divergence - /// between the u128/u64 unaligned-move lowering and plain copies. - #[test] - fn wildcopy_no_overlap_matches_scalar_reference() { - for len in 1usize..=48 { - let src: [u8; 64] = core::array::from_fn(|i| (i as u8).wrapping_mul(7).wrapping_add(1)); - let mut dst = [0u8; 64]; - unsafe { wildcopy_no_overlap(dst.as_mut_ptr(), src.as_ptr(), len) }; - // Only the first `len` bytes are contractually defined; the - // overshoot tail is allowed to differ. Assert the defined - // region matches. - assert_eq!(&dst[..len], &src[..len], "len={len}"); - } - } -} +mod portable_helper_tests; diff --git a/zstd/src/decoding/exec_sequence_inline/inline_helper_tests.rs b/zstd/src/decoding/exec_sequence_inline/inline_helper_tests.rs new file mode 100644 index 000000000..80dea043a --- /dev/null +++ b/zstd/src/decoding/exec_sequence_inline/inline_helper_tests.rs @@ -0,0 +1,85 @@ +use super::x86::{copy16, overlap_copy8, wildcopy_no_overlap, wildcopy_overlap_8byte_stride}; + +#[test] +fn copy16_copies_exactly_16_bytes() { + let src: [u8; 16] = [ + 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, + 0xAF, + ]; + let mut dst = [0u8; 16]; + unsafe { copy16(dst.as_mut_ptr(), src.as_ptr()) }; + assert_eq!(dst, src); +} + +#[test] +fn wildcopy_no_overlap_short_length_overshoots() { + // Length 1 still triggers the unconditional first 16-byte + // store — the wildcopy overshoots up to 15 bytes past the + // declared end, which is the upstream zstd contract. + let src: [u8; 32] = core::array::from_fn(|i| (i + 1) as u8); + let mut dst = [0u8; 32]; + unsafe { wildcopy_no_overlap(dst.as_mut_ptr(), src.as_ptr(), 1) }; + // First 16 bytes copied from src; remaining untouched. + assert_eq!(&dst[..16], &src[..16]); + assert!(dst[16..].iter().all(|&b| b == 0)); +} + +#[test] +fn wildcopy_no_overlap_length_above_16_uses_multiple_iters() { + // Length 24 → first 16-byte store, then one more iter that + // overshoots 8 bytes past the declared end. + let src: [u8; 32] = core::array::from_fn(|i| (i + 1) as u8); + let mut dst = [0u8; 32]; + unsafe { wildcopy_no_overlap(dst.as_mut_ptr(), src.as_ptr(), 24) }; + // 32 bytes get written (two 16-byte stores). + assert_eq!(&dst[..32], &src[..32]); +} + +#[test] +fn wildcopy_overlap_8byte_stride_rle_expansion_offset_8() { + // Offset = 8 means caller has set up src = dst - 8. Each + // 8-byte read picks up bytes the previous iter just wrote, + // expanding the seed pattern across the destination region. + let mut buf = [0u8; 32]; + buf[..8].copy_from_slice(&[1, 2, 3, 4, 5, 6, 7, 8]); + unsafe { + wildcopy_overlap_8byte_stride(buf.as_mut_ptr().add(8), buf.as_ptr(), 16); + } + // Bytes 8..16 = seed; bytes 16..24 = seed again (RLE expansion). + assert_eq!(&buf[8..16], &[1, 2, 3, 4, 5, 6, 7, 8]); + assert_eq!(&buf[16..24], &[1, 2, 3, 4, 5, 6, 7, 8]); +} + +#[test] +fn overlap_copy8_offset_ge_8_does_plain_copy() { + // offset >= 8 path: straight ZSTD_copy8 (8-byte read+write). + let mut buf = [0u8; 32]; + buf[..8].copy_from_slice(&[0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88]); + let (op2, ip2) = unsafe { overlap_copy8(buf.as_mut_ptr().add(8), buf.as_ptr(), 8) }; + // dst advances by 8 bytes, src advances by 8 bytes. + assert_eq!(op2, unsafe { buf.as_mut_ptr().add(16) }); + assert_eq!(ip2, unsafe { buf.as_ptr().add(8) }); + // bytes 8..16 = seed. + assert_eq!( + &buf[8..16], + &[0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88] + ); +} + +#[test] +fn overlap_copy8_offset_lt_8_spreads_source() { + // offset < 8 path: uses dec32table / dec64table to spread + // the source-destination distance so subsequent wildcopy can + // use the ≥ 8 stride. Test offset = 3 (a common short-offset + // RLE pattern). + let mut buf = [0u8; 32]; + buf[..3].copy_from_slice(&[0xAA, 0xBB, 0xCC]); + let (op2, _ip2) = unsafe { overlap_copy8(buf.as_mut_ptr().add(3), buf.as_ptr(), 3) }; + // dst advanced 8 bytes. + assert_eq!(op2, unsafe { buf.as_mut_ptr().add(11) }); + // First 8 bytes of the destination region are the 3-byte + // seed expanded — verify they're non-zero (exact spread + // pattern depends on the lookup tables; upstream zstd parity is the + // contract). + assert!(buf[3..11].iter().any(|&b| b != 0)); +} diff --git a/zstd/src/decoding/exec_sequence_inline/portable_helper_tests.rs b/zstd/src/decoding/exec_sequence_inline/portable_helper_tests.rs new file mode 100644 index 000000000..4fa07d941 --- /dev/null +++ b/zstd/src/decoding/exec_sequence_inline/portable_helper_tests.rs @@ -0,0 +1,88 @@ +use super::portable::{copy16, overlap_copy8, wildcopy_no_overlap, wildcopy_overlap_8byte_stride}; + +#[test] +fn copy16_copies_exactly_16_bytes() { + let src: [u8; 16] = [ + 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, + 0xAF, + ]; + let mut dst = [0u8; 16]; + unsafe { copy16(dst.as_mut_ptr(), src.as_ptr()) }; + assert_eq!(dst, src); +} + +#[test] +fn wildcopy_no_overlap_short_length_overshoots() { + // Length 1 still triggers the unconditional first 16-byte store + // — the wildcopy overshoots up to 15 bytes past the declared + // end, the upstream zstd contract. + let src: [u8; 32] = core::array::from_fn(|i| (i + 1) as u8); + let mut dst = [0u8; 32]; + unsafe { wildcopy_no_overlap(dst.as_mut_ptr(), src.as_ptr(), 1) }; + assert_eq!(&dst[..16], &src[..16]); + assert!(dst[16..].iter().all(|&b| b == 0)); +} + +#[test] +fn wildcopy_no_overlap_length_above_16_uses_multiple_iters() { + // Length 24 → first 16-byte store, then one more iter + // overshooting 8 bytes past the declared end. + let src: [u8; 32] = core::array::from_fn(|i| (i + 1) as u8); + let mut dst = [0u8; 32]; + unsafe { wildcopy_no_overlap(dst.as_mut_ptr(), src.as_ptr(), 24) }; + assert_eq!(&dst[..32], &src[..32]); +} + +#[test] +fn wildcopy_overlap_8byte_stride_rle_expansion_offset_8() { + // Offset = 8: src = dst - 8. Each 8-byte read picks up bytes the + // previous iter just wrote, expanding the seed (RLE). + let mut buf = [0u8; 32]; + buf[..8].copy_from_slice(&[1, 2, 3, 4, 5, 6, 7, 8]); + unsafe { + wildcopy_overlap_8byte_stride(buf.as_mut_ptr().add(8), buf.as_ptr(), 16); + } + assert_eq!(&buf[8..16], &[1, 2, 3, 4, 5, 6, 7, 8]); + assert_eq!(&buf[16..24], &[1, 2, 3, 4, 5, 6, 7, 8]); +} + +#[test] +fn overlap_copy8_offset_ge_8_does_plain_copy() { + // offset >= 8: straight ZSTD_copy8 (8-byte read+write). + let mut buf = [0u8; 32]; + buf[..8].copy_from_slice(&[0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88]); + let (op2, ip2) = unsafe { overlap_copy8(buf.as_mut_ptr().add(8), buf.as_ptr(), 8) }; + assert_eq!(op2, unsafe { buf.as_mut_ptr().add(16) }); + assert_eq!(ip2, unsafe { buf.as_ptr().add(8) }); + assert_eq!( + &buf[8..16], + &[0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88] + ); +} + +#[test] +fn overlap_copy8_offset_lt_8_spreads_source() { + // offset < 8: dec32table / dec64table spread step. offset = 3. + let mut buf = [0u8; 32]; + buf[..3].copy_from_slice(&[0xAA, 0xBB, 0xCC]); + let (op2, _ip2) = unsafe { overlap_copy8(buf.as_mut_ptr().add(3), buf.as_ptr(), 3) }; + assert_eq!(op2, unsafe { buf.as_mut_ptr().add(11) }); + assert!(buf[3..11].iter().any(|&b| b != 0)); +} + +/// Cross-check: the portable helpers must produce byte-identical +/// output to a straightforward scalar reference for the no-overlap +/// copy across a range of lengths. Guards against a divergence +/// between the u128/u64 unaligned-move lowering and plain copies. +#[test] +fn wildcopy_no_overlap_matches_scalar_reference() { + for len in 1usize..=48 { + let src: [u8; 64] = core::array::from_fn(|i| (i as u8).wrapping_mul(7).wrapping_add(1)); + let mut dst = [0u8; 64]; + unsafe { wildcopy_no_overlap(dst.as_mut_ptr(), src.as_ptr(), len) }; + // Only the first `len` bytes are contractually defined; the + // overshoot tail is allowed to differ. Assert the defined + // region matches. + assert_eq!(&dst[..len], &src[..len], "len={len}"); + } +} diff --git a/zstd/src/decoding/flat_buf.rs b/zstd/src/decoding/flat_buf.rs index b1da2af26..f2306419e 100644 --- a/zstd/src/decoding/flat_buf.rs +++ b/zstd/src/decoding/flat_buf.rs @@ -559,282 +559,4 @@ impl BufferBackend for FlatBuf { } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn with_capacity_starts_empty() { - let f = FlatBuf::with_capacity(1024); - assert_eq!(f.len(), 0); - assert_eq!(f.tail(), 0); - assert!(f.cap() >= 1024 + WILDCOPY_OVERLENGTH); - } - - #[test] - fn extend_appends_then_len_matches() { - let mut f = FlatBuf::with_capacity(64); - f.extend(&[1, 2, 3, 4]); - assert_eq!(f.len(), 4); - f.extend(&[5, 6]); - assert_eq!(f.len(), 6); - let (s1, s2) = f.as_slices(); - assert_eq!(s1, &[1, 2, 3, 4, 5, 6]); - assert!(s2.is_empty(), "flat layout never wraps"); - } - - #[test] - fn extend_and_fill_appends_repeated_byte() { - let mut f = FlatBuf::with_capacity(64); - f.extend(&[0xAA]); - f.extend_and_fill(0xBB, 5); - let (s1, _) = f.as_slices(); - assert_eq!(s1, &[0xAA, 0xBB, 0xBB, 0xBB, 0xBB, 0xBB]); - } - - #[test] - fn extend_from_within_unchecked_copies_non_overlapping() { - let mut f = FlatBuf::with_capacity(64); - f.extend(&[10, 20, 30, 40, 50]); - // SAFETY: start+len=3 <= len()=5; capacity covers 5+3. - unsafe { f.extend_from_within_unchecked(0, 3) }; - let (s1, _) = f.as_slices(); - assert_eq!(s1, &[10, 20, 30, 40, 50, 10, 20, 30]); - } - - #[test] - fn drop_first_n_advances_head() { - let mut f = FlatBuf::with_capacity(64); - f.extend(&[1, 2, 3, 4, 5]); - f.drop_first_n(2); - assert_eq!(f.len(), 3); - let (s1, _) = f.as_slices(); - assert_eq!(s1, &[3, 4, 5]); - // Drained bytes remain physically present and back match copies. - // After head=2, logical start=0 maps to physical index 2. - // SAFETY: start+len=3 <= len()=3. - unsafe { f.extend_from_within_unchecked(0, 3) }; - let (s1, _) = f.as_slices(); - assert_eq!(s1, &[3, 4, 5, 3, 4, 5]); - } - - #[test] - fn set_tail_rolls_back() { - let mut f = FlatBuf::with_capacity(64); - f.extend(&[1, 2, 3]); - let saved_tail = f.tail(); - let saved_cap = f.cap(); - f.extend(&[4, 5, 6, 7]); - assert_eq!(f.len(), 7); - assert_eq!(f.cap(), saved_cap, "with_capacity sized to avoid realloc"); - // SAFETY: cap unchanged; new_tail came from prior tail() call. - unsafe { f.set_tail(saved_tail) }; - assert_eq!(f.len(), 3); - let (s1, _) = f.as_slices(); - assert_eq!(s1, &[1, 2, 3]); - } - - #[test] - fn clear_resets() { - let mut f = FlatBuf::with_capacity(64); - f.extend(&[1, 2, 3]); - f.drop_first_n(1); - assert_eq!(f.len(), 2); - f.clear(); - assert_eq!(f.len(), 0); - assert_eq!(f.tail(), 0); - } - - /// Inline executor — verify match-copy correctness against a - /// byte-by-byte reference. Exercises the non-overlap path - /// (offset >= 16), short-offset overlapCopy8 path (offset < 16), - /// and the literal copy16 + wildcopy tail. Runs on every target: on - /// x86_64 it drives the SSE2 `exec_sequence_inline` arm, elsewhere - /// the portable arm (both `cfg`-selected), giving the non-x86 - /// backend method direct coverage. - #[test] - fn exec_sequence_inline_match_copy_correctness() { - for offset in [4usize, 8, 12, 20, 48, 96] { - let mut f = FlatBuf::with_capacity(512); - // Seed bytes 0..256 with deterministic pattern. - let seed: Vec = (0..256u32).map(|i| ((i * 31 + 7) & 0xFF) as u8).collect(); - f.extend(&seed); - let base = f.len(); - let match_length = 96usize; - // Reference: byte-by-byte repeat starting at base, sourced from base-offset. - let mut reference = alloc::vec![0u8; base + match_length]; - reference[..base].copy_from_slice(&seed); - for i in 0..match_length { - reference[base + i] = reference[base + i - offset]; - } - - let lits = [0xAAu8; 16]; - // SAFETY: lit_length = 0 so lit_src is unused beyond a 16-byte - // over-read into the literal scratch (in-bounds). - unsafe { - f.exec_sequence_inline(lits.as_ptr(), 0, offset, match_length) - .unwrap(); - } - assert_eq!(f.len(), base + match_length, "offset={offset}"); - let (s1, _) = f.as_slices(); - for i in 0..match_length { - assert_eq!( - s1[base + i], - reference[base + i], - "offset={offset} byte {i}: got {:#x}, expected {:#x}", - s1[base + i], - reference[base + i], - ); - } - } - } - - /// AVX2 inline executor — verify match-copy correctness for - /// offsets across the SSE2/AVX2 threshold boundary - /// (offset 20 routes to SSE2 16-byte path, offset 32 to AVX2 - /// 32-byte ymm path, offset 64 to deep AVX2 path). - // AVX2 override is x86_64-only; this test calls it directly. The `std` - // feature gate is required: `is_x86_feature_detected!` is `std`-only, - // unavailable in the crate's `#![no_std]` build. - #[cfg(all(target_arch = "x86_64", feature = "std"))] - #[test] - fn exec_sequence_inline_avx2_offset_boundary_correctness() { - if !std::arch::is_x86_feature_detected!("avx2") { - return; - } - for offset in [20usize, 32, 64] { - let mut f = FlatBuf::with_capacity(512); - let seed: Vec = (0..256u32).map(|i| ((i * 31 + 7) & 0xFF) as u8).collect(); - f.extend(&seed); - let base = f.len(); - let match_length = 96usize; - let mut reference = alloc::vec![0u8; base + match_length]; - reference[..base].copy_from_slice(&seed); - for i in 0..match_length { - reference[base + i] = reference[base + i - offset]; - } - - let lits = [0xAAu8; 16]; - // SAFETY: AVX2 detected via runtime feature check above; - // lit_length = 0 → lit_src 16-byte over-read into scratch. - unsafe { - f.exec_sequence_inline_avx2(lits.as_ptr(), 0, offset, match_length) - .unwrap(); - } - assert_eq!(f.len(), base + match_length, "offset={offset}"); - let (s1, _) = f.as_slices(); - for i in 0..match_length { - assert_eq!( - s1[base + i], - reference[base + i], - "offset={offset} byte {i}: got {:#x}, expected {:#x} \ - (regression: AVX2 wildcopy at offset < 32)", - s1[base + i], - reference[base + i], - ); - } - } - } - - /// Fallible capacity guard — `exec_sequence_inline` MUST return - /// `OutputBufferOverflow` instead of writing past `Vec::capacity()` - /// when the requested write + 15-byte SSE2 overshoot would - /// overflow. Mirrors the contract on `UserSliceBackend`. - #[cfg(target_arch = "x86_64")] - #[test] - fn exec_sequence_inline_capacity_overflow_returns_err() { - // Tiny capacity: 32 bytes + WILDCOPY_OVERLENGTH = 64 total. - let mut f = FlatBuf::with_capacity(32); - f.extend(&[0u8; 16]); - // Request `lit_length + match_length + 15 = 17 + 100 + 15 = 132` - // bytes past tail; well over the 64-byte allocation. The literal - // buffer is `lit_length.next_multiple_of(16) = 32` bytes so the call - // satisfies the inline read-slack precondition even if the capacity - // guard later moves past the first literal read. - let lits = [0xAAu8; 32]; - // SAFETY: error-returning path; no writes performed. - let result = unsafe { f.exec_sequence_inline(lits.as_ptr(), 17, 8, 100) }; - assert!( - matches!( - result, - Err(super::super::errors::ExecuteSequencesError::OutputBufferOverflow { .. }) - ), - "expected OutputBufferOverflow, got {result:?}" - ); - } - - /// AVX2 analogue of the capacity guard: the 32-byte-stride variant MUST - /// also return `OutputBufferOverflow` (not write past `Vec::capacity()`) - /// when the requested write plus the 31-byte overshoot exceeds the - /// remaining headroom. Guards the single-compare bounds check on the AVX2 - /// hot path. - // `std` feature gate required: `is_x86_feature_detected!` is `std`-only. - #[cfg(all(target_arch = "x86_64", feature = "std"))] - #[test] - fn exec_sequence_inline_avx2_capacity_overflow_returns_err() { - if !std::arch::is_x86_feature_detected!("avx2") { - return; - } - let mut f = FlatBuf::with_capacity(32); - f.extend(&[0u8; 16]); - // 32-byte literal buffer = `lit_length.next_multiple_of(16)`, so the - // call satisfies the inline read-slack precondition even if the guard - // later moves past the first literal read. - let lits = [0xAAu8; 32]; - // SAFETY: AVX2 detected above; error-returning path performs no writes. - let result = unsafe { f.exec_sequence_inline_avx2(lits.as_ptr(), 17, 8, 100) }; - assert!( - matches!( - result, - Err(super::super::errors::ExecuteSequencesError::OutputBufferOverflow { .. }) - ), - "expected OutputBufferOverflow, got {result:?}" - ); - } - - /// `FlatBuf` is growable, so the per-block decompression-bomb ceiling - /// (`set_max_capacity`) MUST be honoured on its `try_reserve` growth path — - /// the fallback `push`/`repeat` route a malformed single-segment block can - /// take when the inline slack gate fails. A reserve that would grow live - /// output past the ceiling must return `BackendOverflow` instead of growing - /// the `Vec` toward a decompression-bomb OOM. - #[test] - fn try_reserve_rejects_growth_past_block_ceiling() { - let mut f = FlatBuf::with_capacity(64); - f.extend(&[0u8; 32]); - let ceiling = f.len() + 100; // 132 - f.set_max_capacity(ceiling); - // Within the ceiling: succeeds (32 + 50 = 82 <= 132). - assert!(f.try_reserve(50).is_ok()); - // Past the ceiling (32 + 200 = 232 > 132): rejected, no growth. - assert!( - matches!( - f.try_reserve(200), - Err(super::super::buffer_backend::BackendOverflow { .. }) - ), - "reserve past the per-block ceiling must be rejected, not grown" - ); - } - - /// The ceiling bounds this block's OUTPUT, so a reserve past it must be - /// rejected even when it FITS the existing allocation (no growth needed). - /// A large pre-reserved buffer (e.g. a known-FCS single-segment frame) has - /// spare capacity beyond `MAX_BLOCK_SIZE`; without checking the ceiling - /// ahead of the no-growth fast path, an over-producing block could write - /// into that spare and bypass the decompression-bomb guard. - #[test] - fn try_reserve_rejects_within_capacity_but_past_ceiling() { - let mut f = FlatBuf::with_capacity(4096); - f.extend(&[0u8; 32]); - let ceiling = f.len() + 100; // 132, far below the 4 KiB allocation - f.set_max_capacity(ceiling); - // 32 + 500 = 532 <= 4096 capacity (no growth) but > 132 ceiling. - assert!( - matches!( - f.try_reserve(500), - Err(super::super::buffer_backend::BackendOverflow { .. }) - ), - "a reserve past the ceiling must be rejected even when it fits the \ - current capacity without growth" - ); - } -} +mod tests; diff --git a/zstd/src/decoding/flat_buf/tests.rs b/zstd/src/decoding/flat_buf/tests.rs new file mode 100644 index 000000000..b4300c8ab --- /dev/null +++ b/zstd/src/decoding/flat_buf/tests.rs @@ -0,0 +1,277 @@ +use super::*; + +#[test] +fn with_capacity_starts_empty() { + let f = FlatBuf::with_capacity(1024); + assert_eq!(f.len(), 0); + assert_eq!(f.tail(), 0); + assert!(f.cap() >= 1024 + WILDCOPY_OVERLENGTH); +} + +#[test] +fn extend_appends_then_len_matches() { + let mut f = FlatBuf::with_capacity(64); + f.extend(&[1, 2, 3, 4]); + assert_eq!(f.len(), 4); + f.extend(&[5, 6]); + assert_eq!(f.len(), 6); + let (s1, s2) = f.as_slices(); + assert_eq!(s1, &[1, 2, 3, 4, 5, 6]); + assert!(s2.is_empty(), "flat layout never wraps"); +} + +#[test] +fn extend_and_fill_appends_repeated_byte() { + let mut f = FlatBuf::with_capacity(64); + f.extend(&[0xAA]); + f.extend_and_fill(0xBB, 5); + let (s1, _) = f.as_slices(); + assert_eq!(s1, &[0xAA, 0xBB, 0xBB, 0xBB, 0xBB, 0xBB]); +} + +#[test] +fn extend_from_within_unchecked_copies_non_overlapping() { + let mut f = FlatBuf::with_capacity(64); + f.extend(&[10, 20, 30, 40, 50]); + // SAFETY: start+len=3 <= len()=5; capacity covers 5+3. + unsafe { f.extend_from_within_unchecked(0, 3) }; + let (s1, _) = f.as_slices(); + assert_eq!(s1, &[10, 20, 30, 40, 50, 10, 20, 30]); +} + +#[test] +fn drop_first_n_advances_head() { + let mut f = FlatBuf::with_capacity(64); + f.extend(&[1, 2, 3, 4, 5]); + f.drop_first_n(2); + assert_eq!(f.len(), 3); + let (s1, _) = f.as_slices(); + assert_eq!(s1, &[3, 4, 5]); + // Drained bytes remain physically present and back match copies. + // After head=2, logical start=0 maps to physical index 2. + // SAFETY: start+len=3 <= len()=3. + unsafe { f.extend_from_within_unchecked(0, 3) }; + let (s1, _) = f.as_slices(); + assert_eq!(s1, &[3, 4, 5, 3, 4, 5]); +} + +#[test] +fn set_tail_rolls_back() { + let mut f = FlatBuf::with_capacity(64); + f.extend(&[1, 2, 3]); + let saved_tail = f.tail(); + let saved_cap = f.cap(); + f.extend(&[4, 5, 6, 7]); + assert_eq!(f.len(), 7); + assert_eq!(f.cap(), saved_cap, "with_capacity sized to avoid realloc"); + // SAFETY: cap unchanged; new_tail came from prior tail() call. + unsafe { f.set_tail(saved_tail) }; + assert_eq!(f.len(), 3); + let (s1, _) = f.as_slices(); + assert_eq!(s1, &[1, 2, 3]); +} + +#[test] +fn clear_resets() { + let mut f = FlatBuf::with_capacity(64); + f.extend(&[1, 2, 3]); + f.drop_first_n(1); + assert_eq!(f.len(), 2); + f.clear(); + assert_eq!(f.len(), 0); + assert_eq!(f.tail(), 0); +} + +/// Inline executor — verify match-copy correctness against a +/// byte-by-byte reference. Exercises the non-overlap path +/// (offset >= 16), short-offset overlapCopy8 path (offset < 16), +/// and the literal copy16 + wildcopy tail. Runs on every target: on +/// x86_64 it drives the SSE2 `exec_sequence_inline` arm, elsewhere +/// the portable arm (both `cfg`-selected), giving the non-x86 +/// backend method direct coverage. +#[test] +fn exec_sequence_inline_match_copy_correctness() { + for offset in [4usize, 8, 12, 20, 48, 96] { + let mut f = FlatBuf::with_capacity(512); + // Seed bytes 0..256 with deterministic pattern. + let seed: Vec = (0..256u32).map(|i| ((i * 31 + 7) & 0xFF) as u8).collect(); + f.extend(&seed); + let base = f.len(); + let match_length = 96usize; + // Reference: byte-by-byte repeat starting at base, sourced from base-offset. + let mut reference = alloc::vec![0u8; base + match_length]; + reference[..base].copy_from_slice(&seed); + for i in 0..match_length { + reference[base + i] = reference[base + i - offset]; + } + + let lits = [0xAAu8; 16]; + // SAFETY: lit_length = 0 so lit_src is unused beyond a 16-byte + // over-read into the literal scratch (in-bounds). + unsafe { + f.exec_sequence_inline(lits.as_ptr(), 0, offset, match_length) + .unwrap(); + } + assert_eq!(f.len(), base + match_length, "offset={offset}"); + let (s1, _) = f.as_slices(); + for i in 0..match_length { + assert_eq!( + s1[base + i], + reference[base + i], + "offset={offset} byte {i}: got {:#x}, expected {:#x}", + s1[base + i], + reference[base + i], + ); + } + } +} + +/// AVX2 inline executor — verify match-copy correctness for +/// offsets across the SSE2/AVX2 threshold boundary +/// (offset 20 routes to SSE2 16-byte path, offset 32 to AVX2 +/// 32-byte ymm path, offset 64 to deep AVX2 path). +// AVX2 override is x86_64-only; this test calls it directly. The `std` +// feature gate is required: `is_x86_feature_detected!` is `std`-only, +// unavailable in the crate's `#![no_std]` build. +#[cfg(all(target_arch = "x86_64", feature = "std"))] +#[test] +fn exec_sequence_inline_avx2_offset_boundary_correctness() { + if !std::arch::is_x86_feature_detected!("avx2") { + return; + } + for offset in [20usize, 32, 64] { + let mut f = FlatBuf::with_capacity(512); + let seed: Vec = (0..256u32).map(|i| ((i * 31 + 7) & 0xFF) as u8).collect(); + f.extend(&seed); + let base = f.len(); + let match_length = 96usize; + let mut reference = alloc::vec![0u8; base + match_length]; + reference[..base].copy_from_slice(&seed); + for i in 0..match_length { + reference[base + i] = reference[base + i - offset]; + } + + let lits = [0xAAu8; 16]; + // SAFETY: AVX2 detected via runtime feature check above; + // lit_length = 0 → lit_src 16-byte over-read into scratch. + unsafe { + f.exec_sequence_inline_avx2(lits.as_ptr(), 0, offset, match_length) + .unwrap(); + } + assert_eq!(f.len(), base + match_length, "offset={offset}"); + let (s1, _) = f.as_slices(); + for i in 0..match_length { + assert_eq!( + s1[base + i], + reference[base + i], + "offset={offset} byte {i}: got {:#x}, expected {:#x} \ + (regression: AVX2 wildcopy at offset < 32)", + s1[base + i], + reference[base + i], + ); + } + } +} + +/// Fallible capacity guard — `exec_sequence_inline` MUST return +/// `OutputBufferOverflow` instead of writing past `Vec::capacity()` +/// when the requested write + 15-byte SSE2 overshoot would +/// overflow. Mirrors the contract on `UserSliceBackend`. +#[cfg(target_arch = "x86_64")] +#[test] +fn exec_sequence_inline_capacity_overflow_returns_err() { + // Tiny capacity: 32 bytes + WILDCOPY_OVERLENGTH = 64 total. + let mut f = FlatBuf::with_capacity(32); + f.extend(&[0u8; 16]); + // Request `lit_length + match_length + 15 = 17 + 100 + 15 = 132` + // bytes past tail; well over the 64-byte allocation. The literal + // buffer is `lit_length.next_multiple_of(16) = 32` bytes so the call + // satisfies the inline read-slack precondition even if the capacity + // guard later moves past the first literal read. + let lits = [0xAAu8; 32]; + // SAFETY: error-returning path; no writes performed. + let result = unsafe { f.exec_sequence_inline(lits.as_ptr(), 17, 8, 100) }; + assert!( + matches!( + result, + Err(super::super::errors::ExecuteSequencesError::OutputBufferOverflow { .. }) + ), + "expected OutputBufferOverflow, got {result:?}" + ); +} + +/// AVX2 analogue of the capacity guard: the 32-byte-stride variant MUST +/// also return `OutputBufferOverflow` (not write past `Vec::capacity()`) +/// when the requested write plus the 31-byte overshoot exceeds the +/// remaining headroom. Guards the single-compare bounds check on the AVX2 +/// hot path. +// `std` feature gate required: `is_x86_feature_detected!` is `std`-only. +#[cfg(all(target_arch = "x86_64", feature = "std"))] +#[test] +fn exec_sequence_inline_avx2_capacity_overflow_returns_err() { + if !std::arch::is_x86_feature_detected!("avx2") { + return; + } + let mut f = FlatBuf::with_capacity(32); + f.extend(&[0u8; 16]); + // 32-byte literal buffer = `lit_length.next_multiple_of(16)`, so the + // call satisfies the inline read-slack precondition even if the guard + // later moves past the first literal read. + let lits = [0xAAu8; 32]; + // SAFETY: AVX2 detected above; error-returning path performs no writes. + let result = unsafe { f.exec_sequence_inline_avx2(lits.as_ptr(), 17, 8, 100) }; + assert!( + matches!( + result, + Err(super::super::errors::ExecuteSequencesError::OutputBufferOverflow { .. }) + ), + "expected OutputBufferOverflow, got {result:?}" + ); +} + +/// `FlatBuf` is growable, so the per-block decompression-bomb ceiling +/// (`set_max_capacity`) MUST be honoured on its `try_reserve` growth path — +/// the fallback `push`/`repeat` route a malformed single-segment block can +/// take when the inline slack gate fails. A reserve that would grow live +/// output past the ceiling must return `BackendOverflow` instead of growing +/// the `Vec` toward a decompression-bomb OOM. +#[test] +fn try_reserve_rejects_growth_past_block_ceiling() { + let mut f = FlatBuf::with_capacity(64); + f.extend(&[0u8; 32]); + let ceiling = f.len() + 100; // 132 + f.set_max_capacity(ceiling); + // Within the ceiling: succeeds (32 + 50 = 82 <= 132). + assert!(f.try_reserve(50).is_ok()); + // Past the ceiling (32 + 200 = 232 > 132): rejected, no growth. + assert!( + matches!( + f.try_reserve(200), + Err(super::super::buffer_backend::BackendOverflow { .. }) + ), + "reserve past the per-block ceiling must be rejected, not grown" + ); +} + +/// The ceiling bounds this block's OUTPUT, so a reserve past it must be +/// rejected even when it FITS the existing allocation (no growth needed). +/// A large pre-reserved buffer (e.g. a known-FCS single-segment frame) has +/// spare capacity beyond `MAX_BLOCK_SIZE`; without checking the ceiling +/// ahead of the no-growth fast path, an over-producing block could write +/// into that spare and bypass the decompression-bomb guard. +#[test] +fn try_reserve_rejects_within_capacity_but_past_ceiling() { + let mut f = FlatBuf::with_capacity(4096); + f.extend(&[0u8; 32]); + let ceiling = f.len() + 100; // 132, far below the 4 KiB allocation + f.set_max_capacity(ceiling); + // 32 + 500 = 532 <= 4096 capacity (no growth) but > 132 ceiling. + assert!( + matches!( + f.try_reserve(500), + Err(super::super::buffer_backend::BackendOverflow { .. }) + ), + "a reserve past the ceiling must be rejected even when it fits the \ + current capacity without growth" + ); +} diff --git a/zstd/src/decoding/frame_decoder.rs b/zstd/src/decoding/frame_decoder.rs index b50c94661..97b10c8da 100644 --- a/zstd/src/decoding/frame_decoder.rs +++ b/zstd/src/decoding/frame_decoder.rs @@ -3135,2369 +3135,4 @@ impl Read for FrameDecoder { } #[cfg(test)] -mod tests { - extern crate std; - - use super::{DictionaryHandle, FrameDecoder}; - use crate::encoding::{CompressionLevel, FrameCompressor}; - use alloc::vec::Vec; - - #[test] - fn decode_all_tight_and_slack_outputs_match_on_single_segment_frame() { - // Roundtrip a small payload through the encoder, then decode - // it via `decode_all` on two output shapes that select - // different internal sequence-exec paths within the direct - // decode: - // 1. Tight output (exactly `frame_content_size`, no - // WILDCOPY_OVERLENGTH slack) → direct path whose trailing - // sequence(s) take the bounded (non-overshooting) copy in - // `UserSliceBackend::exec_sequence_bounded`. - // 2. Output with WILDCOPY slack → direct path whose - // sequences all take the SIMD wildcopy fast path. - // Both must produce identical output bytes — the bounded tail - // copy must reconstruct the same data as the overshooting fast - // path. This is the regression gate for the relaxed - // direct-decode gate (`cap >= content_size`). - let payload: Vec = (0..4096u32).map(|i| (i & 0xFF) as u8).collect(); - let mut compressor = FrameCompressor::new(CompressionLevel::Default); - compressor.set_source(payload.as_slice()); - let mut compressed = Vec::new(); - compressor.set_drain(&mut compressed); - compressor.compress(); - - // Baseline: tight output → legacy drain path. - let mut dec_a = FrameDecoder::new(); - let mut out_a = alloc::vec![0u8; payload.len()]; - let n_a = dec_a - .decode_all(compressed.as_slice(), &mut out_a) - .expect("decode_all (legacy drain) should succeed"); - assert_eq!(n_a, payload.len()); - assert_eq!(&out_a[..n_a], payload.as_slice()); - - // Direct: output with WILDCOPY slack → direct path. - let slack = super::super::buffer_backend::WILDCOPY_OVERLENGTH; - let mut dec_b = FrameDecoder::new(); - let mut out_b = alloc::vec![0u8; payload.len() + slack]; - let n_b = dec_b - .decode_all(compressed.as_slice(), &mut out_b) - .expect("decode_all (direct path) should succeed"); - assert_eq!( - n_b, - payload.len(), - "direct decode produced wrong byte count" - ); - assert_eq!(&out_b[..n_b], payload.as_slice()); - } - - #[test] - fn decode_all_tight_output_overlapping_tail_match_roundtrips() { - // The bounded tail copy must handle an OVERLAPPING match - // (offset < match_length) as the trailing sequence when the - // output slice is sized to exactly `frame_content_size`. A long - // run of a single byte at the end of the payload encodes as an - // offset-1 match whose length far exceeds the offset, so the - // bounded copy's overlapping (forward byte-by-byte) branch is - // exercised at the buffer tail where the SIMD overshoot would - // otherwise run past `cap`. Decoding into a tight buffer and - // matching the original payload byte-for-byte is the regression - // gate for the overlap branch of `exec_sequence_bounded`. - let mut payload: Vec = (0..256u32).map(|i| (i & 0xFF) as u8).collect(); - payload.extend(core::iter::repeat_n(0xABu8, 8192)); - let mut compressor = FrameCompressor::new(CompressionLevel::Default); - compressor.set_source(payload.as_slice()); - let mut compressed = Vec::new(); - compressor.set_drain(&mut compressed); - compressor.compress(); - - // Anti-vacuous precondition: the 8 KiB trailing run of a single - // byte must compress to a Compressed block dominated by ONE long - // offset-1 (overlapping, offset < match_length) match — not a Raw - // block. If the encoder ever stopped emitting that overlapping - // tail match the test would pass without exercising - // `exec_sequence_bounded`'s overlapping forward-copy branch, so - // gate on the output being a tiny fraction of the input (a raw - // block would be ~`payload.len()`; an offset-1 run match is tens - // of bytes). - assert!( - compressed.len() < payload.len() / 8, - "expected an overlapping-tail match to dominate the frame \ - (compressed={} payload={}); the bounded overlap branch would \ - not be exercised otherwise", - compressed.len(), - payload.len(), - ); - - // Tight output: exactly content_size, no WILDCOPY slack. - let mut dec = FrameDecoder::new(); - let mut out = alloc::vec![0u8; payload.len()]; - let n = dec - .decode_all(compressed.as_slice(), &mut out) - .expect("tight-output decode with overlapping tail match should succeed"); - assert_eq!(n, payload.len()); - assert_eq!(out, payload, "bounded overlap tail copy corrupted output"); - } - - #[test] - fn decode_all_multi_segment_frame_decodes_correctly() { - // Multi-segment frame: payload large enough that the - // encoder's default frame layout has `single_segment_flag = - // false` and `window_size < frame_content_size`. The direct - // path must cap the visible buffer at window_size after each - // block (drop_to_window_size) so match-offset validation - // matches the spec rule `offset <= window_size`, and still - // produce the same bytes as decode_all on the - // FlatBuf/Ring-backed path. - // - // Make the payload structured so multi-segment behavior - // actually kicks in: 2 MiB of repeating + random-ish bytes - // forces window_size lower than content_size at the encoder. - let mut payload: Vec = Vec::with_capacity(2 * 1024 * 1024); - for i in 0..payload.capacity() { - payload.push((i.wrapping_mul(2_654_435_761) & 0xFF) as u8); - } - let mut compressor = FrameCompressor::new(CompressionLevel::Default); - compressor.set_source(payload.as_slice()); - let mut compressed = Vec::new(); - compressor.set_drain(&mut compressed); - compressor.compress(); - - // Baseline: decode_all through the FlatBuf+drain path. - let mut dec_a = FrameDecoder::new(); - let mut out_a = alloc::vec![0u8; payload.len()]; - let n_a = dec_a - .decode_all(compressed.as_slice(), &mut out_a) - .expect("decode_all should succeed"); - assert_eq!(n_a, payload.len()); - assert_eq!(&out_a[..n_a], payload.as_slice()); - - // Direct path: must give identical bytes via UserSliceBackend - // + per-block drop_to_window_size. - let slack = super::super::buffer_backend::WILDCOPY_OVERLENGTH; - let mut dec_b = FrameDecoder::new(); - let mut out_b = alloc::vec![0u8; payload.len() + slack]; - let n_b = dec_b - .decode_all(compressed.as_slice(), &mut out_b) - .expect("decode_all should succeed on multi-segment frame"); - assert_eq!(n_b, payload.len(), "wrong byte count on direct path"); - assert_eq!(&out_b[..n_b], payload.as_slice()); - - // Sanity-check: confirm the encoded frame really IS - // multi-segment. If a future encoder default changes, - // catching the assumption here is better than silently - // testing single_segment on this name. - let mut sanity = FrameDecoder::new(); - sanity.init(&mut compressed.as_slice()).unwrap(); - assert!( - !sanity - .state - .as_ref() - .unwrap() - .frame_header - .descriptor - .single_segment_flag(), - "test precondition violated: frame is single-segment, rename or resize" - ); - } - - #[cfg(feature = "hash")] - #[test] - fn decode_all_propagates_checksum_into_persistent_scratch() { - // Direct path on a checksum-flagged frame: the FrameCompressor - // under `feature = "hash"` sets content_checksum_flag, so the - // decoded frame has a recorded checksum. After - // decode_all we must be able to verify it matches via - // the public get_calculated_checksum() accessor — the digest - // is computed by walking output at end of decode and stored - // into the persistent scratch's hasher. - let payload: Vec = (0..8192u32).map(|i| (i & 0xFF) as u8).collect(); - let mut compressor = FrameCompressor::new(CompressionLevel::Default); - compressor.set_content_checksum(true); - compressor.set_source(payload.as_slice()); - let mut compressed = Vec::new(); - compressor.set_drain(&mut compressed); - compressor.compress(); - - let slack = super::super::buffer_backend::WILDCOPY_OVERLENGTH; - let mut dec = FrameDecoder::new(); - let mut out = alloc::vec![0u8; payload.len() + slack]; - let n = dec - .decode_all(compressed.as_slice(), &mut out) - .expect("decode_all with checksum must succeed"); - assert_eq!(n, payload.len()); - assert_eq!(&out[..n], payload.as_slice()); - - // Both sides must report the same checksum: the frame header - // carries the stored u32, and get_calculated_checksum reads - // the running digest the direct path just propagated. - let stored = dec.get_checksum_from_data(); - let calculated = dec.get_calculated_checksum(); - assert!(stored.is_some(), "frame must carry stored checksum"); - assert!( - calculated.is_some(), - "direct path must propagate calculated checksum" - ); - assert_eq!( - stored, calculated, - "stored vs calculated checksum mismatch on direct path" - ); - } - - #[cfg(feature = "hash")] - #[test] - fn verify_mode_accepts_a_valid_frame() { - use crate::decoding::ContentChecksum; - let payload: Vec = (0..8192u32).map(|i| (i & 0xFF) as u8).collect(); - let mut compressor = FrameCompressor::new(CompressionLevel::Default); - compressor.set_content_checksum(true); - compressor.set_source(payload.as_slice()); - let mut compressed = Vec::new(); - compressor.set_drain(&mut compressed); - compressor.compress(); - - let slack = super::super::buffer_backend::WILDCOPY_OVERLENGTH; - let mut dec = FrameDecoder::new(); - dec.set_content_checksum(ContentChecksum::Verify); - let mut out = alloc::vec![0u8; payload.len() + slack]; - let n = dec - .decode_all(compressed.as_slice(), &mut out) - .expect("Verify mode must accept a frame with a correct checksum"); - assert_eq!(&out[..n], payload.as_slice()); - } - - #[cfg(feature = "hash")] - #[test] - fn verify_mode_rejects_a_corrupted_checksum() { - use crate::decoding::ContentChecksum; - use crate::decoding::errors::FrameDecoderError; - let payload: Vec = (0..8192u32).map(|i| (i & 0xFF) as u8).collect(); - let mut compressor = FrameCompressor::new(CompressionLevel::Default); - compressor.set_content_checksum(true); - compressor.set_source(payload.as_slice()); - let mut compressed = Vec::new(); - compressor.set_drain(&mut compressed); - compressor.compress(); - - // Flip a bit in the trailing 4-byte content checksum: the frame body - // still decodes to the correct bytes, but the stored digest no longer - // matches the one the decoder computes. - let last = compressed.len() - 1; - compressed[last] ^= 0xFF; - - let slack = super::super::buffer_backend::WILDCOPY_OVERLENGTH; - let mut dec = FrameDecoder::new(); - dec.set_content_checksum(ContentChecksum::Verify); - let mut out = alloc::vec![0u8; payload.len() + slack]; - let err = dec - .decode_all(compressed.as_slice(), &mut out) - .expect_err("Verify mode must reject a corrupted checksum"); - assert!( - matches!(err, FrameDecoderError::ChecksumMismatch { .. }), - "expected ChecksumMismatch, got {err:?}" - ); - } - - #[cfg(feature = "hash")] - #[test] - fn decode_from_to_verify_rejects_corrupted_checksum() { - // decode_from_to has its own block loop (not decode_blocks); it must - // still honour Verify and reject a corrupted trailer rather than - // silently accept it. - use crate::decoding::ContentChecksum; - use crate::decoding::errors::FrameDecoderError; - let payload: Vec = (0..8192u32).map(|i| (i & 0xFF) as u8).collect(); - let mut compressor = FrameCompressor::new(CompressionLevel::Default); - compressor.set_content_checksum(true); - compressor.set_source(payload.as_slice()); - let mut compressed = Vec::new(); - compressor.set_drain(&mut compressed); - compressor.compress(); - let last = compressed.len() - 1; - compressed[last] ^= 0xFF; - - let slack = super::super::buffer_backend::WILDCOPY_OVERLENGTH; - let mut dec = FrameDecoder::new(); - dec.set_content_checksum(ContentChecksum::Verify); - let mut out = alloc::vec![0u8; payload.len() + slack]; - - // Split the trailing 4-byte checksum into a SEPARATE call so the - // verification must happen on the checksum-only early-return path (not - // the post-drain path) — the incremental case CodeRabbit flagged. - let split = compressed.len() - 4; - let (_r1, w1) = dec - .decode_from_to(&compressed[..split], &mut out) - .expect("blocks decode without the trailer"); - let err = dec - .decode_from_to(&compressed[split..], &mut out[w1..]) - .expect_err("decode_from_to in Verify mode must reject a corrupted checksum"); - assert!( - matches!(err, FrameDecoderError::ChecksumMismatch { .. }), - "expected ChecksumMismatch, got {err:?}" - ); - } - - #[cfg(feature = "hash")] - #[test] - fn decode_from_to_small_target_split_trailer_flushes_tail() { - // Regression: when a prior call decoded the last block but a small - // `target` left output buffered, the trailer-only call must still flush - // the buffered tail (it used to early-return Ok((4,0)) and lose it). - let payload: Vec = (0..8192u32).map(|i| (i & 0xFF) as u8).collect(); - let mut compressor = FrameCompressor::new(CompressionLevel::Default); - compressor.set_content_checksum(true); - compressor.set_source(payload.as_slice()); - let mut compressed = Vec::new(); - compressor.set_drain(&mut compressed); - compressor.compress(); - - let split = compressed.len() - 4; - let mut dec = FrameDecoder::new(); - let mut out = alloc::vec![0u8; payload.len()]; - // Call 1: all blocks, but a SMALL (64-byte) target leaves the rest - // buffered on the decoder side. - let (_r1, w1) = dec - .decode_from_to(&compressed[..split], &mut out[..64]) - .expect("blocks decode with a small target"); - assert!(w1 <= 64); - // Call 2: the 4-byte trailer alone must flush the buffered tail through - // the shared read path, not return early and drop it. - let (_r2, w2) = dec - .decode_from_to(&compressed[split..], &mut out[w1..]) - .expect("trailer call must flush the buffered tail"); - assert_eq!(w1 + w2, payload.len(), "buffered tail was dropped"); - assert_eq!(&out[..w1 + w2], payload.as_slice()); - } - - #[cfg(feature = "hash")] - #[test] - fn none_mode_skips_the_checksum_pass() { - use crate::decoding::ContentChecksum; - let payload: Vec = (0..8192u32).map(|i| (i & 0xFF) as u8).collect(); - let mut compressor = FrameCompressor::new(CompressionLevel::Default); - compressor.set_content_checksum(true); - compressor.set_source(payload.as_slice()); - let mut compressed = Vec::new(); - compressor.set_drain(&mut compressed); - compressor.compress(); - - let slack = super::super::buffer_backend::WILDCOPY_OVERLENGTH; - let mut dec = FrameDecoder::new(); - dec.set_content_checksum(ContentChecksum::None); - let mut out = alloc::vec![0u8; payload.len() + slack]; - let n = dec - .decode_all(compressed.as_slice(), &mut out) - .expect("None mode must still decode correctly"); - assert_eq!(&out[..n], payload.as_slice()); - // No digest is computed in None mode, even though the frame carries one. - assert!(dec.get_checksum_from_data().is_some()); - assert!(dec.get_calculated_checksum().is_none()); - } - - #[cfg(feature = "hash")] - #[test] - fn encoder_without_checksum_emits_no_trailing_digest() { - let payload: Vec = (0..8192u32).map(|i| (i & 0xFF) as u8).collect(); - - let mut with = Vec::new(); - let mut c_with = FrameCompressor::new(CompressionLevel::Default); - c_with.set_content_checksum(true); - c_with.set_source(payload.as_slice()); - c_with.set_drain(&mut with); - c_with.compress(); - - let mut without = Vec::new(); - let mut c_without = FrameCompressor::new(CompressionLevel::Default); - c_without.set_content_checksum(false); - c_without.set_source(payload.as_slice()); - c_without.set_drain(&mut without); - c_without.compress(); - - // The checksum-off frame is exactly the 4-byte trailing digest shorter. - assert_eq!(with.len(), without.len() + 4); - - let slack = super::super::buffer_backend::WILDCOPY_OVERLENGTH; - let mut dec = FrameDecoder::new(); - let mut out = alloc::vec![0u8; payload.len() + slack]; - let n = dec - .decode_all(without.as_slice(), &mut out) - .expect("a frame without a content checksum must decode"); - assert_eq!(&out[..n], payload.as_slice()); - assert!( - dec.get_checksum_from_data().is_none(), - "no trailing checksum should be reported" - ); - } - - #[test] - fn decode_all_fcs_overflow_via_corrupt_frame_returns_structured_error() { - // Hand-build a corrupt frame that declares - // frame_content_size = 4 but the (last) block carries a - // larger Raw payload. The pre-flight FCS check inside the - // direct path's block loop catches this and returns the - // structured FrameContentSizeMismatch variant — not a - // panic, not a generic TargetTooSmall. - // - // Frame layout (single_segment, FCS=4): - // magic 4 bytes 0xFD2FB528 - // FHD 1 byte single_segment=1, no checksum, - // FCS field size = 0 (-> 1-byte FCS) - // FCS 1 byte 0x04 - // block_header 3 bytes last=1, type=Raw, block_size=10 - // block_payload 10 bytes 0xAA repeated - let mut frame = alloc::vec::Vec::new(); - // magic - frame.extend_from_slice(&0xFD2FB528u32.to_le_bytes()); - // FHD: single_segment=1, fcs_flag=0 (1-byte FCS), no checksum, - // no dict. Bit layout: FCS(7-6)=0, single_segment(5)=1, - // reserved/uncs(4)=0, content_checksum(2)=0, dict(0-1)=00. - frame.push(0b0010_0000); - // FCS: 1 byte - frame.push(4); - // Block header: cBlockSize=10, type=Raw (0), last=1 - // 3-byte LE: bit0=last, bits1-2=type(2 bits), bits3-23=size - let cblock_size: u32 = 10; - let bh: u32 = 1 | (cblock_size << 3); // last=1, type=Raw=0 - frame.push((bh & 0xFF) as u8); - frame.push((bh >> 8) as u8); - frame.push((bh >> 16) as u8); - // Payload — 10 bytes that, if decoded, would exceed FCS=4. - frame.extend(core::iter::repeat_n(0xAAu8, 10)); - - let slack = super::super::buffer_backend::WILDCOPY_OVERLENGTH; - let mut dec = FrameDecoder::new(); - let mut out = alloc::vec![0u8; 4 + slack]; - let err = dec - .decode_all(&frame, &mut out) - .expect_err("FCS-overflow frame must fail decode"); - assert!( - matches!( - err, - super::FrameDecoderError::FrameContentSizeMismatch { .. } - ), - "expected FrameContentSizeMismatch, got {:?}", - err - ); - } - - #[test] - fn decode_all_compressed_block_fcs_overflow_returns_structured_error() { - // Acceptance test for #246: a malformed frame whose *Compressed* - // block expands past the declared `frame_content_size` must - // surface `FrameContentSizeMismatch` from the direct-decode path - // (UserSliceBackend sequence executor), NOT panic and NOT a - // generic FailedToReadBlockBody. The Raw-block sibling above - // covers the `BackendOverflow` arm; this covers the Compressed - // sequence-executor overflow arm (`ExecuteSequencesError:: - // OutputBufferOverflow` folded into FrameContentSizeMismatch in - // `run_direct_decode`). - // - // Construction: compress a compressible payload to get a genuine - // Compressed block + a header-declared FCS, then surgically patch - // the FCS field down to a tiny value. The block body still - // decodes (literals/sequences are independent of FCS) and the - // sequence executor overflows the small output slice. - // Highly compressible payload (repeated phrase) → Compressed - // block whose sequence executor produces ~4 KiB of output. - let unit = b"The quick brown fox jumps over the lazy dog. "; - let mut payload = Vec::with_capacity(4 * 1024); - while payload.len() < 4 * 1024 { - payload.extend_from_slice(unit); - } - let mut compressor = FrameCompressor::new(CompressionLevel::Default); - compressor.set_source(payload.as_slice()); - let mut frame = Vec::new(); - compressor.set_drain(&mut frame); - compressor.compress(); - // Sanity: the encoder actually compressed (=> a Compressed block, - // not a raw-stored fallback) so we exercise the sequence path. - assert!(frame.len() < payload.len()); - - // Locate the FCS field: it is the last `fcs_len` bytes of the - // frame header, whose total size `header_size` includes the magic. - // A ~4 KiB single-segment frame declares FCS = 4096, which lands in - // the 2-byte field range [256, 65791] (RFC 8878 §3.1.1.1.4) — assert - // that so the patch logic below stays a single deterministic branch. - let (header, header_size) = - super::super::frame::read_frame_header(frame.as_slice()).expect("valid header"); - let fcs_len = header - .descriptor - .frame_content_size_bytes() - .expect("fcs present") as usize; - assert_eq!( - fcs_len, 2, - "4 KiB single-segment frame must use a 2-byte FCS" - ); - let fcs_off = header_size as usize - fcs_len; - - // Patch the 2-byte FCS to its floor: stored bytes 0 decode to 256 - // (the field's `+256` bias), far below the 4 KiB the block actually - // produces, so the sequence executor overflows the output slice. - let patched_declared: u64 = 256; - frame[fcs_off] = 0; - frame[fcs_off + 1] = 0; - - // Size the output to declared + WILDCOPY slack so the direct path - // is eligible (output.len() >= content_size + slack) — the - // overflow then comes from the frame, not an undersized buffer. - let slack = super::super::buffer_backend::WILDCOPY_OVERLENGTH; - let mut out = alloc::vec![0u8; patched_declared as usize + slack]; - let mut dec = FrameDecoder::new(); - let err = dec - .decode_all(frame.as_slice(), &mut out) - .expect_err("Compressed block exceeding FCS must fail decode"); - match err { - super::FrameDecoderError::FrameContentSizeMismatch { declared, produced } => { - assert_eq!(declared, patched_declared, "declared echoes patched FCS"); - assert!(produced > declared, "produced must exceed declared"); - } - other => panic!("expected FrameContentSizeMismatch, got {other:?}"), - } - } - - /// Block-precise error positions (#174): a failing block header / body - /// reports its 0-based index and frame-absolute offset, consistent with - /// the encoder's `FrameEmitInfo.blocks[index].offset_in_frame`. - #[cfg(feature = "lsm")] - #[test] - fn block_precise_errors_carry_index_and_offset() { - use crate::encoding::{CompressionLevel, FrameCompressor}; - // ~1.3 MiB of incompressible (xorshift) bytes → many 128 KiB raw - // blocks, so blocks 3 and 7 both exist and are not the last block. - let mut data = alloc::vec::Vec::with_capacity(1_300_000); - let mut s: u64 = 0x2545_F491_4F6C_DD1D; - while data.len() < 1_300_000 { - s ^= s << 13; - s ^= s >> 7; - s ^= s << 17; - data.push((s >> 33) as u8); - } - - let mut frame = alloc::vec::Vec::new(); - let blocks = { - let mut fc = FrameCompressor::new(CompressionLevel::Level(1)); - fc.set_source(data.as_slice()); - fc.set_drain(&mut frame); - fc.compress(); - fc.last_frame_emit_info() - .expect("emit info present under lsm") - .blocks - .clone() - }; - assert!(blocks.len() > 7, "need >7 blocks, got {}", blocks.len()); - - let mut out = alloc::vec![0u8; data.len() + 4096]; - - // (1) Corrupt block 7's header: force its Block_Type to Reserved (3) - // by setting both type bits — fails the header read at block 7. - let off7 = blocks[7].offset_in_frame as usize; - let mut corrupt = frame.clone(); - corrupt[off7] |= 0b0000_0110; - let mut dec = FrameDecoder::new(); - let err = dec - .decode_all(&corrupt, &mut out) - .expect_err("reserved block-7 header must fail"); - match err { - super::FrameDecoderError::FailedToReadBlockHeaderAt { - block_index, - frame_offset, - .. - } => { - assert_eq!(block_index, 7); - assert_eq!(frame_offset, blocks[7].offset_in_frame); - } - other => panic!("expected FailedToReadBlockHeaderAt, got {other:?}"), - } - - // (2) Truncate at block 3's body start: header intact, body missing - // → the body decode fails at block 3 with its FrameBlock metadata. - let body3 = blocks[3].offset_in_frame as usize + blocks[3].header_size as usize; - let mut dec = FrameDecoder::new(); - let err = dec - .decode_all(&frame[..body3], &mut out) - .expect_err("truncated block-3 body must fail"); - match err { - super::FrameDecoderError::FailedToReadBlockBodyAt { - block_index, - frame_offset, - block, - .. - } => { - assert_eq!(block_index, 3); - assert_eq!(frame_offset, blocks[3].offset_in_frame); - assert_eq!(block.offset_in_frame, blocks[3].offset_in_frame); - } - other => panic!("expected FailedToReadBlockBodyAt, got {other:?}"), - } - } - - #[test] - fn decode_all_exact_fit_output_decodes_correctly() { - // Output sized exactly to frame_content_size (no - // WILDCOPY_OVERLENGTH slack) is now eligible for the direct - // path: every output-write site is exact-fit-safe (sequence - // exec falls back to the bounded, non-overshooting copy on the - // trailing sequence(s), Raw/RLE blocks copy exactly). This must - // produce the same bytes as a slack-padded buffer. Exercised on - // x86 through the per-kernel AVX2/SSE2 inline-exec macros, which - // carry the same tight-tail branch. - let payload: Vec = (0..2048u32) - .map(|i| (i.wrapping_mul(31) & 0xFF) as u8) - .collect(); - let mut compressor = FrameCompressor::new(CompressionLevel::Default); - compressor.set_source(payload.as_slice()); - let mut compressed = Vec::new(); - compressor.set_drain(&mut compressed); - compressor.compress(); - - let mut dec = FrameDecoder::new(); - // Exactly payload.len(), no slack. - let mut out = alloc::vec![0u8; payload.len()]; - let n = dec - .decode_all(compressed.as_slice(), &mut out) - .expect("exact-fit decode_all should succeed"); - assert_eq!(n, payload.len()); - assert_eq!(&out[..n], payload.as_slice()); - } - - #[test] - fn decode_all_fallback_validates_fcs_against_total_output() { - // Synthetic single-segment frame: FCS = 20 bytes, but the - // last-block flag fires after only 4 bytes of raw payload. - // On the direct path this would trip the post-block - // `produced > content_size` check; the fallback path - // (eligible=false because output is sized exactly to FCS, - // no WILDCOPY slack) used to silently return Ok(4). With - // the fix it now surfaces `FrameContentSizeMismatch` - // matching the direct path. - // - // Frame layout: 4 B magic | 1 B FHD (single_segment=1, - // FCS_flag=3 → 8-byte FCS) | 8 B FCS=20 | block header - // (Raw, last, size=4) | 4 raw bytes. - let mut wire = Vec::new(); - wire.extend_from_slice(&0xFD2F_B528u32.to_le_bytes()); // magic - // FHD: FCS_flag=3 (8-byte FCS) <<6 | single_segment=1 <<5. - wire.push(0b1110_0000); - wire.extend_from_slice(&20u64.to_le_bytes()); // declared FCS - // Block header: (size << 3) | (block_type << 1) | last_block. - // Raw block (block_type=0), last_block=1, size=4 → 0b00100001 = 0x21. - wire.push(0x21); - wire.push(0x00); - wire.push(0x00); - wire.extend_from_slice(&[1u8, 2, 3, 4]); - - let mut dec = FrameDecoder::new(); - // Size output SMALLER than the declared FCS so direct-decode is - // gated out (`output.len() >= content_size` is false) and the - // frame takes the legacy fallback drain loop — the path this test - // guards. The corrupt frame only produces 4 bytes, so 19 is ample - // room; the point is `19 != declared FCS (20)`. - const DECLARED_FCS: usize = 20; - let mut out = alloc::vec![0u8; DECLARED_FCS - 1]; - assert_ne!( - out.len(), - DECLARED_FCS, - "output must be smaller than FCS to exercise the fallback path", - ); - let err = dec - .decode_all(wire.as_slice(), &mut out) - .expect_err("fallback must reject corrupt FCS underflow"); - match err { - crate::decoding::errors::FrameDecoderError::FrameContentSizeMismatch { - declared, - produced, - } => { - assert_eq!(declared, 20); - assert_eq!(produced, 4); - } - other => panic!("expected FrameContentSizeMismatch, got {other:?}"), - } - } - - #[test] - fn decode_all_fallback_treats_explicit_fcs_zero_as_declared() { - // Synthetic multi-segment frame with FCS_flag=2 (4-byte - // FCS) explicitly set to 0. The header DECLARES zero - // content, but the body carries a 5-byte raw last-block. - // `fcs_declared()` must return true (the field is on the - // wire) so the fallback's post-decode size check sees the - // mismatch — even though `frame_content_size == 0`. This - // is exactly the FCS=0 edge case where the previous - // `content_size > 0` proxy would have silently accepted - // the corrupt frame. - // - // Frame layout: - // 4 B magic — 28 B5 2F FD - // 1 B FHD — FCS_flag=2 (bits 7-6), no - // single_segment, content_checksum=0, - // dict_id_flag=0 → 0b1000_0000 - // 1 B window_descriptor — exp=10, mantissa=0 → window=1 MiB - // 4 B FCS — 0 LE - // 3 B block header — raw, last, size=5 → 0x29 0x00 0x00 - // 5 B raw payload — anything non-empty - let mut wire = Vec::new(); - wire.extend_from_slice(&0xFD2F_B528u32.to_le_bytes()); - wire.push(0b1000_0000); // FHD: FCS_flag=2, others 0. - wire.push(0x50); // window_descriptor: exp=10, mantissa=0. - wire.extend_from_slice(&0u32.to_le_bytes()); // FCS = 0. - // Block header (24-bit LE): (size << 3) | (block_type << 1) | last_block - // = (5 << 3) | (0 << 1) | 1 = 0x29. - wire.push(0x29); - wire.push(0x00); - wire.push(0x00); - wire.extend_from_slice(&[1u8, 2, 3, 4, 5]); - - let mut dec = FrameDecoder::new(); - // FCS=0 declared, so eligibility (`content_size > 0`) - // false — falls through to the drain loop. Output buffer - // size doesn't matter for the eligibility check here; - // give it some room so `read()` can drain the block. - let mut out = alloc::vec![0u8; 16]; - let err = dec - .decode_all(wire.as_slice(), &mut out) - .expect_err("corrupt FCS=0 + 5-byte block must error"); - match err { - crate::decoding::errors::FrameDecoderError::FrameContentSizeMismatch { - declared, - produced, - } => { - assert_eq!(declared, 0); - assert_eq!(produced, 5); - } - other => panic!("expected FrameContentSizeMismatch, got {other:?}"), - } - } - - #[test] - fn decode_all_fallback_accepts_honest_explicit_fcs_zero() { - // Companion to the corrupt-FCS=0 test above: an HONEST - // empty frame with FCS_flag=2 (4-byte FCS) explicitly set - // to 0 AND a 0-byte raw last-block. `fcs_declared()` - // returns true and `content_size == 0 == total_written`, - // so the fallback validation accepts the frame instead of - // misreporting a mismatch. - // - // (Single-segment FCS=0 would test a similar invariant - // but trips header-stage validation: `window_size = - // frame_content_size = 0 < MIN_WINDOW_SIZE` fails the - // window-size sanity check before decode runs. Use the - // multi-segment shape where `window_size` comes from - // `window_descriptor` independently of FCS.) - // - // Frame layout: - // 4 B magic - // 1 B FHD — FCS_flag=2, others 0 → 0x80 - // 1 B window_descriptor — exp=10 → 1 MiB window - // 4 B FCS — 0 LE - // 3 B block header — raw, last, size=0 → 0x01 0x00 0x00 - let mut wire = Vec::new(); - wire.extend_from_slice(&0xFD2F_B528u32.to_le_bytes()); - wire.push(0b1000_0000); - wire.push(0x50); - wire.extend_from_slice(&0u32.to_le_bytes()); - // Block header: (0 << 3) | (0 << 1) | 1 = 0x01. - wire.push(0x01); - wire.push(0x00); - wire.push(0x00); - - let mut dec = FrameDecoder::new(); - let mut out = alloc::vec![0u8; 16]; - let n = dec - .decode_all(wire.as_slice(), &mut out) - .expect("honest FCS=0 + empty block must succeed"); - assert_eq!(n, 0); - } - - #[test] - fn reset_with_dict_handle_applies_dict_when_no_dict_id() { - let payload = b"reset-without-dict-id"; - let mut compressor = FrameCompressor::new(CompressionLevel::Default); - compressor.set_source(payload.as_slice()); - let mut compressed = Vec::new(); - compressor.set_drain(&mut compressed); - compressor.compress(); - - let dict_raw = include_bytes!("../../dict_tests/dictionary"); - let handle = DictionaryHandle::decode_dict(dict_raw).expect("dictionary should parse"); - - let mut decoder = FrameDecoder::new(); - decoder - .reset_with_dict_handle(compressed.as_slice(), &handle) - .expect("reset should succeed"); - let state = decoder.state.as_ref().expect("state should be initialized"); - assert!(state.frame_header.dictionary_id().is_none()); - assert_eq!(state.using_dict, Some(handle.id())); - } - - #[test] - fn reserve_buffer_reserves_the_shortfall_not_the_full_window_again() { - // `Vec::reserve_exact` takes ADDITIONAL capacity. The decode_all - // fallback loop re-enters decode_blocks once per strategy chunk, - // and each entry pre-reserves the window: re-requesting the FULL - // window on a buffer already holding ~window bytes of history - // would grow it toward 2x window, defeating the peak-memory cap - // the exact-growth policy exists for. - use super::DecoderScratchKind; - let window = 1usize << 20; - let mut scratch = DecoderScratchKind::new_flat(window); - scratch.reserve_buffer(window); - let data = alloc::vec![0u8; window]; - match &mut scratch { - super::DecoderScratchKind::Flat(s) => s.buffer.push(&data), - super::DecoderScratchKind::Ring(_) => unreachable!("new_flat builds Flat"), - } - scratch.reserve_buffer(window); - let workspace = scratch.workspace_bytes(); - assert!( - workspace < window * 3 / 2, - "second reserve_buffer grew a full window past the buffered \ - history: workspace {workspace} bytes vs window {window}" - ); - } - - #[test] - fn dict_frame_decodes_through_direct_path() { - // A dictionary frame decoded via `decode_all_with_dict_handle` - // into a buffer sized exactly to FCS takes the direct path - // (UserSliceBackend); matches reaching into the dictionary - // content must resolve through `repeat_from_dict`. The payload - // embeds dictionary content verbatim so the encoder emits - // dict-region matches from the first bytes of the frame. - let dict_raw = include_bytes!("../../dict_tests/dictionary"); - let handle = DictionaryHandle::decode_dict(dict_raw).expect("dictionary should parse"); - let dict_tail: alloc::vec::Vec = handle - .as_dict() - .dict_content - .iter() - .rev() - .take(2048) - .rev() - .copied() - .collect(); - // No in-frame duplicate of the dictionary bytes: with a second - // copy in the payload the encoder may emit the later copy as an - // in-frame match, and the test would stay green even if the - // direct path stopped forwarding the dictionary handle. A - // single copy forces every dict-region match through - // `repeat_from_dict`. - let mut payload = dict_tail; - payload.extend_from_slice(b"unique suffix after dictionary material 0123456789"); - - let mut compressor = FrameCompressor::new(CompressionLevel::Default); - compressor - .set_dictionary_from_bytes(dict_raw) - .expect("dict load"); - compressor.set_source(payload.as_slice()); - let mut compressed = Vec::new(); - compressor.set_drain(&mut compressed); - compressor.compress(); - - // Fixture sanity: the frame must actually depend on the - // dictionary, otherwise the decode below never exercises - // dict-region match resolution. - let mut plain = Vec::new(); - let mut no_dict = FrameCompressor::new(CompressionLevel::Default); - no_dict.set_source(payload.as_slice()); - no_dict.set_drain(&mut plain); - no_dict.compress(); - assert!( - compressed.len() < plain.len(), - "fixture must depend on the dictionary: dict {} bytes vs plain {} bytes", - compressed.len(), - plain.len() - ); - - let mut decoder = FrameDecoder::new(); - let mut out = alloc::vec![0u8; payload.len()]; - let n = decoder - .decode_all_with_dict_handle(compressed.as_slice(), &mut out, &handle) - .expect("dict frame must decode on the direct path"); - assert_eq!(n, payload.len()); - assert_eq!(out, payload, "direct-path dict decode must be byte-exact"); - // Both paths are byte-identical, so pin the dispatch itself: a - // re-introduced dict exclusion in the direct gate would silently - // fall back to the buffered path and leave the asserts above green. - assert_eq!( - decoder.direct_frames, 1, - "dict frame must take the direct path, not the buffered fallback" - ); - } - - #[test] - fn implausible_content_size_skips_eager_alloc_direct_path() { - // Adversarial frame: a 1 KiB window (small ring) but a declared - // content size of 4 MiB, followed by a truncated raw block. The - // direct path would `resize` the caller's Vec to the pledged 4 MiB - // (allocating + zeroing it) BEFORE the truncated body is validated. - // The gate must reject the implausible size (4 MiB cannot come from - // 3 compressed bytes) and fall through to the window-bounded ring - // drain, which errors without ever allocating the pledged size. - // - // Hand-built so the declared size is fully decoupled from the real - // (tiny) input — the encoder always writes a truthful FCS. - let frame: &[u8] = &[ - 0x28, 0xB5, 0x2F, 0xFD, // magic - 0x80, // FHD: multi-segment, 4-byte FCS field, no dict - 0x00, // window descriptor -> 1 KiB window - 0x00, 0x00, 0x40, 0x00, // FCS = 4 MiB - 0x21, 0x03, 0x00, // raw block header: last, size 100, no body - ]; - - let mut dec = FrameDecoder::new(); - let mut src = frame; - dec.init(&mut src).expect("header must parse"); - // `src` now points past the header at the truncated 3-byte block. - let mut out = Vec::new(); - let err = dec.decode_current_frame_to_vec(src, &mut out, None); - assert!( - err.is_err(), - "truncated body must fail regardless of decode path" - ); - assert_eq!( - dec.direct_frames, 0, - "implausible FCS must NOT take the eager-alloc direct path" - ); - } - - #[test] - fn implausible_single_segment_fcs_rejected_before_window_reservation() { - // Single-segment adversarial frame: the window equals the declared - // content size (4 MiB) by definition, so the fallback ring drain would - // pre-reserve that whole window via `useful_window_size()` before the - // truncated body errors — the multi-segment gate test does not cover - // this. The implausible size (4 MiB cannot come from 3 compressed - // bytes) must be rejected up front with a content-size error, NOT a - // block-body error after the reservation. - let frame: &[u8] = &[ - 0x28, 0xB5, 0x2F, 0xFD, // magic - 0xA0, // FHD: single-segment, 4-byte FCS field - 0x00, 0x00, 0x40, 0x00, // FCS = 4 MiB (== window for single-segment) - 0x21, 0x03, 0x00, // raw block header: last, size 100, no body - ]; - - let mut dec = FrameDecoder::new(); - let mut src = frame; - dec.init(&mut src).expect("header must parse"); - let mut out = Vec::new(); - let err = dec - .decode_current_frame_to_vec(src, &mut out, None) - .expect_err("implausible single-segment FCS must be rejected"); - match err { - super::FrameDecoderError::FrameContentSizeMismatch { declared, .. } => { - assert_eq!(declared, 4 * 1024 * 1024); - } - other => panic!( - "expected early FrameContentSizeMismatch (no window reservation), got {other:?}" - ), - } - assert_eq!( - dec.direct_frames, 0, - "implausible FCS must not take the eager-alloc direct path" - ); - } - - #[cfg(feature = "lsm")] - mod expect_validation { - use super::*; - use crate::decoding::errors::FrameDecoderError; - - fn compress(payload: &[u8]) -> Vec { - let mut compressor = FrameCompressor::new(CompressionLevel::Default); - compressor.set_source(payload); - let mut compressed = Vec::new(); - compressor.set_drain(&mut compressed); - compressor.compress(); - compressed - } - - fn compress_with_dict(payload: &[u8], dict_raw: &[u8]) -> Vec { - let mut compressor = FrameCompressor::new(CompressionLevel::Default); - compressor - .set_dictionary_from_bytes(dict_raw) - .expect("dict load"); - compressor.set_source(payload); - let mut compressed = Vec::new(); - compressor.set_drain(&mut compressed); - compressor.compress(); - compressed - } - - #[test] - fn expect_dict_id_none_default_allows_anything() { - let compressed = compress(b"hello-no-expect"); - let mut decoder = FrameDecoder::new(); - decoder - .reset(compressed.as_slice()) - .expect("default None passes"); - } - - #[test] - fn expect_dict_id_zero_matches_frame_without_dict_id() { - // Default-encoded frame has no dict_id; pinning Some(0) - // ("no dictionary expected") must accept it. - let compressed = compress(b"payload"); - let mut decoder = FrameDecoder::new(); - decoder.expect_dict_id(Some(0)); - decoder - .reset(compressed.as_slice()) - .expect("Some(0) ~ None"); - } - - #[test] - fn expect_dict_id_matching_value_passes() { - let dict_raw = include_bytes!("../../dict_tests/dictionary"); - let handle = DictionaryHandle::decode_dict(dict_raw).expect("dict parse"); - let actual_id = handle.id(); - - let compressed = compress_with_dict(b"payload-with-dict", dict_raw); - - let mut decoder = FrameDecoder::new(); - decoder.expect_dict_id(Some(actual_id)); - // Decode requires the dict to be registered; using - // reset_with_dict_handle for that. - decoder - .reset_with_dict_handle(compressed.as_slice(), &handle) - .expect("matching dict_id passes"); - } - - #[test] - fn expect_dict_id_mismatching_value_fails_before_decode() { - let dict_raw = include_bytes!("../../dict_tests/dictionary"); - let handle = DictionaryHandle::decode_dict(dict_raw).expect("dict parse"); - let actual_id = handle.id(); - let wrong_id = actual_id.wrapping_add(1); - - let compressed = compress_with_dict(b"payload-with-dict", dict_raw); - - let mut decoder = FrameDecoder::new(); - decoder.expect_dict_id(Some(wrong_id)); - let err = decoder - .reset_with_dict_handle(compressed.as_slice(), &handle) - .expect_err("mismatch must fail"); - match err { - FrameDecoderError::UnexpectedDictId { expected, found } => { - assert_eq!(expected, Some(wrong_id)); - assert_eq!(found, Some(actual_id)); - } - other => panic!("expected UnexpectedDictId, got {other:?}"), - } - } - - #[test] - fn expect_dict_id_nonzero_fails_on_frame_without_dict_id() { - // Frame has no dict_id; expecting Some(42) (non-zero) - // must fail with found = None. - let compressed = compress(b"no-dict-frame"); - let mut decoder = FrameDecoder::new(); - decoder.expect_dict_id(Some(42)); - let err = decoder - .reset(compressed.as_slice()) - .expect_err("nonzero expectation on dictless frame must fail"); - match err { - FrameDecoderError::UnexpectedDictId { expected, found } => { - assert_eq!(expected, Some(42)); - assert_eq!(found, None); - } - other => panic!("expected UnexpectedDictId, got {other:?}"), - } - } - - #[test] - fn expect_window_descriptor_none_default_allows_anything() { - let compressed = compress(b"hello-no-wd-expect"); - let mut decoder = FrameDecoder::new(); - decoder - .reset(compressed.as_slice()) - .expect("default None passes"); - } - - #[test] - fn expect_window_descriptor_mismatch_fails_before_decode() { - // Compress a payload large enough to force a - // multi-segment frame (window_descriptor on wire). - // Default compression at >256 KiB produces multi- - // segment frames with a real window_descriptor byte. - let payload = alloc::vec![0xABu8; 512 * 1024]; - let compressed = compress(&payload); - - // Read the actual window_descriptor by decoding once - // without expectations, then pin a wrong value. - let mut probe_decoder = FrameDecoder::new(); - probe_decoder.reset(compressed.as_slice()).unwrap(); - let probe_state = probe_decoder.state.as_ref().unwrap(); - let actual_wd = probe_state - .frame_header - .window_descriptor() - .expect("multi-segment frame should expose window_descriptor"); - let wrong_wd = actual_wd.wrapping_add(0x10); // bump exponent - - let mut decoder = FrameDecoder::new(); - decoder.expect_window_descriptor(Some(wrong_wd)); - let err = decoder - .reset(compressed.as_slice()) - .expect_err("wrong window_descriptor must fail"); - match err { - FrameDecoderError::UnexpectedWindowDescriptor { expected, found } => { - assert_eq!(expected, wrong_wd); - assert_eq!(found, Some(actual_wd)); - } - other => panic!("expected UnexpectedWindowDescriptor, got {other:?}"), - } - } - - /// Build a minimal synthetic single-segment zstd frame - /// carrying a 4-byte raw payload. RFC 8878 §3.1.1.1 - /// layout, hand-rolled because our default - /// `FrameCompressor` settings don't emit - /// `single_segment_flag` for tiny inputs. - /// - /// Wire bytes (13 total for 4-byte payload): - /// ```text - /// 28 B5 2F FD magic - /// 20 FHD: single_segment=1, FCS_flag=0 - /// 04 FCS (single byte, value = payload.len()) - /// 21 00 00 block header: raw, last, size=4 - /// .. .. .. .. payload bytes - /// ``` - fn synth_single_segment_frame(payload: &[u8]) -> Vec { - assert!(payload.len() <= 255, "1-byte FCS field caps at 255"); - assert!(payload.len() < (1usize << 21), "block size 21-bit max"); - let mut out = Vec::new(); - // Magic 0xFD2FB528 LE. - out.extend_from_slice(&0xFD2F_B528u32.to_le_bytes()); - // FHD: single_segment_flag (bit 5) set, everything - // else zero. With single_segment + FCS_flag=0 the FCS - // field is 1 byte. No window_descriptor on wire. - out.push(0b0010_0000); - // 1-byte FCS = payload length. - out.push(payload.len() as u8); - // Block header (3 bytes LE): - // last_block=1, block_type=0 (Raw), block_size=payload.len(). - // Encoded: (size << 3) | (block_type << 1) | last_block. - // Block header: last_block flag in bit 0, block_type - // (0 = Raw) in bits 1-2, block size in bits 3+. - let bh: u32 = ((payload.len() as u32) << 3) | 1; - out.push((bh & 0xFF) as u8); - out.push(((bh >> 8) & 0xFF) as u8); - out.push(((bh >> 16) & 0xFF) as u8); - // Raw payload. - out.extend_from_slice(payload); - out - } - - #[test] - fn expect_window_descriptor_on_single_segment_frame_fails_with_found_none() { - // Single-segment frames omit the window_descriptor - // byte from the wire entirely. Setting an expectation - // here must surface `found: None` so callers - // distinguish "wrong descriptor" from "no descriptor - // on the wire" — never silently pass. - let compressed = synth_single_segment_frame(b"tiny"); - - // First sanity-check: the synthetic frame decodes - // cleanly without any expectation. - { - let mut probe = FrameDecoder::new(); - probe - .reset(compressed.as_slice()) - .expect("synth frame parses"); - let probe_state = probe.state.as_ref().unwrap(); - assert!( - probe_state.frame_header.window_descriptor().is_none(), - "synth frame must be single-segment" - ); - } - - let mut decoder = FrameDecoder::new(); - decoder.expect_window_descriptor(Some(0x40)); - let err = decoder - .reset(compressed.as_slice()) - .expect_err("single-segment + expectation must fail"); - match err { - FrameDecoderError::UnexpectedWindowDescriptor { expected, found } => { - assert_eq!(expected, 0x40); - assert_eq!(found, None); - } - other => panic!("expected UnexpectedWindowDescriptor, got {other:?}"), - } - } - - #[test] - fn validation_failure_leaves_decoder_re_resettable() { - // After UnexpectedDictId on a wrong-expectation reset, - // clearing the expectation and re-calling reset must - // succeed on the same source — no lingering failed - // state. - let compressed = compress(b"re-resettable"); - - let mut decoder = FrameDecoder::new(); - decoder.expect_dict_id(Some(42)); - let err = decoder - .reset(compressed.as_slice()) - .expect_err("first reset fails"); - assert!(matches!(err, FrameDecoderError::UnexpectedDictId { .. })); - - // Clear expectation and retry on a fresh source. - decoder.expect_dict_id(None); - decoder - .reset(compressed.as_slice()) - .expect("retry after clearing expectation should succeed"); - } - } - - /// Build a skippable frame on the wire: 4-byte LE magic + 4-byte LE - /// length + payload bytes. RFC 8878 §3.1.2 restricts the magic - /// variant to `0..=15`; assert here so accidental misuse of the - /// helper can't smuggle a non-skippable magic past the tests. - #[cfg(feature = "lsm")] - fn build_skippable_frame(variant: u8, payload: &[u8]) -> Vec { - assert!( - variant <= 15, - "skippable-frame variant {variant} outside RFC 8878 0..=15 range", - ); - let mut out = Vec::with_capacity(8 + payload.len()); - let magic: u32 = 0x184D2A50 + u32::from(variant); - out.extend_from_slice(&magic.to_le_bytes()); - out.extend_from_slice(&u32::try_from(payload.len()).unwrap().to_le_bytes()); - out.extend_from_slice(payload); - out - } - - #[cfg(feature = "lsm")] - #[test] - fn decode_all_with_skippable_visitor_sees_payloads_in_order() { - // Build a stream: skippable(v0, "alpha") + zstd_frame + - // skippable(v3, "beta") + zstd_frame + skippable(v15, "") - // and verify the visitor is invoked exactly three times with - // the correct (variant, payload) pairs in stream order while - // the zstd frames decode normally. - let payload_a: Vec = (0..256u16).map(|i| i as u8).collect(); - let payload_b: Vec = (0..256u16).map(|i| (i ^ 0xAA) as u8).collect(); - - let mut comp_a = Vec::new(); - let mut c = FrameCompressor::new(CompressionLevel::Default); - c.set_source(payload_a.as_slice()); - c.set_drain(&mut comp_a); - c.compress(); - - let mut comp_b = Vec::new(); - let mut c = FrameCompressor::new(CompressionLevel::Default); - c.set_source(payload_b.as_slice()); - c.set_drain(&mut comp_b); - c.compress(); - - let skip0 = build_skippable_frame(0, b"alpha"); - let skip3 = build_skippable_frame(3, b"beta"); - let skip15 = build_skippable_frame(15, &[]); - - let mut stream = Vec::new(); - stream.extend_from_slice(&skip0); - stream.extend_from_slice(&comp_a); - stream.extend_from_slice(&skip3); - stream.extend_from_slice(&comp_b); - stream.extend_from_slice(&skip15); - - let mut decoder = FrameDecoder::new(); - let mut out = alloc::vec![0u8; payload_a.len() + payload_b.len()]; - let mut collected: Vec<(u8, Vec)> = Vec::new(); - let n = decoder - .decode_all_with_skippable_visitor(stream.as_slice(), &mut out, |variant, payload| { - collected.push((variant, payload.to_vec())); - }) - .expect("decode_all_with_skippable_visitor should succeed"); - - // All three skippables visited in stream order. - assert_eq!(collected.len(), 3); - assert_eq!(collected[0], (0u8, b"alpha".to_vec())); - assert_eq!(collected[1], (3u8, b"beta".to_vec())); - assert_eq!(collected[2], (15u8, Vec::::new())); - - // Both zstd frames decoded into `out` back-to-back. - assert_eq!(n, payload_a.len() + payload_b.len()); - assert_eq!(&out[..payload_a.len()], payload_a.as_slice()); - assert_eq!(&out[payload_a.len()..n], payload_b.as_slice()); - } - - #[cfg(feature = "lsm")] - #[test] - fn decode_all_silently_skips_when_no_visitor() { - // Regression gate: plain decode_all must still silently skip - // skippable frames (RFC 8878 mandated behavior) with no - // behavioral change after the visitor refactor. - let payload: Vec = (0..512u16).map(|i| i as u8).collect(); - let mut comp = Vec::new(); - let mut c = FrameCompressor::new(CompressionLevel::Default); - c.set_source(payload.as_slice()); - c.set_drain(&mut comp); - c.compress(); - - let skip = build_skippable_frame(7, b"ignored sidecar"); - let mut stream = Vec::new(); - stream.extend_from_slice(&skip); - stream.extend_from_slice(&comp); - - let mut decoder = FrameDecoder::new(); - let mut out = alloc::vec![0u8; payload.len()]; - let n = decoder - .decode_all(stream.as_slice(), &mut out) - .expect("decode_all should succeed on skippable + zstd stream"); - assert_eq!(n, payload.len()); - assert_eq!(&out[..n], payload.as_slice()); - } - - #[cfg(feature = "lsm")] - #[test] - fn frame_emit_info_describes_emitted_block_layout() { - // Encode a payload large enough to force >1 block, fetch - // FrameEmitInfo, walk blocks[] and verify each block's - // (offset_in_frame, header_size, body_size) matches the bytes - // actually emitted into the drain buffer. - let payload: Vec = (0..200_000u32).map(|i| (i & 0xFF) as u8).collect(); - let mut compressor = FrameCompressor::new(CompressionLevel::Default); - // Content checksum is opt-in (library default mirrors libzstd's - // checksum-off); request it so the checksum_range assertion below - // exercises the hash-gated trailer accounting. - compressor.set_content_checksum(true); - compressor.set_source(payload.as_slice()); - let mut compressed = Vec::new(); - compressor.set_drain(&mut compressed); - compressor.compress(); - - let info = compressor - .last_frame_emit_info() - .expect("last_frame_emit_info populated after compress") - .clone(); - drop(compressor); - - // Frame header range starts at 0 and is non-empty. - assert_eq!(info.frame_header_range.start, 0); - assert!(info.frame_header_range.end > 0); - // Total size matches what was written to the drain. - assert_eq!(info.total_size as usize, compressed.len()); - // At least one block, and the last entry has last_block=true. - assert!(!info.blocks.is_empty()); - assert!(info.blocks.last().unwrap().last_block); - // All non-final blocks have last_block=false. - for b in &info.blocks[..info.blocks.len() - 1] { - assert!(!b.last_block); - } - // Walk and verify each block's header bytes match the - // recorded type / size by re-decoding the 3-byte header. - // Walking arithmetic: offset_in_frame + header_size + body_size - // must land exactly on the next block's offset_in_frame (or, - // for the last block, on the checksum / end of frame). - for (i, b) in info.blocks.iter().enumerate() { - let off = b.offset_in_frame as usize; - assert_eq!(b.header_size, 3); - let mut hdr = [0u8; 4]; - hdr[..3].copy_from_slice(&compressed[off..off + 3]); - let raw = u32::from_le_bytes(hdr); - let last = (raw & 1) != 0; - let ty = (raw >> 1) & 0b11; - let sz = raw >> 3; - assert_eq!(last, b.last_block); - assert_eq!(sz, b.block_size_field); - // body_size is the PHYSICAL length on the wire: spec's - // Block_Size for Raw/Compressed, always 1 for RLE. - let expected_physical = match b.block_type { - crate::encoding::frame_emit_info::BlockType::RLE => 1, - _ => sz, - }; - assert_eq!(b.body_size, expected_physical); - let expected_ty = match b.block_type { - crate::encoding::frame_emit_info::BlockType::Raw => 0, - crate::encoding::frame_emit_info::BlockType::RLE => 1, - crate::encoding::frame_emit_info::BlockType::Compressed => 2, - crate::encoding::frame_emit_info::BlockType::Reserved => 3, - }; - assert_eq!(ty, expected_ty); - // Walking-arithmetic invariant. - let next_off = b.offset_in_frame + b.header_size as u32 + b.body_size; - if let Some(next) = info.blocks.get(i + 1) { - assert_eq!( - next_off, next.offset_in_frame, - "block {i} body_size doesn't reach next block's offset_in_frame", - ); - } else if let Some(cs) = info.checksum_range.as_ref() { - assert_eq!( - next_off, cs.start, - "last block body_size doesn't reach checksum_range.start", - ); - } else { - assert_eq!( - next_off, info.total_size, - "last block body_size doesn't reach total_size", - ); - } - } - // Checksum range present iff `feature = "hash"` is enabled. - assert_eq!(info.checksum_range.is_some(), cfg!(feature = "hash")); - } - - #[cfg(all(feature = "lsm", feature = "hash"))] - #[test] - fn per_block_checksum_round_trip() { - // Encode with per-block checksums enabled. Decode with - // per-block verification. Both sides emit exactly 1 - // checksum per physical block written to / read from the - // wire (encoder hashes per emission site, including each - // post-split partition; decoder hashes each decoded block). - // Cardinality and element-wise contents must match - // round-trip. - let payload: Vec = (0..200_000u32).map(|i| (i & 0xFF) as u8).collect(); - let mut compressor = FrameCompressor::new(CompressionLevel::Default); - compressor.set_source(payload.as_slice()); - compressor.enable_per_block_checksums(); - let mut compressed = Vec::new(); - compressor.set_drain(&mut compressed); - compressor.compress(); - - let encoder_checksums = compressor - .last_frame_block_checksums() - .expect("checksums populated after enable + compress") - .to_vec(); - drop(compressor); - assert!(!encoder_checksums.is_empty()); - - // Decode side: enable verification, decode, compare. - let mut decoder = FrameDecoder::new(); - decoder.enable_per_block_checksums(); - let mut output = alloc::vec![0u8; payload.len()]; - let n = decoder - .decode_all(compressed.as_slice(), &mut output) - .expect("decode_all should succeed"); - assert_eq!(n, payload.len()); - assert_eq!(&output[..n], payload.as_slice()); - - let decoder_checksums = decoder.computed_block_checksums(); - assert_eq!(decoder_checksums, encoder_checksums.as_slice()); - } - - // ── decode_blocks_partial (block-subset partial decode, lsm) ── - - /// Build a multi-block compressible frame and return - /// `(compressed, full_decode, emit_info)`. The emit info's - /// `decompressed_byte_range` maps decompressed offsets to block indices. - #[cfg(feature = "lsm")] - fn multi_block_fixture() -> ( - Vec, - Vec, - crate::encoding::frame_emit_info::FrameEmitInfo, - ) { - let mut data: Vec = Vec::with_capacity(400 * 1024); - let mut x = 0x9E37_79B9u32; - while data.len() < 400 * 1024 { - x ^= x << 13; - x ^= x >> 17; - x ^= x << 5; - let run = 16 + (x as usize % 48); - let byte = (x >> 24) as u8; - for _ in 0..run { - data.push(byte); - } - data.extend_from_slice(b"the quick brown fox jumps over the lazy dog\n"); - } - - let mut compressed = Vec::new(); - let mut compressor = FrameCompressor::new(CompressionLevel::Default); - compressor.set_source(data.as_slice()); - compressor.set_drain(&mut compressed); - compressor.compress(); - let info = compressor - .last_frame_emit_info() - .expect("emit info populated") - .clone(); - drop(compressor); - - let mut dec = FrameDecoder::new(); - let mut full = alloc::vec![0u8; data.len()]; - let n = dec - .decode_all(compressed.as_slice(), &mut full) - .expect("full decode"); - full.truncate(n); - assert_eq!(full, data, "fixture must round-trip"); - (compressed, full, info) - } - - #[cfg(feature = "lsm")] - #[test] - fn decode_blocks_partial_subset_matches_full_decode() { - let (compressed, full, info) = multi_block_fixture(); - let nblocks = info.blocks.len() as u32; - assert!( - nblocks >= 4, - "fixture must have several blocks, got {nblocks}" - ); - let half = nblocks / 2; - // Boundaries: 1 block, 2 blocks, half, all, and a non-zero start. - // `(0, u32::MAX)` exercises the "decode to end of frame" sentinel, - // a distinct public contract from an explicit upper bound. - for &(s, e) in &[ - (0u32, u32::MAX), - (0, 1), - (0, 2), - (0, half), - (0, nblocks), - (1, 2), - (half, nblocks), - ] { - // The sentinel decodes through the last block; map it to nblocks - // for the expected-slice / block-count arithmetic below. - let effective_end = if e == u32::MAX { nblocks } else { e }; - let mut source = compressed.as_slice(); - let mut dec = FrameDecoder::new(); - dec.reset(&mut source).unwrap(); - let pd = dec - .decode_blocks_partial(&mut source, s, e, None, false) - .unwrap_or_else(|err| panic!("range [{s},{e}) errored: {err:?}")); - - let start = info.decompressed_byte_range(s as usize).unwrap().start as usize; - let end = info - .decompressed_byte_range((effective_end - 1) as usize) - .unwrap() - .end as usize; - assert_eq!( - pd.data.as_slice(), - &full[start..end], - "subset bytes must equal the full-decode slice for [{s},{e})" - ); - assert_eq!(pd.start_block, s); - assert_eq!(pd.blocks_decoded, effective_end - s); - assert!(pd.stopped_at.is_none(), "clean range [{s},{e})"); - } - } - - #[cfg(feature = "lsm")] - #[test] - fn decode_blocks_partial_recovers_clean_prefix_on_truncated_block() { - let (compressed, full, info) = multi_block_fixture(); - let nblocks = info.blocks.len(); - let k = nblocks / 2; - assert!(k >= 1, "need a clean prefix before the failing block"); - - // Truncate the source right after block k's 3-byte header, so its body - // read fails regardless of block type (0 body bytes available). - let cut = info.blocks[k].offset_in_frame as usize + info.blocks[k].header_size as usize; - let truncated = &compressed[..cut]; - - let mut source = truncated; - let mut dec = FrameDecoder::new(); - dec.reset(&mut source).unwrap(); - let pd = dec - .decode_blocks_partial(&mut source, 0, u32::MAX, None, false) - .unwrap(); - - let (idx, _err) = pd.stopped_at.expect("must stop on the truncated block"); - assert_eq!(idx, k as u32, "stopped at the truncated block index"); - assert_eq!(pd.blocks_decoded, k as u32, "blocks 0..k decoded cleanly"); - assert!(!pd.frame_finished); - let clean_end = info.decompressed_byte_range(k).unwrap().start as usize; - assert_eq!( - pd.data.as_slice(), - &full[..clean_end], - "clean prefix preserved through the failure" - ); - } - - #[cfg(feature = "lsm")] - #[test] - fn decode_blocks_partial_invalid_range_errors() { - let (compressed, _full, _info) = multi_block_fixture(); - let mut source = compressed.as_slice(); - let mut dec = FrameDecoder::new(); - dec.reset(&mut source).unwrap(); - let err = dec - .decode_blocks_partial(&mut source, 5, 2, None, false) - .expect_err("start > end must error"); - assert!(matches!( - err, - crate::decoding::errors::FrameDecoderError::InvalidBlockRange { - start_block: 5, - end_block: 2, - } - )); - } - - #[cfg(feature = "lsm")] - #[test] - fn decode_blocks_partial_skips_trailing_blocks() { - let (compressed, full, info) = multi_block_fixture(); - assert!(info.blocks.len() >= 3); - let mut source = compressed.as_slice(); - let mut dec = FrameDecoder::new(); - dec.reset(&mut source).unwrap(); - let pd = dec - .decode_blocks_partial(&mut source, 0, 1, None, false) - .unwrap(); - - assert_eq!(pd.blocks_decoded, 1); - assert!(pd.stopped_at.is_none()); - assert!(!pd.frame_finished, "block 0 is not the last block"); - let end = info.decompressed_byte_range(0).unwrap().end as usize; - assert_eq!(pd.data.as_slice(), &full[..end]); - // The trailing blocks + checksum were never consumed from the source. - assert!( - dec.bytes_read_from_source() < u64::from(info.total_size), - "only block 0's region should be consumed, read {} of {}", - dec.bytes_read_from_source(), - info.total_size - ); - } - - #[cfg(feature = "lsm")] - #[test] - fn lsm_style_range_query_partial_recovery() { - // Simulates lsm-tree's range-query path: a key range resolves to a - // decompressed byte window, which maps to inner zstd block indices via - // `decompressed_byte_range`; decode only the covering blocks and check - // the wanted window is recovered exactly (no key outside, all inside). - let (compressed, full, info) = multi_block_fixture(); - let total = full.len() as u64; - let want_start = total / 3; - let want_end = (total * 2) / 3; - - // Map [want_start, want_end) to covering block indices. - let nblocks = info.blocks.len(); - let mut start_block = 0u32; - let mut end_block = nblocks as u32; - for i in 0..nblocks { - let r = info.decompressed_byte_range(i).unwrap(); - if r.start <= want_start && want_start < r.end { - start_block = i as u32; - } - if r.start < want_end && want_end <= r.end { - end_block = i as u32 + 1; - break; - } - } - - let mut source = compressed.as_slice(); - let mut dec = FrameDecoder::new(); - dec.reset(&mut source).unwrap(); - let pd = dec - .decode_blocks_partial(&mut source, start_block, end_block, None, false) - .unwrap(); - assert!(pd.stopped_at.is_none()); - - let covered_start = info - .decompressed_byte_range(start_block as usize) - .unwrap() - .start; - let covered_end = info - .decompressed_byte_range((end_block - 1) as usize) - .unwrap() - .end; - assert!( - covered_start <= want_start && want_end <= covered_end, - "covering blocks must contain the wanted window" - ); - assert_eq!( - pd.data.as_slice(), - &full[covered_start as usize..covered_end as usize], - "covered subset must equal the full-decode slice" - ); - // Slice the exact key range out of the covered subset. - let off = (want_start - covered_start) as usize; - let len = (want_end - want_start) as usize; - assert_eq!( - &pd.data[off..off + len], - &full[want_start as usize..want_end as usize], - "exact key range recovered from the partial decode" - ); - } - - #[cfg(feature = "lsm")] - #[test] - fn decode_blocks_partial_leaves_no_residual_when_no_in_range_block() { - // Regression: when the requested range reaches no in-range block (here - // start_block is past EOF, so every block is decoded only as window - // context), `PartialDecode::data` is empty — but the context bytes must - // NOT linger in the decoder buffer, or a later collect()/read() on the - // same decoder returns out-of-range data. - let (compressed, _full, info) = multi_block_fixture(); - let nblocks = info.blocks.len() as u32; - let mut source = compressed.as_slice(); - let mut dec = FrameDecoder::new(); - dec.reset(&mut source).unwrap(); - let pd = dec - .decode_blocks_partial(&mut source, nblocks + 5, u32::MAX, None, false) - .unwrap(); - assert!(pd.data.is_empty(), "no in-range block → empty data"); - assert_eq!(pd.blocks_decoded, 0); - assert!( - pd.frame_finished, - "frame's last block was reached as context" - ); - assert_eq!( - dec.can_collect(), - 0, - "context bytes must not leak via collect()/read() when data is empty" - ); - } - - #[cfg(feature = "lsm")] - #[test] - fn decode_blocks_partial_empty_range_leaves_no_residual() { - // Companion to the start-past-EOF case: an in-frame empty range `[k, k)` - // (k < EOF) takes the same `prefix_window_len == None` path but with - // `frame_finished == false` and up to `window_size` context bytes still - // physically present. Assert the buffer is fully cleared directly (a - // `can_collect()` check alone would pass even with <= window_size bytes - // retained, because it holds the window back). - let (compressed, _full, info) = multi_block_fixture(); - let k = ((info.blocks.len() as u32) / 2).max(1); - let mut source = compressed.as_slice(); - let mut dec = FrameDecoder::new(); - dec.reset(&mut source).unwrap(); - let pd = dec - .decode_blocks_partial(&mut source, k, k, None, false) - .unwrap(); - - assert!(pd.data.is_empty(), "empty range must yield empty data"); - assert_eq!(pd.blocks_decoded, 0); - assert!( - !pd.frame_finished, - "frame should still have trailing blocks" - ); - assert_eq!( - dec.state.as_ref().unwrap().decoder_scratch.buffer_len(), - 0, - "empty-range partial decode must not retain context bytes" - ); - } - - #[cfg(all(feature = "lsm", feature = "hash"))] - #[test] - fn decode_blocks_partial_captures_per_block_checksums() { - // Regression: with per-block checksums enabled, decode_blocks_partial - // must populate computed_block_checksums just like decode_blocks / - // decode_all — otherwise callers verifying per-block digests silently - // lose them on the partial path. - let (compressed, full, _info) = multi_block_fixture(); - - // Reference digests via decode_blocks (the path that captures them). - let mut ref_dec = FrameDecoder::new(); - ref_dec.enable_per_block_checksums(); - let mut rsrc = compressed.as_slice(); - ref_dec.reset(&mut rsrc).unwrap(); - while !ref_dec.is_finished() { - ref_dec - .decode_blocks(&mut rsrc, crate::decoding::BlockDecodingStrategy::All) - .unwrap(); - } - let expected = ref_dec.computed_block_checksums().to_vec(); - assert!(!expected.is_empty(), "fixture must have multiple blocks"); - let _ = full; - - // Partial decode of the whole frame must capture the same digests. - let mut source = compressed.as_slice(); - let mut dec = FrameDecoder::new(); - dec.enable_per_block_checksums(); - dec.reset(&mut source).unwrap(); - let _ = dec - .decode_blocks_partial(&mut source, 0, u32::MAX, None, false) - .unwrap(); - assert_eq!( - dec.computed_block_checksums(), - expected.as_slice(), - "partial decode must capture the same per-block checksums as full decode" - ); - } - - // ── resume (window-priming + entropy cold resume, lsm) ─────────── - - /// Window size of `compressed`'s frame, read from a freshly-reset decoder. - #[cfg(feature = "lsm")] - fn frame_window_size(compressed: &[u8]) -> usize { - let mut src = compressed; - let mut dec = FrameDecoder::new(); - dec.reset(&mut src).unwrap(); - dec.state - .as_ref() - .unwrap() - .frame_header - .window_size() - .unwrap_or(0) as usize - } - - /// Build a large compressible MULTI-SEGMENT frame (window_size < content, - /// so mid-frame blocks reach back only into a bounded window) and return - /// `(compressed, full_decode, emit_info)`. - #[cfg(feature = "lsm")] - fn multi_segment_block_fixture() -> ( - Vec, - Vec, - crate::encoding::frame_emit_info::FrameEmitInfo, - ) { - // ~3 MiB of compressible (runs + repeated phrase) data — large enough - // that the encoder picks window_size < content_size (multi-segment). - let mut data: Vec = Vec::with_capacity(3 * 1024 * 1024); - let mut x = 0x9E37_79B9u32; - while data.len() < 3 * 1024 * 1024 { - x ^= x << 13; - x ^= x >> 17; - x ^= x << 5; - let run = 16 + (x as usize % 48); - let byte = (x >> 24) as u8; - for _ in 0..run { - data.push(byte); - } - data.extend_from_slice(b"the quick brown fox jumps over the lazy dog\n"); - } - - let mut compressed = Vec::new(); - let mut compressor = FrameCompressor::new(CompressionLevel::Default); - compressor.set_source(data.as_slice()); - compressor.set_drain(&mut compressed); - compressor.compress(); - let info = compressor - .last_frame_emit_info() - .expect("emit info populated") - .clone(); - drop(compressor); - - // Confirm the precondition: the frame must be multi-segment. - let mut sanity = FrameDecoder::new(); - sanity.init(&mut compressed.as_slice()).unwrap(); - assert!( - !sanity - .state - .as_ref() - .unwrap() - .frame_header - .descriptor - .single_segment_flag(), - "fixture precondition: frame must be multi-segment (resize if encoder default changed)" - ); - - let mut dec = FrameDecoder::new(); - let mut full = alloc::vec![0u8; data.len()]; - let n = dec - .decode_all(compressed.as_slice(), &mut full) - .expect("full decode"); - full.truncate(n); - assert_eq!(full, data, "fixture must round-trip"); - (compressed, full, info) - } - - /// Emit a [`ResumeState`] for resuming at block `n` by decoding `[0, n)` on - /// a throwaway decoder with `emit_resume = true`. - #[cfg(feature = "lsm")] - fn emit_resume_state_at(compressed: &[u8], n: u32) -> super::ResumeState { - let mut src = compressed; - let mut dec = FrameDecoder::new(); - dec.reset(&mut src).unwrap(); - let pd = dec - .decode_blocks_partial(&mut src, 0, n, None, true) - .expect("prefix decode for resume-state emission"); - pd.resume_state - .expect("emit_resume should populate resume_state") - } - - #[cfg(feature = "lsm")] - #[test] - fn resume_matches_full_decode_at_first_mid_last() { - // Acceptance criterion: after resuming at block N (cold decoder, primed - // window + restored entropy), decode_blocks_partial yields bytes - // byte-identical to a full decode's [ends[N-1]..ends[end-1]) slice, for - // N in {1, mid, last}. Repeat_Mode entropy blocks are covered because - // the emitted ResumeState carries the carry-over tables. - let (compressed, full, info) = multi_block_fixture(); - let nblocks = info.blocks.len() as u32; - assert!(nblocks >= 4, "need several blocks, got {nblocks}"); - - for &n in &[1u32, nblocks / 2, nblocks - 1] { - // Producer: emit resume state for block n (separate decoder). - let st = emit_resume_state_at(&compressed, n); - assert_eq!(st.block_index(), n); - let output_offset = info.decompressed_byte_range(n as usize).unwrap().start; - assert_eq!(st.output_offset(), output_offset); - - // Consumer: a FRESH (cold) decoder resumes at n. Pass the WHOLE - // decompressed prefix as window_prime; it is capped to one window - // internally, exercising the cap path. - let window_prime = &full[..output_offset as usize]; - let mut header_src = compressed.as_slice(); - let mut dec = FrameDecoder::new(); - dec.reset(&mut header_src).unwrap(); - // Caller positions the source at block n's compressed frame offset. - let off = info.blocks[n as usize].offset_in_frame as usize; - let mut block_src = &compressed[off..]; - let pd = dec - .decode_blocks_partial( - &mut block_src, - n, - u32::MAX, - Some(super::ResumeInput { - window_prime, - state: &st, - }), - false, - ) - .unwrap_or_else(|e| panic!("resume decode at N={n} errored: {e:?}")); - - let start = output_offset as usize; - let end = info - .decompressed_byte_range((nblocks - 1) as usize) - .unwrap() - .end as usize; - assert_eq!( - pd.data.as_slice(), - &full[start..end], - "resumed bytes must equal the full-decode slice for N={n}" - ); - assert_eq!(pd.start_block, n); - assert_eq!(pd.blocks_decoded, nblocks - n); - assert!(pd.stopped_at.is_none(), "clean resume at N={n}"); - assert!(pd.frame_finished, "decoded through the last block"); - } - } - - #[cfg(feature = "lsm")] - #[test] - fn resume_with_exact_window_tail_matches_full_decode() { - // Realistic cold-resume shape on a MULTI-SEGMENT frame: caller supplies - // only the last `window_size` decompressed bytes (not the whole prefix), - // which is all that can ever back a match. - let (compressed, full, info) = multi_segment_block_fixture(); - let nblocks = info.blocks.len() as u32; - let window_size = frame_window_size(&compressed); - // First block whose preceding output exceeds one window, so the tail - // genuinely truncates the prefix. - let n = (1..nblocks) - .find(|&i| { - info.decompressed_byte_range(i as usize).unwrap().start as usize > window_size - }) - .expect("multi-segment frame must have a block past one window"); - let st = emit_resume_state_at(&compressed, n); - let output_offset = info.decompressed_byte_range(n as usize).unwrap().start; - assert!(output_offset as usize > window_size); - let tail_start = output_offset as usize - window_size; - let window_prime = &full[tail_start..output_offset as usize]; - - let mut header_src = compressed.as_slice(); - let mut dec = FrameDecoder::new(); - dec.reset(&mut header_src).unwrap(); - let off = info.blocks[n as usize].offset_in_frame as usize; - let mut block_src = &compressed[off..]; - let pd = dec - .decode_blocks_partial( - &mut block_src, - n, - u32::MAX, - Some(super::ResumeInput { - window_prime, - state: &st, - }), - false, - ) - .unwrap(); - - let end = info - .decompressed_byte_range((nblocks - 1) as usize) - .unwrap() - .end as usize; - assert_eq!(pd.data.as_slice(), &full[output_offset as usize..end]); - assert_eq!(pd.blocks_decoded, nblocks - n); - } - - #[cfg(feature = "lsm")] - #[test] - fn resume_rejects_short_window_prime() { - // Acceptance criterion: a window_prime shorter than the required window - // is rejected with a typed error, not a silent mis-decode. - let (compressed, full, info) = multi_block_fixture(); - let nblocks = info.blocks.len() as u32; - let window_size = frame_window_size(&compressed); - let n = nblocks / 2; - let st = emit_resume_state_at(&compressed, n); - let output_offset = info.decompressed_byte_range(n as usize).unwrap().start; - let required = core::cmp::min(window_size as u64, output_offset) as usize; - assert!(required > 0, "mid block must require a non-empty window"); - - // One byte short of the required window. - let prime = &full[output_offset as usize - (required - 1)..output_offset as usize]; - - let mut header_src = compressed.as_slice(); - let mut dec = FrameDecoder::new(); - dec.reset(&mut header_src).unwrap(); - let off = info.blocks[n as usize].offset_in_frame as usize; - let mut block_src = &compressed[off..]; - let err = dec - .decode_blocks_partial( - &mut block_src, - n, - u32::MAX, - Some(super::ResumeInput { - window_prime: prime, - state: &st, - }), - false, - ) - .expect_err("short window_prime must be rejected"); - match err { - crate::decoding::errors::FrameDecoderError::ResumeWindowTooShort { got, need } => { - assert_eq!(got, required - 1); - assert_eq!(need, required); - } - other => panic!("expected ResumeWindowTooShort, got {other:?}"), - } - } - - #[cfg(feature = "lsm")] - #[test] - fn resume_range_validates_against_effective_start_not_start_block() { - // In resume mode `start_block` is ignored and decoding begins at - // `state.block_index()`. The range guard must therefore validate the - // EFFECTIVE start against `end_block`: `end_block` below the resume - // block is an inverted range and must error, not silently return an - // empty decode. Caller passes the conventional ignored `start_block = 0`. - let (compressed, _full, info) = multi_block_fixture(); - let nblocks = info.blocks.len() as u32; - let n = (nblocks / 2).max(2); - let st = emit_resume_state_at(&compressed, n); - let output_offset = info.decompressed_byte_range(n as usize).unwrap().start; - - let mut header_src = compressed.as_slice(); - let mut dec = FrameDecoder::new(); - dec.reset(&mut header_src).unwrap(); - let off = info.blocks[n as usize].offset_in_frame as usize; - let mut block_src = &compressed[off..]; - // end_block = n - 1 is below the resume block n → inverted range. - let err = dec - .decode_blocks_partial( - &mut block_src, - 0, - n - 1, - Some(super::ResumeInput { - window_prime: &_full[..output_offset as usize], - state: &st, - }), - false, - ) - .expect_err("end_block below the resume block must be an inverted range"); - match err { - crate::decoding::errors::FrameDecoderError::InvalidBlockRange { - start_block, - end_block, - } => { - assert_eq!(start_block, n, "error must report the effective start"); - assert_eq!(end_block, n - 1); - } - other => panic!("expected InvalidBlockRange, got {other:?}"), - } - } - - #[cfg(feature = "lsm")] - #[test] - fn resume_rejects_state_from_a_different_frame() { - // A ResumeState captured from one frame must not be applied to a frame - // with a different decode shape (window size / single-segment / dict): - // restoring foreign entropy tables would yield byte-wrong output. The - // frame-identity guard must reject it up front with a typed error. - let (frame_a, _full_a, info_a) = multi_block_fixture(); - let (frame_b, full_b, _info_b) = multi_segment_block_fixture(); - // Sanity: the two fixtures must differ in decode shape for the guard to - // be exercised (single-segment vs multi-segment here). - let st = emit_resume_state_at(&frame_a, (info_a.blocks.len() as u32 / 2).max(1)); - - let mut header_src = frame_b.as_slice(); - let mut dec = FrameDecoder::new(); - dec.reset(&mut header_src).unwrap(); - // The frame-key check runs before the window-length check, so even a - // valid-length window_prime for frame B is rejected on identity. - let err = dec - .decode_blocks_partial( - &mut frame_b.as_slice(), - st.block_index(), - u32::MAX, - Some(super::ResumeInput { - window_prime: &full_b, - state: &st, - }), - false, - ) - .expect_err("resume state from a different frame must be rejected"); - assert!( - matches!( - err, - crate::decoding::errors::FrameDecoderError::ResumeFrameMismatch - ), - "expected ResumeFrameMismatch, got {err:?}" - ); - } - - #[cfg(all(feature = "lsm", feature = "hash"))] - #[test] - fn resume_rejects_wrong_window_prime_content() { - // Same frame (FrameKey matches) but the caller supplies a window_prime - // with one byte flipped. The shape key cannot catch this; the - // content-exact XXH64 of the window must, rejecting before any restore - // rather than mis-resolving matches against corrupted history. - let (compressed, full, info) = multi_block_fixture(); - let nblocks = info.blocks.len() as u32; - let n = (nblocks / 2).max(1); - let st = emit_resume_state_at(&compressed, n); - let output_offset = info.decompressed_byte_range(n as usize).unwrap().start as usize; - assert!(output_offset > 0); - - // Correct prefix with the last byte corrupted (this byte is inside the - // window the resume block reaches back into). - let mut corrupted = full[..output_offset].to_vec(); - let last = corrupted.len() - 1; - corrupted[last] ^= 0xFF; - - let mut header_src = compressed.as_slice(); - let mut dec = FrameDecoder::new(); - dec.reset(&mut header_src).unwrap(); - let off = info.blocks[n as usize].offset_in_frame as usize; - let mut block_src = &compressed[off..]; - let err = dec - .decode_blocks_partial( - &mut block_src, - n, - u32::MAX, - Some(super::ResumeInput { - window_prime: &corrupted, - state: &st, - }), - false, - ) - .expect_err("corrupted window_prime must be rejected by content hash"); - assert!( - matches!( - err, - crate::decoding::errors::FrameDecoderError::ResumeFrameMismatch - ), - "expected ResumeFrameMismatch, got {err:?}" - ); - } - - #[cfg(feature = "lsm")] - #[test] - fn resume_rejects_state_with_different_active_dictionary() { - // A dictless-header frame can be decoded with an explicit dictionary - // applied at runtime (force_dict / reset_with_dict_handle). Two such - // decodes differ in entropy/repcode/dict context even though the header - // dictionary_id is identically absent, so the resume guard must key on - // the ACTIVE dictionary, not just the header field. Here the snapshot is - // captured with no active dictionary; resuming with one applied must be - // rejected before any state is restored. - let (compressed, full, info) = multi_block_fixture(); - let nblocks = info.blocks.len() as u32; - let n = (nblocks / 2).max(1); - let st = emit_resume_state_at(&compressed, n); // active_dictionary_id = None - let output_offset = info.decompressed_byte_range(n as usize).unwrap().start as usize; - - let raw = std::fs::read("./dict_tests/dictionary").expect("dictionary fixture"); - let dict = crate::decoding::dictionary::Dictionary::decode_dict(&raw).expect("parse dict"); - let dict_id = dict.id; - - let mut header_src = compressed.as_slice(); - let mut dec = FrameDecoder::new(); - dec.add_dict(dict).unwrap(); - dec.reset(&mut header_src).unwrap(); - dec.force_dict(dict_id).unwrap(); // active_dictionary_id = Some(dict_id) - let off = info.blocks[n as usize].offset_in_frame as usize; - let mut block_src = &compressed[off..]; - let err = dec - .decode_blocks_partial( - &mut block_src, - n, - u32::MAX, - Some(super::ResumeInput { - window_prime: &full[..output_offset], - state: &st, - }), - false, - ) - .expect_err("resume with a different active dictionary must be rejected"); - assert!( - matches!( - err, - crate::decoding::errors::FrameDecoderError::ResumeFrameMismatch - ), - "expected ResumeFrameMismatch, got {err:?}" - ); - } - - #[cfg(feature = "lsm")] - #[test] - fn resume_invalid_range_does_not_mutate_decoder_state() { - // An inverted effective range must be rejected WITHOUT priming the - // decoder: no entropy restore, no window prime, no cursor advance. As - // written before the fix, those mutations ran before the range check, - // leaving the decoder in a synthetic resumed state on the error path. - let (compressed, full, info) = multi_block_fixture(); - let nblocks = info.blocks.len() as u32; - let n = (nblocks / 2).max(2); - let st = emit_resume_state_at(&compressed, n); - let output_offset = info.decompressed_byte_range(n as usize).unwrap().start as usize; - - let mut header_src = compressed.as_slice(); - let mut dec = FrameDecoder::new(); - dec.reset(&mut header_src).unwrap(); - // Freshly reset: cursor at block 0. - assert_eq!(dec.state.as_ref().unwrap().block_counter, 0); - - let off = info.blocks[n as usize].offset_in_frame as usize; - let mut block_src = &compressed[off..]; - let err = dec - .decode_blocks_partial( - &mut block_src, - 0, - n - 1, // below the resume block → inverted range - Some(super::ResumeInput { - window_prime: &full[..output_offset], - state: &st, - }), - false, - ) - .expect_err("inverted range must error"); - assert!(matches!( - err, - crate::decoding::errors::FrameDecoderError::InvalidBlockRange { .. } - )); - assert_eq!( - dec.state.as_ref().unwrap().block_counter, - 0, - "error path must not advance the cursor (validate before priming)" - ); - } - - #[cfg(feature = "lsm")] - #[test] - fn emit_resume_state_absent_on_terminal_block() { - // When a decode reaches the frame's last block there is no "next block" - // to resume at: the snapshot's block_index would be one past EOF and the - // caller has no offset_in_frame for it. emit_resume must therefore yield - // None on the terminal block, not a dangling snapshot. - let (compressed, _full, info) = multi_block_fixture(); - let nblocks = info.blocks.len() as u32; - let mut src = compressed.as_slice(); - let mut dec = FrameDecoder::new(); - dec.reset(&mut src).unwrap(); - let pd = dec - .decode_blocks_partial(&mut src, 0, nblocks, None, true) - .unwrap(); - assert!(pd.frame_finished, "decode must reach the last block"); - assert!( - pd.resume_state.is_none(), - "no resume state past the frame's last block" - ); - } - - #[cfg(feature = "lsm")] - #[test] - fn emit_resume_state_absent_when_not_requested() { - // Default partial decode (emit_resume = false) must NOT pay the entropy - // clone: resume_state stays None. - let (compressed, _full, info) = multi_block_fixture(); - let nblocks = info.blocks.len() as u32; - let mut src = compressed.as_slice(); - let mut dec = FrameDecoder::new(); - dec.reset(&mut src).unwrap(); - let pd = dec - .decode_blocks_partial(&mut src, 0, nblocks, None, false) - .unwrap(); - assert!( - pd.resume_state.is_none(), - "resume_state must be None unless emit_resume is set" - ); - } - - #[cfg(feature = "lsm")] - #[test] - fn resume_grow_loop_reconstructs_full() { - // The motivating scenario: a symmetric one-call grow-loop. Each call - // takes the previous ResumeState and emits the next, decoding only the - // new extent — concatenated, the extents reconstruct the full output - // with no prefix ever re-decompressed. - let (compressed, full, info) = multi_block_fixture(); - let nblocks = info.blocks.len() as u32; - assert!(nblocks >= 4); - - // Walk the frame in extents of `step` blocks each. - let step = (nblocks / 3).max(1); - let mut combined: Vec = Vec::new(); - let mut next: u32 = 0; - let mut carry: Option = None; - - while next < nblocks { - let end = (next + step).min(nblocks); - let mut dec = FrameDecoder::new(); - let mut header_src = compressed.as_slice(); - dec.reset(&mut header_src).unwrap(); - - let off = info.blocks[next as usize].offset_in_frame as usize; - let mut block_src = &compressed[off..]; - - let output_offset = info.decompressed_byte_range(next as usize).unwrap().start; - let pd = if let Some(st) = carry.as_ref() { - // Resume from the prior extent's state (cold: fresh decoder). - let window_prime = &full[..output_offset as usize]; - dec.decode_blocks_partial( - &mut block_src, - next, - end, - Some(super::ResumeInput { - window_prime, - state: st, - }), - true, - ) - .unwrap() - } else { - // First extent: no resume input, just emit for the next. - dec.decode_blocks_partial(&mut block_src, next, end, None, true) - .unwrap() - }; - - combined.extend_from_slice(&pd.data); - carry = pd.resume_state; - next = end; - } - - assert_eq!( - combined, full, - "grow-loop extents must reconstruct the full output" - ); - } - - #[cfg(all(feature = "lsm", feature = "hash"))] - #[test] - fn resume_does_not_redecode_prefix_blocks() { - // Instrumented confirmation that blocks < N are not re-decoded on - // resume. With per-block checksums enabled on the resuming decoder, the - // resumed decode must record exactly one digest per in-range block - // (end - N), never one per frame block. - let (compressed, full, info) = multi_block_fixture(); - let nblocks = info.blocks.len() as u32; - let n = nblocks / 2; - let st = emit_resume_state_at(&compressed, n); - let output_offset = info.decompressed_byte_range(n as usize).unwrap().start; - - let mut header_src = compressed.as_slice(); - let mut dec = FrameDecoder::new(); - dec.enable_per_block_checksums(); - dec.reset(&mut header_src).unwrap(); - let off = info.blocks[n as usize].offset_in_frame as usize; - let mut block_src = &compressed[off..]; - let _ = dec - .decode_blocks_partial( - &mut block_src, - n, - u32::MAX, - Some(super::ResumeInput { - window_prime: &full[..output_offset as usize], - state: &st, - }), - false, - ) - .unwrap(); - - assert_eq!( - dec.computed_block_checksums().len() as u32, - nblocks - n, - "resume must decode only in-range blocks, not re-decode the prefix" - ); - } -} +mod tests; diff --git a/zstd/src/decoding/frame_decoder/tests.rs b/zstd/src/decoding/frame_decoder/tests.rs new file mode 100644 index 000000000..7b43aa1d6 --- /dev/null +++ b/zstd/src/decoding/frame_decoder/tests.rs @@ -0,0 +1,2362 @@ +extern crate std; + +use super::{DictionaryHandle, FrameDecoder}; +use crate::encoding::{CompressionLevel, FrameCompressor}; +use alloc::vec::Vec; + +#[test] +fn decode_all_tight_and_slack_outputs_match_on_single_segment_frame() { + // Roundtrip a small payload through the encoder, then decode + // it via `decode_all` on two output shapes that select + // different internal sequence-exec paths within the direct + // decode: + // 1. Tight output (exactly `frame_content_size`, no + // WILDCOPY_OVERLENGTH slack) → direct path whose trailing + // sequence(s) take the bounded (non-overshooting) copy in + // `UserSliceBackend::exec_sequence_bounded`. + // 2. Output with WILDCOPY slack → direct path whose + // sequences all take the SIMD wildcopy fast path. + // Both must produce identical output bytes — the bounded tail + // copy must reconstruct the same data as the overshooting fast + // path. This is the regression gate for the relaxed + // direct-decode gate (`cap >= content_size`). + let payload: Vec = (0..4096u32).map(|i| (i & 0xFF) as u8).collect(); + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_source(payload.as_slice()); + let mut compressed = Vec::new(); + compressor.set_drain(&mut compressed); + compressor.compress(); + + // Baseline: tight output → legacy drain path. + let mut dec_a = FrameDecoder::new(); + let mut out_a = alloc::vec![0u8; payload.len()]; + let n_a = dec_a + .decode_all(compressed.as_slice(), &mut out_a) + .expect("decode_all (legacy drain) should succeed"); + assert_eq!(n_a, payload.len()); + assert_eq!(&out_a[..n_a], payload.as_slice()); + + // Direct: output with WILDCOPY slack → direct path. + let slack = super::super::buffer_backend::WILDCOPY_OVERLENGTH; + let mut dec_b = FrameDecoder::new(); + let mut out_b = alloc::vec![0u8; payload.len() + slack]; + let n_b = dec_b + .decode_all(compressed.as_slice(), &mut out_b) + .expect("decode_all (direct path) should succeed"); + assert_eq!( + n_b, + payload.len(), + "direct decode produced wrong byte count" + ); + assert_eq!(&out_b[..n_b], payload.as_slice()); +} + +#[test] +fn decode_all_tight_output_overlapping_tail_match_roundtrips() { + // The bounded tail copy must handle an OVERLAPPING match + // (offset < match_length) as the trailing sequence when the + // output slice is sized to exactly `frame_content_size`. A long + // run of a single byte at the end of the payload encodes as an + // offset-1 match whose length far exceeds the offset, so the + // bounded copy's overlapping (forward byte-by-byte) branch is + // exercised at the buffer tail where the SIMD overshoot would + // otherwise run past `cap`. Decoding into a tight buffer and + // matching the original payload byte-for-byte is the regression + // gate for the overlap branch of `exec_sequence_bounded`. + let mut payload: Vec = (0..256u32).map(|i| (i & 0xFF) as u8).collect(); + payload.extend(core::iter::repeat_n(0xABu8, 8192)); + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_source(payload.as_slice()); + let mut compressed = Vec::new(); + compressor.set_drain(&mut compressed); + compressor.compress(); + + // Anti-vacuous precondition: the 8 KiB trailing run of a single + // byte must compress to a Compressed block dominated by ONE long + // offset-1 (overlapping, offset < match_length) match — not a Raw + // block. If the encoder ever stopped emitting that overlapping + // tail match the test would pass without exercising + // `exec_sequence_bounded`'s overlapping forward-copy branch, so + // gate on the output being a tiny fraction of the input (a raw + // block would be ~`payload.len()`; an offset-1 run match is tens + // of bytes). + assert!( + compressed.len() < payload.len() / 8, + "expected an overlapping-tail match to dominate the frame \ + (compressed={} payload={}); the bounded overlap branch would \ + not be exercised otherwise", + compressed.len(), + payload.len(), + ); + + // Tight output: exactly content_size, no WILDCOPY slack. + let mut dec = FrameDecoder::new(); + let mut out = alloc::vec![0u8; payload.len()]; + let n = dec + .decode_all(compressed.as_slice(), &mut out) + .expect("tight-output decode with overlapping tail match should succeed"); + assert_eq!(n, payload.len()); + assert_eq!(out, payload, "bounded overlap tail copy corrupted output"); +} + +#[test] +fn decode_all_multi_segment_frame_decodes_correctly() { + // Multi-segment frame: payload large enough that the + // encoder's default frame layout has `single_segment_flag = + // false` and `window_size < frame_content_size`. The direct + // path must cap the visible buffer at window_size after each + // block (drop_to_window_size) so match-offset validation + // matches the spec rule `offset <= window_size`, and still + // produce the same bytes as decode_all on the + // FlatBuf/Ring-backed path. + // + // Make the payload structured so multi-segment behavior + // actually kicks in: 2 MiB of repeating + random-ish bytes + // forces window_size lower than content_size at the encoder. + let mut payload: Vec = Vec::with_capacity(2 * 1024 * 1024); + for i in 0..payload.capacity() { + payload.push((i.wrapping_mul(2_654_435_761) & 0xFF) as u8); + } + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_source(payload.as_slice()); + let mut compressed = Vec::new(); + compressor.set_drain(&mut compressed); + compressor.compress(); + + // Baseline: decode_all through the FlatBuf+drain path. + let mut dec_a = FrameDecoder::new(); + let mut out_a = alloc::vec![0u8; payload.len()]; + let n_a = dec_a + .decode_all(compressed.as_slice(), &mut out_a) + .expect("decode_all should succeed"); + assert_eq!(n_a, payload.len()); + assert_eq!(&out_a[..n_a], payload.as_slice()); + + // Direct path: must give identical bytes via UserSliceBackend + // + per-block drop_to_window_size. + let slack = super::super::buffer_backend::WILDCOPY_OVERLENGTH; + let mut dec_b = FrameDecoder::new(); + let mut out_b = alloc::vec![0u8; payload.len() + slack]; + let n_b = dec_b + .decode_all(compressed.as_slice(), &mut out_b) + .expect("decode_all should succeed on multi-segment frame"); + assert_eq!(n_b, payload.len(), "wrong byte count on direct path"); + assert_eq!(&out_b[..n_b], payload.as_slice()); + + // Sanity-check: confirm the encoded frame really IS + // multi-segment. If a future encoder default changes, + // catching the assumption here is better than silently + // testing single_segment on this name. + let mut sanity = FrameDecoder::new(); + sanity.init(&mut compressed.as_slice()).unwrap(); + assert!( + !sanity + .state + .as_ref() + .unwrap() + .frame_header + .descriptor + .single_segment_flag(), + "test precondition violated: frame is single-segment, rename or resize" + ); +} + +#[cfg(feature = "hash")] +#[test] +fn decode_all_propagates_checksum_into_persistent_scratch() { + // Direct path on a checksum-flagged frame: the FrameCompressor + // under `feature = "hash"` sets content_checksum_flag, so the + // decoded frame has a recorded checksum. After + // decode_all we must be able to verify it matches via + // the public get_calculated_checksum() accessor — the digest + // is computed by walking output at end of decode and stored + // into the persistent scratch's hasher. + let payload: Vec = (0..8192u32).map(|i| (i & 0xFF) as u8).collect(); + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_content_checksum(true); + compressor.set_source(payload.as_slice()); + let mut compressed = Vec::new(); + compressor.set_drain(&mut compressed); + compressor.compress(); + + let slack = super::super::buffer_backend::WILDCOPY_OVERLENGTH; + let mut dec = FrameDecoder::new(); + let mut out = alloc::vec![0u8; payload.len() + slack]; + let n = dec + .decode_all(compressed.as_slice(), &mut out) + .expect("decode_all with checksum must succeed"); + assert_eq!(n, payload.len()); + assert_eq!(&out[..n], payload.as_slice()); + + // Both sides must report the same checksum: the frame header + // carries the stored u32, and get_calculated_checksum reads + // the running digest the direct path just propagated. + let stored = dec.get_checksum_from_data(); + let calculated = dec.get_calculated_checksum(); + assert!(stored.is_some(), "frame must carry stored checksum"); + assert!( + calculated.is_some(), + "direct path must propagate calculated checksum" + ); + assert_eq!( + stored, calculated, + "stored vs calculated checksum mismatch on direct path" + ); +} + +#[cfg(feature = "hash")] +#[test] +fn verify_mode_accepts_a_valid_frame() { + use crate::decoding::ContentChecksum; + let payload: Vec = (0..8192u32).map(|i| (i & 0xFF) as u8).collect(); + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_content_checksum(true); + compressor.set_source(payload.as_slice()); + let mut compressed = Vec::new(); + compressor.set_drain(&mut compressed); + compressor.compress(); + + let slack = super::super::buffer_backend::WILDCOPY_OVERLENGTH; + let mut dec = FrameDecoder::new(); + dec.set_content_checksum(ContentChecksum::Verify); + let mut out = alloc::vec![0u8; payload.len() + slack]; + let n = dec + .decode_all(compressed.as_slice(), &mut out) + .expect("Verify mode must accept a frame with a correct checksum"); + assert_eq!(&out[..n], payload.as_slice()); +} + +#[cfg(feature = "hash")] +#[test] +fn verify_mode_rejects_a_corrupted_checksum() { + use crate::decoding::ContentChecksum; + use crate::decoding::errors::FrameDecoderError; + let payload: Vec = (0..8192u32).map(|i| (i & 0xFF) as u8).collect(); + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_content_checksum(true); + compressor.set_source(payload.as_slice()); + let mut compressed = Vec::new(); + compressor.set_drain(&mut compressed); + compressor.compress(); + + // Flip a bit in the trailing 4-byte content checksum: the frame body + // still decodes to the correct bytes, but the stored digest no longer + // matches the one the decoder computes. + let last = compressed.len() - 1; + compressed[last] ^= 0xFF; + + let slack = super::super::buffer_backend::WILDCOPY_OVERLENGTH; + let mut dec = FrameDecoder::new(); + dec.set_content_checksum(ContentChecksum::Verify); + let mut out = alloc::vec![0u8; payload.len() + slack]; + let err = dec + .decode_all(compressed.as_slice(), &mut out) + .expect_err("Verify mode must reject a corrupted checksum"); + assert!( + matches!(err, FrameDecoderError::ChecksumMismatch { .. }), + "expected ChecksumMismatch, got {err:?}" + ); +} + +#[cfg(feature = "hash")] +#[test] +fn decode_from_to_verify_rejects_corrupted_checksum() { + // decode_from_to has its own block loop (not decode_blocks); it must + // still honour Verify and reject a corrupted trailer rather than + // silently accept it. + use crate::decoding::ContentChecksum; + use crate::decoding::errors::FrameDecoderError; + let payload: Vec = (0..8192u32).map(|i| (i & 0xFF) as u8).collect(); + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_content_checksum(true); + compressor.set_source(payload.as_slice()); + let mut compressed = Vec::new(); + compressor.set_drain(&mut compressed); + compressor.compress(); + let last = compressed.len() - 1; + compressed[last] ^= 0xFF; + + let slack = super::super::buffer_backend::WILDCOPY_OVERLENGTH; + let mut dec = FrameDecoder::new(); + dec.set_content_checksum(ContentChecksum::Verify); + let mut out = alloc::vec![0u8; payload.len() + slack]; + + // Split the trailing 4-byte checksum into a SEPARATE call so the + // verification must happen on the checksum-only early-return path (not + // the post-drain path) — the incremental case CodeRabbit flagged. + let split = compressed.len() - 4; + let (_r1, w1) = dec + .decode_from_to(&compressed[..split], &mut out) + .expect("blocks decode without the trailer"); + let err = dec + .decode_from_to(&compressed[split..], &mut out[w1..]) + .expect_err("decode_from_to in Verify mode must reject a corrupted checksum"); + assert!( + matches!(err, FrameDecoderError::ChecksumMismatch { .. }), + "expected ChecksumMismatch, got {err:?}" + ); +} + +#[cfg(feature = "hash")] +#[test] +fn decode_from_to_small_target_split_trailer_flushes_tail() { + // Regression: when a prior call decoded the last block but a small + // `target` left output buffered, the trailer-only call must still flush + // the buffered tail (it used to early-return Ok((4,0)) and lose it). + let payload: Vec = (0..8192u32).map(|i| (i & 0xFF) as u8).collect(); + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_content_checksum(true); + compressor.set_source(payload.as_slice()); + let mut compressed = Vec::new(); + compressor.set_drain(&mut compressed); + compressor.compress(); + + let split = compressed.len() - 4; + let mut dec = FrameDecoder::new(); + let mut out = alloc::vec![0u8; payload.len()]; + // Call 1: all blocks, but a SMALL (64-byte) target leaves the rest + // buffered on the decoder side. + let (_r1, w1) = dec + .decode_from_to(&compressed[..split], &mut out[..64]) + .expect("blocks decode with a small target"); + assert!(w1 <= 64); + // Call 2: the 4-byte trailer alone must flush the buffered tail through + // the shared read path, not return early and drop it. + let (_r2, w2) = dec + .decode_from_to(&compressed[split..], &mut out[w1..]) + .expect("trailer call must flush the buffered tail"); + assert_eq!(w1 + w2, payload.len(), "buffered tail was dropped"); + assert_eq!(&out[..w1 + w2], payload.as_slice()); +} + +#[cfg(feature = "hash")] +#[test] +fn none_mode_skips_the_checksum_pass() { + use crate::decoding::ContentChecksum; + let payload: Vec = (0..8192u32).map(|i| (i & 0xFF) as u8).collect(); + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_content_checksum(true); + compressor.set_source(payload.as_slice()); + let mut compressed = Vec::new(); + compressor.set_drain(&mut compressed); + compressor.compress(); + + let slack = super::super::buffer_backend::WILDCOPY_OVERLENGTH; + let mut dec = FrameDecoder::new(); + dec.set_content_checksum(ContentChecksum::None); + let mut out = alloc::vec![0u8; payload.len() + slack]; + let n = dec + .decode_all(compressed.as_slice(), &mut out) + .expect("None mode must still decode correctly"); + assert_eq!(&out[..n], payload.as_slice()); + // No digest is computed in None mode, even though the frame carries one. + assert!(dec.get_checksum_from_data().is_some()); + assert!(dec.get_calculated_checksum().is_none()); +} + +#[cfg(feature = "hash")] +#[test] +fn encoder_without_checksum_emits_no_trailing_digest() { + let payload: Vec = (0..8192u32).map(|i| (i & 0xFF) as u8).collect(); + + let mut with = Vec::new(); + let mut c_with = FrameCompressor::new(CompressionLevel::Default); + c_with.set_content_checksum(true); + c_with.set_source(payload.as_slice()); + c_with.set_drain(&mut with); + c_with.compress(); + + let mut without = Vec::new(); + let mut c_without = FrameCompressor::new(CompressionLevel::Default); + c_without.set_content_checksum(false); + c_without.set_source(payload.as_slice()); + c_without.set_drain(&mut without); + c_without.compress(); + + // The checksum-off frame is exactly the 4-byte trailing digest shorter. + assert_eq!(with.len(), without.len() + 4); + + let slack = super::super::buffer_backend::WILDCOPY_OVERLENGTH; + let mut dec = FrameDecoder::new(); + let mut out = alloc::vec![0u8; payload.len() + slack]; + let n = dec + .decode_all(without.as_slice(), &mut out) + .expect("a frame without a content checksum must decode"); + assert_eq!(&out[..n], payload.as_slice()); + assert!( + dec.get_checksum_from_data().is_none(), + "no trailing checksum should be reported" + ); +} + +#[test] +fn decode_all_fcs_overflow_via_corrupt_frame_returns_structured_error() { + // Hand-build a corrupt frame that declares + // frame_content_size = 4 but the (last) block carries a + // larger Raw payload. The pre-flight FCS check inside the + // direct path's block loop catches this and returns the + // structured FrameContentSizeMismatch variant — not a + // panic, not a generic TargetTooSmall. + // + // Frame layout (single_segment, FCS=4): + // magic 4 bytes 0xFD2FB528 + // FHD 1 byte single_segment=1, no checksum, + // FCS field size = 0 (-> 1-byte FCS) + // FCS 1 byte 0x04 + // block_header 3 bytes last=1, type=Raw, block_size=10 + // block_payload 10 bytes 0xAA repeated + let mut frame = alloc::vec::Vec::new(); + // magic + frame.extend_from_slice(&0xFD2FB528u32.to_le_bytes()); + // FHD: single_segment=1, fcs_flag=0 (1-byte FCS), no checksum, + // no dict. Bit layout: FCS(7-6)=0, single_segment(5)=1, + // reserved/uncs(4)=0, content_checksum(2)=0, dict(0-1)=00. + frame.push(0b0010_0000); + // FCS: 1 byte + frame.push(4); + // Block header: cBlockSize=10, type=Raw (0), last=1 + // 3-byte LE: bit0=last, bits1-2=type(2 bits), bits3-23=size + let cblock_size: u32 = 10; + let bh: u32 = 1 | (cblock_size << 3); // last=1, type=Raw=0 + frame.push((bh & 0xFF) as u8); + frame.push((bh >> 8) as u8); + frame.push((bh >> 16) as u8); + // Payload — 10 bytes that, if decoded, would exceed FCS=4. + frame.extend(core::iter::repeat_n(0xAAu8, 10)); + + let slack = super::super::buffer_backend::WILDCOPY_OVERLENGTH; + let mut dec = FrameDecoder::new(); + let mut out = alloc::vec![0u8; 4 + slack]; + let err = dec + .decode_all(&frame, &mut out) + .expect_err("FCS-overflow frame must fail decode"); + assert!( + matches!( + err, + super::FrameDecoderError::FrameContentSizeMismatch { .. } + ), + "expected FrameContentSizeMismatch, got {:?}", + err + ); +} + +#[test] +fn decode_all_compressed_block_fcs_overflow_returns_structured_error() { + // Acceptance test for #246: a malformed frame whose *Compressed* + // block expands past the declared `frame_content_size` must + // surface `FrameContentSizeMismatch` from the direct-decode path + // (UserSliceBackend sequence executor), NOT panic and NOT a + // generic FailedToReadBlockBody. The Raw-block sibling above + // covers the `BackendOverflow` arm; this covers the Compressed + // sequence-executor overflow arm (`ExecuteSequencesError:: + // OutputBufferOverflow` folded into FrameContentSizeMismatch in + // `run_direct_decode`). + // + // Construction: compress a compressible payload to get a genuine + // Compressed block + a header-declared FCS, then surgically patch + // the FCS field down to a tiny value. The block body still + // decodes (literals/sequences are independent of FCS) and the + // sequence executor overflows the small output slice. + // Highly compressible payload (repeated phrase) → Compressed + // block whose sequence executor produces ~4 KiB of output. + let unit = b"The quick brown fox jumps over the lazy dog. "; + let mut payload = Vec::with_capacity(4 * 1024); + while payload.len() < 4 * 1024 { + payload.extend_from_slice(unit); + } + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_source(payload.as_slice()); + let mut frame = Vec::new(); + compressor.set_drain(&mut frame); + compressor.compress(); + // Sanity: the encoder actually compressed (=> a Compressed block, + // not a raw-stored fallback) so we exercise the sequence path. + assert!(frame.len() < payload.len()); + + // Locate the FCS field: it is the last `fcs_len` bytes of the + // frame header, whose total size `header_size` includes the magic. + // A ~4 KiB single-segment frame declares FCS = 4096, which lands in + // the 2-byte field range [256, 65791] (RFC 8878 §3.1.1.1.4) — assert + // that so the patch logic below stays a single deterministic branch. + let (header, header_size) = + super::super::frame::read_frame_header(frame.as_slice()).expect("valid header"); + let fcs_len = header + .descriptor + .frame_content_size_bytes() + .expect("fcs present") as usize; + assert_eq!( + fcs_len, 2, + "4 KiB single-segment frame must use a 2-byte FCS" + ); + let fcs_off = header_size as usize - fcs_len; + + // Patch the 2-byte FCS to its floor: stored bytes 0 decode to 256 + // (the field's `+256` bias), far below the 4 KiB the block actually + // produces, so the sequence executor overflows the output slice. + let patched_declared: u64 = 256; + frame[fcs_off] = 0; + frame[fcs_off + 1] = 0; + + // Size the output to declared + WILDCOPY slack so the direct path + // is eligible (output.len() >= content_size + slack) — the + // overflow then comes from the frame, not an undersized buffer. + let slack = super::super::buffer_backend::WILDCOPY_OVERLENGTH; + let mut out = alloc::vec![0u8; patched_declared as usize + slack]; + let mut dec = FrameDecoder::new(); + let err = dec + .decode_all(frame.as_slice(), &mut out) + .expect_err("Compressed block exceeding FCS must fail decode"); + match err { + super::FrameDecoderError::FrameContentSizeMismatch { declared, produced } => { + assert_eq!(declared, patched_declared, "declared echoes patched FCS"); + assert!(produced > declared, "produced must exceed declared"); + } + other => panic!("expected FrameContentSizeMismatch, got {other:?}"), + } +} + +/// Block-precise error positions (#174): a failing block header / body +/// reports its 0-based index and frame-absolute offset, consistent with +/// the encoder's `FrameEmitInfo.blocks[index].offset_in_frame`. +#[cfg(feature = "lsm")] +#[test] +fn block_precise_errors_carry_index_and_offset() { + use crate::encoding::{CompressionLevel, FrameCompressor}; + // ~1.3 MiB of incompressible (xorshift) bytes → many 128 KiB raw + // blocks, so blocks 3 and 7 both exist and are not the last block. + let mut data = alloc::vec::Vec::with_capacity(1_300_000); + let mut s: u64 = 0x2545_F491_4F6C_DD1D; + while data.len() < 1_300_000 { + s ^= s << 13; + s ^= s >> 7; + s ^= s << 17; + data.push((s >> 33) as u8); + } + + let mut frame = alloc::vec::Vec::new(); + let blocks = { + let mut fc = FrameCompressor::new(CompressionLevel::Level(1)); + fc.set_source(data.as_slice()); + fc.set_drain(&mut frame); + fc.compress(); + fc.last_frame_emit_info() + .expect("emit info present under lsm") + .blocks + .clone() + }; + assert!(blocks.len() > 7, "need >7 blocks, got {}", blocks.len()); + + let mut out = alloc::vec![0u8; data.len() + 4096]; + + // (1) Corrupt block 7's header: force its Block_Type to Reserved (3) + // by setting both type bits — fails the header read at block 7. + let off7 = blocks[7].offset_in_frame as usize; + let mut corrupt = frame.clone(); + corrupt[off7] |= 0b0000_0110; + let mut dec = FrameDecoder::new(); + let err = dec + .decode_all(&corrupt, &mut out) + .expect_err("reserved block-7 header must fail"); + match err { + super::FrameDecoderError::FailedToReadBlockHeaderAt { + block_index, + frame_offset, + .. + } => { + assert_eq!(block_index, 7); + assert_eq!(frame_offset, blocks[7].offset_in_frame); + } + other => panic!("expected FailedToReadBlockHeaderAt, got {other:?}"), + } + + // (2) Truncate at block 3's body start: header intact, body missing + // → the body decode fails at block 3 with its FrameBlock metadata. + let body3 = blocks[3].offset_in_frame as usize + blocks[3].header_size as usize; + let mut dec = FrameDecoder::new(); + let err = dec + .decode_all(&frame[..body3], &mut out) + .expect_err("truncated block-3 body must fail"); + match err { + super::FrameDecoderError::FailedToReadBlockBodyAt { + block_index, + frame_offset, + block, + .. + } => { + assert_eq!(block_index, 3); + assert_eq!(frame_offset, blocks[3].offset_in_frame); + assert_eq!(block.offset_in_frame, blocks[3].offset_in_frame); + } + other => panic!("expected FailedToReadBlockBodyAt, got {other:?}"), + } +} + +#[test] +fn decode_all_exact_fit_output_decodes_correctly() { + // Output sized exactly to frame_content_size (no + // WILDCOPY_OVERLENGTH slack) is now eligible for the direct + // path: every output-write site is exact-fit-safe (sequence + // exec falls back to the bounded, non-overshooting copy on the + // trailing sequence(s), Raw/RLE blocks copy exactly). This must + // produce the same bytes as a slack-padded buffer. Exercised on + // x86 through the per-kernel AVX2/SSE2 inline-exec macros, which + // carry the same tight-tail branch. + let payload: Vec = (0..2048u32) + .map(|i| (i.wrapping_mul(31) & 0xFF) as u8) + .collect(); + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_source(payload.as_slice()); + let mut compressed = Vec::new(); + compressor.set_drain(&mut compressed); + compressor.compress(); + + let mut dec = FrameDecoder::new(); + // Exactly payload.len(), no slack. + let mut out = alloc::vec![0u8; payload.len()]; + let n = dec + .decode_all(compressed.as_slice(), &mut out) + .expect("exact-fit decode_all should succeed"); + assert_eq!(n, payload.len()); + assert_eq!(&out[..n], payload.as_slice()); +} + +#[test] +fn decode_all_fallback_validates_fcs_against_total_output() { + // Synthetic single-segment frame: FCS = 20 bytes, but the + // last-block flag fires after only 4 bytes of raw payload. + // On the direct path this would trip the post-block + // `produced > content_size` check; the fallback path + // (eligible=false because output is sized exactly to FCS, + // no WILDCOPY slack) used to silently return Ok(4). With + // the fix it now surfaces `FrameContentSizeMismatch` + // matching the direct path. + // + // Frame layout: 4 B magic | 1 B FHD (single_segment=1, + // FCS_flag=3 → 8-byte FCS) | 8 B FCS=20 | block header + // (Raw, last, size=4) | 4 raw bytes. + let mut wire = Vec::new(); + wire.extend_from_slice(&0xFD2F_B528u32.to_le_bytes()); // magic + // FHD: FCS_flag=3 (8-byte FCS) <<6 | single_segment=1 <<5. + wire.push(0b1110_0000); + wire.extend_from_slice(&20u64.to_le_bytes()); // declared FCS + // Block header: (size << 3) | (block_type << 1) | last_block. + // Raw block (block_type=0), last_block=1, size=4 → 0b00100001 = 0x21. + wire.push(0x21); + wire.push(0x00); + wire.push(0x00); + wire.extend_from_slice(&[1u8, 2, 3, 4]); + + let mut dec = FrameDecoder::new(); + // Size output SMALLER than the declared FCS so direct-decode is + // gated out (`output.len() >= content_size` is false) and the + // frame takes the legacy fallback drain loop — the path this test + // guards. The corrupt frame only produces 4 bytes, so 19 is ample + // room; the point is `19 != declared FCS (20)`. + const DECLARED_FCS: usize = 20; + let mut out = alloc::vec![0u8; DECLARED_FCS - 1]; + assert_ne!( + out.len(), + DECLARED_FCS, + "output must be smaller than FCS to exercise the fallback path", + ); + let err = dec + .decode_all(wire.as_slice(), &mut out) + .expect_err("fallback must reject corrupt FCS underflow"); + match err { + crate::decoding::errors::FrameDecoderError::FrameContentSizeMismatch { + declared, + produced, + } => { + assert_eq!(declared, 20); + assert_eq!(produced, 4); + } + other => panic!("expected FrameContentSizeMismatch, got {other:?}"), + } +} + +#[test] +fn decode_all_fallback_treats_explicit_fcs_zero_as_declared() { + // Synthetic multi-segment frame with FCS_flag=2 (4-byte + // FCS) explicitly set to 0. The header DECLARES zero + // content, but the body carries a 5-byte raw last-block. + // `fcs_declared()` must return true (the field is on the + // wire) so the fallback's post-decode size check sees the + // mismatch — even though `frame_content_size == 0`. This + // is exactly the FCS=0 edge case where the previous + // `content_size > 0` proxy would have silently accepted + // the corrupt frame. + // + // Frame layout: + // 4 B magic — 28 B5 2F FD + // 1 B FHD — FCS_flag=2 (bits 7-6), no + // single_segment, content_checksum=0, + // dict_id_flag=0 → 0b1000_0000 + // 1 B window_descriptor — exp=10, mantissa=0 → window=1 MiB + // 4 B FCS — 0 LE + // 3 B block header — raw, last, size=5 → 0x29 0x00 0x00 + // 5 B raw payload — anything non-empty + let mut wire = Vec::new(); + wire.extend_from_slice(&0xFD2F_B528u32.to_le_bytes()); + wire.push(0b1000_0000); // FHD: FCS_flag=2, others 0. + wire.push(0x50); // window_descriptor: exp=10, mantissa=0. + wire.extend_from_slice(&0u32.to_le_bytes()); // FCS = 0. + // Block header (24-bit LE): (size << 3) | (block_type << 1) | last_block + // = (5 << 3) | (0 << 1) | 1 = 0x29. + wire.push(0x29); + wire.push(0x00); + wire.push(0x00); + wire.extend_from_slice(&[1u8, 2, 3, 4, 5]); + + let mut dec = FrameDecoder::new(); + // FCS=0 declared, so eligibility (`content_size > 0`) + // false — falls through to the drain loop. Output buffer + // size doesn't matter for the eligibility check here; + // give it some room so `read()` can drain the block. + let mut out = alloc::vec![0u8; 16]; + let err = dec + .decode_all(wire.as_slice(), &mut out) + .expect_err("corrupt FCS=0 + 5-byte block must error"); + match err { + crate::decoding::errors::FrameDecoderError::FrameContentSizeMismatch { + declared, + produced, + } => { + assert_eq!(declared, 0); + assert_eq!(produced, 5); + } + other => panic!("expected FrameContentSizeMismatch, got {other:?}"), + } +} + +#[test] +fn decode_all_fallback_accepts_honest_explicit_fcs_zero() { + // Companion to the corrupt-FCS=0 test above: an HONEST + // empty frame with FCS_flag=2 (4-byte FCS) explicitly set + // to 0 AND a 0-byte raw last-block. `fcs_declared()` + // returns true and `content_size == 0 == total_written`, + // so the fallback validation accepts the frame instead of + // misreporting a mismatch. + // + // (Single-segment FCS=0 would test a similar invariant + // but trips header-stage validation: `window_size = + // frame_content_size = 0 < MIN_WINDOW_SIZE` fails the + // window-size sanity check before decode runs. Use the + // multi-segment shape where `window_size` comes from + // `window_descriptor` independently of FCS.) + // + // Frame layout: + // 4 B magic + // 1 B FHD — FCS_flag=2, others 0 → 0x80 + // 1 B window_descriptor — exp=10 → 1 MiB window + // 4 B FCS — 0 LE + // 3 B block header — raw, last, size=0 → 0x01 0x00 0x00 + let mut wire = Vec::new(); + wire.extend_from_slice(&0xFD2F_B528u32.to_le_bytes()); + wire.push(0b1000_0000); + wire.push(0x50); + wire.extend_from_slice(&0u32.to_le_bytes()); + // Block header: (0 << 3) | (0 << 1) | 1 = 0x01. + wire.push(0x01); + wire.push(0x00); + wire.push(0x00); + + let mut dec = FrameDecoder::new(); + let mut out = alloc::vec![0u8; 16]; + let n = dec + .decode_all(wire.as_slice(), &mut out) + .expect("honest FCS=0 + empty block must succeed"); + assert_eq!(n, 0); +} + +#[test] +fn reset_with_dict_handle_applies_dict_when_no_dict_id() { + let payload = b"reset-without-dict-id"; + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_source(payload.as_slice()); + let mut compressed = Vec::new(); + compressor.set_drain(&mut compressed); + compressor.compress(); + + let dict_raw = include_bytes!("../../../dict_tests/dictionary"); + let handle = DictionaryHandle::decode_dict(dict_raw).expect("dictionary should parse"); + + let mut decoder = FrameDecoder::new(); + decoder + .reset_with_dict_handle(compressed.as_slice(), &handle) + .expect("reset should succeed"); + let state = decoder.state.as_ref().expect("state should be initialized"); + assert!(state.frame_header.dictionary_id().is_none()); + assert_eq!(state.using_dict, Some(handle.id())); +} + +#[test] +fn reserve_buffer_reserves_the_shortfall_not_the_full_window_again() { + // `Vec::reserve_exact` takes ADDITIONAL capacity. The decode_all + // fallback loop re-enters decode_blocks once per strategy chunk, + // and each entry pre-reserves the window: re-requesting the FULL + // window on a buffer already holding ~window bytes of history + // would grow it toward 2x window, defeating the peak-memory cap + // the exact-growth policy exists for. + use super::DecoderScratchKind; + let window = 1usize << 20; + let mut scratch = DecoderScratchKind::new_flat(window); + scratch.reserve_buffer(window); + let data = alloc::vec![0u8; window]; + match &mut scratch { + super::DecoderScratchKind::Flat(s) => s.buffer.push(&data), + super::DecoderScratchKind::Ring(_) => unreachable!("new_flat builds Flat"), + } + scratch.reserve_buffer(window); + let workspace = scratch.workspace_bytes(); + assert!( + workspace < window * 3 / 2, + "second reserve_buffer grew a full window past the buffered \ + history: workspace {workspace} bytes vs window {window}" + ); +} + +#[test] +fn dict_frame_decodes_through_direct_path() { + // A dictionary frame decoded via `decode_all_with_dict_handle` + // into a buffer sized exactly to FCS takes the direct path + // (UserSliceBackend); matches reaching into the dictionary + // content must resolve through `repeat_from_dict`. The payload + // embeds dictionary content verbatim so the encoder emits + // dict-region matches from the first bytes of the frame. + let dict_raw = include_bytes!("../../../dict_tests/dictionary"); + let handle = DictionaryHandle::decode_dict(dict_raw).expect("dictionary should parse"); + let dict_tail: alloc::vec::Vec = handle + .as_dict() + .dict_content + .iter() + .rev() + .take(2048) + .rev() + .copied() + .collect(); + // No in-frame duplicate of the dictionary bytes: with a second + // copy in the payload the encoder may emit the later copy as an + // in-frame match, and the test would stay green even if the + // direct path stopped forwarding the dictionary handle. A + // single copy forces every dict-region match through + // `repeat_from_dict`. + let mut payload = dict_tail; + payload.extend_from_slice(b"unique suffix after dictionary material 0123456789"); + + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor + .set_dictionary_from_bytes(dict_raw) + .expect("dict load"); + compressor.set_source(payload.as_slice()); + let mut compressed = Vec::new(); + compressor.set_drain(&mut compressed); + compressor.compress(); + + // Fixture sanity: the frame must actually depend on the + // dictionary, otherwise the decode below never exercises + // dict-region match resolution. + let mut plain = Vec::new(); + let mut no_dict = FrameCompressor::new(CompressionLevel::Default); + no_dict.set_source(payload.as_slice()); + no_dict.set_drain(&mut plain); + no_dict.compress(); + assert!( + compressed.len() < plain.len(), + "fixture must depend on the dictionary: dict {} bytes vs plain {} bytes", + compressed.len(), + plain.len() + ); + + let mut decoder = FrameDecoder::new(); + let mut out = alloc::vec![0u8; payload.len()]; + let n = decoder + .decode_all_with_dict_handle(compressed.as_slice(), &mut out, &handle) + .expect("dict frame must decode on the direct path"); + assert_eq!(n, payload.len()); + assert_eq!(out, payload, "direct-path dict decode must be byte-exact"); + // Both paths are byte-identical, so pin the dispatch itself: a + // re-introduced dict exclusion in the direct gate would silently + // fall back to the buffered path and leave the asserts above green. + assert_eq!( + decoder.direct_frames, 1, + "dict frame must take the direct path, not the buffered fallback" + ); +} + +#[test] +fn implausible_content_size_skips_eager_alloc_direct_path() { + // Adversarial frame: a 1 KiB window (small ring) but a declared + // content size of 4 MiB, followed by a truncated raw block. The + // direct path would `resize` the caller's Vec to the pledged 4 MiB + // (allocating + zeroing it) BEFORE the truncated body is validated. + // The gate must reject the implausible size (4 MiB cannot come from + // 3 compressed bytes) and fall through to the window-bounded ring + // drain, which errors without ever allocating the pledged size. + // + // Hand-built so the declared size is fully decoupled from the real + // (tiny) input — the encoder always writes a truthful FCS. + let frame: &[u8] = &[ + 0x28, 0xB5, 0x2F, 0xFD, // magic + 0x80, // FHD: multi-segment, 4-byte FCS field, no dict + 0x00, // window descriptor -> 1 KiB window + 0x00, 0x00, 0x40, 0x00, // FCS = 4 MiB + 0x21, 0x03, 0x00, // raw block header: last, size 100, no body + ]; + + let mut dec = FrameDecoder::new(); + let mut src = frame; + dec.init(&mut src).expect("header must parse"); + // `src` now points past the header at the truncated 3-byte block. + let mut out = Vec::new(); + let err = dec.decode_current_frame_to_vec(src, &mut out, None); + assert!( + err.is_err(), + "truncated body must fail regardless of decode path" + ); + assert_eq!( + dec.direct_frames, 0, + "implausible FCS must NOT take the eager-alloc direct path" + ); +} + +#[test] +fn implausible_single_segment_fcs_rejected_before_window_reservation() { + // Single-segment adversarial frame: the window equals the declared + // content size (4 MiB) by definition, so the fallback ring drain would + // pre-reserve that whole window via `useful_window_size()` before the + // truncated body errors — the multi-segment gate test does not cover + // this. The implausible size (4 MiB cannot come from 3 compressed + // bytes) must be rejected up front with a content-size error, NOT a + // block-body error after the reservation. + let frame: &[u8] = &[ + 0x28, 0xB5, 0x2F, 0xFD, // magic + 0xA0, // FHD: single-segment, 4-byte FCS field + 0x00, 0x00, 0x40, 0x00, // FCS = 4 MiB (== window for single-segment) + 0x21, 0x03, 0x00, // raw block header: last, size 100, no body + ]; + + let mut dec = FrameDecoder::new(); + let mut src = frame; + dec.init(&mut src).expect("header must parse"); + let mut out = Vec::new(); + let err = dec + .decode_current_frame_to_vec(src, &mut out, None) + .expect_err("implausible single-segment FCS must be rejected"); + match err { + super::FrameDecoderError::FrameContentSizeMismatch { declared, .. } => { + assert_eq!(declared, 4 * 1024 * 1024); + } + other => { + panic!("expected early FrameContentSizeMismatch (no window reservation), got {other:?}") + } + } + assert_eq!( + dec.direct_frames, 0, + "implausible FCS must not take the eager-alloc direct path" + ); +} + +#[cfg(feature = "lsm")] +mod expect_validation { + use super::*; + use crate::decoding::errors::FrameDecoderError; + + fn compress(payload: &[u8]) -> Vec { + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_source(payload); + let mut compressed = Vec::new(); + compressor.set_drain(&mut compressed); + compressor.compress(); + compressed + } + + fn compress_with_dict(payload: &[u8], dict_raw: &[u8]) -> Vec { + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor + .set_dictionary_from_bytes(dict_raw) + .expect("dict load"); + compressor.set_source(payload); + let mut compressed = Vec::new(); + compressor.set_drain(&mut compressed); + compressor.compress(); + compressed + } + + #[test] + fn expect_dict_id_none_default_allows_anything() { + let compressed = compress(b"hello-no-expect"); + let mut decoder = FrameDecoder::new(); + decoder + .reset(compressed.as_slice()) + .expect("default None passes"); + } + + #[test] + fn expect_dict_id_zero_matches_frame_without_dict_id() { + // Default-encoded frame has no dict_id; pinning Some(0) + // ("no dictionary expected") must accept it. + let compressed = compress(b"payload"); + let mut decoder = FrameDecoder::new(); + decoder.expect_dict_id(Some(0)); + decoder + .reset(compressed.as_slice()) + .expect("Some(0) ~ None"); + } + + #[test] + fn expect_dict_id_matching_value_passes() { + let dict_raw = include_bytes!("../../../dict_tests/dictionary"); + let handle = DictionaryHandle::decode_dict(dict_raw).expect("dict parse"); + let actual_id = handle.id(); + + let compressed = compress_with_dict(b"payload-with-dict", dict_raw); + + let mut decoder = FrameDecoder::new(); + decoder.expect_dict_id(Some(actual_id)); + // Decode requires the dict to be registered; using + // reset_with_dict_handle for that. + decoder + .reset_with_dict_handle(compressed.as_slice(), &handle) + .expect("matching dict_id passes"); + } + + #[test] + fn expect_dict_id_mismatching_value_fails_before_decode() { + let dict_raw = include_bytes!("../../../dict_tests/dictionary"); + let handle = DictionaryHandle::decode_dict(dict_raw).expect("dict parse"); + let actual_id = handle.id(); + let wrong_id = actual_id.wrapping_add(1); + + let compressed = compress_with_dict(b"payload-with-dict", dict_raw); + + let mut decoder = FrameDecoder::new(); + decoder.expect_dict_id(Some(wrong_id)); + let err = decoder + .reset_with_dict_handle(compressed.as_slice(), &handle) + .expect_err("mismatch must fail"); + match err { + FrameDecoderError::UnexpectedDictId { expected, found } => { + assert_eq!(expected, Some(wrong_id)); + assert_eq!(found, Some(actual_id)); + } + other => panic!("expected UnexpectedDictId, got {other:?}"), + } + } + + #[test] + fn expect_dict_id_nonzero_fails_on_frame_without_dict_id() { + // Frame has no dict_id; expecting Some(42) (non-zero) + // must fail with found = None. + let compressed = compress(b"no-dict-frame"); + let mut decoder = FrameDecoder::new(); + decoder.expect_dict_id(Some(42)); + let err = decoder + .reset(compressed.as_slice()) + .expect_err("nonzero expectation on dictless frame must fail"); + match err { + FrameDecoderError::UnexpectedDictId { expected, found } => { + assert_eq!(expected, Some(42)); + assert_eq!(found, None); + } + other => panic!("expected UnexpectedDictId, got {other:?}"), + } + } + + #[test] + fn expect_window_descriptor_none_default_allows_anything() { + let compressed = compress(b"hello-no-wd-expect"); + let mut decoder = FrameDecoder::new(); + decoder + .reset(compressed.as_slice()) + .expect("default None passes"); + } + + #[test] + fn expect_window_descriptor_mismatch_fails_before_decode() { + // Compress a payload large enough to force a + // multi-segment frame (window_descriptor on wire). + // Default compression at >256 KiB produces multi- + // segment frames with a real window_descriptor byte. + let payload = alloc::vec![0xABu8; 512 * 1024]; + let compressed = compress(&payload); + + // Read the actual window_descriptor by decoding once + // without expectations, then pin a wrong value. + let mut probe_decoder = FrameDecoder::new(); + probe_decoder.reset(compressed.as_slice()).unwrap(); + let probe_state = probe_decoder.state.as_ref().unwrap(); + let actual_wd = probe_state + .frame_header + .window_descriptor() + .expect("multi-segment frame should expose window_descriptor"); + let wrong_wd = actual_wd.wrapping_add(0x10); // bump exponent + + let mut decoder = FrameDecoder::new(); + decoder.expect_window_descriptor(Some(wrong_wd)); + let err = decoder + .reset(compressed.as_slice()) + .expect_err("wrong window_descriptor must fail"); + match err { + FrameDecoderError::UnexpectedWindowDescriptor { expected, found } => { + assert_eq!(expected, wrong_wd); + assert_eq!(found, Some(actual_wd)); + } + other => panic!("expected UnexpectedWindowDescriptor, got {other:?}"), + } + } + + /// Build a minimal synthetic single-segment zstd frame + /// carrying a 4-byte raw payload. RFC 8878 §3.1.1.1 + /// layout, hand-rolled because our default + /// `FrameCompressor` settings don't emit + /// `single_segment_flag` for tiny inputs. + /// + /// Wire bytes (13 total for 4-byte payload): + /// ```text + /// 28 B5 2F FD magic + /// 20 FHD: single_segment=1, FCS_flag=0 + /// 04 FCS (single byte, value = payload.len()) + /// 21 00 00 block header: raw, last, size=4 + /// .. .. .. .. payload bytes + /// ``` + fn synth_single_segment_frame(payload: &[u8]) -> Vec { + assert!(payload.len() <= 255, "1-byte FCS field caps at 255"); + assert!(payload.len() < (1usize << 21), "block size 21-bit max"); + let mut out = Vec::new(); + // Magic 0xFD2FB528 LE. + out.extend_from_slice(&0xFD2F_B528u32.to_le_bytes()); + // FHD: single_segment_flag (bit 5) set, everything + // else zero. With single_segment + FCS_flag=0 the FCS + // field is 1 byte. No window_descriptor on wire. + out.push(0b0010_0000); + // 1-byte FCS = payload length. + out.push(payload.len() as u8); + // Block header (3 bytes LE): + // last_block=1, block_type=0 (Raw), block_size=payload.len(). + // Encoded: (size << 3) | (block_type << 1) | last_block. + // Block header: last_block flag in bit 0, block_type + // (0 = Raw) in bits 1-2, block size in bits 3+. + let bh: u32 = ((payload.len() as u32) << 3) | 1; + out.push((bh & 0xFF) as u8); + out.push(((bh >> 8) & 0xFF) as u8); + out.push(((bh >> 16) & 0xFF) as u8); + // Raw payload. + out.extend_from_slice(payload); + out + } + + #[test] + fn expect_window_descriptor_on_single_segment_frame_fails_with_found_none() { + // Single-segment frames omit the window_descriptor + // byte from the wire entirely. Setting an expectation + // here must surface `found: None` so callers + // distinguish "wrong descriptor" from "no descriptor + // on the wire" — never silently pass. + let compressed = synth_single_segment_frame(b"tiny"); + + // First sanity-check: the synthetic frame decodes + // cleanly without any expectation. + { + let mut probe = FrameDecoder::new(); + probe + .reset(compressed.as_slice()) + .expect("synth frame parses"); + let probe_state = probe.state.as_ref().unwrap(); + assert!( + probe_state.frame_header.window_descriptor().is_none(), + "synth frame must be single-segment" + ); + } + + let mut decoder = FrameDecoder::new(); + decoder.expect_window_descriptor(Some(0x40)); + let err = decoder + .reset(compressed.as_slice()) + .expect_err("single-segment + expectation must fail"); + match err { + FrameDecoderError::UnexpectedWindowDescriptor { expected, found } => { + assert_eq!(expected, 0x40); + assert_eq!(found, None); + } + other => panic!("expected UnexpectedWindowDescriptor, got {other:?}"), + } + } + + #[test] + fn validation_failure_leaves_decoder_re_resettable() { + // After UnexpectedDictId on a wrong-expectation reset, + // clearing the expectation and re-calling reset must + // succeed on the same source — no lingering failed + // state. + let compressed = compress(b"re-resettable"); + + let mut decoder = FrameDecoder::new(); + decoder.expect_dict_id(Some(42)); + let err = decoder + .reset(compressed.as_slice()) + .expect_err("first reset fails"); + assert!(matches!(err, FrameDecoderError::UnexpectedDictId { .. })); + + // Clear expectation and retry on a fresh source. + decoder.expect_dict_id(None); + decoder + .reset(compressed.as_slice()) + .expect("retry after clearing expectation should succeed"); + } +} + +/// Build a skippable frame on the wire: 4-byte LE magic + 4-byte LE +/// length + payload bytes. RFC 8878 §3.1.2 restricts the magic +/// variant to `0..=15`; assert here so accidental misuse of the +/// helper can't smuggle a non-skippable magic past the tests. +#[cfg(feature = "lsm")] +fn build_skippable_frame(variant: u8, payload: &[u8]) -> Vec { + assert!( + variant <= 15, + "skippable-frame variant {variant} outside RFC 8878 0..=15 range", + ); + let mut out = Vec::with_capacity(8 + payload.len()); + let magic: u32 = 0x184D2A50 + u32::from(variant); + out.extend_from_slice(&magic.to_le_bytes()); + out.extend_from_slice(&u32::try_from(payload.len()).unwrap().to_le_bytes()); + out.extend_from_slice(payload); + out +} + +#[cfg(feature = "lsm")] +#[test] +fn decode_all_with_skippable_visitor_sees_payloads_in_order() { + // Build a stream: skippable(v0, "alpha") + zstd_frame + + // skippable(v3, "beta") + zstd_frame + skippable(v15, "") + // and verify the visitor is invoked exactly three times with + // the correct (variant, payload) pairs in stream order while + // the zstd frames decode normally. + let payload_a: Vec = (0..256u16).map(|i| i as u8).collect(); + let payload_b: Vec = (0..256u16).map(|i| (i ^ 0xAA) as u8).collect(); + + let mut comp_a = Vec::new(); + let mut c = FrameCompressor::new(CompressionLevel::Default); + c.set_source(payload_a.as_slice()); + c.set_drain(&mut comp_a); + c.compress(); + + let mut comp_b = Vec::new(); + let mut c = FrameCompressor::new(CompressionLevel::Default); + c.set_source(payload_b.as_slice()); + c.set_drain(&mut comp_b); + c.compress(); + + let skip0 = build_skippable_frame(0, b"alpha"); + let skip3 = build_skippable_frame(3, b"beta"); + let skip15 = build_skippable_frame(15, &[]); + + let mut stream = Vec::new(); + stream.extend_from_slice(&skip0); + stream.extend_from_slice(&comp_a); + stream.extend_from_slice(&skip3); + stream.extend_from_slice(&comp_b); + stream.extend_from_slice(&skip15); + + let mut decoder = FrameDecoder::new(); + let mut out = alloc::vec![0u8; payload_a.len() + payload_b.len()]; + let mut collected: Vec<(u8, Vec)> = Vec::new(); + let n = decoder + .decode_all_with_skippable_visitor(stream.as_slice(), &mut out, |variant, payload| { + collected.push((variant, payload.to_vec())); + }) + .expect("decode_all_with_skippable_visitor should succeed"); + + // All three skippables visited in stream order. + assert_eq!(collected.len(), 3); + assert_eq!(collected[0], (0u8, b"alpha".to_vec())); + assert_eq!(collected[1], (3u8, b"beta".to_vec())); + assert_eq!(collected[2], (15u8, Vec::::new())); + + // Both zstd frames decoded into `out` back-to-back. + assert_eq!(n, payload_a.len() + payload_b.len()); + assert_eq!(&out[..payload_a.len()], payload_a.as_slice()); + assert_eq!(&out[payload_a.len()..n], payload_b.as_slice()); +} + +#[cfg(feature = "lsm")] +#[test] +fn decode_all_silently_skips_when_no_visitor() { + // Regression gate: plain decode_all must still silently skip + // skippable frames (RFC 8878 mandated behavior) with no + // behavioral change after the visitor refactor. + let payload: Vec = (0..512u16).map(|i| i as u8).collect(); + let mut comp = Vec::new(); + let mut c = FrameCompressor::new(CompressionLevel::Default); + c.set_source(payload.as_slice()); + c.set_drain(&mut comp); + c.compress(); + + let skip = build_skippable_frame(7, b"ignored sidecar"); + let mut stream = Vec::new(); + stream.extend_from_slice(&skip); + stream.extend_from_slice(&comp); + + let mut decoder = FrameDecoder::new(); + let mut out = alloc::vec![0u8; payload.len()]; + let n = decoder + .decode_all(stream.as_slice(), &mut out) + .expect("decode_all should succeed on skippable + zstd stream"); + assert_eq!(n, payload.len()); + assert_eq!(&out[..n], payload.as_slice()); +} + +#[cfg(feature = "lsm")] +#[test] +fn frame_emit_info_describes_emitted_block_layout() { + // Encode a payload large enough to force >1 block, fetch + // FrameEmitInfo, walk blocks[] and verify each block's + // (offset_in_frame, header_size, body_size) matches the bytes + // actually emitted into the drain buffer. + let payload: Vec = (0..200_000u32).map(|i| (i & 0xFF) as u8).collect(); + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + // Content checksum is opt-in (library default mirrors libzstd's + // checksum-off); request it so the checksum_range assertion below + // exercises the hash-gated trailer accounting. + compressor.set_content_checksum(true); + compressor.set_source(payload.as_slice()); + let mut compressed = Vec::new(); + compressor.set_drain(&mut compressed); + compressor.compress(); + + let info = compressor + .last_frame_emit_info() + .expect("last_frame_emit_info populated after compress") + .clone(); + drop(compressor); + + // Frame header range starts at 0 and is non-empty. + assert_eq!(info.frame_header_range.start, 0); + assert!(info.frame_header_range.end > 0); + // Total size matches what was written to the drain. + assert_eq!(info.total_size as usize, compressed.len()); + // At least one block, and the last entry has last_block=true. + assert!(!info.blocks.is_empty()); + assert!(info.blocks.last().unwrap().last_block); + // All non-final blocks have last_block=false. + for b in &info.blocks[..info.blocks.len() - 1] { + assert!(!b.last_block); + } + // Walk and verify each block's header bytes match the + // recorded type / size by re-decoding the 3-byte header. + // Walking arithmetic: offset_in_frame + header_size + body_size + // must land exactly on the next block's offset_in_frame (or, + // for the last block, on the checksum / end of frame). + for (i, b) in info.blocks.iter().enumerate() { + let off = b.offset_in_frame as usize; + assert_eq!(b.header_size, 3); + let mut hdr = [0u8; 4]; + hdr[..3].copy_from_slice(&compressed[off..off + 3]); + let raw = u32::from_le_bytes(hdr); + let last = (raw & 1) != 0; + let ty = (raw >> 1) & 0b11; + let sz = raw >> 3; + assert_eq!(last, b.last_block); + assert_eq!(sz, b.block_size_field); + // body_size is the PHYSICAL length on the wire: spec's + // Block_Size for Raw/Compressed, always 1 for RLE. + let expected_physical = match b.block_type { + crate::encoding::frame_emit_info::BlockType::RLE => 1, + _ => sz, + }; + assert_eq!(b.body_size, expected_physical); + let expected_ty = match b.block_type { + crate::encoding::frame_emit_info::BlockType::Raw => 0, + crate::encoding::frame_emit_info::BlockType::RLE => 1, + crate::encoding::frame_emit_info::BlockType::Compressed => 2, + crate::encoding::frame_emit_info::BlockType::Reserved => 3, + }; + assert_eq!(ty, expected_ty); + // Walking-arithmetic invariant. + let next_off = b.offset_in_frame + b.header_size as u32 + b.body_size; + if let Some(next) = info.blocks.get(i + 1) { + assert_eq!( + next_off, next.offset_in_frame, + "block {i} body_size doesn't reach next block's offset_in_frame", + ); + } else if let Some(cs) = info.checksum_range.as_ref() { + assert_eq!( + next_off, cs.start, + "last block body_size doesn't reach checksum_range.start", + ); + } else { + assert_eq!( + next_off, info.total_size, + "last block body_size doesn't reach total_size", + ); + } + } + // Checksum range present iff `feature = "hash"` is enabled. + assert_eq!(info.checksum_range.is_some(), cfg!(feature = "hash")); +} + +#[cfg(all(feature = "lsm", feature = "hash"))] +#[test] +fn per_block_checksum_round_trip() { + // Encode with per-block checksums enabled. Decode with + // per-block verification. Both sides emit exactly 1 + // checksum per physical block written to / read from the + // wire (encoder hashes per emission site, including each + // post-split partition; decoder hashes each decoded block). + // Cardinality and element-wise contents must match + // round-trip. + let payload: Vec = (0..200_000u32).map(|i| (i & 0xFF) as u8).collect(); + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_source(payload.as_slice()); + compressor.enable_per_block_checksums(); + let mut compressed = Vec::new(); + compressor.set_drain(&mut compressed); + compressor.compress(); + + let encoder_checksums = compressor + .last_frame_block_checksums() + .expect("checksums populated after enable + compress") + .to_vec(); + drop(compressor); + assert!(!encoder_checksums.is_empty()); + + // Decode side: enable verification, decode, compare. + let mut decoder = FrameDecoder::new(); + decoder.enable_per_block_checksums(); + let mut output = alloc::vec![0u8; payload.len()]; + let n = decoder + .decode_all(compressed.as_slice(), &mut output) + .expect("decode_all should succeed"); + assert_eq!(n, payload.len()); + assert_eq!(&output[..n], payload.as_slice()); + + let decoder_checksums = decoder.computed_block_checksums(); + assert_eq!(decoder_checksums, encoder_checksums.as_slice()); +} + +// ── decode_blocks_partial (block-subset partial decode, lsm) ── + +/// Build a multi-block compressible frame and return +/// `(compressed, full_decode, emit_info)`. The emit info's +/// `decompressed_byte_range` maps decompressed offsets to block indices. +#[cfg(feature = "lsm")] +fn multi_block_fixture() -> ( + Vec, + Vec, + crate::encoding::frame_emit_info::FrameEmitInfo, +) { + let mut data: Vec = Vec::with_capacity(400 * 1024); + let mut x = 0x9E37_79B9u32; + while data.len() < 400 * 1024 { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + let run = 16 + (x as usize % 48); + let byte = (x >> 24) as u8; + for _ in 0..run { + data.push(byte); + } + data.extend_from_slice(b"the quick brown fox jumps over the lazy dog\n"); + } + + let mut compressed = Vec::new(); + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_source(data.as_slice()); + compressor.set_drain(&mut compressed); + compressor.compress(); + let info = compressor + .last_frame_emit_info() + .expect("emit info populated") + .clone(); + drop(compressor); + + let mut dec = FrameDecoder::new(); + let mut full = alloc::vec![0u8; data.len()]; + let n = dec + .decode_all(compressed.as_slice(), &mut full) + .expect("full decode"); + full.truncate(n); + assert_eq!(full, data, "fixture must round-trip"); + (compressed, full, info) +} + +#[cfg(feature = "lsm")] +#[test] +fn decode_blocks_partial_subset_matches_full_decode() { + let (compressed, full, info) = multi_block_fixture(); + let nblocks = info.blocks.len() as u32; + assert!( + nblocks >= 4, + "fixture must have several blocks, got {nblocks}" + ); + let half = nblocks / 2; + // Boundaries: 1 block, 2 blocks, half, all, and a non-zero start. + // `(0, u32::MAX)` exercises the "decode to end of frame" sentinel, + // a distinct public contract from an explicit upper bound. + for &(s, e) in &[ + (0u32, u32::MAX), + (0, 1), + (0, 2), + (0, half), + (0, nblocks), + (1, 2), + (half, nblocks), + ] { + // The sentinel decodes through the last block; map it to nblocks + // for the expected-slice / block-count arithmetic below. + let effective_end = if e == u32::MAX { nblocks } else { e }; + let mut source = compressed.as_slice(); + let mut dec = FrameDecoder::new(); + dec.reset(&mut source).unwrap(); + let pd = dec + .decode_blocks_partial(&mut source, s, e, None, false) + .unwrap_or_else(|err| panic!("range [{s},{e}) errored: {err:?}")); + + let start = info.decompressed_byte_range(s as usize).unwrap().start as usize; + let end = info + .decompressed_byte_range((effective_end - 1) as usize) + .unwrap() + .end as usize; + assert_eq!( + pd.data.as_slice(), + &full[start..end], + "subset bytes must equal the full-decode slice for [{s},{e})" + ); + assert_eq!(pd.start_block, s); + assert_eq!(pd.blocks_decoded, effective_end - s); + assert!(pd.stopped_at.is_none(), "clean range [{s},{e})"); + } +} + +#[cfg(feature = "lsm")] +#[test] +fn decode_blocks_partial_recovers_clean_prefix_on_truncated_block() { + let (compressed, full, info) = multi_block_fixture(); + let nblocks = info.blocks.len(); + let k = nblocks / 2; + assert!(k >= 1, "need a clean prefix before the failing block"); + + // Truncate the source right after block k's 3-byte header, so its body + // read fails regardless of block type (0 body bytes available). + let cut = info.blocks[k].offset_in_frame as usize + info.blocks[k].header_size as usize; + let truncated = &compressed[..cut]; + + let mut source = truncated; + let mut dec = FrameDecoder::new(); + dec.reset(&mut source).unwrap(); + let pd = dec + .decode_blocks_partial(&mut source, 0, u32::MAX, None, false) + .unwrap(); + + let (idx, _err) = pd.stopped_at.expect("must stop on the truncated block"); + assert_eq!(idx, k as u32, "stopped at the truncated block index"); + assert_eq!(pd.blocks_decoded, k as u32, "blocks 0..k decoded cleanly"); + assert!(!pd.frame_finished); + let clean_end = info.decompressed_byte_range(k).unwrap().start as usize; + assert_eq!( + pd.data.as_slice(), + &full[..clean_end], + "clean prefix preserved through the failure" + ); +} + +#[cfg(feature = "lsm")] +#[test] +fn decode_blocks_partial_invalid_range_errors() { + let (compressed, _full, _info) = multi_block_fixture(); + let mut source = compressed.as_slice(); + let mut dec = FrameDecoder::new(); + dec.reset(&mut source).unwrap(); + let err = dec + .decode_blocks_partial(&mut source, 5, 2, None, false) + .expect_err("start > end must error"); + assert!(matches!( + err, + crate::decoding::errors::FrameDecoderError::InvalidBlockRange { + start_block: 5, + end_block: 2, + } + )); +} + +#[cfg(feature = "lsm")] +#[test] +fn decode_blocks_partial_skips_trailing_blocks() { + let (compressed, full, info) = multi_block_fixture(); + assert!(info.blocks.len() >= 3); + let mut source = compressed.as_slice(); + let mut dec = FrameDecoder::new(); + dec.reset(&mut source).unwrap(); + let pd = dec + .decode_blocks_partial(&mut source, 0, 1, None, false) + .unwrap(); + + assert_eq!(pd.blocks_decoded, 1); + assert!(pd.stopped_at.is_none()); + assert!(!pd.frame_finished, "block 0 is not the last block"); + let end = info.decompressed_byte_range(0).unwrap().end as usize; + assert_eq!(pd.data.as_slice(), &full[..end]); + // The trailing blocks + checksum were never consumed from the source. + assert!( + dec.bytes_read_from_source() < u64::from(info.total_size), + "only block 0's region should be consumed, read {} of {}", + dec.bytes_read_from_source(), + info.total_size + ); +} + +#[cfg(feature = "lsm")] +#[test] +fn lsm_style_range_query_partial_recovery() { + // Simulates lsm-tree's range-query path: a key range resolves to a + // decompressed byte window, which maps to inner zstd block indices via + // `decompressed_byte_range`; decode only the covering blocks and check + // the wanted window is recovered exactly (no key outside, all inside). + let (compressed, full, info) = multi_block_fixture(); + let total = full.len() as u64; + let want_start = total / 3; + let want_end = (total * 2) / 3; + + // Map [want_start, want_end) to covering block indices. + let nblocks = info.blocks.len(); + let mut start_block = 0u32; + let mut end_block = nblocks as u32; + for i in 0..nblocks { + let r = info.decompressed_byte_range(i).unwrap(); + if r.start <= want_start && want_start < r.end { + start_block = i as u32; + } + if r.start < want_end && want_end <= r.end { + end_block = i as u32 + 1; + break; + } + } + + let mut source = compressed.as_slice(); + let mut dec = FrameDecoder::new(); + dec.reset(&mut source).unwrap(); + let pd = dec + .decode_blocks_partial(&mut source, start_block, end_block, None, false) + .unwrap(); + assert!(pd.stopped_at.is_none()); + + let covered_start = info + .decompressed_byte_range(start_block as usize) + .unwrap() + .start; + let covered_end = info + .decompressed_byte_range((end_block - 1) as usize) + .unwrap() + .end; + assert!( + covered_start <= want_start && want_end <= covered_end, + "covering blocks must contain the wanted window" + ); + assert_eq!( + pd.data.as_slice(), + &full[covered_start as usize..covered_end as usize], + "covered subset must equal the full-decode slice" + ); + // Slice the exact key range out of the covered subset. + let off = (want_start - covered_start) as usize; + let len = (want_end - want_start) as usize; + assert_eq!( + &pd.data[off..off + len], + &full[want_start as usize..want_end as usize], + "exact key range recovered from the partial decode" + ); +} + +#[cfg(feature = "lsm")] +#[test] +fn decode_blocks_partial_leaves_no_residual_when_no_in_range_block() { + // Regression: when the requested range reaches no in-range block (here + // start_block is past EOF, so every block is decoded only as window + // context), `PartialDecode::data` is empty — but the context bytes must + // NOT linger in the decoder buffer, or a later collect()/read() on the + // same decoder returns out-of-range data. + let (compressed, _full, info) = multi_block_fixture(); + let nblocks = info.blocks.len() as u32; + let mut source = compressed.as_slice(); + let mut dec = FrameDecoder::new(); + dec.reset(&mut source).unwrap(); + let pd = dec + .decode_blocks_partial(&mut source, nblocks + 5, u32::MAX, None, false) + .unwrap(); + assert!(pd.data.is_empty(), "no in-range block → empty data"); + assert_eq!(pd.blocks_decoded, 0); + assert!( + pd.frame_finished, + "frame's last block was reached as context" + ); + assert_eq!( + dec.can_collect(), + 0, + "context bytes must not leak via collect()/read() when data is empty" + ); +} + +#[cfg(feature = "lsm")] +#[test] +fn decode_blocks_partial_empty_range_leaves_no_residual() { + // Companion to the start-past-EOF case: an in-frame empty range `[k, k)` + // (k < EOF) takes the same `prefix_window_len == None` path but with + // `frame_finished == false` and up to `window_size` context bytes still + // physically present. Assert the buffer is fully cleared directly (a + // `can_collect()` check alone would pass even with <= window_size bytes + // retained, because it holds the window back). + let (compressed, _full, info) = multi_block_fixture(); + let k = ((info.blocks.len() as u32) / 2).max(1); + let mut source = compressed.as_slice(); + let mut dec = FrameDecoder::new(); + dec.reset(&mut source).unwrap(); + let pd = dec + .decode_blocks_partial(&mut source, k, k, None, false) + .unwrap(); + + assert!(pd.data.is_empty(), "empty range must yield empty data"); + assert_eq!(pd.blocks_decoded, 0); + assert!( + !pd.frame_finished, + "frame should still have trailing blocks" + ); + assert_eq!( + dec.state.as_ref().unwrap().decoder_scratch.buffer_len(), + 0, + "empty-range partial decode must not retain context bytes" + ); +} + +#[cfg(all(feature = "lsm", feature = "hash"))] +#[test] +fn decode_blocks_partial_captures_per_block_checksums() { + // Regression: with per-block checksums enabled, decode_blocks_partial + // must populate computed_block_checksums just like decode_blocks / + // decode_all — otherwise callers verifying per-block digests silently + // lose them on the partial path. + let (compressed, full, _info) = multi_block_fixture(); + + // Reference digests via decode_blocks (the path that captures them). + let mut ref_dec = FrameDecoder::new(); + ref_dec.enable_per_block_checksums(); + let mut rsrc = compressed.as_slice(); + ref_dec.reset(&mut rsrc).unwrap(); + while !ref_dec.is_finished() { + ref_dec + .decode_blocks(&mut rsrc, crate::decoding::BlockDecodingStrategy::All) + .unwrap(); + } + let expected = ref_dec.computed_block_checksums().to_vec(); + assert!(!expected.is_empty(), "fixture must have multiple blocks"); + let _ = full; + + // Partial decode of the whole frame must capture the same digests. + let mut source = compressed.as_slice(); + let mut dec = FrameDecoder::new(); + dec.enable_per_block_checksums(); + dec.reset(&mut source).unwrap(); + let _ = dec + .decode_blocks_partial(&mut source, 0, u32::MAX, None, false) + .unwrap(); + assert_eq!( + dec.computed_block_checksums(), + expected.as_slice(), + "partial decode must capture the same per-block checksums as full decode" + ); +} + +// ── resume (window-priming + entropy cold resume, lsm) ─────────── + +/// Window size of `compressed`'s frame, read from a freshly-reset decoder. +#[cfg(feature = "lsm")] +fn frame_window_size(compressed: &[u8]) -> usize { + let mut src = compressed; + let mut dec = FrameDecoder::new(); + dec.reset(&mut src).unwrap(); + dec.state + .as_ref() + .unwrap() + .frame_header + .window_size() + .unwrap_or(0) as usize +} + +/// Build a large compressible MULTI-SEGMENT frame (window_size < content, +/// so mid-frame blocks reach back only into a bounded window) and return +/// `(compressed, full_decode, emit_info)`. +#[cfg(feature = "lsm")] +fn multi_segment_block_fixture() -> ( + Vec, + Vec, + crate::encoding::frame_emit_info::FrameEmitInfo, +) { + // ~3 MiB of compressible (runs + repeated phrase) data — large enough + // that the encoder picks window_size < content_size (multi-segment). + let mut data: Vec = Vec::with_capacity(3 * 1024 * 1024); + let mut x = 0x9E37_79B9u32; + while data.len() < 3 * 1024 * 1024 { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + let run = 16 + (x as usize % 48); + let byte = (x >> 24) as u8; + for _ in 0..run { + data.push(byte); + } + data.extend_from_slice(b"the quick brown fox jumps over the lazy dog\n"); + } + + let mut compressed = Vec::new(); + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_source(data.as_slice()); + compressor.set_drain(&mut compressed); + compressor.compress(); + let info = compressor + .last_frame_emit_info() + .expect("emit info populated") + .clone(); + drop(compressor); + + // Confirm the precondition: the frame must be multi-segment. + let mut sanity = FrameDecoder::new(); + sanity.init(&mut compressed.as_slice()).unwrap(); + assert!( + !sanity + .state + .as_ref() + .unwrap() + .frame_header + .descriptor + .single_segment_flag(), + "fixture precondition: frame must be multi-segment (resize if encoder default changed)" + ); + + let mut dec = FrameDecoder::new(); + let mut full = alloc::vec![0u8; data.len()]; + let n = dec + .decode_all(compressed.as_slice(), &mut full) + .expect("full decode"); + full.truncate(n); + assert_eq!(full, data, "fixture must round-trip"); + (compressed, full, info) +} + +/// Emit a [`ResumeState`] for resuming at block `n` by decoding `[0, n)` on +/// a throwaway decoder with `emit_resume = true`. +#[cfg(feature = "lsm")] +fn emit_resume_state_at(compressed: &[u8], n: u32) -> super::ResumeState { + let mut src = compressed; + let mut dec = FrameDecoder::new(); + dec.reset(&mut src).unwrap(); + let pd = dec + .decode_blocks_partial(&mut src, 0, n, None, true) + .expect("prefix decode for resume-state emission"); + pd.resume_state + .expect("emit_resume should populate resume_state") +} + +#[cfg(feature = "lsm")] +#[test] +fn resume_matches_full_decode_at_first_mid_last() { + // Acceptance criterion: after resuming at block N (cold decoder, primed + // window + restored entropy), decode_blocks_partial yields bytes + // byte-identical to a full decode's [ends[N-1]..ends[end-1]) slice, for + // N in {1, mid, last}. Repeat_Mode entropy blocks are covered because + // the emitted ResumeState carries the carry-over tables. + let (compressed, full, info) = multi_block_fixture(); + let nblocks = info.blocks.len() as u32; + assert!(nblocks >= 4, "need several blocks, got {nblocks}"); + + for &n in &[1u32, nblocks / 2, nblocks - 1] { + // Producer: emit resume state for block n (separate decoder). + let st = emit_resume_state_at(&compressed, n); + assert_eq!(st.block_index(), n); + let output_offset = info.decompressed_byte_range(n as usize).unwrap().start; + assert_eq!(st.output_offset(), output_offset); + + // Consumer: a FRESH (cold) decoder resumes at n. Pass the WHOLE + // decompressed prefix as window_prime; it is capped to one window + // internally, exercising the cap path. + let window_prime = &full[..output_offset as usize]; + let mut header_src = compressed.as_slice(); + let mut dec = FrameDecoder::new(); + dec.reset(&mut header_src).unwrap(); + // Caller positions the source at block n's compressed frame offset. + let off = info.blocks[n as usize].offset_in_frame as usize; + let mut block_src = &compressed[off..]; + let pd = dec + .decode_blocks_partial( + &mut block_src, + n, + u32::MAX, + Some(super::ResumeInput { + window_prime, + state: &st, + }), + false, + ) + .unwrap_or_else(|e| panic!("resume decode at N={n} errored: {e:?}")); + + let start = output_offset as usize; + let end = info + .decompressed_byte_range((nblocks - 1) as usize) + .unwrap() + .end as usize; + assert_eq!( + pd.data.as_slice(), + &full[start..end], + "resumed bytes must equal the full-decode slice for N={n}" + ); + assert_eq!(pd.start_block, n); + assert_eq!(pd.blocks_decoded, nblocks - n); + assert!(pd.stopped_at.is_none(), "clean resume at N={n}"); + assert!(pd.frame_finished, "decoded through the last block"); + } +} + +#[cfg(feature = "lsm")] +#[test] +fn resume_with_exact_window_tail_matches_full_decode() { + // Realistic cold-resume shape on a MULTI-SEGMENT frame: caller supplies + // only the last `window_size` decompressed bytes (not the whole prefix), + // which is all that can ever back a match. + let (compressed, full, info) = multi_segment_block_fixture(); + let nblocks = info.blocks.len() as u32; + let window_size = frame_window_size(&compressed); + // First block whose preceding output exceeds one window, so the tail + // genuinely truncates the prefix. + let n = (1..nblocks) + .find(|&i| info.decompressed_byte_range(i as usize).unwrap().start as usize > window_size) + .expect("multi-segment frame must have a block past one window"); + let st = emit_resume_state_at(&compressed, n); + let output_offset = info.decompressed_byte_range(n as usize).unwrap().start; + assert!(output_offset as usize > window_size); + let tail_start = output_offset as usize - window_size; + let window_prime = &full[tail_start..output_offset as usize]; + + let mut header_src = compressed.as_slice(); + let mut dec = FrameDecoder::new(); + dec.reset(&mut header_src).unwrap(); + let off = info.blocks[n as usize].offset_in_frame as usize; + let mut block_src = &compressed[off..]; + let pd = dec + .decode_blocks_partial( + &mut block_src, + n, + u32::MAX, + Some(super::ResumeInput { + window_prime, + state: &st, + }), + false, + ) + .unwrap(); + + let end = info + .decompressed_byte_range((nblocks - 1) as usize) + .unwrap() + .end as usize; + assert_eq!(pd.data.as_slice(), &full[output_offset as usize..end]); + assert_eq!(pd.blocks_decoded, nblocks - n); +} + +#[cfg(feature = "lsm")] +#[test] +fn resume_rejects_short_window_prime() { + // Acceptance criterion: a window_prime shorter than the required window + // is rejected with a typed error, not a silent mis-decode. + let (compressed, full, info) = multi_block_fixture(); + let nblocks = info.blocks.len() as u32; + let window_size = frame_window_size(&compressed); + let n = nblocks / 2; + let st = emit_resume_state_at(&compressed, n); + let output_offset = info.decompressed_byte_range(n as usize).unwrap().start; + let required = core::cmp::min(window_size as u64, output_offset) as usize; + assert!(required > 0, "mid block must require a non-empty window"); + + // One byte short of the required window. + let prime = &full[output_offset as usize - (required - 1)..output_offset as usize]; + + let mut header_src = compressed.as_slice(); + let mut dec = FrameDecoder::new(); + dec.reset(&mut header_src).unwrap(); + let off = info.blocks[n as usize].offset_in_frame as usize; + let mut block_src = &compressed[off..]; + let err = dec + .decode_blocks_partial( + &mut block_src, + n, + u32::MAX, + Some(super::ResumeInput { + window_prime: prime, + state: &st, + }), + false, + ) + .expect_err("short window_prime must be rejected"); + match err { + crate::decoding::errors::FrameDecoderError::ResumeWindowTooShort { got, need } => { + assert_eq!(got, required - 1); + assert_eq!(need, required); + } + other => panic!("expected ResumeWindowTooShort, got {other:?}"), + } +} + +#[cfg(feature = "lsm")] +#[test] +fn resume_range_validates_against_effective_start_not_start_block() { + // In resume mode `start_block` is ignored and decoding begins at + // `state.block_index()`. The range guard must therefore validate the + // EFFECTIVE start against `end_block`: `end_block` below the resume + // block is an inverted range and must error, not silently return an + // empty decode. Caller passes the conventional ignored `start_block = 0`. + let (compressed, _full, info) = multi_block_fixture(); + let nblocks = info.blocks.len() as u32; + let n = (nblocks / 2).max(2); + let st = emit_resume_state_at(&compressed, n); + let output_offset = info.decompressed_byte_range(n as usize).unwrap().start; + + let mut header_src = compressed.as_slice(); + let mut dec = FrameDecoder::new(); + dec.reset(&mut header_src).unwrap(); + let off = info.blocks[n as usize].offset_in_frame as usize; + let mut block_src = &compressed[off..]; + // end_block = n - 1 is below the resume block n → inverted range. + let err = dec + .decode_blocks_partial( + &mut block_src, + 0, + n - 1, + Some(super::ResumeInput { + window_prime: &_full[..output_offset as usize], + state: &st, + }), + false, + ) + .expect_err("end_block below the resume block must be an inverted range"); + match err { + crate::decoding::errors::FrameDecoderError::InvalidBlockRange { + start_block, + end_block, + } => { + assert_eq!(start_block, n, "error must report the effective start"); + assert_eq!(end_block, n - 1); + } + other => panic!("expected InvalidBlockRange, got {other:?}"), + } +} + +#[cfg(feature = "lsm")] +#[test] +fn resume_rejects_state_from_a_different_frame() { + // A ResumeState captured from one frame must not be applied to a frame + // with a different decode shape (window size / single-segment / dict): + // restoring foreign entropy tables would yield byte-wrong output. The + // frame-identity guard must reject it up front with a typed error. + let (frame_a, _full_a, info_a) = multi_block_fixture(); + let (frame_b, full_b, _info_b) = multi_segment_block_fixture(); + // Sanity: the two fixtures must differ in decode shape for the guard to + // be exercised (single-segment vs multi-segment here). + let st = emit_resume_state_at(&frame_a, (info_a.blocks.len() as u32 / 2).max(1)); + + let mut header_src = frame_b.as_slice(); + let mut dec = FrameDecoder::new(); + dec.reset(&mut header_src).unwrap(); + // The frame-key check runs before the window-length check, so even a + // valid-length window_prime for frame B is rejected on identity. + let err = dec + .decode_blocks_partial( + &mut frame_b.as_slice(), + st.block_index(), + u32::MAX, + Some(super::ResumeInput { + window_prime: &full_b, + state: &st, + }), + false, + ) + .expect_err("resume state from a different frame must be rejected"); + assert!( + matches!( + err, + crate::decoding::errors::FrameDecoderError::ResumeFrameMismatch + ), + "expected ResumeFrameMismatch, got {err:?}" + ); +} + +#[cfg(all(feature = "lsm", feature = "hash"))] +#[test] +fn resume_rejects_wrong_window_prime_content() { + // Same frame (FrameKey matches) but the caller supplies a window_prime + // with one byte flipped. The shape key cannot catch this; the + // content-exact XXH64 of the window must, rejecting before any restore + // rather than mis-resolving matches against corrupted history. + let (compressed, full, info) = multi_block_fixture(); + let nblocks = info.blocks.len() as u32; + let n = (nblocks / 2).max(1); + let st = emit_resume_state_at(&compressed, n); + let output_offset = info.decompressed_byte_range(n as usize).unwrap().start as usize; + assert!(output_offset > 0); + + // Correct prefix with the last byte corrupted (this byte is inside the + // window the resume block reaches back into). + let mut corrupted = full[..output_offset].to_vec(); + let last = corrupted.len() - 1; + corrupted[last] ^= 0xFF; + + let mut header_src = compressed.as_slice(); + let mut dec = FrameDecoder::new(); + dec.reset(&mut header_src).unwrap(); + let off = info.blocks[n as usize].offset_in_frame as usize; + let mut block_src = &compressed[off..]; + let err = dec + .decode_blocks_partial( + &mut block_src, + n, + u32::MAX, + Some(super::ResumeInput { + window_prime: &corrupted, + state: &st, + }), + false, + ) + .expect_err("corrupted window_prime must be rejected by content hash"); + assert!( + matches!( + err, + crate::decoding::errors::FrameDecoderError::ResumeFrameMismatch + ), + "expected ResumeFrameMismatch, got {err:?}" + ); +} + +#[cfg(feature = "lsm")] +#[test] +fn resume_rejects_state_with_different_active_dictionary() { + // A dictless-header frame can be decoded with an explicit dictionary + // applied at runtime (force_dict / reset_with_dict_handle). Two such + // decodes differ in entropy/repcode/dict context even though the header + // dictionary_id is identically absent, so the resume guard must key on + // the ACTIVE dictionary, not just the header field. Here the snapshot is + // captured with no active dictionary; resuming with one applied must be + // rejected before any state is restored. + let (compressed, full, info) = multi_block_fixture(); + let nblocks = info.blocks.len() as u32; + let n = (nblocks / 2).max(1); + let st = emit_resume_state_at(&compressed, n); // active_dictionary_id = None + let output_offset = info.decompressed_byte_range(n as usize).unwrap().start as usize; + + let raw = include_bytes!("../../../dict_tests/dictionary"); + let dict = crate::decoding::dictionary::Dictionary::decode_dict(raw).expect("parse dict"); + let dict_id = dict.id; + + let mut header_src = compressed.as_slice(); + let mut dec = FrameDecoder::new(); + dec.add_dict(dict).unwrap(); + dec.reset(&mut header_src).unwrap(); + dec.force_dict(dict_id).unwrap(); // active_dictionary_id = Some(dict_id) + let off = info.blocks[n as usize].offset_in_frame as usize; + let mut block_src = &compressed[off..]; + let err = dec + .decode_blocks_partial( + &mut block_src, + n, + u32::MAX, + Some(super::ResumeInput { + window_prime: &full[..output_offset], + state: &st, + }), + false, + ) + .expect_err("resume with a different active dictionary must be rejected"); + assert!( + matches!( + err, + crate::decoding::errors::FrameDecoderError::ResumeFrameMismatch + ), + "expected ResumeFrameMismatch, got {err:?}" + ); +} + +#[cfg(feature = "lsm")] +#[test] +fn resume_invalid_range_does_not_mutate_decoder_state() { + // An inverted effective range must be rejected WITHOUT priming the + // decoder: no entropy restore, no window prime, no cursor advance. As + // written before the fix, those mutations ran before the range check, + // leaving the decoder in a synthetic resumed state on the error path. + let (compressed, full, info) = multi_block_fixture(); + let nblocks = info.blocks.len() as u32; + let n = (nblocks / 2).max(2); + let st = emit_resume_state_at(&compressed, n); + let output_offset = info.decompressed_byte_range(n as usize).unwrap().start as usize; + + let mut header_src = compressed.as_slice(); + let mut dec = FrameDecoder::new(); + dec.reset(&mut header_src).unwrap(); + // Freshly reset: cursor at block 0. + assert_eq!(dec.state.as_ref().unwrap().block_counter, 0); + + let off = info.blocks[n as usize].offset_in_frame as usize; + let mut block_src = &compressed[off..]; + let err = dec + .decode_blocks_partial( + &mut block_src, + 0, + n - 1, // below the resume block → inverted range + Some(super::ResumeInput { + window_prime: &full[..output_offset], + state: &st, + }), + false, + ) + .expect_err("inverted range must error"); + assert!(matches!( + err, + crate::decoding::errors::FrameDecoderError::InvalidBlockRange { .. } + )); + assert_eq!( + dec.state.as_ref().unwrap().block_counter, + 0, + "error path must not advance the cursor (validate before priming)" + ); +} + +#[cfg(feature = "lsm")] +#[test] +fn emit_resume_state_absent_on_terminal_block() { + // When a decode reaches the frame's last block there is no "next block" + // to resume at: the snapshot's block_index would be one past EOF and the + // caller has no offset_in_frame for it. emit_resume must therefore yield + // None on the terminal block, not a dangling snapshot. + let (compressed, _full, info) = multi_block_fixture(); + let nblocks = info.blocks.len() as u32; + let mut src = compressed.as_slice(); + let mut dec = FrameDecoder::new(); + dec.reset(&mut src).unwrap(); + let pd = dec + .decode_blocks_partial(&mut src, 0, nblocks, None, true) + .unwrap(); + assert!(pd.frame_finished, "decode must reach the last block"); + assert!( + pd.resume_state.is_none(), + "no resume state past the frame's last block" + ); +} + +#[cfg(feature = "lsm")] +#[test] +fn emit_resume_state_absent_when_not_requested() { + // Default partial decode (emit_resume = false) must NOT pay the entropy + // clone: resume_state stays None. + let (compressed, _full, info) = multi_block_fixture(); + let nblocks = info.blocks.len() as u32; + let mut src = compressed.as_slice(); + let mut dec = FrameDecoder::new(); + dec.reset(&mut src).unwrap(); + let pd = dec + .decode_blocks_partial(&mut src, 0, nblocks, None, false) + .unwrap(); + assert!( + pd.resume_state.is_none(), + "resume_state must be None unless emit_resume is set" + ); +} + +#[cfg(feature = "lsm")] +#[test] +fn resume_grow_loop_reconstructs_full() { + // The motivating scenario: a symmetric one-call grow-loop. Each call + // takes the previous ResumeState and emits the next, decoding only the + // new extent — concatenated, the extents reconstruct the full output + // with no prefix ever re-decompressed. + let (compressed, full, info) = multi_block_fixture(); + let nblocks = info.blocks.len() as u32; + assert!(nblocks >= 4); + + // Walk the frame in extents of `step` blocks each. + let step = (nblocks / 3).max(1); + let mut combined: Vec = Vec::new(); + let mut next: u32 = 0; + let mut carry: Option = None; + + while next < nblocks { + let end = (next + step).min(nblocks); + let mut dec = FrameDecoder::new(); + let mut header_src = compressed.as_slice(); + dec.reset(&mut header_src).unwrap(); + + let off = info.blocks[next as usize].offset_in_frame as usize; + let mut block_src = &compressed[off..]; + + let output_offset = info.decompressed_byte_range(next as usize).unwrap().start; + let pd = if let Some(st) = carry.as_ref() { + // Resume from the prior extent's state (cold: fresh decoder). + let window_prime = &full[..output_offset as usize]; + dec.decode_blocks_partial( + &mut block_src, + next, + end, + Some(super::ResumeInput { + window_prime, + state: st, + }), + true, + ) + .unwrap() + } else { + // First extent: no resume input, just emit for the next. + dec.decode_blocks_partial(&mut block_src, next, end, None, true) + .unwrap() + }; + + combined.extend_from_slice(&pd.data); + carry = pd.resume_state; + next = end; + } + + assert_eq!( + combined, full, + "grow-loop extents must reconstruct the full output" + ); +} + +#[cfg(all(feature = "lsm", feature = "hash"))] +#[test] +fn resume_does_not_redecode_prefix_blocks() { + // Instrumented confirmation that blocks < N are not re-decoded on + // resume. With per-block checksums enabled on the resuming decoder, the + // resumed decode must record exactly one digest per in-range block + // (end - N), never one per frame block. + let (compressed, full, info) = multi_block_fixture(); + let nblocks = info.blocks.len() as u32; + let n = nblocks / 2; + let st = emit_resume_state_at(&compressed, n); + let output_offset = info.decompressed_byte_range(n as usize).unwrap().start; + + let mut header_src = compressed.as_slice(); + let mut dec = FrameDecoder::new(); + dec.enable_per_block_checksums(); + dec.reset(&mut header_src).unwrap(); + let off = info.blocks[n as usize].offset_in_frame as usize; + let mut block_src = &compressed[off..]; + let _ = dec + .decode_blocks_partial( + &mut block_src, + n, + u32::MAX, + Some(super::ResumeInput { + window_prime: &full[..output_offset as usize], + state: &st, + }), + false, + ) + .unwrap(); + + assert_eq!( + dec.computed_block_checksums().len() as u32, + nblocks - n, + "resume must decode only in-range blocks, not re-decode the prefix" + ); +} diff --git a/zstd/src/decoding/frame_inspection_tests.rs b/zstd/src/decoding/frame_inspection_tests.rs new file mode 100644 index 000000000..18d660ba2 --- /dev/null +++ b/zstd/src/decoding/frame_inspection_tests.rs @@ -0,0 +1,200 @@ +use super::{ + FrameContentSize, FrameSizeError, find_frame_compressed_size, frame_decompressed_bound, + frame_header_size, read_frame_content_size, read_frame_header_info, +}; +use crate::encoding::{CompressionLevel, compress_slice_to_vec}; +use alloc::vec; +use alloc::vec::Vec; + +fn frame(content: &[u8]) -> Vec { + compress_slice_to_vec(content, CompressionLevel::Default) +} + +/// A hand-built single raw-block frame that omits `Frame_Content_Size` +/// (descriptor `0x00`: FCS_Flag=0, Single_Segment=0) so it carries a +/// Window_Descriptor instead — the only way to exercise the window-size +/// fallback in [`frame_decompressed_bound`] / [`read_frame_header_info`], +/// which the encoder (always declaring FCS) never produces. +fn no_fcs_frame() -> Vec { + vec![ + 0x28, 0xB5, 0x2F, 0xFD, // magic + 0x00, // frame header descriptor: no FCS, multi-segment, no checksum + 0x00, // window descriptor -> windowLog 10 -> 1024 bytes + 0x19, 0x00, 0x00, // block header: last, raw, size 3 + 0xAA, 0xBB, 0xCC, // raw payload + ] +} + +/// As [`no_fcs_frame`] but with the Content_Checksum_flag (descriptor bit 2) +/// set and a 4-byte trailer appended, to cover the checksum-trailer branch. +fn no_fcs_checksum_frame() -> Vec { + vec![ + 0x28, 0xB5, 0x2F, 0xFD, 0x04, // descriptor: checksum flag set + 0x00, 0x19, 0x00, 0x00, 0xAA, 0xBB, 0xCC, // window + block + payload + 0xDE, 0xAD, 0xBE, 0xEF, // content checksum trailer + ] +} + +/// A skippable frame: magic `0x184D2A50`, 4-byte length, then `length` bytes. +fn skippable_frame(payload: &[u8]) -> Vec { + let mut f = vec![0x50, 0x2A, 0x4D, 0x18]; + f.extend_from_slice(&(payload.len() as u32).to_le_bytes()); + f.extend_from_slice(payload); + f +} + +#[test] +fn read_frame_content_size_reports_declared_size() { + let f = frame(&[42u8; 100]); + assert_eq!( + read_frame_content_size(&f).unwrap(), + FrameContentSize::Known(100) + ); +} + +#[test] +fn read_frame_content_size_reports_unknown_without_fcs() { + assert_eq!( + read_frame_content_size(&no_fcs_frame()).unwrap(), + FrameContentSize::Unknown + ); +} + +#[test] +fn read_frame_content_size_errors_on_garbage() { + assert!(read_frame_content_size(&[0xAB; 16]).is_err()); +} + +#[test] +fn find_frame_compressed_size_spans_one_frame_then_the_next() { + let first = frame(&[5u8; 256]); + assert_eq!(find_frame_compressed_size(&first).unwrap(), first.len()); + + let mut two = first.clone(); + two.extend_from_slice(&frame(&[9u8; 50])); + // Still reports only the first frame so a caller can step forward. + assert_eq!(find_frame_compressed_size(&two).unwrap(), first.len()); +} + +#[test] +fn find_frame_compressed_size_measures_skippable_frame() { + let skip = skippable_frame(&[1, 2, 3, 4]); + assert_eq!(find_frame_compressed_size(&skip).unwrap(), skip.len()); +} + +#[test] +fn find_frame_compressed_size_rejects_truncation() { + let f = frame(&[7u8; 512]); + // Drop the trailing block bytes mid-frame. + let err = find_frame_compressed_size(&f[..f.len() - 4]).unwrap_err(); + assert!(matches!(err, FrameSizeError::Truncated)); +} + +#[test] +fn frame_header_size_matches_first_block_offset() { + let f = frame(&[3u8; 2048]); + let hdr = frame_header_size(&f).unwrap(); + assert!((5..=18).contains(&hdr)); + assert!(frame_header_size(&[0u8; 2]).is_err()); +} + +#[test] +fn read_frame_header_info_fills_declared_fields() { + let f = frame(&[7u8; 512]); + let info = read_frame_header_info(&f, false).unwrap(); + assert_eq!(info.content_size, FrameContentSize::Known(512)); + assert!(info.window_size >= 512); + assert_eq!(info.dictionary_id, None); +} + +#[test] +fn read_frame_header_info_derives_window_without_fcs() { + let info = read_frame_header_info(&no_fcs_frame(), false).unwrap(); + assert_eq!(info.content_size, FrameContentSize::Unknown); + assert_eq!(info.window_size, 1024); +} + +#[test] +fn frame_decompressed_bound_returns_declared_size() { + let f = frame(&[4u8; 4096]); + assert_eq!(frame_decompressed_bound(&f).unwrap(), 4096); +} + +#[test] +fn frame_decompressed_bound_uses_block_bound_without_fcs() { + // No declared FCS -> block_count(1) * block_size_max(min(1024,128K)). + assert_eq!(frame_decompressed_bound(&no_fcs_frame()).unwrap(), 1024); +} + +#[test] +fn frame_decompressed_bound_accepts_present_checksum_trailer() { + assert_eq!( + frame_decompressed_bound(&no_fcs_checksum_frame()).unwrap(), + 1024 + ); +} + +#[test] +fn frame_decompressed_bound_rejects_missing_checksum_trailer() { + let mut f = no_fcs_checksum_frame(); + f.truncate(f.len() - 4); // drop the 4-byte trailer the descriptor promises + assert!(matches!( + frame_decompressed_bound(&f).unwrap_err(), + FrameSizeError::Truncated + )); +} + +/// A block header may declare a `Block_Size` larger than the frame's +/// `Block_Maximum_Size` (`min(window, 128 KiB)`). RFC 8878 forbids this; +/// accepting it lets a corrupt frame pass the size query and makes the +/// no-FCS decompressed bound under-count (the raw block regenerates more +/// bytes than `block_count * block_size_max`). Both helpers must reject it. +#[test] +fn size_helpers_reject_oversized_block_header() { + // Window 1024 (WD 0x00) -> Block_Maximum_Size = 1024. Declare a raw + // block of Block_Size 2000 with all 2000 payload bytes present, so the + // failure is the oversized declaration, not truncation. + let block_size = 2000usize; + let raw = ((block_size as u32) << 3) | 1; // last_block flag, Raw type (00) + let mut f = vec![ + 0x28, + 0xB5, + 0x2F, + 0xFD, // magic + 0x00, // descriptor: no FCS, multi-segment, no checksum + 0x00, // window descriptor -> 1024 bytes + (raw & 0xFF) as u8, + ((raw >> 8) & 0xFF) as u8, + ((raw >> 16) & 0xFF) as u8, + ]; + f.resize(f.len() + block_size, 0xAB); + + assert!(matches!( + find_frame_compressed_size(&f).unwrap_err(), + FrameSizeError::OversizedBlock + )); + assert!(matches!( + frame_decompressed_bound(&f).unwrap_err(), + FrameSizeError::OversizedBlock + )); +} + +#[test] +fn frame_decompressed_bound_handles_skippable_frame() { + assert_eq!( + frame_decompressed_bound(&skippable_frame(&[0u8; 8])).unwrap(), + 0 + ); + // A skippable frame whose advertised payload is absent is truncation. + let mut short = skippable_frame(&[0u8; 8]); + short.truncate(short.len() - 2); + assert!(matches!( + frame_decompressed_bound(&short).unwrap_err(), + FrameSizeError::Truncated + )); +} + +#[test] +fn frame_decompressed_bound_errors_on_garbage_header() { + assert!(frame_decompressed_bound(&[0xAB; 16]).is_err()); +} diff --git a/zstd/src/decoding/literals_section_decoder.rs b/zstd/src/decoding/literals_section_decoder.rs index 53796f1ba..03c87f8fd 100644 --- a/zstd/src/decoding/literals_section_decoder.rs +++ b/zstd/src/decoding/literals_section_decoder.rs @@ -844,393 +844,7 @@ unsafe fn run_4stream_burst_loop( } #[cfg(test)] -mod zerocopy_robustness_tests { - //! Regression coverage for `decode_literals_zerocopy` on - //! truncated / corrupt payloads: every branch must return a - //! structured error instead of panicking on out-of-bounds - //! slice indexing. Hit each `*[..n]` / `*[0]` index in the - //! function with a payload one byte short of what the header - //! declares. - // - // Tests live in a separate module so the broader `burst_gate_tests` - // module's helpers don't have to depend on truncated-input - // builders. - use super::{LiteralsView, decode_literals_zerocopy}; - use crate::blocks::literals_section::{LiteralsSection, LiteralsSectionType}; - use crate::decoding::scratch::HuffmanScratch; - use alloc::vec::Vec; - - fn raw_section(regen: u32) -> LiteralsSection { - LiteralsSection { - ls_type: LiteralsSectionType::Raw, - regenerated_size: regen, - compressed_size: None, - num_streams: None, - } - } - - fn rle_section(regen: u32) -> LiteralsSection { - LiteralsSection { - ls_type: LiteralsSectionType::RLE, - regenerated_size: regen, - compressed_size: None, - num_streams: None, - } - } - - fn fresh_scratch() -> HuffmanScratch { - HuffmanScratch::new() - } - - #[test] - fn raw_truncated_source_returns_error_no_panic() { - // Header claims 10 raw literal bytes, source carries 3. - // Indexing `source[0..10]` would panic; the fix must turn - // it into a structured DecompressLiteralsError. - let section = raw_section(10); - let source: [u8; 3] = [1, 2, 3]; - let mut target: Vec = Vec::new(); - let mut scratch = fresh_scratch(); - let result = decode_literals_zerocopy(§ion, &mut scratch, &source, &mut target); - assert!( - result.is_err(), - "truncated raw source must error, not panic; got {:?}", - result.map(|_| ()) - ); - } - - #[test] - fn rle_empty_source_returns_error_no_panic() { - // RLE section needs at least one source byte (the fill byte). - // Indexing `source[0]` on an empty slice would panic. - let section = rle_section(10); - let source: [u8; 0] = []; - let mut target: Vec = Vec::new(); - let mut scratch = fresh_scratch(); - let result = decode_literals_zerocopy(§ion, &mut scratch, &source, &mut target); - assert!( - result.is_err(), - "empty RLE source must error, not panic; got {:?}", - result.map(|_| ()) - ); - } - - #[test] - fn compressed_truncated_source_returns_error_no_panic() { - // Header claims compressed_size = 10 but the source carries 3 bytes. - // Slicing `source[0..10]` for a Compressed/Treeless section would - // panic (decoder DoS on truncated input); the fix must turn it into - // a structured DecompressLiteralsError, matching the Raw/RLE paths. - let section = LiteralsSection { - ls_type: LiteralsSectionType::Compressed, - regenerated_size: 5, - compressed_size: Some(10), - num_streams: Some(1), - }; - let source: [u8; 3] = [1, 2, 3]; - let mut target: Vec = Vec::new(); - let mut scratch = fresh_scratch(); - let result = decode_literals_zerocopy(§ion, &mut scratch, &source, &mut target); - // Pin the EXACT contract: a truncated Compressed section must report - // MissingBytesForLiterals with the precise got/needed, not just "some - // error" (a weaker is_err() would also pass on MissingNumStreams / - // UninitializedHuffmanTable, missing the real regression). - assert!( - matches!( - &result, - Err( - crate::decoding::errors::DecompressLiteralsError::MissingBytesForLiterals { - got: 3, - needed: 10, - } - ) - ), - "truncated compressed source must report MissingBytesForLiterals, got {:?}", - result.map(|_| ()) - ); - } - - #[test] - fn rle_view_excludes_pre_existing_target_bytes() { - // Even if the caller forgot to clear `target`, the returned - // LiteralsView::data must point only at the bytes this call - // produced. The API hardening (`&target[base..]`) is what - // makes this hold. - let mut target: Vec = Vec::from([0xAA, 0xBB, 0xCC]); - let section = rle_section(4); - let source: [u8; 1] = [0x42]; - let mut scratch = fresh_scratch(); - let view = decode_literals_zerocopy(§ion, &mut scratch, &source, &mut target) - .expect("RLE with valid source must succeed"); - assert_eq!(view.data.len(), 4, "view length must match regen_size"); - assert!( - view.data.iter().all(|&b| b == 0x42), - "view must contain only the newly-RLE-expanded bytes, got {:?}", - view.data - ); - // Silence unused-warning if the compiler ever strips - // LiteralsView fields — read bytes_used too. - let _ = LiteralsView { - data: view.data, - bytes_used: view.bytes_used, - }; - } -} +mod zerocopy_robustness_tests; #[cfg(test)] -mod burst_gate_tests { - //! Regression coverage for the HUF 4-stream burst-gate boundary - //! states in `decompress_literals`: - //! - //! 1. `bits_consumed == max_num_bits` — lower boundary of the - //! burst gate, where the gate is entered with zero slack. - //! 2. `bits_consumed + burst_bits == 64` — upper boundary, where - //! the burst consumes all remaining bits in the 64-bit window - //! without overflow. - //! 3. SIMD-fallback → refill → burst re-entry — outer loop falls - //! back to the SIMD 4-symbol path, a `BitReaderReversed` - //! refill occurs, the next iteration re-enters the burst path - //! once `bits_consumed` grows back into burst range. - //! - //! Each named test pins an input shape chosen to drive the gate - //! through the corresponding regime — short skewed input for the - //! initial-entry lower-bound, long mid-cardinality streams for - //! many upper-bound brushes, multi-segment input for repeated - //! SIMD↔burst transitions. The sweep test covers the gate in - //! aggregate across many `(size, alphabet)` combinations. - //! - //! These tests do NOT assert that a specific - //! `(bits_consumed, burst_bits)` configuration is hit deterministically - //! on any single iteration — that would require white-box state - //! instrumentation that the current decoder does not expose. They - //! assert end-to-end roundtrip correctness through the full - //! encoder → 4-stream HUF block → `decode_literals` path; a - //! burst-gate regression that returns the wrong symbol or - //! desynchronises a stream produces either a - //! `DecompressLiteralsError` from the `BitstreamReadMismatch` / - //! `DecodedLiteralCountMismatch` guards or a mismatched decoded - //! buffer — both fail the assertion. The `max_num_bits` range - //! checks in the per-test helper also detect silent drift where - //! the encoder's table-generation choice shifts the test out of - //! the intended gate regime. - use super::*; - use crate::bit_io::BitWriter; - use crate::blocks::literals_section::{LiteralsSection, LiteralsSectionType}; - use crate::decoding::scratch::HuffmanScratch; - use crate::huff0::huff0_encoder::{HuffmanEncoder, HuffmanTable as EncTable}; - use alloc::vec::Vec; - - /// Encode `data` as a 4-stream HUF Compressed literals block (table - /// description + jump table + 4 padded streams) and return the - /// matching `LiteralsSection` header plus the wire bytes. - fn build_huf4x_block(data: &[u8]) -> (LiteralsSection, Vec) { - assert!(data.len() >= 4, "encode4x requires at least 4 bytes"); - let table = EncTable::build_from_data(data); - let mut source: Vec = Vec::new(); - { - let mut writer = BitWriter::from(&mut source); - let mut encoder = HuffmanEncoder::new(&table, &mut writer); - encoder.encode4x(data, true); - writer.flush(); - } - let section = LiteralsSection { - ls_type: LiteralsSectionType::Compressed, - regenerated_size: data.len() as u32, - compressed_size: Some(source.len() as u32), - num_streams: Some(4), - }; - (section, source) - } - - /// Roundtrip `data` through encode4x + decode_literals and assert - /// the decoded buffer matches byte-for-byte. Returns the HUF table's - /// `max_num_bits` so call sites can sanity-check that they actually - /// hit the expected burst-gate regime. - fn roundtrip_assert(data: &[u8]) -> u8 { - let (section, source) = build_huf4x_block(data); - let mut scratch = HuffmanScratch::new(); - let mut target = Vec::new(); - let bytes_read = decode_literals(§ion, &mut scratch, &source, &mut target) - .expect("decode_literals must succeed on a well-formed roundtrip"); - assert_eq!( - bytes_read as usize, - source.len(), - "decoder must consume every byte of the literals block" - ); - assert_eq!( - target, data, - "decoded literals must match the encoder input" - ); - scratch.table.max_num_bits - } - - /// Roundtrip + assertion that the HUF table's `max_num_bits` falls - /// inside the expected range — this is what selects which burst-gate - /// regime the body runs under (`symbols_per_burst = (63 - max) / max`). - fn roundtrip_with_max_bits_range(data: &[u8], expected: core::ops::RangeInclusive) { - let m = roundtrip_assert(data); - assert!( - expected.contains(&m), - "max_num_bits {} outside expected range {:?} for this fixture — \ - test no longer exercises the intended gate regime", - m, - expected - ); - } - - /// Lower boundary: targets `bits_consumed == max_num_bits` on - /// early burst entries. - /// - /// A short stream with a skewed 23-symbol alphabet keeps - /// `max_num_bits` in the 5..=11 band and limits the number of - /// burst iterations, so early iterations run with `bits_consumed` - /// near the gate threshold. The decoder must not lose low stream - /// bits when the shift formula runs close to the threshold; - /// roundtrip correctness over short input is the regression signal. - #[test] - fn burst_gate_lower_boundary_short_skewed_alphabet() { - // 36 bytes, 23 distinct symbols, skewed distribution — - // encoder picks max_num_bits in the 5..=11 band. - let mut data: Vec = Vec::with_capacity(36); - data.extend_from_slice(&[ - 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, - 14, 15, 16, 17, 18, 19, 20, 21, 22, - ]); - roundtrip_with_max_bits_range(&data, 5..=11); - } - - /// Upper boundary: `bits_consumed + burst_bits == 64`. - /// - /// A long, mid-cardinality alphabet drives many full burst windows. - /// Across thousands of iterations the burst-fits-in-64 guard - /// (`bits_consumed + burst_bits <= 64`) is approached and met - /// exactly. A regression that miscalculated the upper boundary - /// would read past the loaded 8-byte window and either crash under - /// debug bounds checks or desynchronise the stream — either way - /// the roundtrip fails. - #[test] - fn burst_gate_upper_boundary_long_mid_alphabet() { - // 4 KiB with a 97-symbol pseudo-random alphabet (kept under the - // encoder's 128-weight raw-table limit). Broad distribution → - // max_num_bits ≈ 7..9, thousands of burst iterations across all - // four streams. - let mut data: Vec = Vec::with_capacity(4096); - for i in 0..4096u32 { - data.push((i.wrapping_mul(0x9E37_79B1) % 97) as u8); - } - roundtrip_with_max_bits_range(&data, 6..=11); - } - - /// SIMD-fallback → refill → burst re-entry transition. - /// - /// After a `BitReaderReversed::refill` (triggered inside - /// `advance_state_by_bits` on the SIMD path), `bits_consumed` - /// rebases to `[0, 7]`. Until it climbs back to `max_num_bits` the - /// burst gate is closed and the outer loop runs the 4-symbol SIMD - /// fallback; on the next outer-loop iteration after `bits_consumed` - /// grows past `max_num_bits` the burst path must re-enter cleanly. - /// - /// Stream length of 16 KiB / 4 ≈ 4 KiB per stream encoded ⇒ each - /// `BitReaderReversed` window crosses many refill boundaries, - /// guaranteeing the SIMD→refill→burst transition fires repeatedly. - #[test] - fn burst_simd_fallback_refill_reentry_long_streams() { - // 67-symbol modulo distribution (`i % 67`, prime modulus spreads - // the alphabet evenly) → max_num_bits typically 7..8, which gives - // `symbols_per_burst = (63 - max) / max ≈ 6..8`. - let mut data: Vec = Vec::with_capacity(16 * 1024); - for i in 0..16 * 1024u32 { - data.push((i % 67) as u8); - } - roundtrip_with_max_bits_range(&data, 5..=8); - } - - /// Parametric sweep across stream lengths and alphabet shapes. - /// - /// The three burst-gate states above are also hit across this matrix - /// at varying `(bits_consumed, max_num_bits, symbols_per_burst)` - /// configurations; any future tweak to the gate that mishandles a - /// specific `(max_num_bits, post-refill bits_consumed)` combo trips - /// at least one cell here. - #[test] - fn burst_gate_sweep_sizes_and_alphabets() { - let sizes = [ - 16usize, 17, 31, 32, 33, 63, 64, 65, 127, 128, 129, 255, 256, 257, 511, 512, 513, 1023, - 1024, 1025, 4096, - ]; - for &n in &sizes { - // Binary alphabet → max_num_bits == 1, symbols_per_burst large. - let mut bin: Vec = Vec::with_capacity(n); - for i in 0..n { - bin.push((i & 1) as u8); - } - roundtrip_assert(&bin); - - // 16-symbol uniform alphabet → max_num_bits ≈ 4. - let mut sm: Vec = Vec::with_capacity(n); - for i in 0..n { - sm.push((i % 16) as u8); - } - roundtrip_assert(&sm); - - // 97-symbol pseudo-random alphabet (where length permits) → - // max_num_bits ≈ 7..9; kept under the encoder's 128-weight - // raw-table cap so the encoder reliably succeeds. - if n >= 128 { - let mut wide: Vec = Vec::with_capacity(n); - for i in 0..n { - wide.push((i.wrapping_mul(2_654_435_761) % 97) as u8); - } - roundtrip_assert(&wide); - } - } - } - - /// Adversarial regression for the `burst_eligible` safety gate. - /// - /// Builds a valid 4-stream HUF block, then forges a `LiteralsSection` - /// header that claims `regenerated_size = 1` while the encoded - /// streams still contain a full block worth of symbols. The shrunk - /// `regenerated_size` collapses `min_seg_len` below - /// `symbols_per_burst`, the exact precondition `burst_eligible` - /// guards against. Without that gate, the burst inner loop would - /// advance `cursors[i]` past `ends[i]` and panic on the - /// `target[cursors[i]]` write — a DoS surface on malformed input. - /// - /// With the gate, the decoder either: - /// - falls through to the SIMD-fallback path which immediately - /// hits the top-of-loop `cursor_exit_olimit` exit and returns - /// a count-mismatch / bitstream-mismatch error, or - /// - returns an error before the loop ever runs. - /// - /// Either way the test asserts `Err(_)` — the contract is "no - /// panic, return an error". - #[test] - fn burst_gate_malformed_small_regen_returns_error() { - // 256 bytes is well above MIN_LITERALS_FOR_4_STREAMS so the - // encoder will happily emit a 4-stream HUF block. The modulo - // alphabet keeps `max_num_bits` small (≤ 8), maximising - // `symbols_per_burst` so the small forged `regenerated_size` - // sits well below it. - let mut data: Vec = Vec::with_capacity(256); - for i in 0..256u32 { - data.push((i % 67) as u8); - } - let (mut section, source) = build_huf4x_block(&data); - - // Forge: claim only 1 regenerated byte. Streams in `source` - // are still encoded for the full 256-byte input. - section.regenerated_size = 1; - - let mut scratch = HuffmanScratch::new(); - let mut target = Vec::new(); - let result = decode_literals(§ion, &mut scratch, &source, &mut target); - - assert!( - result.is_err(), - "decoder must reject the malformed header instead of panicking; \ - got Ok({})", - result.unwrap_or(0) - ); - } -} +mod burst_gate_tests; diff --git a/zstd/src/decoding/literals_section_decoder/burst_gate_tests.rs b/zstd/src/decoding/literals_section_decoder/burst_gate_tests.rs new file mode 100644 index 000000000..092174453 --- /dev/null +++ b/zstd/src/decoding/literals_section_decoder/burst_gate_tests.rs @@ -0,0 +1,254 @@ +//! Regression coverage for the HUF 4-stream burst-gate boundary +//! states in `decompress_literals`: +//! +//! 1. `bits_consumed == max_num_bits` — lower boundary of the +//! burst gate, where the gate is entered with zero slack. +//! 2. `bits_consumed + burst_bits == 64` — upper boundary, where +//! the burst consumes all remaining bits in the 64-bit window +//! without overflow. +//! 3. SIMD-fallback → refill → burst re-entry — outer loop falls +//! back to the SIMD 4-symbol path, a `BitReaderReversed` +//! refill occurs, the next iteration re-enters the burst path +//! once `bits_consumed` grows back into burst range. +//! +//! Each named test pins an input shape chosen to drive the gate +//! through the corresponding regime — short skewed input for the +//! initial-entry lower-bound, long mid-cardinality streams for +//! many upper-bound brushes, multi-segment input for repeated +//! SIMD↔burst transitions. The sweep test covers the gate in +//! aggregate across many `(size, alphabet)` combinations. +//! +//! These tests do NOT assert that a specific +//! `(bits_consumed, burst_bits)` configuration is hit deterministically +//! on any single iteration — that would require white-box state +//! instrumentation that the current decoder does not expose. They +//! assert end-to-end roundtrip correctness through the full +//! encoder → 4-stream HUF block → `decode_literals` path; a +//! burst-gate regression that returns the wrong symbol or +//! desynchronises a stream produces either a +//! `DecompressLiteralsError` from the `BitstreamReadMismatch` / +//! `DecodedLiteralCountMismatch` guards or a mismatched decoded +//! buffer — both fail the assertion. The `max_num_bits` range +//! checks in the per-test helper also detect silent drift where +//! the encoder's table-generation choice shifts the test out of +//! the intended gate regime. +use super::*; +use crate::bit_io::BitWriter; +use crate::blocks::literals_section::{LiteralsSection, LiteralsSectionType}; +use crate::decoding::scratch::HuffmanScratch; +use crate::huff0::huff0_encoder::{HuffmanEncoder, HuffmanTable as EncTable}; +use alloc::vec::Vec; + +/// Encode `data` as a 4-stream HUF Compressed literals block (table +/// description + jump table + 4 padded streams) and return the +/// matching `LiteralsSection` header plus the wire bytes. +fn build_huf4x_block(data: &[u8]) -> (LiteralsSection, Vec) { + assert!(data.len() >= 4, "encode4x requires at least 4 bytes"); + let table = EncTable::build_from_data(data); + let mut source: Vec = Vec::new(); + { + let mut writer = BitWriter::from(&mut source); + let mut encoder = HuffmanEncoder::new(&table, &mut writer); + encoder.encode4x(data, true); + writer.flush(); + } + let section = LiteralsSection { + ls_type: LiteralsSectionType::Compressed, + regenerated_size: data.len() as u32, + compressed_size: Some(source.len() as u32), + num_streams: Some(4), + }; + (section, source) +} + +/// Roundtrip `data` through encode4x + decode_literals and assert +/// the decoded buffer matches byte-for-byte. Returns the HUF table's +/// `max_num_bits` so call sites can sanity-check that they actually +/// hit the expected burst-gate regime. +fn roundtrip_assert(data: &[u8]) -> u8 { + let (section, source) = build_huf4x_block(data); + let mut scratch = HuffmanScratch::new(); + let mut target = Vec::new(); + let bytes_read = decode_literals(§ion, &mut scratch, &source, &mut target) + .expect("decode_literals must succeed on a well-formed roundtrip"); + assert_eq!( + bytes_read as usize, + source.len(), + "decoder must consume every byte of the literals block" + ); + assert_eq!( + target, data, + "decoded literals must match the encoder input" + ); + scratch.table.max_num_bits +} + +/// Roundtrip + assertion that the HUF table's `max_num_bits` falls +/// inside the expected range — this is what selects which burst-gate +/// regime the body runs under (`symbols_per_burst = (63 - max) / max`). +fn roundtrip_with_max_bits_range(data: &[u8], expected: core::ops::RangeInclusive) { + let m = roundtrip_assert(data); + assert!( + expected.contains(&m), + "max_num_bits {} outside expected range {:?} for this fixture — \ + test no longer exercises the intended gate regime", + m, + expected + ); +} + +/// Lower boundary: targets `bits_consumed == max_num_bits` on +/// early burst entries. +/// +/// A short stream with a skewed 23-symbol alphabet keeps +/// `max_num_bits` in the 5..=11 band and limits the number of +/// burst iterations, so early iterations run with `bits_consumed` +/// near the gate threshold. The decoder must not lose low stream +/// bits when the shift formula runs close to the threshold; +/// roundtrip correctness over short input is the regression signal. +#[test] +fn burst_gate_lower_boundary_short_skewed_alphabet() { + // 36 bytes, 23 distinct symbols, skewed distribution — + // encoder picks max_num_bits in the 5..=11 band. + let mut data: Vec = Vec::with_capacity(36); + data.extend_from_slice(&[ + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, + ]); + roundtrip_with_max_bits_range(&data, 5..=11); +} + +/// Upper boundary: `bits_consumed + burst_bits == 64`. +/// +/// A long, mid-cardinality alphabet drives many full burst windows. +/// Across thousands of iterations the burst-fits-in-64 guard +/// (`bits_consumed + burst_bits <= 64`) is approached and met +/// exactly. A regression that miscalculated the upper boundary +/// would read past the loaded 8-byte window and either crash under +/// debug bounds checks or desynchronise the stream — either way +/// the roundtrip fails. +#[test] +fn burst_gate_upper_boundary_long_mid_alphabet() { + // 4 KiB with a 97-symbol pseudo-random alphabet (kept under the + // encoder's 128-weight raw-table limit). Broad distribution → + // max_num_bits ≈ 7..9, thousands of burst iterations across all + // four streams. + let mut data: Vec = Vec::with_capacity(4096); + for i in 0..4096u32 { + data.push((i.wrapping_mul(0x9E37_79B1) % 97) as u8); + } + roundtrip_with_max_bits_range(&data, 6..=11); +} + +/// SIMD-fallback → refill → burst re-entry transition. +/// +/// After a `BitReaderReversed::refill` (triggered inside +/// `advance_state_by_bits` on the SIMD path), `bits_consumed` +/// rebases to `[0, 7]`. Until it climbs back to `max_num_bits` the +/// burst gate is closed and the outer loop runs the 4-symbol SIMD +/// fallback; on the next outer-loop iteration after `bits_consumed` +/// grows past `max_num_bits` the burst path must re-enter cleanly. +/// +/// Stream length of 16 KiB / 4 ≈ 4 KiB per stream encoded ⇒ each +/// `BitReaderReversed` window crosses many refill boundaries, +/// guaranteeing the SIMD→refill→burst transition fires repeatedly. +#[test] +fn burst_simd_fallback_refill_reentry_long_streams() { + // 67-symbol modulo distribution (`i % 67`, prime modulus spreads + // the alphabet evenly) → max_num_bits typically 7..8, which gives + // `symbols_per_burst = (63 - max) / max ≈ 6..8`. + let mut data: Vec = Vec::with_capacity(16 * 1024); + for i in 0..16 * 1024u32 { + data.push((i % 67) as u8); + } + roundtrip_with_max_bits_range(&data, 5..=8); +} + +/// Parametric sweep across stream lengths and alphabet shapes. +/// +/// The three burst-gate states above are also hit across this matrix +/// at varying `(bits_consumed, max_num_bits, symbols_per_burst)` +/// configurations; any future tweak to the gate that mishandles a +/// specific `(max_num_bits, post-refill bits_consumed)` combo trips +/// at least one cell here. +#[test] +fn burst_gate_sweep_sizes_and_alphabets() { + let sizes = [ + 16usize, 17, 31, 32, 33, 63, 64, 65, 127, 128, 129, 255, 256, 257, 511, 512, 513, 1023, + 1024, 1025, 4096, + ]; + for &n in &sizes { + // Binary alphabet → max_num_bits == 1, symbols_per_burst large. + let mut bin: Vec = Vec::with_capacity(n); + for i in 0..n { + bin.push((i & 1) as u8); + } + roundtrip_assert(&bin); + + // 16-symbol uniform alphabet → max_num_bits ≈ 4. + let mut sm: Vec = Vec::with_capacity(n); + for i in 0..n { + sm.push((i % 16) as u8); + } + roundtrip_assert(&sm); + + // 97-symbol pseudo-random alphabet (where length permits) → + // max_num_bits ≈ 7..9; kept under the encoder's 128-weight + // raw-table cap so the encoder reliably succeeds. + if n >= 128 { + let mut wide: Vec = Vec::with_capacity(n); + for i in 0..n { + wide.push((i.wrapping_mul(2_654_435_761) % 97) as u8); + } + roundtrip_assert(&wide); + } + } +} + +/// Adversarial regression for the `burst_eligible` safety gate. +/// +/// Builds a valid 4-stream HUF block, then forges a `LiteralsSection` +/// header that claims `regenerated_size = 1` while the encoded +/// streams still contain a full block worth of symbols. The shrunk +/// `regenerated_size` collapses `min_seg_len` below +/// `symbols_per_burst`, the exact precondition `burst_eligible` +/// guards against. Without that gate, the burst inner loop would +/// advance `cursors[i]` past `ends[i]` and panic on the +/// `target[cursors[i]]` write — a DoS surface on malformed input. +/// +/// With the gate, the decoder either: +/// - falls through to the SIMD-fallback path which immediately +/// hits the top-of-loop `cursor_exit_olimit` exit and returns +/// a count-mismatch / bitstream-mismatch error, or +/// - returns an error before the loop ever runs. +/// +/// Either way the test asserts `Err(_)` — the contract is "no +/// panic, return an error". +#[test] +fn burst_gate_malformed_small_regen_returns_error() { + // 256 bytes is well above MIN_LITERALS_FOR_4_STREAMS so the + // encoder will happily emit a 4-stream HUF block. The modulo + // alphabet keeps `max_num_bits` small (≤ 8), maximising + // `symbols_per_burst` so the small forged `regenerated_size` + // sits well below it. + let mut data: Vec = Vec::with_capacity(256); + for i in 0..256u32 { + data.push((i % 67) as u8); + } + let (mut section, source) = build_huf4x_block(&data); + + // Forge: claim only 1 regenerated byte. Streams in `source` + // are still encoded for the full 256-byte input. + section.regenerated_size = 1; + + let mut scratch = HuffmanScratch::new(); + let mut target = Vec::new(); + let result = decode_literals(§ion, &mut scratch, &source, &mut target); + + assert!( + result.is_err(), + "decoder must reject the malformed header instead of panicking; \ + got Ok({})", + result.unwrap_or(0) + ); +} diff --git a/zstd/src/decoding/literals_section_decoder/zerocopy_robustness_tests.rs b/zstd/src/decoding/literals_section_decoder/zerocopy_robustness_tests.rs new file mode 100644 index 000000000..54cb9c370 --- /dev/null +++ b/zstd/src/decoding/literals_section_decoder/zerocopy_robustness_tests.rs @@ -0,0 +1,130 @@ +//! Regression coverage for `decode_literals_zerocopy` on +//! truncated / corrupt payloads: every branch must return a +//! structured error instead of panicking on out-of-bounds +//! slice indexing. Hit each `*[..n]` / `*[0]` index in the +//! function with a payload one byte short of what the header +//! declares. +// +// Tests live in a separate module so the broader `burst_gate_tests` +// module's helpers don't have to depend on truncated-input +// builders. +use super::{LiteralsView, decode_literals_zerocopy}; +use crate::blocks::literals_section::{LiteralsSection, LiteralsSectionType}; +use crate::decoding::scratch::HuffmanScratch; +use alloc::vec::Vec; + +fn raw_section(regen: u32) -> LiteralsSection { + LiteralsSection { + ls_type: LiteralsSectionType::Raw, + regenerated_size: regen, + compressed_size: None, + num_streams: None, + } +} + +fn rle_section(regen: u32) -> LiteralsSection { + LiteralsSection { + ls_type: LiteralsSectionType::RLE, + regenerated_size: regen, + compressed_size: None, + num_streams: None, + } +} + +fn fresh_scratch() -> HuffmanScratch { + HuffmanScratch::new() +} + +#[test] +fn raw_truncated_source_returns_error_no_panic() { + // Header claims 10 raw literal bytes, source carries 3. + // Indexing `source[0..10]` would panic; the fix must turn + // it into a structured DecompressLiteralsError. + let section = raw_section(10); + let source: [u8; 3] = [1, 2, 3]; + let mut target: Vec = Vec::new(); + let mut scratch = fresh_scratch(); + let result = decode_literals_zerocopy(§ion, &mut scratch, &source, &mut target); + assert!( + result.is_err(), + "truncated raw source must error, not panic; got {:?}", + result.map(|_| ()) + ); +} + +#[test] +fn rle_empty_source_returns_error_no_panic() { + // RLE section needs at least one source byte (the fill byte). + // Indexing `source[0]` on an empty slice would panic. + let section = rle_section(10); + let source: [u8; 0] = []; + let mut target: Vec = Vec::new(); + let mut scratch = fresh_scratch(); + let result = decode_literals_zerocopy(§ion, &mut scratch, &source, &mut target); + assert!( + result.is_err(), + "empty RLE source must error, not panic; got {:?}", + result.map(|_| ()) + ); +} + +#[test] +fn compressed_truncated_source_returns_error_no_panic() { + // Header claims compressed_size = 10 but the source carries 3 bytes. + // Slicing `source[0..10]` for a Compressed/Treeless section would + // panic (decoder DoS on truncated input); the fix must turn it into + // a structured DecompressLiteralsError, matching the Raw/RLE paths. + let section = LiteralsSection { + ls_type: LiteralsSectionType::Compressed, + regenerated_size: 5, + compressed_size: Some(10), + num_streams: Some(1), + }; + let source: [u8; 3] = [1, 2, 3]; + let mut target: Vec = Vec::new(); + let mut scratch = fresh_scratch(); + let result = decode_literals_zerocopy(§ion, &mut scratch, &source, &mut target); + // Pin the EXACT contract: a truncated Compressed section must report + // MissingBytesForLiterals with the precise got/needed, not just "some + // error" (a weaker is_err() would also pass on MissingNumStreams / + // UninitializedHuffmanTable, missing the real regression). + assert!( + matches!( + &result, + Err( + crate::decoding::errors::DecompressLiteralsError::MissingBytesForLiterals { + got: 3, + needed: 10, + } + ) + ), + "truncated compressed source must report MissingBytesForLiterals, got {:?}", + result.map(|_| ()) + ); +} + +#[test] +fn rle_view_excludes_pre_existing_target_bytes() { + // Even if the caller forgot to clear `target`, the returned + // LiteralsView::data must point only at the bytes this call + // produced. The API hardening (`&target[base..]`) is what + // makes this hold. + let mut target: Vec = Vec::from([0xAA, 0xBB, 0xCC]); + let section = rle_section(4); + let source: [u8; 1] = [0x42]; + let mut scratch = fresh_scratch(); + let view = decode_literals_zerocopy(§ion, &mut scratch, &source, &mut target) + .expect("RLE with valid source must succeed"); + assert_eq!(view.data.len(), 4, "view length must match regen_size"); + assert!( + view.data.iter().all(|&b| b == 0x42), + "view must contain only the newly-RLE-expanded bytes, got {:?}", + view.data + ); + // Silence unused-warning if the compiler ever strips + // LiteralsView fields — read bytes_used too. + let _ = LiteralsView { + data: view.data, + bytes_used: view.bytes_used, + }; +} diff --git a/zstd/src/decoding/mod.rs b/zstd/src/decoding/mod.rs index 9e900753a..ff0ea0f21 100644 --- a/zstd/src/decoding/mod.rs +++ b/zstd/src/decoding/mod.rs @@ -420,205 +420,4 @@ pub(crate) mod user_slice_buf; pub(crate) use self::simd_copy::copy_bytes_overshooting_for_bench; #[cfg(test)] -mod frame_inspection_tests { - use super::{ - FrameContentSize, FrameSizeError, find_frame_compressed_size, frame_decompressed_bound, - frame_header_size, read_frame_content_size, read_frame_header_info, - }; - use crate::encoding::{CompressionLevel, compress_slice_to_vec}; - use alloc::vec; - use alloc::vec::Vec; - - fn frame(content: &[u8]) -> Vec { - compress_slice_to_vec(content, CompressionLevel::Default) - } - - /// A hand-built single raw-block frame that omits `Frame_Content_Size` - /// (descriptor `0x00`: FCS_Flag=0, Single_Segment=0) so it carries a - /// Window_Descriptor instead — the only way to exercise the window-size - /// fallback in [`frame_decompressed_bound`] / [`read_frame_header_info`], - /// which the encoder (always declaring FCS) never produces. - fn no_fcs_frame() -> Vec { - vec![ - 0x28, 0xB5, 0x2F, 0xFD, // magic - 0x00, // frame header descriptor: no FCS, multi-segment, no checksum - 0x00, // window descriptor -> windowLog 10 -> 1024 bytes - 0x19, 0x00, 0x00, // block header: last, raw, size 3 - 0xAA, 0xBB, 0xCC, // raw payload - ] - } - - /// As [`no_fcs_frame`] but with the Content_Checksum_flag (descriptor bit 2) - /// set and a 4-byte trailer appended, to cover the checksum-trailer branch. - fn no_fcs_checksum_frame() -> Vec { - vec![ - 0x28, 0xB5, 0x2F, 0xFD, 0x04, // descriptor: checksum flag set - 0x00, 0x19, 0x00, 0x00, 0xAA, 0xBB, 0xCC, // window + block + payload - 0xDE, 0xAD, 0xBE, 0xEF, // content checksum trailer - ] - } - - /// A skippable frame: magic `0x184D2A50`, 4-byte length, then `length` bytes. - fn skippable_frame(payload: &[u8]) -> Vec { - let mut f = vec![0x50, 0x2A, 0x4D, 0x18]; - f.extend_from_slice(&(payload.len() as u32).to_le_bytes()); - f.extend_from_slice(payload); - f - } - - #[test] - fn read_frame_content_size_reports_declared_size() { - let f = frame(&[42u8; 100]); - assert_eq!( - read_frame_content_size(&f).unwrap(), - FrameContentSize::Known(100) - ); - } - - #[test] - fn read_frame_content_size_reports_unknown_without_fcs() { - assert_eq!( - read_frame_content_size(&no_fcs_frame()).unwrap(), - FrameContentSize::Unknown - ); - } - - #[test] - fn read_frame_content_size_errors_on_garbage() { - assert!(read_frame_content_size(&[0xAB; 16]).is_err()); - } - - #[test] - fn find_frame_compressed_size_spans_one_frame_then_the_next() { - let first = frame(&[5u8; 256]); - assert_eq!(find_frame_compressed_size(&first).unwrap(), first.len()); - - let mut two = first.clone(); - two.extend_from_slice(&frame(&[9u8; 50])); - // Still reports only the first frame so a caller can step forward. - assert_eq!(find_frame_compressed_size(&two).unwrap(), first.len()); - } - - #[test] - fn find_frame_compressed_size_measures_skippable_frame() { - let skip = skippable_frame(&[1, 2, 3, 4]); - assert_eq!(find_frame_compressed_size(&skip).unwrap(), skip.len()); - } - - #[test] - fn find_frame_compressed_size_rejects_truncation() { - let f = frame(&[7u8; 512]); - // Drop the trailing block bytes mid-frame. - let err = find_frame_compressed_size(&f[..f.len() - 4]).unwrap_err(); - assert!(matches!(err, FrameSizeError::Truncated)); - } - - #[test] - fn frame_header_size_matches_first_block_offset() { - let f = frame(&[3u8; 2048]); - let hdr = frame_header_size(&f).unwrap(); - assert!((5..=18).contains(&hdr)); - assert!(frame_header_size(&[0u8; 2]).is_err()); - } - - #[test] - fn read_frame_header_info_fills_declared_fields() { - let f = frame(&[7u8; 512]); - let info = read_frame_header_info(&f, false).unwrap(); - assert_eq!(info.content_size, FrameContentSize::Known(512)); - assert!(info.window_size >= 512); - assert_eq!(info.dictionary_id, None); - } - - #[test] - fn read_frame_header_info_derives_window_without_fcs() { - let info = read_frame_header_info(&no_fcs_frame(), false).unwrap(); - assert_eq!(info.content_size, FrameContentSize::Unknown); - assert_eq!(info.window_size, 1024); - } - - #[test] - fn frame_decompressed_bound_returns_declared_size() { - let f = frame(&[4u8; 4096]); - assert_eq!(frame_decompressed_bound(&f).unwrap(), 4096); - } - - #[test] - fn frame_decompressed_bound_uses_block_bound_without_fcs() { - // No declared FCS -> block_count(1) * block_size_max(min(1024,128K)). - assert_eq!(frame_decompressed_bound(&no_fcs_frame()).unwrap(), 1024); - } - - #[test] - fn frame_decompressed_bound_accepts_present_checksum_trailer() { - assert_eq!( - frame_decompressed_bound(&no_fcs_checksum_frame()).unwrap(), - 1024 - ); - } - - #[test] - fn frame_decompressed_bound_rejects_missing_checksum_trailer() { - let mut f = no_fcs_checksum_frame(); - f.truncate(f.len() - 4); // drop the 4-byte trailer the descriptor promises - assert!(matches!( - frame_decompressed_bound(&f).unwrap_err(), - FrameSizeError::Truncated - )); - } - - /// A block header may declare a `Block_Size` larger than the frame's - /// `Block_Maximum_Size` (`min(window, 128 KiB)`). RFC 8878 forbids this; - /// accepting it lets a corrupt frame pass the size query and makes the - /// no-FCS decompressed bound under-count (the raw block regenerates more - /// bytes than `block_count * block_size_max`). Both helpers must reject it. - #[test] - fn size_helpers_reject_oversized_block_header() { - // Window 1024 (WD 0x00) -> Block_Maximum_Size = 1024. Declare a raw - // block of Block_Size 2000 with all 2000 payload bytes present, so the - // failure is the oversized declaration, not truncation. - let block_size = 2000usize; - let raw = ((block_size as u32) << 3) | 1; // last_block flag, Raw type (00) - let mut f = vec![ - 0x28, - 0xB5, - 0x2F, - 0xFD, // magic - 0x00, // descriptor: no FCS, multi-segment, no checksum - 0x00, // window descriptor -> 1024 bytes - (raw & 0xFF) as u8, - ((raw >> 8) & 0xFF) as u8, - ((raw >> 16) & 0xFF) as u8, - ]; - f.resize(f.len() + block_size, 0xAB); - - assert!(matches!( - find_frame_compressed_size(&f).unwrap_err(), - FrameSizeError::OversizedBlock - )); - assert!(matches!( - frame_decompressed_bound(&f).unwrap_err(), - FrameSizeError::OversizedBlock - )); - } - - #[test] - fn frame_decompressed_bound_handles_skippable_frame() { - assert_eq!( - frame_decompressed_bound(&skippable_frame(&[0u8; 8])).unwrap(), - 0 - ); - // A skippable frame whose advertised payload is absent is truncation. - let mut short = skippable_frame(&[0u8; 8]); - short.truncate(short.len() - 2); - assert!(matches!( - frame_decompressed_bound(&short).unwrap_err(), - FrameSizeError::Truncated - )); - } - - #[test] - fn frame_decompressed_bound_errors_on_garbage_header() { - assert!(frame_decompressed_bound(&[0xAB; 16]).is_err()); - } -} +mod frame_inspection_tests; diff --git a/zstd/src/decoding/ringbuffer.rs b/zstd/src/decoding/ringbuffer.rs index bafc97899..7aaabdee1 100644 --- a/zstd/src/decoding/ringbuffer.rs +++ b/zstd/src/decoding/ringbuffer.rs @@ -1536,562 +1536,4 @@ unsafe fn copy_with_nobranch_check( } #[cfg(test)] -mod tests { - use alloc::vec; - - use super::{RingBuffer, copy_with_checks, copy_with_nobranch_check}; - use crate::decoding::simd_copy; - - fn assert_buffers_equal(expected: &RingBuffer, actual: &RingBuffer) { - assert_eq!(expected.len(), actual.len()); - assert_eq!(expected.as_slices(), actual.as_slices()); - assert_eq!(expected.head, actual.head); - assert_eq!(expected.tail, actual.tail); - assert_eq!(expected.cap, actual.cap); - } - - fn assert_branchless_matches_checked( - mut checked: RingBuffer, - mut branchless: RingBuffer, - start: usize, - len: usize, - ) { - assert!(checked.free() >= len); - assert!(branchless.free() >= len); - - unsafe { - checked.extend_from_within_unchecked(start, len); - branchless.extend_from_within_unchecked_branchless(start, len); - } - - assert_buffers_equal(&checked, &branchless); - } - - #[test] - fn inline_exec_ok_respects_block_output_ceiling() { - // The inline sequence-exec path bypasses `try_reserve`, so it must - // itself honour the per-block output ceiling (`max_capacity`, armed - // by `set_block_output_ceiling(MAX_BLOCK_SIZE)`). Otherwise a reused - // large-window ring with physical slack could emit past the ceiling - // — weakening the streaming-path decompression-bomb guard. - use super::super::buffer_backend::BufferBackend; - let mut rb = RingBuffer::new(); - rb.reserve(64 * 1024); // plenty of physical slack - rb.extend(&[0u8; 1000]); // head = 0, tail = 1000 - // Per-block ceiling with only 100 bytes of output budget remaining. - rb.set_max_capacity(1000 + 100); - // lit+match = 500 exceeds the 100-byte budget but fits physically - // (1531 < cap): the inline gate must reject it. - assert!( - !rb.inline_exec_ok(500, 0, 1), - "inline_exec_ok must reject a write past the per-block output ceiling" - ); - // A write within the budget stays eligible for the inline path. - assert!( - rb.inline_exec_ok(50, 0, 1), - "inline_exec_ok must allow a write within the per-block ceiling" - ); - } - - #[test] - fn inline_exec_ok_admits_contiguous_wrapped_sequence() { - // After the ring wraps (`head > tail`), the inline path stays - // eligible for a sequence whose linear write fits in the free gap - // before `head` AND whose match source is the contiguous lower live - // segment (`offset <= tail + lit`). A far-back match (source across the - // wrap) or a write that would reach `head` must still be vetoed. - use super::super::buffer_backend::BufferBackend; - let mut rb = RingBuffer::new(); - rb.reserve(4096); - let cap = rb.cap; - // Force a wrapped layout: head well ahead of a small tail. - rb.head = cap - 64; - rb.tail = 32; - // Free gap is [32, cap-64): write 16 lit + 16 match + 31 overshoot = 63 - // ends at 95, far below head, and offset 8 <= tail+lit = 48 -> eligible. - assert!( - rb.inline_exec_ok(16, 16, 8), - "wrapped ring with contiguous write + in-segment source must stay inline-eligible" - ); - // Match source crosses the wrap (offset 100 > tail+lit = 48) -> veto. - assert!( - !rb.inline_exec_ok(16, 16, 100), - "wrapped ring with match source across the wrap must veto the inline path" - ); - // Write + overshoot would reach `head` (huge match) -> veto. - let huge_match = cap; // tail + lit + huge_match overflows past head - assert!( - !rb.inline_exec_ok(16, huge_match, 8), - "wrapped ring whose write would reach the upper live segment must veto" - ); - } - - #[test] - fn smoke() { - let mut rb = RingBuffer::new(); - - rb.reserve(15); - assert_eq!(17, rb.cap); - - rb.extend(b"0123456789"); - assert_eq!(rb.len(), 10); - assert_eq!(rb.as_slices().0, b"0123456789"); - assert_eq!(rb.as_slices().1, b""); - - rb.drop_first_n(5); - assert_eq!(rb.len(), 5); - assert_eq!(rb.as_slices().0, b"56789"); - assert_eq!(rb.as_slices().1, b""); - - rb.extend_from_within(2, 3); - assert_eq!(rb.len(), 8); - assert_eq!(rb.as_slices().0, b"56789789"); - assert_eq!(rb.as_slices().1, b""); - - rb.extend_from_within(0, 3); - assert_eq!(rb.len(), 11); - assert_eq!(rb.as_slices().0, b"56789789567"); - assert_eq!(rb.as_slices().1, b""); - - rb.extend_from_within(0, 2); - assert_eq!(rb.len(), 13); - assert_eq!(rb.as_slices().0, b"567897895675"); - assert_eq!(rb.as_slices().1, b"6"); - - rb.drop_first_n(11); - assert_eq!(rb.len(), 2); - assert_eq!(rb.as_slices().0, b"5"); - assert_eq!(rb.as_slices().1, b"6"); - - rb.extend(b"0123456789"); - assert_eq!(rb.len(), 12); - assert_eq!(rb.as_slices().0, b"5"); - assert_eq!(rb.as_slices().1, b"60123456789"); - - rb.drop_first_n(11); - assert_eq!(rb.len(), 1); - assert_eq!(rb.as_slices().0, b"9"); - assert_eq!(rb.as_slices().1, b""); - - rb.extend(b"0123456789"); - assert_eq!(rb.len(), 11); - assert_eq!(rb.as_slices().0, b"9012345"); - assert_eq!(rb.as_slices().1, b"6789"); - } - - #[test] - fn edge_cases() { - // Fill exactly, then empty then fill again - let mut rb = RingBuffer::new(); - rb.reserve(16); - assert_eq!(17, rb.cap); - rb.extend(b"0123456789012345"); - assert_eq!(17, rb.cap); - assert_eq!(16, rb.len()); - assert_eq!(0, rb.free()); - rb.drop_first_n(16); - assert_eq!(0, rb.len()); - assert_eq!(16, rb.free()); - rb.extend(b"0123456789012345"); - assert_eq!(16, rb.len()); - assert_eq!(0, rb.free()); - assert_eq!(17, rb.cap); - assert_eq!(1, rb.as_slices().0.len()); - assert_eq!(15, rb.as_slices().1.len()); - - rb.clear(); - - // data in both slices and then reserve - rb.extend(b"0123456789012345"); - rb.drop_first_n(8); - rb.extend(b"67890123"); - assert_eq!(16, rb.len()); - assert_eq!(0, rb.free()); - assert_eq!(17, rb.cap); - assert_eq!(9, rb.as_slices().0.len()); - assert_eq!(7, rb.as_slices().1.len()); - rb.reserve(1); - assert_eq!(16, rb.len()); - assert_eq!(16, rb.free()); - assert_eq!(33, rb.cap); - assert_eq!(16, rb.as_slices().0.len()); - assert_eq!(0, rb.as_slices().1.len()); - - rb.clear(); - - // fill exactly, then extend from within - rb.extend(b"0123456789012345"); - rb.extend_from_within(0, 16); - assert_eq!(32, rb.len()); - assert_eq!(0, rb.free()); - assert_eq!(33, rb.cap); - assert_eq!(32, rb.as_slices().0.len()); - assert_eq!(0, rb.as_slices().1.len()); - - // extend from within cases - let mut rb = RingBuffer::new(); - rb.reserve(8); - rb.extend(b"01234567"); - rb.drop_first_n(5); - rb.extend_from_within(0, 3); - assert_eq!(4, rb.as_slices().0.len()); - assert_eq!(2, rb.as_slices().1.len()); - - rb.drop_first_n(2); - assert_eq!(2, rb.as_slices().0.len()); - assert_eq!(2, rb.as_slices().1.len()); - rb.extend_from_within(0, 4); - assert_eq!(2, rb.as_slices().0.len()); - assert_eq!(6, rb.as_slices().1.len()); - - rb.drop_first_n(2); - assert_eq!(6, rb.as_slices().0.len()); - assert_eq!(0, rb.as_slices().1.len()); - rb.drop_first_n(2); - assert_eq!(4, rb.as_slices().0.len()); - assert_eq!(0, rb.as_slices().1.len()); - rb.extend_from_within(0, 4); - assert_eq!(7, rb.as_slices().0.len()); - assert_eq!(1, rb.as_slices().1.len()); - - let mut rb = RingBuffer::new(); - rb.reserve(8); - rb.extend(b"11111111"); - rb.drop_first_n(7); - rb.extend(b"111"); - assert_eq!(2, rb.as_slices().0.len()); - assert_eq!(2, rb.as_slices().1.len()); - rb.extend_from_within(0, 4); - assert_eq!(b"11", rb.as_slices().0); - assert_eq!(b"111111", rb.as_slices().1); - } - - #[test] - fn extend_from_within_branchless_matches_checked_across_layouts() { - let contiguous = || { - let mut rb = RingBuffer::new(); - rb.reserve(16); - rb.extend(b"0123456789"); - rb - }; - assert_branchless_matches_checked(contiguous(), contiguous(), 2, 5); - - let wrapped_write = || { - let mut rb = RingBuffer::new(); - rb.reserve(16); - rb.extend(b"0123456789ABC"); - rb.drop_first_n(2); - rb - }; - assert_branchless_matches_checked(wrapped_write(), wrapped_write(), 1, 5); - - let wrapped_data = || { - let mut rb = RingBuffer::new(); - rb.reserve(32); - rb.extend(b"0123456789abcdefghijklmn"); - rb.drop_first_n(18); - rb.extend(b"wxyz012345"); - rb - }; - assert_branchless_matches_checked(wrapped_data(), wrapped_data(), 8, 2); - assert_branchless_matches_checked(wrapped_data(), wrapped_data(), 2, 8); - } - - #[test] - fn copy_with_nobranch_check_matches_checked_for_all_valid_case_masks() { - let cases = [ - (0, 0, 0, 0), - (1, 0, 0, 0), - (0, 1, 0, 0), - (0, 0, 1, 0), - (0, 0, 0, 1), - (1, 1, 0, 0), - (1, 0, 1, 0), - (1, 0, 0, 1), - (0, 1, 0, 1), - (0, 0, 1, 1), - (1, 1, 0, 1), - (1, 0, 1, 1), - ]; - - for (m1_in_f1, m2_in_f1, m1_in_f2, m2_in_f2) in cases { - let m1 = [11_u8, 12, 13, 14]; - let m2 = [21_u8, 22, 23, 24]; - let mut expected = [0_u8; 8]; - let mut actual = [0_u8; 8]; - - unsafe { - copy_with_checks( - m1.as_ptr(), - m2.as_ptr(), - expected.as_mut_ptr(), - expected.as_mut_ptr().add(4), - m1_in_f1, - m2_in_f1, - m1_in_f2, - m2_in_f2, - ); - copy_with_nobranch_check( - m1.as_ptr(), - m2.as_ptr(), - actual.as_mut_ptr(), - actual.as_mut_ptr().add(4), - m1_in_f1, - m2_in_f1, - m1_in_f2, - m2_in_f2, - ); - } - - assert_eq!( - expected, actual, - "case=({}, {}, {}, {})", - m1_in_f1, m2_in_f1, m1_in_f2, m2_in_f2 - ); - } - } - - #[test] - fn copy_bytes_overshooting_preserves_prefix_for_runtime_chunk_lengths() { - // Validate correctness for lengths derived from the active runtime chunk: - // - single chunk (`chunk`) - // - multi chunk (`2 * chunk`) - // - fallback shape (`chunk + 1`) - // This checks copy semantics across runtime-selected strategies. - let chunk = simd_copy::active_chunk_size_for_tests(); - let single_len = chunk; - let multi_len = chunk * 2; - let fallback_len = chunk + 1; - let overshoot_cap = chunk * 2; - let cap = multi_len + chunk; - - let src_single = vec![1_u8; cap]; - let mut dst_single = vec![0_u8; cap]; - unsafe { - simd_copy::copy_bytes_overshooting( - (src_single.as_ptr(), single_len), - (dst_single.as_mut_ptr(), single_len), - single_len, - ); - } - assert_eq!(&dst_single[..single_len], &src_single[..single_len]); - - let src_multi = vec![2_u8; cap]; - let mut dst_multi = vec![0_u8; cap]; - unsafe { - simd_copy::copy_bytes_overshooting( - (src_multi.as_ptr(), multi_len), - (dst_multi.as_mut_ptr(), multi_len), - multi_len, - ); - } - assert_eq!(&dst_multi[..multi_len], &src_multi[..multi_len]); - - let src_fallback = vec![3_u8; cap]; - let mut dst_fallback = vec![0_u8; cap]; - unsafe { - simd_copy::copy_bytes_overshooting( - (src_fallback.as_ptr(), fallback_len), - (dst_fallback.as_mut_ptr(), fallback_len), - fallback_len, - ); - } - assert_eq!(&dst_fallback[..fallback_len], &src_fallback[..fallback_len]); - - let src_overshoot = vec![4_u8; cap + 1]; - let mut dst_overshoot = vec![0_u8; cap + 1]; - unsafe { - simd_copy::copy_bytes_overshooting( - (src_overshoot.as_ptr().add(1), overshoot_cap), - (dst_overshoot.as_mut_ptr().add(1), overshoot_cap), - fallback_len, - ); - } - assert_eq!( - &dst_overshoot[1..1 + fallback_len], - &src_overshoot[1..1 + fallback_len] - ); - } - - /// Helper: drive the ringbuffer into a wrapped layout where the two - /// free-slice halves straddle the physical end of the backing buffer. - /// Returns a buffer whose `len() == fill_len` of `pre_byte` data and - /// whose free region wraps. - fn build_wrapped_buffer(cap: usize, fill_len: usize, pre_byte: u8) -> RingBuffer { - let mut rb = RingBuffer::new(); - rb.reserve(cap); - let actual_cap = rb.cap; - // Push to near-end of the physical buffer. - let pre_len = actual_cap - 2; - let prefix = alloc::vec![pre_byte; pre_len]; - rb.extend(&prefix); - // Drop those bytes so head advances past them; tail now sits near - // the end of `cap`, head is in the middle. Subsequent inserts will - // wrap across the physical end. - rb.drop_first_n(pre_len - fill_len); - assert_eq!(rb.len(), fill_len); - assert!(rb.tail > rb.head, "tail should still trail tape end"); - rb - } - - #[test] - fn extend_and_fill_contiguous_layout() { - let mut rb = RingBuffer::new(); - rb.extend_and_fill(0xAB, 7); - assert_eq!(rb.len(), 7); - let (s1, s2) = rb.as_slices(); - let mut combined = alloc::vec::Vec::with_capacity(7); - combined.extend_from_slice(s1); - combined.extend_from_slice(s2); - assert_eq!(combined, alloc::vec![0xAB; 7]); - } - - /// Construct a RingBuffer in the wrapped state where `tail < head`. - /// Data occupies `[head, cap)` followed by `[0, tail)`, leaving the - /// free region as `[tail, head)` — i.e. the **first** free slice - /// returned by `free_slice_parts` is the inner gap, NOT a span that - /// ends at `cap`. - fn build_tail_before_head(cap_hint: usize, head_pos: usize, tail_pos: usize) -> RingBuffer { - assert!(tail_pos < head_pos); - let mut rb = RingBuffer::new(); - rb.reserve(cap_hint); - let actual_cap = rb.cap; - // Fill almost the whole buffer so we can carve out tail < head. - let fill_len = actual_cap - 2; - let prefix = alloc::vec![0xCD; fill_len]; - rb.extend(&prefix); - // Drop bytes so `head` lands at the target. Tail stays at fill_len. - rb.drop_first_n(head_pos); - // Now extend just enough to wrap tail past `cap` and land it at - // `tail_pos`. From the current state (head=head_pos, tail=fill_len), - // we need `tail_pos = (fill_len + extra) % cap`, so - // `extra = (tail_pos + cap - fill_len) % cap`. - let extra = (tail_pos + actual_cap - fill_len) % actual_cap; - rb.extend(&alloc::vec![0xCD; extra]); - assert_eq!(rb.head, head_pos); - assert_eq!(rb.tail, tail_pos); - assert!( - rb.tail < rb.head, - "expected wrapped layout: tail={} head={}", - rb.tail, - rb.head - ); - rb - } - - #[test] - fn extend_wrapped_layout_preserves_bytes_past_head() { - // Regression test for the WILDCOPY_OVERLENGTH inflation bug. - // When `tail < head` the first free slice returned by - // `free_slice_parts` is the inner `[tail, head)` gap — it does NOT - // end at `cap`. A previous version of `RingBuffer::extend` always - // added WILDCOPY_OVERLENGTH to the destination capacity passed to - // `simd_copy::copy_bytes_overshooting`, which on this layout would - // allow wildcopy overshoot writes to clobber bytes at `[head, - // head+16)` — still-readable data the caller has not consumed yet. - // The current code gates the inflation on `tail >= head`; this - // test asserts that bytes at `head..head+16` survive an extend() - // that fills the inner gap nearly to capacity. - let mut rb = build_tail_before_head(128, 80, 10); - let head_before = rb.head; - let cap = rb.cap; - // Sample a window of bytes immediately past `head` and confirm - // they are the prefill (0xCD); these are the bytes any erroneous - // wildcopy overshoot would corrupt. - let mut sentinel = [0u8; 16]; - unsafe { - for (i, slot) in sentinel.iter_mut().enumerate() { - *slot = rb.buf.as_ptr().add((head_before + i) % cap).read(); - } - } - assert!(sentinel.iter().all(|&b| b == 0xCD), "pre-state sentinel"); - - // Fill the inner free region with a recognisable pattern, leaving - // exactly the 1-byte sentinel gap the RingBuffer always reserves. - let free_before = rb.free(); - let payload = alloc::vec![0x42; free_before]; - rb.extend(&payload); - - // The bytes at [head, head+16) must still be the original 0xCD - // prefill — any overshoot would have written 0x42 over them. - unsafe { - for i in 0..16usize { - let actual = rb.buf.as_ptr().add((head_before + i) % cap).read(); - assert_eq!( - actual, - 0xCD, - "byte at head+{i} (raw idx {}) was clobbered: got {:#04x}", - (head_before + i) % cap, - actual - ); - } - } - } - - #[test] - fn extend_and_fill_wrapped_layout() { - // Pre-fill so the free region straddles the wrap boundary, then - // verify both halves are written with the fill byte. - let mut rb = build_wrapped_buffer(16, 2, 0x11); - let extra = rb.cap - 2; // fills exactly to capacity, forcing a wrap - rb.extend_and_fill(0x22, extra); - assert_eq!(rb.len(), 2 + extra); - let (s1, s2) = rb.as_slices(); - let mut combined = alloc::vec::Vec::with_capacity(rb.len()); - combined.extend_from_slice(s1); - combined.extend_from_slice(s2); - let mut expected = alloc::vec![0x11; 2]; - expected.extend(alloc::vec![0x22; extra]); - assert_eq!(combined, expected); - } - - #[test] - fn extend_from_reader_contiguous_layout() { - let mut rb = RingBuffer::new(); - let src: [u8; 6] = [1, 2, 3, 4, 5, 6]; - rb.extend_from_reader(&src[..], 6).unwrap(); - assert_eq!(rb.len(), 6); - let (s1, s2) = rb.as_slices(); - let mut combined = alloc::vec::Vec::with_capacity(6); - combined.extend_from_slice(s1); - combined.extend_from_slice(s2); - assert_eq!(combined, src); - } - - #[test] - fn extend_from_reader_wrapped_layout() { - let mut rb = build_wrapped_buffer(16, 3, 0xAA); - let extra = rb.cap - 3; - let src: alloc::vec::Vec = (0..extra as u8).collect(); - rb.extend_from_reader(src.as_slice(), extra).unwrap(); - assert_eq!(rb.len(), 3 + extra); - let (s1, s2) = rb.as_slices(); - let mut combined = alloc::vec::Vec::with_capacity(rb.len()); - combined.extend_from_slice(s1); - combined.extend_from_slice(s2); - let mut expected = alloc::vec![0xAA; 3]; - expected.extend_from_slice(&src); - assert_eq!(combined, expected); - } - - #[test] - fn extend_from_reader_eof_leaves_state_unchanged() { - let mut rb = RingBuffer::new(); - rb.extend(b"prefix"); - let snapshot_len = rb.len(); - let snapshot_slices: (alloc::vec::Vec, alloc::vec::Vec) = { - let (a, b) = rb.as_slices(); - (a.to_vec(), b.to_vec()) - }; - - // Reader yields only 2 bytes but we ask for 10 → `read_exact` fails - // on the second chunk, and `tail` must not advance. - let short: [u8; 2] = [0xCC, 0xDD]; - let err = rb.extend_from_reader(&short[..], 10); - assert!(err.is_err(), "short reader must propagate IO error"); - assert_eq!(rb.len(), snapshot_len, "len() must be unchanged on error"); - let (a, b) = rb.as_slices(); - assert_eq!(a, snapshot_slices.0.as_slice()); - assert_eq!(b, snapshot_slices.1.as_slice()); - } -} +mod tests; diff --git a/zstd/src/decoding/ringbuffer/tests.rs b/zstd/src/decoding/ringbuffer/tests.rs new file mode 100644 index 000000000..722fb6148 --- /dev/null +++ b/zstd/src/decoding/ringbuffer/tests.rs @@ -0,0 +1,557 @@ +use alloc::vec; + +use super::{RingBuffer, copy_with_checks, copy_with_nobranch_check}; +use crate::decoding::simd_copy; + +fn assert_buffers_equal(expected: &RingBuffer, actual: &RingBuffer) { + assert_eq!(expected.len(), actual.len()); + assert_eq!(expected.as_slices(), actual.as_slices()); + assert_eq!(expected.head, actual.head); + assert_eq!(expected.tail, actual.tail); + assert_eq!(expected.cap, actual.cap); +} + +fn assert_branchless_matches_checked( + mut checked: RingBuffer, + mut branchless: RingBuffer, + start: usize, + len: usize, +) { + assert!(checked.free() >= len); + assert!(branchless.free() >= len); + + unsafe { + checked.extend_from_within_unchecked(start, len); + branchless.extend_from_within_unchecked_branchless(start, len); + } + + assert_buffers_equal(&checked, &branchless); +} + +#[test] +fn inline_exec_ok_respects_block_output_ceiling() { + // The inline sequence-exec path bypasses `try_reserve`, so it must + // itself honour the per-block output ceiling (`max_capacity`, armed + // by `set_block_output_ceiling(MAX_BLOCK_SIZE)`). Otherwise a reused + // large-window ring with physical slack could emit past the ceiling + // — weakening the streaming-path decompression-bomb guard. + use super::super::buffer_backend::BufferBackend; + let mut rb = RingBuffer::new(); + rb.reserve(64 * 1024); // plenty of physical slack + rb.extend(&[0u8; 1000]); // head = 0, tail = 1000 + // Per-block ceiling with only 100 bytes of output budget remaining. + rb.set_max_capacity(1000 + 100); + // lit+match = 500 exceeds the 100-byte budget but fits physically + // (1531 < cap): the inline gate must reject it. + assert!( + !rb.inline_exec_ok(500, 0, 1), + "inline_exec_ok must reject a write past the per-block output ceiling" + ); + // A write within the budget stays eligible for the inline path. + assert!( + rb.inline_exec_ok(50, 0, 1), + "inline_exec_ok must allow a write within the per-block ceiling" + ); +} + +#[test] +fn inline_exec_ok_admits_contiguous_wrapped_sequence() { + // After the ring wraps (`head > tail`), the inline path stays + // eligible for a sequence whose linear write fits in the free gap + // before `head` AND whose match source is the contiguous lower live + // segment (`offset <= tail + lit`). A far-back match (source across the + // wrap) or a write that would reach `head` must still be vetoed. + use super::super::buffer_backend::BufferBackend; + let mut rb = RingBuffer::new(); + rb.reserve(4096); + let cap = rb.cap; + // Force a wrapped layout: head well ahead of a small tail. + rb.head = cap - 64; + rb.tail = 32; + // Free gap is [32, cap-64): write 16 lit + 16 match + 31 overshoot = 63 + // ends at 95, far below head, and offset 8 <= tail+lit = 48 -> eligible. + assert!( + rb.inline_exec_ok(16, 16, 8), + "wrapped ring with contiguous write + in-segment source must stay inline-eligible" + ); + // Match source crosses the wrap (offset 100 > tail+lit = 48) -> veto. + assert!( + !rb.inline_exec_ok(16, 16, 100), + "wrapped ring with match source across the wrap must veto the inline path" + ); + // Write + overshoot would reach `head` (huge match) -> veto. + let huge_match = cap; // tail + lit + huge_match overflows past head + assert!( + !rb.inline_exec_ok(16, huge_match, 8), + "wrapped ring whose write would reach the upper live segment must veto" + ); +} + +#[test] +fn smoke() { + let mut rb = RingBuffer::new(); + + rb.reserve(15); + assert_eq!(17, rb.cap); + + rb.extend(b"0123456789"); + assert_eq!(rb.len(), 10); + assert_eq!(rb.as_slices().0, b"0123456789"); + assert_eq!(rb.as_slices().1, b""); + + rb.drop_first_n(5); + assert_eq!(rb.len(), 5); + assert_eq!(rb.as_slices().0, b"56789"); + assert_eq!(rb.as_slices().1, b""); + + rb.extend_from_within(2, 3); + assert_eq!(rb.len(), 8); + assert_eq!(rb.as_slices().0, b"56789789"); + assert_eq!(rb.as_slices().1, b""); + + rb.extend_from_within(0, 3); + assert_eq!(rb.len(), 11); + assert_eq!(rb.as_slices().0, b"56789789567"); + assert_eq!(rb.as_slices().1, b""); + + rb.extend_from_within(0, 2); + assert_eq!(rb.len(), 13); + assert_eq!(rb.as_slices().0, b"567897895675"); + assert_eq!(rb.as_slices().1, b"6"); + + rb.drop_first_n(11); + assert_eq!(rb.len(), 2); + assert_eq!(rb.as_slices().0, b"5"); + assert_eq!(rb.as_slices().1, b"6"); + + rb.extend(b"0123456789"); + assert_eq!(rb.len(), 12); + assert_eq!(rb.as_slices().0, b"5"); + assert_eq!(rb.as_slices().1, b"60123456789"); + + rb.drop_first_n(11); + assert_eq!(rb.len(), 1); + assert_eq!(rb.as_slices().0, b"9"); + assert_eq!(rb.as_slices().1, b""); + + rb.extend(b"0123456789"); + assert_eq!(rb.len(), 11); + assert_eq!(rb.as_slices().0, b"9012345"); + assert_eq!(rb.as_slices().1, b"6789"); +} + +#[test] +fn edge_cases() { + // Fill exactly, then empty then fill again + let mut rb = RingBuffer::new(); + rb.reserve(16); + assert_eq!(17, rb.cap); + rb.extend(b"0123456789012345"); + assert_eq!(17, rb.cap); + assert_eq!(16, rb.len()); + assert_eq!(0, rb.free()); + rb.drop_first_n(16); + assert_eq!(0, rb.len()); + assert_eq!(16, rb.free()); + rb.extend(b"0123456789012345"); + assert_eq!(16, rb.len()); + assert_eq!(0, rb.free()); + assert_eq!(17, rb.cap); + assert_eq!(1, rb.as_slices().0.len()); + assert_eq!(15, rb.as_slices().1.len()); + + rb.clear(); + + // data in both slices and then reserve + rb.extend(b"0123456789012345"); + rb.drop_first_n(8); + rb.extend(b"67890123"); + assert_eq!(16, rb.len()); + assert_eq!(0, rb.free()); + assert_eq!(17, rb.cap); + assert_eq!(9, rb.as_slices().0.len()); + assert_eq!(7, rb.as_slices().1.len()); + rb.reserve(1); + assert_eq!(16, rb.len()); + assert_eq!(16, rb.free()); + assert_eq!(33, rb.cap); + assert_eq!(16, rb.as_slices().0.len()); + assert_eq!(0, rb.as_slices().1.len()); + + rb.clear(); + + // fill exactly, then extend from within + rb.extend(b"0123456789012345"); + rb.extend_from_within(0, 16); + assert_eq!(32, rb.len()); + assert_eq!(0, rb.free()); + assert_eq!(33, rb.cap); + assert_eq!(32, rb.as_slices().0.len()); + assert_eq!(0, rb.as_slices().1.len()); + + // extend from within cases + let mut rb = RingBuffer::new(); + rb.reserve(8); + rb.extend(b"01234567"); + rb.drop_first_n(5); + rb.extend_from_within(0, 3); + assert_eq!(4, rb.as_slices().0.len()); + assert_eq!(2, rb.as_slices().1.len()); + + rb.drop_first_n(2); + assert_eq!(2, rb.as_slices().0.len()); + assert_eq!(2, rb.as_slices().1.len()); + rb.extend_from_within(0, 4); + assert_eq!(2, rb.as_slices().0.len()); + assert_eq!(6, rb.as_slices().1.len()); + + rb.drop_first_n(2); + assert_eq!(6, rb.as_slices().0.len()); + assert_eq!(0, rb.as_slices().1.len()); + rb.drop_first_n(2); + assert_eq!(4, rb.as_slices().0.len()); + assert_eq!(0, rb.as_slices().1.len()); + rb.extend_from_within(0, 4); + assert_eq!(7, rb.as_slices().0.len()); + assert_eq!(1, rb.as_slices().1.len()); + + let mut rb = RingBuffer::new(); + rb.reserve(8); + rb.extend(b"11111111"); + rb.drop_first_n(7); + rb.extend(b"111"); + assert_eq!(2, rb.as_slices().0.len()); + assert_eq!(2, rb.as_slices().1.len()); + rb.extend_from_within(0, 4); + assert_eq!(b"11", rb.as_slices().0); + assert_eq!(b"111111", rb.as_slices().1); +} + +#[test] +fn extend_from_within_branchless_matches_checked_across_layouts() { + let contiguous = || { + let mut rb = RingBuffer::new(); + rb.reserve(16); + rb.extend(b"0123456789"); + rb + }; + assert_branchless_matches_checked(contiguous(), contiguous(), 2, 5); + + let wrapped_write = || { + let mut rb = RingBuffer::new(); + rb.reserve(16); + rb.extend(b"0123456789ABC"); + rb.drop_first_n(2); + rb + }; + assert_branchless_matches_checked(wrapped_write(), wrapped_write(), 1, 5); + + let wrapped_data = || { + let mut rb = RingBuffer::new(); + rb.reserve(32); + rb.extend(b"0123456789abcdefghijklmn"); + rb.drop_first_n(18); + rb.extend(b"wxyz012345"); + rb + }; + assert_branchless_matches_checked(wrapped_data(), wrapped_data(), 8, 2); + assert_branchless_matches_checked(wrapped_data(), wrapped_data(), 2, 8); +} + +#[test] +fn copy_with_nobranch_check_matches_checked_for_all_valid_case_masks() { + let cases = [ + (0, 0, 0, 0), + (1, 0, 0, 0), + (0, 1, 0, 0), + (0, 0, 1, 0), + (0, 0, 0, 1), + (1, 1, 0, 0), + (1, 0, 1, 0), + (1, 0, 0, 1), + (0, 1, 0, 1), + (0, 0, 1, 1), + (1, 1, 0, 1), + (1, 0, 1, 1), + ]; + + for (m1_in_f1, m2_in_f1, m1_in_f2, m2_in_f2) in cases { + let m1 = [11_u8, 12, 13, 14]; + let m2 = [21_u8, 22, 23, 24]; + let mut expected = [0_u8; 8]; + let mut actual = [0_u8; 8]; + + unsafe { + copy_with_checks( + m1.as_ptr(), + m2.as_ptr(), + expected.as_mut_ptr(), + expected.as_mut_ptr().add(4), + m1_in_f1, + m2_in_f1, + m1_in_f2, + m2_in_f2, + ); + copy_with_nobranch_check( + m1.as_ptr(), + m2.as_ptr(), + actual.as_mut_ptr(), + actual.as_mut_ptr().add(4), + m1_in_f1, + m2_in_f1, + m1_in_f2, + m2_in_f2, + ); + } + + assert_eq!( + expected, actual, + "case=({}, {}, {}, {})", + m1_in_f1, m2_in_f1, m1_in_f2, m2_in_f2 + ); + } +} + +#[test] +fn copy_bytes_overshooting_preserves_prefix_for_runtime_chunk_lengths() { + // Validate correctness for lengths derived from the active runtime chunk: + // - single chunk (`chunk`) + // - multi chunk (`2 * chunk`) + // - fallback shape (`chunk + 1`) + // This checks copy semantics across runtime-selected strategies. + let chunk = simd_copy::active_chunk_size_for_tests(); + let single_len = chunk; + let multi_len = chunk * 2; + let fallback_len = chunk + 1; + let overshoot_cap = chunk * 2; + let cap = multi_len + chunk; + + let src_single = vec![1_u8; cap]; + let mut dst_single = vec![0_u8; cap]; + unsafe { + simd_copy::copy_bytes_overshooting( + (src_single.as_ptr(), single_len), + (dst_single.as_mut_ptr(), single_len), + single_len, + ); + } + assert_eq!(&dst_single[..single_len], &src_single[..single_len]); + + let src_multi = vec![2_u8; cap]; + let mut dst_multi = vec![0_u8; cap]; + unsafe { + simd_copy::copy_bytes_overshooting( + (src_multi.as_ptr(), multi_len), + (dst_multi.as_mut_ptr(), multi_len), + multi_len, + ); + } + assert_eq!(&dst_multi[..multi_len], &src_multi[..multi_len]); + + let src_fallback = vec![3_u8; cap]; + let mut dst_fallback = vec![0_u8; cap]; + unsafe { + simd_copy::copy_bytes_overshooting( + (src_fallback.as_ptr(), fallback_len), + (dst_fallback.as_mut_ptr(), fallback_len), + fallback_len, + ); + } + assert_eq!(&dst_fallback[..fallback_len], &src_fallback[..fallback_len]); + + let src_overshoot = vec![4_u8; cap + 1]; + let mut dst_overshoot = vec![0_u8; cap + 1]; + unsafe { + simd_copy::copy_bytes_overshooting( + (src_overshoot.as_ptr().add(1), overshoot_cap), + (dst_overshoot.as_mut_ptr().add(1), overshoot_cap), + fallback_len, + ); + } + assert_eq!( + &dst_overshoot[1..1 + fallback_len], + &src_overshoot[1..1 + fallback_len] + ); +} + +/// Helper: drive the ringbuffer into a wrapped layout where the two +/// free-slice halves straddle the physical end of the backing buffer. +/// Returns a buffer whose `len() == fill_len` of `pre_byte` data and +/// whose free region wraps. +fn build_wrapped_buffer(cap: usize, fill_len: usize, pre_byte: u8) -> RingBuffer { + let mut rb = RingBuffer::new(); + rb.reserve(cap); + let actual_cap = rb.cap; + // Push to near-end of the physical buffer. + let pre_len = actual_cap - 2; + let prefix = alloc::vec![pre_byte; pre_len]; + rb.extend(&prefix); + // Drop those bytes so head advances past them; tail now sits near + // the end of `cap`, head is in the middle. Subsequent inserts will + // wrap across the physical end. + rb.drop_first_n(pre_len - fill_len); + assert_eq!(rb.len(), fill_len); + assert!(rb.tail > rb.head, "tail should still trail tape end"); + rb +} + +#[test] +fn extend_and_fill_contiguous_layout() { + let mut rb = RingBuffer::new(); + rb.extend_and_fill(0xAB, 7); + assert_eq!(rb.len(), 7); + let (s1, s2) = rb.as_slices(); + let mut combined = alloc::vec::Vec::with_capacity(7); + combined.extend_from_slice(s1); + combined.extend_from_slice(s2); + assert_eq!(combined, alloc::vec![0xAB; 7]); +} + +/// Construct a RingBuffer in the wrapped state where `tail < head`. +/// Data occupies `[head, cap)` followed by `[0, tail)`, leaving the +/// free region as `[tail, head)` — i.e. the **first** free slice +/// returned by `free_slice_parts` is the inner gap, NOT a span that +/// ends at `cap`. +fn build_tail_before_head(cap_hint: usize, head_pos: usize, tail_pos: usize) -> RingBuffer { + assert!(tail_pos < head_pos); + let mut rb = RingBuffer::new(); + rb.reserve(cap_hint); + let actual_cap = rb.cap; + // Fill almost the whole buffer so we can carve out tail < head. + let fill_len = actual_cap - 2; + let prefix = alloc::vec![0xCD; fill_len]; + rb.extend(&prefix); + // Drop bytes so `head` lands at the target. Tail stays at fill_len. + rb.drop_first_n(head_pos); + // Now extend just enough to wrap tail past `cap` and land it at + // `tail_pos`. From the current state (head=head_pos, tail=fill_len), + // we need `tail_pos = (fill_len + extra) % cap`, so + // `extra = (tail_pos + cap - fill_len) % cap`. + let extra = (tail_pos + actual_cap - fill_len) % actual_cap; + rb.extend(&alloc::vec![0xCD; extra]); + assert_eq!(rb.head, head_pos); + assert_eq!(rb.tail, tail_pos); + assert!( + rb.tail < rb.head, + "expected wrapped layout: tail={} head={}", + rb.tail, + rb.head + ); + rb +} + +#[test] +fn extend_wrapped_layout_preserves_bytes_past_head() { + // Regression test for the WILDCOPY_OVERLENGTH inflation bug. + // When `tail < head` the first free slice returned by + // `free_slice_parts` is the inner `[tail, head)` gap — it does NOT + // end at `cap`. A previous version of `RingBuffer::extend` always + // added WILDCOPY_OVERLENGTH to the destination capacity passed to + // `simd_copy::copy_bytes_overshooting`, which on this layout would + // allow wildcopy overshoot writes to clobber bytes at `[head, + // head+16)` — still-readable data the caller has not consumed yet. + // The current code gates the inflation on `tail >= head`; this + // test asserts that bytes at `head..head+16` survive an extend() + // that fills the inner gap nearly to capacity. + let mut rb = build_tail_before_head(128, 80, 10); + let head_before = rb.head; + let cap = rb.cap; + // Sample a window of bytes immediately past `head` and confirm + // they are the prefill (0xCD); these are the bytes any erroneous + // wildcopy overshoot would corrupt. + let mut sentinel = [0u8; 16]; + unsafe { + for (i, slot) in sentinel.iter_mut().enumerate() { + *slot = rb.buf.as_ptr().add((head_before + i) % cap).read(); + } + } + assert!(sentinel.iter().all(|&b| b == 0xCD), "pre-state sentinel"); + + // Fill the inner free region with a recognisable pattern, leaving + // exactly the 1-byte sentinel gap the RingBuffer always reserves. + let free_before = rb.free(); + let payload = alloc::vec![0x42; free_before]; + rb.extend(&payload); + + // The bytes at [head, head+16) must still be the original 0xCD + // prefill — any overshoot would have written 0x42 over them. + unsafe { + for i in 0..16usize { + let actual = rb.buf.as_ptr().add((head_before + i) % cap).read(); + assert_eq!( + actual, + 0xCD, + "byte at head+{i} (raw idx {}) was clobbered: got {:#04x}", + (head_before + i) % cap, + actual + ); + } + } +} + +#[test] +fn extend_and_fill_wrapped_layout() { + // Pre-fill so the free region straddles the wrap boundary, then + // verify both halves are written with the fill byte. + let mut rb = build_wrapped_buffer(16, 2, 0x11); + let extra = rb.cap - 2; // fills exactly to capacity, forcing a wrap + rb.extend_and_fill(0x22, extra); + assert_eq!(rb.len(), 2 + extra); + let (s1, s2) = rb.as_slices(); + let mut combined = alloc::vec::Vec::with_capacity(rb.len()); + combined.extend_from_slice(s1); + combined.extend_from_slice(s2); + let mut expected = alloc::vec![0x11; 2]; + expected.extend(alloc::vec![0x22; extra]); + assert_eq!(combined, expected); +} + +#[test] +fn extend_from_reader_contiguous_layout() { + let mut rb = RingBuffer::new(); + let src: [u8; 6] = [1, 2, 3, 4, 5, 6]; + rb.extend_from_reader(&src[..], 6).unwrap(); + assert_eq!(rb.len(), 6); + let (s1, s2) = rb.as_slices(); + let mut combined = alloc::vec::Vec::with_capacity(6); + combined.extend_from_slice(s1); + combined.extend_from_slice(s2); + assert_eq!(combined, src); +} + +#[test] +fn extend_from_reader_wrapped_layout() { + let mut rb = build_wrapped_buffer(16, 3, 0xAA); + let extra = rb.cap - 3; + let src: alloc::vec::Vec = (0..extra as u8).collect(); + rb.extend_from_reader(src.as_slice(), extra).unwrap(); + assert_eq!(rb.len(), 3 + extra); + let (s1, s2) = rb.as_slices(); + let mut combined = alloc::vec::Vec::with_capacity(rb.len()); + combined.extend_from_slice(s1); + combined.extend_from_slice(s2); + let mut expected = alloc::vec![0xAA; 3]; + expected.extend_from_slice(&src); + assert_eq!(combined, expected); +} + +#[test] +fn extend_from_reader_eof_leaves_state_unchanged() { + let mut rb = RingBuffer::new(); + rb.extend(b"prefix"); + let snapshot_len = rb.len(); + let snapshot_slices: (alloc::vec::Vec, alloc::vec::Vec) = { + let (a, b) = rb.as_slices(); + (a.to_vec(), b.to_vec()) + }; + + // Reader yields only 2 bytes but we ask for 10 → `read_exact` fails + // on the second chunk, and `tail` must not advance. + let short: [u8; 2] = [0xCC, 0xDD]; + let err = rb.extend_from_reader(&short[..], 10); + assert!(err.is_err(), "short reader must propagate IO error"); + assert_eq!(rb.len(), snapshot_len, "len() must be unchanged on error"); + let (a, b) = rb.as_slices(); + assert_eq!(a, snapshot_slices.0.as_slice()); + assert_eq!(b, snapshot_slices.1.as_slice()); +} diff --git a/zstd/src/decoding/scratch.rs b/zstd/src/decoding/scratch.rs index 743a77de3..601d95075 100644 --- a/zstd/src/decoding/scratch.rs +++ b/zstd/src/decoding/scratch.rs @@ -567,112 +567,4 @@ impl DerefMut for AlignedFSETable { } #[cfg(test)] -mod tests { - use super::*; - use crate::decoding::dictionary::Dictionary; - - #[test] - fn init_from_dict_marks_fse_ddict_is_cold() { - // Upstream zstd parity: `ZSTD_decompressBegin_usingDDict` sets - // `dctx->ddictIsCold = 1`. Mirror: `init_from_dict` must - // leave `fse.ddict_is_cold = true` so the first - // sequence-section decode of the frame engages the prefetch - // pipeline regardless of `offsets_long_share`. - extern crate std; - let dict_raw = - std::fs::read("./dict_tests/dictionary").expect("dictionary fixture should load"); - let dict = DictionaryHandle::from_dictionary( - Dictionary::decode_dict(&dict_raw).expect("dictionary should parse"), - ); - let mut scratch: DecoderScratch = DecoderScratch::new(1024); - assert!( - !scratch.fse.ddict_is_cold, - "fresh DecoderScratch must not advertise a cold dict" - ); - scratch.init_from_dict(&dict); - assert!( - scratch.fse.ddict_is_cold, - "init_from_dict must set ddict_is_cold = true" - ); - } - - #[test] - fn reinit_from_clears_cold_dict_flag() { - // `reinit_from` materialises a local-only snapshot (LSM resume - // export/restore). It detaches the dict and forces every axis to - // Local, so it must also clear `ddict_is_cold` — otherwise a stale - // cold-dict pipeline gate from a prior dictionary frame would be - // carried into restored entropy state that has no dictionary. - let mut dst = FSEScratch::new(); - dst.ddict_is_cold = true; // simulate a prior cold-dict frame's flag - let src = FSEScratch::new(); // clean local-only source, not cold - dst.reinit_from(&src); - assert!( - !dst.ddict_is_cold, - "reinit_from must clear ddict_is_cold on a local-only snapshot" - ); - } - - #[test] - fn init_from_dict_is_zero_copy_cow_then_reset_detaches() { - // Copy-on-write contract: `init_from_dict` must NOT copy the - // dictionary's sequence FSE table bytes into the per-frame local - // scratch; the live accessor resolves straight to the shared - // dictionary's table. `reset` must then detach so a reused - // scratch never reads the previous frame's dictionary tables. - extern crate std; - let dict_raw = - std::fs::read("./dict_tests/dictionary").expect("dictionary fixture should load"); - let dict = DictionaryHandle::from_dictionary( - Dictionary::decode_dict(&dict_raw).expect("dictionary should parse"), - ); - let mut scratch: DecoderScratch = DecoderScratch::new(1024); - scratch.init_from_dict(&dict); - - let dict_ll_len = dict.as_dict().fse.literal_lengths.decode().len(); - assert!( - dict_ll_len > 0, - "dict fixture should carry a built LL table" - ); - // Dict-sourced axis resolves to the dictionary's table... - assert_eq!( - scratch.fse.ll_table().decode().len(), - dict_ll_len, - "Dict-sourced axis must resolve to the shared dictionary's table" - ); - // ...with the local buffer left untouched (no eager copy). - assert!( - scratch.fse.literal_lengths.decode().is_empty(), - "init_from_dict must not copy table bytes into local scratch (COW)" - ); - - // Same copy-on-write contract for the Huffman literals table: the - // accessor resolves to the dictionary's built table while the - // local table stays untouched (max_num_bits == 0 means unbuilt). - let dict_huf_bits = dict.as_dict().huf.table.max_num_bits; - assert!( - dict_huf_bits > 0, - "dict fixture should carry a built HUF table" - ); - assert_eq!( - scratch.huf.huf_table().max_num_bits, - dict_huf_bits, - "Dict-sourced HUF axis must resolve to the dictionary's table" - ); - assert_eq!( - scratch.huf.table.max_num_bits, 0, - "init_from_dict must not copy the HUF table into local scratch (COW)" - ); - - scratch.reset(1024); - assert!( - scratch.fse.ll_table().decode().is_empty(), - "reset must detach the dictionary copy-on-write source" - ); - assert_eq!( - scratch.huf.huf_table().max_num_bits, - 0, - "reset must detach the HUF dictionary copy-on-write source" - ); - } -} +mod tests; diff --git a/zstd/src/decoding/scratch/tests.rs b/zstd/src/decoding/scratch/tests.rs new file mode 100644 index 000000000..3781540ed --- /dev/null +++ b/zstd/src/decoding/scratch/tests.rs @@ -0,0 +1,107 @@ +use super::*; +use crate::decoding::dictionary::Dictionary; + +#[test] +fn init_from_dict_marks_fse_ddict_is_cold() { + // Upstream zstd parity: `ZSTD_decompressBegin_usingDDict` sets + // `dctx->ddictIsCold = 1`. Mirror: `init_from_dict` must + // leave `fse.ddict_is_cold = true` so the first + // sequence-section decode of the frame engages the prefetch + // pipeline regardless of `offsets_long_share`. + extern crate std; + let dict_raw = + std::fs::read("./dict_tests/dictionary").expect("dictionary fixture should load"); + let dict = DictionaryHandle::from_dictionary( + Dictionary::decode_dict(&dict_raw).expect("dictionary should parse"), + ); + let mut scratch: DecoderScratch = DecoderScratch::new(1024); + assert!( + !scratch.fse.ddict_is_cold, + "fresh DecoderScratch must not advertise a cold dict" + ); + scratch.init_from_dict(&dict); + assert!( + scratch.fse.ddict_is_cold, + "init_from_dict must set ddict_is_cold = true" + ); +} + +#[test] +fn reinit_from_clears_cold_dict_flag() { + // `reinit_from` materialises a local-only snapshot (LSM resume + // export/restore). It detaches the dict and forces every axis to + // Local, so it must also clear `ddict_is_cold` — otherwise a stale + // cold-dict pipeline gate from a prior dictionary frame would be + // carried into restored entropy state that has no dictionary. + let mut dst = FSEScratch::new(); + dst.ddict_is_cold = true; // simulate a prior cold-dict frame's flag + let src = FSEScratch::new(); // clean local-only source, not cold + dst.reinit_from(&src); + assert!( + !dst.ddict_is_cold, + "reinit_from must clear ddict_is_cold on a local-only snapshot" + ); +} + +#[test] +fn init_from_dict_is_zero_copy_cow_then_reset_detaches() { + // Copy-on-write contract: `init_from_dict` must NOT copy the + // dictionary's sequence FSE table bytes into the per-frame local + // scratch; the live accessor resolves straight to the shared + // dictionary's table. `reset` must then detach so a reused + // scratch never reads the previous frame's dictionary tables. + extern crate std; + let dict_raw = + std::fs::read("./dict_tests/dictionary").expect("dictionary fixture should load"); + let dict = DictionaryHandle::from_dictionary( + Dictionary::decode_dict(&dict_raw).expect("dictionary should parse"), + ); + let mut scratch: DecoderScratch = DecoderScratch::new(1024); + scratch.init_from_dict(&dict); + + let dict_ll_len = dict.as_dict().fse.literal_lengths.decode().len(); + assert!( + dict_ll_len > 0, + "dict fixture should carry a built LL table" + ); + // Dict-sourced axis resolves to the dictionary's table... + assert_eq!( + scratch.fse.ll_table().decode().len(), + dict_ll_len, + "Dict-sourced axis must resolve to the shared dictionary's table" + ); + // ...with the local buffer left untouched (no eager copy). + assert!( + scratch.fse.literal_lengths.decode().is_empty(), + "init_from_dict must not copy table bytes into local scratch (COW)" + ); + + // Same copy-on-write contract for the Huffman literals table: the + // accessor resolves to the dictionary's built table while the + // local table stays untouched (max_num_bits == 0 means unbuilt). + let dict_huf_bits = dict.as_dict().huf.table.max_num_bits; + assert!( + dict_huf_bits > 0, + "dict fixture should carry a built HUF table" + ); + assert_eq!( + scratch.huf.huf_table().max_num_bits, + dict_huf_bits, + "Dict-sourced HUF axis must resolve to the dictionary's table" + ); + assert_eq!( + scratch.huf.table.max_num_bits, 0, + "init_from_dict must not copy the HUF table into local scratch (COW)" + ); + + scratch.reset(1024); + assert!( + scratch.fse.ll_table().decode().is_empty(), + "reset must detach the dictionary copy-on-write source" + ); + assert_eq!( + scratch.huf.huf_table().max_num_bits, + 0, + "reset must detach the HUF dictionary copy-on-write source" + ); +} diff --git a/zstd/src/decoding/sequence_execution.rs b/zstd/src/decoding/sequence_execution.rs index ec681181f..3600b5651 100644 --- a/zstd/src/decoding/sequence_execution.rs +++ b/zstd/src/decoding/sequence_execution.rs @@ -151,70 +151,4 @@ fn do_offset_history_repcode(offset_value: u32, lit_len: u32, scratch: &mut [u32 } #[cfg(test)] -mod tests { - use super::do_offset_history; - - #[test] - fn offset_history_lit_non_zero_rep1_keeps_history() { - let mut hist = [10, 20, 30]; - let actual = do_offset_history(1, 5, &mut hist); - assert_eq!(actual, 10); - assert_eq!(hist, [10, 20, 30]); - } - - #[test] - fn offset_history_lit_non_zero_rep2_rotates_first_two() { - let mut hist = [10, 20, 30]; - let actual = do_offset_history(2, 5, &mut hist); - assert_eq!(actual, 20); - assert_eq!(hist, [20, 10, 30]); - } - - #[test] - fn offset_history_lit_non_zero_rep3_full_rotate() { - let mut hist = [10, 20, 30]; - let actual = do_offset_history(3, 5, &mut hist); - assert_eq!(actual, 30); - assert_eq!(hist, [30, 10, 20]); - } - - #[test] - fn offset_history_lit_zero_rep1_uses_second_history() { - let mut hist = [10, 20, 30]; - let actual = do_offset_history(1, 0, &mut hist); - assert_eq!(actual, 20); - assert_eq!(hist, [20, 10, 30]); - } - - #[test] - fn offset_history_lit_zero_rep2_uses_third_history() { - let mut hist = [10, 20, 30]; - let actual = do_offset_history(2, 0, &mut hist); - assert_eq!(actual, 30); - assert_eq!(hist, [30, 10, 20]); - } - - #[test] - fn offset_history_lit_zero_rep3_minus_one() { - let mut hist = [10, 20, 30]; - let actual = do_offset_history(3, 0, &mut hist); - assert_eq!(actual, 9); - assert_eq!(hist, [9, 10, 20]); - } - - #[test] - fn offset_history_new_offset_path() { - let mut hist = [10, 20, 30]; - let actual = do_offset_history(9, 1, &mut hist); - assert_eq!(actual, 6); - assert_eq!(hist, [6, 10, 20]); - } - - #[test] - fn offset_history_zero_offset_preserves_error_path() { - let mut hist = [10, 20, 30]; - let actual = do_offset_history(0, 1, &mut hist); - assert_eq!(actual, 0); - assert_eq!(hist, [10, 20, 30]); - } -} +mod tests; diff --git a/zstd/src/decoding/sequence_execution/tests.rs b/zstd/src/decoding/sequence_execution/tests.rs new file mode 100644 index 000000000..3aa821bb1 --- /dev/null +++ b/zstd/src/decoding/sequence_execution/tests.rs @@ -0,0 +1,65 @@ +use super::do_offset_history; + +#[test] +fn offset_history_lit_non_zero_rep1_keeps_history() { + let mut hist = [10, 20, 30]; + let actual = do_offset_history(1, 5, &mut hist); + assert_eq!(actual, 10); + assert_eq!(hist, [10, 20, 30]); +} + +#[test] +fn offset_history_lit_non_zero_rep2_rotates_first_two() { + let mut hist = [10, 20, 30]; + let actual = do_offset_history(2, 5, &mut hist); + assert_eq!(actual, 20); + assert_eq!(hist, [20, 10, 30]); +} + +#[test] +fn offset_history_lit_non_zero_rep3_full_rotate() { + let mut hist = [10, 20, 30]; + let actual = do_offset_history(3, 5, &mut hist); + assert_eq!(actual, 30); + assert_eq!(hist, [30, 10, 20]); +} + +#[test] +fn offset_history_lit_zero_rep1_uses_second_history() { + let mut hist = [10, 20, 30]; + let actual = do_offset_history(1, 0, &mut hist); + assert_eq!(actual, 20); + assert_eq!(hist, [20, 10, 30]); +} + +#[test] +fn offset_history_lit_zero_rep2_uses_third_history() { + let mut hist = [10, 20, 30]; + let actual = do_offset_history(2, 0, &mut hist); + assert_eq!(actual, 30); + assert_eq!(hist, [30, 10, 20]); +} + +#[test] +fn offset_history_lit_zero_rep3_minus_one() { + let mut hist = [10, 20, 30]; + let actual = do_offset_history(3, 0, &mut hist); + assert_eq!(actual, 9); + assert_eq!(hist, [9, 10, 20]); +} + +#[test] +fn offset_history_new_offset_path() { + let mut hist = [10, 20, 30]; + let actual = do_offset_history(9, 1, &mut hist); + assert_eq!(actual, 6); + assert_eq!(hist, [6, 10, 20]); +} + +#[test] +fn offset_history_zero_offset_preserves_error_path() { + let mut hist = [10, 20, 30]; + let actual = do_offset_history(0, 1, &mut hist); + assert_eq!(actual, 0); + assert_eq!(hist, [10, 20, 30]); +} diff --git a/zstd/src/decoding/sequence_section_decoder.rs b/zstd/src/decoding/sequence_section_decoder.rs index 2511fcfaf..0a742d940 100644 --- a/zstd/src/decoding/sequence_section_decoder.rs +++ b/zstd/src/decoding/sequence_section_decoder.rs @@ -10,8 +10,6 @@ use crate::common::MAX_BLOCK_SIZE; use crate::decoding::errors::{DecodeSequenceError, DecompressBlockError, ExecuteSequencesError}; use crate::decoding::sequence_execution::do_offset_history; use crate::fse::SeqFSEDecoder; -#[cfg(test)] -use alloc::vec::Vec; // 8-slot software pipeline mirroring upstream zstd // `ZSTD_decompressSequencesLong_body`'s `STORED_SEQS = 8`. The @@ -1691,500 +1689,5 @@ const OFFSET_DEFAULT_DISTRIBUTION: [i32; 29] = [ 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1, ]; -/// Regression gate for the predefined FSE table cache: every cached -/// table must be byte-identical to the table the rebuild path would -/// produce on the next call. If the cache ever drifts from the -/// rebuild output (different `decode` entries, different -/// `accuracy_log`, different `offsets_long_share` for OF) the -/// dispatch in `maybe_update_fse_tables` would silently decode -/// against a stale table — the bench delta would still look fine -/// but cross-validation against the upstream zstd would diverge on the -/// next ratio gate. -#[cfg(feature = "std")] -#[test] -fn predefined_fse_caches_match_rebuild_output() { - use crate::fse::SeqFSETable; - - let mut ll_rebuild = SeqFSETable::new(MAX_LITERAL_LENGTH_CODE); - ll_rebuild - .build_from_probabilities( - LL_DEFAULT_ACC_LOG, - &Vec::from(&LITERALS_LENGTH_DEFAULT_DISTRIBUTION[..]), - ) - .unwrap(); - ll_rebuild.enrich_with_packed_seq_meta(&LL_META); - let ll_cached = predefined_ll_table(); - assert_eq!(ll_rebuild.accuracy_log, ll_cached.accuracy_log); - assert_eq!(ll_rebuild.decode().len(), ll_cached.decode().len()); - for (i, (a, b)) in ll_rebuild - .decode() - .iter() - .zip(ll_cached.decode().iter()) - .enumerate() - { - assert_eq!(a.num_bits, b.num_bits, "LL entry {i} num_bits mismatch"); - assert_eq!(a.new_state, b.new_state, "LL entry {i} new_state mismatch"); - assert_eq!( - a.base_value, b.base_value, - "LL entry {i} base_value mismatch" - ); - assert_eq!( - a.num_additional_bits, b.num_additional_bits, - "LL entry {i} num_additional_bits mismatch" - ); - } - - let mut ml_rebuild = SeqFSETable::new(MAX_MATCH_LENGTH_CODE); - ml_rebuild - .build_from_probabilities( - ML_DEFAULT_ACC_LOG, - &Vec::from(&MATCH_LENGTH_DEFAULT_DISTRIBUTION[..]), - ) - .unwrap(); - ml_rebuild.enrich_with_packed_seq_meta(&ML_META); - let ml_cached = predefined_ml_table(); - assert_eq!(ml_rebuild.accuracy_log, ml_cached.accuracy_log); - assert_eq!(ml_rebuild.decode().len(), ml_cached.decode().len()); - for (i, (a, b)) in ml_rebuild - .decode() - .iter() - .zip(ml_cached.decode().iter()) - .enumerate() - { - assert_eq!(a.num_bits, b.num_bits, "ML entry {i} num_bits mismatch"); - assert_eq!(a.new_state, b.new_state, "ML entry {i} new_state mismatch"); - assert_eq!( - a.base_value, b.base_value, - "ML entry {i} base_value mismatch" - ); - assert_eq!( - a.num_additional_bits, b.num_additional_bits, - "ML entry {i} num_additional_bits mismatch" - ); - } - - let mut of_rebuild = SeqFSETable::new(MAX_OFFSET_CODE); - of_rebuild - .build_from_probabilities( - OF_DEFAULT_ACC_LOG, - &Vec::from(&OFFSET_DEFAULT_DISTRIBUTION[..]), - ) - .unwrap(); - of_rebuild.enrich_for_offsets(); - let of_rebuild_share = compute_offsets_long_share(&of_rebuild); - let (of_cached, of_cached_share) = predefined_of_table(); - assert_eq!(of_rebuild.accuracy_log, of_cached.accuracy_log); - assert_eq!(of_rebuild.decode().len(), of_cached.decode().len()); - assert_eq!( - of_rebuild_share, of_cached_share, - "OF offsets_long_share mismatch" - ); - for (i, (a, b)) in of_rebuild - .decode() - .iter() - .zip(of_cached.decode().iter()) - .enumerate() - { - assert_eq!(a.num_bits, b.num_bits, "OF entry {i} num_bits mismatch"); - assert_eq!(a.new_state, b.new_state, "OF entry {i} new_state mismatch"); - assert_eq!( - a.base_value, b.base_value, - "OF entry {i} base_value mismatch" - ); - assert_eq!( - a.num_additional_bits, b.num_additional_bits, - "OF entry {i} num_additional_bits mismatch" - ); - } -} - -#[test] -fn test_ll_default() { - let mut table = crate::fse::SeqFSETable::new(MAX_LITERAL_LENGTH_CODE); - table - .build_from_probabilities( - LL_DEFAULT_ACC_LOG, - &Vec::from(&LITERALS_LENGTH_DEFAULT_DISTRIBUTION[..]), - ) - .unwrap(); - - assert!(table.decode().len() == 64); - - //just test a few values. TODO test all values - assert!(table.decode()[0].num_bits == 4); - assert!(table.decode()[0].new_state == 0); - - assert!(table.decode()[19].num_bits == 6); - assert!(table.decode()[19].new_state == 0); - - assert!(table.decode()[39].num_bits == 4); - assert!(table.decode()[39].new_state == 16); - - assert!(table.decode()[60].num_bits == 6); - assert!(table.decode()[60].new_state == 0); - - assert!(table.decode()[59].num_bits == 5); - assert!(table.decode()[59].new_state == 32); -} - -#[cfg(test)] -mod offsets_long_share_tests { - use super::compute_offsets_long_share; - - /// Construct a synthetic offsets [`SeqFSETable`] with the given - /// offset code per entry. Mirrors the post-`enrich_for_offsets` - /// shape used by `compute_offsets_long_share`: each entry's - /// `num_additional_bits` holds the code (== source byte for - /// `code < 32`, the only range this helper reads). - fn synthetic_offsets_table(accuracy_log: u8, symbols: &[u8]) -> crate::fse::SeqFSETable { - use crate::fse::SeqSymbol; - let size = 1usize << accuracy_log; - assert_eq!( - symbols.len(), - size, - "symbols.len() must equal 1 << accuracy_log" - ); - let mut t = crate::fse::SeqFSETable::new(31); - t.accuracy_log = accuracy_log; - let entries: alloc::vec::Vec = symbols - .iter() - .map(|&s| SeqSymbol { - new_state: 0, - num_bits: 0, - num_additional_bits: s, - base_value: 0, - }) - .collect(); - t.set_decode_for_test(&entries); - t - } - - #[test] - fn zero_long_codes_returns_zero_share() { - // A table with only short offset codes (all symbols <= 22). - // Upstream zstd parity: share is the count of symbols > 22, scaled to - // OffFSELog = 8 — with zero such symbols, share is 0 - // regardless of accuracy_log. - for log in [3u8, 5, 6, 8] { - let size = 1usize << log; - let symbols: alloc::vec::Vec = (0..size).map(|i| (i as u8) % 22).collect(); - let table = synthetic_offsets_table(log, &symbols); - assert_eq!( - compute_offsets_long_share(&table), - 0, - "log={log}: pure short-offset table must score 0" - ); - } - } - - #[test] - fn long_codes_scale_to_offset_fse_log_reference() { - // accuracy_log = 5 → 32-entry table. One symbol at code 23 - // (just above the threshold of 22), the rest at 0. Upstream zstd - // scales the raw count by `OffFSELog - accuracy_log` = - // `8 - 5 = 3`, so 1 << 3 = 8 should land at the 64-bit - // `MIN_LONG_OFFSET_SHARE = 7` threshold (just over). - let mut symbols = [0u8; 32]; - symbols[7] = 23; - let table = synthetic_offsets_table(5, &symbols); - assert_eq!(compute_offsets_long_share(&table), 8); - } - - #[test] - fn raw_count_at_offset_fse_log_passes_through_unscaled() { - // accuracy_log = OffFSELog = 8 → 256-entry table. No scaling - // applied (shift by zero), so the share equals the raw count - // of symbols > 22. - let mut symbols = [0u8; 256]; - for sym in symbols.iter_mut().take(15) { - *sym = 25; - } - let table = synthetic_offsets_table(8, &symbols); - assert_eq!(compute_offsets_long_share(&table), 15); - } - - #[test] - fn threshold_is_strict_greater_than() { - // Symbol == LONG_OFFSET_CODE_THRESHOLD (22) does NOT count — - // matches upstream zstd `> 22` strict-greater predicate. Only - // symbols 23..MAX raise the share. - let mut symbols = [0u8; 256]; - for sym in symbols.iter_mut().take(50) { - *sym = 22; - } - let table = synthetic_offsets_table(8, &symbols); - assert_eq!(compute_offsets_long_share(&table), 0); - symbols[0] = 23; - let table = synthetic_offsets_table(8, &symbols); - assert_eq!(compute_offsets_long_share(&table), 1); - } -} - -#[cfg(test)] -mod compute_use_long_pipeline_tests { - use super::{ADVANCE, compute_use_long_pipeline}; - - /// Per-target `MIN_LONG_OFFSET_SHARE` (mirrors the cfg in - /// `compute_use_long_pipeline`). Keep in sync with the constant - /// in production code so the tests follow target_pointer_width. - #[cfg(target_pointer_width = "64")] - const MIN_SHARE: u32 = 7; - #[cfg(not(target_pointer_width = "64"))] - const MIN_SHARE: u32 = 20; - const HISTORY_THRESHOLD: usize = 1 << 24; - - #[test] - fn rejects_when_num_sequences_below_2x_advance() { - // Below `ADVANCE * 2` (= 16): never engage the long pipeline, - // regardless of cold-dict / history / share signals. - assert!(!compute_use_long_pipeline( - ADVANCE * 2 - 1, - true, - HISTORY_THRESHOLD + 1, - u32::MAX - )); - } - - #[test] - fn cold_dict_forces_long_pipeline_at_min_seq_count() { - // At the `ADVANCE * 2` boundary, `ddict_is_cold == true` is a - // sufficient override — history / share are not read. - assert!(compute_use_long_pipeline(ADVANCE * 2, true, 0, 0)); - } - - #[test] - fn history_at_threshold_does_not_engage() { - // Gate uses `>` not `>=`: history exactly at threshold fails. - assert!(!compute_use_long_pipeline( - ADVANCE * 2, - false, - HISTORY_THRESHOLD, - MIN_SHARE, - )); - } - - #[test] - fn history_just_above_threshold_engages_when_share_meets_min() { - // Strictly above threshold + share at the per-target min: engage. - assert!(compute_use_long_pipeline( - ADVANCE * 2, - false, - HISTORY_THRESHOLD + 1, - MIN_SHARE, - )); - } - - #[test] - fn share_below_min_blocks_engagement_even_with_large_history() { - // Even with the share one below the per-target minimum, no engage. - const _: () = assert!(MIN_SHARE > 0, "test invariant: MIN_SHARE > 0 required"); - assert!(!compute_use_long_pipeline( - ADVANCE * 2, - false, - HISTORY_THRESHOLD + 1, - MIN_SHARE - 1, - )); - } - - #[test] - fn saturating_history_engages_when_share_meets_min() { - // `total_history = usize::MAX` (the saturating-add fallback path - // from the per-tier wrappers) is well above the threshold. - assert!(compute_use_long_pipeline( - ADVANCE * 2, - false, - usize::MAX, - MIN_SHARE, - )); - } -} - #[cfg(test)] -mod init_sequence_stream_tests { - use super::super::scratch::FSEScratch; - use super::init_sequence_stream; - use crate::blocks::sequence_section::SequencesHeader; - use crate::cpu_kernel::ScalarKernel; - use crate::decoding::decode_buffer::DecodeBuffer; - use crate::decoding::errors::{DecodeSequenceError, DecompressBlockError}; - use crate::decoding::ringbuffer::RingBuffer; - - /// The sequence bitstream must end with a single `1` sentinel bit in - /// the final byte; the decoder skips trailing `0` padding looking for - /// it. A final byte (read first, MSB-down) that is all zeros has no - /// sentinel, so more than 8 padding bits are skipped — that must be - /// rejected as `ExtraPadding`, not decoded. - #[test] - fn rejects_bitstream_with_excess_padding() { - let mut header = SequencesHeader::new(); - // `0x01` = one sequence; `0x00` modes byte = all axes Predefined, - // so `maybe_update_fse_tables` reads zero table bytes and the - // whole source is the (malformed) bitstream. - header.parse_from_header(&[0x01, 0x00]).unwrap(); - - let mut fse = FSEScratch::new(); - let mut buffer = DecodeBuffer::::new(4 * 1024); - // All-zero bitstream: no sentinel `1` bit anywhere. - let source = [0u8; 2]; - - // `SeqStreamSetup` is not `Debug`, so assert on the `Result` - // directly via `matches!` rather than unwrapping the Ok side. - let result = init_sequence_stream::( - &header, - &source, - &mut fse, - &mut buffer, - ); - assert!( - matches!( - result, - Err(DecompressBlockError::DecodeSequenceError( - DecodeSequenceError::ExtraPadding { .. } - )) - ), - "all-zero padding must be rejected as ExtraPadding" - ); - } - - /// One-sequence, all-Predefined header. Pairs with an ample non-zero - /// bitstream so `init_sequence_stream` returns `Ok` (the sequence loop - /// may still error afterwards). Used to drive the per-tier monolith - /// preambles directly: on x86_64 CI only the avx2 tier is selected at - /// runtime, so scalar / bmi2 / vbmi2 and the K-generic impl would - /// otherwise never execute their (shared) preamble. - fn predefined_one_sequence_header() -> SequencesHeader { - let mut header = SequencesHeader::new(); - header.parse_from_header(&[0x01, 0x00]).unwrap(); - header - } - - /// Drives the always-available decoders (the portable scalar tier and - /// the K-generic impl that backs the aarch64 NEON/SVE path) through a - /// well-formed preamble. The result is intentionally ignored: a - /// crafted bitstream need not yield a valid sequence, only reach and - /// pass the preamble. - #[test] - fn scalar_tier_and_generic_impl_run_preamble() { - let header = predefined_one_sequence_header(); - let source = [0xFFu8; 8]; - let lits = [0u8; 32]; - - let mut fse = FSEScratch::new(); - let mut buf = DecodeBuffer::::new(4 * 1024); - let mut offset_hist = [1u32, 4, 8]; - let _ = crate::decoding::seq_decoder_scalar::decode_and_execute_sequences_scalar( - &header, - &source, - &mut fse, - &mut buf, - &mut offset_hist, - &lits, - ); - - let mut fse = FSEScratch::new(); - let mut buf = DecodeBuffer::::new(4 * 1024); - let mut offset_hist = [1u32, 4, 8]; - let _ = super::decode_and_execute_sequences_impl::( - &header, - &source, - &mut fse, - &mut buf, - &mut offset_hist, - &lits, - ); - } - - /// Drive the BMI2 monolith preamble directly. The runtime kernel - /// selector prefers the avx2 tier on any CPU that has BMI2, so this - /// tier never runs through the normal dispatch on CI hardware; call it - /// directly (guarded on the same feature it requires). - #[cfg(all(feature = "std", target_arch = "x86_64", feature = "kernel_bmi2"))] - #[test] - fn bmi2_tier_runs_preamble() { - if !std::is_x86_feature_detected!("bmi2") { - return; - } - let header = predefined_one_sequence_header(); - let source = [0xFFu8; 8]; - let lits = [0u8; 32]; - let mut fse = FSEScratch::new(); - let mut buf = DecodeBuffer::::new(4 * 1024); - let mut offset_hist = [1u32, 4, 8]; - // SAFETY: BMI2 confirmed available by the runtime check above. - let _ = unsafe { - crate::decoding::seq_decoder_bmi2::decode_and_execute_sequences_bmi2::( - &header, - &source, - &mut fse, - &mut buf, - &mut offset_hist, - &lits, - ) - }; - } - - /// Drive the AVX2 monolith preamble directly. The avx2 tier is the - /// production path on AVX2 hardware, so this is usually also covered by - /// the normal decode tests; the explicit call keeps the preamble - /// covered even on a runner that lacks AVX2. - #[cfg(all(feature = "std", target_arch = "x86_64", feature = "kernel_avx2"))] - #[test] - fn avx2_tier_runs_preamble() { - if !(std::is_x86_feature_detected!("avx2") && std::is_x86_feature_detected!("bmi2")) { - return; - } - let header = predefined_one_sequence_header(); - let source = [0xFFu8; 8]; - let lits = [0u8; 32]; - let mut fse = FSEScratch::new(); - let mut buf = DecodeBuffer::::new(4 * 1024); - let mut offset_hist = [1u32, 4, 8]; - // SAFETY: AVX2 + BMI2 confirmed available by the runtime check. - let _ = unsafe { - crate::decoding::seq_decoder_avx2::decode_and_execute_sequences_avx2::( - &header, - &source, - &mut fse, - &mut buf, - &mut offset_hist, - &lits, - ) - }; - } - - /// Drive the VBMI2 monolith preamble directly. Requires AVX-512 VBMI2, - /// which most CI runners lack; the test self-skips there, so this tier - /// is only covered on AVX-512 hardware. - #[cfg(all(feature = "std", target_arch = "x86_64", feature = "kernel_vbmi2"))] - #[test] - fn vbmi2_tier_runs_preamble() { - // Mirror the production dispatch gate: the unsafe monolith is annotated - // `target_feature(bmi2,avx2,avx512vbmi2,avx512f,avx512vl,avx512bw)`, so a - // bare `avx512vbmi2` probe could SIGILL on a CPU that reports VBMI2 but - // lacks a companion feature. `detect_cpu_kernel() == Vbmi2` verifies the - // full set before we call it. - if !matches!( - crate::cpu_kernel::detect_cpu_kernel(), - crate::cpu_kernel::CpuKernelTag::Vbmi2 - ) { - return; - } - let header = predefined_one_sequence_header(); - let source = [0xFFu8; 8]; - let lits = [0u8; 32]; - let mut fse = FSEScratch::new(); - let mut buf = DecodeBuffer::::new(4 * 1024); - let mut offset_hist = [1u32, 4, 8]; - // SAFETY: AVX-512 VBMI2 confirmed available by the runtime check. - let _ = unsafe { - crate::decoding::seq_decoder_vbmi2::decode_and_execute_sequences_vbmi2::( - &header, - &source, - &mut fse, - &mut buf, - &mut offset_hist, - &lits, - ) - }; - } -} +mod tests; diff --git a/zstd/src/decoding/sequence_section_decoder/tests.rs b/zstd/src/decoding/sequence_section_decoder/tests.rs new file mode 100644 index 000000000..1c18f12e2 --- /dev/null +++ b/zstd/src/decoding/sequence_section_decoder/tests.rs @@ -0,0 +1,500 @@ +use super::*; +use alloc::vec::Vec; + +/// Regression gate for the predefined FSE table cache: every cached +/// table must be byte-identical to the table the rebuild path would +/// produce on the next call. If the cache ever drifts from the +/// rebuild output (different `decode` entries, different +/// `accuracy_log`, different `offsets_long_share` for OF) the +/// dispatch in `maybe_update_fse_tables` would silently decode +/// against a stale table — the bench delta would still look fine +/// but cross-validation against the upstream zstd would diverge on the +/// next ratio gate. +#[cfg(feature = "std")] +#[test] +fn predefined_fse_caches_match_rebuild_output() { + use crate::fse::SeqFSETable; + + let mut ll_rebuild = SeqFSETable::new(MAX_LITERAL_LENGTH_CODE); + ll_rebuild + .build_from_probabilities( + LL_DEFAULT_ACC_LOG, + &Vec::from(&LITERALS_LENGTH_DEFAULT_DISTRIBUTION[..]), + ) + .unwrap(); + ll_rebuild.enrich_with_packed_seq_meta(&LL_META); + let ll_cached = predefined_ll_table(); + assert_eq!(ll_rebuild.accuracy_log, ll_cached.accuracy_log); + assert_eq!(ll_rebuild.decode().len(), ll_cached.decode().len()); + for (i, (a, b)) in ll_rebuild + .decode() + .iter() + .zip(ll_cached.decode().iter()) + .enumerate() + { + assert_eq!(a.num_bits, b.num_bits, "LL entry {i} num_bits mismatch"); + assert_eq!(a.new_state, b.new_state, "LL entry {i} new_state mismatch"); + assert_eq!( + a.base_value, b.base_value, + "LL entry {i} base_value mismatch" + ); + assert_eq!( + a.num_additional_bits, b.num_additional_bits, + "LL entry {i} num_additional_bits mismatch" + ); + } + + let mut ml_rebuild = SeqFSETable::new(MAX_MATCH_LENGTH_CODE); + ml_rebuild + .build_from_probabilities( + ML_DEFAULT_ACC_LOG, + &Vec::from(&MATCH_LENGTH_DEFAULT_DISTRIBUTION[..]), + ) + .unwrap(); + ml_rebuild.enrich_with_packed_seq_meta(&ML_META); + let ml_cached = predefined_ml_table(); + assert_eq!(ml_rebuild.accuracy_log, ml_cached.accuracy_log); + assert_eq!(ml_rebuild.decode().len(), ml_cached.decode().len()); + for (i, (a, b)) in ml_rebuild + .decode() + .iter() + .zip(ml_cached.decode().iter()) + .enumerate() + { + assert_eq!(a.num_bits, b.num_bits, "ML entry {i} num_bits mismatch"); + assert_eq!(a.new_state, b.new_state, "ML entry {i} new_state mismatch"); + assert_eq!( + a.base_value, b.base_value, + "ML entry {i} base_value mismatch" + ); + assert_eq!( + a.num_additional_bits, b.num_additional_bits, + "ML entry {i} num_additional_bits mismatch" + ); + } + + let mut of_rebuild = SeqFSETable::new(MAX_OFFSET_CODE); + of_rebuild + .build_from_probabilities( + OF_DEFAULT_ACC_LOG, + &Vec::from(&OFFSET_DEFAULT_DISTRIBUTION[..]), + ) + .unwrap(); + of_rebuild.enrich_for_offsets(); + let of_rebuild_share = compute_offsets_long_share(&of_rebuild); + let (of_cached, of_cached_share) = predefined_of_table(); + assert_eq!(of_rebuild.accuracy_log, of_cached.accuracy_log); + assert_eq!(of_rebuild.decode().len(), of_cached.decode().len()); + assert_eq!( + of_rebuild_share, of_cached_share, + "OF offsets_long_share mismatch" + ); + for (i, (a, b)) in of_rebuild + .decode() + .iter() + .zip(of_cached.decode().iter()) + .enumerate() + { + assert_eq!(a.num_bits, b.num_bits, "OF entry {i} num_bits mismatch"); + assert_eq!(a.new_state, b.new_state, "OF entry {i} new_state mismatch"); + assert_eq!( + a.base_value, b.base_value, + "OF entry {i} base_value mismatch" + ); + assert_eq!( + a.num_additional_bits, b.num_additional_bits, + "OF entry {i} num_additional_bits mismatch" + ); + } +} + +#[test] +fn test_ll_default() { + let mut table = crate::fse::SeqFSETable::new(MAX_LITERAL_LENGTH_CODE); + table + .build_from_probabilities( + LL_DEFAULT_ACC_LOG, + &Vec::from(&LITERALS_LENGTH_DEFAULT_DISTRIBUTION[..]), + ) + .unwrap(); + + assert!(table.decode().len() == 64); + + //just test a few values. TODO test all values + assert!(table.decode()[0].num_bits == 4); + assert!(table.decode()[0].new_state == 0); + + assert!(table.decode()[19].num_bits == 6); + assert!(table.decode()[19].new_state == 0); + + assert!(table.decode()[39].num_bits == 4); + assert!(table.decode()[39].new_state == 16); + + assert!(table.decode()[60].num_bits == 6); + assert!(table.decode()[60].new_state == 0); + + assert!(table.decode()[59].num_bits == 5); + assert!(table.decode()[59].new_state == 32); +} + +#[cfg(test)] +mod offsets_long_share_tests { + use super::super::compute_offsets_long_share; + + /// Construct a synthetic offsets [`SeqFSETable`] with the given + /// offset code per entry. Mirrors the post-`enrich_for_offsets` + /// shape used by `compute_offsets_long_share`: each entry's + /// `num_additional_bits` holds the code (== source byte for + /// `code < 32`, the only range this helper reads). + fn synthetic_offsets_table(accuracy_log: u8, symbols: &[u8]) -> crate::fse::SeqFSETable { + use crate::fse::SeqSymbol; + let size = 1usize << accuracy_log; + assert_eq!( + symbols.len(), + size, + "symbols.len() must equal 1 << accuracy_log" + ); + let mut t = crate::fse::SeqFSETable::new(31); + t.accuracy_log = accuracy_log; + let entries: alloc::vec::Vec = symbols + .iter() + .map(|&s| SeqSymbol { + new_state: 0, + num_bits: 0, + num_additional_bits: s, + base_value: 0, + }) + .collect(); + t.set_decode_for_test(&entries); + t + } + + #[test] + fn zero_long_codes_returns_zero_share() { + // A table with only short offset codes (all symbols <= 22). + // Upstream zstd parity: share is the count of symbols > 22, scaled to + // OffFSELog = 8 — with zero such symbols, share is 0 + // regardless of accuracy_log. + for log in [3u8, 5, 6, 8] { + let size = 1usize << log; + let symbols: alloc::vec::Vec = (0..size).map(|i| (i as u8) % 22).collect(); + let table = synthetic_offsets_table(log, &symbols); + assert_eq!( + compute_offsets_long_share(&table), + 0, + "log={log}: pure short-offset table must score 0" + ); + } + } + + #[test] + fn long_codes_scale_to_offset_fse_log_reference() { + // accuracy_log = 5 → 32-entry table. One symbol at code 23 + // (just above the threshold of 22), the rest at 0. Upstream zstd + // scales the raw count by `OffFSELog - accuracy_log` = + // `8 - 5 = 3`, so 1 << 3 = 8 should land at the 64-bit + // `MIN_LONG_OFFSET_SHARE = 7` threshold (just over). + let mut symbols = [0u8; 32]; + symbols[7] = 23; + let table = synthetic_offsets_table(5, &symbols); + assert_eq!(compute_offsets_long_share(&table), 8); + } + + #[test] + fn raw_count_at_offset_fse_log_passes_through_unscaled() { + // accuracy_log = OffFSELog = 8 → 256-entry table. No scaling + // applied (shift by zero), so the share equals the raw count + // of symbols > 22. + let mut symbols = [0u8; 256]; + for sym in symbols.iter_mut().take(15) { + *sym = 25; + } + let table = synthetic_offsets_table(8, &symbols); + assert_eq!(compute_offsets_long_share(&table), 15); + } + + #[test] + fn threshold_is_strict_greater_than() { + // Symbol == LONG_OFFSET_CODE_THRESHOLD (22) does NOT count — + // matches upstream zstd `> 22` strict-greater predicate. Only + // symbols 23..MAX raise the share. + let mut symbols = [0u8; 256]; + for sym in symbols.iter_mut().take(50) { + *sym = 22; + } + let table = synthetic_offsets_table(8, &symbols); + assert_eq!(compute_offsets_long_share(&table), 0); + symbols[0] = 23; + let table = synthetic_offsets_table(8, &symbols); + assert_eq!(compute_offsets_long_share(&table), 1); + } +} + +#[cfg(test)] +mod compute_use_long_pipeline_tests { + use super::super::{ADVANCE, compute_use_long_pipeline}; + + /// Per-target `MIN_LONG_OFFSET_SHARE` (mirrors the cfg in + /// `compute_use_long_pipeline`). Keep in sync with the constant + /// in production code so the tests follow target_pointer_width. + #[cfg(target_pointer_width = "64")] + const MIN_SHARE: u32 = 7; + #[cfg(not(target_pointer_width = "64"))] + const MIN_SHARE: u32 = 20; + const HISTORY_THRESHOLD: usize = 1 << 24; + + #[test] + fn rejects_when_num_sequences_below_2x_advance() { + // Below `ADVANCE * 2` (= 16): never engage the long pipeline, + // regardless of cold-dict / history / share signals. + assert!(!compute_use_long_pipeline( + ADVANCE * 2 - 1, + true, + HISTORY_THRESHOLD + 1, + u32::MAX + )); + } + + #[test] + fn cold_dict_forces_long_pipeline_at_min_seq_count() { + // At the `ADVANCE * 2` boundary, `ddict_is_cold == true` is a + // sufficient override — history / share are not read. + assert!(compute_use_long_pipeline(ADVANCE * 2, true, 0, 0)); + } + + #[test] + fn history_at_threshold_does_not_engage() { + // Gate uses `>` not `>=`: history exactly at threshold fails. + assert!(!compute_use_long_pipeline( + ADVANCE * 2, + false, + HISTORY_THRESHOLD, + MIN_SHARE, + )); + } + + #[test] + fn history_just_above_threshold_engages_when_share_meets_min() { + // Strictly above threshold + share at the per-target min: engage. + assert!(compute_use_long_pipeline( + ADVANCE * 2, + false, + HISTORY_THRESHOLD + 1, + MIN_SHARE, + )); + } + + #[test] + fn share_below_min_blocks_engagement_even_with_large_history() { + // Even with the share one below the per-target minimum, no engage. + const _: () = assert!(MIN_SHARE > 0, "test invariant: MIN_SHARE > 0 required"); + assert!(!compute_use_long_pipeline( + ADVANCE * 2, + false, + HISTORY_THRESHOLD + 1, + MIN_SHARE - 1, + )); + } + + #[test] + fn saturating_history_engages_when_share_meets_min() { + // `total_history = usize::MAX` (the saturating-add fallback path + // from the per-tier wrappers) is well above the threshold. + assert!(compute_use_long_pipeline( + ADVANCE * 2, + false, + usize::MAX, + MIN_SHARE, + )); + } +} + +#[cfg(test)] +mod init_sequence_stream_tests { + use super::super::super::scratch::FSEScratch; + use super::super::init_sequence_stream; + use crate::blocks::sequence_section::SequencesHeader; + use crate::cpu_kernel::ScalarKernel; + use crate::decoding::decode_buffer::DecodeBuffer; + use crate::decoding::errors::{DecodeSequenceError, DecompressBlockError}; + use crate::decoding::ringbuffer::RingBuffer; + + /// The sequence bitstream must end with a single `1` sentinel bit in + /// the final byte; the decoder skips trailing `0` padding looking for + /// it. A final byte (read first, MSB-down) that is all zeros has no + /// sentinel, so more than 8 padding bits are skipped — that must be + /// rejected as `ExtraPadding`, not decoded. + #[test] + fn rejects_bitstream_with_excess_padding() { + let mut header = SequencesHeader::new(); + // `0x01` = one sequence; `0x00` modes byte = all axes Predefined, + // so `maybe_update_fse_tables` reads zero table bytes and the + // whole source is the (malformed) bitstream. + header.parse_from_header(&[0x01, 0x00]).unwrap(); + + let mut fse = FSEScratch::new(); + let mut buffer = DecodeBuffer::::new(4 * 1024); + // All-zero bitstream: no sentinel `1` bit anywhere. + let source = [0u8; 2]; + + // `SeqStreamSetup` is not `Debug`, so assert on the `Result` + // directly via `matches!` rather than unwrapping the Ok side. + let result = init_sequence_stream::( + &header, + &source, + &mut fse, + &mut buffer, + ); + assert!( + matches!( + result, + Err(DecompressBlockError::DecodeSequenceError( + DecodeSequenceError::ExtraPadding { .. } + )) + ), + "all-zero padding must be rejected as ExtraPadding" + ); + } + + /// One-sequence, all-Predefined header. Pairs with an ample non-zero + /// bitstream so `init_sequence_stream` returns `Ok` (the sequence loop + /// may still error afterwards). Used to drive the per-tier monolith + /// preambles directly: on x86_64 CI only the avx2 tier is selected at + /// runtime, so scalar / bmi2 / vbmi2 and the K-generic impl would + /// otherwise never execute their (shared) preamble. + fn predefined_one_sequence_header() -> SequencesHeader { + let mut header = SequencesHeader::new(); + header.parse_from_header(&[0x01, 0x00]).unwrap(); + header + } + + /// Drives the always-available decoders (the portable scalar tier and + /// the K-generic impl that backs the aarch64 NEON/SVE path) through a + /// well-formed preamble. The result is intentionally ignored: a + /// crafted bitstream need not yield a valid sequence, only reach and + /// pass the preamble. + #[test] + fn scalar_tier_and_generic_impl_run_preamble() { + let header = predefined_one_sequence_header(); + let source = [0xFFu8; 8]; + let lits = [0u8; 32]; + + let mut fse = FSEScratch::new(); + let mut buf = DecodeBuffer::::new(4 * 1024); + let mut offset_hist = [1u32, 4, 8]; + let _ = crate::decoding::seq_decoder_scalar::decode_and_execute_sequences_scalar( + &header, + &source, + &mut fse, + &mut buf, + &mut offset_hist, + &lits, + ); + + let mut fse = FSEScratch::new(); + let mut buf = DecodeBuffer::::new(4 * 1024); + let mut offset_hist = [1u32, 4, 8]; + let _ = super::super::decode_and_execute_sequences_impl::( + &header, + &source, + &mut fse, + &mut buf, + &mut offset_hist, + &lits, + ); + } + + /// Drive the BMI2 monolith preamble directly. The runtime kernel + /// selector prefers the avx2 tier on any CPU that has BMI2, so this + /// tier never runs through the normal dispatch on CI hardware; call it + /// directly (guarded on the same feature it requires). + #[cfg(all(feature = "std", target_arch = "x86_64", feature = "kernel_bmi2"))] + #[test] + fn bmi2_tier_runs_preamble() { + if !std::is_x86_feature_detected!("bmi2") { + return; + } + let header = predefined_one_sequence_header(); + let source = [0xFFu8; 8]; + let lits = [0u8; 32]; + let mut fse = FSEScratch::new(); + let mut buf = DecodeBuffer::::new(4 * 1024); + let mut offset_hist = [1u32, 4, 8]; + // SAFETY: BMI2 confirmed available by the runtime check above. + let _ = unsafe { + crate::decoding::seq_decoder_bmi2::decode_and_execute_sequences_bmi2::( + &header, + &source, + &mut fse, + &mut buf, + &mut offset_hist, + &lits, + ) + }; + } + + /// Drive the AVX2 monolith preamble directly. The avx2 tier is the + /// production path on AVX2 hardware, so this is usually also covered by + /// the normal decode tests; the explicit call keeps the preamble + /// covered even on a runner that lacks AVX2. + #[cfg(all(feature = "std", target_arch = "x86_64", feature = "kernel_avx2"))] + #[test] + fn avx2_tier_runs_preamble() { + if !(std::is_x86_feature_detected!("avx2") && std::is_x86_feature_detected!("bmi2")) { + return; + } + let header = predefined_one_sequence_header(); + let source = [0xFFu8; 8]; + let lits = [0u8; 32]; + let mut fse = FSEScratch::new(); + let mut buf = DecodeBuffer::::new(4 * 1024); + let mut offset_hist = [1u32, 4, 8]; + // SAFETY: AVX2 + BMI2 confirmed available by the runtime check. + let _ = unsafe { + crate::decoding::seq_decoder_avx2::decode_and_execute_sequences_avx2::( + &header, + &source, + &mut fse, + &mut buf, + &mut offset_hist, + &lits, + ) + }; + } + + /// Drive the VBMI2 monolith preamble directly. Requires AVX-512 VBMI2, + /// which most CI runners lack; the test self-skips there, so this tier + /// is only covered on AVX-512 hardware. + #[cfg(all(feature = "std", target_arch = "x86_64", feature = "kernel_vbmi2"))] + #[test] + fn vbmi2_tier_runs_preamble() { + // Mirror the production dispatch gate: the unsafe monolith is annotated + // `target_feature(bmi2,avx2,avx512vbmi2,avx512f,avx512vl,avx512bw)`, so a + // bare `avx512vbmi2` probe could SIGILL on a CPU that reports VBMI2 but + // lacks a companion feature. `detect_cpu_kernel() == Vbmi2` verifies the + // full set before we call it. + if !matches!( + crate::cpu_kernel::detect_cpu_kernel(), + crate::cpu_kernel::CpuKernelTag::Vbmi2 + ) { + return; + } + let header = predefined_one_sequence_header(); + let source = [0xFFu8; 8]; + let lits = [0u8; 32]; + let mut fse = FSEScratch::new(); + let mut buf = DecodeBuffer::::new(4 * 1024); + let mut offset_hist = [1u32, 4, 8]; + // SAFETY: AVX-512 VBMI2 confirmed available by the runtime check. + let _ = unsafe { + crate::decoding::seq_decoder_vbmi2::decode_and_execute_sequences_vbmi2::( + &header, + &source, + &mut fse, + &mut buf, + &mut offset_hist, + &lits, + ) + }; + } +} diff --git a/zstd/src/decoding/simd_copy.rs b/zstd/src/decoding/simd_copy.rs index f565a7bba..8db286d50 100644 --- a/zstd/src/decoding/simd_copy.rs +++ b/zstd/src/decoding/simd_copy.rs @@ -1180,169 +1180,4 @@ pub(crate) unsafe fn copy_exact_medium(src: *const u8, dst: *mut u8, len: usize) } #[cfg(test)] -mod tests { - use super::*; - use alloc::vec; - - #[test] - fn copy_exact_medium_matches_memcpy_all_sizes() { - // Exact byte-for-byte equivalence across the medium range, incl. - // non-multiples of every tier width (16/32) so the overlapping - // tail is exercised. - let src: vec::Vec = (0..4096u32) - .map(|i| (i.wrapping_mul(2654435761) >> 24) as u8) - .collect(); - for len in 33..2048usize { - let mut got = vec![0u8; len]; - unsafe { copy_exact_medium(src.as_ptr(), got.as_mut_ptr(), len) }; - assert_eq!( - &got[..], - &src[..len], - "copy_exact_medium mismatch at len={len}" - ); - } - } - - #[test] - fn copy_bytes_overshooting_zero_len_is_noop() { - let src = [1_u8, 2, 3, 4]; - let mut dst = [9_u8, 9, 9, 9]; - unsafe { - copy_bytes_overshooting((src.as_ptr(), src.len()), (dst.as_mut_ptr(), dst.len()), 0); - } - assert_eq!(dst, [9_u8, 9, 9, 9]); - } - - #[test] - fn copy_bytes_overshooting_fallback_exact_copy_when_caps_are_tight() { - // Pick a size that exceeds the single-op fast path threshold (16) - // and the next chunk size on every supported arch, so the fallback - // path is exercised regardless of which kernel a given build picks. - let len = 65; // > AVX-512 chunk - let src = vec![5_u8; len]; - let mut dst = vec![0_u8; len]; - - unsafe { - copy_bytes_overshooting((src.as_ptr(), len), (dst.as_mut_ptr(), len), len); - } - - assert_eq!(dst, src); - } - - #[test] - fn copy_bytes_overshooting_single_op_small() { - // Sub-16 copy with full 16-byte slack on both sides: single-op fast - // path covers it via one SIMD store (or two overlapping u64 stores - // on archs without 128-bit SIMD). - for len in 1..=16 { - let mut src = [0u8; 32]; - for (i, b) in src.iter_mut().enumerate() { - *b = i as u8; - } - let mut dst = [0u8; 32]; - unsafe { - copy_bytes_overshooting((src.as_ptr(), 32), (dst.as_mut_ptr(), 32), len); - } - assert_eq!(&dst[..len], &src[..len], "len={len}"); - } - } - - #[test] - fn copy_scalar_copies_requested_bytes() { - let src = [11_u8, 12, 13, 14, 15, 16, 17, 18]; - let mut dst = [0_u8; 8]; - unsafe { copy_scalar(src.as_ptr(), dst.as_mut_ptr(), src.len()) }; - assert_eq!(dst, src); - } - - #[cfg(all( - feature = "std", - feature = "kernel_sse2", - any(target_arch = "x86", target_arch = "x86_64") - ))] - #[test] - fn copy_sse2_copies_full_chunk_when_available() { - if !std::arch::is_x86_feature_detected!("sse2") { - return; - } - let src = [7_u8; 16]; - let mut dst = [0_u8; 16]; - unsafe { copy_sse2(src.as_ptr(), dst.as_mut_ptr(), 16) }; - assert_eq!(dst, src); - } - - #[cfg(all( - feature = "std", - feature = "kernel_avx2", - any(target_arch = "x86", target_arch = "x86_64") - ))] - #[test] - fn copy_avx2_copies_full_chunk_when_available() { - if !std::arch::is_x86_feature_detected!("avx2") { - return; - } - // Single 32-byte vector (no unrolled body, tail-only path). - let src = [8_u8; 32]; - let mut dst = [0_u8; 32]; - unsafe { copy_avx2(src.as_ptr(), dst.as_mut_ptr(), 32) }; - assert_eq!(dst, src); - } - - /// Exercises one full iteration of the 64-byte unrolled body - /// (`v0` + `v1` load/store pair) with no residual tail. - #[cfg(all( - feature = "std", - feature = "kernel_avx2", - any(target_arch = "x86", target_arch = "x86_64") - ))] - #[test] - fn copy_avx2_copies_full_unroll2_iteration() { - use alloc::vec::Vec; - if !std::arch::is_x86_feature_detected!("avx2") { - return; - } - let src: Vec = (0..64u8).collect(); - let mut dst = [0_u8; 64]; - unsafe { copy_avx2(src.as_ptr(), dst.as_mut_ptr(), 64) }; - assert_eq!(&dst[..], &src[..]); - } - - /// Exercises ONE unrolled 64-byte iteration PLUS the single- - /// vector 32-byte residual tail (96 = 64 + 32). Validates that - /// the tail branch doesn't overwrite preceding bytes and copies - /// the correct source offset. - #[cfg(all( - feature = "std", - feature = "kernel_avx2", - any(target_arch = "x86", target_arch = "x86_64") - ))] - #[test] - fn copy_avx2_copies_unroll2_loop_plus_residual_tail() { - use alloc::vec::Vec; - if !std::arch::is_x86_feature_detected!("avx2") { - return; - } - let src: Vec = (0..96u8).collect(); - let mut dst = [0_u8; 96]; - unsafe { copy_avx2(src.as_ptr(), dst.as_mut_ptr(), 96) }; - assert_eq!(&dst[..], &src[..]); - // Spot-check tail boundary: bytes 60..68 span the unroll/tail seam. - assert_eq!(&dst[60..68], &[60, 61, 62, 63, 64, 65, 66, 67]); - } - - #[cfg(all( - feature = "std", - feature = "kernel_vbmi2", - any(target_arch = "x86", target_arch = "x86_64") - ))] - #[test] - fn copy_avx512_copies_full_chunk_when_available() { - if !std::arch::is_x86_feature_detected!("avx512f") { - return; - } - let src = [9_u8; 64]; - let mut dst = [0_u8; 64]; - unsafe { copy_avx512(src.as_ptr(), dst.as_mut_ptr(), 64) }; - assert_eq!(dst, src); - } -} +mod tests; diff --git a/zstd/src/decoding/simd_copy/tests.rs b/zstd/src/decoding/simd_copy/tests.rs new file mode 100644 index 000000000..79ff1e5df --- /dev/null +++ b/zstd/src/decoding/simd_copy/tests.rs @@ -0,0 +1,164 @@ +use super::*; +use alloc::vec; + +#[test] +fn copy_exact_medium_matches_memcpy_all_sizes() { + // Exact byte-for-byte equivalence across the medium range, incl. + // non-multiples of every tier width (16/32) so the overlapping + // tail is exercised. + let src: vec::Vec = (0..4096u32) + .map(|i| (i.wrapping_mul(2654435761) >> 24) as u8) + .collect(); + for len in 33..2048usize { + let mut got = vec![0u8; len]; + unsafe { copy_exact_medium(src.as_ptr(), got.as_mut_ptr(), len) }; + assert_eq!( + &got[..], + &src[..len], + "copy_exact_medium mismatch at len={len}" + ); + } +} + +#[test] +fn copy_bytes_overshooting_zero_len_is_noop() { + let src = [1_u8, 2, 3, 4]; + let mut dst = [9_u8, 9, 9, 9]; + unsafe { + copy_bytes_overshooting((src.as_ptr(), src.len()), (dst.as_mut_ptr(), dst.len()), 0); + } + assert_eq!(dst, [9_u8, 9, 9, 9]); +} + +#[test] +fn copy_bytes_overshooting_fallback_exact_copy_when_caps_are_tight() { + // Pick a size that exceeds the single-op fast path threshold (16) + // and the next chunk size on every supported arch, so the fallback + // path is exercised regardless of which kernel a given build picks. + let len = 65; // > AVX-512 chunk + let src = vec![5_u8; len]; + let mut dst = vec![0_u8; len]; + + unsafe { + copy_bytes_overshooting((src.as_ptr(), len), (dst.as_mut_ptr(), len), len); + } + + assert_eq!(dst, src); +} + +#[test] +fn copy_bytes_overshooting_single_op_small() { + // Sub-16 copy with full 16-byte slack on both sides: single-op fast + // path covers it via one SIMD store (or two overlapping u64 stores + // on archs without 128-bit SIMD). + for len in 1..=16 { + let mut src = [0u8; 32]; + for (i, b) in src.iter_mut().enumerate() { + *b = i as u8; + } + let mut dst = [0u8; 32]; + unsafe { + copy_bytes_overshooting((src.as_ptr(), 32), (dst.as_mut_ptr(), 32), len); + } + assert_eq!(&dst[..len], &src[..len], "len={len}"); + } +} + +#[test] +fn copy_scalar_copies_requested_bytes() { + let src = [11_u8, 12, 13, 14, 15, 16, 17, 18]; + let mut dst = [0_u8; 8]; + unsafe { copy_scalar(src.as_ptr(), dst.as_mut_ptr(), src.len()) }; + assert_eq!(dst, src); +} + +#[cfg(all( + feature = "std", + feature = "kernel_sse2", + any(target_arch = "x86", target_arch = "x86_64") +))] +#[test] +fn copy_sse2_copies_full_chunk_when_available() { + if !std::arch::is_x86_feature_detected!("sse2") { + return; + } + let src = [7_u8; 16]; + let mut dst = [0_u8; 16]; + unsafe { copy_sse2(src.as_ptr(), dst.as_mut_ptr(), 16) }; + assert_eq!(dst, src); +} + +#[cfg(all( + feature = "std", + feature = "kernel_avx2", + any(target_arch = "x86", target_arch = "x86_64") +))] +#[test] +fn copy_avx2_copies_full_chunk_when_available() { + if !std::arch::is_x86_feature_detected!("avx2") { + return; + } + // Single 32-byte vector (no unrolled body, tail-only path). + let src = [8_u8; 32]; + let mut dst = [0_u8; 32]; + unsafe { copy_avx2(src.as_ptr(), dst.as_mut_ptr(), 32) }; + assert_eq!(dst, src); +} + +/// Exercises one full iteration of the 64-byte unrolled body +/// (`v0` + `v1` load/store pair) with no residual tail. +#[cfg(all( + feature = "std", + feature = "kernel_avx2", + any(target_arch = "x86", target_arch = "x86_64") +))] +#[test] +fn copy_avx2_copies_full_unroll2_iteration() { + use alloc::vec::Vec; + if !std::arch::is_x86_feature_detected!("avx2") { + return; + } + let src: Vec = (0..64u8).collect(); + let mut dst = [0_u8; 64]; + unsafe { copy_avx2(src.as_ptr(), dst.as_mut_ptr(), 64) }; + assert_eq!(&dst[..], &src[..]); +} + +/// Exercises ONE unrolled 64-byte iteration PLUS the single- +/// vector 32-byte residual tail (96 = 64 + 32). Validates that +/// the tail branch doesn't overwrite preceding bytes and copies +/// the correct source offset. +#[cfg(all( + feature = "std", + feature = "kernel_avx2", + any(target_arch = "x86", target_arch = "x86_64") +))] +#[test] +fn copy_avx2_copies_unroll2_loop_plus_residual_tail() { + use alloc::vec::Vec; + if !std::arch::is_x86_feature_detected!("avx2") { + return; + } + let src: Vec = (0..96u8).collect(); + let mut dst = [0_u8; 96]; + unsafe { copy_avx2(src.as_ptr(), dst.as_mut_ptr(), 96) }; + assert_eq!(&dst[..], &src[..]); + // Spot-check tail boundary: bytes 60..68 span the unroll/tail seam. + assert_eq!(&dst[60..68], &[60, 61, 62, 63, 64, 65, 66, 67]); +} + +#[cfg(all( + feature = "std", + feature = "kernel_vbmi2", + any(target_arch = "x86", target_arch = "x86_64") +))] +#[test] +fn copy_avx512_copies_full_chunk_when_available() { + if !std::arch::is_x86_feature_detected!("avx512f") { + return; + } + let src = [9_u8; 64]; + let mut dst = [0_u8; 64]; + unsafe { copy_avx512(src.as_ptr(), dst.as_mut_ptr(), 64) }; + assert_eq!(dst, src); +} diff --git a/zstd/src/decoding/streaming_decoder.rs b/zstd/src/decoding/streaming_decoder.rs index 6cb8b66c8..f5b3df1a3 100644 --- a/zstd/src/decoding/streaming_decoder.rs +++ b/zstd/src/decoding/streaming_decoder.rs @@ -354,330 +354,4 @@ impl> Read for StreamingDecoder = (0..8192u32).map(|i| (i & 0xFF) as u8).collect(); - let mut compressor = FrameCompressor::new(CompressionLevel::Default); - // Checksum is the subject under test; the encoder default is off - // (upstream library parity). - compressor.set_content_checksum(true); - compressor.set_source(payload.as_slice()); - let mut compressed = Vec::new(); - compressor.set_drain(&mut compressed); - compressor.compress(); - - // Corrupt the trailing 4-byte content checksum: the body still decodes - // to the right bytes, but the stored digest no longer matches. - let last = compressed.len() - 1; - compressed[last] ^= 0xFF; - - let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); - decoder - .decoder - .set_content_checksum(ContentChecksum::Verify); - - // A buffer large enough to drain the whole frame in one call: this call - // finishes the frame AND writes every payload byte. The mismatch must - // NOT abort it (that would drop the delivered bytes). - let mut buf = vec![0u8; payload.len() + 4096]; - let n = decoder - .read(&mut buf) - .expect("a read that delivered bytes must not return the checksum Err"); - assert_eq!(n, payload.len()); - assert_eq!(&buf[..n], payload.as_slice()); - - // The deferred mismatch surfaces on the terminating zero-byte read. - let err = decoder - .read(&mut buf) - .expect_err("deferred checksum mismatch must surface on the terminating read"); - assert_eq!(err.kind(), ErrorKind::Other); - } - - /// A fresh `read_to_end` must take the single-copy decode-in-place path - /// (FCS-declared frame decoded straight into the output `Vec`, no ring - /// drain) AND reproduce the payload byte-for-byte. - #[cfg(feature = "std")] - #[test] - fn read_to_end_decode_in_place_matches_and_takes_direct_path() { - use crate::encoding::{CompressionLevel, FrameCompressor}; - use alloc::vec::Vec; - - let payload: Vec = (0..20_000u32) - .map(|i| (i.wrapping_mul(2654435761) >> 24) as u8) - .collect(); - let mut compressor = FrameCompressor::new(CompressionLevel::Default); - compressor.set_source(payload.as_slice()); - let mut compressed = Vec::new(); - compressor.set_drain(&mut compressed); - compressor.compress(); - - let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); - let mut out = Vec::new(); - let n = decoder.read_to_end(&mut out).unwrap(); - assert_eq!(n, payload.len()); - assert_eq!(out, payload); - // FrameCompressor declares FCS, so the fresh fast path used the direct - // (decode-in-place) route, not the ring drain. - assert_eq!(decoder.decoder.direct_frames(), 1); - } - - /// `read_to_end` after a partial `read` must still produce the full - /// payload. The decoder is mid-frame, so the fast path is skipped and the - /// generic grow-and-drain fallback runs (no direct frame). - #[cfg(feature = "std")] - #[test] - fn read_to_end_after_partial_read_is_complete() { - use crate::encoding::{CompressionLevel, FrameCompressor}; - use alloc::vec; - use alloc::vec::Vec; - - let payload: Vec = (0..20_000u32).map(|i| (i & 0xFF) as u8).collect(); - let mut compressor = FrameCompressor::new(CompressionLevel::Default); - compressor.set_source(payload.as_slice()); - let mut compressed = Vec::new(); - compressor.set_drain(&mut compressed); - compressor.compress(); - - let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); - let mut head = vec![0u8; 4096]; - let got = decoder.read(&mut head).unwrap(); - assert!(got > 0 && got <= head.len()); - - let mut out = Vec::new(); - out.extend_from_slice(&head[..got]); - decoder.read_to_end(&mut out).unwrap(); - assert_eq!(out, payload); - // Mid-frame entry → fallback path, never the direct route. - assert_eq!(decoder.decoder.direct_frames(), 0); - } - - /// `read_to_end` reads the WHOLE source to EOF: a stream of concatenated - /// frames must decode every frame, not just the first. (The fast path - /// buffers the whole source, so dropping the trailing frame would lose - /// data.) - #[cfg(feature = "std")] - #[test] - fn read_to_end_decodes_all_concatenated_frames() { - use crate::encoding::{CompressionLevel, compress_slice_to_vec}; - use alloc::vec::Vec; - - let a: Vec = (0..5000u32).map(|i| (i & 0xFF) as u8).collect(); - let b: Vec = (0..3000u32) - .map(|i| ((i.wrapping_mul(7)) & 0xFF) as u8) - .collect(); - let mut stream = compress_slice_to_vec(&a, CompressionLevel::Level(3)); - stream.extend_from_slice(&compress_slice_to_vec(&b, CompressionLevel::Level(3))); - - let mut decoder = StreamingDecoder::new(stream.as_slice()).unwrap(); - let mut out = Vec::new(); - decoder.read_to_end(&mut out).unwrap(); - - let mut expected = a.clone(); - expected.extend_from_slice(&b); - assert_eq!(out, expected); - // Both FCS-declared frames took the direct path. - assert_eq!(decoder.decoder.direct_frames(), 2); - } - - /// `read_to_end` after a partial `read` must STILL consume the source to - /// EOF across concatenated frames, not stop at the current frame's end. The - /// partial read forces the mid-frame fallback path; with two concatenated - /// frames the fallback must finish frame 1, then advance through frame 2. - #[cfg(feature = "std")] - #[test] - fn read_to_end_after_partial_read_decodes_all_concatenated_frames() { - use crate::encoding::{CompressionLevel, compress_slice_to_vec}; - use alloc::vec; - use alloc::vec::Vec; - - let a: Vec = (0..6000u32).map(|i| (i & 0xFF) as u8).collect(); - let b: Vec = (0..4000u32) - .map(|i| ((i.wrapping_mul(11)) & 0xFF) as u8) - .collect(); - let mut stream = compress_slice_to_vec(&a, CompressionLevel::Level(3)); - stream.extend_from_slice(&compress_slice_to_vec(&b, CompressionLevel::Level(3))); - - let mut decoder = StreamingDecoder::new(stream.as_slice()).unwrap(); - // Partial read of frame 1 → mid-frame, so read_to_end takes the fallback. - let mut head = vec![0u8; 2048]; - let got = decoder.read(&mut head).unwrap(); - assert!(got > 0 && got <= head.len()); - - let mut out = Vec::new(); - out.extend_from_slice(&head[..got]); - decoder.read_to_end(&mut out).unwrap(); - - let mut expected = a.clone(); - expected.extend_from_slice(&b); - assert_eq!( - out, expected, - "fallback path must decode frame 2 too, not stop at frame 1 EOF" - ); - } - - /// `read_to_end` on a stream of concatenated DICTIONARY frames must decode - /// every frame WITH the dictionary the decoder was constructed with. The - /// fast-path concatenated loop re-initialises following frames, and a plain - /// re-init resolves dictionaries by frame id only — losing the forced - /// dictionary for frames that omit (or can't resolve) the id. - #[cfg(feature = "std")] - #[test] - fn read_to_end_concatenated_dict_frames_decode_with_dictionary() { - use crate::encoding::{CompressionLevel, FrameCompressor}; - use alloc::vec::Vec; - - let dict_raw = include_bytes!("../../dict_tests/dictionary"); - let compress_with_dict = |payload: &[u8]| -> Vec { - let mut compressor = FrameCompressor::new(CompressionLevel::Default); - compressor - .set_dictionary_from_bytes(dict_raw) - .expect("dict load"); - compressor.set_source(payload); - let mut compressed = Vec::new(); - compressor.set_drain(&mut compressed); - compressor.compress(); - compressed - }; - - let a = b"first dictionary-compressed frame payload".to_vec(); - let b = b"second dictionary-compressed frame payload".to_vec(); - let mut stream = compress_with_dict(&a); - stream.extend_from_slice(&compress_with_dict(&b)); - - let mut decoder = - StreamingDecoder::new_with_dictionary_bytes(stream.as_slice(), dict_raw).unwrap(); - let mut out = Vec::new(); - decoder - .read_to_end(&mut out) - .expect("both dict frames must decode with the forced dictionary"); - - let mut expected = a.clone(); - expected.extend_from_slice(&b); - assert_eq!(out, expected); - } - - /// A direct-path decode error must NOT leave non-decoded bytes in `output`. - /// The fast path resizes `output` to the declared content size before - /// decoding; if decode fails, the enlarged (zeroed) tail must be truncated - /// away so callers never observe bytes that were never decoded. - #[cfg(feature = "std")] - #[test] - fn read_to_end_truncates_output_on_direct_decode_error() { - use crate::encoding::{CompressionLevel, FrameCompressor}; - use alloc::vec::Vec; - - let payload: Vec = (0..5000u32).map(|i| (i & 0xFF) as u8).collect(); - let mut compressor = FrameCompressor::new(CompressionLevel::Default); - compressor.set_source(payload.as_slice()); - let mut compressed = Vec::new(); - compressor.set_drain(&mut compressed); - compressor.compress(); - // Truncate the block bytes (the FCS-bearing header at the front stays - // intact) so the header parses but the direct-path block decode hits a - // premature end → error after `output` was already resized. - compressed.truncate(compressed.len() - 40); - - let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); - let mut out = b"SENTINEL".to_vec(); - let result = decoder.read_to_end(&mut out); - assert!(result.is_err(), "truncated block must fail the decode"); - assert_eq!( - out, b"SENTINEL", - "failed direct decode must not append non-decoded bytes to output" - ); - } - - /// The mid-frame fallback grows `output` by `MAX_BLOCK_SIZE` before each - /// `self.read`. When that read errors (truncated current frame), the grown - /// zero-filled tail must be truncated away before the error propagates, so - /// the caller never observes `MAX_BLOCK_SIZE` worth of bytes that were never - /// decoded. - #[cfg(feature = "std")] - #[test] - fn read_to_end_truncates_output_on_midframe_fallback_error() { - use crate::encoding::{CompressionLevel, CompressionParameters, FrameCompressor}; - use alloc::vec; - use alloc::vec::Vec; - - // Incompressible payload with a window (128 KiB) SMALLER than the input, - // so the frame holds several blocks and bytes become collectable while - // the frame is still mid-decode. Without a sub-input window the decoder - // retains the whole input until the frame finishes, and a partial read - // could only ever finish or error, never leave a truncated remainder for - // the fallback to trip on. - let payload: Vec = (0..320_000u32) - .map(|i| (i.wrapping_mul(2654435761) >> 24) as u8) - .collect(); - let params = CompressionParameters::builder(CompressionLevel::Default) - .window_log(17) - .build() - .expect("window_log within bounds"); - let mut compressor = FrameCompressor::new(CompressionLevel::Default); - compressor.set_parameters(¶ms); - compressor.set_source(payload.as_slice()); - let mut compressed = Vec::new(); - compressor.set_drain(&mut compressed); - compressor.compress(); - // Truncate the tail so the final block decode fails partway through. - compressed.truncate(compressed.len() - 40); - - let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); - // A partial `read` first: leaves the decoder mid-frame so `read_to_end` - // takes the grow-and-drain fallback (not the decode-in-place fast path). - let mut head = vec![0u8; 4096]; - let got = decoder.read(&mut head).unwrap(); - assert!(got > 0); - - let mut out = Vec::new(); - out.extend_from_slice(&head[..got]); - let result = decoder.read_to_end(&mut out); - assert!( - result.is_err(), - "truncated current frame must fail the decode" - ); - assert!( - out.len() <= payload.len(), - "failed fallback read must not leave a zero-filled tail (len {} > payload {})", - out.len(), - payload.len() - ); - assert_eq!( - out.as_slice(), - &payload[..out.len()], - "decoded prefix must match the payload, with no appended non-decoded bytes" - ); - } - - /// An empty (`Frame_Content_Size = 0`) frame decodes to nothing through the - /// `read_to_end` fast path — the declared-size validation accepts the valid - /// case (produced == 0) instead of erroring. - #[cfg(feature = "std")] - #[test] - fn read_to_end_empty_frame_decodes_to_empty() { - use crate::encoding::{CompressionLevel, compress_slice_to_vec}; - use alloc::vec::Vec; - - let compressed = compress_slice_to_vec(&[], CompressionLevel::Level(3)); - let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); - let mut out = Vec::new(); - decoder.read_to_end(&mut out).unwrap(); - assert!(out.is_empty()); - } -} +mod tests; diff --git a/zstd/src/decoding/streaming_decoder/tests.rs b/zstd/src/decoding/streaming_decoder/tests.rs new file mode 100644 index 000000000..e4ee22dc1 --- /dev/null +++ b/zstd/src/decoding/streaming_decoder/tests.rs @@ -0,0 +1,325 @@ +use super::StreamingDecoder; +use crate::io::Read; + +/// `Read::read` must not return `Err` after it has already written bytes +/// into the caller's buffer (the trait mandates that an error implies no +/// bytes were read). When a single `read` call both drains the final bytes +/// of a `Verify`-mode frame AND finishes it, a checksum mismatch must be +/// deferred: those bytes are delivered as `Ok(n)` and the error surfaces on +/// the next (zero-byte) call, where returning `Err` violates no contract. +#[cfg(feature = "hash")] +#[test] +fn read_delivering_bytes_defers_checksum_error_to_next_call() { + use crate::decoding::ContentChecksum; + use crate::encoding::{CompressionLevel, FrameCompressor}; + use crate::io::ErrorKind; + use alloc::vec; + use alloc::vec::Vec; + + let payload: Vec = (0..8192u32).map(|i| (i & 0xFF) as u8).collect(); + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + // Checksum is the subject under test; the encoder default is off + // (upstream library parity). + compressor.set_content_checksum(true); + compressor.set_source(payload.as_slice()); + let mut compressed = Vec::new(); + compressor.set_drain(&mut compressed); + compressor.compress(); + + // Corrupt the trailing 4-byte content checksum: the body still decodes + // to the right bytes, but the stored digest no longer matches. + let last = compressed.len() - 1; + compressed[last] ^= 0xFF; + + let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); + decoder + .decoder + .set_content_checksum(ContentChecksum::Verify); + + // A buffer large enough to drain the whole frame in one call: this call + // finishes the frame AND writes every payload byte. The mismatch must + // NOT abort it (that would drop the delivered bytes). + let mut buf = vec![0u8; payload.len() + 4096]; + let n = decoder + .read(&mut buf) + .expect("a read that delivered bytes must not return the checksum Err"); + assert_eq!(n, payload.len()); + assert_eq!(&buf[..n], payload.as_slice()); + + // The deferred mismatch surfaces on the terminating zero-byte read. + let err = decoder + .read(&mut buf) + .expect_err("deferred checksum mismatch must surface on the terminating read"); + assert_eq!(err.kind(), ErrorKind::Other); +} + +/// A fresh `read_to_end` must take the single-copy decode-in-place path +/// (FCS-declared frame decoded straight into the output `Vec`, no ring +/// drain) AND reproduce the payload byte-for-byte. +#[cfg(feature = "std")] +#[test] +fn read_to_end_decode_in_place_matches_and_takes_direct_path() { + use crate::encoding::{CompressionLevel, FrameCompressor}; + use alloc::vec::Vec; + + let payload: Vec = (0..20_000u32) + .map(|i| (i.wrapping_mul(2654435761) >> 24) as u8) + .collect(); + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_source(payload.as_slice()); + let mut compressed = Vec::new(); + compressor.set_drain(&mut compressed); + compressor.compress(); + + let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); + let mut out = Vec::new(); + let n = decoder.read_to_end(&mut out).unwrap(); + assert_eq!(n, payload.len()); + assert_eq!(out, payload); + // FrameCompressor declares FCS, so the fresh fast path used the direct + // (decode-in-place) route, not the ring drain. + assert_eq!(decoder.decoder.direct_frames(), 1); +} + +/// `read_to_end` after a partial `read` must still produce the full +/// payload. The decoder is mid-frame, so the fast path is skipped and the +/// generic grow-and-drain fallback runs (no direct frame). +#[cfg(feature = "std")] +#[test] +fn read_to_end_after_partial_read_is_complete() { + use crate::encoding::{CompressionLevel, FrameCompressor}; + use alloc::vec; + use alloc::vec::Vec; + + let payload: Vec = (0..20_000u32).map(|i| (i & 0xFF) as u8).collect(); + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_source(payload.as_slice()); + let mut compressed = Vec::new(); + compressor.set_drain(&mut compressed); + compressor.compress(); + + let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); + let mut head = vec![0u8; 4096]; + let got = decoder.read(&mut head).unwrap(); + assert!(got > 0 && got <= head.len()); + + let mut out = Vec::new(); + out.extend_from_slice(&head[..got]); + decoder.read_to_end(&mut out).unwrap(); + assert_eq!(out, payload); + // Mid-frame entry → fallback path, never the direct route. + assert_eq!(decoder.decoder.direct_frames(), 0); +} + +/// `read_to_end` reads the WHOLE source to EOF: a stream of concatenated +/// frames must decode every frame, not just the first. (The fast path +/// buffers the whole source, so dropping the trailing frame would lose +/// data.) +#[cfg(feature = "std")] +#[test] +fn read_to_end_decodes_all_concatenated_frames() { + use crate::encoding::{CompressionLevel, compress_slice_to_vec}; + use alloc::vec::Vec; + + let a: Vec = (0..5000u32).map(|i| (i & 0xFF) as u8).collect(); + let b: Vec = (0..3000u32) + .map(|i| ((i.wrapping_mul(7)) & 0xFF) as u8) + .collect(); + let mut stream = compress_slice_to_vec(&a, CompressionLevel::Level(3)); + stream.extend_from_slice(&compress_slice_to_vec(&b, CompressionLevel::Level(3))); + + let mut decoder = StreamingDecoder::new(stream.as_slice()).unwrap(); + let mut out = Vec::new(); + decoder.read_to_end(&mut out).unwrap(); + + let mut expected = a.clone(); + expected.extend_from_slice(&b); + assert_eq!(out, expected); + // Both FCS-declared frames took the direct path. + assert_eq!(decoder.decoder.direct_frames(), 2); +} + +/// `read_to_end` after a partial `read` must STILL consume the source to +/// EOF across concatenated frames, not stop at the current frame's end. The +/// partial read forces the mid-frame fallback path; with two concatenated +/// frames the fallback must finish frame 1, then advance through frame 2. +#[cfg(feature = "std")] +#[test] +fn read_to_end_after_partial_read_decodes_all_concatenated_frames() { + use crate::encoding::{CompressionLevel, compress_slice_to_vec}; + use alloc::vec; + use alloc::vec::Vec; + + let a: Vec = (0..6000u32).map(|i| (i & 0xFF) as u8).collect(); + let b: Vec = (0..4000u32) + .map(|i| ((i.wrapping_mul(11)) & 0xFF) as u8) + .collect(); + let mut stream = compress_slice_to_vec(&a, CompressionLevel::Level(3)); + stream.extend_from_slice(&compress_slice_to_vec(&b, CompressionLevel::Level(3))); + + let mut decoder = StreamingDecoder::new(stream.as_slice()).unwrap(); + // Partial read of frame 1 → mid-frame, so read_to_end takes the fallback. + let mut head = vec![0u8; 2048]; + let got = decoder.read(&mut head).unwrap(); + assert!(got > 0 && got <= head.len()); + + let mut out = Vec::new(); + out.extend_from_slice(&head[..got]); + decoder.read_to_end(&mut out).unwrap(); + + let mut expected = a.clone(); + expected.extend_from_slice(&b); + assert_eq!( + out, expected, + "fallback path must decode frame 2 too, not stop at frame 1 EOF" + ); +} + +/// `read_to_end` on a stream of concatenated DICTIONARY frames must decode +/// every frame WITH the dictionary the decoder was constructed with. The +/// fast-path concatenated loop re-initialises following frames, and a plain +/// re-init resolves dictionaries by frame id only — losing the forced +/// dictionary for frames that omit (or can't resolve) the id. +#[cfg(feature = "std")] +#[test] +fn read_to_end_concatenated_dict_frames_decode_with_dictionary() { + use crate::encoding::{CompressionLevel, FrameCompressor}; + use alloc::vec::Vec; + + let dict_raw = include_bytes!("../../../dict_tests/dictionary"); + let compress_with_dict = |payload: &[u8]| -> Vec { + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor + .set_dictionary_from_bytes(dict_raw) + .expect("dict load"); + compressor.set_source(payload); + let mut compressed = Vec::new(); + compressor.set_drain(&mut compressed); + compressor.compress(); + compressed + }; + + let a = b"first dictionary-compressed frame payload".to_vec(); + let b = b"second dictionary-compressed frame payload".to_vec(); + let mut stream = compress_with_dict(&a); + stream.extend_from_slice(&compress_with_dict(&b)); + + let mut decoder = + StreamingDecoder::new_with_dictionary_bytes(stream.as_slice(), dict_raw).unwrap(); + let mut out = Vec::new(); + decoder + .read_to_end(&mut out) + .expect("both dict frames must decode with the forced dictionary"); + + let mut expected = a.clone(); + expected.extend_from_slice(&b); + assert_eq!(out, expected); +} + +/// A direct-path decode error must NOT leave non-decoded bytes in `output`. +/// The fast path resizes `output` to the declared content size before +/// decoding; if decode fails, the enlarged (zeroed) tail must be truncated +/// away so callers never observe bytes that were never decoded. +#[cfg(feature = "std")] +#[test] +fn read_to_end_truncates_output_on_direct_decode_error() { + use crate::encoding::{CompressionLevel, FrameCompressor}; + use alloc::vec::Vec; + + let payload: Vec = (0..5000u32).map(|i| (i & 0xFF) as u8).collect(); + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_source(payload.as_slice()); + let mut compressed = Vec::new(); + compressor.set_drain(&mut compressed); + compressor.compress(); + // Truncate the block bytes (the FCS-bearing header at the front stays + // intact) so the header parses but the direct-path block decode hits a + // premature end → error after `output` was already resized. + compressed.truncate(compressed.len() - 40); + + let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); + let mut out = b"SENTINEL".to_vec(); + let result = decoder.read_to_end(&mut out); + assert!(result.is_err(), "truncated block must fail the decode"); + assert_eq!( + out, b"SENTINEL", + "failed direct decode must not append non-decoded bytes to output" + ); +} + +/// The mid-frame fallback grows `output` by `MAX_BLOCK_SIZE` before each +/// `self.read`. When that read errors (truncated current frame), the grown +/// zero-filled tail must be truncated away before the error propagates, so +/// the caller never observes `MAX_BLOCK_SIZE` worth of bytes that were never +/// decoded. +#[cfg(feature = "std")] +#[test] +fn read_to_end_truncates_output_on_midframe_fallback_error() { + use crate::encoding::{CompressionLevel, CompressionParameters, FrameCompressor}; + use alloc::vec; + use alloc::vec::Vec; + + // Incompressible payload with a window (128 KiB) SMALLER than the input, + // so the frame holds several blocks and bytes become collectable while + // the frame is still mid-decode. Without a sub-input window the decoder + // retains the whole input until the frame finishes, and a partial read + // could only ever finish or error, never leave a truncated remainder for + // the fallback to trip on. + let payload: Vec = (0..320_000u32) + .map(|i| (i.wrapping_mul(2654435761) >> 24) as u8) + .collect(); + let params = CompressionParameters::builder(CompressionLevel::Default) + .window_log(17) + .build() + .expect("window_log within bounds"); + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_parameters(¶ms); + compressor.set_source(payload.as_slice()); + let mut compressed = Vec::new(); + compressor.set_drain(&mut compressed); + compressor.compress(); + // Truncate the tail so the final block decode fails partway through. + compressed.truncate(compressed.len() - 40); + + let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); + // A partial `read` first: leaves the decoder mid-frame so `read_to_end` + // takes the grow-and-drain fallback (not the decode-in-place fast path). + let mut head = vec![0u8; 4096]; + let got = decoder.read(&mut head).unwrap(); + assert!(got > 0); + + let mut out = Vec::new(); + out.extend_from_slice(&head[..got]); + let result = decoder.read_to_end(&mut out); + assert!( + result.is_err(), + "truncated current frame must fail the decode" + ); + assert!( + out.len() <= payload.len(), + "failed fallback read must not leave a zero-filled tail (len {} > payload {})", + out.len(), + payload.len() + ); + assert_eq!( + out.as_slice(), + &payload[..out.len()], + "decoded prefix must match the payload, with no appended non-decoded bytes" + ); +} + +/// An empty (`Frame_Content_Size = 0`) frame decodes to nothing through the +/// `read_to_end` fast path — the declared-size validation accepts the valid +/// case (produced == 0) instead of erroring. +#[cfg(feature = "std")] +#[test] +fn read_to_end_empty_frame_decodes_to_empty() { + use crate::encoding::{CompressionLevel, compress_slice_to_vec}; + use alloc::vec::Vec; + + let compressed = compress_slice_to_vec(&[], CompressionLevel::Level(3)); + let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); + let mut out = Vec::new(); + decoder.read_to_end(&mut out).unwrap(); + assert!(out.is_empty()); +} diff --git a/zstd/src/decoding/user_slice_buf.rs b/zstd/src/decoding/user_slice_buf.rs index 500421eca..d644c9d75 100644 --- a/zstd/src/decoding/user_slice_buf.rs +++ b/zstd/src/decoding/user_slice_buf.rs @@ -1015,452 +1015,4 @@ const _: () = { }; #[cfg(test)] -mod tests { - extern crate alloc; - use super::*; - use alloc::vec; - use alloc::vec::Vec; - - #[test] - fn extend_writes_at_tail() { - let mut buf = vec![0u8; 32]; - let mut b = UserSliceBackend::from_slice(&mut buf); - b.extend(&[1, 2, 3, 4]); - assert_eq!(b.len(), 4); - assert_eq!(b.tail(), 4); - b.extend(&[5, 6]); - let (s, t) = b.as_slices(); - assert_eq!(s, &[1, 2, 3, 4, 5, 6]); - assert!(t.is_empty()); - } - - #[test] - fn extend_and_fill_repeats_byte() { - let mut buf = vec![0u8; 16]; - let mut b = UserSliceBackend::from_slice(&mut buf); - b.extend(&[0xAA]); - b.extend_and_fill(0xBB, 4); - let (s, _) = b.as_slices(); - assert_eq!(s, &[0xAA, 0xBB, 0xBB, 0xBB, 0xBB]); - } - - #[test] - fn extend_from_within_unchecked_copies_non_overlapping() { - let mut buf = vec![0u8; 32]; - let mut b = UserSliceBackend::from_slice(&mut buf); - b.extend(&[10, 20, 30, 40, 50]); - // SAFETY: 0+3 <= 5 = len; cap 32 covers 5+3. - unsafe { b.extend_from_within_unchecked(0, 3) }; - let (s, _) = b.as_slices(); - assert_eq!(s, &[10, 20, 30, 40, 50, 10, 20, 30]); - } - - #[test] - fn drop_first_n_advances_head_keeps_history() { - let mut buf = vec![0u8; 32]; - let mut b = UserSliceBackend::from_slice(&mut buf); - b.extend(&[1, 2, 3, 4, 5]); - b.drop_first_n(2); - assert_eq!(b.len(), 3); - let (s, _) = b.as_slices(); - assert_eq!(s, &[3, 4, 5]); - // After drop, drained bytes remain physically present and can - // back a match copy via `start` indexed from the post-drop head. - unsafe { b.extend_from_within_unchecked(0, 3) }; - let (s, _) = b.as_slices(); - assert_eq!(s, &[3, 4, 5, 3, 4, 5]); - } - - #[test] - fn set_tail_rollback() { - let mut buf = vec![0u8; 32]; - let mut b = UserSliceBackend::from_slice(&mut buf); - b.extend(&[1, 2, 3]); - let saved = b.tail(); - b.extend(&[4, 5, 6, 7]); - assert_eq!(b.len(), 7); - unsafe { b.set_tail(saved) }; - assert_eq!(b.len(), 3); - let (s, _) = b.as_slices(); - assert_eq!(s, &[1, 2, 3]); - } - - #[test] - fn clear_resets_cursors() { - let mut buf = vec![0u8; 32]; - let mut b = UserSliceBackend::from_slice(&mut buf); - b.extend(&[1, 2, 3]); - b.drop_first_n(1); - b.clear(); - assert_eq!(b.len(), 0); - assert_eq!(b.tail(), 0); - } - - #[test] - fn extend_from_reader_into_slice() { - let mut buf = vec![0u8; 16]; - let mut b = UserSliceBackend::from_slice(&mut buf); - let src = [9u8, 8, 7, 6, 5]; - b.extend_from_reader(&src[..], 5).unwrap(); - let (s, _) = b.as_slices(); - assert_eq!(s, &[9, 8, 7, 6, 5]); - } - - #[test] - fn extend_from_reader_over_capacity_errors() { - let mut buf = vec![0u8; 4]; - let mut b = UserSliceBackend::from_slice(&mut buf); - let src = [9u8, 8, 7, 6, 5]; - // 5 bytes requested, only 4 cap -> error, tail unchanged. - assert!(b.extend_from_reader(&src[..], 5).is_err()); - assert_eq!(b.tail(), 0); - } - - // Coverage for the fallible try_* surface. Exercises: - // - happy paths (exact-fit + room to spare), - // - capacity-overflow paths (returns Err with diagnostic fields), - // - integer-overflow wrap-guards (checked_add ok_or branch). - - use super::super::buffer_backend::BufferBackend; - - #[test] - fn try_extend_exact_fit_succeeds_and_advances_tail() { - let mut buf = vec![0u8; 4]; - let mut b = UserSliceBackend::from_slice(&mut buf); - assert!(b.try_extend(&[1, 2, 3, 4]).is_ok()); - assert_eq!(b.tail(), 4); - let (s, _) = b.as_slices(); - assert_eq!(s, &[1, 2, 3, 4]); - } - - #[test] - fn try_extend_over_capacity_returns_overflow_and_keeps_tail() { - let mut buf = vec![0u8; 4]; - let mut b = UserSliceBackend::from_slice(&mut buf); - let err = b.try_extend(&[1, 2, 3, 4, 5]).unwrap_err(); - assert_eq!(err.tail, 0); - assert_eq!(err.requested, 5); - assert_eq!(err.capacity, 4); - assert_eq!(b.tail(), 0); - } - - #[test] - fn try_extend_partially_full_overshoot_reports_current_tail() { - let mut buf = vec![0u8; 4]; - let mut b = UserSliceBackend::from_slice(&mut buf); - b.extend(&[1, 2]); - // 3 more bytes would need 5 total, capacity is 4. - let err = b.try_extend(&[3, 4, 5]).unwrap_err(); - assert_eq!(err.tail, 2); - assert_eq!(err.requested, 3); - assert_eq!(err.capacity, 4); - assert_eq!(b.tail(), 2); - } - - #[test] - fn try_extend_zero_length_succeeds_and_leaves_tail_unchanged() { - // The `checked_add(tail, len)` wrap branch in `try_extend` is - // a defense-in-depth guard for corrupted input that names - // `regenerated_size` near `usize::MAX`. Constructing such a - // `&[u8]` from safe Rust is not expressible — `from_raw_parts` - // with a forged length is UB. The wrap branch is exercised - // only from the fuzz harness under `feature = "fuzz_exports"` - // (which routes a controlled `len` through `try_*`) and from - // the real malformed-frame decode path that the harness - // emulates. - // - // This test covers the adjacent normal case — a zero-length - // `try_extend` MUST succeed regardless of current `tail` - // (the new_tail = tail + 0 = tail comparison both passes - // checked_add and the upper-bound check). Without this case - // the early-return on `len == 0` could regress silently. - let mut buf = vec![0u8; 8]; - let mut b = UserSliceBackend::from_slice(&mut buf); - assert!(b.try_extend(&[]).is_ok()); - assert_eq!(b.tail(), 0); - b.extend(&[1, 2, 3]); - assert!(b.try_extend(&[]).is_ok()); - assert_eq!(b.tail(), 3); - } - - #[test] - fn try_extend_and_fill_exact_fit_writes_pattern() { - let mut buf = vec![0u8; 4]; - let mut b = UserSliceBackend::from_slice(&mut buf); - assert!(b.try_extend_and_fill(0xAB, 4).is_ok()); - assert_eq!(b.tail(), 4); - let (s, _) = b.as_slices(); - assert_eq!(s, &[0xAB, 0xAB, 0xAB, 0xAB]); - } - - #[test] - fn try_extend_and_fill_over_capacity_returns_overflow() { - let mut buf = vec![0u8; 4]; - let mut b = UserSliceBackend::from_slice(&mut buf); - b.extend(&[1, 2]); - let err = b.try_extend_and_fill(0xCD, 5).unwrap_err(); - assert_eq!(err.tail, 2); - assert_eq!(err.requested, 5); - assert_eq!(err.capacity, 4); - assert_eq!(b.tail(), 2); - } - - #[test] - fn try_extend_from_within_within_bounds_repeats_history() { - let mut buf = vec![0u8; 8]; - let mut b = UserSliceBackend::from_slice(&mut buf); - b.extend(&[1, 2, 3]); - // Repeat the first 3 bytes from history into the next 3 slots. - assert!(b.try_extend_from_within(0, 3).is_ok()); - let (s, _) = b.as_slices(); - assert_eq!(s, &[1, 2, 3, 1, 2, 3]); - assert_eq!(b.tail(), 6); - } - - #[test] - fn try_extend_from_within_source_past_tail_returns_overflow() { - let mut buf = vec![0u8; 8]; - let mut b = UserSliceBackend::from_slice(&mut buf); - b.extend(&[1, 2]); - // start=0, len=5 — source range needs bytes 0..5 but tail=2. - let err = b.try_extend_from_within(0, 5).unwrap_err(); - assert_eq!(err.tail, 2); - assert_eq!(err.requested, 5); - assert_eq!(b.tail(), 2); - } - - #[test] - fn try_extend_from_within_destination_overflow_returns_err() { - let mut buf = vec![0u8; 4]; - let mut b = UserSliceBackend::from_slice(&mut buf); - b.extend(&[1, 2, 3]); - // Source 0..2 valid, but writing 2 more bytes would push tail - // from 3 to 5, past capacity 4. - let err = b.try_extend_from_within(0, 2).unwrap_err(); - assert_eq!(err.tail, 3); - assert_eq!(err.requested, 2); - assert_eq!(err.capacity, 4); - assert_eq!(b.tail(), 3); - } - - /// Direct tests for `exec_sequence_inline` — exercise the - /// x86_64 inline body so coverage attributes its 40 lines to - /// these tests, not through the deep `decode_all` - /// pipeline where `cargo llvm-cov` sometimes loses the inlined - /// callee. Tests cover: short-literal + short-match, long - /// literal (wildcopy tail), short-offset match (overlapCopy8 + - /// 8-byte stride), long-offset match (wildcopy_no_overlap). - #[test] - fn exec_sequence_inline_overflow_returns_output_buffer_overflow() { - // #246: the upstream zstd-inline sequence executor must return - // `ExecuteSequencesError::OutputBufferOverflow` (NOT panic, NOT - // an out-of-bounds unsafe write) when a sequence's literal+match - // copy plus wildcopy overshoot would land past the fixed slice. - // This is the per-sequence guard that keeps the unsafe write - // surface inside the user slice on the way to the post-block FCS - // check; the higher-level acceptance test - // (`decode_all_compressed_block_fcs_overflow_returns_structured_error`) - // proves the whole chain folds into `FrameContentSizeMismatch`. - use super::super::errors::ExecuteSequencesError; - // Tiny slice: 16-byte capacity, tail already at 8. A sequence - // writing 8 literals + 8 match (16 bytes) plus the 15-byte - // wildcopy overshoot cannot fit in `16 - 8 = 8` remaining bytes. - let mut buf = vec![0u8; 16]; - for (i, slot) in buf.iter_mut().take(8).enumerate() { - *slot = i as u8; - } - let mut b = UserSliceBackend::from_slice(&mut buf); - b.tail = 8; - let lits = [0xAAu8; 16]; - // SAFETY: `lits` is a 16-byte parent buffer (upstream zstd 16-byte read - // slack satisfied); the call must return Err before any write - // past the slice end. - let err = unsafe { b.exec_sequence_inline(lits.as_ptr(), 8, 4, 8) } - .expect_err("overshoot must return OutputBufferOverflow"); - assert!( - matches!(err, ExecuteSequencesError::OutputBufferOverflow { .. }), - "expected OutputBufferOverflow, got {err:?}" - ); - // Backend left untouched on Err — tail not advanced. - assert_eq!(b.tail, 8, "tail must not advance on overflow"); - } - - #[test] - fn exec_sequence_inline_short_literal_plus_long_offset_match() { - // Layout: pre-fill `tail = 8` with a "history" region so - // a match copy at offset 16 reaches inside the slice. Then - // bump tail past that history and exercise upstream zstd_exec. - // Buffer sized with WILDCOPY_OVERLENGTH slack at the end. - const WILDCOPY: usize = super::super::buffer_backend::WILDCOPY_OVERLENGTH; - let mut buf = vec![0u8; 256 + WILDCOPY]; - // Seed history: bytes 0..32 = ascending values, so a later - // match at offset 16 picks up bytes 16..32. - for (i, slot) in buf.iter_mut().take(32).enumerate() { - *slot = i as u8; - } - let mut b = UserSliceBackend::from_slice(&mut buf); - b.tail = 32; // Pretend 32 history bytes are already written. - - // 8-byte literals to write (upstream zstd's litLength <= 16 fast - // path — no wildcopy tail). Match length 8 at offset 16. - let lits: [u8; 16] = [ - 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, 0x11, 0x22, 0xA1, 0xB1, 0xC1, 0xD1, 0xE1, 0xF1, - 0x10, 0x20, - ]; - unsafe { - b.exec_sequence_inline(lits.as_ptr(), 8, 16, 8).unwrap(); - } - // tail advanced by 8 lit + 8 match = 16. - assert_eq!(b.tail, 48); - // Literals landed at tail = 32..40. - assert_eq!(&buf[32..40], &lits[..8]); - // Match: at output position 40..48, source = tail + lit_len - - // offset = 32 + 8 - 16 = 24. So buf[40..48] == buf[24..32] - // (history bytes 24..32 = [24, 25, 26, 27, 28, 29, 30, 31]). - assert_eq!(&buf[40..48], &[24u8, 25, 26, 27, 28, 29, 30, 31]); - } - - #[test] - fn exec_sequence_inline_long_literal_uses_wildcopy_tail() { - // litLength > 16 path: unconditional copy16 + wildcopy tail - // for the remaining literal bytes. - const WILDCOPY: usize = super::super::buffer_backend::WILDCOPY_OVERLENGTH; - let mut buf = vec![0u8; 256 + WILDCOPY]; - for (i, slot) in buf.iter_mut().take(32).enumerate() { - *slot = i as u8; - } - let mut b = UserSliceBackend::from_slice(&mut buf); - b.tail = 32; - - // 40-byte literals (forces wildcopy tail) — needs 40-byte - // source buffer with extra read slack for the final - // 16-byte load. - let lits: Vec = (0..40u8 + 16).map(|i| 0x80 + i).collect(); - unsafe { - b.exec_sequence_inline(lits.as_ptr(), 40, 16, 8).unwrap(); - } - assert_eq!(b.tail, 80); - assert_eq!(&buf[32..72], &lits[..40]); - // Match at offset 16: src = 32 + 40 - 16 = 56. buf[72..80] - // == buf[56..64] (which we just wrote = lits[24..32]). - assert_eq!(&buf[72..80], &lits[24..32]); - } - - #[test] - fn exec_sequence_inline_short_offset_match_uses_overlap_copy() { - // offset < 16 takes the overlapCopy8 + 8-byte stride path - // (vs. the offset >= 16 wildcopy_no_overlap path). - // - // What we actually assert here: - // 1. `tail` advances by `lit_length + match_length`. - // 2. The literal payload lands at `buf[tail..tail+ll]`. - // 3. The FIRST 4 match-output bytes match seed[4..8] — - // that prefix of the match copy reads source bytes - // that the literal `copy16` overshoot did NOT - // overwrite (the upstream zstd `copy16` writes 16 bytes at - // `tail`, so source bytes BEFORE `tail` survive). - // - // We do NOT cross-validate against the legacy `extend` + - // `repeat_in_chunks` chain: those paths don't perform the - // 16-byte literal overshoot, so they produce a different - // output for the same logical sequence (different bytes in - // positions where the upstream zstd's overshoot is consumed by the - // match copy). End-to-end parity is covered by the higher - // level `roundtrip_integrity::*` tests in lib.rs which - // decode whole frames and compare to the encoder input. - const WILDCOPY: usize = super::super::buffer_backend::WILDCOPY_OVERLENGTH; - let mut buf = vec![0u8; 256 + WILDCOPY]; - // Seed last 8 bytes of history with a recognisable pattern. - let seed = [0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7]; - buf[24..32].copy_from_slice(&seed); - let mut b = UserSliceBackend::from_slice(&mut buf); - b.tail = 32; - - let lits: [u8; 16] = [0xFF; 16]; - // litLength=4, offset=8, matchLength=12. offset<16 → short - // path (overlapCopy8 + 8-byte stride). - unsafe { - b.exec_sequence_inline(lits.as_ptr(), 4, 8, 12).unwrap(); - } - // tail = 32 + 4 + 12 = 48. - assert_eq!(b.tail, 48); - // Literal copy: bytes 32..36 are the literal payload. - assert_eq!(&buf[32..36], &lits[..4]); - // Match copy: the first 4 output bytes (positions 36..40) - // are the FIRST 4 source bytes (positions 28..32), which the - // literal copy16 has NOT overwritten (it wrote at 32..48, so - // 28..32 remain as the seed tail). Verify those. - assert_eq!(&buf[36..40], &seed[4..8]); - // The remaining 8 match bytes (40..48) get fed by the - // 8-byte-stride wildcopy reading from positions inside the - // match-destination region, which the literal copy16 already - // overwrote with 0xFF. That's the upstream zstd invariant — the - // overshoot is consumed correctly. We don't pin the exact - // bytes (they're a function of overlap_copy8's spread - // tables) but the output length must be right. - } - - /// `exec_sequence_inline_avx2` correctness across the - /// match-copy offset cases. Issue #279 round 4 regression: - /// the AVX2 32-byte ymm wildcopy threshold MUST be `offset >= 32` - /// (not >= 16). At offset 16..31 a 32-byte load from `match_src` - /// reads bytes inside the destination region BEFORE the first - /// store has written them; SSE2 16-byte fallback covers those - /// offsets safely. Test exercises: - /// - offset 20 (mid-range, must route to SSE2 fallback) - /// - offset 32 (boundary, AVX2 path) - /// - offset 64 (deep AVX2 path) - /// - /// against a byte-by-byte reference of the same repeat semantics. - // `std` feature gates the test: `is_x86_feature_detected!` is `std`-only - // (runtime CPU detection), unavailable in the crate's `#![no_std]` build. - #[cfg(all(target_arch = "x86_64", feature = "std"))] - #[test] - fn exec_sequence_inline_avx2_offset_boundary_correctness() { - if !std::arch::is_x86_feature_detected!("avx2") { - return; - } - const WILDCOPY: usize = super::super::buffer_backend::WILDCOPY_OVERLENGTH; - - for offset in [20usize, 32, 64] { - let mut buf = vec![0u8; 512 + WILDCOPY]; - // Seed bytes 0..200 with a deterministic, byte-position-derived - // pattern so the reference match-copy is unambiguous. - for (i, slot) in buf.iter_mut().take(200).enumerate() { - *slot = ((i * 31 + 7) & 0xFF) as u8; - } - // Compute reference: byte-by-byte repeat from offset 100..(100+offset) - // up to match_length bytes, written at position 200. - let match_length = 96usize; - let mut reference = buf.clone(); - for i in 0..match_length { - reference[200 + i] = reference[200 + i - offset]; - } - - // Now run the AVX2 inline executor on the actual buffer. - let mut b = UserSliceBackend::from_slice(&mut buf); - b.tail = 200; - let dummy_lits: [u8; 16] = [0xAA; 16]; - unsafe { - b.exec_sequence_inline_avx2(dummy_lits.as_ptr(), 0, offset, match_length) - .unwrap(); - } - assert_eq!(b.tail, 200 + match_length); - - // Compare match-copy region against the reference. - // Bytes 200..(200+match_length) must match the byte-by-byte - // repeat. The overshoot region past match_length is not pinned. - for i in 0..match_length { - assert_eq!( - buf[200 + i], - reference[200 + i], - "exec_sequence_inline_avx2 offset={offset} byte {i}: \ - got {:#x}, expected {:#x} (regression: AVX2 wildcopy \ - reading past first-store boundary)", - buf[200 + i], - reference[200 + i], - ); - } - } - } -} +mod tests; diff --git a/zstd/src/decoding/user_slice_buf/tests.rs b/zstd/src/decoding/user_slice_buf/tests.rs new file mode 100644 index 000000000..b6570426a --- /dev/null +++ b/zstd/src/decoding/user_slice_buf/tests.rs @@ -0,0 +1,447 @@ +extern crate alloc; +use super::*; +use alloc::vec; +use alloc::vec::Vec; + +#[test] +fn extend_writes_at_tail() { + let mut buf = vec![0u8; 32]; + let mut b = UserSliceBackend::from_slice(&mut buf); + b.extend(&[1, 2, 3, 4]); + assert_eq!(b.len(), 4); + assert_eq!(b.tail(), 4); + b.extend(&[5, 6]); + let (s, t) = b.as_slices(); + assert_eq!(s, &[1, 2, 3, 4, 5, 6]); + assert!(t.is_empty()); +} + +#[test] +fn extend_and_fill_repeats_byte() { + let mut buf = vec![0u8; 16]; + let mut b = UserSliceBackend::from_slice(&mut buf); + b.extend(&[0xAA]); + b.extend_and_fill(0xBB, 4); + let (s, _) = b.as_slices(); + assert_eq!(s, &[0xAA, 0xBB, 0xBB, 0xBB, 0xBB]); +} + +#[test] +fn extend_from_within_unchecked_copies_non_overlapping() { + let mut buf = vec![0u8; 32]; + let mut b = UserSliceBackend::from_slice(&mut buf); + b.extend(&[10, 20, 30, 40, 50]); + // SAFETY: 0+3 <= 5 = len; cap 32 covers 5+3. + unsafe { b.extend_from_within_unchecked(0, 3) }; + let (s, _) = b.as_slices(); + assert_eq!(s, &[10, 20, 30, 40, 50, 10, 20, 30]); +} + +#[test] +fn drop_first_n_advances_head_keeps_history() { + let mut buf = vec![0u8; 32]; + let mut b = UserSliceBackend::from_slice(&mut buf); + b.extend(&[1, 2, 3, 4, 5]); + b.drop_first_n(2); + assert_eq!(b.len(), 3); + let (s, _) = b.as_slices(); + assert_eq!(s, &[3, 4, 5]); + // After drop, drained bytes remain physically present and can + // back a match copy via `start` indexed from the post-drop head. + unsafe { b.extend_from_within_unchecked(0, 3) }; + let (s, _) = b.as_slices(); + assert_eq!(s, &[3, 4, 5, 3, 4, 5]); +} + +#[test] +fn set_tail_rollback() { + let mut buf = vec![0u8; 32]; + let mut b = UserSliceBackend::from_slice(&mut buf); + b.extend(&[1, 2, 3]); + let saved = b.tail(); + b.extend(&[4, 5, 6, 7]); + assert_eq!(b.len(), 7); + unsafe { b.set_tail(saved) }; + assert_eq!(b.len(), 3); + let (s, _) = b.as_slices(); + assert_eq!(s, &[1, 2, 3]); +} + +#[test] +fn clear_resets_cursors() { + let mut buf = vec![0u8; 32]; + let mut b = UserSliceBackend::from_slice(&mut buf); + b.extend(&[1, 2, 3]); + b.drop_first_n(1); + b.clear(); + assert_eq!(b.len(), 0); + assert_eq!(b.tail(), 0); +} + +#[test] +fn extend_from_reader_into_slice() { + let mut buf = vec![0u8; 16]; + let mut b = UserSliceBackend::from_slice(&mut buf); + let src = [9u8, 8, 7, 6, 5]; + b.extend_from_reader(&src[..], 5).unwrap(); + let (s, _) = b.as_slices(); + assert_eq!(s, &[9, 8, 7, 6, 5]); +} + +#[test] +fn extend_from_reader_over_capacity_errors() { + let mut buf = vec![0u8; 4]; + let mut b = UserSliceBackend::from_slice(&mut buf); + let src = [9u8, 8, 7, 6, 5]; + // 5 bytes requested, only 4 cap -> error, tail unchanged. + assert!(b.extend_from_reader(&src[..], 5).is_err()); + assert_eq!(b.tail(), 0); +} + +// Coverage for the fallible try_* surface. Exercises: +// - happy paths (exact-fit + room to spare), +// - capacity-overflow paths (returns Err with diagnostic fields), +// - integer-overflow wrap-guards (checked_add ok_or branch). + +use super::super::buffer_backend::BufferBackend; + +#[test] +fn try_extend_exact_fit_succeeds_and_advances_tail() { + let mut buf = vec![0u8; 4]; + let mut b = UserSliceBackend::from_slice(&mut buf); + assert!(b.try_extend(&[1, 2, 3, 4]).is_ok()); + assert_eq!(b.tail(), 4); + let (s, _) = b.as_slices(); + assert_eq!(s, &[1, 2, 3, 4]); +} + +#[test] +fn try_extend_over_capacity_returns_overflow_and_keeps_tail() { + let mut buf = vec![0u8; 4]; + let mut b = UserSliceBackend::from_slice(&mut buf); + let err = b.try_extend(&[1, 2, 3, 4, 5]).unwrap_err(); + assert_eq!(err.tail, 0); + assert_eq!(err.requested, 5); + assert_eq!(err.capacity, 4); + assert_eq!(b.tail(), 0); +} + +#[test] +fn try_extend_partially_full_overshoot_reports_current_tail() { + let mut buf = vec![0u8; 4]; + let mut b = UserSliceBackend::from_slice(&mut buf); + b.extend(&[1, 2]); + // 3 more bytes would need 5 total, capacity is 4. + let err = b.try_extend(&[3, 4, 5]).unwrap_err(); + assert_eq!(err.tail, 2); + assert_eq!(err.requested, 3); + assert_eq!(err.capacity, 4); + assert_eq!(b.tail(), 2); +} + +#[test] +fn try_extend_zero_length_succeeds_and_leaves_tail_unchanged() { + // The `checked_add(tail, len)` wrap branch in `try_extend` is + // a defense-in-depth guard for corrupted input that names + // `regenerated_size` near `usize::MAX`. Constructing such a + // `&[u8]` from safe Rust is not expressible — `from_raw_parts` + // with a forged length is UB. The wrap branch is exercised + // only from the fuzz harness under `feature = "fuzz_exports"` + // (which routes a controlled `len` through `try_*`) and from + // the real malformed-frame decode path that the harness + // emulates. + // + // This test covers the adjacent normal case — a zero-length + // `try_extend` MUST succeed regardless of current `tail` + // (the new_tail = tail + 0 = tail comparison both passes + // checked_add and the upper-bound check). Without this case + // the early-return on `len == 0` could regress silently. + let mut buf = vec![0u8; 8]; + let mut b = UserSliceBackend::from_slice(&mut buf); + assert!(b.try_extend(&[]).is_ok()); + assert_eq!(b.tail(), 0); + b.extend(&[1, 2, 3]); + assert!(b.try_extend(&[]).is_ok()); + assert_eq!(b.tail(), 3); +} + +#[test] +fn try_extend_and_fill_exact_fit_writes_pattern() { + let mut buf = vec![0u8; 4]; + let mut b = UserSliceBackend::from_slice(&mut buf); + assert!(b.try_extend_and_fill(0xAB, 4).is_ok()); + assert_eq!(b.tail(), 4); + let (s, _) = b.as_slices(); + assert_eq!(s, &[0xAB, 0xAB, 0xAB, 0xAB]); +} + +#[test] +fn try_extend_and_fill_over_capacity_returns_overflow() { + let mut buf = vec![0u8; 4]; + let mut b = UserSliceBackend::from_slice(&mut buf); + b.extend(&[1, 2]); + let err = b.try_extend_and_fill(0xCD, 5).unwrap_err(); + assert_eq!(err.tail, 2); + assert_eq!(err.requested, 5); + assert_eq!(err.capacity, 4); + assert_eq!(b.tail(), 2); +} + +#[test] +fn try_extend_from_within_within_bounds_repeats_history() { + let mut buf = vec![0u8; 8]; + let mut b = UserSliceBackend::from_slice(&mut buf); + b.extend(&[1, 2, 3]); + // Repeat the first 3 bytes from history into the next 3 slots. + assert!(b.try_extend_from_within(0, 3).is_ok()); + let (s, _) = b.as_slices(); + assert_eq!(s, &[1, 2, 3, 1, 2, 3]); + assert_eq!(b.tail(), 6); +} + +#[test] +fn try_extend_from_within_source_past_tail_returns_overflow() { + let mut buf = vec![0u8; 8]; + let mut b = UserSliceBackend::from_slice(&mut buf); + b.extend(&[1, 2]); + // start=0, len=5 — source range needs bytes 0..5 but tail=2. + let err = b.try_extend_from_within(0, 5).unwrap_err(); + assert_eq!(err.tail, 2); + assert_eq!(err.requested, 5); + assert_eq!(b.tail(), 2); +} + +#[test] +fn try_extend_from_within_destination_overflow_returns_err() { + let mut buf = vec![0u8; 4]; + let mut b = UserSliceBackend::from_slice(&mut buf); + b.extend(&[1, 2, 3]); + // Source 0..2 valid, but writing 2 more bytes would push tail + // from 3 to 5, past capacity 4. + let err = b.try_extend_from_within(0, 2).unwrap_err(); + assert_eq!(err.tail, 3); + assert_eq!(err.requested, 2); + assert_eq!(err.capacity, 4); + assert_eq!(b.tail(), 3); +} + +/// Direct tests for `exec_sequence_inline` — exercise the +/// x86_64 inline body so coverage attributes its 40 lines to +/// these tests, not through the deep `decode_all` +/// pipeline where `cargo llvm-cov` sometimes loses the inlined +/// callee. Tests cover: short-literal + short-match, long +/// literal (wildcopy tail), short-offset match (overlapCopy8 + +/// 8-byte stride), long-offset match (wildcopy_no_overlap). +#[test] +fn exec_sequence_inline_overflow_returns_output_buffer_overflow() { + // #246: the upstream zstd-inline sequence executor must return + // `ExecuteSequencesError::OutputBufferOverflow` (NOT panic, NOT + // an out-of-bounds unsafe write) when a sequence's literal+match + // copy plus wildcopy overshoot would land past the fixed slice. + // This is the per-sequence guard that keeps the unsafe write + // surface inside the user slice on the way to the post-block FCS + // check; the higher-level acceptance test + // (`decode_all_compressed_block_fcs_overflow_returns_structured_error`) + // proves the whole chain folds into `FrameContentSizeMismatch`. + use super::super::errors::ExecuteSequencesError; + // Tiny slice: 16-byte capacity, tail already at 8. A sequence + // writing 8 literals + 8 match (16 bytes) plus the 15-byte + // wildcopy overshoot cannot fit in `16 - 8 = 8` remaining bytes. + let mut buf = vec![0u8; 16]; + for (i, slot) in buf.iter_mut().take(8).enumerate() { + *slot = i as u8; + } + let mut b = UserSliceBackend::from_slice(&mut buf); + b.tail = 8; + let lits = [0xAAu8; 16]; + // SAFETY: `lits` is a 16-byte parent buffer (upstream zstd 16-byte read + // slack satisfied); the call must return Err before any write + // past the slice end. + let err = unsafe { b.exec_sequence_inline(lits.as_ptr(), 8, 4, 8) } + .expect_err("overshoot must return OutputBufferOverflow"); + assert!( + matches!(err, ExecuteSequencesError::OutputBufferOverflow { .. }), + "expected OutputBufferOverflow, got {err:?}" + ); + // Backend left untouched on Err — tail not advanced. + assert_eq!(b.tail, 8, "tail must not advance on overflow"); +} + +#[test] +fn exec_sequence_inline_short_literal_plus_long_offset_match() { + // Layout: pre-fill `tail = 8` with a "history" region so + // a match copy at offset 16 reaches inside the slice. Then + // bump tail past that history and exercise upstream zstd_exec. + // Buffer sized with WILDCOPY_OVERLENGTH slack at the end. + const WILDCOPY: usize = super::super::buffer_backend::WILDCOPY_OVERLENGTH; + let mut buf = vec![0u8; 256 + WILDCOPY]; + // Seed history: bytes 0..32 = ascending values, so a later + // match at offset 16 picks up bytes 16..32. + for (i, slot) in buf.iter_mut().take(32).enumerate() { + *slot = i as u8; + } + let mut b = UserSliceBackend::from_slice(&mut buf); + b.tail = 32; // Pretend 32 history bytes are already written. + + // 8-byte literals to write (upstream zstd's litLength <= 16 fast + // path — no wildcopy tail). Match length 8 at offset 16. + let lits: [u8; 16] = [ + 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, 0x11, 0x22, 0xA1, 0xB1, 0xC1, 0xD1, 0xE1, 0xF1, 0x10, + 0x20, + ]; + unsafe { + b.exec_sequence_inline(lits.as_ptr(), 8, 16, 8).unwrap(); + } + // tail advanced by 8 lit + 8 match = 16. + assert_eq!(b.tail, 48); + // Literals landed at tail = 32..40. + assert_eq!(&buf[32..40], &lits[..8]); + // Match: at output position 40..48, source = tail + lit_len - + // offset = 32 + 8 - 16 = 24. So buf[40..48] == buf[24..32] + // (history bytes 24..32 = [24, 25, 26, 27, 28, 29, 30, 31]). + assert_eq!(&buf[40..48], &[24u8, 25, 26, 27, 28, 29, 30, 31]); +} + +#[test] +fn exec_sequence_inline_long_literal_uses_wildcopy_tail() { + // litLength > 16 path: unconditional copy16 + wildcopy tail + // for the remaining literal bytes. + const WILDCOPY: usize = super::super::buffer_backend::WILDCOPY_OVERLENGTH; + let mut buf = vec![0u8; 256 + WILDCOPY]; + for (i, slot) in buf.iter_mut().take(32).enumerate() { + *slot = i as u8; + } + let mut b = UserSliceBackend::from_slice(&mut buf); + b.tail = 32; + + // 40-byte literals (forces wildcopy tail) — needs 40-byte + // source buffer with extra read slack for the final + // 16-byte load. + let lits: Vec = (0..40u8 + 16).map(|i| 0x80 + i).collect(); + unsafe { + b.exec_sequence_inline(lits.as_ptr(), 40, 16, 8).unwrap(); + } + assert_eq!(b.tail, 80); + assert_eq!(&buf[32..72], &lits[..40]); + // Match at offset 16: src = 32 + 40 - 16 = 56. buf[72..80] + // == buf[56..64] (which we just wrote = lits[24..32]). + assert_eq!(&buf[72..80], &lits[24..32]); +} + +#[test] +fn exec_sequence_inline_short_offset_match_uses_overlap_copy() { + // offset < 16 takes the overlapCopy8 + 8-byte stride path + // (vs. the offset >= 16 wildcopy_no_overlap path). + // + // What we actually assert here: + // 1. `tail` advances by `lit_length + match_length`. + // 2. The literal payload lands at `buf[tail..tail+ll]`. + // 3. The FIRST 4 match-output bytes match seed[4..8] — + // that prefix of the match copy reads source bytes + // that the literal `copy16` overshoot did NOT + // overwrite (the upstream zstd `copy16` writes 16 bytes at + // `tail`, so source bytes BEFORE `tail` survive). + // + // We do NOT cross-validate against the legacy `extend` + + // `repeat_in_chunks` chain: those paths don't perform the + // 16-byte literal overshoot, so they produce a different + // output for the same logical sequence (different bytes in + // positions where the upstream zstd's overshoot is consumed by the + // match copy). End-to-end parity is covered by the higher + // level `roundtrip_integrity::*` tests in lib.rs which + // decode whole frames and compare to the encoder input. + const WILDCOPY: usize = super::super::buffer_backend::WILDCOPY_OVERLENGTH; + let mut buf = vec![0u8; 256 + WILDCOPY]; + // Seed last 8 bytes of history with a recognisable pattern. + let seed = [0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7]; + buf[24..32].copy_from_slice(&seed); + let mut b = UserSliceBackend::from_slice(&mut buf); + b.tail = 32; + + let lits: [u8; 16] = [0xFF; 16]; + // litLength=4, offset=8, matchLength=12. offset<16 → short + // path (overlapCopy8 + 8-byte stride). + unsafe { + b.exec_sequence_inline(lits.as_ptr(), 4, 8, 12).unwrap(); + } + // tail = 32 + 4 + 12 = 48. + assert_eq!(b.tail, 48); + // Literal copy: bytes 32..36 are the literal payload. + assert_eq!(&buf[32..36], &lits[..4]); + // Match copy: the first 4 output bytes (positions 36..40) + // are the FIRST 4 source bytes (positions 28..32), which the + // literal copy16 has NOT overwritten (it wrote at 32..48, so + // 28..32 remain as the seed tail). Verify those. + assert_eq!(&buf[36..40], &seed[4..8]); + // The remaining 8 match bytes (40..48) get fed by the + // 8-byte-stride wildcopy reading from positions inside the + // match-destination region, which the literal copy16 already + // overwrote with 0xFF. That's the upstream zstd invariant — the + // overshoot is consumed correctly. We don't pin the exact + // bytes (they're a function of overlap_copy8's spread + // tables) but the output length must be right. +} + +/// `exec_sequence_inline_avx2` correctness across the +/// match-copy offset cases. Issue #279 round 4 regression: +/// the AVX2 32-byte ymm wildcopy threshold MUST be `offset >= 32` +/// (not >= 16). At offset 16..31 a 32-byte load from `match_src` +/// reads bytes inside the destination region BEFORE the first +/// store has written them; SSE2 16-byte fallback covers those +/// offsets safely. Test exercises: +/// - offset 20 (mid-range, must route to SSE2 fallback) +/// - offset 32 (boundary, AVX2 path) +/// - offset 64 (deep AVX2 path) +/// +/// against a byte-by-byte reference of the same repeat semantics. +// `std` feature gates the test: `is_x86_feature_detected!` is `std`-only +// (runtime CPU detection), unavailable in the crate's `#![no_std]` build. +#[cfg(all(target_arch = "x86_64", feature = "std"))] +#[test] +fn exec_sequence_inline_avx2_offset_boundary_correctness() { + if !std::arch::is_x86_feature_detected!("avx2") { + return; + } + const WILDCOPY: usize = super::super::buffer_backend::WILDCOPY_OVERLENGTH; + + for offset in [20usize, 32, 64] { + let mut buf = vec![0u8; 512 + WILDCOPY]; + // Seed bytes 0..200 with a deterministic, byte-position-derived + // pattern so the reference match-copy is unambiguous. + for (i, slot) in buf.iter_mut().take(200).enumerate() { + *slot = ((i * 31 + 7) & 0xFF) as u8; + } + // Compute reference: byte-by-byte repeat from offset 100..(100+offset) + // up to match_length bytes, written at position 200. + let match_length = 96usize; + let mut reference = buf.clone(); + for i in 0..match_length { + reference[200 + i] = reference[200 + i - offset]; + } + + // Now run the AVX2 inline executor on the actual buffer. + let mut b = UserSliceBackend::from_slice(&mut buf); + b.tail = 200; + let dummy_lits: [u8; 16] = [0xAA; 16]; + unsafe { + b.exec_sequence_inline_avx2(dummy_lits.as_ptr(), 0, offset, match_length) + .unwrap(); + } + assert_eq!(b.tail, 200 + match_length); + + // Compare match-copy region against the reference. + // Bytes 200..(200+match_length) must match the byte-by-byte + // repeat. The overshoot region past match_length is not pinned. + for i in 0..match_length { + assert_eq!( + buf[200 + i], + reference[200 + i], + "exec_sequence_inline_avx2 offset={offset} byte {i}: \ + got {:#x}, expected {:#x} (regression: AVX2 wildcopy \ + reading past first-store boundary)", + buf[200 + i], + reference[200 + i], + ); + } + } +} diff --git a/zstd/src/dictionary/fastcover.rs b/zstd/src/dictionary/fastcover.rs index 2c2e58640..1fc566b9d 100644 --- a/zstd/src/dictionary/fastcover.rs +++ b/zstd/src/dictionary/fastcover.rs @@ -328,139 +328,4 @@ pub fn optimize_fastcover_raw( } #[cfg(test)] -mod tests { - use super::*; - use std::format; - - fn corpus() -> Vec { - let mut data = Vec::new(); - for i in 0..500u32 { - data.extend_from_slice( - format!("tenant=demo table=orders key={i} region=eu payload=aaaaabbbbbccccdddd\n") - .as_bytes(), - ); - } - data - } - - #[test] - fn fastcover_raw_produces_non_empty_dict() { - let sample = corpus(); - let dict = train_fastcover_raw( - sample.as_slice(), - 4096, - FastCoverParams { - k: 256, - d: 8, - f: 20, - accel: 1, - }, - ); - assert!(!dict.is_empty()); - assert!(dict.len() <= 4096); - } - - #[test] - fn fastcover_raw_returns_empty_for_empty_or_zero_budget() { - let sample = corpus(); - let params = FastCoverParams { - k: 256, - d: 8, - f: 20, - accel: 1, - }; - assert!(train_fastcover_raw(&[], 1024, params).is_empty()); - assert!(train_fastcover_raw(sample.as_slice(), 0, params).is_empty()); - } - - #[test] - fn fastcover_optimizer_selects_valid_params() { - let sample = corpus(); - let (dict, tuned) = optimize_fastcover_raw( - sample.as_slice(), - 4096, - 0.75, - 1, - &[6, 8], - &[18, 20], - &[128, 256], - ); - assert!(!dict.is_empty()); - assert!([6, 8].contains(&tuned.d)); - assert!([18, 20].contains(&tuned.f)); - assert!([128, 256].contains(&tuned.k)); - } - - #[test] - fn fastcover_optimizer_falls_back_when_k_candidates_empty() { - let sample = corpus(); - let (dict, tuned) = - optimize_fastcover_raw(sample.as_slice(), 4096, 0.75, 1, &[6, 8], &[18, 20], &[]); - assert!(!dict.is_empty()); - assert!(DEFAULT_K_CANDIDATES.contains(&tuned.k)); - } - - #[test] - fn fastcover_optimizer_handles_one_byte_sample_without_panic() { - let sample = [0xAB]; - let (dict, tuned) = optimize_fastcover_raw(&sample, 16, 0.75, 1, &[], &[], &[]); - assert!(!dict.is_empty()); - assert!(dict.len() <= 16); - assert!(DEFAULT_K_CANDIDATES.contains(&tuned.k)); - assert!(DEFAULT_D_CANDIDATES.contains(&tuned.d)); - assert!(DEFAULT_F_CANDIDATES.contains(&tuned.f)); - } - - #[test] - fn fastcover_optimizer_seeds_winner_when_all_scores_are_zero() { - let sample = b"abcdefghijklmnopqrst"; - let (dict, tuned) = optimize_fastcover_raw(sample, 16, 0.9, 1, &[6], &[16], &[8]); - assert!(!dict.is_empty()); - assert_eq!(tuned.k, 16); - assert_eq!(tuned.d, 6); - assert_eq!(tuned.f, 16); - assert_eq!(tuned.score, 0); - } - - #[test] - fn fastcover_optimizer_handles_zero_dict_budget() { - let sample = corpus(); - let (dict, tuned) = optimize_fastcover_raw( - sample.as_slice(), - 0, - 0.75, - 1, - &[6, 8], - &[18, 20], - &[128, 256], - ); - assert!(dict.is_empty()); - assert!([6, 8].contains(&tuned.d)); - assert!([18, 20].contains(&tuned.f)); - assert!([128, 256].contains(&tuned.k)); - } - - #[test] - fn fastcover_optimizer_clamps_extreme_split_points() { - let sample = corpus(); - let (dict_low, tuned_low) = - optimize_fastcover_raw(sample.as_slice(), 2048, 0.0, 1, &[6], &[18], &[128]); - let (dict_high, tuned_high) = - optimize_fastcover_raw(sample.as_slice(), 2048, 1.0, 1, &[6], &[18], &[128]); - assert!(!dict_low.is_empty()); - assert!(!dict_high.is_empty()); - assert_eq!(tuned_low.k, 128); - assert_eq!(tuned_high.k, 128); - } - - #[test] - fn fastcover_optimizer_reports_normalized_params() { - let sample = corpus(); - let (dict, tuned) = - optimize_fastcover_raw(sample.as_slice(), 1024, 0.75, 1, &[64], &[42], &[8]); - assert!(!dict.is_empty()); - assert_eq!(tuned.d, 32); - assert_eq!(tuned.f, 20); - assert_eq!(tuned.k, 32); - } -} +mod tests; diff --git a/zstd/src/dictionary/fastcover/tests.rs b/zstd/src/dictionary/fastcover/tests.rs new file mode 100644 index 000000000..71d346373 --- /dev/null +++ b/zstd/src/dictionary/fastcover/tests.rs @@ -0,0 +1,134 @@ +use super::*; +use std::format; + +fn corpus() -> Vec { + let mut data = Vec::new(); + for i in 0..500u32 { + data.extend_from_slice( + format!("tenant=demo table=orders key={i} region=eu payload=aaaaabbbbbccccdddd\n") + .as_bytes(), + ); + } + data +} + +#[test] +fn fastcover_raw_produces_non_empty_dict() { + let sample = corpus(); + let dict = train_fastcover_raw( + sample.as_slice(), + 4096, + FastCoverParams { + k: 256, + d: 8, + f: 20, + accel: 1, + }, + ); + assert!(!dict.is_empty()); + assert!(dict.len() <= 4096); +} + +#[test] +fn fastcover_raw_returns_empty_for_empty_or_zero_budget() { + let sample = corpus(); + let params = FastCoverParams { + k: 256, + d: 8, + f: 20, + accel: 1, + }; + assert!(train_fastcover_raw(&[], 1024, params).is_empty()); + assert!(train_fastcover_raw(sample.as_slice(), 0, params).is_empty()); +} + +#[test] +fn fastcover_optimizer_selects_valid_params() { + let sample = corpus(); + let (dict, tuned) = optimize_fastcover_raw( + sample.as_slice(), + 4096, + 0.75, + 1, + &[6, 8], + &[18, 20], + &[128, 256], + ); + assert!(!dict.is_empty()); + assert!([6, 8].contains(&tuned.d)); + assert!([18, 20].contains(&tuned.f)); + assert!([128, 256].contains(&tuned.k)); +} + +#[test] +fn fastcover_optimizer_falls_back_when_k_candidates_empty() { + let sample = corpus(); + let (dict, tuned) = + optimize_fastcover_raw(sample.as_slice(), 4096, 0.75, 1, &[6, 8], &[18, 20], &[]); + assert!(!dict.is_empty()); + assert!(DEFAULT_K_CANDIDATES.contains(&tuned.k)); +} + +#[test] +fn fastcover_optimizer_handles_one_byte_sample_without_panic() { + let sample = [0xAB]; + let (dict, tuned) = optimize_fastcover_raw(&sample, 16, 0.75, 1, &[], &[], &[]); + assert!(!dict.is_empty()); + assert!(dict.len() <= 16); + assert!(DEFAULT_K_CANDIDATES.contains(&tuned.k)); + assert!(DEFAULT_D_CANDIDATES.contains(&tuned.d)); + assert!(DEFAULT_F_CANDIDATES.contains(&tuned.f)); +} + +#[test] +fn fastcover_optimizer_seeds_winner_when_all_scores_are_zero() { + let sample = b"abcdefghijklmnopqrst"; + let (dict, tuned) = optimize_fastcover_raw(sample, 16, 0.9, 1, &[6], &[16], &[8]); + assert!(!dict.is_empty()); + assert_eq!(tuned.k, 16); + assert_eq!(tuned.d, 6); + assert_eq!(tuned.f, 16); + assert_eq!(tuned.score, 0); +} + +#[test] +fn fastcover_optimizer_handles_zero_dict_budget() { + let sample = corpus(); + let (dict, tuned) = optimize_fastcover_raw( + sample.as_slice(), + 0, + 0.75, + 1, + &[6, 8], + &[18, 20], + &[128, 256], + ); + assert!(dict.is_empty()); + assert!([6, 8].contains(&tuned.d)); + assert!([18, 20].contains(&tuned.f)); + assert!([128, 256].contains(&tuned.k)); +} + +#[test] +fn fastcover_optimizer_clamps_extreme_split_points() { + let sample = corpus(); + let (dict_low, tuned_low) = + optimize_fastcover_raw(sample.as_slice(), 2048, 0.0, 1, &[6], &[18], &[128]); + let (dict_high, tuned_high) = + optimize_fastcover_raw(sample.as_slice(), 2048, 1.0, 1, &[6], &[18], &[128]); + assert!(!dict_low.is_empty()); + assert!(!dict_high.is_empty()); + assert_eq!(tuned_low.k, 128); + assert_eq!(tuned_high.k, 128); +} + +#[test] +fn fastcover_optimizer_reports_normalized_params() { + let sample = corpus(); + let (dict, tuned) = + optimize_fastcover_raw(sample.as_slice(), 1024, 0.75, 1, &[64], &[42], &[8]); + assert!(!dict.is_empty()); + assert_eq!(tuned.d, 32); + assert_eq!(tuned.f, 20); + assert_eq!(tuned.k, 32); +} diff --git a/zstd/src/dictionary/frequency.rs b/zstd/src/dictionary/frequency.rs index 26085c4ac..9bc0b6ba4 100644 --- a/zstd/src/dictionary/frequency.rs +++ b/zstd/src/dictionary/frequency.rs @@ -52,26 +52,4 @@ pub fn estimate_frequency(pattern: &[u8], body: &[u8]) -> usize { } #[cfg(test)] -mod tests { - use super::estimate_frequency; - #[test] - fn dead_beef() { - assert_eq!( - estimate_frequency(&[0xde, 0xad], &[0xde, 0xad, 0xbe, 0xef, 0xde, 0xad]), - 2 - ); - } - - #[test] - fn smallest_body() { - assert_eq!(estimate_frequency(&[0x00, 0xff], &[0x00, 0xff]), 1); - } - - #[test] - fn no_match() { - assert_eq!( - estimate_frequency(&[0xff, 0xff], &[0xde, 0xad, 0xbe, 0xef, 0xde, 0xad]), - 0 - ); - } -} +mod tests; diff --git a/zstd/src/dictionary/frequency/tests.rs b/zstd/src/dictionary/frequency/tests.rs new file mode 100644 index 000000000..a919aa047 --- /dev/null +++ b/zstd/src/dictionary/frequency/tests.rs @@ -0,0 +1,21 @@ +use super::estimate_frequency; +#[test] +fn dead_beef() { + assert_eq!( + estimate_frequency(&[0xde, 0xad], &[0xde, 0xad, 0xbe, 0xef, 0xde, 0xad]), + 2 + ); +} + +#[test] +fn smallest_body() { + assert_eq!(estimate_frequency(&[0x00, 0xff], &[0x00, 0xff]), 1); +} + +#[test] +fn no_match() { + assert_eq!( + estimate_frequency(&[0xff, 0xff], &[0xde, 0xad, 0xbe, 0xef, 0xde, 0xad]), + 0 + ); +} diff --git a/zstd/src/dictionary/mod.rs b/zstd/src/dictionary/mod.rs index 75185c63a..8c6c1b895 100644 --- a/zstd/src/dictionary/mod.rs +++ b/zstd/src/dictionary/mod.rs @@ -675,266 +675,4 @@ pub(crate) fn dict_roundtrip_fixture() -> ( } #[cfg(test)] -mod tests { - use super::*; - use crate::decoding::Dictionary; - use std::io::Cursor; - use std::string::ToString; - - fn training_data() -> Vec { - let mut data = Vec::new(); - for i in 0..512u32 { - data.extend_from_slice( - format!( - "tenant=demo table=orders key={i} region=eu payload=aaaaabbbbbcccccdddddeeeee\n" - ) - .as_bytes(), - ); - } - data - } - - #[test] - fn create_fastcover_dict_from_source_writes_non_empty_output() { - let sample = training_data(); - let mut out = Vec::new(); - let tuned = create_fastcover_dict_from_source( - Cursor::new(sample.as_slice()), - &mut out, - 4096, - &FastCoverOptions::default(), - FinalizeOptions::default(), - ) - .expect("fastcover+finalize should succeed"); - assert!(!out.is_empty()); - assert!(tuned.k > 0); - assert!(tuned.d > 0); - } - - #[test] - fn create_fastcover_raw_dict_from_source_rejects_empty_source() { - let mut out = Vec::new(); - let err = create_fastcover_raw_dict_from_source( - Cursor::new(Vec::::new()), - &mut out, - 1024, - &FastCoverOptions::default(), - ) - .expect_err("empty source must be rejected"); - assert_eq!(err.kind(), io::ErrorKind::InvalidInput); - } - - #[test] - fn create_fastcover_dict_from_source_propagates_finalize_error() { - let sample = training_data(); - let mut out = Vec::new(); - let err = create_fastcover_dict_from_source( - Cursor::new(sample.as_slice()), - &mut out, - 32, - &FastCoverOptions::default(), - FinalizeOptions::default(), - ) - .expect_err("too-small dictionary budget must fail during finalize"); - assert_eq!(err.kind(), io::ErrorKind::InvalidInput); - assert!(err.to_string().contains("dictionary size too small")); - } - - #[test] - fn create_fastcover_dict_from_source_rejects_empty_source() { - let mut out = Vec::new(); - let err = create_fastcover_dict_from_source( - Cursor::new(Vec::::new()), - &mut out, - 1024, - &FastCoverOptions::default(), - FinalizeOptions::default(), - ) - .expect_err("empty source must be rejected"); - assert_eq!(err.kind(), io::ErrorKind::InvalidInput); - } - - #[test] - fn create_raw_dict_from_source_early_returns_on_zero_dict_size() { - let sample = training_data(); - let mut out = Vec::new(); - create_raw_dict_from_source(Cursor::new(sample.as_slice()), sample.len(), &mut out, 0) - .expect("zero dict size should no-op"); - assert!(out.is_empty()); - } - - #[test] - fn create_raw_dict_from_source_treats_source_size_as_hint() { - let sample = training_data(); - let mut out = Vec::new(); - create_raw_dict_from_source(Cursor::new(sample.as_slice()), 0, &mut out, 1024) - .expect("raw dictionary training should succeed"); - assert!(!out.is_empty()); - } - - #[test] - fn create_raw_dict_from_source_handles_tiny_source_without_epochs() { - let sample = b"short"; - let mut out = Vec::new(); - create_raw_dict_from_source(Cursor::new(sample.as_slice()), sample.len(), &mut out, 3) - .expect("tiny source path should succeed"); - assert_eq!(out, b"ort"); - } - - #[test] - fn create_raw_dict_from_source_propagates_read_error() { - struct FailingReader; - impl io::Read for FailingReader { - fn read(&mut self, _buf: &mut [u8]) -> io::Result { - Err(io::Error::other("read failed")) - } - } - - let mut out = Vec::new(); - let err = create_raw_dict_from_source(FailingReader, 1024, &mut out, 1024) - .expect_err("read failures must be returned"); - assert_eq!(err.kind(), io::ErrorKind::Other); - assert_eq!(err.to_string(), "read failed"); - } - - #[test] - fn create_raw_dict_from_source_propagates_write_error() { - struct FailingWriter; - impl io::Write for FailingWriter { - fn write(&mut self, _buf: &[u8]) -> io::Result { - Err(io::Error::other("write failed")) - } - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } - } - - let sample = b"short"; - let mut out = FailingWriter; - let err = - create_raw_dict_from_source(Cursor::new(sample.as_slice()), sample.len(), &mut out, 3) - .expect_err("write failures must be returned"); - assert_eq!(err.kind(), io::ErrorKind::Other); - assert_eq!(err.to_string(), "write failed"); - } - - #[test] - fn create_raw_dict_from_source_never_exceeds_requested_size() { - let dict_size = 4096usize; - let source: Vec = core::iter::repeat_n(b'a', 320_001).collect(); - let mut out = Vec::new(); - create_raw_dict_from_source( - Cursor::new(source.as_slice()), - source.len(), - &mut out, - dict_size, - ) - .expect("raw dictionary training should succeed"); - assert!( - out.len() <= dict_size, - "raw dictionary exceeded requested size: {} > {}", - out.len(), - dict_size - ); - } - - #[test] - fn train_fastcover_raw_from_slice_rejects_empty_sample() { - let err = train_fastcover_raw_from_slice(&[], 1024, &FastCoverOptions::default()) - .expect_err("empty sample must be rejected"); - assert_eq!(err.kind(), io::ErrorKind::InvalidInput); - } - - #[test] - fn train_fastcover_raw_from_slice_supports_non_optimized_params() { - let sample = training_data(); - let options = FastCoverOptions { - optimize: false, - k: 128, - d: 6, - f: 18, - ..FastCoverOptions::default() - }; - let (dict, tuned) = - train_fastcover_raw_from_slice(sample.as_slice(), 2048, &options).expect("must train"); - assert!(!dict.is_empty()); - assert!(dict.len() <= 2048); - assert_eq!(tuned.k, 128); - assert_eq!(tuned.d, 6); - assert_eq!(tuned.f, 18); - assert_eq!(tuned.score, 0); - } - - #[test] - fn train_fastcover_raw_from_slice_rejects_tiny_sample_with_empty_dict() { - let sample = b"tiny"; - let err = train_fastcover_raw_from_slice(sample, 1024, &FastCoverOptions::default()) - .expect_err("tiny sample should not produce an empty dictionary successfully"); - assert_eq!(err.kind(), io::ErrorKind::InvalidInput); - assert_eq!( - err.to_string(), - "training sample is too small for FastCOVER" - ); - } - - #[test] - fn train_fastcover_raw_from_slice_normalizes_non_optimized_params() { - let sample = training_data(); - let options = FastCoverOptions { - optimize: false, - k: 8, - d: 64, - f: 42, - ..FastCoverOptions::default() - }; - let (_, tuned) = - train_fastcover_raw_from_slice(sample.as_slice(), 2048, &options).expect("must train"); - assert_eq!(tuned.k, 32); - assert_eq!(tuned.d, 32); - assert_eq!(tuned.f, 20); - } - - #[test] - fn finalize_raw_dict_rejects_empty_raw_content() { - let sample = training_data(); - let err = finalize_raw_dict(&[], sample.as_slice(), 4096, FinalizeOptions::default()) - .expect_err("empty raw dictionary must be rejected"); - assert_eq!(err.kind(), io::ErrorKind::InvalidInput); - } - - #[test] - fn finalize_raw_dict_rejects_too_small_budget() { - let sample = training_data(); - let raw = b"some-raw-bytes"; - let err = finalize_raw_dict(raw, sample.as_slice(), 32, FinalizeOptions::default()) - .expect_err("tiny dict_size must fail"); - assert_eq!(err.kind(), io::ErrorKind::InvalidInput); - assert!(err.to_string().contains("dictionary size too small")); - } - - #[test] - fn finalize_raw_dict_pads_to_minimum_content_size() { - let sample = training_data(); - let raw = b"x"; - let finalized = finalize_raw_dict(raw, sample.as_slice(), 4096, FinalizeOptions::default()) - .expect("finalize should pad small raw content"); - let parsed = Dictionary::decode_dict(finalized.as_slice()).expect("finalized dict parses"); - assert!(parsed.dict_content.len() >= 8); - assert_eq!(parsed.dict_content.last(), Some(&b'x')); - } - - #[test] - fn finalize_raw_dict_rejects_zero_dict_id() { - let sample = training_data(); - let raw = b"raw-fastcover-bytes"; - let err = finalize_raw_dict( - raw, - sample.as_slice(), - 4096, - FinalizeOptions { dict_id: Some(0) }, - ) - .expect_err("dict_id=0 must be rejected"); - assert_eq!(err.kind(), io::ErrorKind::InvalidInput); - assert_eq!(err.to_string(), "dictionary id must be non-zero"); - } -} +mod tests; diff --git a/zstd/src/dictionary/reservoir.rs b/zstd/src/dictionary/reservoir.rs index 6fb318c91..20a5bcd1c 100644 --- a/zstd/src/dictionary/reservoir.rs +++ b/zstd/src/dictionary/reservoir.rs @@ -108,37 +108,4 @@ impl Reservoir { } #[cfg(test)] -mod tests { - use super::Reservoir; - use alloc::vec; - - #[test] - fn initial_fill() { - // Create a reservoir 16 bytes in size and read - // 16 bytes into it - let r = Reservoir::new(16); - let test_data = vec![0_u8; 16]; - let output = r.fill(&mut test_data.as_slice()); - assert_eq!(test_data, output); - } - - #[test] - fn shrinks_for_small_sample() { - // Create a reservoir larger than the sample. - // The output should be smaller. - let r = Reservoir::new(32); - let test_data = vec![0_u8; 28]; - let output = r.fill(&mut test_data.as_slice()); - assert!(output.len() == 28); - } - - #[test] - fn lake_doesnt_grow() { - // Create a sample larger than the reservoir - // The output should be smaller. - let r = Reservoir::new(32); - let test_data = vec![0_u8; 16_000_000]; - let output = r.fill(&mut test_data.as_slice()); - assert!(output.len() == 32); - } -} +mod tests; diff --git a/zstd/src/dictionary/reservoir/tests.rs b/zstd/src/dictionary/reservoir/tests.rs new file mode 100644 index 000000000..788200f6a --- /dev/null +++ b/zstd/src/dictionary/reservoir/tests.rs @@ -0,0 +1,32 @@ +use super::Reservoir; +use alloc::vec; + +#[test] +fn initial_fill() { + // Create a reservoir 16 bytes in size and read + // 16 bytes into it + let r = Reservoir::new(16); + let test_data = vec![0_u8; 16]; + let output = r.fill(&mut test_data.as_slice()); + assert_eq!(test_data, output); +} + +#[test] +fn shrinks_for_small_sample() { + // Create a reservoir larger than the sample. + // The output should be smaller. + let r = Reservoir::new(32); + let test_data = vec![0_u8; 28]; + let output = r.fill(&mut test_data.as_slice()); + assert!(output.len() == 28); +} + +#[test] +fn lake_doesnt_grow() { + // Create a sample larger than the reservoir + // The output should be smaller. + let r = Reservoir::new(32); + let test_data = vec![0_u8; 16_000_000]; + let output = r.fill(&mut test_data.as_slice()); + assert!(output.len() == 32); +} diff --git a/zstd/src/dictionary/tests.rs b/zstd/src/dictionary/tests.rs new file mode 100644 index 000000000..2c01a6553 --- /dev/null +++ b/zstd/src/dictionary/tests.rs @@ -0,0 +1,261 @@ +use super::*; +use crate::decoding::Dictionary; +use std::io::Cursor; +use std::string::ToString; + +fn training_data() -> Vec { + let mut data = Vec::new(); + for i in 0..512u32 { + data.extend_from_slice( + format!( + "tenant=demo table=orders key={i} region=eu payload=aaaaabbbbbcccccdddddeeeee\n" + ) + .as_bytes(), + ); + } + data +} + +#[test] +fn create_fastcover_dict_from_source_writes_non_empty_output() { + let sample = training_data(); + let mut out = Vec::new(); + let tuned = create_fastcover_dict_from_source( + Cursor::new(sample.as_slice()), + &mut out, + 4096, + &FastCoverOptions::default(), + FinalizeOptions::default(), + ) + .expect("fastcover+finalize should succeed"); + assert!(!out.is_empty()); + assert!(tuned.k > 0); + assert!(tuned.d > 0); +} + +#[test] +fn create_fastcover_raw_dict_from_source_rejects_empty_source() { + let mut out = Vec::new(); + let err = create_fastcover_raw_dict_from_source( + Cursor::new(Vec::::new()), + &mut out, + 1024, + &FastCoverOptions::default(), + ) + .expect_err("empty source must be rejected"); + assert_eq!(err.kind(), io::ErrorKind::InvalidInput); +} + +#[test] +fn create_fastcover_dict_from_source_propagates_finalize_error() { + let sample = training_data(); + let mut out = Vec::new(); + let err = create_fastcover_dict_from_source( + Cursor::new(sample.as_slice()), + &mut out, + 32, + &FastCoverOptions::default(), + FinalizeOptions::default(), + ) + .expect_err("too-small dictionary budget must fail during finalize"); + assert_eq!(err.kind(), io::ErrorKind::InvalidInput); + assert!(err.to_string().contains("dictionary size too small")); +} + +#[test] +fn create_fastcover_dict_from_source_rejects_empty_source() { + let mut out = Vec::new(); + let err = create_fastcover_dict_from_source( + Cursor::new(Vec::::new()), + &mut out, + 1024, + &FastCoverOptions::default(), + FinalizeOptions::default(), + ) + .expect_err("empty source must be rejected"); + assert_eq!(err.kind(), io::ErrorKind::InvalidInput); +} + +#[test] +fn create_raw_dict_from_source_early_returns_on_zero_dict_size() { + let sample = training_data(); + let mut out = Vec::new(); + create_raw_dict_from_source(Cursor::new(sample.as_slice()), sample.len(), &mut out, 0) + .expect("zero dict size should no-op"); + assert!(out.is_empty()); +} + +#[test] +fn create_raw_dict_from_source_treats_source_size_as_hint() { + let sample = training_data(); + let mut out = Vec::new(); + create_raw_dict_from_source(Cursor::new(sample.as_slice()), 0, &mut out, 1024) + .expect("raw dictionary training should succeed"); + assert!(!out.is_empty()); +} + +#[test] +fn create_raw_dict_from_source_handles_tiny_source_without_epochs() { + let sample = b"short"; + let mut out = Vec::new(); + create_raw_dict_from_source(Cursor::new(sample.as_slice()), sample.len(), &mut out, 3) + .expect("tiny source path should succeed"); + assert_eq!(out, b"ort"); +} + +#[test] +fn create_raw_dict_from_source_propagates_read_error() { + struct FailingReader; + impl io::Read for FailingReader { + fn read(&mut self, _buf: &mut [u8]) -> io::Result { + Err(io::Error::other("read failed")) + } + } + + let mut out = Vec::new(); + let err = create_raw_dict_from_source(FailingReader, 1024, &mut out, 1024) + .expect_err("read failures must be returned"); + assert_eq!(err.kind(), io::ErrorKind::Other); + assert_eq!(err.to_string(), "read failed"); +} + +#[test] +fn create_raw_dict_from_source_propagates_write_error() { + struct FailingWriter; + impl io::Write for FailingWriter { + fn write(&mut self, _buf: &[u8]) -> io::Result { + Err(io::Error::other("write failed")) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + let sample = b"short"; + let mut out = FailingWriter; + let err = + create_raw_dict_from_source(Cursor::new(sample.as_slice()), sample.len(), &mut out, 3) + .expect_err("write failures must be returned"); + assert_eq!(err.kind(), io::ErrorKind::Other); + assert_eq!(err.to_string(), "write failed"); +} + +#[test] +fn create_raw_dict_from_source_never_exceeds_requested_size() { + let dict_size = 4096usize; + let source: Vec = core::iter::repeat_n(b'a', 320_001).collect(); + let mut out = Vec::new(); + create_raw_dict_from_source( + Cursor::new(source.as_slice()), + source.len(), + &mut out, + dict_size, + ) + .expect("raw dictionary training should succeed"); + assert!( + out.len() <= dict_size, + "raw dictionary exceeded requested size: {} > {}", + out.len(), + dict_size + ); +} + +#[test] +fn train_fastcover_raw_from_slice_rejects_empty_sample() { + let err = train_fastcover_raw_from_slice(&[], 1024, &FastCoverOptions::default()) + .expect_err("empty sample must be rejected"); + assert_eq!(err.kind(), io::ErrorKind::InvalidInput); +} + +#[test] +fn train_fastcover_raw_from_slice_supports_non_optimized_params() { + let sample = training_data(); + let options = FastCoverOptions { + optimize: false, + k: 128, + d: 6, + f: 18, + ..FastCoverOptions::default() + }; + let (dict, tuned) = + train_fastcover_raw_from_slice(sample.as_slice(), 2048, &options).expect("must train"); + assert!(!dict.is_empty()); + assert!(dict.len() <= 2048); + assert_eq!(tuned.k, 128); + assert_eq!(tuned.d, 6); + assert_eq!(tuned.f, 18); + assert_eq!(tuned.score, 0); +} + +#[test] +fn train_fastcover_raw_from_slice_rejects_tiny_sample_with_empty_dict() { + let sample = b"tiny"; + let err = train_fastcover_raw_from_slice(sample, 1024, &FastCoverOptions::default()) + .expect_err("tiny sample should not produce an empty dictionary successfully"); + assert_eq!(err.kind(), io::ErrorKind::InvalidInput); + assert_eq!( + err.to_string(), + "training sample is too small for FastCOVER" + ); +} + +#[test] +fn train_fastcover_raw_from_slice_normalizes_non_optimized_params() { + let sample = training_data(); + let options = FastCoverOptions { + optimize: false, + k: 8, + d: 64, + f: 42, + ..FastCoverOptions::default() + }; + let (_, tuned) = + train_fastcover_raw_from_slice(sample.as_slice(), 2048, &options).expect("must train"); + assert_eq!(tuned.k, 32); + assert_eq!(tuned.d, 32); + assert_eq!(tuned.f, 20); +} + +#[test] +fn finalize_raw_dict_rejects_empty_raw_content() { + let sample = training_data(); + let err = finalize_raw_dict(&[], sample.as_slice(), 4096, FinalizeOptions::default()) + .expect_err("empty raw dictionary must be rejected"); + assert_eq!(err.kind(), io::ErrorKind::InvalidInput); +} + +#[test] +fn finalize_raw_dict_rejects_too_small_budget() { + let sample = training_data(); + let raw = b"some-raw-bytes"; + let err = finalize_raw_dict(raw, sample.as_slice(), 32, FinalizeOptions::default()) + .expect_err("tiny dict_size must fail"); + assert_eq!(err.kind(), io::ErrorKind::InvalidInput); + assert!(err.to_string().contains("dictionary size too small")); +} + +#[test] +fn finalize_raw_dict_pads_to_minimum_content_size() { + let sample = training_data(); + let raw = b"x"; + let finalized = finalize_raw_dict(raw, sample.as_slice(), 4096, FinalizeOptions::default()) + .expect("finalize should pad small raw content"); + let parsed = Dictionary::decode_dict(finalized.as_slice()).expect("finalized dict parses"); + assert!(parsed.dict_content.len() >= 8); + assert_eq!(parsed.dict_content.last(), Some(&b'x')); +} + +#[test] +fn finalize_raw_dict_rejects_zero_dict_id() { + let sample = training_data(); + let raw = b"raw-fastcover-bytes"; + let err = finalize_raw_dict( + raw, + sample.as_slice(), + 4096, + FinalizeOptions { dict_id: Some(0) }, + ) + .expect_err("dict_id=0 must be rejected"); + assert_eq!(err.kind(), io::ErrorKind::InvalidInput); + assert_eq!(err.to_string(), "dictionary id must be non-zero"); +} diff --git a/zstd/src/encoding/block_header.rs b/zstd/src/encoding/block_header.rs index 42faae9b7..fbe410e2e 100644 --- a/zstd/src/encoding/block_header.rs +++ b/zstd/src/encoding/block_header.rs @@ -47,28 +47,4 @@ impl BlockHeader { } #[cfg(test)] -mod tests { - use super::BlockHeader; - use crate::{blocks::block::BlockType, decoding::block_decoder}; - use alloc::vec::Vec; - - #[test] - fn block_header_serialize() { - let header = BlockHeader { - last_block: true, - block_type: super::BlockType::Compressed, - block_size: 69, - }; - let mut serialized_header = Vec::new(); - header.serialize(&mut serialized_header); - let mut decoder = block_decoder::new(); - let parsed_header = decoder - .read_block_header(serialized_header.as_slice()) - .unwrap() - .0; - - assert!(parsed_header.last_block); - assert_eq!(parsed_header.block_type, BlockType::Compressed); - assert_eq!(parsed_header.content_size, 69); - } -} +mod tests; diff --git a/zstd/src/encoding/block_header/tests.rs b/zstd/src/encoding/block_header/tests.rs new file mode 100644 index 000000000..9ad443cb1 --- /dev/null +++ b/zstd/src/encoding/block_header/tests.rs @@ -0,0 +1,23 @@ +use super::BlockHeader; +use crate::{blocks::block::BlockType, decoding::block_decoder}; +use alloc::vec::Vec; + +#[test] +fn block_header_serialize() { + let header = BlockHeader { + last_block: true, + block_type: super::BlockType::Compressed, + block_size: 69, + }; + let mut serialized_header = Vec::new(); + header.serialize(&mut serialized_header); + let mut decoder = block_decoder::new(); + let parsed_header = decoder + .read_block_header(serialized_header.as_slice()) + .unwrap() + .0; + + assert!(parsed_header.last_block); + assert_eq!(parsed_header.block_type, BlockType::Compressed); + assert_eq!(parsed_header.content_size, 69); +} diff --git a/zstd/src/encoding/blocks/compressed.rs b/zstd/src/encoding/blocks/compressed.rs index 553df5be7..d3ffb34f2 100644 --- a/zstd/src/encoding/blocks/compressed.rs +++ b/zstd/src/encoding/blocks/compressed.rs @@ -2325,594 +2325,4 @@ fn compress_literals( } #[cfg(test)] -mod tests { - - use super::{ - FseTableMode, RawSequence, choose_table, emit_single_sequence_block, encode_match_len, - encode_offset_with_history, min_gain, min_literals_to_compress, previous_table, - remember_last_used_tables, - }; - use crate::encoding::frame_compressor::{CompressState, FseTables, PreviousFseTable}; - use crate::encoding::strategy::StrategyTag; - use crate::fse::fse_encoder::build_table_from_symbol_counts; - use crate::huff0::huff0_encoder; - use alloc::vec::Vec; - - fn tables_match( - lhs: &crate::fse::fse_encoder::FSETable, - rhs: &crate::fse::fse_encoder::FSETable, - ) -> bool { - lhs.table_size == rhs.table_size - && (0..=255u8) - .all(|symbol| lhs.symbol_probability(symbol) == rhs.symbol_probability(symbol)) - } - - #[test] - fn repeat_offset_codes_follow_rfc_mapping() { - let mut hist = [10, 20, 30]; - assert_eq!(encode_offset_with_history(10, 5, &mut hist), 1); - assert_eq!(hist, [10, 20, 30]); - - let mut hist = [10, 20, 30]; - assert_eq!(encode_offset_with_history(20, 5, &mut hist), 2); - assert_eq!(hist, [20, 10, 30]); - - let mut hist = [10, 20, 30]; - assert_eq!(encode_offset_with_history(30, 5, &mut hist), 3); - assert_eq!(hist, [30, 10, 20]); - - let mut hist = [10, 20, 30]; - assert_eq!(encode_offset_with_history(20, 0, &mut hist), 1); - assert_eq!(hist, [20, 10, 30]); - - let mut hist = [10, 20, 30]; - assert_eq!(encode_offset_with_history(30, 0, &mut hist), 2); - assert_eq!(hist, [30, 10, 20]); - - let mut hist = [10, 20, 30]; - assert_eq!(encode_offset_with_history(9, 0, &mut hist), 3); - assert_eq!(hist, [9, 10, 20]); - } - - #[test] - fn min_literals_to_compress_returns_per_strategy_floor() { - for strat in [ - StrategyTag::Fast, - StrategyTag::Dfast, - StrategyTag::Greedy, - StrategyTag::Lazy, - StrategyTag::Btlazy2, - ] { - assert_eq!(min_literals_to_compress(strat, false), 64); - assert_eq!(min_literals_to_compress(strat, true), 6); - } - assert_eq!(min_literals_to_compress(StrategyTag::BtOpt, false), 32); - assert_eq!(min_literals_to_compress(StrategyTag::BtOpt, true), 6); - assert_eq!(min_literals_to_compress(StrategyTag::BtUltra, false), 16); - assert_eq!(min_literals_to_compress(StrategyTag::BtUltra, true), 6); - assert_eq!(min_literals_to_compress(StrategyTag::BtUltra2, false), 8); - assert_eq!(min_literals_to_compress(StrategyTag::BtUltra2, true), 6); - } - - #[test] - fn min_gain_returns_per_strategy_margin() { - let src = 4096usize; - for strat in [ - StrategyTag::Fast, - StrategyTag::Dfast, - StrategyTag::Greedy, - StrategyTag::Lazy, - StrategyTag::Btlazy2, - StrategyTag::BtOpt, - ] { - assert_eq!(min_gain(src, strat), (src >> 6) + 2); - } - assert_eq!(min_gain(src, StrategyTag::BtUltra), (src >> 7) + 2); - assert_eq!(min_gain(src, StrategyTag::BtUltra2), (src >> 8) + 2); - assert_eq!(min_gain(0, StrategyTag::Fast), 2); - assert_eq!(min_gain(63, StrategyTag::Fast), 2); - assert_eq!(min_gain(64, StrategyTag::Fast), 3); - } - - #[test] - fn use_raw_literal_fallback_uses_payload_vs_srcsize_threshold() { - use super::{compressed_literals_header_bytes, use_raw_literal_fallback}; - // Upstream zstd formula: `huf_section_size >= literals_len - min_gain`, - // payload-vs-srcSize (no headers on either side). Verify the - // gate is symmetric in header overhead by hitting the boundary - // where the old on-wire `total >= raw_section - mg` form would - // have disagreed. - let strategy = StrategyTag::Fast; // min_gain(20, Fast) = (20>>6)+2 = 2 - - // literals_len = 20: raw_header = 1, compressed_header = 3. - // New threshold (payload-vs-srcSize): - // keep huf iff huf_section_size < 20 - 2 = 18 - // fallback iff huf_section_size >= 18 - // - // Old (regressed) threshold on-wire-vs-on-wire: - // total = huf_section_size + 3 - // fallback iff total >= (20 + 1) - 2 = 19 - // iff huf_section_size >= 16 - // - // Gap where formulas disagree: huf_section_size in [16, 18). - // The new formula MUST keep huf in this gap. - let literals_len = 20usize; - // Sanity-check the literals-header constants the math relies on. - assert_eq!(compressed_literals_header_bytes(literals_len), 3); - assert_eq!(super::uncompressed_literals_header_bytes(literals_len), 1); - - // Inside the gap — new keeps, old would have rejected: - assert!(!use_raw_literal_fallback(16, literals_len, strategy)); - assert!(!use_raw_literal_fallback(17, literals_len, strategy)); - - // At/above new threshold — both new and old fall back: - assert!(use_raw_literal_fallback(18, literals_len, strategy)); - assert!(use_raw_literal_fallback(19, literals_len, strategy)); - - // Below the old threshold — both keep: - assert!(!use_raw_literal_fallback(15, literals_len, strategy)); - assert!(!use_raw_literal_fallback(0, literals_len, strategy)); - } - - #[test] - fn prefer_repeat_eligible_applies_gate() { - use super::prefer_repeat_eligible; - // Upstream zstd `zstd_compress_literals.c:165`: - // strategy < ZSTD_lazy && srcSize <= 1024 -> HUF_flags_preferRepeat - // ZSTD_lazy == 4 in upstream zstd enum; our `< Lazy` set is - // {Fast, Dfast, Greedy}. Verify the gate fires for each - // and stays off for the rest. - for lit_len in [0usize, 1, 64, 256, 1024] { - assert!( - prefer_repeat_eligible(StrategyTag::Fast, lit_len), - "Fast/{lit_len}" - ); - assert!( - prefer_repeat_eligible(StrategyTag::Dfast, lit_len), - "Dfast/{lit_len}" - ); - assert!( - prefer_repeat_eligible(StrategyTag::Greedy, lit_len), - "Greedy/{lit_len}" - ); - assert!( - !prefer_repeat_eligible(StrategyTag::Lazy, lit_len), - "Lazy/{lit_len}" - ); - assert!( - !prefer_repeat_eligible(StrategyTag::Btlazy2, lit_len), - "Btlazy2/{lit_len}" - ); - assert!( - !prefer_repeat_eligible(StrategyTag::BtOpt, lit_len), - "BtOpt/{lit_len}" - ); - assert!( - !prefer_repeat_eligible(StrategyTag::BtUltra, lit_len), - "BtUltra/{lit_len}" - ); - assert!( - !prefer_repeat_eligible(StrategyTag::BtUltra2, lit_len), - "BtUltra2/{lit_len}" - ); - } - // Above the 1024-byte size threshold the gate stays off - // for ALL strategies (upstream zstd `srcSize <= 1024` is the - // closed upper bound). - for lit_len in [1025usize, 2048, 16384] { - assert!(!prefer_repeat_eligible(StrategyTag::Fast, lit_len)); - assert!(!prefer_repeat_eligible(StrategyTag::Dfast, lit_len)); - assert!(!prefer_repeat_eligible(StrategyTag::Greedy, lit_len)); - assert!(!prefer_repeat_eligible(StrategyTag::Btlazy2, lit_len)); - } - } - - #[test] - fn decide_huff_reuse_prefer_repeat_forces_reuse_for_fast_band() { - use super::{decide_huff_reuse_like_encoder, huff0_encoder}; - // Fixture chosen so size-comparison heuristic and the - // preferRepeat short-circuit DISAGREE: `prev` is built - // from a broad uniform sweep so it can encode any byte; - // the literals payload is heavily skewed (240 zeros + 16 - // outliers) so a freshly-built `new_tbl` would compress - // strictly better than `prev`. Without preferRepeat, the - // heuristic picks `new` (returns true); WITH preferRepeat - // the fast-band gate forces reuse (returns false). - // Removing the short-circuit flips Fast/Dfast/Greedy to - // true and breaks this test — that's the regression gate. - let prev_training: Vec = (0..1024u32).map(|i| (i % 256) as u8).collect(); - let prev = huff0_encoder::HuffmanTable::build_from_data(&prev_training); - let mut skewed_literals: Vec = Vec::with_capacity(256); - skewed_literals.extend(core::iter::repeat_n(0u8, 240)); - skewed_literals.extend((0..16u8).map(|i| 200 + i)); - let new_tbl = huff0_encoder::HuffmanTable::build_from_data(&skewed_literals); - let new_desc = new_tbl - .writeable_table_description_size() - .expect("non-empty table emits a description"); - - // Distinguishing precondition: WITHOUT preferRepeat the - // size comparison must prefer new (else the test isn't - // exercising the override). Verify by running with a - // strategy outside the eligible band: Lazy returns - // true (=new) on this fixture. - assert!( - decide_huff_reuse_like_encoder( - &new_tbl, - Some(&prev), - new_desc, - &skewed_literals, - StrategyTag::Lazy, - ), - "fixture precondition: size-comparison must prefer new for Lazy on skewed literals" - ); - - // Eligible fast band: preferRepeat forces reuse (=false) - // despite size-comparison preferring new. - for strategy in [StrategyTag::Fast, StrategyTag::Dfast, StrategyTag::Greedy] { - assert!( - !decide_huff_reuse_like_encoder( - &new_tbl, - Some(&prev), - new_desc, - &skewed_literals, - strategy, - ), - "{strategy:?} <= 1024 must short-circuit to reuse despite size-comparison favouring new" - ); - } - - // Above the 1024-byte threshold the gate stays off even - // for Fast/Dfast/Greedy — falls through to the size - // heuristic, which on this fixture prefers new. - let mut big_skewed = skewed_literals.clone(); - big_skewed.extend(core::iter::repeat_n(0u8, 1024)); - assert!( - big_skewed.len() > 1024, - "fixture must exceed 1024 to disable preferRepeat" - ); - assert!( - decide_huff_reuse_like_encoder( - &new_tbl, - Some(&prev), - new_desc, - &big_skewed, - StrategyTag::Fast, - ), - "Fast at len > 1024 must NOT short-circuit (gate disabled), falls through to size heuristic" - ); - } - - #[test] - fn estimator_literals_section_mirrors_emit_for_short_inputs() { - use super::{ - CompressedBlockScratch, EntropyOnlyMatcher, EstimatorWorkspace, - encode_block_parts_with_sequence_scratch, estimate_block_parts_size, - }; - // For each strategy at boundary literal lengths around `min_lits` - // and across the all-identical RLE pre-check (fires for any - // non-empty all-identical input under every strategy/HUF state), - // estimator's predicted size MUST equal the bytes the emitter - // actually writes. Cases include: (a) fresh state with - // `last_huff_table: None` covering the strategy-specific - // `min_lits` band (8/16/32/64), (b) seeded HUF-reuse state at - // the lowered floor of 6 — under the prior hardcoded `len >= 8` - // gate 6/7-byte all-identical sections went raw, now they pass - // `min_lits == 6` and the upstream zstd-parity HUF+`cLitSize==1` path - // would route them to RLE; the pre-check shortcuts that path, - // (c) sub-`min_lits` all-identical sections that take the - // RLE pre-check regardless of strategy. - type Inputs = &'static [(usize, bool)]; - let cases: &[(StrategyTag, bool, Inputs)] = &[ - // (strategy, seed_huff_reuse, [(len, all_identical)]) - ( - StrategyTag::Fast, - false, - &[ - (1, true), // sub-min_lits all-identical → RLE - (5, true), // sub-min_lits all-identical → RLE - (8, true), - (8, false), - (63, true), - (63, false), - (64, false), - ], - ), - ( - StrategyTag::BtUltra2, - false, - &[(7, true), (7, false), (8, true), (8, false), (16, false)], - ), - ( - StrategyTag::BtOpt, - false, - &[(8, true), (31, true), (32, false)], - ), - // HUF reuse path: floor drops to 6. Under the prior - // hardcoded `len >= 8` gate 6/7-byte sections went raw; - // post-fix, all-identical 6/7-byte sections take the RLE - // pre-check and stay byte-equivalent estimator-vs-emit - // (upstream zstd parity would route them HUF→cLitSize==1→RLE). - // Also exercise non-identical 6-byte raw fallback and - // 16-byte HUF reuse path. - ( - StrategyTag::Lazy, - true, - &[(6, true), (7, true), (6, false), (16, false)], - ), - ]; - - for (strat, seed_huff, inputs) in cases { - for (len, identical) in *inputs { - let literals: Vec = if *identical { - alloc::vec![0x5Au8; *len] - } else { - (0..*len as u8).collect() - }; - // Seed both estimator and emit state with the same - // synthetic HUF table when `seed_huff` is true so the - // reuse path's `min_lits == 6` floor is exercised. - // Counts from a varied byte sequence give a valid - // (writeable) table that survives the `decide_huff_reuse` - // decision when literals are large enough to consider it. - let seed_table = if *seed_huff { - let mut counts = [0usize; 256]; - for b in (0..=63u8).chain(64..=127u8) { - counts[b as usize] = 1; - } - Some(huff0_encoder::HuffmanTable::build_from_counts(&counts)) - } else { - None - }; - let mut est_state = CompressState:: { - matcher: EntropyOnlyMatcher, - last_huff_table: seed_table.clone(), - huff_table_spare: None, - fse_tables: FseTables::new(), - block_scratch: CompressedBlockScratch::new(), - offset_hist: [1, 4, 8], - strategy_tag: *strat, - huf_optimal_search: true, - }; - let mut emit_state = CompressState:: { - matcher: EntropyOnlyMatcher, - last_huff_table: seed_table, - huff_table_spare: None, - fse_tables: FseTables::new(), - block_scratch: CompressedBlockScratch::new(), - offset_hist: [1, 4, 8], - strategy_tag: *strat, - huf_optimal_search: true, - }; - let mut workspace = EstimatorWorkspace::default(); - let est = estimate_block_parts_size(&mut est_state, &literals, &[], &mut workspace); - let mut emitted: Vec = Vec::new(); - let mut scratch: Vec = Vec::new(); - encode_block_parts_with_sequence_scratch( - &mut emit_state, - &literals, - &[], - &mut emitted, - &mut scratch, - ); - assert_eq!( - est, - emitted.len(), - "estimator/emit parity broken: strategy={:?} seed_huff={} len={} identical={} est={} emit={}", - strat, - seed_huff, - len, - identical, - est, - emitted.len(), - ); - } - } - } - - #[test] - fn encode_match_len_uses_correct_upper_range_base() { - assert_eq!(encode_match_len(65539), (52, 0, 16)); - assert_eq!(encode_match_len(65540), (52, 1, 16)); - assert_eq!(encode_match_len(131074), (52, 65535, 16)); - } - - #[test] - fn raw_partition_fallback_restores_repeat_offset_history() { - let mut state = CompressState { - matcher: super::EntropyOnlyMatcher, - last_huff_table: None, - huff_table_spare: None, - fse_tables: FseTables::new(), - block_scratch: super::CompressedBlockScratch::new(), - offset_hist: [10, 20, 30], - strategy_tag: crate::encoding::strategy::StrategyTag::Fast, - huf_optimal_search: true, - }; - let source = [0xA5; 8]; - let sequences = [RawSequence { - ll: 0, - ml: 5, - offset: 20, - }]; - let mut output = Vec::new(); - let mut compressed_scratch = Vec::new(); - let mut sequence_scratch = Vec::new(); - - let mut emit_buffers = super::SingleSequenceEmitBuffers { - output: &mut output, - compressed: &mut compressed_scratch, - sequence_scratch: &mut sequence_scratch, - }; - let emitted_raw = emit_single_sequence_block( - &mut state, - true, - source.len(), - &[], - &sequences, - &mut emit_buffers, - ); - if emitted_raw { - output.extend_from_slice(&source); - } - - assert_eq!( - state.offset_hist, - [10, 20, 30], - "raw post-split fallback must not advance decoder repeat-offset history" - ); - assert_eq!( - (output[0] >> 1) & 0b11, - 0, - "fixture should force the partition to fall back to a Raw block" - ); - } - - #[test] - fn remember_last_used_tables_keeps_predefined_and_repeat_modes() { - let mut fse_tables = FseTables::new(); - - remember_last_used_tables( - &mut fse_tables, - Some(PreviousFseTable::Default), - Some(PreviousFseTable::Default), - Some(PreviousFseTable::Default), - ); - - assert!(tables_match( - previous_table(fse_tables.ll_previous.as_ref(), fse_tables.ll_default_ref()).unwrap(), - fse_tables.ll_default_ref() - )); - assert!(tables_match( - previous_table(fse_tables.ml_previous.as_ref(), fse_tables.ml_default_ref()).unwrap(), - fse_tables.ml_default_ref() - )); - assert!(tables_match( - previous_table(fse_tables.of_previous.as_ref(), fse_tables.of_default_ref()).unwrap(), - fse_tables.of_default_ref() - )); - - let sample_codes = [0u8, 1u8]; - // Lazy is a non-fast-band strategy, so this exercises the cost-based - // repeat decision (not the fast-band shortcut). - let strat = crate::encoding::strategy::StrategyTag::Lazy; - let ll_repeat = choose_table( - fse_tables.ll_previous.as_ref(), - fse_tables.ll_default_ref(), - sample_codes.iter().copied(), - 9, - strat, - ); - let ml_repeat = choose_table( - fse_tables.ml_previous.as_ref(), - fse_tables.ml_default_ref(), - sample_codes.iter().copied(), - 9, - strat, - ); - let of_repeat = choose_table( - fse_tables.of_previous.as_ref(), - fse_tables.of_default_ref(), - sample_codes.iter().copied(), - 8, - strat, - ); - - assert!(matches!(ll_repeat, FseTableMode::RepeatLast(_))); - assert!(matches!(ml_repeat, FseTableMode::RepeatLast(_))); - assert!(matches!(of_repeat, FseTableMode::RepeatLast(_))); - } - - /// Fast-band strategies (fast/dfast/greedy) reuse a covering previous FSE - /// table without building a new one (upstream zstd `preferRepeat`), even on a - /// fresh distribution where the cost-based path could pick a new table. - /// A non-fast-band strategy on an identical distribution takes the - /// cost-based path instead. - #[test] - fn fast_band_strategies_prefer_repeat_fse_table() { - use crate::encoding::strategy::StrategyTag; - let prev = build_table_from_symbol_counts(&[8, 1], 9, false); - let previous = - PreviousFseTable::Custom(crate::encoding::frame_compressor::SharedFseTable::new(prev)); - let fse_tables = FseTables::new(); - // Distribution over symbols {0,1}, both covered by `previous`. - let mut counts = [0usize; 256]; - counts[0] = 4; - counts[1] = 6; - let total = 10; - - // All fast-band strategies (Fast, Dfast, Greedy) unconditionally - // reuse the covering previous table; cover every eligible arm so an - // enum-arm regression in the implementation branch is caught. - for strategy in [StrategyTag::Fast, StrategyTag::Dfast, StrategyTag::Greedy] { - let mode = super::choose_table_from_counts( - Some(&previous), - fse_tables.ll_default_ref(), - &counts, - total, - 1, // highest non-zero code in {0,1} - 9, - strategy, - ); - assert!( - matches!(mode, FseTableMode::RepeatLast(_)), - "fast-band {strategy:?} must reuse the covering previous table", - ); - } - } - - #[test] - fn remember_last_used_tables_reuses_existing_custom_slot_for_repeat() { - let mut fse_tables = FseTables::new(); - let custom = build_table_from_symbol_counts(&[1, 1], 5, false); - fse_tables.ll_previous = Some(PreviousFseTable::Custom( - crate::encoding::frame_compressor::SharedFseTable::new(custom), - )); - - let before = core::ptr::from_ref( - previous_table(fse_tables.ll_previous.as_ref(), fse_tables.ll_default_ref()).unwrap(), - ); - - remember_last_used_tables( - &mut fse_tables, - None, - Some(PreviousFseTable::Default), - Some(PreviousFseTable::Default), - ); - - let after = core::ptr::from_ref( - previous_table(fse_tables.ll_previous.as_ref(), fse_tables.ll_default_ref()).unwrap(), - ); - - assert_eq!(before, after); - assert!(matches!( - fse_tables.ll_previous.as_ref(), - Some(PreviousFseTable::Custom(_)) - )); - } - - #[test] - fn choose_table_handles_single_symbol_distribution() { - let fse_tables = FseTables::new(); - let mode = choose_table( - None, - fse_tables.ll_default_ref(), - core::iter::repeat_n(0u8, 32), - 9, - crate::encoding::strategy::StrategyTag::Lazy, - ); - assert!(matches!(mode, FseTableMode::Rle(0))); - } - - #[test] - fn choose_table_without_previous_does_not_unwrap_none() { - let only_zero_one_table = build_table_from_symbol_counts(&[1, 1], 5, false); - let mode = choose_table( - None, - &only_zero_one_table, - [1u8, 2].into_iter().cycle().take(32), - 5, - crate::encoding::strategy::StrategyTag::Lazy, - ); - assert!(matches!(mode, FseTableMode::Encoded(_))); - } -} +mod tests; diff --git a/zstd/src/encoding/blocks/compressed/tests.rs b/zstd/src/encoding/blocks/compressed/tests.rs new file mode 100644 index 000000000..513a0bfde --- /dev/null +++ b/zstd/src/encoding/blocks/compressed/tests.rs @@ -0,0 +1,588 @@ +use super::{ + FseTableMode, RawSequence, choose_table, emit_single_sequence_block, encode_match_len, + encode_offset_with_history, min_gain, min_literals_to_compress, previous_table, + remember_last_used_tables, +}; +use crate::encoding::frame_compressor::{CompressState, FseTables, PreviousFseTable}; +use crate::encoding::strategy::StrategyTag; +use crate::fse::fse_encoder::build_table_from_symbol_counts; +use crate::huff0::huff0_encoder; +use alloc::vec::Vec; + +fn tables_match( + lhs: &crate::fse::fse_encoder::FSETable, + rhs: &crate::fse::fse_encoder::FSETable, +) -> bool { + lhs.table_size == rhs.table_size + && (0..=255u8) + .all(|symbol| lhs.symbol_probability(symbol) == rhs.symbol_probability(symbol)) +} + +#[test] +fn repeat_offset_codes_follow_rfc_mapping() { + let mut hist = [10, 20, 30]; + assert_eq!(encode_offset_with_history(10, 5, &mut hist), 1); + assert_eq!(hist, [10, 20, 30]); + + let mut hist = [10, 20, 30]; + assert_eq!(encode_offset_with_history(20, 5, &mut hist), 2); + assert_eq!(hist, [20, 10, 30]); + + let mut hist = [10, 20, 30]; + assert_eq!(encode_offset_with_history(30, 5, &mut hist), 3); + assert_eq!(hist, [30, 10, 20]); + + let mut hist = [10, 20, 30]; + assert_eq!(encode_offset_with_history(20, 0, &mut hist), 1); + assert_eq!(hist, [20, 10, 30]); + + let mut hist = [10, 20, 30]; + assert_eq!(encode_offset_with_history(30, 0, &mut hist), 2); + assert_eq!(hist, [30, 10, 20]); + + let mut hist = [10, 20, 30]; + assert_eq!(encode_offset_with_history(9, 0, &mut hist), 3); + assert_eq!(hist, [9, 10, 20]); +} + +#[test] +fn min_literals_to_compress_returns_per_strategy_floor() { + for strat in [ + StrategyTag::Fast, + StrategyTag::Dfast, + StrategyTag::Greedy, + StrategyTag::Lazy, + StrategyTag::Btlazy2, + ] { + assert_eq!(min_literals_to_compress(strat, false), 64); + assert_eq!(min_literals_to_compress(strat, true), 6); + } + assert_eq!(min_literals_to_compress(StrategyTag::BtOpt, false), 32); + assert_eq!(min_literals_to_compress(StrategyTag::BtOpt, true), 6); + assert_eq!(min_literals_to_compress(StrategyTag::BtUltra, false), 16); + assert_eq!(min_literals_to_compress(StrategyTag::BtUltra, true), 6); + assert_eq!(min_literals_to_compress(StrategyTag::BtUltra2, false), 8); + assert_eq!(min_literals_to_compress(StrategyTag::BtUltra2, true), 6); +} + +#[test] +fn min_gain_returns_per_strategy_margin() { + let src = 4096usize; + for strat in [ + StrategyTag::Fast, + StrategyTag::Dfast, + StrategyTag::Greedy, + StrategyTag::Lazy, + StrategyTag::Btlazy2, + StrategyTag::BtOpt, + ] { + assert_eq!(min_gain(src, strat), (src >> 6) + 2); + } + assert_eq!(min_gain(src, StrategyTag::BtUltra), (src >> 7) + 2); + assert_eq!(min_gain(src, StrategyTag::BtUltra2), (src >> 8) + 2); + assert_eq!(min_gain(0, StrategyTag::Fast), 2); + assert_eq!(min_gain(63, StrategyTag::Fast), 2); + assert_eq!(min_gain(64, StrategyTag::Fast), 3); +} + +#[test] +fn use_raw_literal_fallback_uses_payload_vs_srcsize_threshold() { + use super::{compressed_literals_header_bytes, use_raw_literal_fallback}; + // Upstream zstd formula: `huf_section_size >= literals_len - min_gain`, + // payload-vs-srcSize (no headers on either side). Verify the + // gate is symmetric in header overhead by hitting the boundary + // where the old on-wire `total >= raw_section - mg` form would + // have disagreed. + let strategy = StrategyTag::Fast; // min_gain(20, Fast) = (20>>6)+2 = 2 + + // literals_len = 20: raw_header = 1, compressed_header = 3. + // New threshold (payload-vs-srcSize): + // keep huf iff huf_section_size < 20 - 2 = 18 + // fallback iff huf_section_size >= 18 + // + // Old (regressed) threshold on-wire-vs-on-wire: + // total = huf_section_size + 3 + // fallback iff total >= (20 + 1) - 2 = 19 + // iff huf_section_size >= 16 + // + // Gap where formulas disagree: huf_section_size in [16, 18). + // The new formula MUST keep huf in this gap. + let literals_len = 20usize; + // Sanity-check the literals-header constants the math relies on. + assert_eq!(compressed_literals_header_bytes(literals_len), 3); + assert_eq!(super::uncompressed_literals_header_bytes(literals_len), 1); + + // Inside the gap — new keeps, old would have rejected: + assert!(!use_raw_literal_fallback(16, literals_len, strategy)); + assert!(!use_raw_literal_fallback(17, literals_len, strategy)); + + // At/above new threshold — both new and old fall back: + assert!(use_raw_literal_fallback(18, literals_len, strategy)); + assert!(use_raw_literal_fallback(19, literals_len, strategy)); + + // Below the old threshold — both keep: + assert!(!use_raw_literal_fallback(15, literals_len, strategy)); + assert!(!use_raw_literal_fallback(0, literals_len, strategy)); +} + +#[test] +fn prefer_repeat_eligible_applies_gate() { + use super::prefer_repeat_eligible; + // Upstream zstd `zstd_compress_literals.c:165`: + // strategy < ZSTD_lazy && srcSize <= 1024 -> HUF_flags_preferRepeat + // ZSTD_lazy == 4 in upstream zstd enum; our `< Lazy` set is + // {Fast, Dfast, Greedy}. Verify the gate fires for each + // and stays off for the rest. + for lit_len in [0usize, 1, 64, 256, 1024] { + assert!( + prefer_repeat_eligible(StrategyTag::Fast, lit_len), + "Fast/{lit_len}" + ); + assert!( + prefer_repeat_eligible(StrategyTag::Dfast, lit_len), + "Dfast/{lit_len}" + ); + assert!( + prefer_repeat_eligible(StrategyTag::Greedy, lit_len), + "Greedy/{lit_len}" + ); + assert!( + !prefer_repeat_eligible(StrategyTag::Lazy, lit_len), + "Lazy/{lit_len}" + ); + assert!( + !prefer_repeat_eligible(StrategyTag::Btlazy2, lit_len), + "Btlazy2/{lit_len}" + ); + assert!( + !prefer_repeat_eligible(StrategyTag::BtOpt, lit_len), + "BtOpt/{lit_len}" + ); + assert!( + !prefer_repeat_eligible(StrategyTag::BtUltra, lit_len), + "BtUltra/{lit_len}" + ); + assert!( + !prefer_repeat_eligible(StrategyTag::BtUltra2, lit_len), + "BtUltra2/{lit_len}" + ); + } + // Above the 1024-byte size threshold the gate stays off + // for ALL strategies (upstream zstd `srcSize <= 1024` is the + // closed upper bound). + for lit_len in [1025usize, 2048, 16384] { + assert!(!prefer_repeat_eligible(StrategyTag::Fast, lit_len)); + assert!(!prefer_repeat_eligible(StrategyTag::Dfast, lit_len)); + assert!(!prefer_repeat_eligible(StrategyTag::Greedy, lit_len)); + assert!(!prefer_repeat_eligible(StrategyTag::Btlazy2, lit_len)); + } +} + +#[test] +fn decide_huff_reuse_prefer_repeat_forces_reuse_for_fast_band() { + use super::{decide_huff_reuse_like_encoder, huff0_encoder}; + // Fixture chosen so size-comparison heuristic and the + // preferRepeat short-circuit DISAGREE: `prev` is built + // from a broad uniform sweep so it can encode any byte; + // the literals payload is heavily skewed (240 zeros + 16 + // outliers) so a freshly-built `new_tbl` would compress + // strictly better than `prev`. Without preferRepeat, the + // heuristic picks `new` (returns true); WITH preferRepeat + // the fast-band gate forces reuse (returns false). + // Removing the short-circuit flips Fast/Dfast/Greedy to + // true and breaks this test — that's the regression gate. + let prev_training: Vec = (0..1024u32).map(|i| (i % 256) as u8).collect(); + let prev = huff0_encoder::HuffmanTable::build_from_data(&prev_training); + let mut skewed_literals: Vec = Vec::with_capacity(256); + skewed_literals.extend(core::iter::repeat_n(0u8, 240)); + skewed_literals.extend((0..16u8).map(|i| 200 + i)); + let new_tbl = huff0_encoder::HuffmanTable::build_from_data(&skewed_literals); + let new_desc = new_tbl + .writeable_table_description_size() + .expect("non-empty table emits a description"); + + // Distinguishing precondition: WITHOUT preferRepeat the + // size comparison must prefer new (else the test isn't + // exercising the override). Verify by running with a + // strategy outside the eligible band: Lazy returns + // true (=new) on this fixture. + assert!( + decide_huff_reuse_like_encoder( + &new_tbl, + Some(&prev), + new_desc, + &skewed_literals, + StrategyTag::Lazy, + ), + "fixture precondition: size-comparison must prefer new for Lazy on skewed literals" + ); + + // Eligible fast band: preferRepeat forces reuse (=false) + // despite size-comparison preferring new. + for strategy in [StrategyTag::Fast, StrategyTag::Dfast, StrategyTag::Greedy] { + assert!( + !decide_huff_reuse_like_encoder( + &new_tbl, + Some(&prev), + new_desc, + &skewed_literals, + strategy, + ), + "{strategy:?} <= 1024 must short-circuit to reuse despite size-comparison favouring new" + ); + } + + // Above the 1024-byte threshold the gate stays off even + // for Fast/Dfast/Greedy — falls through to the size + // heuristic, which on this fixture prefers new. + let mut big_skewed = skewed_literals.clone(); + big_skewed.extend(core::iter::repeat_n(0u8, 1024)); + assert!( + big_skewed.len() > 1024, + "fixture must exceed 1024 to disable preferRepeat" + ); + assert!( + decide_huff_reuse_like_encoder( + &new_tbl, + Some(&prev), + new_desc, + &big_skewed, + StrategyTag::Fast, + ), + "Fast at len > 1024 must NOT short-circuit (gate disabled), falls through to size heuristic" + ); +} + +#[test] +fn estimator_literals_section_mirrors_emit_for_short_inputs() { + use super::{ + CompressedBlockScratch, EntropyOnlyMatcher, EstimatorWorkspace, + encode_block_parts_with_sequence_scratch, estimate_block_parts_size, + }; + // For each strategy at boundary literal lengths around `min_lits` + // and across the all-identical RLE pre-check (fires for any + // non-empty all-identical input under every strategy/HUF state), + // estimator's predicted size MUST equal the bytes the emitter + // actually writes. Cases include: (a) fresh state with + // `last_huff_table: None` covering the strategy-specific + // `min_lits` band (8/16/32/64), (b) seeded HUF-reuse state at + // the lowered floor of 6 — under the prior hardcoded `len >= 8` + // gate 6/7-byte all-identical sections went raw, now they pass + // `min_lits == 6` and the upstream zstd-parity HUF+`cLitSize==1` path + // would route them to RLE; the pre-check shortcuts that path, + // (c) sub-`min_lits` all-identical sections that take the + // RLE pre-check regardless of strategy. + type Inputs = &'static [(usize, bool)]; + let cases: &[(StrategyTag, bool, Inputs)] = &[ + // (strategy, seed_huff_reuse, [(len, all_identical)]) + ( + StrategyTag::Fast, + false, + &[ + (1, true), // sub-min_lits all-identical → RLE + (5, true), // sub-min_lits all-identical → RLE + (8, true), + (8, false), + (63, true), + (63, false), + (64, false), + ], + ), + ( + StrategyTag::BtUltra2, + false, + &[(7, true), (7, false), (8, true), (8, false), (16, false)], + ), + ( + StrategyTag::BtOpt, + false, + &[(8, true), (31, true), (32, false)], + ), + // HUF reuse path: floor drops to 6. Under the prior + // hardcoded `len >= 8` gate 6/7-byte sections went raw; + // post-fix, all-identical 6/7-byte sections take the RLE + // pre-check and stay byte-equivalent estimator-vs-emit + // (upstream zstd parity would route them HUF→cLitSize==1→RLE). + // Also exercise non-identical 6-byte raw fallback and + // 16-byte HUF reuse path. + ( + StrategyTag::Lazy, + true, + &[(6, true), (7, true), (6, false), (16, false)], + ), + ]; + + for (strat, seed_huff, inputs) in cases { + for (len, identical) in *inputs { + let literals: Vec = if *identical { + alloc::vec![0x5Au8; *len] + } else { + (0..*len as u8).collect() + }; + // Seed both estimator and emit state with the same + // synthetic HUF table when `seed_huff` is true so the + // reuse path's `min_lits == 6` floor is exercised. + // Counts from a varied byte sequence give a valid + // (writeable) table that survives the `decide_huff_reuse` + // decision when literals are large enough to consider it. + let seed_table = if *seed_huff { + let mut counts = [0usize; 256]; + for b in (0..=63u8).chain(64..=127u8) { + counts[b as usize] = 1; + } + Some(huff0_encoder::HuffmanTable::build_from_counts(&counts)) + } else { + None + }; + let mut est_state = CompressState:: { + matcher: EntropyOnlyMatcher, + last_huff_table: seed_table.clone(), + huff_table_spare: None, + fse_tables: FseTables::new(), + block_scratch: CompressedBlockScratch::new(), + offset_hist: [1, 4, 8], + strategy_tag: *strat, + huf_optimal_search: true, + }; + let mut emit_state = CompressState:: { + matcher: EntropyOnlyMatcher, + last_huff_table: seed_table, + huff_table_spare: None, + fse_tables: FseTables::new(), + block_scratch: CompressedBlockScratch::new(), + offset_hist: [1, 4, 8], + strategy_tag: *strat, + huf_optimal_search: true, + }; + let mut workspace = EstimatorWorkspace::default(); + let est = estimate_block_parts_size(&mut est_state, &literals, &[], &mut workspace); + let mut emitted: Vec = Vec::new(); + let mut scratch: Vec = Vec::new(); + encode_block_parts_with_sequence_scratch( + &mut emit_state, + &literals, + &[], + &mut emitted, + &mut scratch, + ); + assert_eq!( + est, + emitted.len(), + "estimator/emit parity broken: strategy={:?} seed_huff={} len={} identical={} est={} emit={}", + strat, + seed_huff, + len, + identical, + est, + emitted.len(), + ); + } + } +} + +#[test] +fn encode_match_len_uses_correct_upper_range_base() { + assert_eq!(encode_match_len(65539), (52, 0, 16)); + assert_eq!(encode_match_len(65540), (52, 1, 16)); + assert_eq!(encode_match_len(131074), (52, 65535, 16)); +} + +#[test] +fn raw_partition_fallback_restores_repeat_offset_history() { + let mut state = CompressState { + matcher: super::EntropyOnlyMatcher, + last_huff_table: None, + huff_table_spare: None, + fse_tables: FseTables::new(), + block_scratch: super::CompressedBlockScratch::new(), + offset_hist: [10, 20, 30], + strategy_tag: crate::encoding::strategy::StrategyTag::Fast, + huf_optimal_search: true, + }; + let source = [0xA5; 8]; + let sequences = [RawSequence { + ll: 0, + ml: 5, + offset: 20, + }]; + let mut output = Vec::new(); + let mut compressed_scratch = Vec::new(); + let mut sequence_scratch = Vec::new(); + + let mut emit_buffers = super::SingleSequenceEmitBuffers { + output: &mut output, + compressed: &mut compressed_scratch, + sequence_scratch: &mut sequence_scratch, + }; + let emitted_raw = emit_single_sequence_block( + &mut state, + true, + source.len(), + &[], + &sequences, + &mut emit_buffers, + ); + if emitted_raw { + output.extend_from_slice(&source); + } + + assert_eq!( + state.offset_hist, + [10, 20, 30], + "raw post-split fallback must not advance decoder repeat-offset history" + ); + assert_eq!( + (output[0] >> 1) & 0b11, + 0, + "fixture should force the partition to fall back to a Raw block" + ); +} + +#[test] +fn remember_last_used_tables_keeps_predefined_and_repeat_modes() { + let mut fse_tables = FseTables::new(); + + remember_last_used_tables( + &mut fse_tables, + Some(PreviousFseTable::Default), + Some(PreviousFseTable::Default), + Some(PreviousFseTable::Default), + ); + + assert!(tables_match( + previous_table(fse_tables.ll_previous.as_ref(), fse_tables.ll_default_ref()).unwrap(), + fse_tables.ll_default_ref() + )); + assert!(tables_match( + previous_table(fse_tables.ml_previous.as_ref(), fse_tables.ml_default_ref()).unwrap(), + fse_tables.ml_default_ref() + )); + assert!(tables_match( + previous_table(fse_tables.of_previous.as_ref(), fse_tables.of_default_ref()).unwrap(), + fse_tables.of_default_ref() + )); + + let sample_codes = [0u8, 1u8]; + // Lazy is a non-fast-band strategy, so this exercises the cost-based + // repeat decision (not the fast-band shortcut). + let strat = crate::encoding::strategy::StrategyTag::Lazy; + let ll_repeat = choose_table( + fse_tables.ll_previous.as_ref(), + fse_tables.ll_default_ref(), + sample_codes.iter().copied(), + 9, + strat, + ); + let ml_repeat = choose_table( + fse_tables.ml_previous.as_ref(), + fse_tables.ml_default_ref(), + sample_codes.iter().copied(), + 9, + strat, + ); + let of_repeat = choose_table( + fse_tables.of_previous.as_ref(), + fse_tables.of_default_ref(), + sample_codes.iter().copied(), + 8, + strat, + ); + + assert!(matches!(ll_repeat, FseTableMode::RepeatLast(_))); + assert!(matches!(ml_repeat, FseTableMode::RepeatLast(_))); + assert!(matches!(of_repeat, FseTableMode::RepeatLast(_))); +} + +/// Fast-band strategies (fast/dfast/greedy) reuse a covering previous FSE +/// table without building a new one (upstream zstd `preferRepeat`), even on a +/// fresh distribution where the cost-based path could pick a new table. +/// A non-fast-band strategy on an identical distribution takes the +/// cost-based path instead. +#[test] +fn fast_band_strategies_prefer_repeat_fse_table() { + use crate::encoding::strategy::StrategyTag; + let prev = build_table_from_symbol_counts(&[8, 1], 9, false); + let previous = + PreviousFseTable::Custom(crate::encoding::frame_compressor::SharedFseTable::new(prev)); + let fse_tables = FseTables::new(); + // Distribution over symbols {0,1}, both covered by `previous`. + let mut counts = [0usize; 256]; + counts[0] = 4; + counts[1] = 6; + let total = 10; + + // All fast-band strategies (Fast, Dfast, Greedy) unconditionally + // reuse the covering previous table; cover every eligible arm so an + // enum-arm regression in the implementation branch is caught. + for strategy in [StrategyTag::Fast, StrategyTag::Dfast, StrategyTag::Greedy] { + let mode = super::choose_table_from_counts( + Some(&previous), + fse_tables.ll_default_ref(), + &counts, + total, + 1, // highest non-zero code in {0,1} + 9, + strategy, + ); + assert!( + matches!(mode, FseTableMode::RepeatLast(_)), + "fast-band {strategy:?} must reuse the covering previous table", + ); + } +} + +#[test] +fn remember_last_used_tables_reuses_existing_custom_slot_for_repeat() { + let mut fse_tables = FseTables::new(); + let custom = build_table_from_symbol_counts(&[1, 1], 5, false); + fse_tables.ll_previous = Some(PreviousFseTable::Custom( + crate::encoding::frame_compressor::SharedFseTable::new(custom), + )); + + let before = core::ptr::from_ref( + previous_table(fse_tables.ll_previous.as_ref(), fse_tables.ll_default_ref()).unwrap(), + ); + + remember_last_used_tables( + &mut fse_tables, + None, + Some(PreviousFseTable::Default), + Some(PreviousFseTable::Default), + ); + + let after = core::ptr::from_ref( + previous_table(fse_tables.ll_previous.as_ref(), fse_tables.ll_default_ref()).unwrap(), + ); + + assert_eq!(before, after); + assert!(matches!( + fse_tables.ll_previous.as_ref(), + Some(PreviousFseTable::Custom(_)) + )); +} + +#[test] +fn choose_table_handles_single_symbol_distribution() { + let fse_tables = FseTables::new(); + let mode = choose_table( + None, + fse_tables.ll_default_ref(), + core::iter::repeat_n(0u8, 32), + 9, + crate::encoding::strategy::StrategyTag::Lazy, + ); + assert!(matches!(mode, FseTableMode::Rle(0))); +} + +#[test] +fn choose_table_without_previous_does_not_unwrap_none() { + let only_zero_one_table = build_table_from_symbol_counts(&[1, 1], 5, false); + let mode = choose_table( + None, + &only_zero_one_table, + [1u8, 2].into_iter().cycle().take(32), + 5, + crate::encoding::strategy::StrategyTag::Lazy, + ); + assert!(matches!(mode, FseTableMode::Encoded(_))); +} diff --git a/zstd/src/encoding/bt/ldm_helper_tests.rs b/zstd/src/encoding/bt/ldm_helper_tests.rs new file mode 100644 index 000000000..c200a622c --- /dev/null +++ b/zstd/src/encoding/bt/ldm_helper_tests.rs @@ -0,0 +1,331 @@ +//! Unit coverage for the LDM helper cluster that batch 14 lifted +//! onto `impl BtMatcher`. The helpers walk the raw-LDM seq store +//! and translate its (literals, match) sequences into the optimal +//! parser's `MatchCandidate` shape. They are pure logic on +//! [`HcRawSeqStore`] / [`HcOptLdmState`] — no SIMD, no `MatchTable` +//! contact — so they exercise on every CI target. +use alloc::vec; +use alloc::vec::Vec; + +use super::*; +use crate::encoding::opt::ldm::{HcOptLdmState, HcRawSeq, HcRawSeqStore}; + +fn raw(lit: usize, mat: usize, off: usize) -> HcRawSeq { + HcRawSeq { + lit_length: lit, + offset: off, + match_length: mat, + } +} + +fn bt_with_seqs(seqs: Vec) -> BtMatcher { + let mut bt = BtMatcher::new(); + bt.ldm_sequences = seqs; + bt +} + +fn store(size: usize) -> HcRawSeqStore { + HcRawSeqStore { + pos: 0, + pos_in_sequence: 0, + size, + } +} + +/// Regression test for PR #139 Copilot review — threads #1/#2: +/// `prepare_ldm_candidates` must translate the absolute +/// `current_abs_start` (`history_abs_start + window_size - +/// current_len`) into a slice index inside the supplied +/// `history` slice. Before the fix, the code passed +/// `current_abs_start` directly as `block_start` to +/// `LdmProducer::generate_into`, which would index out of +/// bounds (or into the wrong byte range) as soon as +/// `history_abs_start != 0` after window eviction. +/// +/// We construct a `BtMatcher` with an active `LdmProducer` +/// whose table starts empty, hand it a small `history` slice, +/// and call `prepare_ldm_candidates` with absolute coordinates +/// shifted by a non-zero `history_abs_start`. The call must +/// complete without panic — under the pre-fix code it would +/// panic on the `history[block_start..block_end]` slice access +/// inside `generate_into`. +#[test] +#[cfg(feature = "hash")] +fn prepare_ldm_candidates_translates_absolute_positions_to_slice_indices() { + use crate::encoding::ldm::{LdmProducer, params::LdmParams}; + + let mut bt = BtMatcher::new(); + // Activate the producer with hand-tuned small params — + // the upstream zstd btultra2 defaults (`with_window_and_strategy(27, 9)`) + // would allocate `1 << 23 = 8M` entries × 8 bytes = 64 MiB + // for a table this test never actually populates beyond a + // handful of entries. Build a 10-bit hash log instead so + // the table is ~8 KiB — keeps the suite stable under + // parallel nextest on 32-bit / memory-tight CI shards. + bt.ldm_producer = Some(LdmProducer::new(LdmParams { + window_log: 27, + hash_log: 10, + hash_rate_log: 4, + min_match_length: 32, + bucket_size_log: 4, + })); + + // 256 bytes of arbitrary content — enough for the gear + // hash to make progress against the upstream zstd btultra2 + // `min_match_length = 32`. + let history: alloc::vec::Vec = (0u8..=255).collect(); + + // Simulate a frame whose window has already evicted some + // bytes: `history_abs_start = 1024`, and the current block + // covers absolute `[1024, 1280)` which maps to slice + // `[0, 256)` inside `history`. + let history_abs_start = 1024usize; + let current_abs_start = 1024usize; + let current_len = history.len(); + + // Pre-fix: would panic with `slice index out of bounds` + // on `history[1024..1280]`. Post-fix: translates to + // `[0..256]` and runs cleanly. + bt.prepare_ldm_candidates(&history, history_abs_start, current_abs_start, current_len); + // Cross-block fixture isn't engineered to emit a match — + // the assertion is simply that the call succeeded. +} + +#[test] +fn ldm_skip_raw_seq_store_bytes_walks_whole_sequences() { + let bt = bt_with_seqs(vec![raw(2, 3, 100), raw(4, 1, 200), raw(0, 5, 50)]); + let mut s = store(3); + // 5 = first seq (lit+match=5) → advance to seq 1 with pos_in_sequence=0 + bt.ldm_skip_raw_seq_store_bytes(&mut s, 5); + assert_eq!(s.pos, 1); + assert_eq!(s.pos_in_sequence, 0); +} + +#[test] +fn ldm_skip_raw_seq_store_bytes_leaves_partial_offset() { + let bt = bt_with_seqs(vec![raw(2, 3, 100), raw(4, 1, 200)]); + let mut s = store(2); + // 4 < first seq's 5 bytes → stay on seq 0, pos_in_sequence=4 + bt.ldm_skip_raw_seq_store_bytes(&mut s, 4); + assert_eq!(s.pos, 0); + assert_eq!(s.pos_in_sequence, 4); +} + +#[test] +fn ldm_skip_raw_seq_store_bytes_exhausts_store_when_distance_too_large() { + let bt = bt_with_seqs(vec![raw(1, 1, 1), raw(1, 1, 1)]); + let mut s = store(2); + // 10 bytes total but store only holds 4 (2 seqs × 2 bytes each) + bt.ldm_skip_raw_seq_store_bytes(&mut s, 10); + assert_eq!(s.pos, 2); + assert_eq!(s.pos_in_sequence, 0); +} + +#[test] +fn ldm_get_next_match_empty_store_clears_window() { + let bt = bt_with_seqs(vec![]); + let mut opt_ldm = HcOptLdmState::default(); + opt_ldm.seq_store.size = 0; + bt.ldm_get_next_match_and_update_seq_store(&mut opt_ldm, 10, 100); + assert_eq!(opt_ldm.start_pos_in_block, usize::MAX); + assert_eq!(opt_ldm.end_pos_in_block, usize::MAX); +} + +#[test] +fn ldm_get_next_match_marks_window_inside_block() { + let bt = bt_with_seqs(vec![raw(2, 5, 17)]); + let mut opt_ldm = HcOptLdmState::default(); + opt_ldm.seq_store.size = 1; + // 100 bytes remain in block, current pos 10 → window starts at + // 10 + lit_length (2) = 12 and ends 12 + match_length (5) = 17. + bt.ldm_get_next_match_and_update_seq_store(&mut opt_ldm, 10, 100); + assert_eq!(opt_ldm.start_pos_in_block, 12); + assert_eq!(opt_ldm.end_pos_in_block, 17); + assert_eq!(opt_ldm.offset, 17); +} + +#[test] +fn ldm_get_next_match_skips_when_literals_exceed_remaining_block() { + let bt = bt_with_seqs(vec![raw(50, 5, 17)]); + let mut opt_ldm = HcOptLdmState::default(); + opt_ldm.seq_store.size = 1; + // Only 30 bytes left in block but seq starts with 50 literals → + // window stays "no LDM in flight" and cursor advances past the + // remaining bytes. + bt.ldm_get_next_match_and_update_seq_store(&mut opt_ldm, 0, 30); + assert_eq!(opt_ldm.start_pos_in_block, usize::MAX); + assert_eq!(opt_ldm.end_pos_in_block, usize::MAX); + assert_eq!(opt_ldm.seq_store.pos_in_sequence, 30); +} + +#[test] +fn ldm_maybe_add_match_returns_none_before_window() { + let bt = bt_with_seqs(vec![]); + let opt_ldm = HcOptLdmState { + seq_store: store(0), + start_pos_in_block: 20, + end_pos_in_block: 30, + offset: 7, + }; + assert!(bt.ldm_maybe_add_match(&opt_ldm, 10, 4).is_none()); +} + +#[test] +fn ldm_maybe_add_match_returns_none_past_window() { + let bt = bt_with_seqs(vec![]); + let opt_ldm = HcOptLdmState { + seq_store: store(0), + start_pos_in_block: 20, + end_pos_in_block: 30, + offset: 7, + }; + assert!(bt.ldm_maybe_add_match(&opt_ldm, 30, 4).is_none()); + assert!(bt.ldm_maybe_add_match(&opt_ldm, 35, 4).is_none()); +} + +#[test] +fn ldm_maybe_add_match_returns_none_below_min_match() { + let bt = bt_with_seqs(vec![]); + let opt_ldm = HcOptLdmState { + seq_store: store(0), + start_pos_in_block: 20, + end_pos_in_block: 23, + offset: 7, + }; + // window length is 3 < min_match (4) → reject + assert!(bt.ldm_maybe_add_match(&opt_ldm, 20, 4).is_none()); +} + +#[test] +fn ldm_maybe_add_match_emits_candidate_inside_window() { + let bt = bt_with_seqs(vec![]); + let opt_ldm = HcOptLdmState { + seq_store: store(0), + start_pos_in_block: 20, + end_pos_in_block: 30, + offset: 7, + }; + let cand = bt + .ldm_maybe_add_match(&opt_ldm, 22, 4) + .expect("should emit"); + assert_eq!(cand.start, 22); + assert_eq!(cand.offset, 7); + // candidate length = window - (pos - start) = 10 - 2 = 8 + assert_eq!(cand.match_len, 8); +} + +#[test] +fn ldm_process_match_candidate_returns_none_when_store_exhausted() { + let mut bt = bt_with_seqs(vec![raw(2, 5, 17)]); + bt.ldm_sequences.clear(); + let mut opt_ldm = HcOptLdmState::default(); + opt_ldm.seq_store.size = 0; + assert!( + bt.ldm_process_match_candidate(&mut opt_ldm, 0, 100, 4) + .is_none() + ); +} + +#[test] +fn push_candidate_ladder_rejects_below_min_match() { + let mut out: Vec = vec![]; + let mut best = 0usize; + let accepted = BtMatcher::push_candidate_ladder( + &mut out, + &mut best, + MatchCandidate { + start: 10, + offset: 5, + match_len: 2, + }, + 4, + ); + assert!(!accepted); + assert!(out.is_empty()); + assert_eq!(best, 0); +} + +#[test] +fn push_candidate_ladder_rejects_when_not_strictly_better() { + let mut out: Vec = vec![]; + let mut best = 8usize; + let accepted = BtMatcher::push_candidate_ladder( + &mut out, + &mut best, + MatchCandidate { + start: 10, + offset: 5, + match_len: 8, + }, + 4, + ); + // match_len == best_len_for_skip → not strictly greater → rejected + assert!(!accepted); + assert!(out.is_empty()); + assert_eq!(best, 8); +} + +#[test] +fn ldm_get_next_match_clamps_window_to_block_end() { + // Seq match window extends past the block boundary — the + // function clamps `end_pos_in_block` to `curr_block_end_pos` + // and skips through the seq store accordingly. + let bt = bt_with_seqs(vec![raw(0, 100, 99)]); + let mut opt_ldm = HcOptLdmState::default(); + opt_ldm.seq_store.size = 1; + bt.ldm_get_next_match_and_update_seq_store(&mut opt_ldm, 10, 30); + // Window would naturally be [10, 110); clamped to [10, 40) + // because remaining block bytes only allow that span. + assert_eq!(opt_ldm.start_pos_in_block, 10); + assert_eq!(opt_ldm.end_pos_in_block, 40); + assert_eq!(opt_ldm.offset, 99); +} + +#[test] +fn ldm_process_match_candidate_handles_overshoot_into_next_seq() { + // Parser jumped past the active LDM window — `pos_overshoot` + // forces the seq-store cursor forward before re-seeding. + let bt = bt_with_seqs(vec![raw(0, 5, 17), raw(0, 5, 33)]); + let mut opt_ldm = HcOptLdmState { + seq_store: HcRawSeqStore { + pos: 1, + pos_in_sequence: 0, + size: 2, + }, + start_pos_in_block: 0, + end_pos_in_block: 5, + offset: 17, + }; + // curr_pos = 12 (overshot the previous window's end by 7) → must + // consume the overshoot from the next seq, then re-seed. + let _ = bt.ldm_process_match_candidate(&mut opt_ldm, 12, 100, 4); + // After overshoot consumes 7 bytes from seq 1 (which only had + // 5), cursor advances to pos=2 (store exhausted). + assert_eq!(opt_ldm.seq_store.pos, 2); +} + +#[test] +fn ldm_process_match_candidate_reseeds_after_overshoot() { + // Two sequences. Seq 0 was just emitted (cursor advanced past it), + // so the active window points at seq 1's offset (33). The parser + // arrives at position 5 — past the seq 0 window — and must + // re-seed onto seq 1's window without overshoot logic re-reading + // seq 0. + let bt = bt_with_seqs(vec![raw(0, 5, 17), raw(0, 5, 33)]); + let mut opt_ldm = HcOptLdmState { + seq_store: HcRawSeqStore { + pos: 1, + pos_in_sequence: 0, + size: 2, + }, + start_pos_in_block: 0, + end_pos_in_block: 5, + offset: 17, + }; + let cand = bt + .ldm_process_match_candidate(&mut opt_ldm, 5, 100, 4) + .expect("should emit reseeded candidate"); + assert_eq!(cand.start, 5); + assert_eq!(cand.offset, 33); + assert_eq!(cand.match_len, 5); +} diff --git a/zstd/src/encoding/bt/mod.rs b/zstd/src/encoding/bt/mod.rs index d0a2a1823..2dca29262 100644 --- a/zstd/src/encoding/bt/mod.rs +++ b/zstd/src/encoding/bt/mod.rs @@ -752,336 +752,4 @@ impl BtMatcher { } #[cfg(test)] -mod ldm_helper_tests { - //! Unit coverage for the LDM helper cluster that batch 14 lifted - //! onto `impl BtMatcher`. The helpers walk the raw-LDM seq store - //! and translate its (literals, match) sequences into the optimal - //! parser's `MatchCandidate` shape. They are pure logic on - //! [`HcRawSeqStore`] / [`HcOptLdmState`] — no SIMD, no `MatchTable` - //! contact — so they exercise on every CI target. - use alloc::vec; - use alloc::vec::Vec; - - use super::*; - use crate::encoding::opt::ldm::{HcOptLdmState, HcRawSeq, HcRawSeqStore}; - - fn raw(lit: usize, mat: usize, off: usize) -> HcRawSeq { - HcRawSeq { - lit_length: lit, - offset: off, - match_length: mat, - } - } - - fn bt_with_seqs(seqs: Vec) -> BtMatcher { - let mut bt = BtMatcher::new(); - bt.ldm_sequences = seqs; - bt - } - - fn store(size: usize) -> HcRawSeqStore { - HcRawSeqStore { - pos: 0, - pos_in_sequence: 0, - size, - } - } - - /// Regression test for PR #139 Copilot review — threads #1/#2: - /// `prepare_ldm_candidates` must translate the absolute - /// `current_abs_start` (`history_abs_start + window_size - - /// current_len`) into a slice index inside the supplied - /// `history` slice. Before the fix, the code passed - /// `current_abs_start` directly as `block_start` to - /// `LdmProducer::generate_into`, which would index out of - /// bounds (or into the wrong byte range) as soon as - /// `history_abs_start != 0` after window eviction. - /// - /// We construct a `BtMatcher` with an active `LdmProducer` - /// whose table starts empty, hand it a small `history` slice, - /// and call `prepare_ldm_candidates` with absolute coordinates - /// shifted by a non-zero `history_abs_start`. The call must - /// complete without panic — under the pre-fix code it would - /// panic on the `history[block_start..block_end]` slice access - /// inside `generate_into`. - #[test] - #[cfg(feature = "hash")] - fn prepare_ldm_candidates_translates_absolute_positions_to_slice_indices() { - use crate::encoding::ldm::{LdmProducer, params::LdmParams}; - - let mut bt = BtMatcher::new(); - // Activate the producer with hand-tuned small params — - // the upstream zstd btultra2 defaults (`with_window_and_strategy(27, 9)`) - // would allocate `1 << 23 = 8M` entries × 8 bytes = 64 MiB - // for a table this test never actually populates beyond a - // handful of entries. Build a 10-bit hash log instead so - // the table is ~8 KiB — keeps the suite stable under - // parallel nextest on 32-bit / memory-tight CI shards. - bt.ldm_producer = Some(LdmProducer::new(LdmParams { - window_log: 27, - hash_log: 10, - hash_rate_log: 4, - min_match_length: 32, - bucket_size_log: 4, - })); - - // 256 bytes of arbitrary content — enough for the gear - // hash to make progress against the upstream zstd btultra2 - // `min_match_length = 32`. - let history: alloc::vec::Vec = (0u8..=255).collect(); - - // Simulate a frame whose window has already evicted some - // bytes: `history_abs_start = 1024`, and the current block - // covers absolute `[1024, 1280)` which maps to slice - // `[0, 256)` inside `history`. - let history_abs_start = 1024usize; - let current_abs_start = 1024usize; - let current_len = history.len(); - - // Pre-fix: would panic with `slice index out of bounds` - // on `history[1024..1280]`. Post-fix: translates to - // `[0..256]` and runs cleanly. - bt.prepare_ldm_candidates(&history, history_abs_start, current_abs_start, current_len); - // Cross-block fixture isn't engineered to emit a match — - // the assertion is simply that the call succeeded. - } - - #[test] - fn ldm_skip_raw_seq_store_bytes_walks_whole_sequences() { - let bt = bt_with_seqs(vec![raw(2, 3, 100), raw(4, 1, 200), raw(0, 5, 50)]); - let mut s = store(3); - // 5 = first seq (lit+match=5) → advance to seq 1 with pos_in_sequence=0 - bt.ldm_skip_raw_seq_store_bytes(&mut s, 5); - assert_eq!(s.pos, 1); - assert_eq!(s.pos_in_sequence, 0); - } - - #[test] - fn ldm_skip_raw_seq_store_bytes_leaves_partial_offset() { - let bt = bt_with_seqs(vec![raw(2, 3, 100), raw(4, 1, 200)]); - let mut s = store(2); - // 4 < first seq's 5 bytes → stay on seq 0, pos_in_sequence=4 - bt.ldm_skip_raw_seq_store_bytes(&mut s, 4); - assert_eq!(s.pos, 0); - assert_eq!(s.pos_in_sequence, 4); - } - - #[test] - fn ldm_skip_raw_seq_store_bytes_exhausts_store_when_distance_too_large() { - let bt = bt_with_seqs(vec![raw(1, 1, 1), raw(1, 1, 1)]); - let mut s = store(2); - // 10 bytes total but store only holds 4 (2 seqs × 2 bytes each) - bt.ldm_skip_raw_seq_store_bytes(&mut s, 10); - assert_eq!(s.pos, 2); - assert_eq!(s.pos_in_sequence, 0); - } - - #[test] - fn ldm_get_next_match_empty_store_clears_window() { - let bt = bt_with_seqs(vec![]); - let mut opt_ldm = HcOptLdmState::default(); - opt_ldm.seq_store.size = 0; - bt.ldm_get_next_match_and_update_seq_store(&mut opt_ldm, 10, 100); - assert_eq!(opt_ldm.start_pos_in_block, usize::MAX); - assert_eq!(opt_ldm.end_pos_in_block, usize::MAX); - } - - #[test] - fn ldm_get_next_match_marks_window_inside_block() { - let bt = bt_with_seqs(vec![raw(2, 5, 17)]); - let mut opt_ldm = HcOptLdmState::default(); - opt_ldm.seq_store.size = 1; - // 100 bytes remain in block, current pos 10 → window starts at - // 10 + lit_length (2) = 12 and ends 12 + match_length (5) = 17. - bt.ldm_get_next_match_and_update_seq_store(&mut opt_ldm, 10, 100); - assert_eq!(opt_ldm.start_pos_in_block, 12); - assert_eq!(opt_ldm.end_pos_in_block, 17); - assert_eq!(opt_ldm.offset, 17); - } - - #[test] - fn ldm_get_next_match_skips_when_literals_exceed_remaining_block() { - let bt = bt_with_seqs(vec![raw(50, 5, 17)]); - let mut opt_ldm = HcOptLdmState::default(); - opt_ldm.seq_store.size = 1; - // Only 30 bytes left in block but seq starts with 50 literals → - // window stays "no LDM in flight" and cursor advances past the - // remaining bytes. - bt.ldm_get_next_match_and_update_seq_store(&mut opt_ldm, 0, 30); - assert_eq!(opt_ldm.start_pos_in_block, usize::MAX); - assert_eq!(opt_ldm.end_pos_in_block, usize::MAX); - assert_eq!(opt_ldm.seq_store.pos_in_sequence, 30); - } - - #[test] - fn ldm_maybe_add_match_returns_none_before_window() { - let bt = bt_with_seqs(vec![]); - let opt_ldm = HcOptLdmState { - seq_store: store(0), - start_pos_in_block: 20, - end_pos_in_block: 30, - offset: 7, - }; - assert!(bt.ldm_maybe_add_match(&opt_ldm, 10, 4).is_none()); - } - - #[test] - fn ldm_maybe_add_match_returns_none_past_window() { - let bt = bt_with_seqs(vec![]); - let opt_ldm = HcOptLdmState { - seq_store: store(0), - start_pos_in_block: 20, - end_pos_in_block: 30, - offset: 7, - }; - assert!(bt.ldm_maybe_add_match(&opt_ldm, 30, 4).is_none()); - assert!(bt.ldm_maybe_add_match(&opt_ldm, 35, 4).is_none()); - } - - #[test] - fn ldm_maybe_add_match_returns_none_below_min_match() { - let bt = bt_with_seqs(vec![]); - let opt_ldm = HcOptLdmState { - seq_store: store(0), - start_pos_in_block: 20, - end_pos_in_block: 23, - offset: 7, - }; - // window length is 3 < min_match (4) → reject - assert!(bt.ldm_maybe_add_match(&opt_ldm, 20, 4).is_none()); - } - - #[test] - fn ldm_maybe_add_match_emits_candidate_inside_window() { - let bt = bt_with_seqs(vec![]); - let opt_ldm = HcOptLdmState { - seq_store: store(0), - start_pos_in_block: 20, - end_pos_in_block: 30, - offset: 7, - }; - let cand = bt - .ldm_maybe_add_match(&opt_ldm, 22, 4) - .expect("should emit"); - assert_eq!(cand.start, 22); - assert_eq!(cand.offset, 7); - // candidate length = window - (pos - start) = 10 - 2 = 8 - assert_eq!(cand.match_len, 8); - } - - #[test] - fn ldm_process_match_candidate_returns_none_when_store_exhausted() { - let mut bt = bt_with_seqs(vec![raw(2, 5, 17)]); - bt.ldm_sequences.clear(); - let mut opt_ldm = HcOptLdmState::default(); - opt_ldm.seq_store.size = 0; - assert!( - bt.ldm_process_match_candidate(&mut opt_ldm, 0, 100, 4) - .is_none() - ); - } - - #[test] - fn push_candidate_ladder_rejects_below_min_match() { - let mut out: Vec = vec![]; - let mut best = 0usize; - let accepted = BtMatcher::push_candidate_ladder( - &mut out, - &mut best, - MatchCandidate { - start: 10, - offset: 5, - match_len: 2, - }, - 4, - ); - assert!(!accepted); - assert!(out.is_empty()); - assert_eq!(best, 0); - } - - #[test] - fn push_candidate_ladder_rejects_when_not_strictly_better() { - let mut out: Vec = vec![]; - let mut best = 8usize; - let accepted = BtMatcher::push_candidate_ladder( - &mut out, - &mut best, - MatchCandidate { - start: 10, - offset: 5, - match_len: 8, - }, - 4, - ); - // match_len == best_len_for_skip → not strictly greater → rejected - assert!(!accepted); - assert!(out.is_empty()); - assert_eq!(best, 8); - } - - #[test] - fn ldm_get_next_match_clamps_window_to_block_end() { - // Seq match window extends past the block boundary — the - // function clamps `end_pos_in_block` to `curr_block_end_pos` - // and skips through the seq store accordingly. - let bt = bt_with_seqs(vec![raw(0, 100, 99)]); - let mut opt_ldm = HcOptLdmState::default(); - opt_ldm.seq_store.size = 1; - bt.ldm_get_next_match_and_update_seq_store(&mut opt_ldm, 10, 30); - // Window would naturally be [10, 110); clamped to [10, 40) - // because remaining block bytes only allow that span. - assert_eq!(opt_ldm.start_pos_in_block, 10); - assert_eq!(opt_ldm.end_pos_in_block, 40); - assert_eq!(opt_ldm.offset, 99); - } - - #[test] - fn ldm_process_match_candidate_handles_overshoot_into_next_seq() { - // Parser jumped past the active LDM window — `pos_overshoot` - // forces the seq-store cursor forward before re-seeding. - let bt = bt_with_seqs(vec![raw(0, 5, 17), raw(0, 5, 33)]); - let mut opt_ldm = HcOptLdmState { - seq_store: HcRawSeqStore { - pos: 1, - pos_in_sequence: 0, - size: 2, - }, - start_pos_in_block: 0, - end_pos_in_block: 5, - offset: 17, - }; - // curr_pos = 12 (overshot the previous window's end by 7) → must - // consume the overshoot from the next seq, then re-seed. - let _ = bt.ldm_process_match_candidate(&mut opt_ldm, 12, 100, 4); - // After overshoot consumes 7 bytes from seq 1 (which only had - // 5), cursor advances to pos=2 (store exhausted). - assert_eq!(opt_ldm.seq_store.pos, 2); - } - - #[test] - fn ldm_process_match_candidate_reseeds_after_overshoot() { - // Two sequences. Seq 0 was just emitted (cursor advanced past it), - // so the active window points at seq 1's offset (33). The parser - // arrives at position 5 — past the seq 0 window — and must - // re-seed onto seq 1's window without overshoot logic re-reading - // seq 0. - let bt = bt_with_seqs(vec![raw(0, 5, 17), raw(0, 5, 33)]); - let mut opt_ldm = HcOptLdmState { - seq_store: HcRawSeqStore { - pos: 1, - pos_in_sequence: 0, - size: 2, - }, - start_pos_in_block: 0, - end_pos_in_block: 5, - offset: 17, - }; - let cand = bt - .ldm_process_match_candidate(&mut opt_ldm, 5, 100, 4) - .expect("should emit reseeded candidate"); - assert_eq!(cand.start, 5); - assert_eq!(cand.offset, 33); - assert_eq!(cand.match_len, 5); - } -} +mod ldm_helper_tests; diff --git a/zstd/src/encoding/compress_bound_tests.rs b/zstd/src/encoding/compress_bound_tests.rs new file mode 100644 index 000000000..a64ecf71c --- /dev/null +++ b/zstd/src/encoding/compress_bound_tests.rs @@ -0,0 +1,30 @@ +use super::{CompressionLevel, compress_bound, compress_slice_to_vec}; + +#[test] +fn matches_upstream_formula_below_threshold() { + // src_size + (src_size >> 8) + ((128 KiB - src_size) >> 11). + assert_eq!(compress_bound(0), 64); + assert_eq!(compress_bound(4096), 4096 + 16 + 62); +} + +#[test] +fn drops_margin_at_and_above_threshold() { + let lower = 128 * 1024; + assert_eq!(compress_bound(lower), lower + (lower >> 8)); + assert_eq!(compress_bound(lower + 1), (lower + 1) + ((lower + 1) >> 8)); +} + +#[test] +fn saturates_instead_of_wrapping() { + // No allocation this large can exist; the ceiling is the right sentinel. + assert_eq!(compress_bound(usize::MAX), usize::MAX); +} + +#[test] +fn always_fits_real_compressed_output() { + for len in [0usize, 1, 100, 4096, 200_000] { + let data = alloc::vec![7u8; len]; + let out = compress_slice_to_vec(&data, CompressionLevel::Default); + assert!(out.len() <= compress_bound(len), "len={len}"); + } +} diff --git a/zstd/src/encoding/cparams.rs b/zstd/src/encoding/cparams.rs index 18881e9ae..4a4bb969b 100644 --- a/zstd/src/encoding/cparams.rs +++ b/zstd/src/encoding/cparams.rs @@ -326,30 +326,4 @@ pub(crate) fn create_cdict_table_logs( } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn create_cdict_table_logs_downsizes_for_unknown_source_with_dict() { - // create_cdict + a dictionary + unknown source size assumes a minimal - // (513-byte) source, so the prepared CDict tables down-size far below - // the requested input logs instead of holding max-size tables - // (`ZSTD_adjustCParams_internal` with `ZSTD_cpm_createCDict`). - let (hash_log, chain_log) = create_cdict_table_logs( - 27, 27, 28, /* uses_bt */ true, /* dict_size */ 4096, - ); - assert!( - hash_log < 27, - "hash_log {hash_log} must shrink for a tiny CDict source", - ); - assert!(chain_log <= 28, "chain_log {chain_log} must not grow"); - - // A zero-dict CDict does NOT force the minSrcSize assumption (the - // `dict_size != 0` guard), so the unknown-source path leaves the logs as - // requested. - let (h2, c2) = - create_cdict_table_logs(20, 20, 21, /* uses_bt */ false, /* dict_size */ 0); - assert_eq!(h2, 20, "no-dict CDict keeps the requested hash_log"); - assert_eq!(c2, 21, "no-dict CDict keeps the requested chain_log"); - } -} +mod tests; diff --git a/zstd/src/encoding/cparams/tests.rs b/zstd/src/encoding/cparams/tests.rs new file mode 100644 index 000000000..0d6deacad --- /dev/null +++ b/zstd/src/encoding/cparams/tests.rs @@ -0,0 +1,25 @@ +use super::*; + +#[test] +fn create_cdict_table_logs_downsizes_for_unknown_source_with_dict() { + // create_cdict + a dictionary + unknown source size assumes a minimal + // (513-byte) source, so the prepared CDict tables down-size far below + // the requested input logs instead of holding max-size tables + // (`ZSTD_adjustCParams_internal` with `ZSTD_cpm_createCDict`). + let (hash_log, chain_log) = create_cdict_table_logs( + 27, 27, 28, /* uses_bt */ true, /* dict_size */ 4096, + ); + assert!( + hash_log < 27, + "hash_log {hash_log} must shrink for a tiny CDict source", + ); + assert!(chain_log <= 28, "chain_log {chain_log} must not grow"); + + // A zero-dict CDict does NOT force the minSrcSize assumption (the + // `dict_size != 0` guard), so the unknown-source path leaves the logs as + // requested. + let (h2, c2) = + create_cdict_table_logs(20, 20, 21, /* uses_bt */ false, /* dict_size */ 0); + assert_eq!(h2, 20, "no-dict CDict keeps the requested hash_log"); + assert_eq!(c2, 21, "no-dict CDict keeps the requested chain_log"); +} diff --git a/zstd/src/encoding/dfast/extend_with_repcode_tests.rs b/zstd/src/encoding/dfast/extend_with_repcode_tests.rs new file mode 100644 index 000000000..c2d28c54d --- /dev/null +++ b/zstd/src/encoding/dfast/extend_with_repcode_tests.rs @@ -0,0 +1,412 @@ +//! Targeted regression coverage for `extend_with_repcode_after_match`. +//! +//! These tests intentionally bypass the higher-level +//! `compress_to_vec` roundtrip path used by `cross_validation` so +//! that a failure pinpoints the post-match rep helper rather than +//! firing somewhere downstream (block writer / huff0 / FSE / decode). +//! The capture closure records the exact sequence stream the matcher +//! emits, which is what the assertions check. +use alloc::vec; +use alloc::vec::Vec; + +use super::*; + +/// Capture every sequence the matcher emits into an owned record, +/// so the assertions can match on `lit_len` / `offset` / `match_len` +/// shape directly. `Sequence::Triple` carries borrowed literals; we +/// take their length and discard the bytes (the test only cares +/// about the structural shape, not the literal content). +#[derive(Debug, Clone, PartialEq, Eq)] +enum CapturedSeq { + Triple { + lit_len: usize, + offset: usize, + match_len: usize, + }, + Literals { + lit_len: usize, + }, +} + +fn record_seq<'a>(out: &'a mut Vec) -> impl FnMut(Sequence<'_>) + 'a { + move |seq| match seq { + Sequence::Triple { + literals, + offset, + match_len, + } => out.push(CapturedSeq::Triple { + lit_len: literals.len(), + offset, + match_len, + }), + Sequence::Literals { literals } => out.push(CapturedSeq::Literals { + lit_len: literals.len(), + }), + } +} + +fn build_dfast_with(data: &[u8]) -> DfastMatchGenerator { + // Window sized to the block so the matcher does not start + // trimming history mid-test. + let mut dfast = DfastMatchGenerator::new(data.len().next_power_of_two().max(64)); + dfast.ensure_hash_tables(); + dfast.add_data(data.to_vec(), |_| {}); + dfast +} + +/// Direct call into [`DfastMatchGenerator::extend_with_repcode_after_match`] +/// with a hand-built post-primary-match state. Going through +/// `start_matching` is unreliable for this assertion because the +/// primary `best_match` greedily consumes a constant run in a +/// single `Triple` (offset 1, match_len = block - 1), leaving the +/// helper nothing to extend. Instead we set up the state the +/// helper expects after a primary emit and verify it chains +/// rep-0 sequences for as many bytes as the rep predicate +/// matches. +#[test] +fn dfast_repcode_extension_emits_zero_literal_rep_on_constant_run() { + let data: Vec = vec![b'A'; 64]; + let mut dfast = build_dfast_with(&data); + + // Post-primary-match state: pretend a previous sequence emitted + // with offset = 4 (`offset_hist[0]`). Under the upstream zstd swap the + // post-match rep probe consults `offset_hist[1]`, here set to + // 1 so every subsequent byte (constant 'A') matches its + // predecessor. + dfast.offset_hist = [4, 1, 8]; + let current_abs_start = dfast.history_abs_start + dfast.window_size - data.len(); + let current_len = data.len(); + // Start the helper mid-block; the leading bytes are the + // "literals + match" the (simulated) primary would have + // covered. `literals_start == pos` is the post-emit invariant + // — `lit_len` for the next sequence is zero. + let pos = 10usize; + let mut literals_start = pos; + + let mut seqs = Vec::new(); + let new_pos = { + let mut rec = record_seq(&mut seqs); + dfast.extend_with_repcode_after_match( + current_abs_start, + current_len, + pos, + &mut literals_start, + &mut rec, + ) + }; + + assert!( + new_pos > pos, + "helper must advance pos past at least one rep match \ + (pos={pos}, new_pos={new_pos})" + ); + assert_eq!( + literals_start, new_pos, + "helper must keep literals_start == new_pos so the caller's main \ + loop sees zero pending literals after the rep chain" + ); + assert!(!seqs.is_empty(), "helper must emit at least one Triple"); + for seq in &seqs { + match seq { + CapturedSeq::Triple { + lit_len, + offset, + match_len: _, + } => { + assert_eq!( + *lit_len, 0, + "rep emission must be zero-literal (got {seq:?})" + ); + assert_eq!( + *offset, 1, + "rep emission must use the swapped-in offset_hist[1] = 1 \ + (got {seq:?})" + ); + } + CapturedSeq::Literals { .. } => { + panic!("rep extension must not emit a Literals tail: {seq:?}"); + } + } + } +} + +/// Cross-block / retained-history case: probe with `offset > pos` +/// (where `pos` is block-local) so the candidate lives in retained +/// history from a previously committed block. The +/// CodeRabbit-flagged `rep > pos` guard would have rejected +/// exactly this path — the current implementation only gates on +/// `cur_idx.checked_sub(rep)` so the helper accepts the cross- +/// block offset and emits the rep sequence. +#[test] +fn dfast_repcode_extension_walks_into_retained_history() { + let block_a: Vec = vec![b'C'; 64]; + let block_b: Vec = vec![b'C'; 32]; + let mut dfast = DfastMatchGenerator::new(256); + dfast.ensure_hash_tables(); + dfast.add_data(block_a, |_| {}); + dfast.add_data(block_b.clone(), |_| {}); + + // Post-primary-match state targeting cross-block rep: probe + // offset = 40 (a candidate inside block A bytes), block-local + // cursor = 5 (so `rep > pos` under the rejected guard). + dfast.offset_hist = [4, 40, 8]; + let current_len = block_b.len(); + let current_abs_start = dfast.history_abs_start + dfast.window_size - current_len; + let pos = 5usize; + let mut literals_start = pos; + + let mut seqs = Vec::new(); + let new_pos = { + let mut rec = record_seq(&mut seqs); + dfast.extend_with_repcode_after_match( + current_abs_start, + current_len, + pos, + &mut literals_start, + &mut rec, + ) + }; + + assert!( + new_pos > pos, + "rep with offset > block-local pos must still emit a match when the \ + candidate lives in retained history (pos={pos}, new_pos={new_pos})" + ); + assert_eq!(seqs.len(), 1, "expected one rep emit, got {seqs:?}"); + match &seqs[0] { + CapturedSeq::Triple { + lit_len, + offset, + match_len: _, + } => { + assert_eq!(*lit_len, 0, "rep emit must be zero-literal"); + assert_eq!(*offset, 40, "rep emit must use the cross-block offset 40"); + } + other => panic!("expected Triple, got {other:?}"), + } +} + +/// The helper accepts 4-byte rep extensions (upstream zstd `MINMATCH = 4`), +/// not the main-loop `DFAST_MIN_MATCH_LEN = 5` floor. A regression +/// back above 4 would still pass the constant-run / cross-block tests +/// above (their rep matches extend much further), so this fixture +/// is built so the rep matches EXACTLY 4 bytes before terminating: +/// the byte at `pos + 4` differs from the byte at `pos + 4 - rep`. +/// +/// Fixture (32 bytes, indices `0..=31`): +/// `"ABCD????ABCD!??????????ABCDX????"` +/// 01234567890123456789012345678901 (ones digit) +/// 1111111111222222222233 (tens digit, aligned) +/// +/// Probe state: +/// * `offset_hist[1] = 8` → rep probe reads `concat[pos - 8..]`. +/// * `pos = 8` → `concat[8..12] = "ABCD"`, `concat[0..4] = "ABCD"` +/// → 4 bytes match. +/// * `concat[12] = '!'` vs `concat[4] = '?'` → 5th byte mismatch, +/// so the rep extension stops at exactly 4 bytes. +#[test] +fn dfast_repcode_extension_accepts_exactly_four_byte_rep() { + // Block: "ABCD????" (8) + "ABCD!" (5) + "??????????" (10) + + // "ABCDX" (5) + "????" (4) = 32 bytes total. The + // important invariants are `concat[0..4] == "ABCD"`, + // `concat[8..12] == "ABCD"`, and `concat[12] = '!'` + // (so byte 12 ≠ byte 4 = '?', stopping the rep at + // length 4). The trailing bytes are irrelevant — we + // only iterate the helper at `pos = 8` and the rep + // chain terminates after one 4-byte emit because the + // next rep probe (post-swap) would need bytes at + // `pos + 4` to match a different offset. + let data: Vec = b"ABCD????ABCD!??????????ABCDX????".to_vec(); + assert_eq!(data.len(), 32, "fixture invariant: 32 bytes"); + let mut dfast = DfastMatchGenerator::new(64); + dfast.ensure_hash_tables(); + dfast.add_data(data.clone(), |_| {}); + + dfast.offset_hist = [12, 8, 4]; + let current_abs_start = dfast.history_abs_start + dfast.window_size - data.len(); + let current_len = data.len(); + let pos = 8usize; + let mut literals_start = pos; + + let mut seqs = Vec::new(); + let new_pos = { + let mut rec = record_seq(&mut seqs); + dfast.extend_with_repcode_after_match( + current_abs_start, + current_len, + pos, + &mut literals_start, + &mut rec, + ) + }; + + // Helper must emit a single 4-byte rep, then stop because + // the 5th byte mismatches. + assert_eq!(seqs.len(), 1, "expected one 4-byte rep emit, got {seqs:?}"); + match &seqs[0] { + CapturedSeq::Triple { + lit_len, + offset, + match_len, + } => { + assert_eq!(*lit_len, 0, "rep emit must be zero-literal"); + assert_eq!(*offset, 8, "rep emit must use offset 8 (offset_hist[1])"); + assert_eq!( + *match_len, 4, + "rep emit must be exactly 4 bytes (upstream zstd MINMATCH floor). \ + A regression back to DFAST_MIN_MATCH_LEN > 4 would skip \ + this emission entirely and the test would fail with 0 seqs." + ); + } + other => panic!("expected Triple, got {other:?}"), + } + assert_eq!(new_pos, pos + 4, "pos must advance by exactly 4"); + assert_eq!(literals_start, new_pos, "literals_start must follow pos"); +} + +/// Integration coverage for the **fast-loop** call sites of +/// `extend_with_repcode_after_match` inside +/// `start_matching_fast_loop`. The direct-call tests above pin +/// down the helper's contract; this test drives the full fast +/// loop end-to-end through the production `compress_to_vec` +/// pipeline on a fixture engineered to exercise the post-match +/// rep chain on the fast-loop path. +/// +/// `CompressionLevel::Default` is the production config that +/// enables `use_fast_loop = true` (see `Matcher::reset` in +/// `match_generator.rs`). The fixture alternates 60-byte runs of +/// `'A'` with single `'B'` break bytes and a short `'A'` tail +/// per cycle — the breaks terminate the fast loop's primary match +/// early, so subsequent iterations have runway for the helper to +/// chain additional reps. A regression that broke either fast- +/// loop helper call site surfaces as a roundtrip failure (decoded +/// != input) or a ratio explosion. Constructing +/// `DfastMatchGenerator` directly and asserting on captured +/// sequences was attempted but the fixture engineering is +/// brittle: the fast loop's primary match on simple constant +/// fixtures consumes the entire remaining block in a single +/// Triple, leaving no bytes for the helper to extend. The +/// high-level roundtrip sidesteps that fragility while still +/// routing through the same call site via the production driver. +/// +/// Gated on `feature = "std"`: the `Read::read_to_end` method +/// used to drain `StreamingDecoder` resolves to `std::io::Read` +/// only when std is enabled. Under no-std `StreamingDecoder` +/// implements the crate's `io_nostd::Read` alias instead, and +/// the call site has to be rewritten through that trait. The +/// fast-loop helper itself is exercised under both +/// configurations by the direct-call tests above plus the +/// `cross_validation` Default-level roundtrip — gating this one +/// integration test on std loses no coverage, only saves the +/// dual-trait rewrite. +#[cfg(feature = "std")] +#[test] +fn dfast_default_level_roundtrip_with_repetitive_breaks_exercises_fast_loop() { + // ~4 KiB of input: 64 cycles of [60 'A's, 1 'B', 3 'A's]. + let mut data: Vec = Vec::with_capacity(64 * 64); + for _ in 0..64 { + data.extend_from_slice(&[b'A'; 60]); + data.push(b'B'); + data.extend_from_slice(&[b'A'; 3]); + } + assert!( + data.len() > 4000, + "fixture invariant: long enough for fast loop" + ); + + let compressed = crate::encoding::compress_to_vec( + data.as_slice(), + crate::encoding::CompressionLevel::Default, + ); + + // Decompress and assert byte-for-byte parity. A regression + // that broke the fast-loop helper call would either produce + // invalid frames (decode error) or wrong bytes (mismatch). + let mut decoder = crate::decoding::StreamingDecoder::new(compressed.as_slice()) + .expect("default-level frame must decode"); + let mut decoded = Vec::with_capacity(data.len()); + // Under `feature = "std"` (the gate above) `StreamingDecoder` + // implements `std::io::Read`, so `Read::read_to_end` resolves + // through the standard library's blanket implementation. + std::io::Read::read_to_end(&mut decoder, &mut decoded) + .expect("fast-loop output must round-trip cleanly"); + assert_eq!( + decoded, data, + "Default-level (use_fast_loop = true) roundtrip must be \ + byte-for-byte exact on the repetitive-breaks fixture" + ); + + // Ratio sanity: the post-match rep helper is what makes + // repetitive runs compress aggressively on the fast-loop + // path. A regression to a no-op helper would still produce + // some compression via the primary match, but the ratio + // would degrade. A 2:1 floor is conservative enough not to + // flake on small fixture changes while still catching + // structural failures of the fast loop. + assert!( + compressed.len() * 2 < data.len(), + "fast loop must compress repetitive runs to at least 2:1, \ + got {} → {} bytes", + data.len(), + compressed.len() + ); +} + +/// The borrowed one-shot scan (no per-block `commit_space` copy) must +/// emit the byte-identical sequence stream as the owned `history`-copying +/// path for an in-window input: the window far exceeds the input so the +/// owned path never evicts, making its accumulated `history` identical to +/// the borrowed buffer, and the borrowed scan's absolute positions / +/// seam re-seed reproduce the same match decisions. This is the +/// correctness contract behind routing dfast through `borrowed_eligible`. +#[test] +fn dfast_borrowed_window_matches_owned_sequence_stream() { + // Repeating pattern (period 64) so the matcher emits real matches, + // split across two blocks. + let mut whole: Vec = Vec::with_capacity(512); + for i in 0..512usize { + whole.push((i % 64) as u8); + } + let split = 300usize; + let window = 1usize << 16; // 64 KiB >> 512-byte input: no eviction. + + // Owned: commit each block, then scan. + let mut owned = DfastMatchGenerator::new(window); + owned.ensure_hash_tables(); + let mut owned_seqs: Vec = Vec::new(); + owned.add_data(whole[..split].to_vec(), |_| {}); + { + let mut rec = record_seq(&mut owned_seqs); + owned.start_matching(&mut rec); + } + owned.add_data(whole[split..].to_vec(), |_| {}); + { + let mut rec = record_seq(&mut owned_seqs); + owned.start_matching(&mut rec); + } + + // Borrowed: same bytes, scanned in place by absolute block range. + let mut borrowed = DfastMatchGenerator::new(window); + borrowed.ensure_hash_tables(); + let mut borrowed_seqs: Vec = Vec::new(); + // SAFETY: `whole` outlives both scans; `borrowed` is dropped at end + // of scope before `whole`, so the window never dangles. + unsafe { borrowed.set_borrowed_window(&whole) }; + { + let mut rec = record_seq(&mut borrowed_seqs); + borrowed.start_matching_borrowed(0, split, &mut rec); + } + { + let mut rec = record_seq(&mut borrowed_seqs); + borrowed.start_matching_borrowed(split, whole.len(), &mut rec); + } + + assert_eq!( + owned_seqs, borrowed_seqs, + "dfast borrowed window must emit the identical sequence stream as the owned path", + ); + assert_eq!( + owned.offset_hist, borrowed.offset_hist, + "rep state must match after both scans", + ); +} diff --git a/zstd/src/encoding/dfast/mod.rs b/zstd/src/encoding/dfast/mod.rs index eb0b85574..dfb137911 100644 --- a/zstd/src/encoding/dfast/mod.rs +++ b/zstd/src/encoding/dfast/mod.rs @@ -19,10 +19,11 @@ use super::blocks::encode_offset_with_history; use super::dict_attach::DictAttach; use super::fastpath::{FastpathKernel, select_kernel}; use super::incompressible::block_looks_incompressible; +use super::levels::config::MIN_WINDOW_LOG; use super::match_generator::{ DFAST_EMPTY_SLOT, DFAST_HASH_BITS, DFAST_INCOMPRESSIBLE_SKIP_STEP, DFAST_MAX_SKIP_STEP, DFAST_MIN_MATCH_LEN, DFAST_REBASE_GUARD_BAND, DFAST_SHORT_HASH_BITS_DELTA, - DFAST_SHORT_HASH_LOOKAHEAD, DFAST_SKIP_STEP_GROWTH_INTERVAL, MIN_WINDOW_LOG, + DFAST_SHORT_HASH_LOOKAHEAD, DFAST_SKIP_STEP_GROWTH_INTERVAL, }; use super::match_table::helpers::{common_prefix_len_with_kernel, extend_backwards_shared}; use super::match_table::storage::{REBASE_RESET_FLOOR_CEILING, check_stream_abs_headroom}; @@ -2862,417 +2863,4 @@ impl DfastMatchGenerator { } #[cfg(test)] -mod extend_with_repcode_tests { - //! Targeted regression coverage for `extend_with_repcode_after_match`. - //! - //! These tests intentionally bypass the higher-level - //! `compress_to_vec` roundtrip path used by `cross_validation` so - //! that a failure pinpoints the post-match rep helper rather than - //! firing somewhere downstream (block writer / huff0 / FSE / decode). - //! The capture closure records the exact sequence stream the matcher - //! emits, which is what the assertions check. - use alloc::vec; - use alloc::vec::Vec; - - use super::*; - - /// Capture every sequence the matcher emits into an owned record, - /// so the assertions can match on `lit_len` / `offset` / `match_len` - /// shape directly. `Sequence::Triple` carries borrowed literals; we - /// take their length and discard the bytes (the test only cares - /// about the structural shape, not the literal content). - #[derive(Debug, Clone, PartialEq, Eq)] - enum CapturedSeq { - Triple { - lit_len: usize, - offset: usize, - match_len: usize, - }, - Literals { - lit_len: usize, - }, - } - - fn record_seq<'a>(out: &'a mut Vec) -> impl FnMut(Sequence<'_>) + 'a { - move |seq| match seq { - Sequence::Triple { - literals, - offset, - match_len, - } => out.push(CapturedSeq::Triple { - lit_len: literals.len(), - offset, - match_len, - }), - Sequence::Literals { literals } => out.push(CapturedSeq::Literals { - lit_len: literals.len(), - }), - } - } - - fn build_dfast_with(data: &[u8]) -> DfastMatchGenerator { - // Window sized to the block so the matcher does not start - // trimming history mid-test. - let mut dfast = DfastMatchGenerator::new(data.len().next_power_of_two().max(64)); - dfast.ensure_hash_tables(); - dfast.add_data(data.to_vec(), |_| {}); - dfast - } - - /// Direct call into [`DfastMatchGenerator::extend_with_repcode_after_match`] - /// with a hand-built post-primary-match state. Going through - /// `start_matching` is unreliable for this assertion because the - /// primary `best_match` greedily consumes a constant run in a - /// single `Triple` (offset 1, match_len = block - 1), leaving the - /// helper nothing to extend. Instead we set up the state the - /// helper expects after a primary emit and verify it chains - /// rep-0 sequences for as many bytes as the rep predicate - /// matches. - #[test] - fn dfast_repcode_extension_emits_zero_literal_rep_on_constant_run() { - let data: Vec = vec![b'A'; 64]; - let mut dfast = build_dfast_with(&data); - - // Post-primary-match state: pretend a previous sequence emitted - // with offset = 4 (`offset_hist[0]`). Under the upstream zstd swap the - // post-match rep probe consults `offset_hist[1]`, here set to - // 1 so every subsequent byte (constant 'A') matches its - // predecessor. - dfast.offset_hist = [4, 1, 8]; - let current_abs_start = dfast.history_abs_start + dfast.window_size - data.len(); - let current_len = data.len(); - // Start the helper mid-block; the leading bytes are the - // "literals + match" the (simulated) primary would have - // covered. `literals_start == pos` is the post-emit invariant - // — `lit_len` for the next sequence is zero. - let pos = 10usize; - let mut literals_start = pos; - - let mut seqs = Vec::new(); - let new_pos = { - let mut rec = record_seq(&mut seqs); - dfast.extend_with_repcode_after_match( - current_abs_start, - current_len, - pos, - &mut literals_start, - &mut rec, - ) - }; - - assert!( - new_pos > pos, - "helper must advance pos past at least one rep match \ - (pos={pos}, new_pos={new_pos})" - ); - assert_eq!( - literals_start, new_pos, - "helper must keep literals_start == new_pos so the caller's main \ - loop sees zero pending literals after the rep chain" - ); - assert!(!seqs.is_empty(), "helper must emit at least one Triple"); - for seq in &seqs { - match seq { - CapturedSeq::Triple { - lit_len, - offset, - match_len: _, - } => { - assert_eq!( - *lit_len, 0, - "rep emission must be zero-literal (got {seq:?})" - ); - assert_eq!( - *offset, 1, - "rep emission must use the swapped-in offset_hist[1] = 1 \ - (got {seq:?})" - ); - } - CapturedSeq::Literals { .. } => { - panic!("rep extension must not emit a Literals tail: {seq:?}"); - } - } - } - } - - /// Cross-block / retained-history case: probe with `offset > pos` - /// (where `pos` is block-local) so the candidate lives in retained - /// history from a previously committed block. The - /// CodeRabbit-flagged `rep > pos` guard would have rejected - /// exactly this path — the current implementation only gates on - /// `cur_idx.checked_sub(rep)` so the helper accepts the cross- - /// block offset and emits the rep sequence. - #[test] - fn dfast_repcode_extension_walks_into_retained_history() { - let block_a: Vec = vec![b'C'; 64]; - let block_b: Vec = vec![b'C'; 32]; - let mut dfast = DfastMatchGenerator::new(256); - dfast.ensure_hash_tables(); - dfast.add_data(block_a, |_| {}); - dfast.add_data(block_b.clone(), |_| {}); - - // Post-primary-match state targeting cross-block rep: probe - // offset = 40 (a candidate inside block A bytes), block-local - // cursor = 5 (so `rep > pos` under the rejected guard). - dfast.offset_hist = [4, 40, 8]; - let current_len = block_b.len(); - let current_abs_start = dfast.history_abs_start + dfast.window_size - current_len; - let pos = 5usize; - let mut literals_start = pos; - - let mut seqs = Vec::new(); - let new_pos = { - let mut rec = record_seq(&mut seqs); - dfast.extend_with_repcode_after_match( - current_abs_start, - current_len, - pos, - &mut literals_start, - &mut rec, - ) - }; - - assert!( - new_pos > pos, - "rep with offset > block-local pos must still emit a match when the \ - candidate lives in retained history (pos={pos}, new_pos={new_pos})" - ); - assert_eq!(seqs.len(), 1, "expected one rep emit, got {seqs:?}"); - match &seqs[0] { - CapturedSeq::Triple { - lit_len, - offset, - match_len: _, - } => { - assert_eq!(*lit_len, 0, "rep emit must be zero-literal"); - assert_eq!(*offset, 40, "rep emit must use the cross-block offset 40"); - } - other => panic!("expected Triple, got {other:?}"), - } - } - - /// The helper accepts 4-byte rep extensions (upstream zstd `MINMATCH = 4`), - /// not the main-loop `DFAST_MIN_MATCH_LEN = 5` floor. A regression - /// back above 4 would still pass the constant-run / cross-block tests - /// above (their rep matches extend much further), so this fixture - /// is built so the rep matches EXACTLY 4 bytes before terminating: - /// the byte at `pos + 4` differs from the byte at `pos + 4 - rep`. - /// - /// Fixture (32 bytes, indices `0..=31`): - /// `"ABCD????ABCD!??????????ABCDX????"` - /// 01234567890123456789012345678901 (ones digit) - /// 1111111111222222222233 (tens digit, aligned) - /// - /// Probe state: - /// * `offset_hist[1] = 8` → rep probe reads `concat[pos - 8..]`. - /// * `pos = 8` → `concat[8..12] = "ABCD"`, `concat[0..4] = "ABCD"` - /// → 4 bytes match. - /// * `concat[12] = '!'` vs `concat[4] = '?'` → 5th byte mismatch, - /// so the rep extension stops at exactly 4 bytes. - #[test] - fn dfast_repcode_extension_accepts_exactly_four_byte_rep() { - // Block: "ABCD????" (8) + "ABCD!" (5) + "??????????" (10) + - // "ABCDX" (5) + "????" (4) = 32 bytes total. The - // important invariants are `concat[0..4] == "ABCD"`, - // `concat[8..12] == "ABCD"`, and `concat[12] = '!'` - // (so byte 12 ≠ byte 4 = '?', stopping the rep at - // length 4). The trailing bytes are irrelevant — we - // only iterate the helper at `pos = 8` and the rep - // chain terminates after one 4-byte emit because the - // next rep probe (post-swap) would need bytes at - // `pos + 4` to match a different offset. - let data: Vec = b"ABCD????ABCD!??????????ABCDX????".to_vec(); - assert_eq!(data.len(), 32, "fixture invariant: 32 bytes"); - let mut dfast = DfastMatchGenerator::new(64); - dfast.ensure_hash_tables(); - dfast.add_data(data.clone(), |_| {}); - - dfast.offset_hist = [12, 8, 4]; - let current_abs_start = dfast.history_abs_start + dfast.window_size - data.len(); - let current_len = data.len(); - let pos = 8usize; - let mut literals_start = pos; - - let mut seqs = Vec::new(); - let new_pos = { - let mut rec = record_seq(&mut seqs); - dfast.extend_with_repcode_after_match( - current_abs_start, - current_len, - pos, - &mut literals_start, - &mut rec, - ) - }; - - // Helper must emit a single 4-byte rep, then stop because - // the 5th byte mismatches. - assert_eq!(seqs.len(), 1, "expected one 4-byte rep emit, got {seqs:?}"); - match &seqs[0] { - CapturedSeq::Triple { - lit_len, - offset, - match_len, - } => { - assert_eq!(*lit_len, 0, "rep emit must be zero-literal"); - assert_eq!(*offset, 8, "rep emit must use offset 8 (offset_hist[1])"); - assert_eq!( - *match_len, 4, - "rep emit must be exactly 4 bytes (upstream zstd MINMATCH floor). \ - A regression back to DFAST_MIN_MATCH_LEN > 4 would skip \ - this emission entirely and the test would fail with 0 seqs." - ); - } - other => panic!("expected Triple, got {other:?}"), - } - assert_eq!(new_pos, pos + 4, "pos must advance by exactly 4"); - assert_eq!(literals_start, new_pos, "literals_start must follow pos"); - } - - /// Integration coverage for the **fast-loop** call sites of - /// `extend_with_repcode_after_match` inside - /// `start_matching_fast_loop`. The direct-call tests above pin - /// down the helper's contract; this test drives the full fast - /// loop end-to-end through the production `compress_to_vec` - /// pipeline on a fixture engineered to exercise the post-match - /// rep chain on the fast-loop path. - /// - /// `CompressionLevel::Default` is the production config that - /// enables `use_fast_loop = true` (see `Matcher::reset` in - /// `match_generator.rs`). The fixture alternates 60-byte runs of - /// `'A'` with single `'B'` break bytes and a short `'A'` tail - /// per cycle — the breaks terminate the fast loop's primary match - /// early, so subsequent iterations have runway for the helper to - /// chain additional reps. A regression that broke either fast- - /// loop helper call site surfaces as a roundtrip failure (decoded - /// != input) or a ratio explosion. Constructing - /// `DfastMatchGenerator` directly and asserting on captured - /// sequences was attempted but the fixture engineering is - /// brittle: the fast loop's primary match on simple constant - /// fixtures consumes the entire remaining block in a single - /// Triple, leaving no bytes for the helper to extend. The - /// high-level roundtrip sidesteps that fragility while still - /// routing through the same call site via the production driver. - /// - /// Gated on `feature = "std"`: the `Read::read_to_end` method - /// used to drain `StreamingDecoder` resolves to `std::io::Read` - /// only when std is enabled. Under no-std `StreamingDecoder` - /// implements the crate's `io_nostd::Read` alias instead, and - /// the call site has to be rewritten through that trait. The - /// fast-loop helper itself is exercised under both - /// configurations by the direct-call tests above plus the - /// `cross_validation` Default-level roundtrip — gating this one - /// integration test on std loses no coverage, only saves the - /// dual-trait rewrite. - #[cfg(feature = "std")] - #[test] - fn dfast_default_level_roundtrip_with_repetitive_breaks_exercises_fast_loop() { - // ~4 KiB of input: 64 cycles of [60 'A's, 1 'B', 3 'A's]. - let mut data: Vec = Vec::with_capacity(64 * 64); - for _ in 0..64 { - data.extend_from_slice(&[b'A'; 60]); - data.push(b'B'); - data.extend_from_slice(&[b'A'; 3]); - } - assert!( - data.len() > 4000, - "fixture invariant: long enough for fast loop" - ); - - let compressed = crate::encoding::compress_to_vec( - data.as_slice(), - crate::encoding::CompressionLevel::Default, - ); - - // Decompress and assert byte-for-byte parity. A regression - // that broke the fast-loop helper call would either produce - // invalid frames (decode error) or wrong bytes (mismatch). - let mut decoder = crate::decoding::StreamingDecoder::new(compressed.as_slice()) - .expect("default-level frame must decode"); - let mut decoded = Vec::with_capacity(data.len()); - // Under `feature = "std"` (the gate above) `StreamingDecoder` - // implements `std::io::Read`, so `Read::read_to_end` resolves - // through the standard library's blanket implementation. - std::io::Read::read_to_end(&mut decoder, &mut decoded) - .expect("fast-loop output must round-trip cleanly"); - assert_eq!( - decoded, data, - "Default-level (use_fast_loop = true) roundtrip must be \ - byte-for-byte exact on the repetitive-breaks fixture" - ); - - // Ratio sanity: the post-match rep helper is what makes - // repetitive runs compress aggressively on the fast-loop - // path. A regression to a no-op helper would still produce - // some compression via the primary match, but the ratio - // would degrade. A 2:1 floor is conservative enough not to - // flake on small fixture changes while still catching - // structural failures of the fast loop. - assert!( - compressed.len() * 2 < data.len(), - "fast loop must compress repetitive runs to at least 2:1, \ - got {} → {} bytes", - data.len(), - compressed.len() - ); - } - - /// The borrowed one-shot scan (no per-block `commit_space` copy) must - /// emit the byte-identical sequence stream as the owned `history`-copying - /// path for an in-window input: the window far exceeds the input so the - /// owned path never evicts, making its accumulated `history` identical to - /// the borrowed buffer, and the borrowed scan's absolute positions / - /// seam re-seed reproduce the same match decisions. This is the - /// correctness contract behind routing dfast through `borrowed_eligible`. - #[test] - fn dfast_borrowed_window_matches_owned_sequence_stream() { - // Repeating pattern (period 64) so the matcher emits real matches, - // split across two blocks. - let mut whole: Vec = Vec::with_capacity(512); - for i in 0..512usize { - whole.push((i % 64) as u8); - } - let split = 300usize; - let window = 1usize << 16; // 64 KiB >> 512-byte input: no eviction. - - // Owned: commit each block, then scan. - let mut owned = DfastMatchGenerator::new(window); - owned.ensure_hash_tables(); - let mut owned_seqs: Vec = Vec::new(); - owned.add_data(whole[..split].to_vec(), |_| {}); - { - let mut rec = record_seq(&mut owned_seqs); - owned.start_matching(&mut rec); - } - owned.add_data(whole[split..].to_vec(), |_| {}); - { - let mut rec = record_seq(&mut owned_seqs); - owned.start_matching(&mut rec); - } - - // Borrowed: same bytes, scanned in place by absolute block range. - let mut borrowed = DfastMatchGenerator::new(window); - borrowed.ensure_hash_tables(); - let mut borrowed_seqs: Vec = Vec::new(); - // SAFETY: `whole` outlives both scans; `borrowed` is dropped at end - // of scope before `whole`, so the window never dangles. - unsafe { borrowed.set_borrowed_window(&whole) }; - { - let mut rec = record_seq(&mut borrowed_seqs); - borrowed.start_matching_borrowed(0, split, &mut rec); - } - { - let mut rec = record_seq(&mut borrowed_seqs); - borrowed.start_matching_borrowed(split, whole.len(), &mut rec); - } - - assert_eq!( - owned_seqs, borrowed_seqs, - "dfast borrowed window must emit the identical sequence stream as the owned path", - ); - assert_eq!( - owned.offset_hist, borrowed.offset_hist, - "rep state must match after both scans", - ); - } -} +mod extend_with_repcode_tests; diff --git a/zstd/src/encoding/dict_attach.rs b/zstd/src/encoding/dict_attach.rs index 08945501a..a1cace877 100644 --- a/zstd/src/encoding/dict_attach.rs +++ b/zstd/src/encoding/dict_attach.rs @@ -162,33 +162,4 @@ impl DictAttach { } #[cfg(test)] -mod tests { - use super::*; - use alloc::{vec, vec::Vec}; - - #[test] - fn lifecycle_attach_prime_invalidate() { - let mut da: DictAttach> = DictAttach::new(); - assert!(!da.is_attached()); - assert!(!da.is_primed()); - assert_eq!(da.region_len(), 0); - - da.set_region_len(128); - // mark_primed is a no-op while no table exists. - da.mark_primed(); - assert!(!da.is_primed()); - - da.table_mut_or_init(|| vec![0u32; 16]).fill(7); - assert!(da.is_attached()); - assert_eq!(da.table().unwrap()[0], 7); - - da.mark_primed(); - assert!(da.is_primed()); - assert_eq!(da.region_len(), 128); - - da.invalidate(); - assert!(!da.is_attached()); - assert!(!da.is_primed()); - assert_eq!(da.region_len(), 0); - } -} +mod tests; diff --git a/zstd/src/encoding/dict_attach/tests.rs b/zstd/src/encoding/dict_attach/tests.rs new file mode 100644 index 000000000..4eda61fca --- /dev/null +++ b/zstd/src/encoding/dict_attach/tests.rs @@ -0,0 +1,28 @@ +use super::*; +use alloc::{vec, vec::Vec}; + +#[test] +fn lifecycle_attach_prime_invalidate() { + let mut da: DictAttach> = DictAttach::new(); + assert!(!da.is_attached()); + assert!(!da.is_primed()); + assert_eq!(da.region_len(), 0); + + da.set_region_len(128); + // mark_primed is a no-op while no table exists. + da.mark_primed(); + assert!(!da.is_primed()); + + da.table_mut_or_init(|| vec![0u32; 16]).fill(7); + assert!(da.is_attached()); + assert_eq!(da.table().unwrap()[0], 7); + + da.mark_primed(); + assert!(da.is_primed()); + assert_eq!(da.region_len(), 128); + + da.invalidate(); + assert!(!da.is_attached()); + assert!(!da.is_primed()); + assert_eq!(da.region_len(), 0); +} diff --git a/zstd/src/encoding/fastpath/mod.rs b/zstd/src/encoding/fastpath/mod.rs index e8eddaa15..55a0bd530 100644 --- a/zstd/src/encoding/fastpath/mod.rs +++ b/zstd/src/encoding/fastpath/mod.rs @@ -364,50 +364,4 @@ pub(crate) unsafe fn dispatch_common_prefix_len_ptr_with_kernel( } #[cfg(test)] -mod tests { - use super::{FastpathKernel, detect_kernel_uncached, select_kernel}; - - #[test] - fn select_kernel_returns_supported_variant() { - let k = select_kernel(); - // Cached and direct calls must agree. - assert_eq!(k, detect_kernel_uncached()); - // Whatever the kernel is, it must be one of the variants compiled in - // for this target. - match k { - FastpathKernel::Scalar => {} - #[cfg(all(target_arch = "aarch64", target_endian = "little"))] - FastpathKernel::Neon => {} - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - FastpathKernel::Sse42 => {} - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - FastpathKernel::Avx2Bmi2 => {} - #[cfg(all( - target_arch = "wasm32", - target_feature = "simd128", - feature = "kernel_simd128" - ))] - FastpathKernel::Simd128 => {} - } - } - - #[cfg(all(target_arch = "aarch64", target_endian = "little"))] - #[test] - fn aarch64_picks_neon_when_crc_available() { - // The dispatcher gates the NEON kernel on both `neon` (baseline) - // and the optional `crc` extension. Mirror that runtime/compile-time - // gate so the test stays accurate on AArch64 CPUs (or CI runners) - // where `crc` is not reported. - #[cfg(feature = "std")] - let crc_available = std::arch::is_aarch64_feature_detected!("crc"); - #[cfg(not(feature = "std"))] - let crc_available = cfg!(target_feature = "crc"); - - let expected = if crc_available { - FastpathKernel::Neon - } else { - FastpathKernel::Scalar - }; - assert_eq!(detect_kernel_uncached(), expected); - } -} +mod tests; diff --git a/zstd/src/encoding/fastpath/neon.rs b/zstd/src/encoding/fastpath/neon.rs index fe7127509..c9464b4e8 100644 --- a/zstd/src/encoding/fastpath/neon.rs +++ b/zstd/src/encoding/fastpath/neon.rs @@ -113,31 +113,4 @@ pub(crate) unsafe fn count_match_from_indices( } #[cfg(test)] -mod tests { - use super::*; - use alloc::vec::Vec; - - #[test] - fn neon_prefix_len_matches_scalar_on_long_run() { - // 40-byte runs cover both the SIMD 16-byte loop and the scalar tail. - let a = b"abcdefghijklmnopqrstuvwxyz0123456789-+=*"; - let mut b: Vec = a.to_vec(); - b[25] = b'!'; - let max = a.len(); - let neon = unsafe { common_prefix_len_ptr(a.as_ptr(), b.as_ptr(), max) }; - let scl = unsafe { scalar::common_prefix_len_ptr(a.as_ptr(), b.as_ptr(), max) }; - assert_eq!(neon, scl); - assert_eq!(neon, 25); - } - - #[test] - fn neon_handles_short_input() { - let a = b"abc"; - let b = b"abc"; - let max = a.len(); - assert_eq!( - unsafe { common_prefix_len_ptr(a.as_ptr(), b.as_ptr(), max) }, - 3 - ); - } -} +mod tests; diff --git a/zstd/src/encoding/fastpath/neon/tests.rs b/zstd/src/encoding/fastpath/neon/tests.rs new file mode 100644 index 000000000..30f99935e --- /dev/null +++ b/zstd/src/encoding/fastpath/neon/tests.rs @@ -0,0 +1,26 @@ +use super::*; +use alloc::vec::Vec; + +#[test] +fn neon_prefix_len_matches_scalar_on_long_run() { + // 40-byte runs cover both the SIMD 16-byte loop and the scalar tail. + let a = b"abcdefghijklmnopqrstuvwxyz0123456789-+=*"; + let mut b: Vec = a.to_vec(); + b[25] = b'!'; + let max = a.len(); + let neon = unsafe { common_prefix_len_ptr(a.as_ptr(), b.as_ptr(), max) }; + let scl = unsafe { scalar::common_prefix_len_ptr(a.as_ptr(), b.as_ptr(), max) }; + assert_eq!(neon, scl); + assert_eq!(neon, 25); +} + +#[test] +fn neon_handles_short_input() { + let a = b"abc"; + let b = b"abc"; + let max = a.len(); + assert_eq!( + unsafe { common_prefix_len_ptr(a.as_ptr(), b.as_ptr(), max) }, + 3 + ); +} diff --git a/zstd/src/encoding/fastpath/scalar.rs b/zstd/src/encoding/fastpath/scalar.rs index eccbe85d6..ad3d674b0 100644 --- a/zstd/src/encoding/fastpath/scalar.rs +++ b/zstd/src/encoding/fastpath/scalar.rs @@ -108,40 +108,4 @@ pub(crate) unsafe fn count_match_from_indices( } #[cfg(test)] -mod tests { - use super::*; - use alloc::vec::Vec; - - #[test] - fn scalar_prefix_len_matches_naive_reference() { - let a = b"abcdef_long string here for the test 1234567890"; - let mut b: Vec = a.to_vec(); - b[20] = b'!'; - let max = a.len().min(b.len()); - let len = unsafe { common_prefix_len_ptr(a.as_ptr(), b.as_ptr(), max) }; - assert_eq!(len, 20); - } - - #[test] - fn scalar_prefix_len_empty_inputs() { - let p = core::ptr::NonNull::::dangling().as_ptr(); - assert_eq!(unsafe { common_prefix_len_ptr(p, p, 0) }, 0); - } - - #[test] - #[cfg(target_endian = "little")] - fn mismatch_byte_index_finds_low_bit_byte_le() { - // diff == 0x_FF_00 -> mismatch is byte 1 (bits 8..16) on LE. - let diff: usize = 0xff00; - assert_eq!(mismatch_byte_index(diff), 1); - } - - #[test] - #[cfg(target_endian = "big")] - fn mismatch_byte_index_finds_low_bit_byte_be() { - // Mirror of the LE case for the BE branch: the LSB byte sits at the - // tail of the word, so the mismatch byte index is `bytes_per_usize-1`. - let diff: usize = 0xff; - assert_eq!(mismatch_byte_index(diff), core::mem::size_of::() - 1); - } -} +mod tests; diff --git a/zstd/src/encoding/fastpath/scalar/tests.rs b/zstd/src/encoding/fastpath/scalar/tests.rs new file mode 100644 index 000000000..e6fa1c4a9 --- /dev/null +++ b/zstd/src/encoding/fastpath/scalar/tests.rs @@ -0,0 +1,35 @@ +use super::*; +use alloc::vec::Vec; + +#[test] +fn scalar_prefix_len_matches_naive_reference() { + let a = b"abcdef_long string here for the test 1234567890"; + let mut b: Vec = a.to_vec(); + b[20] = b'!'; + let max = a.len().min(b.len()); + let len = unsafe { common_prefix_len_ptr(a.as_ptr(), b.as_ptr(), max) }; + assert_eq!(len, 20); +} + +#[test] +fn scalar_prefix_len_empty_inputs() { + let p = core::ptr::NonNull::::dangling().as_ptr(); + assert_eq!(unsafe { common_prefix_len_ptr(p, p, 0) }, 0); +} + +#[test] +#[cfg(target_endian = "little")] +fn mismatch_byte_index_finds_low_bit_byte_le() { + // diff == 0x_FF_00 -> mismatch is byte 1 (bits 8..16) on LE. + let diff: usize = 0xff00; + assert_eq!(mismatch_byte_index(diff), 1); +} + +#[test] +#[cfg(target_endian = "big")] +fn mismatch_byte_index_finds_low_bit_byte_be() { + // Mirror of the LE case for the BE branch: the LSB byte sits at the + // tail of the word, so the mismatch byte index is `bytes_per_usize-1`. + let diff: usize = 0xff; + assert_eq!(mismatch_byte_index(diff), core::mem::size_of::() - 1); +} diff --git a/zstd/src/encoding/fastpath/simd128.rs b/zstd/src/encoding/fastpath/simd128.rs index 4698b1ea3..123d33e09 100644 --- a/zstd/src/encoding/fastpath/simd128.rs +++ b/zstd/src/encoding/fastpath/simd128.rs @@ -100,29 +100,4 @@ pub(crate) unsafe fn count_match_from_indices( } #[cfg(test)] -mod tests { - use super::*; - use alloc::vec::Vec; - - #[test] - fn simd128_prefix_len_matches_scalar_on_long_run() { - let a = b"abcdefghijklmnopqrstuvwxyz0123456789-+=*"; - let mut b: Vec = a.to_vec(); - b[25] = b'!'; - let max = a.len(); - let got = unsafe { common_prefix_len_ptr(a.as_ptr(), b.as_ptr(), max) }; - let scl = unsafe { scalar::common_prefix_len_ptr(a.as_ptr(), b.as_ptr(), max) }; - assert_eq!(got, scl); - assert_eq!(got, 25); - } - - #[test] - fn simd128_handles_short_input() { - let a = b"abc"; - let b = b"abc"; - assert_eq!( - unsafe { common_prefix_len_ptr(a.as_ptr(), b.as_ptr(), a.len()) }, - 3 - ); - } -} +mod tests; diff --git a/zstd/src/encoding/fastpath/simd128/tests.rs b/zstd/src/encoding/fastpath/simd128/tests.rs new file mode 100644 index 000000000..1b4e65b5c --- /dev/null +++ b/zstd/src/encoding/fastpath/simd128/tests.rs @@ -0,0 +1,24 @@ +use super::*; +use alloc::vec::Vec; + +#[test] +fn simd128_prefix_len_matches_scalar_on_long_run() { + let a = b"abcdefghijklmnopqrstuvwxyz0123456789-+=*"; + let mut b: Vec = a.to_vec(); + b[25] = b'!'; + let max = a.len(); + let got = unsafe { common_prefix_len_ptr(a.as_ptr(), b.as_ptr(), max) }; + let scl = unsafe { scalar::common_prefix_len_ptr(a.as_ptr(), b.as_ptr(), max) }; + assert_eq!(got, scl); + assert_eq!(got, 25); +} + +#[test] +fn simd128_handles_short_input() { + let a = b"abc"; + let b = b"abc"; + assert_eq!( + unsafe { common_prefix_len_ptr(a.as_ptr(), b.as_ptr(), a.len()) }, + 3 + ); +} diff --git a/zstd/src/encoding/fastpath/tests.rs b/zstd/src/encoding/fastpath/tests.rs new file mode 100644 index 000000000..299324da4 --- /dev/null +++ b/zstd/src/encoding/fastpath/tests.rs @@ -0,0 +1,45 @@ +use super::{FastpathKernel, detect_kernel_uncached, select_kernel}; + +#[test] +fn select_kernel_returns_supported_variant() { + let k = select_kernel(); + // Cached and direct calls must agree. + assert_eq!(k, detect_kernel_uncached()); + // Whatever the kernel is, it must be one of the variants compiled in + // for this target. + match k { + FastpathKernel::Scalar => {} + #[cfg(all(target_arch = "aarch64", target_endian = "little"))] + FastpathKernel::Neon => {} + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + FastpathKernel::Sse42 => {} + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + FastpathKernel::Avx2Bmi2 => {} + #[cfg(all( + target_arch = "wasm32", + target_feature = "simd128", + feature = "kernel_simd128" + ))] + FastpathKernel::Simd128 => {} + } +} + +#[cfg(all(target_arch = "aarch64", target_endian = "little"))] +#[test] +fn aarch64_picks_neon_when_crc_available() { + // The dispatcher gates the NEON kernel on both `neon` (baseline) + // and the optional `crc` extension. Mirror that runtime/compile-time + // gate so the test stays accurate on AArch64 CPUs (or CI runners) + // where `crc` is not reported. + #[cfg(feature = "std")] + let crc_available = std::arch::is_aarch64_feature_detected!("crc"); + #[cfg(not(feature = "std"))] + let crc_available = cfg!(target_feature = "crc"); + + let expected = if crc_available { + FastpathKernel::Neon + } else { + FastpathKernel::Scalar + }; + assert_eq!(detect_kernel_uncached(), expected); +} diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index d64058c43..b37e2eac1 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -642,7 +642,7 @@ pub(crate) fn optimal_block_size( block_size_max: usize, savings: i64, ) -> usize { - let Some(split_level) = crate::encoding::match_generator::level_pre_split(level) else { + let Some(split_level) = crate::encoding::levels::config::level_pre_split(level) else { return remaining_src_size.min(block_size_max); }; if remaining_src_size < MAX_BLOCK_SIZE as usize || block_size_max < MAX_BLOCK_SIZE as usize { @@ -1250,7 +1250,7 @@ impl FrameCompressor { if savings >= 3 && input.len() - start >= MAX_BLOCK_SIZE as usize && block_capacity >= MAX_BLOCK_SIZE as usize - && crate::encoding::match_generator::level_pre_split(self.compression_level) + && crate::encoding::levels::config::level_pre_split(self.compression_level) .is_some() { warm_presplit_window(&input[start..start + MAX_BLOCK_SIZE as usize]); @@ -1613,7 +1613,7 @@ impl FrameCompressor { // gate in sync with the matcher's attach cutoff so Fast never // captures/restores a copy snapshot it can no longer use. crate::encoding::strategy::StrategyTag::Fast => { - crate::encoding::match_generator::FAST_ATTACH_DICT_CUTOFF_LOG + crate::encoding::levels::config::FAST_ATTACH_DICT_CUTOFF_LOG } crate::encoding::strategy::StrategyTag::BtUltra | crate::encoding::strategy::StrategyTag::BtUltra2 => 13, @@ -1632,7 +1632,7 @@ impl FrameCompressor { .reapply_resident_dictionary(dict.inner.offset_hist); } else { let prefer_copy_snapshot = initial_size_hint.is_some_and(|s| { - crate::encoding::match_generator::source_size_ceil_log(s) > cutoff_log + crate::encoding::levels::config::source_size_ceil_log(s) > cutoff_log }); let restored = prefer_copy_snapshot && self @@ -2367,2549 +2367,4 @@ impl FrameCompressor { } #[cfg(test)] -mod tests { - // `format!` is used by ungated tests (e.g. the btlazy2 dict-reuse - // byte-identity test), so the import must not be feature-gated — under - // default features (no `dict_builder`) the gated form left `format!` - // unresolved when the test module is compiled. - use alloc::format; - use alloc::vec; - - use super::FrameCompressor; - use crate::common::{MAGIC_NUM, MAX_BLOCK_SIZE}; - use crate::decoding::FrameDecoder; - use crate::encoding::{Matcher, Sequence}; - use alloc::vec::Vec; - - fn generate_data(seed: u64, len: usize) -> Vec { - let mut state = seed; - let mut data = Vec::with_capacity(len); - for _ in 0..len { - state = state - .wrapping_mul(6364136223846793005) - .wrapping_add(1442695040888963407); - data.push((state >> 33) as u8); - } - data - } - - // Cross-implementation parity tests (compress here, decode through the C - // bindings) moved to `ffi-bench/tests/frame_compressor_ffi.rs` so the - // library crate never links libzstd. - - struct NoDictionaryMatcher { - last_space: Vec, - window_size: u64, - } - - impl NoDictionaryMatcher { - fn new(window_size: u64) -> Self { - Self { - last_space: Vec::new(), - window_size, - } - } - } - - impl Matcher for NoDictionaryMatcher { - fn get_next_space(&mut self) -> Vec { - vec![0; self.window_size as usize] - } - - fn get_last_space(&mut self) -> &[u8] { - self.last_space.as_slice() - } - - fn commit_space(&mut self, space: Vec) { - self.last_space = space; - } - - fn skip_matching(&mut self) {} - - fn start_matching(&mut self, mut handle_sequence: impl for<'a> FnMut(Sequence<'a>)) { - handle_sequence(Sequence::Literals { - literals: self.last_space.as_slice(), - }); - } - - fn reset(&mut self, _level: super::CompressionLevel) { - self.last_space.clear(); - } - - fn window_size(&self) -> u64 { - self.window_size - } - } - - #[test] - fn frame_starts_with_magic_num() { - let mock_data = [1_u8, 2, 3].as_slice(); - let mut output: Vec = Vec::new(); - let mut compressor = FrameCompressor::new(super::CompressionLevel::Uncompressed); - compressor.set_source(mock_data); - compressor.set_drain(&mut output); - - compressor.compress(); - assert!(output.starts_with(&MAGIC_NUM.to_le_bytes())); - } - - #[test] - fn very_simple_raw_compress() { - let mock_data = [1_u8, 2, 3].as_slice(); - let mut output: Vec = Vec::new(); - let mut compressor = FrameCompressor::new(super::CompressionLevel::Uncompressed); - compressor.set_source(mock_data); - compressor.set_drain(&mut output); - - compressor.compress(); - } - - #[test] - fn very_simple_compress() { - let mut mock_data = vec![0; 1 << 17]; - mock_data.extend(vec![1; (1 << 17) - 1]); - mock_data.extend(vec![2; (1 << 18) - 1]); - mock_data.extend(vec![2; 1 << 17]); - mock_data.extend(vec![3; (1 << 17) - 1]); - let mut output: Vec = Vec::new(); - let mut compressor = FrameCompressor::new(super::CompressionLevel::Uncompressed); - compressor.set_source(mock_data.as_slice()); - compressor.set_drain(&mut output); - - compressor.compress(); - - let mut decoder = FrameDecoder::new(); - let mut decoded = Vec::with_capacity(mock_data.len()); - decoder.decode_all_to_vec(&output, &mut decoded).unwrap(); - assert_eq!(mock_data, decoded); - } - - #[test] - fn rle_compress() { - let mock_data = vec![0; 1 << 19]; - let mut output: Vec = Vec::new(); - let mut compressor = FrameCompressor::new(super::CompressionLevel::Uncompressed); - compressor.set_source(mock_data.as_slice()); - compressor.set_drain(&mut output); - - compressor.compress(); - - let mut decoder = FrameDecoder::new(); - let mut decoded = Vec::with_capacity(mock_data.len()); - decoder.decode_all_to_vec(&output, &mut decoded).unwrap(); - assert_eq!(mock_data, decoded); - } - - #[test] - fn aaa_compress() { - let mock_data = vec![0, 1, 3, 4, 5]; - let mut output: Vec = Vec::new(); - let mut compressor = FrameCompressor::new(super::CompressionLevel::Uncompressed); - compressor.set_source(mock_data.as_slice()); - compressor.set_drain(&mut output); - - compressor.compress(); - - let mut decoder = FrameDecoder::new(); - let mut decoded = Vec::with_capacity(mock_data.len()); - decoder.decode_all_to_vec(&output, &mut decoded).unwrap(); - assert_eq!(mock_data, decoded); - } - - #[test] - fn dictionary_compression_sets_required_dict_id_and_roundtrips() { - let dict_raw = include_bytes!("../../dict_tests/dictionary"); - let dict_for_encoder = crate::decoding::Dictionary::decode_dict(dict_raw).unwrap(); - let dict_for_decoder = crate::decoding::Dictionary::decode_dict(dict_raw).unwrap(); - - let mut data = Vec::new(); - for _ in 0..8 { - data.extend_from_slice(&dict_for_decoder.dict_content[..2048]); - } - - let mut with_dict = Vec::new(); - let mut compressor = FrameCompressor::new(super::CompressionLevel::Fastest); - let previous = compressor - .set_dictionary_from_bytes(dict_raw) - .expect("dictionary bytes should parse"); - assert!( - previous.is_none(), - "first dictionary insert should return None" - ); - assert_eq!( - compressor - .set_dictionary(dict_for_encoder) - .expect("valid dictionary should attach") - .expect("set_dictionary_from_bytes inserted previous dictionary") - .id(), - dict_for_decoder.id - ); - compressor.set_source(data.as_slice()); - compressor.set_drain(&mut with_dict); - compressor.compress(); - - let (frame_header, _) = crate::decoding::frame::read_frame_header(with_dict.as_slice()) - .expect("encoded stream should have a frame header"); - assert_eq!(frame_header.dictionary_id(), Some(dict_for_decoder.id)); - - let mut decoder = FrameDecoder::new(); - let mut missing_dict_target = Vec::with_capacity(data.len()); - let err = decoder - .decode_all_to_vec(&with_dict, &mut missing_dict_target) - .unwrap_err(); - assert!( - matches!( - &err, - crate::decoding::errors::FrameDecoderError::DictNotProvided { .. } - ), - "dict-compressed stream should require dictionary id, got: {err:?}" - ); - - let mut decoder = FrameDecoder::new(); - decoder.add_dict(dict_for_decoder).unwrap(); - let mut decoded = Vec::with_capacity(data.len()); - decoder.decode_all_to_vec(&with_dict, &mut decoded).unwrap(); - assert_eq!(decoded, data); - } - - #[cfg(all(feature = "dict_builder", feature = "std"))] - #[test] - fn dictionary_compression_roundtrips_with_dict_builder_dictionary() { - use std::io::Cursor; - - let mut training = Vec::new(); - for idx in 0..256u32 { - training.extend_from_slice( - format!("tenant=demo table=orders key={idx} region=eu\n").as_bytes(), - ); - } - let mut raw_dict = Vec::new(); - crate::dictionary::create_raw_dict_from_source( - Cursor::new(training.as_slice()), - training.len(), - &mut raw_dict, - 4096, - ) - .expect("dict_builder training should succeed"); - assert!( - !raw_dict.is_empty(), - "dict_builder produced an empty dictionary" - ); - - let dict_id = 0xD1C7_0008; - let encoder_dict = - crate::decoding::Dictionary::from_raw_content(dict_id, raw_dict.clone()).unwrap(); - let decoder_dict = - crate::decoding::Dictionary::from_raw_content(dict_id, raw_dict.clone()).unwrap(); - - let mut payload = Vec::new(); - for idx in 0..96u32 { - payload.extend_from_slice( - format!( - "tenant=demo table=orders op=put key={idx} value=aaaaabbbbbcccccdddddeeeee\n" - ) - .as_bytes(), - ); - } - - let mut without_dict = Vec::new(); - let mut baseline = FrameCompressor::new(super::CompressionLevel::Fastest); - baseline.set_source(payload.as_slice()); - baseline.set_drain(&mut without_dict); - baseline.compress(); - - let mut with_dict = Vec::new(); - let mut compressor = FrameCompressor::new(super::CompressionLevel::Fastest); - compressor - .set_dictionary(encoder_dict) - .expect("valid dict_builder dictionary should attach"); - compressor.set_source(payload.as_slice()); - compressor.set_drain(&mut with_dict); - compressor.compress(); - - let (frame_header, _) = crate::decoding::frame::read_frame_header(with_dict.as_slice()) - .expect("encoded stream should have a frame header"); - assert_eq!(frame_header.dictionary_id(), Some(dict_id)); - let mut decoder = FrameDecoder::new(); - decoder.add_dict(decoder_dict).unwrap(); - let mut decoded = Vec::with_capacity(payload.len()); - decoder.decode_all_to_vec(&with_dict, &mut decoded).unwrap(); - assert_eq!(decoded, payload); - assert!( - with_dict.len() < without_dict.len(), - "trained dictionary should improve compression for this small payload" - ); - } - - #[test] - fn set_dictionary_from_bytes_seeds_entropy_tables_for_first_block() { - let dict_raw = include_bytes!("../../dict_tests/dictionary"); - let mut output = Vec::new(); - let input = b""; - - let mut compressor = FrameCompressor::new(super::CompressionLevel::Fastest); - let previous = compressor - .set_dictionary_from_bytes(dict_raw) - .expect("dictionary bytes should parse"); - assert!(previous.is_none()); - - compressor.set_source(input.as_slice()); - compressor.set_drain(&mut output); - compressor.compress(); - - assert!( - compressor.state.last_huff_table.is_some(), - "dictionary entropy should seed previous huffman table before first block" - ); - assert!( - compressor.state.fse_tables.ll_previous.is_some(), - "dictionary entropy should seed previous ll table before first block" - ); - assert!( - compressor.state.fse_tables.ml_previous.is_some(), - "dictionary entropy should seed previous ml table before first block" - ); - assert!( - compressor.state.fse_tables.of_previous.is_some(), - "dictionary entropy should seed previous of table before first block" - ); - } - - // `set_content_size_flag(false)`: the header must omit the FCS field - // (and the single-segment layout that requires it) while the frame - // still round-trips through our decoder. - #[test] - fn content_size_flag_off_omits_fcs_and_roundtrips() { - let payload = alloc::vec![0x42u8; 4096]; - - let mut compressor: FrameCompressor = - FrameCompressor::new(super::CompressionLevel::Fastest); - let mut with_fcs = Vec::new(); - compressor.compress_independent_frame_into(&payload, &mut with_fcs); - - compressor.set_content_size_flag(false); - let mut without_fcs = Vec::new(); - compressor.compress_independent_frame_into(&payload, &mut without_fcs); - - let parsed_with = crate::decoding::frame::read_frame_header(with_fcs.as_slice()) - .expect("flag-on frame header must parse") - .0; - assert_eq!(parsed_with.frame_content_size(), 4096); - - let parsed_without = crate::decoding::frame::read_frame_header(without_fcs.as_slice()) - .expect("flag-off frame header must parse") - .0; - // 0 is the decoder's "unknown content size" sentinel... - assert_eq!( - parsed_without.frame_content_size(), - 0, - "FCS must be omitted with the content-size flag off" - ); - // ...and the descriptor must confirm the field is ABSENT (0 bytes), - // not present with an explicit zero value. - assert_eq!( - parsed_without - .descriptor - .frame_content_size_bytes() - .expect("descriptor must parse"), - 0, - "the FCS field itself must be omitted, not written as zero" - ); - - let mut decoder = crate::decoding::FrameDecoder::new(); - // `decode_all_to_vec` fills existing capacity (no FCS to pre-size - // from with the flag off), so reserve the expected payload upfront. - let mut decoded = Vec::with_capacity(payload.len() + 64); - decoder - .decode_all_to_vec(&without_fcs, &mut decoded) - .expect("flag-off frame must decode"); - assert_eq!(decoded, payload); - } - - // `set_dictionary_id_flag(false)`: a dict-compressed frame must omit - // the dictionary ID and still decode when the dictionary is handed to - // the decoder explicitly. - #[test] - fn dict_id_flag_off_omits_dictionary_id_and_roundtrips() { - let dict_raw = include_bytes!("../../dict_tests/dictionary"); - let payload = b"dictionary-keyed payload dictionary-keyed payload".repeat(8); - - let mut compressor: FrameCompressor = - FrameCompressor::new(super::CompressionLevel::Fastest); - compressor - .set_dictionary_from_bytes(dict_raw) - .expect("dictionary bytes should parse"); - compressor.set_dictionary_id_flag(false); - let mut frame = Vec::new(); - compressor.compress_independent_frame_into(&payload, &mut frame); - - let parsed = crate::decoding::frame::read_frame_header(frame.as_slice()) - .expect("frame header must parse") - .0; - assert_eq!( - parsed.dictionary_id(), - None, - "dictionary id must be omitted with the dict-id flag off" - ); - - // With the ID omitted the decoder cannot look the dictionary up by - // header; hand it explicitly (the `reset_with_dict_handle` path). - let mut sd = crate::decoding::StreamingDecoder::new_with_dictionary_bytes( - frame.as_slice(), - dict_raw, - ) - .expect("decoder must accept the dictionary"); - let mut dec = Vec::new(); - std::io::Read::read_to_end(&mut sd, &mut dec) - .expect("frame must decode with the dictionary handed explicitly"); - assert_eq!(dec, payload); - } - - // The output reservation must track the observed compression ratio, not - // the whole-input `compress_bound`: a multi-MiB compressible stream's - // output buffer stays at output scale (the old up-front bound held an - // input-sized allocation for the whole frame). Incompressible input may - // still re-estimate to ~the full bound — that is the genuine worst case. - #[test] - fn compressible_stream_output_capacity_stays_at_output_scale() { - // 4 MiB of highly repetitive log-like lines. - let line = b"ts=2026-03-26T21:39:28Z level=INFO msg=\"flush memtable\" tenant=demo\n"; - let mut input = Vec::with_capacity(4 << 20); - while input.len() < (4 << 20) { - let take = line.len().min((4 << 20) - input.len()); - input.extend_from_slice(&line[..take]); - } - - let mut compressor: FrameCompressor = - FrameCompressor::new(super::CompressionLevel::Fastest); - let mut out = Vec::new(); - compressor.compress_independent_frame_into(&input, &mut out); - - assert!(!out.is_empty()); - assert!( - out.capacity() < input.len() / 4, - "capacity {} must stay at output scale (input {}, output {})", - out.capacity(), - input.len(), - out.len() - ); - - // Round-trip: the adaptive reservation must not affect the bytes. - let mut decoder = crate::decoding::FrameDecoder::new(); - let mut decoded = Vec::with_capacity(input.len() + 64); - decoder - .decode_all_to_vec(&out, &mut decoded) - .expect("frame must decode"); - assert_eq!(decoded, input); - } - - // A dictionary frame with a known content size that fits the window - // must take the single-segment layout (reference parity): the - // dictionary is decoder setup state, not part of the regenerated - // segment, so it must not force the windowed multi-segment layout. - #[test] - fn dict_frame_with_known_size_is_single_segment() { - let dict_raw = include_bytes!("../../dict_tests/dictionary"); - let payload = b"dictionary-keyed payload dictionary-keyed payload".repeat(64); - - let mut compressor: FrameCompressor = - FrameCompressor::new(super::CompressionLevel::Fastest); - compressor - .set_dictionary_from_bytes(dict_raw) - .expect("dictionary bytes should parse"); - let mut frame = Vec::new(); - compressor.compress_independent_frame_into(&payload, &mut frame); - - let parsed = crate::decoding::frame::read_frame_header(frame.as_slice()) - .expect("frame header must parse") - .0; - assert!( - parsed.descriptor.single_segment_flag(), - "dict frame with known size <= window must be single-segment" - ); - assert!(parsed.dictionary_id().is_some()); - assert_eq!(parsed.frame_content_size(), payload.len() as u64); - - // Round-trip through our own decoder with the dictionary. - let mut decoder = crate::decoding::FrameDecoder::new(); - decoder - .add_dict_from_bytes(dict_raw) - .expect("decoder must accept the dictionary"); - let mut decoded = Vec::with_capacity(payload.len() + 64); - decoder - .decode_all_to_vec(&frame, &mut decoded) - .expect("single-segment dict frame must decode"); - assert_eq!(decoded, payload); - } - - // Regression: after `clear_dictionary()` a reused compressor must fully - // deactivate the dictionary state. The dict-active matcher reset rewinds the - // hash/chain tables to the origin (`position_base = 0`) and DEFERS the table - // clear to the next prime/restore. If `clear_dictionary` leaves the matcher - // marked dictionary-active, a subsequent NO-dictionary frame hits that - // deferred-clear branch but never runs prime/restore, so stale dict-region - // entries (old absolute positions) survive at the rewound base and can - // surface as bogus matches. The no-dict frame must still round-trip without - // the dictionary. Uses Level(16) (btopt → HashChain backend, whose storage - // owns the deferred-clear path). - #[test] - fn clear_dictionary_then_nodict_frame_roundtrips() { - let dict_raw = include_bytes!("../../dict_tests/dictionary"); - // Payload B embeds dictionary bytes up front so a surviving dict-region - // chain entry would be a tempting (wrong) match candidate. - let mut payload_b = Vec::new(); - payload_b.extend_from_slice(&dict_raw[..dict_raw.len().min(2048)]); - payload_b.extend_from_slice( - b"no-dictionary tail no-dictionary tail" - .repeat(16) - .as_slice(), - ); - - let mut compressor: FrameCompressor = - FrameCompressor::new(super::CompressionLevel::Level(16)); - compressor - .set_dictionary_from_bytes(dict_raw) - .expect("dictionary bytes should parse"); - // Frame A primes the dictionary (sets the matcher dictionary-active). - let mut frame_a = Vec::new(); - compressor.compress_independent_frame_into( - b"dictionary-keyed payload dictionary-keyed payload" - .repeat(8) - .as_slice(), - &mut frame_a, - ); - // Remove the dictionary, then compress a no-dictionary frame on the - // SAME reused compressor. - compressor.clear_dictionary(); - let mut frame_b = Vec::new(); - compressor.compress_independent_frame_into(&payload_b, &mut frame_b); - - // Frame B must decode WITHOUT any dictionary. - let mut decoder = crate::decoding::FrameDecoder::new(); - let mut decoded = Vec::with_capacity(payload_b.len() + 64); - decoder - .decode_all_to_vec(&frame_b, &mut decoded) - .expect("no-dict frame after clear_dictionary must decode"); - assert_eq!( - decoded, payload_b, - "no-dict frame after clear_dictionary must round-trip exactly" - ); - } - - // Regression test: `heap_size()` must count the retained Huffman tables - // (the active `last_huff_table` and the recycled `huff_table_spare`). - // A reused context that parks a table would otherwise under-report its - // footprint through the public size API. - #[test] - fn heap_size_counts_active_and_spare_huffman_tables() { - let mut compressor: FrameCompressor = - FrameCompressor::new(super::CompressionLevel::Fastest); - let base = compressor.heap_size(); - - let active = crate::huff0::huff0_encoder::HuffmanTable::build_from_data( - b"abacabadabacabaeabacabadabacaba", - ); - let active_bytes = active.heap_size(); - assert!(active_bytes > 0, "built table must own heap buffers"); - compressor.state.last_huff_table = Some(active); - assert_eq!( - compressor.heap_size(), - base + active_bytes, - "heap_size must include the active last_huff_table" - ); - - let spare = crate::huff0::huff0_encoder::HuffmanTable::build_from_data( - b"the quick brown fox jumps over the lazy dog", - ); - let spare_bytes = spare.heap_size(); - assert!(spare_bytes > 0, "built table must own heap buffers"); - compressor.state.huff_table_spare = Some(spare); - assert_eq!( - compressor.heap_size(), - base + active_bytes + spare_bytes, - "heap_size must include the parked huff_table_spare" - ); - } - - #[test] - fn set_encoder_dictionary_reattaches_prepared_dict_without_reparse() { - let dict_raw = include_bytes!("../../dict_tests/dictionary"); - let payload = b"tenant=demo table=orders op=put key=1 value=aaaaabbbbbcccccdddddeeeee\n\ - tenant=demo table=orders op=put key=2 value=aaaaabbbbbcccccdddddeeeee\n"; - - // Prepare the EncoderDictionary once, then attach it via the prepared- - // dictionary API (no raw-blob reparse at attach time). - let prepared = - super::EncoderDictionary::from_bytes(dict_raw).expect("dict bytes should parse"); - let dict_id = prepared.id(); - - let mut with_dict = Vec::new(); - let mut compressor = FrameCompressor::new(super::CompressionLevel::Fastest); - let previous = compressor - .set_encoder_dictionary(prepared) - .expect("prepared dictionary should attach"); - assert!(previous.is_none()); - compressor.set_source(payload.as_slice()); - compressor.set_drain(&mut with_dict); - compressor.compress(); - // clear_dictionary hands the prepared dictionary back (last use of - // `compressor`, so its `&mut with_dict` drain borrow ends here). - let returned = compressor - .clear_dictionary() - .expect("dictionary was attached"); - assert_eq!(returned.id(), dict_id); - - // The reattached dictionary drives the frame: its id is advertised and - // the stream round-trips through a decoder primed with the same dict. - let (frame_header, _) = crate::decoding::frame::read_frame_header(with_dict.as_slice()) - .expect("encoded stream should have a frame header"); - assert_eq!(frame_header.dictionary_id(), Some(dict_id)); - let decoder_dict = crate::decoding::Dictionary::decode_dict(dict_raw).unwrap(); - let mut decoder = FrameDecoder::new(); - decoder.add_dict(decoder_dict).unwrap(); - let mut decoded = Vec::with_capacity(payload.len()); - decoder.decode_all_to_vec(&with_dict, &mut decoded).unwrap(); - assert_eq!(decoded.as_slice(), payload.as_slice()); - - // The dictionary handed back by clear_dictionary reattaches to another - // compressor without touching the raw bytes again, producing an - // identical frame. - let mut with_dict2 = Vec::new(); - let mut compressor2 = FrameCompressor::new(super::CompressionLevel::Fastest); - compressor2 - .set_encoder_dictionary(returned) - .expect("returned dictionary should reattach"); - compressor2.set_source(payload.as_slice()); - compressor2.set_drain(&mut with_dict2); - compressor2.compress(); - assert_eq!( - with_dict2, with_dict, - "reattached prepared dict must produce an identical frame" - ); - } - - #[test] - fn dict_primed_matcher_snapshot_reused_across_frames_is_byte_identical() { - // CDict-equivalent: a compressor reused across frames with the same - // dictionary restores the primed matcher snapshot on frames 2..N - // (a table copy) instead of re-hashing the dictionary. The restored - // state must reproduce the first-frame (freshly-primed) output - // byte-for-byte, and every frame must round-trip through a - // dict-primed decoder. - let dict_raw = include_bytes!("../../dict_tests/dictionary"); - // Source must exceed the Fast strategy's 8 KiB attach cutoff so the - // copy-snapshot (restore) path is taken on frame 2 — at or below the - // cutoff the upstream zstd attaches by reference and we fall back to re-prime, - // which would not exercise restore. - let mut payload = Vec::new(); - while payload.len() < 16 * 1024 { - payload.extend_from_slice( - b"tenant=demo table=orders op=put key=1 value=aaaaabbbbbcccccdddddeeeee\n", - ); - } - - let prepared = - super::EncoderDictionary::from_bytes(dict_raw).expect("dict bytes should parse"); - let dict_id = prepared.id(); - let mut compressor: FrameCompressor = - FrameCompressor::new(super::CompressionLevel::Fastest); - compressor - .set_encoder_dictionary(prepared) - .expect("prepared dictionary should attach"); - - // Frame 1 primes + captures the snapshot; frame 2 restores it. - let frame1 = compressor.compress_independent_frame(payload.as_slice()); - let frame2 = compressor.compress_independent_frame(payload.as_slice()); - assert_eq!( - frame1, frame2, - "restored prime snapshot must reproduce the freshly-primed frame byte-for-byte" - ); - - // Both frames advertise the dict id and round-trip through a - // dict-primed decoder. - for frame in [&frame1, &frame2] { - let (hdr, _) = - crate::decoding::frame::read_frame_header(frame.as_slice()).expect("frame header"); - assert_eq!(hdr.dictionary_id(), Some(dict_id)); - let mut decoder = FrameDecoder::new(); - decoder - .add_dict(crate::decoding::Dictionary::decode_dict(dict_raw).unwrap()) - .unwrap(); - let mut decoded = Vec::with_capacity(payload.len()); - decoder.decode_all_to_vec(frame, &mut decoded).unwrap(); - assert_eq!(decoded.as_slice(), payload.as_slice()); - } - } - - #[test] - fn dict_primed_matcher_cache_reused_across_small_attach_frames_is_byte_identical() { - // CDict-equivalent ATTACH path (small source, at/below the Fast 8 KiB - // attach cutoff): frames 2..N re-prime — re-committing the dict bytes - // to history — but reuse the already-built dict table instead of - // re-hashing it. The cached-table frame must reproduce the - // freshly-primed first frame byte-for-byte, and a fresh single-frame - // compressor (no prior dict cache) must produce the identical bytes - // too, proving the cache changes timing, not output. - let dict_raw = include_bytes!("../../dict_tests/dictionary"); - // Stay under the 8 KiB cutoff so the attach (re-prime) path is taken - // every frame rather than the copy-snapshot restore. - let mut payload = Vec::new(); - while payload.len() < 2 * 1024 { - payload.extend_from_slice(b"tenant=demo op=put key=1 value=aaaaabbbbbcccccddddd\n"); - } - - let prepared = - super::EncoderDictionary::from_bytes(dict_raw).expect("dict bytes should parse"); - let dict_id = prepared.id(); - let mut compressor: FrameCompressor = - FrameCompressor::new(super::CompressionLevel::Fastest); - compressor - .set_encoder_dictionary(prepared) - .expect("prepared dictionary should attach"); - - // Frame 1 builds + marks the dict table; frame 2 reuses it. - let frame1 = compressor.compress_independent_frame(payload.as_slice()); - let frame2 = compressor.compress_independent_frame(payload.as_slice()); - assert_eq!( - frame1, frame2, - "reused dict table (attach path) must reproduce the freshly-built frame byte-for-byte" - ); - - // A fresh compressor (cold dict cache) must emit the same bytes — the - // cache is a timing optimization, never a content change. - let fresh_prepared = - super::EncoderDictionary::from_bytes(dict_raw).expect("dict bytes should parse"); - let mut fresh: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Fastest); - fresh - .set_encoder_dictionary(fresh_prepared) - .expect("prepared dictionary should attach"); - let fresh_frame = fresh.compress_independent_frame(payload.as_slice()); - assert_eq!( - fresh_frame, frame1, - "cold-cache compressor must match the warm-cache frame byte-for-byte" - ); - - for frame in [&frame1, &frame2] { - let (hdr, _) = - crate::decoding::frame::read_frame_header(frame.as_slice()).expect("frame header"); - assert_eq!(hdr.dictionary_id(), Some(dict_id)); - let mut decoder = FrameDecoder::new(); - decoder - .add_dict(crate::decoding::Dictionary::decode_dict(dict_raw).unwrap()) - .unwrap(); - let mut decoded = Vec::with_capacity(payload.len()); - decoder.decode_all_to_vec(frame, &mut decoded).unwrap(); - assert_eq!(decoded.as_slice(), payload.as_slice()); - } - } - - #[test] - fn dict_reused_across_many_lazy_frames_stays_applied() { - // Regression: a reused HashChain-backed dictionary frame (lazy levels) - // re-primes the dict at the live `history_abs_start` every frame. The - // floor-advance reset let that base climb frame-over-frame until the - // freshly-primed dict region dropped below `window_low`, after which - // every dict match was silently lost and the output ballooned to the - // no-dict size (observed at frame 3-4 of a reused compressor). Drive - // many frames and require every one to stay byte-identical to the - // first — the dict must keep applying, not decay after a few frames. - // Multiple distinct lines so the dictionary is load-bearing: without it - // frame 0 must emit each distinct line as literals; with it those lines - // match the primed dict immediately. A single repeated line matches - // in-frame regardless and would hide the regression. - let lines: &[&[u8]] = &[ - b"ts=2026 level=INFO msg=\"flush memtable\" tenant=demo table=orders\n", - b"ts=2026 level=INFO msg=\"rotate segment\" tenant=demo table=orders\n", - b"ts=2026 level=INFO msg=\"compact level\" tenant=demo table=orders\n", - b"ts=2026 level=INFO msg=\"write block\" tenant=demo table=orders\n", - ]; - let fill = |n: usize| -> Vec { - let mut b = Vec::with_capacity(n); - while b.len() < n { - for l in lines { - if b.len() >= n { - break; - } - let take = (n - b.len()).min(l.len()); - b.extend_from_slice(&l[..take]); - } - } - b - }; - let dict = fill(8 * 1024); - let payload = fill(16 * 1024); - let dict_obj = - crate::decoding::Dictionary::from_raw_content(1, dict).expect("raw dict should build"); - - let mut compressor: FrameCompressor = - FrameCompressor::new(super::CompressionLevel::Level(6)); - compressor.set_dictionary_id_flag(false); - compressor - .set_dictionary(dict_obj) - .expect("dict should attach"); - - let first = compressor.compress_independent_frame(payload.as_slice()); - - // No-dict baseline at the same level: the dictionary must be - // load-bearing, so the dict-applied frame has to beat it. Without this - // anchor the equal-length loop below would also pass if EVERY frame - // decayed to the no-dict size in lockstep. - let mut nodict: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Level(6)); - let no_dict_frame = nodict.compress_independent_frame(payload.as_slice()); - assert!( - first.len() < no_dict_frame.len(), - "dict must be load-bearing: dict frame {} should beat the no-dict baseline {}", - first.len(), - no_dict_frame.len(), - ); - - for i in 1..16 { - let frame = compressor.compress_independent_frame(payload.as_slice()); - // Byte-identity, not just equal length: a same-size divergence (e.g. - // a different match decision once the resident dict bookkeeping - // drifts) would slip past a length-only check. - assert_eq!( - frame, first, - "frame {i} of a reused dict compressor must stay byte-identical to \ - the first (dict still applied, no decay or bookkeeping drift)" - ); - } - } - - #[test] - fn dict_fast_epoch_reset_many_frames_and_attach_copy_alternation_byte_identical() { - // The Fast attach path invalidates the main hash table between - // frames with an epoch-bias advance instead of a memset. Two things - // need proving against a fresh-compressor reference: - // 1. the bias accumulates across MANY reused frames without ever - // letting a stale entry through (every frame byte-identical); - // 2. crossing the 8 KiB attach/copy cutoff in both directions - // (attach → copy clears the bias for the raw-slice kernel, - // copy → attach re-enters epoch mode) stays byte-identical. - let dict_raw = include_bytes!("../../dict_tests/dictionary"); - let mut small = Vec::new(); - while small.len() < 2 * 1024 { - small.extend_from_slice(b"tenant=demo op=put key=1 value=aaaaabbbbbcccccddddd\n"); - } - // Over the Fast 8 KiB attach cutoff → copy-mode frame. - let mut large = Vec::new(); - while large.len() < 64 * 1024 { - large.extend_from_slice(b"tenant=demo op=scan range=[k0,k9) limit=500 order=asc\n"); - } - - let mut reused: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Fastest); - reused - .set_encoder_dictionary( - super::EncoderDictionary::from_bytes(dict_raw).expect("dict bytes should parse"), - ) - .expect("prepared dictionary should attach"); - - let reference = |payload: &[u8]| -> alloc::vec::Vec { - let mut fresh: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Fastest); - fresh - .set_encoder_dictionary( - super::EncoderDictionary::from_bytes(dict_raw) - .expect("dict bytes should parse"), - ) - .expect("prepared dictionary should attach"); - fresh.compress_independent_frame(payload) - }; - - let small_expected = reference(&small); - let large_expected = reference(&large); - - // 1. Long attach-only run: every frame advances the epoch bias. - for i in 0..32 { - let frame = reused.compress_independent_frame(small.as_slice()); - assert_eq!( - frame, small_expected, - "attach frame {i} diverged from the fresh-compressor reference" - ); - } - // 2. Cutoff alternation: attach → copy → attach → copy. - for i in 0..4 { - let frame = reused.compress_independent_frame(large.as_slice()); - assert_eq!( - frame, large_expected, - "copy frame {i} diverged from the fresh-compressor reference" - ); - let frame = reused.compress_independent_frame(small.as_slice()); - assert_eq!( - frame, small_expected, - "attach frame after copy {i} diverged from the fresh-compressor reference" - ); - } - } - - #[test] - fn dict_primed_btlazy2_reused_across_attach_and_copy_boundary_is_byte_identical() { - // Btlazy2 (Level 15) uses the 32 KiB dict attach/copy cutoff in - // prepare_frame. Exercise BOTH sides of that boundary on a reused - // compressor: a sub-cutoff payload (re-prime/attach path) and an - // over-cutoff payload (copy-snapshot restore path). In each case the - // warm-cache second frame must reproduce the cold-cache first frame - // byte-for-byte (the dict cache is a timing optimization, never a - // content change), and every frame must round-trip. - let dict_raw = include_bytes!("../../dict_tests/dictionary"); - let dict_id = super::EncoderDictionary::from_bytes(dict_raw) - .expect("dict bytes should parse") - .id(); - // Distinct lines so the payload does not trivially self-compress; the - // BT finder + dict dual-probe both get exercised. - let make_payload = |target: usize| { - let mut p = Vec::with_capacity(target); - let mut i = 0u64; - while p.len() < target { - p.extend_from_slice( - format!( - "tenant=demo op=put key={i} value=aaaaabbbbbcccccddddd-{}\n", - i % 97 - ) - .as_bytes(), - ); - i += 1; - } - p - }; - // Below the 32 KiB cutoff (attach/re-prime) and above it (copy-snapshot). - for target in [16 * 1024usize, 64 * 1024usize] { - let payload = make_payload(target); - let mut warm: FrameCompressor = - FrameCompressor::new(super::CompressionLevel::Level(15)); - warm.set_encoder_dictionary( - super::EncoderDictionary::from_bytes(dict_raw).expect("dict parse"), - ) - .expect("dict attach"); - // Frame 1 builds + marks the dict tables; frame 2 reuses them. - let frame1 = warm.compress_independent_frame(payload.as_slice()); - let frame2 = warm.compress_independent_frame(payload.as_slice()); - assert_eq!( - frame1, frame2, - "reused dict cache must reproduce the freshly-primed frame byte-for-byte \ - (Level 15, target={target})" - ); - // Cold-cache compressor: must match the warm-cache bytes. - let mut cold: FrameCompressor = - FrameCompressor::new(super::CompressionLevel::Level(15)); - cold.set_encoder_dictionary( - super::EncoderDictionary::from_bytes(dict_raw).expect("dict parse"), - ) - .expect("dict attach"); - let cold_frame = cold.compress_independent_frame(payload.as_slice()); - assert_eq!( - cold_frame, frame1, - "cold-cache compressor must match warm-cache frame (Level 15, target={target})" - ); - // Round-trip through a decoder primed with the same dict. - for frame in [&frame1, &frame2] { - let (hdr, _) = crate::decoding::frame::read_frame_header(frame.as_slice()) - .expect("frame header"); - assert_eq!(hdr.dictionary_id(), Some(dict_id)); - let mut decoder = FrameDecoder::new(); - decoder - .add_dict(crate::decoding::Dictionary::decode_dict(dict_raw).unwrap()) - .unwrap(); - let mut decoded = Vec::with_capacity(payload.len()); - decoder.decode_all_to_vec(frame, &mut decoded).unwrap(); - assert_eq!(decoded.as_slice(), payload.as_slice()); - } - } - } - - #[test] - fn dict_primed_btultra2_restore_is_floor_safe_and_byte_identical() { - // Regression guard for the dictionary primed-snapshot RESTORE path on - // the binary-tree (btultra2 / Level 22) backend — the path a minimal / - // decoupled prepared-dict refactor rewrites. - // - // The trap it pins: a reused compressor compresses frame A (which fills - // the live hash/chain tables with frame-A positions and advances the - // window floor), then frame B of the SAME resolved shape (same size → - // same PrimedKey → the snapshot RESTORE path) but DIFFERENT content. The - // restore must reinstate the clean post-prime dict state with NO live - // frame-A entries surviving above the restored floor; a restore that - // leaks stale frame-A positions would surface FALSE matches and produce - // a different (or undecodable) frame B. The invariant: a snapshot - // restore is a pure timing optimization and MUST be byte-identical to a - // cold compressor compressing frame B from scratch, and must round-trip. - let dict_raw = include_bytes!("../../dict_tests/dictionary"); - let dict_id = super::EncoderDictionary::from_bytes(dict_raw) - .expect("dict bytes should parse") - .id(); - // 48 KiB > the btultra2 8 KiB attach cutoff → the copy-snapshot - // capture/restore path. Two distinct payloads of the SAME size so frame - // B resolves to frame A's snapshot key and takes the restore path. - let make_payload = |seed: u64, target: usize| { - let mut p = Vec::with_capacity(target); - let mut i = seed; - while p.len() < target { - p.extend_from_slice( - format!( - "tenant=demo op=put key={i} value=aaaaabbbbbcccccddddd-{}\n", - i % 89 - ) - .as_bytes(), - ); - i = i.wrapping_add(1); - } - p.truncate(target); - p - }; - let size = 48 * 1024usize; - let frame_a = make_payload(0, size); - let frame_b = make_payload(1_000_000, size); - - let mut warm: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Level(22)); - warm.set_encoder_dictionary( - super::EncoderDictionary::from_bytes(dict_raw).expect("dict parse"), - ) - .expect("dict attach"); - // Frame A: cold cache — primes the dict + captures the snapshot, and - // fills the live tables with frame-A positions. - let _wa = warm.compress_independent_frame(frame_a.as_slice()); - // Frame B: warm cache — takes the snapshot RESTORE path (same size). - let warm_b = warm.compress_independent_frame(frame_b.as_slice()); - - // Cold compressor compressing frame B from scratch: the ground truth. - let mut cold: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Level(22)); - cold.set_encoder_dictionary( - super::EncoderDictionary::from_bytes(dict_raw).expect("dict parse"), - ) - .expect("dict attach"); - let cold_b = cold.compress_independent_frame(frame_b.as_slice()); - - assert_eq!( - warm_b, cold_b, - "frame B via snapshot restore must be byte-identical to a cold compress \ - (a restore that leaks frame-A live-table entries would diverge here)" - ); - - // Round-trip frame B through a dict-primed decoder. - let (hdr, _) = - crate::decoding::frame::read_frame_header(warm_b.as_slice()).expect("frame header"); - assert_eq!(hdr.dictionary_id(), Some(dict_id)); - let mut decoder = FrameDecoder::new(); - decoder - .add_dict(crate::decoding::Dictionary::decode_dict(dict_raw).unwrap()) - .unwrap(); - let mut decoded = Vec::with_capacity(frame_b.len()); - decoder - .decode_all_to_vec(warm_b.as_slice(), &mut decoded) - .unwrap(); - assert_eq!(decoded.as_slice(), frame_b.as_slice()); - } - - #[test] - fn dict_primed_btultra2_ldm_restore_is_byte_identical() { - // Same restore-path byte-identity guard as - // `dict_primed_btultra2_restore_is_floor_safe_and_byte_identical`, but - // with long-distance matching ENABLED. The BtMatcher's LDM producer is - // part of the snapshot; a refactor that decouples it (so the snapshot - // does not retain the empty LDM table) must reinstate an equivalent - // empty producer on restore. This pins that the warm-cache (restore) - // frame stays byte-identical to a cold compress when LDM is on. - let dict_raw = include_bytes!("../../dict_tests/dictionary"); - let dict_id = super::EncoderDictionary::from_bytes(dict_raw) - .expect("dict bytes should parse") - .id(); - let make_payload = |seed: u64, target: usize| { - let mut p = Vec::with_capacity(target); - let mut i = seed; - while p.len() < target { - p.extend_from_slice( - format!( - "tenant=demo op=put key={i} value=aaaaabbbbbcccccddddd-{}\n", - i % 89 - ) - .as_bytes(), - ); - i = i.wrapping_add(1); - } - p.truncate(target); - p - }; - let ldm_params = - crate::encoding::CompressionParameters::builder(super::CompressionLevel::Level(22)) - .enable_long_distance_matching(true) - .build() - .expect("LDM-only params build"); - let size = 48 * 1024usize; - let frame_a = make_payload(0, size); - let frame_b = make_payload(1_000_000, size); - - let mut warm: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Level(22)); - warm.set_parameters(&ldm_params); - warm.set_encoder_dictionary( - super::EncoderDictionary::from_bytes(dict_raw).expect("dict parse"), - ) - .expect("dict attach"); - let _wa = warm.compress_independent_frame(frame_a.as_slice()); - let warm_b = warm.compress_independent_frame(frame_b.as_slice()); - - let mut cold: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Level(22)); - cold.set_parameters(&ldm_params); - cold.set_encoder_dictionary( - super::EncoderDictionary::from_bytes(dict_raw).expect("dict parse"), - ) - .expect("dict attach"); - let cold_b = cold.compress_independent_frame(frame_b.as_slice()); - - assert_eq!( - warm_b, cold_b, - "LDM-on frame B via snapshot restore must be byte-identical to a cold compress" - ); - - let (hdr, _) = - crate::decoding::frame::read_frame_header(warm_b.as_slice()).expect("frame header"); - assert_eq!(hdr.dictionary_id(), Some(dict_id)); - let mut decoder = FrameDecoder::new(); - decoder - .add_dict(crate::decoding::Dictionary::decode_dict(dict_raw).unwrap()) - .unwrap(); - let mut decoded = Vec::with_capacity(frame_b.len()); - decoder - .decode_all_to_vec(warm_b.as_slice(), &mut decoded) - .unwrap(); - assert_eq!(decoded.as_slice(), frame_b.as_slice()); - } - - #[test] - fn set_dictionary_from_bytes_matches_full_decode_byte_for_byte() { - // The encoder-only dict parse (`decode_dict_for_encoding`, used by - // `set_dictionary_from_bytes`) skips the FSE/HUF decoder-table build and - // the enrich passes. The encoder entropy tables are derived purely from - // the symbol probabilities / Huffman weights, so the compressed output - // MUST be byte-identical to the full-decode path. This pins the - // load-bearing equivalence so a future FSE/HUF parsing refactor that - // still round-trips but silently diverges on the probabilities/weights - // fails loudly here instead of producing a different (but valid) frame. - let dict_raw = include_bytes!("../../dict_tests/dictionary"); - let payload = b"tenant=demo table=orders op=put key=1 value=aaaaabbbbbcccccdddddeeeee\n\ - tenant=demo table=orders op=put key=2 value=aaaaabbbbbcccccdddddeeeee\n"; - - // Path A: encoder-only parse straight from the raw blob. - let mut from_bytes_out = Vec::new(); - { - let mut compressor = FrameCompressor::new(super::CompressionLevel::Fastest); - compressor - .set_dictionary_from_bytes(dict_raw) - .expect("dictionary bytes should parse"); - compressor.set_source(payload.as_slice()); - compressor.set_drain(&mut from_bytes_out); - compressor.compress(); - } - - // Path B: full decode (builds the decoder tables too), then attach for - // encoding via the `Dictionary` setter. - let full_decode = crate::decoding::Dictionary::decode_dict(dict_raw) - .expect("dictionary bytes should fully decode"); - let mut full_decode_out = Vec::new(); - { - let mut compressor = FrameCompressor::new(super::CompressionLevel::Fastest); - compressor - .set_dictionary(full_decode) - .expect("full-decode dictionary should attach"); - compressor.set_source(payload.as_slice()); - compressor.set_drain(&mut full_decode_out); - compressor.compress(); - } - - assert_eq!( - from_bytes_out, full_decode_out, - "encoder-only dict parse must produce byte-identical output to the full decode" - ); - } - - #[test] - fn set_dictionary_rejects_zero_dictionary_id() { - let invalid = crate::decoding::Dictionary { - id: 0, - fse: crate::decoding::scratch::FSEScratch::new(), - huf: crate::decoding::scratch::HuffmanScratch::new(), - dict_content: vec![1, 2, 3], - offset_hist: [1, 4, 8], - }; - - let mut compressor: FrameCompressor< - &[u8], - Vec, - crate::encoding::match_generator::MatchGeneratorDriver, - > = FrameCompressor::new(super::CompressionLevel::Fastest); - let result = compressor.set_dictionary(invalid); - assert!(matches!( - result, - Err(crate::decoding::errors::DictionaryDecodeError::ZeroDictionaryId) - )); - } - - #[test] - fn set_dictionary_rejects_zero_repeat_offsets() { - let invalid = crate::decoding::Dictionary { - id: 1, - fse: crate::decoding::scratch::FSEScratch::new(), - huf: crate::decoding::scratch::HuffmanScratch::new(), - dict_content: vec![1, 2, 3], - offset_hist: [0, 4, 8], - }; - - let mut compressor: FrameCompressor< - &[u8], - Vec, - crate::encoding::match_generator::MatchGeneratorDriver, - > = FrameCompressor::new(super::CompressionLevel::Fastest); - let result = compressor.set_dictionary(invalid); - assert!(matches!( - result, - Err( - crate::decoding::errors::DictionaryDecodeError::ZeroRepeatOffsetInDictionary { - index: 0 - } - ) - )); - } - - #[test] - fn uncompressed_mode_does_not_require_dictionary() { - let dict_id = 0xABCD_0001; - let dict = - crate::decoding::Dictionary::from_raw_content(dict_id, b"shared-history".to_vec()) - .expect("raw dictionary should be valid"); - - let payload = b"plain-bytes-that-should-stay-raw"; - let mut output = Vec::new(); - let mut compressor = FrameCompressor::new(super::CompressionLevel::Uncompressed); - compressor - .set_dictionary(dict) - .expect("dictionary should attach in uncompressed mode"); - compressor.set_source(payload.as_slice()); - compressor.set_drain(&mut output); - compressor.compress(); - - let (frame_header, _) = crate::decoding::frame::read_frame_header(output.as_slice()) - .expect("encoded frame should have a header"); - assert_eq!( - frame_header.dictionary_id(), - None, - "raw/uncompressed frames must not advertise dictionary dependency" - ); - - let mut decoder = FrameDecoder::new(); - let mut decoded = Vec::with_capacity(payload.len()); - decoder.decode_all_to_vec(&output, &mut decoded).unwrap(); - assert_eq!(decoded, payload); - } - - #[test] - fn default_level_tiny_raw_dict_compresses_cleanly() { - // Coverage for the dfast dict-attach fast path with a - // sub-min-match raw-content dictionary: the dict-table probe in - // `start_matching_fast_loop` is gated on the dict table actually - // existing (`table().is_some()`), not merely on `is_attached()`, - // so a dictionary whose hashable region is shorter than the - // short-hash lookahead (where `prime_dict_tables_for_range` - // returns before allocating the tables) never dereferences a - // null dict pointer. Compressing at the default (dfast) level - // with such a dict must succeed. - let dict_id = 0xABCD_0009; - let dict = crate::decoding::Dictionary::from_raw_content(dict_id, b"abc".to_vec()) - .expect("raw dictionary should be valid"); - let payload = b"the quick brown fox jumps over the lazy dog, repeatedly and at length"; - let mut output = Vec::new(); - let mut compressor = FrameCompressor::new(super::CompressionLevel::Default); - compressor - .set_dictionary(dict) - .expect("tiny raw dictionary should attach"); - compressor.set_source(payload.as_slice()); - compressor.set_drain(&mut output); - compressor.compress(); - assert!(!output.is_empty(), "compression should produce a frame"); - - // The emitted frame must advertise the attached dictionary id, proving - // the tiny-dict path stayed active (the payload round-trips either way, - // so without this the test would also pass on a silent no-dict frame). - let (frame_header, _) = crate::decoding::frame::read_frame_header(output.as_slice()) - .expect("encoded frame should have a readable header"); - assert_eq!( - frame_header.dictionary_id(), - Some(dict_id), - "tiny raw dict frame should still advertise its dictionary id", - ); - - // Full roundtrip: decode the dict-compressed frame with the SAME - // dictionary attached and confirm byte-exact recovery — proves the - // tiny-dict fast path produces a correct frame, not just a non-empty - // one. - let decode_dict = crate::decoding::Dictionary::from_raw_content(dict_id, b"abc".to_vec()) - .expect("raw dictionary should be valid"); - let mut decoder = FrameDecoder::new(); - decoder - .add_dict(decode_dict) - .expect("decoder dict should attach"); - let mut decoded = Vec::with_capacity(payload.len()); - decoder - .decode_all_to_vec(&output, &mut decoded) - .expect("dict roundtrip should decode"); - assert_eq!(decoded, payload, "tiny-dict roundtrip mismatch"); - } - - /// Exercises the dictionary dual-probe (live + immutable dict tables) - /// in the Fast / dfast / Row match finders with a dict whose content - /// the payload actually reuses, so each backend's dict long/short - /// probe (and the dfast `ip+1` dict-long retry) is reached and the - /// dict-compressed frame round-trips through a decoder primed with the - /// same dict. The 3-byte-dict test above only proves the null-table - /// guard; this proves the full attach path produces correct frames. - #[test] - fn dict_attach_roundtrips_across_backends_with_matching_payload() { - let dict_id = 0xD1C7_0001; - // Distinct lines so the payload does NOT self-compress: each line - // appears exactly once in the payload, so without the dictionary there - // are no in-frame back-references to exploit. The dictionary holds the - // SAME lines, so the only way the output shrinks is if the dict probe - // actually fires. A no-dict baseline below pins that the dict path ran - // (self-compressible payloads would round-trip + stay small via - // in-frame matches alone, proving nothing). - let line = |i: u32| { - alloc::format!( - "ts=2026-03-26T21:{:02}:{:02}Z level=INFO msg=\"event {i:05}\" tenant=t{i} region=eu\n", - i / 60 % 60, - i % 60, - ) - .into_bytes() - }; - let mut dict_content = Vec::new(); - for i in 0..256u32 { - dict_content.extend_from_slice(&line(i)); - } - // Payload = the same distinct lines in a different (stride) order, each - // once → no self-repeats, every line is a dictionary match. - let mut payload = Vec::new(); - let mut i = 0u32; - for _ in 0..256u32 { - payload.extend_from_slice(&line(i)); - i = (i + 97) % 256; // coprime stride → permutation, no adjacency - } - - let compress_at = |level, dict: Option>| -> Vec { - let mut compressor = FrameCompressor::new(level); - if let Some(bytes) = dict { - let d = crate::decoding::Dictionary::from_raw_content(dict_id, bytes) - .expect("raw dictionary should be valid"); - compressor - .set_dictionary(d) - .expect("dictionary should attach"); - } - let mut out = Vec::new(); - compressor.set_source(payload.as_slice()); - compressor.set_drain(&mut out); - compressor.compress(); - out - }; - - for level in [ - super::CompressionLevel::Level(-5), // Fast (negative) - super::CompressionLevel::Level(1), // Fast - super::CompressionLevel::Default, // dfast (L3) - super::CompressionLevel::Level(8), // Row-backed lazy2 - ] { - let out = compress_at(level, Some(dict_content.clone())); - let no_dict = compress_at(level, None); - // The dict path MUST measurably beat no-dict on this - // non-self-compressible payload — otherwise the dict probe never - // fired and the roundtrip below would prove nothing. - assert!( - out.len() < no_dict.len(), - "level {level:?}: dict-primed output ({}) must beat no-dict ({}) — dict probe did not fire", - out.len(), - no_dict.len(), - ); - - let ddict = - crate::decoding::Dictionary::from_raw_content(dict_id, dict_content.clone()) - .expect("raw dictionary should be valid"); - let mut decoder = FrameDecoder::new(); - decoder.add_dict(ddict).expect("decoder dict should attach"); - let mut decoded = Vec::with_capacity(payload.len()); - decoder - .decode_all_to_vec(&out, &mut decoded) - .unwrap_or_else(|e| panic!("level {level:?}: dict roundtrip decode failed: {e:?}")); - assert_eq!(decoded, payload, "level {level:?}: dict roundtrip mismatch"); - } - } - - /// Reusing one compressor across independent frames with DIFFERENT - /// dictionaries must drop the per-backend dict cache on each swap - /// (Simple/Dfast/Row keep the attach index across frames). Without the - /// invalidation a later frame would reuse the previous dict's rows. - /// Each frame round-trips through a decoder primed with its own dict. - #[test] - fn dict_swap_across_reused_compressor_roundtrips() { - // Distinct lines per dict (not a single repeated line) so payloads do - // NOT self-compress: each line appears once, so a frame only shrinks if - // the dict probe fires, and — crucially for the invalidation check — if - // frame B reused dict A's stale rows it would emit offsets into A's - // distinct content, which decode under dict B reconstructs as WRONG - // bytes (caught by the roundtrip). A single repeated line would hide - // pollution behind in-frame matches. - let lines = |tag: &str| -> (Vec, Vec) { - let line = - |i: u32| alloc::format!("{tag} record {i:05} field=value{i} end\n").into_bytes(); - let mut dict = Vec::new(); - for i in 0..256u32 { - dict.extend_from_slice(&line(i)); - } - let mut payload = Vec::new(); - let mut i = 0u32; - for _ in 0..256u32 { - payload.extend_from_slice(&line(i)); - i = (i + 97) % 256; - } - (dict, payload) - }; - let (dict_a, payload_a) = lines("alpha"); - let (dict_b, payload_b) = lines("bravo"); - - for level in [ - super::CompressionLevel::Default, - super::CompressionLevel::Level(8), - ] { - let no_dict = |payload: &[u8]| -> usize { - let mut c: FrameCompressor = FrameCompressor::new(level); - c.compress_independent_frame(payload).len() - }; - let no_dict_a = no_dict(&payload_a); - let no_dict_b = no_dict(&payload_b); - - let mut compressor: FrameCompressor = FrameCompressor::new(level); - for (dict_bytes, payload, no_dict_len) in [ - (&dict_a, &payload_a, no_dict_a), - (&dict_b, &payload_b, no_dict_b), - ] { - let dict = - crate::decoding::Dictionary::from_raw_content(0xD1C7_0002, dict_bytes.clone()) - .expect("raw dictionary should be valid"); - compressor - .set_dictionary(dict) - .expect("dictionary should attach"); - let out = compressor.compress_independent_frame(payload.as_slice()); - assert!( - out.len() < no_dict_len, - "level {level:?}: dict frame ({}) must beat no-dict ({}) — dict probe did not fire", - out.len(), - no_dict_len, - ); - - let ddict = - crate::decoding::Dictionary::from_raw_content(0xD1C7_0002, dict_bytes.clone()) - .expect("raw dictionary should be valid"); - let mut decoder = FrameDecoder::new(); - decoder.add_dict(ddict).expect("decoder dict should attach"); - let mut decoded = Vec::with_capacity(payload.len()); - decoder - .decode_all_to_vec(&out, &mut decoded) - .unwrap_or_else(|e| panic!("level {level:?}: dict-swap decode failed: {e:?}")); - assert_eq!( - decoded, *payload, - "level {level:?}: dict-swap roundtrip mismatch (stale dict rows?)" - ); - } - } - } - - #[test] - fn dictionary_roundtrip_stays_valid_after_output_exceeds_window() { - use crate::encoding::match_generator::MatchGeneratorDriver; - - let dict_id = 0xABCD_0002; - let dict = crate::decoding::Dictionary::from_raw_content(dict_id, b"abcdefgh".to_vec()) - .expect("raw dictionary should be valid"); - let dict_for_decoder = - crate::decoding::Dictionary::from_raw_content(dict_id, b"abcdefgh".to_vec()) - .expect("raw dictionary should be valid"); - - // Payload must exceed the encoder's advertised window (512 KiB - // for Fastest after `window_log = 19` alignment with upstream zstd's - // L1 fast row in `clevels.h`) so the test actually exercises - // cross-window-boundary behavior. - let payload = b"abcdefgh".repeat(512 * 1024 / 8 + 64); - let matcher = MatchGeneratorDriver::new(1024, 1); - - let mut no_dict_output = Vec::new(); - let mut no_dict_compressor = - FrameCompressor::new_with_matcher(matcher, super::CompressionLevel::Fastest); - no_dict_compressor.set_source(payload.as_slice()); - no_dict_compressor.set_drain(&mut no_dict_output); - no_dict_compressor.compress(); - let (no_dict_frame_header, _) = - crate::decoding::frame::read_frame_header(no_dict_output.as_slice()) - .expect("baseline frame should have a header"); - let no_dict_window = no_dict_frame_header - .window_size() - .expect("window size should be present"); - - let mut output = Vec::new(); - let matcher = MatchGeneratorDriver::new(1024, 1); - let mut compressor = - FrameCompressor::new_with_matcher(matcher, super::CompressionLevel::Fastest); - compressor - .set_dictionary(dict) - .expect("dictionary should attach"); - compressor.set_source(payload.as_slice()); - compressor.set_drain(&mut output); - compressor.compress(); - - let (frame_header, _) = crate::decoding::frame::read_frame_header(output.as_slice()) - .expect("encoded frame should have a header"); - let advertised_window = frame_header - .window_size() - .expect("window size should be present"); - assert_eq!( - advertised_window, no_dict_window, - "dictionary priming must not inflate advertised window size" - ); - assert!( - payload.len() > advertised_window as usize, - "test must cross the advertised window boundary" - ); - - let mut decoder = FrameDecoder::new(); - decoder.add_dict(dict_for_decoder).unwrap(); - let mut decoded = Vec::with_capacity(payload.len()); - decoder.decode_all_to_vec(&output, &mut decoded).unwrap(); - assert_eq!(decoded, payload); - } - - #[test] - fn source_size_hint_with_dictionary_keeps_roundtrip_and_nonincreasing_window() { - let dict_id = 0xABCD_0004; - let dict_content = b"abcd".repeat(1024); // 4 KiB dictionary history - let dict = crate::decoding::Dictionary::from_raw_content(dict_id, dict_content).unwrap(); - let dict_for_decoder = - crate::decoding::Dictionary::from_raw_content(dict_id, b"abcd".repeat(1024)).unwrap(); - let payload = b"abcdabcdabcdabcd".repeat(128); - - let mut hinted_output = Vec::new(); - let mut hinted = FrameCompressor::new(super::CompressionLevel::Fastest); - hinted.set_dictionary(dict).unwrap(); - hinted.set_source_size_hint(1); - hinted.set_source(payload.as_slice()); - hinted.set_drain(&mut hinted_output); - hinted.compress(); - - let mut no_hint_output = Vec::new(); - let mut no_hint = FrameCompressor::new(super::CompressionLevel::Fastest); - no_hint - .set_dictionary( - crate::decoding::Dictionary::from_raw_content(dict_id, b"abcd".repeat(1024)) - .unwrap(), - ) - .unwrap(); - no_hint.set_source(payload.as_slice()); - no_hint.set_drain(&mut no_hint_output); - no_hint.compress(); - - let hinted_window = crate::decoding::frame::read_frame_header(hinted_output.as_slice()) - .expect("encoded frame should have a header") - .0 - .window_size() - .expect("window size should be present"); - let no_hint_window = crate::decoding::frame::read_frame_header(no_hint_output.as_slice()) - .expect("encoded frame should have a header") - .0 - .window_size() - .expect("window size should be present"); - assert!( - hinted_window <= no_hint_window, - "source-size hint should not increase advertised window with dictionary priming", - ); - - let mut decoder = FrameDecoder::new(); - decoder.add_dict(dict_for_decoder).unwrap(); - let mut decoded = Vec::with_capacity(payload.len()); - decoder - .decode_all_to_vec(&hinted_output, &mut decoded) - .unwrap(); - assert_eq!(decoded, payload); - } - - /// A dictionary segment embedded ONCE in otherwise-incompressible - /// input must be matched against the dictionary. Before the fix the - /// raw-fast-path (which skips matching) fired on the - /// incompressible-looking block and the dictionary was never searched, - /// so `with_dict` came out the same size as `no_dict` (the embedded - /// match was lost). Now the block compresses against the dict. - #[test] - fn dictionary_segment_in_incompressible_input_is_matched() { - // Deterministic LCG bytes: high-entropy, so the only compressible - // content is the embedded dictionary segment. - fn lcg(seed: u64, n: usize) -> alloc::vec::Vec { - let mut s = seed; - (0..n) - .map(|_| { - s = s - .wrapping_mul(6364136223846793005) - .wrapping_add(1442695040888963407); - (s >> 56) as u8 - }) - .collect() - } - let dict_id = 0x00DC_7777; - let r = lcg(1, 512); // the dictionary content - let mut payload = lcg(2, 2000); // incompressible filler before - payload.extend_from_slice(&r); // the single dict-matchable segment - payload.extend_from_slice(&lcg(3, 1500)); // filler after - - // Precondition: the payload must actually look incompressible so - // that the raw-fast-path WOULD fire (and skip matching) without - // the fix. If the heuristic ever changes and this no longer holds, - // the test below would pass vacuously — assert it up front. - assert!( - crate::encoding::incompressible::block_looks_incompressible(&payload), - "test payload must look incompressible to exercise the raw-fast-path", - ); - - let compress = - |level: super::CompressionLevel, dict: Option<&[u8]>| -> alloc::vec::Vec { - let mut out = alloc::vec::Vec::new(); - let mut c = FrameCompressor::new(level); - if let Some(d) = dict { - c.set_dictionary( - crate::decoding::Dictionary::from_raw_content(dict_id, d.to_vec()).unwrap(), - ) - .unwrap(); - } - c.set_source(payload.as_slice()); - c.set_drain(&mut out); - c.compress(); - out - }; - - for lvl in [ - super::CompressionLevel::Level(2), - super::CompressionLevel::Level(6), - super::CompressionLevel::Level(19), - ] { - let with_dict = compress(lvl, Some(&r)); - let no_dict = compress(lvl, None); - // The 512-byte dict segment should be matched, saving most of - // its length (generous slack for sequence/header coding). - assert!( - with_dict.len() + 300 < no_dict.len(), - "{lvl:?}: dict segment not matched (with_dict={}, no_dict={})", - with_dict.len(), - no_dict.len(), - ); - // The dict-compressed frame must round-trip through the decoder. - let mut decoder = FrameDecoder::new(); - decoder - .add_dict( - crate::decoding::Dictionary::from_raw_content(dict_id, r.clone()).unwrap(), - ) - .unwrap(); - let mut decoded = Vec::with_capacity(payload.len()); - decoder.decode_all_to_vec(&with_dict, &mut decoded).unwrap(); - assert_eq!(decoded, payload, "{lvl:?}: dict round-trip mismatch"); - - // A dictionary that does NOT appear in the input must not make - // the output larger than the no-dict (raw) encoding: the - // post-compress raw fallback covers incompressible-with-dict. - let unrelated = lcg(99, 512); - let with_bad_dict = compress(lvl, Some(&unrelated)); - assert!( - with_bad_dict.len() <= no_dict.len() + 16, - "{lvl:?}: unhelpful dict expanded output (with={}, no_dict={})", - with_bad_dict.len(), - no_dict.len(), - ); - } - } - - #[test] - fn source_size_hint_with_dictionary_keeps_roundtrip_for_larger_payload() { - let dict_id = 0xABCD_0005; - let dict_content = b"abcd".repeat(1024); // 4 KiB dictionary history - let dict = crate::decoding::Dictionary::from_raw_content(dict_id, dict_content).unwrap(); - let dict_for_decoder = - crate::decoding::Dictionary::from_raw_content(dict_id, b"abcd".repeat(1024)).unwrap(); - let payload = b"abcd".repeat(1024); // 4 KiB payload - let payload_len = payload.len() as u64; - - let mut hinted_output = Vec::new(); - let mut hinted = FrameCompressor::new(super::CompressionLevel::Fastest); - hinted.set_dictionary(dict).unwrap(); - hinted.set_source_size_hint(payload_len); - hinted.set_source(payload.as_slice()); - hinted.set_drain(&mut hinted_output); - hinted.compress(); - - let mut no_hint_output = Vec::new(); - let mut no_hint = FrameCompressor::new(super::CompressionLevel::Fastest); - no_hint - .set_dictionary( - crate::decoding::Dictionary::from_raw_content(dict_id, b"abcd".repeat(1024)) - .unwrap(), - ) - .unwrap(); - no_hint.set_source(payload.as_slice()); - no_hint.set_drain(&mut no_hint_output); - no_hint.compress(); - - let hinted_window = crate::decoding::frame::read_frame_header(hinted_output.as_slice()) - .expect("encoded frame should have a header") - .0 - .window_size() - .expect("window size should be present"); - let no_hint_window = crate::decoding::frame::read_frame_header(no_hint_output.as_slice()) - .expect("encoded frame should have a header") - .0 - .window_size() - .expect("window size should be present"); - assert!( - hinted_window <= no_hint_window, - "source-size hint should not increase advertised window with dictionary priming", - ); - - let mut decoder = FrameDecoder::new(); - decoder.add_dict(dict_for_decoder).unwrap(); - let mut decoded = Vec::with_capacity(payload.len()); - decoder - .decode_all_to_vec(&hinted_output, &mut decoded) - .unwrap(); - assert_eq!(decoded, payload); - } - - #[test] - fn custom_matcher_without_dictionary_priming_does_not_advertise_dict_id() { - let dict_id = 0xABCD_0003; - let dict = crate::decoding::Dictionary::from_raw_content(dict_id, b"abcdefgh".to_vec()) - .expect("raw dictionary should be valid"); - let payload = b"abcdefghabcdefgh"; - - let mut output = Vec::new(); - let matcher = NoDictionaryMatcher::new(64); - let mut compressor = - FrameCompressor::new_with_matcher(matcher, super::CompressionLevel::Fastest); - compressor - .set_dictionary(dict) - .expect("dictionary should attach"); - compressor.set_source(payload.as_slice()); - compressor.set_drain(&mut output); - compressor.compress(); - - let (frame_header, _) = crate::decoding::frame::read_frame_header(output.as_slice()) - .expect("encoded frame should have a header"); - assert_eq!( - frame_header.dictionary_id(), - None, - "matchers that do not support dictionary priming must not advertise dictionary dependency" - ); - - let mut decoder = FrameDecoder::new(); - let mut decoded = Vec::with_capacity(payload.len()); - decoder.decode_all_to_vec(&output, &mut decoded).unwrap(); - assert_eq!(decoded, payload); - } - - #[cfg(feature = "hash")] - #[test] - fn checksum_two_frames_reused_compressor() { - // Compress the same data twice using the same compressor and verify that: - // 1. The checksum written in each frame matches what the decoder calculates. - // 2. The hasher is correctly reset between frames (no cross-contamination). - // If the hasher were NOT reset, the second frame's calculated checksum - // would differ from the one stored in the frame data, causing assert_eq to fail. - let data: Vec = (0u8..=255).cycle().take(1024).collect(); - - let mut compressor = FrameCompressor::new(super::CompressionLevel::Uncompressed); - - // --- Frame 1 --- - let mut compressed1 = Vec::new(); - compressor.set_source(data.as_slice()); - compressor.set_drain(&mut compressed1); - compressor.compress(); - - // --- Frame 2 (reuse the same compressor) --- - let mut compressed2 = Vec::new(); - compressor.set_source(data.as_slice()); - compressor.set_drain(&mut compressed2); - compressor.compress(); - - fn decode_and_collect(compressed: &[u8]) -> (Vec, Option, Option) { - let mut decoder = FrameDecoder::new(); - let mut source = compressed; - decoder.reset(&mut source).unwrap(); - while !decoder.is_finished() { - decoder - .decode_blocks(&mut source, crate::decoding::BlockDecodingStrategy::All) - .unwrap(); - } - let mut decoded = Vec::new(); - decoder.collect_to_writer(&mut decoded).unwrap(); - ( - decoded, - decoder.get_checksum_from_data(), - decoder.get_calculated_checksum(), - ) - } - - let (decoded1, chksum_from_data1, chksum_calculated1) = decode_and_collect(&compressed1); - assert_eq!(decoded1, data, "frame 1: decoded data mismatch"); - assert_eq!( - chksum_from_data1, chksum_calculated1, - "frame 1: checksum mismatch" - ); - - let (decoded2, chksum_from_data2, chksum_calculated2) = decode_and_collect(&compressed2); - assert_eq!(decoded2, data, "frame 2: decoded data mismatch"); - assert_eq!( - chksum_from_data2, chksum_calculated2, - "frame 2: checksum mismatch" - ); - - // Same data compressed twice must produce the same checksum. - // If state leaked across frames, the second calculated checksum would differ. - assert_eq!( - chksum_from_data1, chksum_from_data2, - "frame 1 and frame 2 should have the same checksum (same data, hash must reset per frame)" - ); - } - - #[cfg(feature = "lsm")] - #[test] - fn frame_emit_info_decompressed_ranges_match_decoded_output() { - // Part A correctness: the per-block `decompressed_size` captured during - // encode (and the `decompressed_byte_range` prefix sum derived from it) - // must describe the real decoded output exactly — one entry per - // physical block, contiguous, summing to the full decompressed length. - // A multi-block compressible payload exercises the Compressed-block - // path (whose regenerated size is NOT on the wire, so it relies on the - // encode-side capture this test guards). - let data = emit_info_fixture_data(); - - // Cover both the single-block-per-chunk path (Default) and the - // Level(16..=22) post-split path (multiple physical partitions per - // input chunk), since lsm-tree compresses at zstd:22 and post-split - // is the riskiest capture site (per-partition `src_size`). - for level in [ - super::CompressionLevel::Default, - super::CompressionLevel::Level(22), - ] { - let mut compressed = Vec::new(); - let mut compressor = FrameCompressor::new(level); - // Pledge the source size so the high-level (22) window shrinks to - // fit the payload, keeping the frame compact (no oversized window - // descriptor for a small input). Still >= 128 KiB, so post-split - // eligibility is preserved. - compressor.set_source_size_hint(data.len() as u64); - compressor.set_source(data.as_slice()); - compressor.set_drain(&mut compressed); - compressor.compress(); - - let info = compressor - .last_frame_emit_info() - .expect("emit info populated after compress") - .clone(); - - // Reference: full decode of the same frame. - let mut decoder = FrameDecoder::new(); - let mut source = compressed.as_slice(); - decoder.reset(&mut source).unwrap(); - while !decoder.is_finished() { - decoder - .decode_blocks(&mut source, crate::decoding::BlockDecodingStrategy::All) - .unwrap(); - } - let mut decoded = Vec::new(); - decoder.collect_to_writer(&mut decoded).unwrap(); - assert_eq!(decoded, data, "sanity: frame must round-trip ({level:?})"); - - assert!( - info.blocks.len() >= 2, - "fixture must span multiple blocks to exercise the mapping ({level:?}, got {})", - info.blocks.len() - ); - assert!( - info.blocks.last().unwrap().last_block, - "final block must carry last_block ({level:?})" - ); - - // Pin the Level(22) post-split path: the owned loop feeds the - // encoder MAX_BLOCK_SIZE input chunks, so without post-split the - // block count cannot exceed the chunk count. More blocks than - // chunks proves at least one chunk was split into multiple physical - // partitions (the per-partition `src_size` capture under test). - if matches!(level, super::CompressionLevel::Level(22)) { - let max_block = crate::common::MAX_BLOCK_SIZE as usize; - let n_chunks = data.len().div_ceil(max_block); - assert!( - info.blocks.len() > n_chunks, - "Level(22) must exercise post-split: {} blocks for {} input chunks", - info.blocks.len(), - n_chunks - ); - } - - // Per-block ranges: contiguous, zero-based, summing to the full output. - let mut expected_start = 0u64; - for i in 0..info.blocks.len() { - let range = info - .decompressed_byte_range(i) - .expect("in-bounds block has a range"); - assert_eq!( - range.start, expected_start, - "block {i} range must start where the previous ended ({level:?})" - ); - assert_eq!( - u64::from(info.blocks[i].decompressed_size), - range.end - range.start, - "block {i} decompressed_size must equal its range width ({level:?})" - ); - // Validate the mapping against REAL per-block bytes, not just - // prefix-sum consistency: decode block `i` alone and require it - // to equal the corresponding slice of the full decode. A - // sidecar that swapped sizes between adjacent blocks (same sum, - // same contiguity) would fail here. - let mut psrc = compressed.as_slice(); - let mut pdec = FrameDecoder::new(); - pdec.reset(&mut psrc).unwrap(); - let pd = pdec - .decode_blocks_partial(&mut psrc, i as u32, i as u32 + 1, None, false) - .unwrap(); - assert!( - pd.stopped_at.is_none(), - "block {i} must decode cleanly ({level:?})" - ); - assert_eq!( - pd.data.as_slice(), - &decoded[range.start as usize..range.end as usize], - "block {i} partial-decode bytes must equal the full-decode slice ({level:?})" - ); - expected_start = range.end; - } - assert_eq!( - expected_start, - decoded.len() as u64, - "block decompressed sizes must sum to the full decoded length ({level:?})" - ); - assert_eq!( - info.decompressed_byte_range(info.blocks.len()), - None, - "out-of-range index yields None ({level:?})" - ); - } - } - - /// ~400 KiB semi-repetitive payload (long runs interleaved with a stride - /// phrase) that compresses into several multi-block frames across levels. - #[cfg(feature = "lsm")] - fn emit_info_fixture_data() -> Vec { - let mut data: Vec = Vec::with_capacity(400 * 1024); - let mut x = 0x9E37_79B9u32; - while data.len() < 400 * 1024 { - x ^= x << 13; - x ^= x >> 17; - x ^= x << 5; - let run = 16 + (x as usize % 48); - let byte = (x >> 24) as u8; - for _ in 0..run { - data.push(byte); - } - data.extend_from_slice(b"the quick brown fox jumps over the lazy dog\n"); - } - data - } - - #[cfg(feature = "lsm")] - #[test] - fn frame_emit_info_decompressed_ranges_match_on_borrowed_oneshot_path() { - // The borrowed one-shot path (`compress_independent_frame` -> - // `run_borrowed_block_loop` -> `compress_block_encoded_borrowed`) - // threads the decompressed-size sidecar through a DIFFERENT emit site - // than the owned/streaming loop, so it needs its own per-block mapping - // check. A Fast level keeps the encoder on the borrowed-eligible - // (Simple matcher) path. - let data = emit_info_fixture_data(); - - let mut compressor: FrameCompressor = - FrameCompressor::new(super::CompressionLevel::Fastest); - let compressed = compressor.compress_independent_frame(data.as_slice()); - let info = compressor - .last_frame_emit_info() - .expect("emit info populated after compress_independent_frame") - .clone(); - // Pin the compressed-block path: without this the fixture could regress - // into the raw-fast fallback and still pass via the Raw wire-size - // fallback in populate_frame_emit_info, never exercising the borrowed - // compressed-block sidecar capture this test targets. - assert!( - info.blocks - .iter() - .any(|b| matches!(b.block_type, crate::blocks::block::BlockType::Compressed)), - "borrowed-path fixture must emit at least one compressed block" - ); - assert!( - info.blocks.len() >= 2, - "borrowed fixture must span multiple blocks (got {})", - info.blocks.len() - ); - assert!(info.blocks.last().unwrap().last_block); - - // Full decode reference. - let mut decoder = FrameDecoder::new(); - let mut source = compressed.as_slice(); - decoder.reset(&mut source).unwrap(); - while !decoder.is_finished() { - decoder - .decode_blocks(&mut source, crate::decoding::BlockDecodingStrategy::All) - .unwrap(); - } - let mut decoded = Vec::new(); - decoder.collect_to_writer(&mut decoded).unwrap(); - assert_eq!(decoded, data, "borrowed one-shot frame must round-trip"); - - // Each block's mapping must match real per-block bytes. - let mut expected_start = 0u64; - for i in 0..info.blocks.len() { - let range = info.decompressed_byte_range(i).unwrap(); - assert_eq!(range.start, expected_start, "block {i} range contiguity"); - let mut psrc = compressed.as_slice(); - let mut pdec = FrameDecoder::new(); - pdec.reset(&mut psrc).unwrap(); - let pd = pdec - .decode_blocks_partial(&mut psrc, i as u32, i as u32 + 1, None, false) - .unwrap(); - assert!(pd.stopped_at.is_none(), "block {i} must decode cleanly"); - assert_eq!( - pd.data.as_slice(), - &decoded[range.start as usize..range.end as usize], - "borrowed block {i} partial-decode bytes must equal the full-decode slice" - ); - expected_start = range.end; - } - assert_eq!( - expected_start, - decoded.len() as u64, - "ranges sum to full length" - ); - } - - // The fuzz-artifact interop replay (C-compress -> our-decode and - // our-compress -> C-decode) moved to `ffi-bench/tests/fuzz_interop.rs` so - // the library crate never links libzstd. - - /// Homogeneous input — every byte the same — must NOT be split: - /// both border histograms are identical (all 512 hits on a single - /// slot), so `presplit_fingerprints_differ` returns `false` and the - /// function takes the early-return path at - /// `zstd_preSplit.c:214` returning `blockSize`. - #[test] - fn split_block_from_borders_keeps_homogeneous_block() { - let block = vec![0xAAu8; MAX_BLOCK_SIZE as usize]; - let split = super::split_block_from_borders(&block); - assert_eq!(split, MAX_BLOCK_SIZE as usize); - } - - /// Heterogeneous input — first half all zeros, second half a - /// counter sequence — has clearly distinguishable border - /// histograms, so the borders heuristic decides to split. - /// - /// The transition sits at exactly the block midpoint, so the - /// middle 512-byte sample (`block[mid-256..mid+256]`) is half - /// zeros + half counter values. That makes it roughly - /// equidistant from both border fingerprints — the - /// `abs_diff(dist_from_begin, dist_from_end) < min_distance` - /// branch fires and the heuristic returns the midpoint (64 KiB) - /// per `zstd_preSplit.c:222`. The test asserts the exact value - /// rather than just "one of {32K, 64K, 96K}" so a regression - /// to a different quantised arm cannot silently slip through. - #[test] - fn split_block_from_borders_returns_midpoint_for_centred_transition() { - let mut block = vec![0u8; MAX_BLOCK_SIZE as usize]; - for (i, byte) in block - .iter_mut() - .enumerate() - .skip(MAX_BLOCK_SIZE as usize / 2) - { - *byte = (i % 251 + 1) as u8; - } - let split = super::split_block_from_borders(&block); - assert_eq!( - split, - 64 * 1024, - "centred-transition fixture must take the symmetric \ - midpoint arm (`abs_diff < min_distance`), got {split}" - ); - } - - /// `level_pre_split` resolves the per-level split knob through the - /// `LevelParams` table, returning the EFFECTIVE upstream `ZSTD_splitBlock` - /// level (`splitLevels[strategy] - 2`, clamped at 0) that - /// `ZSTD_optimalBlockSize` dispatches: fast/dfast/greedy/lazy(d1) → 0 - /// (from-borders), lazy2/btlazy2 → 1 (byChunks rate 43), - /// btopt/btultra/btultra2 → 2 (byChunks rate 11). `Uncompressed` has no - /// numeric level so it stays `None`. - #[test] - fn pre_split_level_dispatches_by_compression_level() { - use crate::encoding::CompressionLevel; - use crate::encoding::match_generator::level_pre_split; - assert_eq!(level_pre_split(CompressionLevel::Uncompressed), None); - // Fastest = level 1 (fast) → 0 (from-borders). - assert_eq!(level_pre_split(CompressionLevel::Fastest), Some(0)); - // Default = level 3 (dfast) → 0 (splitLevels 1 - 2, clamped). - assert_eq!(level_pre_split(CompressionLevel::Default), Some(0)); - // Better is a pure alias for level 7 (lazy): same as Level(7). - assert_eq!( - level_pre_split(CompressionLevel::Better), - level_pre_split(CompressionLevel::Level(7)), - ); - // Best resolves to the level-13 table row (btlazy2): pin it to that - // numeric route so the named path can't drift from the pre-split - // table. - assert_eq!( - level_pre_split(CompressionLevel::Best), - level_pre_split(CompressionLevel::Level(13)), - ); - assert_eq!(level_pre_split(CompressionLevel::Level(2)), Some(0)); // fast - assert_eq!(level_pre_split(CompressionLevel::Level(4)), Some(0)); // dfast - assert_eq!(level_pre_split(CompressionLevel::Level(5)), Some(0)); // greedy - assert_eq!(level_pre_split(CompressionLevel::Level(7)), Some(0)); // lazy (depth 1) - // lazy2 / btlazy2: splitLevels 3 - 2 = 1 (byChunks rate 43, hashLog 8). - // The coarse byte-histogram tier is robust to the periodic-input - // phantom-split the rate-5/hashLog-10 tier suffered, so it matches - // upstream AND stays whole on periodic input (`periodic_stream_not_oversplit`). - assert_eq!(level_pre_split(CompressionLevel::Level(8)), Some(1)); // lazy2 lower bound - assert_eq!(level_pre_split(CompressionLevel::Level(11)), Some(1)); // lazy2 (depth 2) - assert_eq!(level_pre_split(CompressionLevel::Level(12)), Some(1)); // lazy2 upper bound - assert_eq!(level_pre_split(CompressionLevel::Level(13)), Some(1)); // btlazy2 lower bound - assert_eq!(level_pre_split(CompressionLevel::Level(15)), Some(1)); // btlazy2 (depth 2) - assert_eq!(level_pre_split(CompressionLevel::Level(16)), Some(2)); // btopt - assert_eq!(level_pre_split(CompressionLevel::Level(22)), Some(2)); // btultra2 - } - - /// Regression: a homogeneous but periodic multi-block stream must not be - /// pre-split into tiny blocks at the lazy2 / btlazy2 levels. The rate-5 - /// chunk sampler used to phantom-split such input at every 8 KB chunk, - /// cascading a large stream into hundreds of tiny blocks whose per-block - /// headers ballooned the output (~5x vs the lazy level next door). With - /// the rate-1 full-scan splitter the periodic stream is seen as uniform - /// and stays a few full blocks. We assert the lazy2 (L8) and btlazy2 (L15) - /// outputs stay within 2x of the lazy (L7) output on the same input, and - /// that every output round-trips. - #[test] - fn periodic_stream_not_oversplit() { - use crate::encoding::{CompressionLevel, compress_slice_to_vec}; - const LINES: &[&str] = &[ - "ts=2026-03-26T21:39:28Z level=INFO msg=\"flush memtable\" tenant=demo table=orders region=eu-west\n", - "ts=2026-03-26T21:39:29Z level=INFO msg=\"rotate segment\" tenant=demo table=orders region=eu-west\n", - "ts=2026-03-26T21:39:30Z level=INFO msg=\"compact level\" tenant=demo table=orders region=eu-west\n", - "ts=2026-03-26T21:39:31Z level=INFO msg=\"write block\" tenant=demo table=orders region=eu-west\n", - ]; - // 512 KB = 4 upstream zstd blocks, enough for the cascade to manifest. - let target = 512 * 1024usize; - let mut data = Vec::with_capacity(target); - let mut i = 0; - while data.len() < target { - let line = LINES[i % LINES.len()].as_bytes(); - let take = line.len().min(target - data.len()); - data.extend_from_slice(&line[..take]); - i += 1; - } - let l7 = compress_slice_to_vec(&data, CompressionLevel::Level(7)); // lazy depth1 - let l8 = compress_slice_to_vec(&data, CompressionLevel::Level(8)); // lazy2 - let l15 = compress_slice_to_vec(&data, CompressionLevel::Level(15)); // btlazy2 - assert!( - l8.len() < l7.len() * 2, - "lazy2 over-split periodic stream: l7={} l8={}", - l7.len(), - l8.len() - ); - assert!( - l15.len() < l7.len() * 2, - "btlazy2 over-split periodic stream: l7={} l15={}", - l7.len(), - l15.len() - ); - for out in [&l7, &l8, &l15] { - let mut decoder = FrameDecoder::new(); - let mut round = Vec::with_capacity(data.len()); - decoder - .decode_all_to_vec(out, &mut round) - .expect("decode periodic stream"); - assert_eq!(round, data, "periodic stream roundtrip mismatch"); - } - } - - /// End-to-end: a 256 KB payload whose SECOND 128 KB upstream zstd block carries - /// an intra-block fingerprint transition, compressed at Level(5) - /// (greedy, the pre-split path this revision routes through the cheap - /// chunk splitter), round-trips through the crate's own decoder. - /// - /// The transition lives in the second block on purpose: the upstream zstd - /// `savings < 3` gate skips splitting the first block (savings start at - /// 0), so the first block is a homogeneous compressible run that banks - /// savings, and the second block is the one whose intra-block transition - /// `split_block_by_chunks()` resolves into a sub-block boundary (the - /// `pending_input.split_off(...)` path). The test asserts that split - /// decision directly so it cannot silently stop exercising the path if - /// the fixture or params drift, then proves the emitted split frame - /// round-trips. Level 13 (lazy) no longer pre-splits, hence Level 5. - #[test] - fn greedy_chunk_split_roundtrips_through_own_decoder() { - use crate::encoding::CompressionLevel; - let mut data = vec![0u8; 256 * 1024]; - // First 128 KB: homogeneous low-entropy run (compressible, banks - // the savings the upstream zstd gate needs). Second 128 KB: low-entropy run - // for its first half, then a counter sequence: a clear intra-block - // fingerprint transition at the 192 KB midpoint for the chunk - // splitter to find. - for (i, byte) in data.iter_mut().enumerate() { - *byte = if i < 192 * 1024 { - (i & 0x07) as u8 - } else { - (i % 251 + 1) as u8 - }; - } - - // Directly assert the chunk splitter resolves the second block's - // intra-block transition into a sub-block boundary once savings have - // accrued (the compressible first block banks well over the gate). - let second_block = &data[128 * 1024..]; - let split = super::optimal_block_size( - CompressionLevel::Level(5), - second_block, - second_block.len(), - MAX_BLOCK_SIZE as usize, - 100, - ); - assert!( - split < MAX_BLOCK_SIZE as usize, - "second upstream zstd block must chunk-split at its intra-block transition, got {split}", - ); - - let mut compressed = Vec::new(); - let mut compressor = FrameCompressor::new(CompressionLevel::Level(5)); - compressor.set_source(data.as_slice()); - compressor.set_drain(&mut compressed); - compressor.compress(); - - let mut decoder = FrameDecoder::new(); - let mut source = compressed.as_slice(); - decoder - .reset(&mut source) - .expect("frame header should parse"); - while !decoder.is_finished() { - decoder - .decode_blocks(&mut source, crate::decoding::BlockDecodingStrategy::All) - .expect("decode should succeed"); - } - let mut decoded = Vec::with_capacity(data.len()); - decoder.collect_to_writer(&mut decoded).unwrap(); - assert_eq!(decoded, data, "roundtrip must reproduce the input verbatim"); - } - - /// Outside-diff coverage for the FAST one-shot path. - /// `compress_slice_to_vec` / `compress_independent_frame` on a Fast level - /// routes through `run_borrowed_block_loop` (not the owned loop the test - /// above covers), which must honour `optimal_block_size` and emit a - /// sub-`MAX_BLOCK_SIZE` boundary rather than fixed 128 KiB blocks. A - /// 256 KiB input is two 128 KiB blocks when unsplit; a chunk boundary in - /// the second block yields >= 3 decoded blocks, asserted on the round-trip. - #[test] - fn fast_oneshot_borrowed_split_emits_subblock() { - use crate::encoding::CompressionLevel; - // First 192 KiB: homogeneous zero run (banks the savings the split - // gate needs). The second 128 KiB block flips to a counter sequence - // at its 64 KiB midpoint (the 192 KiB mark) — a fingerprint - // transition the Fast from-borders splitter (split level 0) resolves - // into a sub-block boundary. - let mut data = vec![0u8; 256 * 1024]; - for (i, byte) in data.iter_mut().enumerate() { - if i >= 192 * 1024 { - *byte = (i % 251 + 1) as u8; - } - } - - // Pin the splitter decision for the Fast path directly (mirrors the - // greedy test): the second upstream zstd block must resolve to a sub-block - // boundary, so the >= 3 block count below cannot pass vacuously. - let second_block = &data[128 * 1024..]; - assert!( - super::optimal_block_size( - CompressionLevel::Fastest, - second_block, - second_block.len(), - MAX_BLOCK_SIZE as usize, - 100, - ) < MAX_BLOCK_SIZE as usize, - "fixture must resolve to a sub-block split in the second upstream zstd block", - ); - - // Drive the borrowed one-shot route explicitly (Fast level -> - // run_borrowed_block_loop via compress_independent_frame). - let mut compressor: FrameCompressor = FrameCompressor::new(CompressionLevel::Fastest); - let frame = compressor.compress_independent_frame(&data); - - let mut decoder = FrameDecoder::new(); - let mut source = frame.as_slice(); - decoder - .reset(&mut source) - .expect("frame header should parse"); - while !decoder.is_finished() { - decoder - .decode_blocks(&mut source, crate::decoding::BlockDecodingStrategy::All) - .expect("decode should succeed"); - } - let mut decoded = Vec::with_capacity(data.len()); - decoder.collect_to_writer(&mut decoded).unwrap(); - assert_eq!(decoded, data, "roundtrip must reproduce the input verbatim"); - assert!( - decoder.blocks_decoded() >= 3, - "fast one-shot borrowed path must split the second upstream zstd block \ - (256 KiB unsplit = 2 blocks), got {} blocks", - decoder.blocks_decoded(), - ); - } - - /// Regression: `set_compression_level` followed by `compress()` must - /// refresh `state.strategy_tag` through the reset-time sync so the - /// literal-compression gates (`min_literals_to_compress`, - /// `min_gain`) use the NEW level's strategy. Picks a level pair - /// that genuinely crosses strategy bands — `Fastest` resolves to - /// `Fast`, `Level(20)` resolves to `BtUltra2` — so a missed sync - /// would leave the construction-time tag visible and trip the - /// assertion. `CompressionLevel::Best` would also pass type-wise - /// but resolves to `Lazy` today, which keeps `min_literals_to_compress` - /// in the same `shift=3 → 64-byte` band as `Fast` and weakens the - /// signal that the gate floor actually moved. - #[cfg(feature = "std")] - #[test] - fn set_compression_level_then_compress_refreshes_strategy_tag() { - use super::CompressionLevel; - use crate::encoding::strategy::StrategyTag; - - let data = vec![0xABu8; 256]; - let mut out = Vec::new(); - let mut compressor = FrameCompressor::new(CompressionLevel::Fastest); - let initial_tag = compressor.state.strategy_tag; - assert_eq!( - initial_tag, - StrategyTag::for_compression_level(CompressionLevel::Fastest), - "construction-time strategy_tag must reflect initial level", - ); - - // Switch to a level whose resolved strategy lives in a different - // band, then run a full compress cycle — the matcher.reset() - // inside `compress` is the only site that can refresh the tag. - let new_level = CompressionLevel::Level(20); - compressor.set_compression_level(new_level); - compressor.set_source(data.as_slice()); - compressor.set_drain(&mut out); - compressor.compress(); - - let new_tag = compressor.state.strategy_tag; - let expected = StrategyTag::for_compression_level(new_level); - assert_eq!( - new_tag, expected, - "strategy_tag must follow set_compression_level → compress, \ - got {new_tag:?} expected {expected:?}", - ); - assert_eq!( - expected, - StrategyTag::BtUltra2, - "test fixture invariant: Level(20) must resolve to BtUltra2 \ - so the post-switch tag visibly crosses the band boundary", - ); - assert_ne!( - new_tag, initial_tag, - "test fixture invariant: chosen levels must resolve to \ - different StrategyTag variants", - ); - } - - /// Magicless mode (`ZSTD_f_zstd1_magicless`): encoded frame - /// MUST NOT start with the 4-byte magic prefix, AND must - /// round-trip through a magicless-aware decoder. - #[test] - fn magicless_frame_omits_magic_and_roundtrips() { - use crate::common::MAGIC_NUM; - let input: alloc::vec::Vec = (0..512u32).map(|i| (i ^ 0xA5) as u8).collect(); - - // Encode with magicless = true. - let mut output: Vec = Vec::new(); - let mut compressor = FrameCompressor::new(super::CompressionLevel::Default); - compressor.set_magicless(true); - compressor.set_source(input.as_slice()); - compressor.set_drain(&mut output); - compressor.compress(); - - // 1. Encoded output must NOT begin with the zstd magic number. - assert!( - !output.starts_with(&MAGIC_NUM.to_le_bytes()), - "magicless frame must omit the 4-byte magic prefix", - ); - - // 2. A magicless-aware decoder must round-trip the payload. - let mut decoder = crate::decoding::FrameDecoder::new(); - decoder.set_magicless(true); - let mut cursor: &[u8] = output.as_slice(); - decoder.init(&mut cursor).expect("magicless init"); - decoder - .decode_blocks(&mut cursor, crate::decoding::BlockDecodingStrategy::All) - .expect("decode_blocks"); - let mut decoded: Vec = Vec::new(); - decoder - .collect_to_writer(&mut decoded) - .expect("collect_to_writer"); - assert_eq!(decoded, input, "magicless roundtrip must preserve bytes"); - - // 3. A standard (magicful) decoder MUST reject a magicless - // frame at the header-read step — the first 4 bytes are - // the frame-header descriptor + window / dictionary / FCS - // metadata, not the magic. We accept either - // `BadMagicNumber` (typical case: first 4 bytes don't - // match `MAGIC_NUM` and don't fall in the skippable-frame - // magic range) or `SkipFrame` (rare: the first 4 bytes - // coincidentally land in `0x184D2A50..=0x184D2A5F`). Both - // prove the standard decoder did not treat the bytes as a - // real magicful frame. - use crate::decoding::errors::{FrameDecoderError, ReadFrameHeaderError}; - let mut std_decoder = crate::decoding::FrameDecoder::new(); - let std_init = std_decoder.init(output.as_slice()); - match std_init { - Err(FrameDecoderError::ReadFrameHeaderError( - ReadFrameHeaderError::BadMagicNumber(_) | ReadFrameHeaderError::SkipFrame { .. }, - )) => {} - other => panic!( - "standard decoder must reject a magicless frame with \ - ReadFrameHeaderError::BadMagicNumber or SkipFrame, got {other:?}", - ), - } - } - - /// A reused `FrameCompressor` must emit byte-identical frames to a - /// fresh compressor per input across both the borrowed (Fast) and - /// owned (Dfast/Lazy/Greedy/Uncompressed) backends. This proves - /// `prepare_frame` fully resets the per-frame state (matcher window, - /// content hasher, FSE/Huffman seeds) between independent frames; a - /// missed reset would corrupt frame N>=2's header checksum or matches. - /// Each emitted frame must also round-trip. - #[test] - fn compress_independent_frame_reuse_matches_fresh_and_roundtrips() { - use crate::encoding::{CompressionLevel, compress_slice_to_vec}; - let levels = [ - CompressionLevel::Uncompressed, - CompressionLevel::Fastest, - CompressionLevel::Default, - CompressionLevel::Better, - CompressionLevel::Best, - CompressionLevel::Level(5), - ]; - let inputs: Vec> = vec![ - Vec::new(), - vec![0x00], - b"the quick brown fox jumps over the lazy dog\n".to_vec(), - vec![0x7Eu8; 50_000], // highly compressible - generate_data(0xABCD, 70_000), // pseudo-random - generate_data(0x1234, 200_000), - ]; - for level in levels { - let mut cctx: FrameCompressor = FrameCompressor::new(level); - for data in &inputs { - let reused = cctx.compress_independent_frame(data); - let fresh = compress_slice_to_vec(data, level); - assert_eq!( - reused, - fresh, - "reused frame != fresh frame for len={} level={:?}", - data.len(), - level, - ); - let mut decoder = FrameDecoder::new(); - let mut decoded = Vec::with_capacity(data.len()); - decoder.decode_all_to_vec(&reused, &mut decoded).unwrap(); - assert_eq!( - decoded, - *data, - "roundtrip failed for len={} level={:?}", - data.len(), - level, - ); - } - } - } - - /// `compress_independent_frame_into` must replace (not append to) the - /// caller's buffer each call, so a smaller frame after a larger one - /// yields exactly the smaller frame, and the reused buffer's content - /// matches a fresh compression of the same input. - #[test] - fn compress_independent_frame_into_replaces_buffer_contents() { - use crate::encoding::{CompressionLevel, compress_slice_to_vec}; - let large = vec![0x11u8; 40_000]; - let small = b"short payload".to_vec(); - let mut cctx: FrameCompressor = FrameCompressor::new(CompressionLevel::Default); - let mut out = Vec::new(); - cctx.compress_independent_frame_into(&large, &mut out); - let frame_large = out.clone(); - // Reusing the same buffer for a smaller frame must clear it first. - cctx.compress_independent_frame_into(&small, &mut out); - assert_eq!( - out, - compress_slice_to_vec(&small, CompressionLevel::Default), - "reused buffer must hold exactly the second frame", - ); - // The first frame, captured before reuse, still round-trips. - let mut decoder = FrameDecoder::new(); - let mut decoded = Vec::with_capacity(large.len()); - decoder - .decode_all_to_vec(&frame_large, &mut decoded) - .unwrap(); - assert_eq!(decoded, large); - } - - /// A sticky dictionary set once on a reused compressor must be primed - /// into every independent frame (mirroring `ZSTD_CCtx_loadDictionary`): - /// each frame decodes with the dictionary and is byte-identical to a - /// fresh compressor carrying the same dictionary. This proves - /// `prepare_frame` re-primes the dictionary (matcher content + offset - /// history + entropy seed) every call rather than only on the first. - #[test] - fn compress_independent_frame_reuses_sticky_dictionary() { - use crate::encoding::CompressionLevel; - let dict_raw = include_bytes!("../../dict_tests/dictionary"); - let dict_content = crate::decoding::Dictionary::decode_dict(dict_raw).unwrap(); - let mut payload_a = Vec::new(); - for _ in 0..8 { - payload_a.extend_from_slice(&dict_content.dict_content[..2048]); - } - let payload_b = b"a different second frame payload, still dict-attached".to_vec(); - let inputs = [payload_a, payload_b]; - - let mut cctx: FrameCompressor = FrameCompressor::new(CompressionLevel::Fastest); - cctx.set_dictionary_from_bytes(dict_raw) - .expect("dictionary bytes should parse"); - - for data in &inputs { - let reused = cctx.compress_independent_frame(data); - // Fresh compressor carrying the same sticky dictionary. - let mut fresh_enc: FrameCompressor = FrameCompressor::new(CompressionLevel::Fastest); - fresh_enc - .set_dictionary_from_bytes(dict_raw) - .expect("dictionary bytes should parse"); - let fresh = fresh_enc.compress_independent_frame(data); - assert_eq!( - reused, - fresh, - "reused dict frame != fresh dict frame, len={}", - data.len(), - ); - // Round-trip with the dictionary on the decode side. - let dict_for_decoder = crate::decoding::Dictionary::decode_dict(dict_raw).unwrap(); - let mut decoder = FrameDecoder::new(); - decoder.add_dict(dict_for_decoder).unwrap(); - let mut decoded = Vec::with_capacity(data.len()); - decoder.decode_all_to_vec(&reused, &mut decoded).unwrap(); - assert_eq!(&decoded, data, "dict roundtrip failed, len={}", data.len()); - } - } -} +mod tests; diff --git a/zstd/src/encoding/frame_compressor/tests.rs b/zstd/src/encoding/frame_compressor/tests.rs new file mode 100644 index 000000000..69a0f5f69 --- /dev/null +++ b/zstd/src/encoding/frame_compressor/tests.rs @@ -0,0 +1,2516 @@ +// `format!` is used by ungated tests (e.g. the btlazy2 dict-reuse +// byte-identity test), so the import must not be feature-gated — under +// default features (no `dict_builder`) the gated form left `format!` +// unresolved when the test module is compiled. +use alloc::format; +use alloc::vec; + +use super::FrameCompressor; +use crate::common::{MAGIC_NUM, MAX_BLOCK_SIZE}; +use crate::decoding::FrameDecoder; +use crate::encoding::{Matcher, Sequence}; +use alloc::vec::Vec; + +fn generate_data(seed: u64, len: usize) -> Vec { + let mut state = seed; + let mut data = Vec::with_capacity(len); + for _ in 0..len { + state = state + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + data.push((state >> 33) as u8); + } + data +} + +// Cross-implementation parity tests (compress here, decode through the C +// bindings) moved to `ffi-bench/tests/frame_compressor_ffi.rs` so the +// library crate never links libzstd. + +struct NoDictionaryMatcher { + last_space: Vec, + window_size: u64, +} + +impl NoDictionaryMatcher { + fn new(window_size: u64) -> Self { + Self { + last_space: Vec::new(), + window_size, + } + } +} + +impl Matcher for NoDictionaryMatcher { + fn get_next_space(&mut self) -> Vec { + vec![0; self.window_size as usize] + } + + fn get_last_space(&mut self) -> &[u8] { + self.last_space.as_slice() + } + + fn commit_space(&mut self, space: Vec) { + self.last_space = space; + } + + fn skip_matching(&mut self) {} + + fn start_matching(&mut self, mut handle_sequence: impl for<'a> FnMut(Sequence<'a>)) { + handle_sequence(Sequence::Literals { + literals: self.last_space.as_slice(), + }); + } + + fn reset(&mut self, _level: super::CompressionLevel) { + self.last_space.clear(); + } + + fn window_size(&self) -> u64 { + self.window_size + } +} + +#[test] +fn frame_starts_with_magic_num() { + let mock_data = [1_u8, 2, 3].as_slice(); + let mut output: Vec = Vec::new(); + let mut compressor = FrameCompressor::new(super::CompressionLevel::Uncompressed); + compressor.set_source(mock_data); + compressor.set_drain(&mut output); + + compressor.compress(); + assert!(output.starts_with(&MAGIC_NUM.to_le_bytes())); +} + +#[test] +fn very_simple_raw_compress() { + let mock_data = [1_u8, 2, 3].as_slice(); + let mut output: Vec = Vec::new(); + let mut compressor = FrameCompressor::new(super::CompressionLevel::Uncompressed); + compressor.set_source(mock_data); + compressor.set_drain(&mut output); + + compressor.compress(); +} + +#[test] +fn very_simple_compress() { + let mut mock_data = vec![0; 1 << 17]; + mock_data.extend(vec![1; (1 << 17) - 1]); + mock_data.extend(vec![2; (1 << 18) - 1]); + mock_data.extend(vec![2; 1 << 17]); + mock_data.extend(vec![3; (1 << 17) - 1]); + let mut output: Vec = Vec::new(); + let mut compressor = FrameCompressor::new(super::CompressionLevel::Uncompressed); + compressor.set_source(mock_data.as_slice()); + compressor.set_drain(&mut output); + + compressor.compress(); + + let mut decoder = FrameDecoder::new(); + let mut decoded = Vec::with_capacity(mock_data.len()); + decoder.decode_all_to_vec(&output, &mut decoded).unwrap(); + assert_eq!(mock_data, decoded); +} + +#[test] +fn rle_compress() { + let mock_data = vec![0; 1 << 19]; + let mut output: Vec = Vec::new(); + let mut compressor = FrameCompressor::new(super::CompressionLevel::Uncompressed); + compressor.set_source(mock_data.as_slice()); + compressor.set_drain(&mut output); + + compressor.compress(); + + let mut decoder = FrameDecoder::new(); + let mut decoded = Vec::with_capacity(mock_data.len()); + decoder.decode_all_to_vec(&output, &mut decoded).unwrap(); + assert_eq!(mock_data, decoded); +} + +#[test] +fn aaa_compress() { + let mock_data = vec![0, 1, 3, 4, 5]; + let mut output: Vec = Vec::new(); + let mut compressor = FrameCompressor::new(super::CompressionLevel::Uncompressed); + compressor.set_source(mock_data.as_slice()); + compressor.set_drain(&mut output); + + compressor.compress(); + + let mut decoder = FrameDecoder::new(); + let mut decoded = Vec::with_capacity(mock_data.len()); + decoder.decode_all_to_vec(&output, &mut decoded).unwrap(); + assert_eq!(mock_data, decoded); +} + +#[test] +fn dictionary_compression_sets_required_dict_id_and_roundtrips() { + let dict_raw = include_bytes!("../../../dict_tests/dictionary"); + let dict_for_encoder = crate::decoding::Dictionary::decode_dict(dict_raw).unwrap(); + let dict_for_decoder = crate::decoding::Dictionary::decode_dict(dict_raw).unwrap(); + + let mut data = Vec::new(); + for _ in 0..8 { + data.extend_from_slice(&dict_for_decoder.dict_content[..2048]); + } + + let mut with_dict = Vec::new(); + let mut compressor = FrameCompressor::new(super::CompressionLevel::Fastest); + let previous = compressor + .set_dictionary_from_bytes(dict_raw) + .expect("dictionary bytes should parse"); + assert!( + previous.is_none(), + "first dictionary insert should return None" + ); + assert_eq!( + compressor + .set_dictionary(dict_for_encoder) + .expect("valid dictionary should attach") + .expect("set_dictionary_from_bytes inserted previous dictionary") + .id(), + dict_for_decoder.id + ); + compressor.set_source(data.as_slice()); + compressor.set_drain(&mut with_dict); + compressor.compress(); + + let (frame_header, _) = crate::decoding::frame::read_frame_header(with_dict.as_slice()) + .expect("encoded stream should have a frame header"); + assert_eq!(frame_header.dictionary_id(), Some(dict_for_decoder.id)); + + let mut decoder = FrameDecoder::new(); + let mut missing_dict_target = Vec::with_capacity(data.len()); + let err = decoder + .decode_all_to_vec(&with_dict, &mut missing_dict_target) + .unwrap_err(); + assert!( + matches!( + &err, + crate::decoding::errors::FrameDecoderError::DictNotProvided { .. } + ), + "dict-compressed stream should require dictionary id, got: {err:?}" + ); + + let mut decoder = FrameDecoder::new(); + decoder.add_dict(dict_for_decoder).unwrap(); + let mut decoded = Vec::with_capacity(data.len()); + decoder.decode_all_to_vec(&with_dict, &mut decoded).unwrap(); + assert_eq!(decoded, data); +} + +#[cfg(all(feature = "dict_builder", feature = "std"))] +#[test] +fn dictionary_compression_roundtrips_with_dict_builder_dictionary() { + use std::io::Cursor; + + let mut training = Vec::new(); + for idx in 0..256u32 { + training.extend_from_slice( + format!("tenant=demo table=orders key={idx} region=eu\n").as_bytes(), + ); + } + let mut raw_dict = Vec::new(); + crate::dictionary::create_raw_dict_from_source( + Cursor::new(training.as_slice()), + training.len(), + &mut raw_dict, + 4096, + ) + .expect("dict_builder training should succeed"); + assert!( + !raw_dict.is_empty(), + "dict_builder produced an empty dictionary" + ); + + let dict_id = 0xD1C7_0008; + let encoder_dict = + crate::decoding::Dictionary::from_raw_content(dict_id, raw_dict.clone()).unwrap(); + let decoder_dict = + crate::decoding::Dictionary::from_raw_content(dict_id, raw_dict.clone()).unwrap(); + + let mut payload = Vec::new(); + for idx in 0..96u32 { + payload.extend_from_slice( + format!("tenant=demo table=orders op=put key={idx} value=aaaaabbbbbcccccdddddeeeee\n") + .as_bytes(), + ); + } + + let mut without_dict = Vec::new(); + let mut baseline = FrameCompressor::new(super::CompressionLevel::Fastest); + baseline.set_source(payload.as_slice()); + baseline.set_drain(&mut without_dict); + baseline.compress(); + + let mut with_dict = Vec::new(); + let mut compressor = FrameCompressor::new(super::CompressionLevel::Fastest); + compressor + .set_dictionary(encoder_dict) + .expect("valid dict_builder dictionary should attach"); + compressor.set_source(payload.as_slice()); + compressor.set_drain(&mut with_dict); + compressor.compress(); + + let (frame_header, _) = crate::decoding::frame::read_frame_header(with_dict.as_slice()) + .expect("encoded stream should have a frame header"); + assert_eq!(frame_header.dictionary_id(), Some(dict_id)); + let mut decoder = FrameDecoder::new(); + decoder.add_dict(decoder_dict).unwrap(); + let mut decoded = Vec::with_capacity(payload.len()); + decoder.decode_all_to_vec(&with_dict, &mut decoded).unwrap(); + assert_eq!(decoded, payload); + assert!( + with_dict.len() < without_dict.len(), + "trained dictionary should improve compression for this small payload" + ); +} + +#[test] +fn set_dictionary_from_bytes_seeds_entropy_tables_for_first_block() { + let dict_raw = include_bytes!("../../../dict_tests/dictionary"); + let mut output = Vec::new(); + let input = b""; + + let mut compressor = FrameCompressor::new(super::CompressionLevel::Fastest); + let previous = compressor + .set_dictionary_from_bytes(dict_raw) + .expect("dictionary bytes should parse"); + assert!(previous.is_none()); + + compressor.set_source(input.as_slice()); + compressor.set_drain(&mut output); + compressor.compress(); + + assert!( + compressor.state.last_huff_table.is_some(), + "dictionary entropy should seed previous huffman table before first block" + ); + assert!( + compressor.state.fse_tables.ll_previous.is_some(), + "dictionary entropy should seed previous ll table before first block" + ); + assert!( + compressor.state.fse_tables.ml_previous.is_some(), + "dictionary entropy should seed previous ml table before first block" + ); + assert!( + compressor.state.fse_tables.of_previous.is_some(), + "dictionary entropy should seed previous of table before first block" + ); +} + +// `set_content_size_flag(false)`: the header must omit the FCS field +// (and the single-segment layout that requires it) while the frame +// still round-trips through our decoder. +#[test] +fn content_size_flag_off_omits_fcs_and_roundtrips() { + let payload = alloc::vec![0x42u8; 4096]; + + let mut compressor: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Fastest); + let mut with_fcs = Vec::new(); + compressor.compress_independent_frame_into(&payload, &mut with_fcs); + + compressor.set_content_size_flag(false); + let mut without_fcs = Vec::new(); + compressor.compress_independent_frame_into(&payload, &mut without_fcs); + + let parsed_with = crate::decoding::frame::read_frame_header(with_fcs.as_slice()) + .expect("flag-on frame header must parse") + .0; + assert_eq!(parsed_with.frame_content_size(), 4096); + + let parsed_without = crate::decoding::frame::read_frame_header(without_fcs.as_slice()) + .expect("flag-off frame header must parse") + .0; + // 0 is the decoder's "unknown content size" sentinel... + assert_eq!( + parsed_without.frame_content_size(), + 0, + "FCS must be omitted with the content-size flag off" + ); + // ...and the descriptor must confirm the field is ABSENT (0 bytes), + // not present with an explicit zero value. + assert_eq!( + parsed_without + .descriptor + .frame_content_size_bytes() + .expect("descriptor must parse"), + 0, + "the FCS field itself must be omitted, not written as zero" + ); + + let mut decoder = crate::decoding::FrameDecoder::new(); + // `decode_all_to_vec` fills existing capacity (no FCS to pre-size + // from with the flag off), so reserve the expected payload upfront. + let mut decoded = Vec::with_capacity(payload.len() + 64); + decoder + .decode_all_to_vec(&without_fcs, &mut decoded) + .expect("flag-off frame must decode"); + assert_eq!(decoded, payload); +} + +// `set_dictionary_id_flag(false)`: a dict-compressed frame must omit +// the dictionary ID and still decode when the dictionary is handed to +// the decoder explicitly. +#[test] +fn dict_id_flag_off_omits_dictionary_id_and_roundtrips() { + let dict_raw = include_bytes!("../../../dict_tests/dictionary"); + let payload = b"dictionary-keyed payload dictionary-keyed payload".repeat(8); + + let mut compressor: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Fastest); + compressor + .set_dictionary_from_bytes(dict_raw) + .expect("dictionary bytes should parse"); + compressor.set_dictionary_id_flag(false); + let mut frame = Vec::new(); + compressor.compress_independent_frame_into(&payload, &mut frame); + + let parsed = crate::decoding::frame::read_frame_header(frame.as_slice()) + .expect("frame header must parse") + .0; + assert_eq!( + parsed.dictionary_id(), + None, + "dictionary id must be omitted with the dict-id flag off" + ); + + // With the ID omitted the decoder cannot look the dictionary up by + // header; hand it explicitly (the `reset_with_dict_handle` path). + let mut sd = + crate::decoding::StreamingDecoder::new_with_dictionary_bytes(frame.as_slice(), dict_raw) + .expect("decoder must accept the dictionary"); + let mut dec = Vec::new(); + std::io::Read::read_to_end(&mut sd, &mut dec) + .expect("frame must decode with the dictionary handed explicitly"); + assert_eq!(dec, payload); +} + +// The output reservation must track the observed compression ratio, not +// the whole-input `compress_bound`: a multi-MiB compressible stream's +// output buffer stays at output scale (the old up-front bound held an +// input-sized allocation for the whole frame). Incompressible input may +// still re-estimate to ~the full bound — that is the genuine worst case. +#[test] +fn compressible_stream_output_capacity_stays_at_output_scale() { + // 4 MiB of highly repetitive log-like lines. + let line = b"ts=2026-03-26T21:39:28Z level=INFO msg=\"flush memtable\" tenant=demo\n"; + let mut input = Vec::with_capacity(4 << 20); + while input.len() < (4 << 20) { + let take = line.len().min((4 << 20) - input.len()); + input.extend_from_slice(&line[..take]); + } + + let mut compressor: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Fastest); + let mut out = Vec::new(); + compressor.compress_independent_frame_into(&input, &mut out); + + assert!(!out.is_empty()); + assert!( + out.capacity() < input.len() / 4, + "capacity {} must stay at output scale (input {}, output {})", + out.capacity(), + input.len(), + out.len() + ); + + // Round-trip: the adaptive reservation must not affect the bytes. + let mut decoder = crate::decoding::FrameDecoder::new(); + let mut decoded = Vec::with_capacity(input.len() + 64); + decoder + .decode_all_to_vec(&out, &mut decoded) + .expect("frame must decode"); + assert_eq!(decoded, input); +} + +// A dictionary frame with a known content size that fits the window +// must take the single-segment layout (reference parity): the +// dictionary is decoder setup state, not part of the regenerated +// segment, so it must not force the windowed multi-segment layout. +#[test] +fn dict_frame_with_known_size_is_single_segment() { + let dict_raw = include_bytes!("../../../dict_tests/dictionary"); + let payload = b"dictionary-keyed payload dictionary-keyed payload".repeat(64); + + let mut compressor: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Fastest); + compressor + .set_dictionary_from_bytes(dict_raw) + .expect("dictionary bytes should parse"); + let mut frame = Vec::new(); + compressor.compress_independent_frame_into(&payload, &mut frame); + + let parsed = crate::decoding::frame::read_frame_header(frame.as_slice()) + .expect("frame header must parse") + .0; + assert!( + parsed.descriptor.single_segment_flag(), + "dict frame with known size <= window must be single-segment" + ); + assert!(parsed.dictionary_id().is_some()); + assert_eq!(parsed.frame_content_size(), payload.len() as u64); + + // Round-trip through our own decoder with the dictionary. + let mut decoder = crate::decoding::FrameDecoder::new(); + decoder + .add_dict_from_bytes(dict_raw) + .expect("decoder must accept the dictionary"); + let mut decoded = Vec::with_capacity(payload.len() + 64); + decoder + .decode_all_to_vec(&frame, &mut decoded) + .expect("single-segment dict frame must decode"); + assert_eq!(decoded, payload); +} + +// Regression: after `clear_dictionary()` a reused compressor must fully +// deactivate the dictionary state. The dict-active matcher reset rewinds the +// hash/chain tables to the origin (`position_base = 0`) and DEFERS the table +// clear to the next prime/restore. If `clear_dictionary` leaves the matcher +// marked dictionary-active, a subsequent NO-dictionary frame hits that +// deferred-clear branch but never runs prime/restore, so stale dict-region +// entries (old absolute positions) survive at the rewound base and can +// surface as bogus matches. The no-dict frame must still round-trip without +// the dictionary. Uses Level(16) (btopt → HashChain backend, whose storage +// owns the deferred-clear path). +#[test] +fn clear_dictionary_then_nodict_frame_roundtrips() { + let dict_raw = include_bytes!("../../../dict_tests/dictionary"); + // Payload B embeds dictionary bytes up front so a surviving dict-region + // chain entry would be a tempting (wrong) match candidate. + let mut payload_b = Vec::new(); + payload_b.extend_from_slice(&dict_raw[..dict_raw.len().min(2048)]); + payload_b.extend_from_slice( + b"no-dictionary tail no-dictionary tail" + .repeat(16) + .as_slice(), + ); + + let mut compressor: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Level(16)); + compressor + .set_dictionary_from_bytes(dict_raw) + .expect("dictionary bytes should parse"); + // Frame A primes the dictionary (sets the matcher dictionary-active). + let mut frame_a = Vec::new(); + compressor.compress_independent_frame_into( + b"dictionary-keyed payload dictionary-keyed payload" + .repeat(8) + .as_slice(), + &mut frame_a, + ); + // Remove the dictionary, then compress a no-dictionary frame on the + // SAME reused compressor. + compressor.clear_dictionary(); + let mut frame_b = Vec::new(); + compressor.compress_independent_frame_into(&payload_b, &mut frame_b); + + // Frame B must decode WITHOUT any dictionary. + let mut decoder = crate::decoding::FrameDecoder::new(); + let mut decoded = Vec::with_capacity(payload_b.len() + 64); + decoder + .decode_all_to_vec(&frame_b, &mut decoded) + .expect("no-dict frame after clear_dictionary must decode"); + assert_eq!( + decoded, payload_b, + "no-dict frame after clear_dictionary must round-trip exactly" + ); +} + +// Regression test: `heap_size()` must count the retained Huffman tables +// (the active `last_huff_table` and the recycled `huff_table_spare`). +// A reused context that parks a table would otherwise under-report its +// footprint through the public size API. +#[test] +fn heap_size_counts_active_and_spare_huffman_tables() { + let mut compressor: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Fastest); + let base = compressor.heap_size(); + + let active = crate::huff0::huff0_encoder::HuffmanTable::build_from_data( + b"abacabadabacabaeabacabadabacaba", + ); + let active_bytes = active.heap_size(); + assert!(active_bytes > 0, "built table must own heap buffers"); + compressor.state.last_huff_table = Some(active); + assert_eq!( + compressor.heap_size(), + base + active_bytes, + "heap_size must include the active last_huff_table" + ); + + let spare = crate::huff0::huff0_encoder::HuffmanTable::build_from_data( + b"the quick brown fox jumps over the lazy dog", + ); + let spare_bytes = spare.heap_size(); + assert!(spare_bytes > 0, "built table must own heap buffers"); + compressor.state.huff_table_spare = Some(spare); + assert_eq!( + compressor.heap_size(), + base + active_bytes + spare_bytes, + "heap_size must include the parked huff_table_spare" + ); +} + +#[test] +fn set_encoder_dictionary_reattaches_prepared_dict_without_reparse() { + let dict_raw = include_bytes!("../../../dict_tests/dictionary"); + let payload = b"tenant=demo table=orders op=put key=1 value=aaaaabbbbbcccccdddddeeeee\n\ + tenant=demo table=orders op=put key=2 value=aaaaabbbbbcccccdddddeeeee\n"; + + // Prepare the EncoderDictionary once, then attach it via the prepared- + // dictionary API (no raw-blob reparse at attach time). + let prepared = super::EncoderDictionary::from_bytes(dict_raw).expect("dict bytes should parse"); + let dict_id = prepared.id(); + + let mut with_dict = Vec::new(); + let mut compressor = FrameCompressor::new(super::CompressionLevel::Fastest); + let previous = compressor + .set_encoder_dictionary(prepared) + .expect("prepared dictionary should attach"); + assert!(previous.is_none()); + compressor.set_source(payload.as_slice()); + compressor.set_drain(&mut with_dict); + compressor.compress(); + // clear_dictionary hands the prepared dictionary back (last use of + // `compressor`, so its `&mut with_dict` drain borrow ends here). + let returned = compressor + .clear_dictionary() + .expect("dictionary was attached"); + assert_eq!(returned.id(), dict_id); + + // The reattached dictionary drives the frame: its id is advertised and + // the stream round-trips through a decoder primed with the same dict. + let (frame_header, _) = crate::decoding::frame::read_frame_header(with_dict.as_slice()) + .expect("encoded stream should have a frame header"); + assert_eq!(frame_header.dictionary_id(), Some(dict_id)); + let decoder_dict = crate::decoding::Dictionary::decode_dict(dict_raw).unwrap(); + let mut decoder = FrameDecoder::new(); + decoder.add_dict(decoder_dict).unwrap(); + let mut decoded = Vec::with_capacity(payload.len()); + decoder.decode_all_to_vec(&with_dict, &mut decoded).unwrap(); + assert_eq!(decoded.as_slice(), payload.as_slice()); + + // The dictionary handed back by clear_dictionary reattaches to another + // compressor without touching the raw bytes again, producing an + // identical frame. + let mut with_dict2 = Vec::new(); + let mut compressor2 = FrameCompressor::new(super::CompressionLevel::Fastest); + compressor2 + .set_encoder_dictionary(returned) + .expect("returned dictionary should reattach"); + compressor2.set_source(payload.as_slice()); + compressor2.set_drain(&mut with_dict2); + compressor2.compress(); + assert_eq!( + with_dict2, with_dict, + "reattached prepared dict must produce an identical frame" + ); +} + +#[test] +fn dict_primed_matcher_snapshot_reused_across_frames_is_byte_identical() { + // CDict-equivalent: a compressor reused across frames with the same + // dictionary restores the primed matcher snapshot on frames 2..N + // (a table copy) instead of re-hashing the dictionary. The restored + // state must reproduce the first-frame (freshly-primed) output + // byte-for-byte, and every frame must round-trip through a + // dict-primed decoder. + let dict_raw = include_bytes!("../../../dict_tests/dictionary"); + // Source must exceed the Fast strategy's 8 KiB attach cutoff so the + // copy-snapshot (restore) path is taken on frame 2 — at or below the + // cutoff the upstream zstd attaches by reference and we fall back to re-prime, + // which would not exercise restore. + let mut payload = Vec::new(); + while payload.len() < 16 * 1024 { + payload.extend_from_slice( + b"tenant=demo table=orders op=put key=1 value=aaaaabbbbbcccccdddddeeeee\n", + ); + } + + let prepared = super::EncoderDictionary::from_bytes(dict_raw).expect("dict bytes should parse"); + let dict_id = prepared.id(); + let mut compressor: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Fastest); + compressor + .set_encoder_dictionary(prepared) + .expect("prepared dictionary should attach"); + + // Frame 1 primes + captures the snapshot; frame 2 restores it. + let frame1 = compressor.compress_independent_frame(payload.as_slice()); + let frame2 = compressor.compress_independent_frame(payload.as_slice()); + assert_eq!( + frame1, frame2, + "restored prime snapshot must reproduce the freshly-primed frame byte-for-byte" + ); + + // Both frames advertise the dict id and round-trip through a + // dict-primed decoder. + for frame in [&frame1, &frame2] { + let (hdr, _) = + crate::decoding::frame::read_frame_header(frame.as_slice()).expect("frame header"); + assert_eq!(hdr.dictionary_id(), Some(dict_id)); + let mut decoder = FrameDecoder::new(); + decoder + .add_dict(crate::decoding::Dictionary::decode_dict(dict_raw).unwrap()) + .unwrap(); + let mut decoded = Vec::with_capacity(payload.len()); + decoder.decode_all_to_vec(frame, &mut decoded).unwrap(); + assert_eq!(decoded.as_slice(), payload.as_slice()); + } +} + +#[test] +fn dict_primed_matcher_cache_reused_across_small_attach_frames_is_byte_identical() { + // CDict-equivalent ATTACH path (small source, at/below the Fast 8 KiB + // attach cutoff): frames 2..N re-prime — re-committing the dict bytes + // to history — but reuse the already-built dict table instead of + // re-hashing it. The cached-table frame must reproduce the + // freshly-primed first frame byte-for-byte, and a fresh single-frame + // compressor (no prior dict cache) must produce the identical bytes + // too, proving the cache changes timing, not output. + let dict_raw = include_bytes!("../../../dict_tests/dictionary"); + // Stay under the 8 KiB cutoff so the attach (re-prime) path is taken + // every frame rather than the copy-snapshot restore. + let mut payload = Vec::new(); + while payload.len() < 2 * 1024 { + payload.extend_from_slice(b"tenant=demo op=put key=1 value=aaaaabbbbbcccccddddd\n"); + } + + let prepared = super::EncoderDictionary::from_bytes(dict_raw).expect("dict bytes should parse"); + let dict_id = prepared.id(); + let mut compressor: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Fastest); + compressor + .set_encoder_dictionary(prepared) + .expect("prepared dictionary should attach"); + + // Frame 1 builds + marks the dict table; frame 2 reuses it. + let frame1 = compressor.compress_independent_frame(payload.as_slice()); + let frame2 = compressor.compress_independent_frame(payload.as_slice()); + assert_eq!( + frame1, frame2, + "reused dict table (attach path) must reproduce the freshly-built frame byte-for-byte" + ); + + // A fresh compressor (cold dict cache) must emit the same bytes — the + // cache is a timing optimization, never a content change. + let fresh_prepared = + super::EncoderDictionary::from_bytes(dict_raw).expect("dict bytes should parse"); + let mut fresh: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Fastest); + fresh + .set_encoder_dictionary(fresh_prepared) + .expect("prepared dictionary should attach"); + let fresh_frame = fresh.compress_independent_frame(payload.as_slice()); + assert_eq!( + fresh_frame, frame1, + "cold-cache compressor must match the warm-cache frame byte-for-byte" + ); + + for frame in [&frame1, &frame2] { + let (hdr, _) = + crate::decoding::frame::read_frame_header(frame.as_slice()).expect("frame header"); + assert_eq!(hdr.dictionary_id(), Some(dict_id)); + let mut decoder = FrameDecoder::new(); + decoder + .add_dict(crate::decoding::Dictionary::decode_dict(dict_raw).unwrap()) + .unwrap(); + let mut decoded = Vec::with_capacity(payload.len()); + decoder.decode_all_to_vec(frame, &mut decoded).unwrap(); + assert_eq!(decoded.as_slice(), payload.as_slice()); + } +} + +#[test] +fn dict_reused_across_many_lazy_frames_stays_applied() { + // Regression: a reused HashChain-backed dictionary frame (lazy levels) + // re-primes the dict at the live `history_abs_start` every frame. The + // floor-advance reset let that base climb frame-over-frame until the + // freshly-primed dict region dropped below `window_low`, after which + // every dict match was silently lost and the output ballooned to the + // no-dict size (observed at frame 3-4 of a reused compressor). Drive + // many frames and require every one to stay byte-identical to the + // first — the dict must keep applying, not decay after a few frames. + // Multiple distinct lines so the dictionary is load-bearing: without it + // frame 0 must emit each distinct line as literals; with it those lines + // match the primed dict immediately. A single repeated line matches + // in-frame regardless and would hide the regression. + let lines: &[&[u8]] = &[ + b"ts=2026 level=INFO msg=\"flush memtable\" tenant=demo table=orders\n", + b"ts=2026 level=INFO msg=\"rotate segment\" tenant=demo table=orders\n", + b"ts=2026 level=INFO msg=\"compact level\" tenant=demo table=orders\n", + b"ts=2026 level=INFO msg=\"write block\" tenant=demo table=orders\n", + ]; + let fill = |n: usize| -> Vec { + let mut b = Vec::with_capacity(n); + while b.len() < n { + for l in lines { + if b.len() >= n { + break; + } + let take = (n - b.len()).min(l.len()); + b.extend_from_slice(&l[..take]); + } + } + b + }; + let dict = fill(8 * 1024); + let payload = fill(16 * 1024); + let dict_obj = + crate::decoding::Dictionary::from_raw_content(1, dict).expect("raw dict should build"); + + let mut compressor: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Level(6)); + compressor.set_dictionary_id_flag(false); + compressor + .set_dictionary(dict_obj) + .expect("dict should attach"); + + let first = compressor.compress_independent_frame(payload.as_slice()); + + // No-dict baseline at the same level: the dictionary must be + // load-bearing, so the dict-applied frame has to beat it. Without this + // anchor the equal-length loop below would also pass if EVERY frame + // decayed to the no-dict size in lockstep. + let mut nodict: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Level(6)); + let no_dict_frame = nodict.compress_independent_frame(payload.as_slice()); + assert!( + first.len() < no_dict_frame.len(), + "dict must be load-bearing: dict frame {} should beat the no-dict baseline {}", + first.len(), + no_dict_frame.len(), + ); + + for i in 1..16 { + let frame = compressor.compress_independent_frame(payload.as_slice()); + // Byte-identity, not just equal length: a same-size divergence (e.g. + // a different match decision once the resident dict bookkeeping + // drifts) would slip past a length-only check. + assert_eq!( + frame, first, + "frame {i} of a reused dict compressor must stay byte-identical to \ + the first (dict still applied, no decay or bookkeeping drift)" + ); + } +} + +#[test] +fn dict_fast_epoch_reset_many_frames_and_attach_copy_alternation_byte_identical() { + // The Fast attach path invalidates the main hash table between + // frames with an epoch-bias advance instead of a memset. Two things + // need proving against a fresh-compressor reference: + // 1. the bias accumulates across MANY reused frames without ever + // letting a stale entry through (every frame byte-identical); + // 2. crossing the 8 KiB attach/copy cutoff in both directions + // (attach → copy clears the bias for the raw-slice kernel, + // copy → attach re-enters epoch mode) stays byte-identical. + let dict_raw = include_bytes!("../../../dict_tests/dictionary"); + let mut small = Vec::new(); + while small.len() < 2 * 1024 { + small.extend_from_slice(b"tenant=demo op=put key=1 value=aaaaabbbbbcccccddddd\n"); + } + // Over the Fast 8 KiB attach cutoff → copy-mode frame. + let mut large = Vec::new(); + while large.len() < 64 * 1024 { + large.extend_from_slice(b"tenant=demo op=scan range=[k0,k9) limit=500 order=asc\n"); + } + + let mut reused: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Fastest); + reused + .set_encoder_dictionary( + super::EncoderDictionary::from_bytes(dict_raw).expect("dict bytes should parse"), + ) + .expect("prepared dictionary should attach"); + + let reference = |payload: &[u8]| -> alloc::vec::Vec { + let mut fresh: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Fastest); + fresh + .set_encoder_dictionary( + super::EncoderDictionary::from_bytes(dict_raw).expect("dict bytes should parse"), + ) + .expect("prepared dictionary should attach"); + fresh.compress_independent_frame(payload) + }; + + let small_expected = reference(&small); + let large_expected = reference(&large); + + // 1. Long attach-only run: every frame advances the epoch bias. + for i in 0..32 { + let frame = reused.compress_independent_frame(small.as_slice()); + assert_eq!( + frame, small_expected, + "attach frame {i} diverged from the fresh-compressor reference" + ); + } + // 2. Cutoff alternation: attach → copy → attach → copy. + for i in 0..4 { + let frame = reused.compress_independent_frame(large.as_slice()); + assert_eq!( + frame, large_expected, + "copy frame {i} diverged from the fresh-compressor reference" + ); + let frame = reused.compress_independent_frame(small.as_slice()); + assert_eq!( + frame, small_expected, + "attach frame after copy {i} diverged from the fresh-compressor reference" + ); + } +} + +#[test] +fn dict_primed_btlazy2_reused_across_attach_and_copy_boundary_is_byte_identical() { + // Btlazy2 (Level 15) uses the 32 KiB dict attach/copy cutoff in + // prepare_frame. Exercise BOTH sides of that boundary on a reused + // compressor: a sub-cutoff payload (re-prime/attach path) and an + // over-cutoff payload (copy-snapshot restore path). In each case the + // warm-cache second frame must reproduce the cold-cache first frame + // byte-for-byte (the dict cache is a timing optimization, never a + // content change), and every frame must round-trip. + let dict_raw = include_bytes!("../../../dict_tests/dictionary"); + let dict_id = super::EncoderDictionary::from_bytes(dict_raw) + .expect("dict bytes should parse") + .id(); + // Distinct lines so the payload does not trivially self-compress; the + // BT finder + dict dual-probe both get exercised. + let make_payload = |target: usize| { + let mut p = Vec::with_capacity(target); + let mut i = 0u64; + while p.len() < target { + p.extend_from_slice( + format!( + "tenant=demo op=put key={i} value=aaaaabbbbbcccccddddd-{}\n", + i % 97 + ) + .as_bytes(), + ); + i += 1; + } + p + }; + // Below the 32 KiB cutoff (attach/re-prime) and above it (copy-snapshot). + for target in [16 * 1024usize, 64 * 1024usize] { + let payload = make_payload(target); + let mut warm: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Level(15)); + warm.set_encoder_dictionary( + super::EncoderDictionary::from_bytes(dict_raw).expect("dict parse"), + ) + .expect("dict attach"); + // Frame 1 builds + marks the dict tables; frame 2 reuses them. + let frame1 = warm.compress_independent_frame(payload.as_slice()); + let frame2 = warm.compress_independent_frame(payload.as_slice()); + assert_eq!( + frame1, frame2, + "reused dict cache must reproduce the freshly-primed frame byte-for-byte \ + (Level 15, target={target})" + ); + // Cold-cache compressor: must match the warm-cache bytes. + let mut cold: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Level(15)); + cold.set_encoder_dictionary( + super::EncoderDictionary::from_bytes(dict_raw).expect("dict parse"), + ) + .expect("dict attach"); + let cold_frame = cold.compress_independent_frame(payload.as_slice()); + assert_eq!( + cold_frame, frame1, + "cold-cache compressor must match warm-cache frame (Level 15, target={target})" + ); + // Round-trip through a decoder primed with the same dict. + for frame in [&frame1, &frame2] { + let (hdr, _) = + crate::decoding::frame::read_frame_header(frame.as_slice()).expect("frame header"); + assert_eq!(hdr.dictionary_id(), Some(dict_id)); + let mut decoder = FrameDecoder::new(); + decoder + .add_dict(crate::decoding::Dictionary::decode_dict(dict_raw).unwrap()) + .unwrap(); + let mut decoded = Vec::with_capacity(payload.len()); + decoder.decode_all_to_vec(frame, &mut decoded).unwrap(); + assert_eq!(decoded.as_slice(), payload.as_slice()); + } + } +} + +#[test] +fn dict_primed_btultra2_restore_is_floor_safe_and_byte_identical() { + // Regression guard for the dictionary primed-snapshot RESTORE path on + // the binary-tree (btultra2 / Level 22) backend — the path a minimal / + // decoupled prepared-dict refactor rewrites. + // + // The trap it pins: a reused compressor compresses frame A (which fills + // the live hash/chain tables with frame-A positions and advances the + // window floor), then frame B of the SAME resolved shape (same size → + // same PrimedKey → the snapshot RESTORE path) but DIFFERENT content. The + // restore must reinstate the clean post-prime dict state with NO live + // frame-A entries surviving above the restored floor; a restore that + // leaks stale frame-A positions would surface FALSE matches and produce + // a different (or undecodable) frame B. The invariant: a snapshot + // restore is a pure timing optimization and MUST be byte-identical to a + // cold compressor compressing frame B from scratch, and must round-trip. + let dict_raw = include_bytes!("../../../dict_tests/dictionary"); + let dict_id = super::EncoderDictionary::from_bytes(dict_raw) + .expect("dict bytes should parse") + .id(); + // 48 KiB > the btultra2 8 KiB attach cutoff → the copy-snapshot + // capture/restore path. Two distinct payloads of the SAME size so frame + // B resolves to frame A's snapshot key and takes the restore path. + let make_payload = |seed: u64, target: usize| { + let mut p = Vec::with_capacity(target); + let mut i = seed; + while p.len() < target { + p.extend_from_slice( + format!( + "tenant=demo op=put key={i} value=aaaaabbbbbcccccddddd-{}\n", + i % 89 + ) + .as_bytes(), + ); + i = i.wrapping_add(1); + } + p.truncate(target); + p + }; + let size = 48 * 1024usize; + let frame_a = make_payload(0, size); + let frame_b = make_payload(1_000_000, size); + + let mut warm: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Level(22)); + warm.set_encoder_dictionary( + super::EncoderDictionary::from_bytes(dict_raw).expect("dict parse"), + ) + .expect("dict attach"); + // Frame A: cold cache — primes the dict + captures the snapshot, and + // fills the live tables with frame-A positions. + let _wa = warm.compress_independent_frame(frame_a.as_slice()); + // Frame B: warm cache — takes the snapshot RESTORE path (same size). + let warm_b = warm.compress_independent_frame(frame_b.as_slice()); + + // Cold compressor compressing frame B from scratch: the ground truth. + let mut cold: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Level(22)); + cold.set_encoder_dictionary( + super::EncoderDictionary::from_bytes(dict_raw).expect("dict parse"), + ) + .expect("dict attach"); + let cold_b = cold.compress_independent_frame(frame_b.as_slice()); + + assert_eq!( + warm_b, cold_b, + "frame B via snapshot restore must be byte-identical to a cold compress \ + (a restore that leaks frame-A live-table entries would diverge here)" + ); + + // Round-trip frame B through a dict-primed decoder. + let (hdr, _) = + crate::decoding::frame::read_frame_header(warm_b.as_slice()).expect("frame header"); + assert_eq!(hdr.dictionary_id(), Some(dict_id)); + let mut decoder = FrameDecoder::new(); + decoder + .add_dict(crate::decoding::Dictionary::decode_dict(dict_raw).unwrap()) + .unwrap(); + let mut decoded = Vec::with_capacity(frame_b.len()); + decoder + .decode_all_to_vec(warm_b.as_slice(), &mut decoded) + .unwrap(); + assert_eq!(decoded.as_slice(), frame_b.as_slice()); +} + +#[test] +fn dict_primed_btultra2_ldm_restore_is_byte_identical() { + // Same restore-path byte-identity guard as + // `dict_primed_btultra2_restore_is_floor_safe_and_byte_identical`, but + // with long-distance matching ENABLED. The BtMatcher's LDM producer is + // part of the snapshot; a refactor that decouples it (so the snapshot + // does not retain the empty LDM table) must reinstate an equivalent + // empty producer on restore. This pins that the warm-cache (restore) + // frame stays byte-identical to a cold compress when LDM is on. + let dict_raw = include_bytes!("../../../dict_tests/dictionary"); + let dict_id = super::EncoderDictionary::from_bytes(dict_raw) + .expect("dict bytes should parse") + .id(); + let make_payload = |seed: u64, target: usize| { + let mut p = Vec::with_capacity(target); + let mut i = seed; + while p.len() < target { + p.extend_from_slice( + format!( + "tenant=demo op=put key={i} value=aaaaabbbbbcccccddddd-{}\n", + i % 89 + ) + .as_bytes(), + ); + i = i.wrapping_add(1); + } + p.truncate(target); + p + }; + let ldm_params = + crate::encoding::CompressionParameters::builder(super::CompressionLevel::Level(22)) + .enable_long_distance_matching(true) + .build() + .expect("LDM-only params build"); + let size = 48 * 1024usize; + let frame_a = make_payload(0, size); + let frame_b = make_payload(1_000_000, size); + + let mut warm: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Level(22)); + warm.set_parameters(&ldm_params); + warm.set_encoder_dictionary( + super::EncoderDictionary::from_bytes(dict_raw).expect("dict parse"), + ) + .expect("dict attach"); + let _wa = warm.compress_independent_frame(frame_a.as_slice()); + let warm_b = warm.compress_independent_frame(frame_b.as_slice()); + + let mut cold: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Level(22)); + cold.set_parameters(&ldm_params); + cold.set_encoder_dictionary( + super::EncoderDictionary::from_bytes(dict_raw).expect("dict parse"), + ) + .expect("dict attach"); + let cold_b = cold.compress_independent_frame(frame_b.as_slice()); + + assert_eq!( + warm_b, cold_b, + "LDM-on frame B via snapshot restore must be byte-identical to a cold compress" + ); + + let (hdr, _) = + crate::decoding::frame::read_frame_header(warm_b.as_slice()).expect("frame header"); + assert_eq!(hdr.dictionary_id(), Some(dict_id)); + let mut decoder = FrameDecoder::new(); + decoder + .add_dict(crate::decoding::Dictionary::decode_dict(dict_raw).unwrap()) + .unwrap(); + let mut decoded = Vec::with_capacity(frame_b.len()); + decoder + .decode_all_to_vec(warm_b.as_slice(), &mut decoded) + .unwrap(); + assert_eq!(decoded.as_slice(), frame_b.as_slice()); +} + +#[test] +fn set_dictionary_from_bytes_matches_full_decode_byte_for_byte() { + // The encoder-only dict parse (`decode_dict_for_encoding`, used by + // `set_dictionary_from_bytes`) skips the FSE/HUF decoder-table build and + // the enrich passes. The encoder entropy tables are derived purely from + // the symbol probabilities / Huffman weights, so the compressed output + // MUST be byte-identical to the full-decode path. This pins the + // load-bearing equivalence so a future FSE/HUF parsing refactor that + // still round-trips but silently diverges on the probabilities/weights + // fails loudly here instead of producing a different (but valid) frame. + let dict_raw = include_bytes!("../../../dict_tests/dictionary"); + let payload = b"tenant=demo table=orders op=put key=1 value=aaaaabbbbbcccccdddddeeeee\n\ + tenant=demo table=orders op=put key=2 value=aaaaabbbbbcccccdddddeeeee\n"; + + // Path A: encoder-only parse straight from the raw blob. + let mut from_bytes_out = Vec::new(); + { + let mut compressor = FrameCompressor::new(super::CompressionLevel::Fastest); + compressor + .set_dictionary_from_bytes(dict_raw) + .expect("dictionary bytes should parse"); + compressor.set_source(payload.as_slice()); + compressor.set_drain(&mut from_bytes_out); + compressor.compress(); + } + + // Path B: full decode (builds the decoder tables too), then attach for + // encoding via the `Dictionary` setter. + let full_decode = crate::decoding::Dictionary::decode_dict(dict_raw) + .expect("dictionary bytes should fully decode"); + let mut full_decode_out = Vec::new(); + { + let mut compressor = FrameCompressor::new(super::CompressionLevel::Fastest); + compressor + .set_dictionary(full_decode) + .expect("full-decode dictionary should attach"); + compressor.set_source(payload.as_slice()); + compressor.set_drain(&mut full_decode_out); + compressor.compress(); + } + + assert_eq!( + from_bytes_out, full_decode_out, + "encoder-only dict parse must produce byte-identical output to the full decode" + ); +} + +#[test] +fn set_dictionary_rejects_zero_dictionary_id() { + let invalid = crate::decoding::Dictionary { + id: 0, + fse: crate::decoding::scratch::FSEScratch::new(), + huf: crate::decoding::scratch::HuffmanScratch::new(), + dict_content: vec![1, 2, 3], + offset_hist: [1, 4, 8], + }; + + let mut compressor: FrameCompressor< + &[u8], + Vec, + crate::encoding::match_generator::MatchGeneratorDriver, + > = FrameCompressor::new(super::CompressionLevel::Fastest); + let result = compressor.set_dictionary(invalid); + assert!(matches!( + result, + Err(crate::decoding::errors::DictionaryDecodeError::ZeroDictionaryId) + )); +} + +#[test] +fn set_dictionary_rejects_zero_repeat_offsets() { + let invalid = crate::decoding::Dictionary { + id: 1, + fse: crate::decoding::scratch::FSEScratch::new(), + huf: crate::decoding::scratch::HuffmanScratch::new(), + dict_content: vec![1, 2, 3], + offset_hist: [0, 4, 8], + }; + + let mut compressor: FrameCompressor< + &[u8], + Vec, + crate::encoding::match_generator::MatchGeneratorDriver, + > = FrameCompressor::new(super::CompressionLevel::Fastest); + let result = compressor.set_dictionary(invalid); + assert!(matches!( + result, + Err( + crate::decoding::errors::DictionaryDecodeError::ZeroRepeatOffsetInDictionary { + index: 0 + } + ) + )); +} + +#[test] +fn uncompressed_mode_does_not_require_dictionary() { + let dict_id = 0xABCD_0001; + let dict = crate::decoding::Dictionary::from_raw_content(dict_id, b"shared-history".to_vec()) + .expect("raw dictionary should be valid"); + + let payload = b"plain-bytes-that-should-stay-raw"; + let mut output = Vec::new(); + let mut compressor = FrameCompressor::new(super::CompressionLevel::Uncompressed); + compressor + .set_dictionary(dict) + .expect("dictionary should attach in uncompressed mode"); + compressor.set_source(payload.as_slice()); + compressor.set_drain(&mut output); + compressor.compress(); + + let (frame_header, _) = crate::decoding::frame::read_frame_header(output.as_slice()) + .expect("encoded frame should have a header"); + assert_eq!( + frame_header.dictionary_id(), + None, + "raw/uncompressed frames must not advertise dictionary dependency" + ); + + let mut decoder = FrameDecoder::new(); + let mut decoded = Vec::with_capacity(payload.len()); + decoder.decode_all_to_vec(&output, &mut decoded).unwrap(); + assert_eq!(decoded, payload); +} + +#[test] +fn default_level_tiny_raw_dict_compresses_cleanly() { + // Coverage for the dfast dict-attach fast path with a + // sub-min-match raw-content dictionary: the dict-table probe in + // `start_matching_fast_loop` is gated on the dict table actually + // existing (`table().is_some()`), not merely on `is_attached()`, + // so a dictionary whose hashable region is shorter than the + // short-hash lookahead (where `prime_dict_tables_for_range` + // returns before allocating the tables) never dereferences a + // null dict pointer. Compressing at the default (dfast) level + // with such a dict must succeed. + let dict_id = 0xABCD_0009; + let dict = crate::decoding::Dictionary::from_raw_content(dict_id, b"abc".to_vec()) + .expect("raw dictionary should be valid"); + let payload = b"the quick brown fox jumps over the lazy dog, repeatedly and at length"; + let mut output = Vec::new(); + let mut compressor = FrameCompressor::new(super::CompressionLevel::Default); + compressor + .set_dictionary(dict) + .expect("tiny raw dictionary should attach"); + compressor.set_source(payload.as_slice()); + compressor.set_drain(&mut output); + compressor.compress(); + assert!(!output.is_empty(), "compression should produce a frame"); + + // The emitted frame must advertise the attached dictionary id, proving + // the tiny-dict path stayed active (the payload round-trips either way, + // so without this the test would also pass on a silent no-dict frame). + let (frame_header, _) = crate::decoding::frame::read_frame_header(output.as_slice()) + .expect("encoded frame should have a readable header"); + assert_eq!( + frame_header.dictionary_id(), + Some(dict_id), + "tiny raw dict frame should still advertise its dictionary id", + ); + + // Full roundtrip: decode the dict-compressed frame with the SAME + // dictionary attached and confirm byte-exact recovery — proves the + // tiny-dict fast path produces a correct frame, not just a non-empty + // one. + let decode_dict = crate::decoding::Dictionary::from_raw_content(dict_id, b"abc".to_vec()) + .expect("raw dictionary should be valid"); + let mut decoder = FrameDecoder::new(); + decoder + .add_dict(decode_dict) + .expect("decoder dict should attach"); + let mut decoded = Vec::with_capacity(payload.len()); + decoder + .decode_all_to_vec(&output, &mut decoded) + .expect("dict roundtrip should decode"); + assert_eq!(decoded, payload, "tiny-dict roundtrip mismatch"); +} + +/// Exercises the dictionary dual-probe (live + immutable dict tables) +/// in the Fast / dfast / Row match finders with a dict whose content +/// the payload actually reuses, so each backend's dict long/short +/// probe (and the dfast `ip+1` dict-long retry) is reached and the +/// dict-compressed frame round-trips through a decoder primed with the +/// same dict. The 3-byte-dict test above only proves the null-table +/// guard; this proves the full attach path produces correct frames. +#[test] +fn dict_attach_roundtrips_across_backends_with_matching_payload() { + let dict_id = 0xD1C7_0001; + // Distinct lines so the payload does NOT self-compress: each line + // appears exactly once in the payload, so without the dictionary there + // are no in-frame back-references to exploit. The dictionary holds the + // SAME lines, so the only way the output shrinks is if the dict probe + // actually fires. A no-dict baseline below pins that the dict path ran + // (self-compressible payloads would round-trip + stay small via + // in-frame matches alone, proving nothing). + let line = |i: u32| { + alloc::format!( + "ts=2026-03-26T21:{:02}:{:02}Z level=INFO msg=\"event {i:05}\" tenant=t{i} region=eu\n", + i / 60 % 60, + i % 60, + ) + .into_bytes() + }; + let mut dict_content = Vec::new(); + for i in 0..256u32 { + dict_content.extend_from_slice(&line(i)); + } + // Payload = the same distinct lines in a different (stride) order, each + // once → no self-repeats, every line is a dictionary match. + let mut payload = Vec::new(); + let mut i = 0u32; + for _ in 0..256u32 { + payload.extend_from_slice(&line(i)); + i = (i + 97) % 256; // coprime stride → permutation, no adjacency + } + + let compress_at = |level, dict: Option>| -> Vec { + let mut compressor = FrameCompressor::new(level); + if let Some(bytes) = dict { + let d = crate::decoding::Dictionary::from_raw_content(dict_id, bytes) + .expect("raw dictionary should be valid"); + compressor + .set_dictionary(d) + .expect("dictionary should attach"); + } + let mut out = Vec::new(); + compressor.set_source(payload.as_slice()); + compressor.set_drain(&mut out); + compressor.compress(); + out + }; + + for level in [ + super::CompressionLevel::Level(-5), // Fast (negative) + super::CompressionLevel::Level(1), // Fast + super::CompressionLevel::Default, // dfast (L3) + super::CompressionLevel::Level(8), // Row-backed lazy2 + ] { + let out = compress_at(level, Some(dict_content.clone())); + let no_dict = compress_at(level, None); + // The dict path MUST measurably beat no-dict on this + // non-self-compressible payload — otherwise the dict probe never + // fired and the roundtrip below would prove nothing. + assert!( + out.len() < no_dict.len(), + "level {level:?}: dict-primed output ({}) must beat no-dict ({}) — dict probe did not fire", + out.len(), + no_dict.len(), + ); + + let ddict = crate::decoding::Dictionary::from_raw_content(dict_id, dict_content.clone()) + .expect("raw dictionary should be valid"); + let mut decoder = FrameDecoder::new(); + decoder.add_dict(ddict).expect("decoder dict should attach"); + let mut decoded = Vec::with_capacity(payload.len()); + decoder + .decode_all_to_vec(&out, &mut decoded) + .unwrap_or_else(|e| panic!("level {level:?}: dict roundtrip decode failed: {e:?}")); + assert_eq!(decoded, payload, "level {level:?}: dict roundtrip mismatch"); + } +} + +/// Reusing one compressor across independent frames with DIFFERENT +/// dictionaries must drop the per-backend dict cache on each swap +/// (Simple/Dfast/Row keep the attach index across frames). Without the +/// invalidation a later frame would reuse the previous dict's rows. +/// Each frame round-trips through a decoder primed with its own dict. +#[test] +fn dict_swap_across_reused_compressor_roundtrips() { + // Distinct lines per dict (not a single repeated line) so payloads do + // NOT self-compress: each line appears once, so a frame only shrinks if + // the dict probe fires, and — crucially for the invalidation check — if + // frame B reused dict A's stale rows it would emit offsets into A's + // distinct content, which decode under dict B reconstructs as WRONG + // bytes (caught by the roundtrip). A single repeated line would hide + // pollution behind in-frame matches. + let lines = |tag: &str| -> (Vec, Vec) { + let line = |i: u32| alloc::format!("{tag} record {i:05} field=value{i} end\n").into_bytes(); + let mut dict = Vec::new(); + for i in 0..256u32 { + dict.extend_from_slice(&line(i)); + } + let mut payload = Vec::new(); + let mut i = 0u32; + for _ in 0..256u32 { + payload.extend_from_slice(&line(i)); + i = (i + 97) % 256; + } + (dict, payload) + }; + let (dict_a, payload_a) = lines("alpha"); + let (dict_b, payload_b) = lines("bravo"); + + for level in [ + super::CompressionLevel::Default, + super::CompressionLevel::Level(8), + ] { + let no_dict = |payload: &[u8]| -> usize { + let mut c: FrameCompressor = FrameCompressor::new(level); + c.compress_independent_frame(payload).len() + }; + let no_dict_a = no_dict(&payload_a); + let no_dict_b = no_dict(&payload_b); + + let mut compressor: FrameCompressor = FrameCompressor::new(level); + for (dict_bytes, payload, no_dict_len) in [ + (&dict_a, &payload_a, no_dict_a), + (&dict_b, &payload_b, no_dict_b), + ] { + let dict = + crate::decoding::Dictionary::from_raw_content(0xD1C7_0002, dict_bytes.clone()) + .expect("raw dictionary should be valid"); + compressor + .set_dictionary(dict) + .expect("dictionary should attach"); + let out = compressor.compress_independent_frame(payload.as_slice()); + assert!( + out.len() < no_dict_len, + "level {level:?}: dict frame ({}) must beat no-dict ({}) — dict probe did not fire", + out.len(), + no_dict_len, + ); + + let ddict = + crate::decoding::Dictionary::from_raw_content(0xD1C7_0002, dict_bytes.clone()) + .expect("raw dictionary should be valid"); + let mut decoder = FrameDecoder::new(); + decoder.add_dict(ddict).expect("decoder dict should attach"); + let mut decoded = Vec::with_capacity(payload.len()); + decoder + .decode_all_to_vec(&out, &mut decoded) + .unwrap_or_else(|e| panic!("level {level:?}: dict-swap decode failed: {e:?}")); + assert_eq!( + decoded, *payload, + "level {level:?}: dict-swap roundtrip mismatch (stale dict rows?)" + ); + } + } +} + +#[test] +fn dictionary_roundtrip_stays_valid_after_output_exceeds_window() { + use crate::encoding::match_generator::MatchGeneratorDriver; + + let dict_id = 0xABCD_0002; + let dict = crate::decoding::Dictionary::from_raw_content(dict_id, b"abcdefgh".to_vec()) + .expect("raw dictionary should be valid"); + let dict_for_decoder = + crate::decoding::Dictionary::from_raw_content(dict_id, b"abcdefgh".to_vec()) + .expect("raw dictionary should be valid"); + + // Payload must exceed the encoder's advertised window (512 KiB + // for Fastest after `window_log = 19` alignment with upstream zstd's + // L1 fast row in `clevels.h`) so the test actually exercises + // cross-window-boundary behavior. + let payload = b"abcdefgh".repeat(512 * 1024 / 8 + 64); + let matcher = MatchGeneratorDriver::new(1024, 1); + + let mut no_dict_output = Vec::new(); + let mut no_dict_compressor = + FrameCompressor::new_with_matcher(matcher, super::CompressionLevel::Fastest); + no_dict_compressor.set_source(payload.as_slice()); + no_dict_compressor.set_drain(&mut no_dict_output); + no_dict_compressor.compress(); + let (no_dict_frame_header, _) = + crate::decoding::frame::read_frame_header(no_dict_output.as_slice()) + .expect("baseline frame should have a header"); + let no_dict_window = no_dict_frame_header + .window_size() + .expect("window size should be present"); + + let mut output = Vec::new(); + let matcher = MatchGeneratorDriver::new(1024, 1); + let mut compressor = + FrameCompressor::new_with_matcher(matcher, super::CompressionLevel::Fastest); + compressor + .set_dictionary(dict) + .expect("dictionary should attach"); + compressor.set_source(payload.as_slice()); + compressor.set_drain(&mut output); + compressor.compress(); + + let (frame_header, _) = crate::decoding::frame::read_frame_header(output.as_slice()) + .expect("encoded frame should have a header"); + let advertised_window = frame_header + .window_size() + .expect("window size should be present"); + assert_eq!( + advertised_window, no_dict_window, + "dictionary priming must not inflate advertised window size" + ); + assert!( + payload.len() > advertised_window as usize, + "test must cross the advertised window boundary" + ); + + let mut decoder = FrameDecoder::new(); + decoder.add_dict(dict_for_decoder).unwrap(); + let mut decoded = Vec::with_capacity(payload.len()); + decoder.decode_all_to_vec(&output, &mut decoded).unwrap(); + assert_eq!(decoded, payload); +} + +#[test] +fn source_size_hint_with_dictionary_keeps_roundtrip_and_nonincreasing_window() { + let dict_id = 0xABCD_0004; + let dict_content = b"abcd".repeat(1024); // 4 KiB dictionary history + let dict = crate::decoding::Dictionary::from_raw_content(dict_id, dict_content).unwrap(); + let dict_for_decoder = + crate::decoding::Dictionary::from_raw_content(dict_id, b"abcd".repeat(1024)).unwrap(); + let payload = b"abcdabcdabcdabcd".repeat(128); + + let mut hinted_output = Vec::new(); + let mut hinted = FrameCompressor::new(super::CompressionLevel::Fastest); + hinted.set_dictionary(dict).unwrap(); + hinted.set_source_size_hint(1); + hinted.set_source(payload.as_slice()); + hinted.set_drain(&mut hinted_output); + hinted.compress(); + + let mut no_hint_output = Vec::new(); + let mut no_hint = FrameCompressor::new(super::CompressionLevel::Fastest); + no_hint + .set_dictionary( + crate::decoding::Dictionary::from_raw_content(dict_id, b"abcd".repeat(1024)).unwrap(), + ) + .unwrap(); + no_hint.set_source(payload.as_slice()); + no_hint.set_drain(&mut no_hint_output); + no_hint.compress(); + + let hinted_window = crate::decoding::frame::read_frame_header(hinted_output.as_slice()) + .expect("encoded frame should have a header") + .0 + .window_size() + .expect("window size should be present"); + let no_hint_window = crate::decoding::frame::read_frame_header(no_hint_output.as_slice()) + .expect("encoded frame should have a header") + .0 + .window_size() + .expect("window size should be present"); + assert!( + hinted_window <= no_hint_window, + "source-size hint should not increase advertised window with dictionary priming", + ); + + let mut decoder = FrameDecoder::new(); + decoder.add_dict(dict_for_decoder).unwrap(); + let mut decoded = Vec::with_capacity(payload.len()); + decoder + .decode_all_to_vec(&hinted_output, &mut decoded) + .unwrap(); + assert_eq!(decoded, payload); +} + +/// A dictionary segment embedded ONCE in otherwise-incompressible +/// input must be matched against the dictionary. Before the fix the +/// raw-fast-path (which skips matching) fired on the +/// incompressible-looking block and the dictionary was never searched, +/// so `with_dict` came out the same size as `no_dict` (the embedded +/// match was lost). Now the block compresses against the dict. +#[test] +fn dictionary_segment_in_incompressible_input_is_matched() { + // Deterministic LCG bytes: high-entropy, so the only compressible + // content is the embedded dictionary segment. + fn lcg(seed: u64, n: usize) -> alloc::vec::Vec { + let mut s = seed; + (0..n) + .map(|_| { + s = s + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + (s >> 56) as u8 + }) + .collect() + } + let dict_id = 0x00DC_7777; + let r = lcg(1, 512); // the dictionary content + let mut payload = lcg(2, 2000); // incompressible filler before + payload.extend_from_slice(&r); // the single dict-matchable segment + payload.extend_from_slice(&lcg(3, 1500)); // filler after + + // Precondition: the payload must actually look incompressible so + // that the raw-fast-path WOULD fire (and skip matching) without + // the fix. If the heuristic ever changes and this no longer holds, + // the test below would pass vacuously — assert it up front. + assert!( + crate::encoding::incompressible::block_looks_incompressible(&payload), + "test payload must look incompressible to exercise the raw-fast-path", + ); + + let compress = |level: super::CompressionLevel, dict: Option<&[u8]>| -> alloc::vec::Vec { + let mut out = alloc::vec::Vec::new(); + let mut c = FrameCompressor::new(level); + if let Some(d) = dict { + c.set_dictionary( + crate::decoding::Dictionary::from_raw_content(dict_id, d.to_vec()).unwrap(), + ) + .unwrap(); + } + c.set_source(payload.as_slice()); + c.set_drain(&mut out); + c.compress(); + out + }; + + for lvl in [ + super::CompressionLevel::Level(2), + super::CompressionLevel::Level(6), + super::CompressionLevel::Level(19), + ] { + let with_dict = compress(lvl, Some(&r)); + let no_dict = compress(lvl, None); + // The 512-byte dict segment should be matched, saving most of + // its length (generous slack for sequence/header coding). + assert!( + with_dict.len() + 300 < no_dict.len(), + "{lvl:?}: dict segment not matched (with_dict={}, no_dict={})", + with_dict.len(), + no_dict.len(), + ); + // The dict-compressed frame must round-trip through the decoder. + let mut decoder = FrameDecoder::new(); + decoder + .add_dict(crate::decoding::Dictionary::from_raw_content(dict_id, r.clone()).unwrap()) + .unwrap(); + let mut decoded = Vec::with_capacity(payload.len()); + decoder.decode_all_to_vec(&with_dict, &mut decoded).unwrap(); + assert_eq!(decoded, payload, "{lvl:?}: dict round-trip mismatch"); + + // A dictionary that does NOT appear in the input must not make + // the output larger than the no-dict (raw) encoding: the + // post-compress raw fallback covers incompressible-with-dict. + let unrelated = lcg(99, 512); + let with_bad_dict = compress(lvl, Some(&unrelated)); + assert!( + with_bad_dict.len() <= no_dict.len() + 16, + "{lvl:?}: unhelpful dict expanded output (with={}, no_dict={})", + with_bad_dict.len(), + no_dict.len(), + ); + } +} + +#[test] +fn source_size_hint_with_dictionary_keeps_roundtrip_for_larger_payload() { + let dict_id = 0xABCD_0005; + let dict_content = b"abcd".repeat(1024); // 4 KiB dictionary history + let dict = crate::decoding::Dictionary::from_raw_content(dict_id, dict_content).unwrap(); + let dict_for_decoder = + crate::decoding::Dictionary::from_raw_content(dict_id, b"abcd".repeat(1024)).unwrap(); + let payload = b"abcd".repeat(1024); // 4 KiB payload + let payload_len = payload.len() as u64; + + let mut hinted_output = Vec::new(); + let mut hinted = FrameCompressor::new(super::CompressionLevel::Fastest); + hinted.set_dictionary(dict).unwrap(); + hinted.set_source_size_hint(payload_len); + hinted.set_source(payload.as_slice()); + hinted.set_drain(&mut hinted_output); + hinted.compress(); + + let mut no_hint_output = Vec::new(); + let mut no_hint = FrameCompressor::new(super::CompressionLevel::Fastest); + no_hint + .set_dictionary( + crate::decoding::Dictionary::from_raw_content(dict_id, b"abcd".repeat(1024)).unwrap(), + ) + .unwrap(); + no_hint.set_source(payload.as_slice()); + no_hint.set_drain(&mut no_hint_output); + no_hint.compress(); + + let hinted_window = crate::decoding::frame::read_frame_header(hinted_output.as_slice()) + .expect("encoded frame should have a header") + .0 + .window_size() + .expect("window size should be present"); + let no_hint_window = crate::decoding::frame::read_frame_header(no_hint_output.as_slice()) + .expect("encoded frame should have a header") + .0 + .window_size() + .expect("window size should be present"); + assert!( + hinted_window <= no_hint_window, + "source-size hint should not increase advertised window with dictionary priming", + ); + + let mut decoder = FrameDecoder::new(); + decoder.add_dict(dict_for_decoder).unwrap(); + let mut decoded = Vec::with_capacity(payload.len()); + decoder + .decode_all_to_vec(&hinted_output, &mut decoded) + .unwrap(); + assert_eq!(decoded, payload); +} + +#[test] +fn custom_matcher_without_dictionary_priming_does_not_advertise_dict_id() { + let dict_id = 0xABCD_0003; + let dict = crate::decoding::Dictionary::from_raw_content(dict_id, b"abcdefgh".to_vec()) + .expect("raw dictionary should be valid"); + let payload = b"abcdefghabcdefgh"; + + let mut output = Vec::new(); + let matcher = NoDictionaryMatcher::new(64); + let mut compressor = + FrameCompressor::new_with_matcher(matcher, super::CompressionLevel::Fastest); + compressor + .set_dictionary(dict) + .expect("dictionary should attach"); + compressor.set_source(payload.as_slice()); + compressor.set_drain(&mut output); + compressor.compress(); + + let (frame_header, _) = crate::decoding::frame::read_frame_header(output.as_slice()) + .expect("encoded frame should have a header"); + assert_eq!( + frame_header.dictionary_id(), + None, + "matchers that do not support dictionary priming must not advertise dictionary dependency" + ); + + let mut decoder = FrameDecoder::new(); + let mut decoded = Vec::with_capacity(payload.len()); + decoder.decode_all_to_vec(&output, &mut decoded).unwrap(); + assert_eq!(decoded, payload); +} + +#[cfg(feature = "hash")] +#[test] +fn checksum_two_frames_reused_compressor() { + // Compress the same data twice using the same compressor and verify that: + // 1. The checksum written in each frame matches what the decoder calculates. + // 2. The hasher is correctly reset between frames (no cross-contamination). + // If the hasher were NOT reset, the second frame's calculated checksum + // would differ from the one stored in the frame data, causing assert_eq to fail. + let data: Vec = (0u8..=255).cycle().take(1024).collect(); + + let mut compressor = FrameCompressor::new(super::CompressionLevel::Uncompressed); + + // --- Frame 1 --- + let mut compressed1 = Vec::new(); + compressor.set_source(data.as_slice()); + compressor.set_drain(&mut compressed1); + compressor.compress(); + + // --- Frame 2 (reuse the same compressor) --- + let mut compressed2 = Vec::new(); + compressor.set_source(data.as_slice()); + compressor.set_drain(&mut compressed2); + compressor.compress(); + + fn decode_and_collect(compressed: &[u8]) -> (Vec, Option, Option) { + let mut decoder = FrameDecoder::new(); + let mut source = compressed; + decoder.reset(&mut source).unwrap(); + while !decoder.is_finished() { + decoder + .decode_blocks(&mut source, crate::decoding::BlockDecodingStrategy::All) + .unwrap(); + } + let mut decoded = Vec::new(); + decoder.collect_to_writer(&mut decoded).unwrap(); + ( + decoded, + decoder.get_checksum_from_data(), + decoder.get_calculated_checksum(), + ) + } + + let (decoded1, chksum_from_data1, chksum_calculated1) = decode_and_collect(&compressed1); + assert_eq!(decoded1, data, "frame 1: decoded data mismatch"); + assert_eq!( + chksum_from_data1, chksum_calculated1, + "frame 1: checksum mismatch" + ); + + let (decoded2, chksum_from_data2, chksum_calculated2) = decode_and_collect(&compressed2); + assert_eq!(decoded2, data, "frame 2: decoded data mismatch"); + assert_eq!( + chksum_from_data2, chksum_calculated2, + "frame 2: checksum mismatch" + ); + + // Same data compressed twice must produce the same checksum. + // If state leaked across frames, the second calculated checksum would differ. + assert_eq!( + chksum_from_data1, chksum_from_data2, + "frame 1 and frame 2 should have the same checksum (same data, hash must reset per frame)" + ); +} + +#[cfg(feature = "lsm")] +#[test] +fn frame_emit_info_decompressed_ranges_match_decoded_output() { + // Part A correctness: the per-block `decompressed_size` captured during + // encode (and the `decompressed_byte_range` prefix sum derived from it) + // must describe the real decoded output exactly — one entry per + // physical block, contiguous, summing to the full decompressed length. + // A multi-block compressible payload exercises the Compressed-block + // path (whose regenerated size is NOT on the wire, so it relies on the + // encode-side capture this test guards). + let data = emit_info_fixture_data(); + + // Cover both the single-block-per-chunk path (Default) and the + // Level(16..=22) post-split path (multiple physical partitions per + // input chunk), since lsm-tree compresses at zstd:22 and post-split + // is the riskiest capture site (per-partition `src_size`). + for level in [ + super::CompressionLevel::Default, + super::CompressionLevel::Level(22), + ] { + let mut compressed = Vec::new(); + let mut compressor = FrameCompressor::new(level); + // Pledge the source size so the high-level (22) window shrinks to + // fit the payload, keeping the frame compact (no oversized window + // descriptor for a small input). Still >= 128 KiB, so post-split + // eligibility is preserved. + compressor.set_source_size_hint(data.len() as u64); + compressor.set_source(data.as_slice()); + compressor.set_drain(&mut compressed); + compressor.compress(); + + let info = compressor + .last_frame_emit_info() + .expect("emit info populated after compress") + .clone(); + + // Reference: full decode of the same frame. + let mut decoder = FrameDecoder::new(); + let mut source = compressed.as_slice(); + decoder.reset(&mut source).unwrap(); + while !decoder.is_finished() { + decoder + .decode_blocks(&mut source, crate::decoding::BlockDecodingStrategy::All) + .unwrap(); + } + let mut decoded = Vec::new(); + decoder.collect_to_writer(&mut decoded).unwrap(); + assert_eq!(decoded, data, "sanity: frame must round-trip ({level:?})"); + + assert!( + info.blocks.len() >= 2, + "fixture must span multiple blocks to exercise the mapping ({level:?}, got {})", + info.blocks.len() + ); + assert!( + info.blocks.last().unwrap().last_block, + "final block must carry last_block ({level:?})" + ); + + // Pin the Level(22) post-split path: the owned loop feeds the + // encoder MAX_BLOCK_SIZE input chunks, so without post-split the + // block count cannot exceed the chunk count. More blocks than + // chunks proves at least one chunk was split into multiple physical + // partitions (the per-partition `src_size` capture under test). + if matches!(level, super::CompressionLevel::Level(22)) { + let max_block = crate::common::MAX_BLOCK_SIZE as usize; + let n_chunks = data.len().div_ceil(max_block); + assert!( + info.blocks.len() > n_chunks, + "Level(22) must exercise post-split: {} blocks for {} input chunks", + info.blocks.len(), + n_chunks + ); + } + + // Per-block ranges: contiguous, zero-based, summing to the full output. + let mut expected_start = 0u64; + for i in 0..info.blocks.len() { + let range = info + .decompressed_byte_range(i) + .expect("in-bounds block has a range"); + assert_eq!( + range.start, expected_start, + "block {i} range must start where the previous ended ({level:?})" + ); + assert_eq!( + u64::from(info.blocks[i].decompressed_size), + range.end - range.start, + "block {i} decompressed_size must equal its range width ({level:?})" + ); + // Validate the mapping against REAL per-block bytes, not just + // prefix-sum consistency: decode block `i` alone and require it + // to equal the corresponding slice of the full decode. A + // sidecar that swapped sizes between adjacent blocks (same sum, + // same contiguity) would fail here. + let mut psrc = compressed.as_slice(); + let mut pdec = FrameDecoder::new(); + pdec.reset(&mut psrc).unwrap(); + let pd = pdec + .decode_blocks_partial(&mut psrc, i as u32, i as u32 + 1, None, false) + .unwrap(); + assert!( + pd.stopped_at.is_none(), + "block {i} must decode cleanly ({level:?})" + ); + assert_eq!( + pd.data.as_slice(), + &decoded[range.start as usize..range.end as usize], + "block {i} partial-decode bytes must equal the full-decode slice ({level:?})" + ); + expected_start = range.end; + } + assert_eq!( + expected_start, + decoded.len() as u64, + "block decompressed sizes must sum to the full decoded length ({level:?})" + ); + assert_eq!( + info.decompressed_byte_range(info.blocks.len()), + None, + "out-of-range index yields None ({level:?})" + ); + } +} + +/// ~400 KiB semi-repetitive payload (long runs interleaved with a stride +/// phrase) that compresses into several multi-block frames across levels. +#[cfg(feature = "lsm")] +fn emit_info_fixture_data() -> Vec { + let mut data: Vec = Vec::with_capacity(400 * 1024); + let mut x = 0x9E37_79B9u32; + while data.len() < 400 * 1024 { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + let run = 16 + (x as usize % 48); + let byte = (x >> 24) as u8; + for _ in 0..run { + data.push(byte); + } + data.extend_from_slice(b"the quick brown fox jumps over the lazy dog\n"); + } + data +} + +#[cfg(feature = "lsm")] +#[test] +fn frame_emit_info_decompressed_ranges_match_on_borrowed_oneshot_path() { + // The borrowed one-shot path (`compress_independent_frame` -> + // `run_borrowed_block_loop` -> `compress_block_encoded_borrowed`) + // threads the decompressed-size sidecar through a DIFFERENT emit site + // than the owned/streaming loop, so it needs its own per-block mapping + // check. A Fast level keeps the encoder on the borrowed-eligible + // (Simple matcher) path. + let data = emit_info_fixture_data(); + + let mut compressor: FrameCompressor = FrameCompressor::new(super::CompressionLevel::Fastest); + let compressed = compressor.compress_independent_frame(data.as_slice()); + let info = compressor + .last_frame_emit_info() + .expect("emit info populated after compress_independent_frame") + .clone(); + // Pin the compressed-block path: without this the fixture could regress + // into the raw-fast fallback and still pass via the Raw wire-size + // fallback in populate_frame_emit_info, never exercising the borrowed + // compressed-block sidecar capture this test targets. + assert!( + info.blocks + .iter() + .any(|b| matches!(b.block_type, crate::blocks::block::BlockType::Compressed)), + "borrowed-path fixture must emit at least one compressed block" + ); + assert!( + info.blocks.len() >= 2, + "borrowed fixture must span multiple blocks (got {})", + info.blocks.len() + ); + assert!(info.blocks.last().unwrap().last_block); + + // Full decode reference. + let mut decoder = FrameDecoder::new(); + let mut source = compressed.as_slice(); + decoder.reset(&mut source).unwrap(); + while !decoder.is_finished() { + decoder + .decode_blocks(&mut source, crate::decoding::BlockDecodingStrategy::All) + .unwrap(); + } + let mut decoded = Vec::new(); + decoder.collect_to_writer(&mut decoded).unwrap(); + assert_eq!(decoded, data, "borrowed one-shot frame must round-trip"); + + // Each block's mapping must match real per-block bytes. + let mut expected_start = 0u64; + for i in 0..info.blocks.len() { + let range = info.decompressed_byte_range(i).unwrap(); + assert_eq!(range.start, expected_start, "block {i} range contiguity"); + let mut psrc = compressed.as_slice(); + let mut pdec = FrameDecoder::new(); + pdec.reset(&mut psrc).unwrap(); + let pd = pdec + .decode_blocks_partial(&mut psrc, i as u32, i as u32 + 1, None, false) + .unwrap(); + assert!(pd.stopped_at.is_none(), "block {i} must decode cleanly"); + assert_eq!( + pd.data.as_slice(), + &decoded[range.start as usize..range.end as usize], + "borrowed block {i} partial-decode bytes must equal the full-decode slice" + ); + expected_start = range.end; + } + assert_eq!( + expected_start, + decoded.len() as u64, + "ranges sum to full length" + ); +} + +// The fuzz-artifact interop replay (C-compress -> our-decode and +// our-compress -> C-decode) moved to `ffi-bench/tests/fuzz_interop.rs` so +// the library crate never links libzstd. + +/// Homogeneous input — every byte the same — must NOT be split: +/// both border histograms are identical (all 512 hits on a single +/// slot), so `presplit_fingerprints_differ` returns `false` and the +/// function takes the early-return path at +/// `zstd_preSplit.c:214` returning `blockSize`. +#[test] +fn split_block_from_borders_keeps_homogeneous_block() { + let block = vec![0xAAu8; MAX_BLOCK_SIZE as usize]; + let split = super::split_block_from_borders(&block); + assert_eq!(split, MAX_BLOCK_SIZE as usize); +} + +/// Heterogeneous input — first half all zeros, second half a +/// counter sequence — has clearly distinguishable border +/// histograms, so the borders heuristic decides to split. +/// +/// The transition sits at exactly the block midpoint, so the +/// middle 512-byte sample (`block[mid-256..mid+256]`) is half +/// zeros + half counter values. That makes it roughly +/// equidistant from both border fingerprints — the +/// `abs_diff(dist_from_begin, dist_from_end) < min_distance` +/// branch fires and the heuristic returns the midpoint (64 KiB) +/// per `zstd_preSplit.c:222`. The test asserts the exact value +/// rather than just "one of {32K, 64K, 96K}" so a regression +/// to a different quantised arm cannot silently slip through. +#[test] +fn split_block_from_borders_returns_midpoint_for_centred_transition() { + let mut block = vec![0u8; MAX_BLOCK_SIZE as usize]; + for (i, byte) in block + .iter_mut() + .enumerate() + .skip(MAX_BLOCK_SIZE as usize / 2) + { + *byte = (i % 251 + 1) as u8; + } + let split = super::split_block_from_borders(&block); + assert_eq!( + split, + 64 * 1024, + "centred-transition fixture must take the symmetric \ + midpoint arm (`abs_diff < min_distance`), got {split}" + ); +} + +/// `level_pre_split` resolves the per-level split knob through the +/// `LevelParams` table, returning the EFFECTIVE upstream `ZSTD_splitBlock` +/// level (`splitLevels[strategy] - 2`, clamped at 0) that +/// `ZSTD_optimalBlockSize` dispatches: fast/dfast/greedy/lazy(d1) → 0 +/// (from-borders), lazy2/btlazy2 → 1 (byChunks rate 43), +/// btopt/btultra/btultra2 → 2 (byChunks rate 11). `Uncompressed` has no +/// numeric level so it stays `None`. +#[test] +fn pre_split_level_dispatches_by_compression_level() { + use crate::encoding::CompressionLevel; + use crate::encoding::levels::config::level_pre_split; + assert_eq!(level_pre_split(CompressionLevel::Uncompressed), None); + // Fastest = level 1 (fast) → 0 (from-borders). + assert_eq!(level_pre_split(CompressionLevel::Fastest), Some(0)); + // Default = level 3 (dfast) → 0 (splitLevels 1 - 2, clamped). + assert_eq!(level_pre_split(CompressionLevel::Default), Some(0)); + // Better is a pure alias for level 7 (lazy): same as Level(7). + assert_eq!( + level_pre_split(CompressionLevel::Better), + level_pre_split(CompressionLevel::Level(7)), + ); + // Best resolves to the level-13 table row (btlazy2): pin it to that + // numeric route so the named path can't drift from the pre-split + // table. + assert_eq!( + level_pre_split(CompressionLevel::Best), + level_pre_split(CompressionLevel::Level(13)), + ); + assert_eq!(level_pre_split(CompressionLevel::Level(2)), Some(0)); // fast + assert_eq!(level_pre_split(CompressionLevel::Level(4)), Some(0)); // dfast + assert_eq!(level_pre_split(CompressionLevel::Level(5)), Some(0)); // greedy + assert_eq!(level_pre_split(CompressionLevel::Level(7)), Some(0)); // lazy (depth 1) + // lazy2 / btlazy2: splitLevels 3 - 2 = 1 (byChunks rate 43, hashLog 8). + // The coarse byte-histogram tier is robust to the periodic-input + // phantom-split the rate-5/hashLog-10 tier suffered, so it matches + // upstream AND stays whole on periodic input (`periodic_stream_not_oversplit`). + assert_eq!(level_pre_split(CompressionLevel::Level(8)), Some(1)); // lazy2 lower bound + assert_eq!(level_pre_split(CompressionLevel::Level(11)), Some(1)); // lazy2 (depth 2) + assert_eq!(level_pre_split(CompressionLevel::Level(12)), Some(1)); // lazy2 upper bound + assert_eq!(level_pre_split(CompressionLevel::Level(13)), Some(1)); // btlazy2 lower bound + assert_eq!(level_pre_split(CompressionLevel::Level(15)), Some(1)); // btlazy2 (depth 2) + assert_eq!(level_pre_split(CompressionLevel::Level(16)), Some(2)); // btopt + assert_eq!(level_pre_split(CompressionLevel::Level(22)), Some(2)); // btultra2 +} + +/// Regression: a homogeneous but periodic multi-block stream must not be +/// pre-split into tiny blocks at the lazy2 / btlazy2 levels. The rate-5 +/// chunk sampler used to phantom-split such input at every 8 KB chunk, +/// cascading a large stream into hundreds of tiny blocks whose per-block +/// headers ballooned the output (~5x vs the lazy level next door). With +/// the rate-1 full-scan splitter the periodic stream is seen as uniform +/// and stays a few full blocks. We assert the lazy2 (L8) and btlazy2 (L15) +/// outputs stay within 2x of the lazy (L7) output on the same input, and +/// that every output round-trips. +#[test] +fn periodic_stream_not_oversplit() { + use crate::encoding::{CompressionLevel, compress_slice_to_vec}; + const LINES: &[&str] = &[ + "ts=2026-03-26T21:39:28Z level=INFO msg=\"flush memtable\" tenant=demo table=orders region=eu-west\n", + "ts=2026-03-26T21:39:29Z level=INFO msg=\"rotate segment\" tenant=demo table=orders region=eu-west\n", + "ts=2026-03-26T21:39:30Z level=INFO msg=\"compact level\" tenant=demo table=orders region=eu-west\n", + "ts=2026-03-26T21:39:31Z level=INFO msg=\"write block\" tenant=demo table=orders region=eu-west\n", + ]; + // 512 KB = 4 upstream zstd blocks, enough for the cascade to manifest. + let target = 512 * 1024usize; + let mut data = Vec::with_capacity(target); + let mut i = 0; + while data.len() < target { + let line = LINES[i % LINES.len()].as_bytes(); + let take = line.len().min(target - data.len()); + data.extend_from_slice(&line[..take]); + i += 1; + } + let l7 = compress_slice_to_vec(&data, CompressionLevel::Level(7)); // lazy depth1 + let l8 = compress_slice_to_vec(&data, CompressionLevel::Level(8)); // lazy2 + let l15 = compress_slice_to_vec(&data, CompressionLevel::Level(15)); // btlazy2 + assert!( + l8.len() < l7.len() * 2, + "lazy2 over-split periodic stream: l7={} l8={}", + l7.len(), + l8.len() + ); + assert!( + l15.len() < l7.len() * 2, + "btlazy2 over-split periodic stream: l7={} l15={}", + l7.len(), + l15.len() + ); + for out in [&l7, &l8, &l15] { + let mut decoder = FrameDecoder::new(); + let mut round = Vec::with_capacity(data.len()); + decoder + .decode_all_to_vec(out, &mut round) + .expect("decode periodic stream"); + assert_eq!(round, data, "periodic stream roundtrip mismatch"); + } +} + +/// End-to-end: a 256 KB payload whose SECOND 128 KB upstream zstd block carries +/// an intra-block fingerprint transition, compressed at Level(5) +/// (greedy, the pre-split path this revision routes through the cheap +/// chunk splitter), round-trips through the crate's own decoder. +/// +/// The transition lives in the second block on purpose: the upstream zstd +/// `savings < 3` gate skips splitting the first block (savings start at +/// 0), so the first block is a homogeneous compressible run that banks +/// savings, and the second block is the one whose intra-block transition +/// `split_block_by_chunks()` resolves into a sub-block boundary (the +/// `pending_input.split_off(...)` path). The test asserts that split +/// decision directly so it cannot silently stop exercising the path if +/// the fixture or params drift, then proves the emitted split frame +/// round-trips. Level 13 (lazy) no longer pre-splits, hence Level 5. +#[test] +fn greedy_chunk_split_roundtrips_through_own_decoder() { + use crate::encoding::CompressionLevel; + let mut data = vec![0u8; 256 * 1024]; + // First 128 KB: homogeneous low-entropy run (compressible, banks + // the savings the upstream zstd gate needs). Second 128 KB: low-entropy run + // for its first half, then a counter sequence: a clear intra-block + // fingerprint transition at the 192 KB midpoint for the chunk + // splitter to find. + for (i, byte) in data.iter_mut().enumerate() { + *byte = if i < 192 * 1024 { + (i & 0x07) as u8 + } else { + (i % 251 + 1) as u8 + }; + } + + // Directly assert the chunk splitter resolves the second block's + // intra-block transition into a sub-block boundary once savings have + // accrued (the compressible first block banks well over the gate). + let second_block = &data[128 * 1024..]; + let split = super::optimal_block_size( + CompressionLevel::Level(5), + second_block, + second_block.len(), + MAX_BLOCK_SIZE as usize, + 100, + ); + assert!( + split < MAX_BLOCK_SIZE as usize, + "second upstream zstd block must chunk-split at its intra-block transition, got {split}", + ); + + let mut compressed = Vec::new(); + let mut compressor = FrameCompressor::new(CompressionLevel::Level(5)); + compressor.set_source(data.as_slice()); + compressor.set_drain(&mut compressed); + compressor.compress(); + + let mut decoder = FrameDecoder::new(); + let mut source = compressed.as_slice(); + decoder + .reset(&mut source) + .expect("frame header should parse"); + while !decoder.is_finished() { + decoder + .decode_blocks(&mut source, crate::decoding::BlockDecodingStrategy::All) + .expect("decode should succeed"); + } + let mut decoded = Vec::with_capacity(data.len()); + decoder.collect_to_writer(&mut decoded).unwrap(); + assert_eq!(decoded, data, "roundtrip must reproduce the input verbatim"); +} + +/// Outside-diff coverage for the FAST one-shot path. +/// `compress_slice_to_vec` / `compress_independent_frame` on a Fast level +/// routes through `run_borrowed_block_loop` (not the owned loop the test +/// above covers), which must honour `optimal_block_size` and emit a +/// sub-`MAX_BLOCK_SIZE` boundary rather than fixed 128 KiB blocks. A +/// 256 KiB input is two 128 KiB blocks when unsplit; a chunk boundary in +/// the second block yields >= 3 decoded blocks, asserted on the round-trip. +#[test] +fn fast_oneshot_borrowed_split_emits_subblock() { + use crate::encoding::CompressionLevel; + // First 192 KiB: homogeneous zero run (banks the savings the split + // gate needs). The second 128 KiB block flips to a counter sequence + // at its 64 KiB midpoint (the 192 KiB mark) — a fingerprint + // transition the Fast from-borders splitter (split level 0) resolves + // into a sub-block boundary. + let mut data = vec![0u8; 256 * 1024]; + for (i, byte) in data.iter_mut().enumerate() { + if i >= 192 * 1024 { + *byte = (i % 251 + 1) as u8; + } + } + + // Pin the splitter decision for the Fast path directly (mirrors the + // greedy test): the second upstream zstd block must resolve to a sub-block + // boundary, so the >= 3 block count below cannot pass vacuously. + let second_block = &data[128 * 1024..]; + assert!( + super::optimal_block_size( + CompressionLevel::Fastest, + second_block, + second_block.len(), + MAX_BLOCK_SIZE as usize, + 100, + ) < MAX_BLOCK_SIZE as usize, + "fixture must resolve to a sub-block split in the second upstream zstd block", + ); + + // Drive the borrowed one-shot route explicitly (Fast level -> + // run_borrowed_block_loop via compress_independent_frame). + let mut compressor: FrameCompressor = FrameCompressor::new(CompressionLevel::Fastest); + let frame = compressor.compress_independent_frame(&data); + + let mut decoder = FrameDecoder::new(); + let mut source = frame.as_slice(); + decoder + .reset(&mut source) + .expect("frame header should parse"); + while !decoder.is_finished() { + decoder + .decode_blocks(&mut source, crate::decoding::BlockDecodingStrategy::All) + .expect("decode should succeed"); + } + let mut decoded = Vec::with_capacity(data.len()); + decoder.collect_to_writer(&mut decoded).unwrap(); + assert_eq!(decoded, data, "roundtrip must reproduce the input verbatim"); + assert!( + decoder.blocks_decoded() >= 3, + "fast one-shot borrowed path must split the second upstream zstd block \ + (256 KiB unsplit = 2 blocks), got {} blocks", + decoder.blocks_decoded(), + ); +} + +/// Regression: `set_compression_level` followed by `compress()` must +/// refresh `state.strategy_tag` through the reset-time sync so the +/// literal-compression gates (`min_literals_to_compress`, +/// `min_gain`) use the NEW level's strategy. Picks a level pair +/// that genuinely crosses strategy bands — `Fastest` resolves to +/// `Fast`, `Level(20)` resolves to `BtUltra2` — so a missed sync +/// would leave the construction-time tag visible and trip the +/// assertion. `CompressionLevel::Best` would also pass type-wise +/// but resolves to `Lazy` today, which keeps `min_literals_to_compress` +/// in the same `shift=3 → 64-byte` band as `Fast` and weakens the +/// signal that the gate floor actually moved. +#[cfg(feature = "std")] +#[test] +fn set_compression_level_then_compress_refreshes_strategy_tag() { + use super::CompressionLevel; + use crate::encoding::strategy::StrategyTag; + + let data = vec![0xABu8; 256]; + let mut out = Vec::new(); + let mut compressor = FrameCompressor::new(CompressionLevel::Fastest); + let initial_tag = compressor.state.strategy_tag; + assert_eq!( + initial_tag, + StrategyTag::for_compression_level(CompressionLevel::Fastest), + "construction-time strategy_tag must reflect initial level", + ); + + // Switch to a level whose resolved strategy lives in a different + // band, then run a full compress cycle — the matcher.reset() + // inside `compress` is the only site that can refresh the tag. + let new_level = CompressionLevel::Level(20); + compressor.set_compression_level(new_level); + compressor.set_source(data.as_slice()); + compressor.set_drain(&mut out); + compressor.compress(); + + let new_tag = compressor.state.strategy_tag; + let expected = StrategyTag::for_compression_level(new_level); + assert_eq!( + new_tag, expected, + "strategy_tag must follow set_compression_level → compress, \ + got {new_tag:?} expected {expected:?}", + ); + assert_eq!( + expected, + StrategyTag::BtUltra2, + "test fixture invariant: Level(20) must resolve to BtUltra2 \ + so the post-switch tag visibly crosses the band boundary", + ); + assert_ne!( + new_tag, initial_tag, + "test fixture invariant: chosen levels must resolve to \ + different StrategyTag variants", + ); +} + +/// Magicless mode (`ZSTD_f_zstd1_magicless`): encoded frame +/// MUST NOT start with the 4-byte magic prefix, AND must +/// round-trip through a magicless-aware decoder. +#[test] +fn magicless_frame_omits_magic_and_roundtrips() { + use crate::common::MAGIC_NUM; + let input: alloc::vec::Vec = (0..512u32).map(|i| (i ^ 0xA5) as u8).collect(); + + // Encode with magicless = true. + let mut output: Vec = Vec::new(); + let mut compressor = FrameCompressor::new(super::CompressionLevel::Default); + compressor.set_magicless(true); + compressor.set_source(input.as_slice()); + compressor.set_drain(&mut output); + compressor.compress(); + + // 1. Encoded output must NOT begin with the zstd magic number. + assert!( + !output.starts_with(&MAGIC_NUM.to_le_bytes()), + "magicless frame must omit the 4-byte magic prefix", + ); + + // 2. A magicless-aware decoder must round-trip the payload. + let mut decoder = crate::decoding::FrameDecoder::new(); + decoder.set_magicless(true); + let mut cursor: &[u8] = output.as_slice(); + decoder.init(&mut cursor).expect("magicless init"); + decoder + .decode_blocks(&mut cursor, crate::decoding::BlockDecodingStrategy::All) + .expect("decode_blocks"); + let mut decoded: Vec = Vec::new(); + decoder + .collect_to_writer(&mut decoded) + .expect("collect_to_writer"); + assert_eq!(decoded, input, "magicless roundtrip must preserve bytes"); + + // 3. A standard (magicful) decoder MUST reject a magicless + // frame at the header-read step — the first 4 bytes are + // the frame-header descriptor + window / dictionary / FCS + // metadata, not the magic. We accept either + // `BadMagicNumber` (typical case: first 4 bytes don't + // match `MAGIC_NUM` and don't fall in the skippable-frame + // magic range) or `SkipFrame` (rare: the first 4 bytes + // coincidentally land in `0x184D2A50..=0x184D2A5F`). Both + // prove the standard decoder did not treat the bytes as a + // real magicful frame. + use crate::decoding::errors::{FrameDecoderError, ReadFrameHeaderError}; + let mut std_decoder = crate::decoding::FrameDecoder::new(); + let std_init = std_decoder.init(output.as_slice()); + match std_init { + Err(FrameDecoderError::ReadFrameHeaderError( + ReadFrameHeaderError::BadMagicNumber(_) | ReadFrameHeaderError::SkipFrame { .. }, + )) => {} + other => panic!( + "standard decoder must reject a magicless frame with \ + ReadFrameHeaderError::BadMagicNumber or SkipFrame, got {other:?}", + ), + } +} + +/// A reused `FrameCompressor` must emit byte-identical frames to a +/// fresh compressor per input across both the borrowed (Fast) and +/// owned (Dfast/Lazy/Greedy/Uncompressed) backends. This proves +/// `prepare_frame` fully resets the per-frame state (matcher window, +/// content hasher, FSE/Huffman seeds) between independent frames; a +/// missed reset would corrupt frame N>=2's header checksum or matches. +/// Each emitted frame must also round-trip. +#[test] +fn compress_independent_frame_reuse_matches_fresh_and_roundtrips() { + use crate::encoding::{CompressionLevel, compress_slice_to_vec}; + let levels = [ + CompressionLevel::Uncompressed, + CompressionLevel::Fastest, + CompressionLevel::Default, + CompressionLevel::Better, + CompressionLevel::Best, + CompressionLevel::Level(5), + ]; + let inputs: Vec> = vec![ + Vec::new(), + vec![0x00], + b"the quick brown fox jumps over the lazy dog\n".to_vec(), + vec![0x7Eu8; 50_000], // highly compressible + generate_data(0xABCD, 70_000), // pseudo-random + generate_data(0x1234, 200_000), + ]; + for level in levels { + let mut cctx: FrameCompressor = FrameCompressor::new(level); + for data in &inputs { + let reused = cctx.compress_independent_frame(data); + let fresh = compress_slice_to_vec(data, level); + assert_eq!( + reused, + fresh, + "reused frame != fresh frame for len={} level={:?}", + data.len(), + level, + ); + let mut decoder = FrameDecoder::new(); + let mut decoded = Vec::with_capacity(data.len()); + decoder.decode_all_to_vec(&reused, &mut decoded).unwrap(); + assert_eq!( + decoded, + *data, + "roundtrip failed for len={} level={:?}", + data.len(), + level, + ); + } + } +} + +/// `compress_independent_frame_into` must replace (not append to) the +/// caller's buffer each call, so a smaller frame after a larger one +/// yields exactly the smaller frame, and the reused buffer's content +/// matches a fresh compression of the same input. +#[test] +fn compress_independent_frame_into_replaces_buffer_contents() { + use crate::encoding::{CompressionLevel, compress_slice_to_vec}; + let large = vec![0x11u8; 40_000]; + let small = b"short payload".to_vec(); + let mut cctx: FrameCompressor = FrameCompressor::new(CompressionLevel::Default); + let mut out = Vec::new(); + cctx.compress_independent_frame_into(&large, &mut out); + let frame_large = out.clone(); + // Reusing the same buffer for a smaller frame must clear it first. + cctx.compress_independent_frame_into(&small, &mut out); + assert_eq!( + out, + compress_slice_to_vec(&small, CompressionLevel::Default), + "reused buffer must hold exactly the second frame", + ); + // The first frame, captured before reuse, still round-trips. + let mut decoder = FrameDecoder::new(); + let mut decoded = Vec::with_capacity(large.len()); + decoder + .decode_all_to_vec(&frame_large, &mut decoded) + .unwrap(); + assert_eq!(decoded, large); +} + +/// A sticky dictionary set once on a reused compressor must be primed +/// into every independent frame (mirroring `ZSTD_CCtx_loadDictionary`): +/// each frame decodes with the dictionary and is byte-identical to a +/// fresh compressor carrying the same dictionary. This proves +/// `prepare_frame` re-primes the dictionary (matcher content + offset +/// history + entropy seed) every call rather than only on the first. +#[test] +fn compress_independent_frame_reuses_sticky_dictionary() { + use crate::encoding::CompressionLevel; + let dict_raw = include_bytes!("../../../dict_tests/dictionary"); + let dict_content = crate::decoding::Dictionary::decode_dict(dict_raw).unwrap(); + let mut payload_a = Vec::new(); + for _ in 0..8 { + payload_a.extend_from_slice(&dict_content.dict_content[..2048]); + } + let payload_b = b"a different second frame payload, still dict-attached".to_vec(); + let inputs = [payload_a, payload_b]; + + let mut cctx: FrameCompressor = FrameCompressor::new(CompressionLevel::Fastest); + cctx.set_dictionary_from_bytes(dict_raw) + .expect("dictionary bytes should parse"); + + for data in &inputs { + let reused = cctx.compress_independent_frame(data); + // Fresh compressor carrying the same sticky dictionary. + let mut fresh_enc: FrameCompressor = FrameCompressor::new(CompressionLevel::Fastest); + fresh_enc + .set_dictionary_from_bytes(dict_raw) + .expect("dictionary bytes should parse"); + let fresh = fresh_enc.compress_independent_frame(data); + assert_eq!( + reused, + fresh, + "reused dict frame != fresh dict frame, len={}", + data.len(), + ); + // Round-trip with the dictionary on the decode side. + let dict_for_decoder = crate::decoding::Dictionary::decode_dict(dict_raw).unwrap(); + let mut decoder = FrameDecoder::new(); + decoder.add_dict(dict_for_decoder).unwrap(); + let mut decoded = Vec::with_capacity(data.len()); + decoder.decode_all_to_vec(&reused, &mut decoded).unwrap(); + assert_eq!(&decoded, data, "dict roundtrip failed, len={}", data.len()); + } +} diff --git a/zstd/src/encoding/frame_header.rs b/zstd/src/encoding/frame_header.rs index 29d3ac203..a05983e85 100644 --- a/zstd/src/encoding/frame_header.rs +++ b/zstd/src/encoding/frame_header.rs @@ -159,142 +159,4 @@ fn write_fcs(val: u64, field_size: usize, output: &mut Vec) { } #[cfg(test)] -mod tests { - use super::FrameHeader; - use crate::decoding::frame::{FrameDescriptor, read_frame_header}; - use alloc::vec::Vec; - - #[test] - fn frame_header_descriptor_decode() { - let header = FrameHeader { - frame_content_size: Some(1), - single_segment: true, - content_checksum: false, - dictionary_id: None, - window_size: None, - magicless: false, - }; - let descriptor = header.descriptor(); - let decoded_descriptor = FrameDescriptor(descriptor); - assert_eq!(decoded_descriptor.frame_content_size_bytes().unwrap(), 1); - assert!(!decoded_descriptor.content_checksum_flag()); - assert_eq!(decoded_descriptor.dictionary_id_bytes().unwrap(), 0); - } - - #[test] - fn frame_header_decode() { - let header = FrameHeader { - frame_content_size: Some(1), - single_segment: true, - content_checksum: false, - dictionary_id: None, - window_size: None, - magicless: false, - }; - - let mut serialized_header = Vec::new(); - header.serialize(&mut serialized_header); - let parsed_header = read_frame_header(serialized_header.as_slice()).unwrap().0; - assert!(parsed_header.dictionary_id().is_none()); - assert_eq!(parsed_header.frame_content_size(), 1); - } - - // Locks the descriptor/FCS field-size class boundaries: 255/256 is the - // 1-byte (single-segment) vs 2-byte edge, 65791/65792 the 2-byte - // (+256 offset) vs 4-byte edge. A drift between `find_fcs_field_size`, - // the descriptor flag mapping, and the FCS write would round-trip to a - // wrong size through the decoder. - #[test] - fn frame_header_fcs_boundaries_round_trip() { - for (frame_content_size, single_segment) in [ - (255, true), - (256, true), - (65791, true), - (65792, true), - (255, false), - (256, false), - (65791, false), - (65792, false), - ] { - let header = FrameHeader { - frame_content_size: Some(frame_content_size), - single_segment, - content_checksum: false, - dictionary_id: None, - window_size: if single_segment { None } else { Some(1024) }, - magicless: false, - }; - - let mut serialized_header = Vec::new(); - header.serialize(&mut serialized_header); - let parsed_header = read_frame_header(serialized_header.as_slice()) - .expect("serialized header must parse") - .0; - assert_eq!( - parsed_header.frame_content_size(), - frame_content_size, - "FCS must round-trip at boundary {frame_content_size} (single_segment={single_segment})" - ); - } - } - - // The dictionary-id field-size classes (1/2/4 bytes by value magnitude) - // share the descriptor's bits 0-1; lock their boundaries through the - // decoder as well. - #[test] - fn frame_header_dictionary_id_boundaries_round_trip() { - for dictionary_id in [1, 255, 256, 65535, 65536, u32::MAX as u64] { - let header = FrameHeader { - frame_content_size: Some(4096), - single_segment: false, - content_checksum: false, - dictionary_id: Some(dictionary_id), - window_size: Some(1024), - magicless: false, - }; - - let mut serialized_header = Vec::new(); - header.serialize(&mut serialized_header); - let parsed_header = read_frame_header(serialized_header.as_slice()) - .expect("serialized header must parse") - .0; - assert_eq!( - parsed_header.dictionary_id(), - Some(dictionary_id as u32), - "dictionary id must round-trip at boundary {dictionary_id}" - ); - } - } - - #[test] - #[should_panic] - fn catches_single_segment_no_fcs() { - let header = FrameHeader { - frame_content_size: None, - single_segment: true, - content_checksum: false, - dictionary_id: None, - window_size: Some(1), - magicless: false, - }; - - let mut serialized_header = Vec::new(); - header.serialize(&mut serialized_header); - } - - #[test] - #[should_panic] - fn catches_single_segment_no_winsize() { - let header = FrameHeader { - frame_content_size: Some(7), - single_segment: false, - content_checksum: false, - dictionary_id: None, - window_size: None, - magicless: false, - }; - - let mut serialized_header = Vec::new(); - header.serialize(&mut serialized_header); - } -} +mod tests; diff --git a/zstd/src/encoding/frame_header/tests.rs b/zstd/src/encoding/frame_header/tests.rs new file mode 100644 index 000000000..55e42e242 --- /dev/null +++ b/zstd/src/encoding/frame_header/tests.rs @@ -0,0 +1,137 @@ +use super::FrameHeader; +use crate::decoding::frame::{FrameDescriptor, read_frame_header}; +use alloc::vec::Vec; + +#[test] +fn frame_header_descriptor_decode() { + let header = FrameHeader { + frame_content_size: Some(1), + single_segment: true, + content_checksum: false, + dictionary_id: None, + window_size: None, + magicless: false, + }; + let descriptor = header.descriptor(); + let decoded_descriptor = FrameDescriptor(descriptor); + assert_eq!(decoded_descriptor.frame_content_size_bytes().unwrap(), 1); + assert!(!decoded_descriptor.content_checksum_flag()); + assert_eq!(decoded_descriptor.dictionary_id_bytes().unwrap(), 0); +} + +#[test] +fn frame_header_decode() { + let header = FrameHeader { + frame_content_size: Some(1), + single_segment: true, + content_checksum: false, + dictionary_id: None, + window_size: None, + magicless: false, + }; + + let mut serialized_header = Vec::new(); + header.serialize(&mut serialized_header); + let parsed_header = read_frame_header(serialized_header.as_slice()).unwrap().0; + assert!(parsed_header.dictionary_id().is_none()); + assert_eq!(parsed_header.frame_content_size(), 1); +} + +// Locks the descriptor/FCS field-size class boundaries: 255/256 is the +// 1-byte (single-segment) vs 2-byte edge, 65791/65792 the 2-byte +// (+256 offset) vs 4-byte edge. A drift between `find_fcs_field_size`, +// the descriptor flag mapping, and the FCS write would round-trip to a +// wrong size through the decoder. +#[test] +fn frame_header_fcs_boundaries_round_trip() { + for (frame_content_size, single_segment) in [ + (255, true), + (256, true), + (65791, true), + (65792, true), + (255, false), + (256, false), + (65791, false), + (65792, false), + ] { + let header = FrameHeader { + frame_content_size: Some(frame_content_size), + single_segment, + content_checksum: false, + dictionary_id: None, + window_size: if single_segment { None } else { Some(1024) }, + magicless: false, + }; + + let mut serialized_header = Vec::new(); + header.serialize(&mut serialized_header); + let parsed_header = read_frame_header(serialized_header.as_slice()) + .expect("serialized header must parse") + .0; + assert_eq!( + parsed_header.frame_content_size(), + frame_content_size, + "FCS must round-trip at boundary {frame_content_size} (single_segment={single_segment})" + ); + } +} + +// The dictionary-id field-size classes (1/2/4 bytes by value magnitude) +// share the descriptor's bits 0-1; lock their boundaries through the +// decoder as well. +#[test] +fn frame_header_dictionary_id_boundaries_round_trip() { + for dictionary_id in [1, 255, 256, 65535, 65536, u32::MAX as u64] { + let header = FrameHeader { + frame_content_size: Some(4096), + single_segment: false, + content_checksum: false, + dictionary_id: Some(dictionary_id), + window_size: Some(1024), + magicless: false, + }; + + let mut serialized_header = Vec::new(); + header.serialize(&mut serialized_header); + let parsed_header = read_frame_header(serialized_header.as_slice()) + .expect("serialized header must parse") + .0; + assert_eq!( + parsed_header.dictionary_id(), + Some(dictionary_id as u32), + "dictionary id must round-trip at boundary {dictionary_id}" + ); + } +} + +#[test] +#[should_panic] +fn catches_single_segment_no_fcs() { + let header = FrameHeader { + frame_content_size: None, + single_segment: true, + content_checksum: false, + dictionary_id: None, + window_size: Some(1), + magicless: false, + }; + + let mut serialized_header = Vec::new(); + header.serialize(&mut serialized_header); +} + +#[test] +#[should_panic] +fn catches_single_segment_no_winsize() { + let header = FrameHeader { + frame_content_size: Some(7), + single_segment: false, + content_checksum: false, + dictionary_id: None, + window_size: None, + magicless: false, + }; + + let mut serialized_header = Vec::new(); + header.serialize(&mut serialized_header); +} diff --git a/zstd/src/encoding/hc/generator.rs b/zstd/src/encoding/hc/generator.rs new file mode 100644 index 000000000..fb64cffce --- /dev/null +++ b/zstd/src/encoding/hc/generator.rs @@ -0,0 +1,1261 @@ +//! HashChain / binary-tree match generator (`HcMatchGenerator`). +//! +//! The HC chain matcher plus the BT-backed optimal-parse machinery the lazy / +//! btopt / btultra strategies run on: the `HcBackend` discriminator, the +//! `HcMatchGenerator` storage + methods, the `btlazy2` cost helpers, the +//! optimal-plan body macros, and the `build_optimal_plan` driver. Moved +//! verbatim from `match_generator.rs` (no behaviour change); encoding-level +//! paths are absolute (`crate::encoding::…`) for the deeper module. + +use alloc::vec::Vec; + +use crate::encoding::Sequence; +use crate::encoding::blocks::encode_offset_with_history; +use crate::encoding::cost_model::HC_PREDEF_THRESHOLD; +use crate::encoding::hc::{HC_MIN_MATCH_LEN, MAX_HC_SEARCH_DEPTH}; +use crate::encoding::levels::config::HcConfig; +use crate::encoding::match_generator::{HC_SEARCH_DEPTH, HC_TARGET_LEN, HcBackend}; +use crate::encoding::match_table::storage::HC3_HASH_LOG; + +impl HcBackend { + /// Heap bytes held by the backend. `Hc` is zero-sized; `Bt` boxes a + /// `BtMatcher`, so count the boxed payload plus its own scratch heap. + pub(crate) fn heap_size(&self) -> usize { + match self { + Self::Hc => 0, + Self::Bt(bt) => core::mem::size_of::() + bt.heap_size(), + } + } + + /// Mutable accessor on the BT matcher; panics if the active + /// backend is `Hc`. The HC-or-Bt branches in orchestrator code use + /// `let HcBackend::Bt(bt) = &self.backend` directly for readonly + /// access — this helper exists so macro bodies that already drive + /// a mutable BT update through the optimal parser can write + /// `$self.backend.bt_mut().X` without an outer `match` ladder. + #[inline(always)] + pub(crate) fn bt_mut(&mut self) -> &mut crate::encoding::bt::BtMatcher { + match self { + Self::Bt(bt) => bt, + Self::Hc => unreachable!("BT-only accessor called in HC mode"), + } + } +} + +#[derive(Clone)] +pub(crate) struct HcMatchGenerator { + /// Shared match-finder storage (window, history, hash / chain / + /// hash3 tables, dictionary-priming flags). Used identically by HC + /// and BT modes; backend-specific table interpretation lives in the + /// matcher methods on this struct. + pub(crate) table: crate::encoding::match_table::storage::MatchTable, + /// HC runtime knobs (lazy_depth, search_depth, target_len). Always + /// present — BT modes still consult `hc.search_depth` for repcode + /// probing and chain candidate enumeration. + pub(crate) hc: crate::encoding::hc::HcMatcher, + /// Backend discriminator. [`HcBackend::Hc`] is zero-sized for the + /// lazy / lazy2 path so HC-only generators don't carry the BT + /// optimal-parser scratch buffers. [`HcBackend::Bt`] holds the + /// `BtMatcher` when an optimal mode is configured. + pub(crate) backend: HcBackend, + /// Compile-time strategy tag mirrored from + /// [`MatchGeneratorDriver::strategy_tag`] during `configure()`. + /// The driver hot path never reads this — it dispatches to + /// `compress_block::` from its own tag — but the + /// `#[cfg(test)] start_matching` helper consumes it so artificial + /// test setups still pick the correct concrete `S` for the + /// const-generic optimal parser (BtOpt vs BtUltra vs BtUltra2). + /// Without this field the test path would have to collapse + /// `BtOpt` and `BtUltra` onto the same monomorphisation since + /// `table.uses_bt` / `table.is_btultra2` alone can't tell them + /// apart. + pub(crate) strategy_tag: crate::encoding::strategy::StrategyTag, +} + +// Plain-data types relocated to [`crate::encoding::opt::types`] and +// [`crate::encoding::opt::ldm`] by #111 Phase 1. The use statements at +// the top of this file bring them back into scope so the existing +// methods on `HcMatchGenerator` compile unchanged. + +/// `bt_insert_step_no_rebase` body parameterized over the per-CPU +/// `count_match_from_indices` symbol. Each kernel-specific wrapper invokes +/// the macro with its own `fastpath::::count_match_from_indices` +/// path so the call resolves inside the wrapper's `#[target_feature]` +/// umbrella and inlines instead of paying the function-call ABI per BT walk +/// iteration. Used only by `HcMatchGenerator` BT walk wrappers below. +/// +/// Crate-private: the macro body references private `encoding::*` +/// modules via `$crate::...`, so it is unusable downstream and is +/// re-exported only inside this crate via `pub(crate) use` below. +macro_rules! bt_insert_step_no_rebase_body { + ($table:expr, $search_depth:expr, $abs_pos:ident, $current_abs_end:ident, $target_abs:ident, $cmf:path) => {{ + let idx = $abs_pos - $table.history_abs_start; + // Borrowed-aware live region (owned: `history[history_start..]`; + // borrowed: the in-place input `[0, block_end)`). Reborrow-then-raw-ptr + // so the slice holds NO borrow and coexists with the `&mut $table` + // binary-tree writes below. Owned is byte-identical (same bytes). + let concat: &[u8] = unsafe { + let lh = $table.live_history(); + core::slice::from_raw_parts(lh.as_ptr(), lh.len()) + }; + if idx + 8 > concat.len() { + return 1; + } + debug_assert!( + $abs_pos <= $current_abs_end, + "BT walker called past current block end" + ); + let tail_limit = $current_abs_end - $abs_pos; + let hash = $crate::encoding::match_table::storage::MatchTable::hash_position_at( + concat, + idx, + $table.hash_log, + $table.search_mls, + ); + // Prefetch the hash bucket now. For the large L16+ hash table over + // high-entropy input the bucket is L3/DRAM-cold, and unlike upstream's + // monolithic ZSTD_btGetAllMatches (which overlaps this miss with its + // inline rep/hash3 prologue) the read+write of `hash_table[hash]` + // below is reached with nothing to hide it behind — it stalled a large + // share of this function's cycles. Issuing the hint here lets the miss + // overlap the address setup that follows. + #[cfg(all( + target_feature = "sse", + any(target_arch = "x86", target_arch = "x86_64") + ))] + { + #[cfg(target_arch = "x86")] + use core::arch::x86::{_MM_HINT_T0, _mm_prefetch}; + #[cfg(target_arch = "x86_64")] + use core::arch::x86_64::{_MM_HINT_T0, _mm_prefetch}; + // SAFETY: prefetch is a hint that never faults; `hash` indexes + // `hash_table` directly below, so it is in bounds. + unsafe { + _mm_prefetch($table.hash_table.as_ptr().add(hash).cast(), _MM_HINT_T0); + } + // Prefetch the NEXT position's bucket too. The optimal-parser DP + // advances one position per iteration, so this miss is issued a + // full BT walk plus the next iteration's pre-collect work ahead of + // the collect that will read it — far more lead than the same-call + // hint above, enough to hide the full DRAM latency. + if idx + 1 + 8 <= concat.len() { + let hash_next = + $crate::encoding::match_table::storage::MatchTable::hash_position_at( + concat, + idx + 1, + $table.hash_log, + $table.search_mls, + ); + // SAFETY: prefetch never faults; an out-of-range index is a + // harmless no-op hint. + unsafe { + _mm_prefetch( + $table.hash_table.as_ptr().add(hash_next).cast(), + _MM_HINT_T0, + ); + } + } + } + let Some(relative_pos) = $table.relative_position($abs_pos) else { + return 1; + }; + let stored = relative_pos + 1; + let bt_mask = $table.bt_mask(); + // `abs_pos < bt_mask` legitimately happens for the first BT walk of + // a fresh frame (bt_low effectively "no floor"). Saturating keeps + // the floor at 0 so the `candidate_abs <= bt_low` check never + // triggers early; raw subtraction would underflow into a huge + // sentinel that ALWAYS triggers. + let bt_low = $abs_pos.saturating_sub(bt_mask); + // Hoist the BT pointer-pair base out of `self` once — see the + // collect-matches body for the full rationale (per-step Vec reload + + // bounds check through `&mut self` vs the upstream zstd's raw `U32*` walk). + let chain_ptr = $table.chain_table.as_mut_ptr(); + debug_assert_eq!($table.chain_table.len(), 2 << $table.bt_log()); + let window_low = $table.window_low_abs_for_target($target_abs); + // `abs_pos + 9` is safe in raw form: `MatchTable::add_data` caps + // total input at `usize::MAX - STREAM_ABS_HEADROOM` (where + // `STREAM_ABS_HEADROOM = HC_OPT_NUM + 16`), so every + // frame-lifetime absolute cursor passed to the BT walker stays + // below `usize::MAX - 9` regardless of stream length or + // pointer width. The guard is hoisted to the data-ingest + // boundary so this per-position site pays zero arithmetic + // overhead in the hot loop. + let mut match_end_abs = $abs_pos + 9; + let mut best_len = 8usize; + let mut compares_left = $search_depth; + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + let pair_idx = $table.bt_pair_index_for_abs($abs_pos); + let mut smaller_slot = pair_idx; + let mut larger_slot = pair_idx + 1; + let mut match_stored = $table.hash_table[hash]; + $table.hash_table[hash] = stored; + + while compares_left > 0 { + if match_stored == $crate::encoding::match_table::storage::HC_EMPTY { + break; + } + // Reject stale post-rebase slots whose pre-shift position is below + // `index_shift` explicitly. A `wrapping_sub` maps such a slot to a + // near-`usize::MAX` value that the `>= abs_pos` test only rejects + // while `abs_pos` is far from the integer ceiling; on a + // long-running rebased stream (reachable on 32-bit) `abs_pos` can + // approach the ceiling and the wrapped value can land back inside + // `[window_low, abs_pos)`. `checked_sub` ends the walk on the + // underflow instead. `match_stored != HC_EMPTY` here, so the `- 1` + // cannot underflow. + let Some(candidate_abs) = ($table.position_base + (match_stored as usize - 1)) + .checked_sub($table.index_shift) + else { + break; + }; + if candidate_abs < window_low || candidate_abs >= $abs_pos { + break; + } + compares_left -= 1; + + let next_pair_idx = $table.bt_pair_index_for_abs(candidate_abs); + // SAFETY: `next_pair_idx (+1)` = `2*(candidate_abs & bt_mask) (+1)` + // ≤ `chain_table.len()-1`; `chain_ptr` is the hoisted live base, + // table not realloc'd during the walk. + let next_smaller = unsafe { *chain_ptr.add(next_pair_idx) }; + let next_larger = unsafe { *chain_ptr.add(next_pair_idx + 1) }; + let seed_len = common_length_smaller.min(common_length_larger); + let candidate_idx = candidate_abs - $table.history_abs_start; + // SAFETY: BT walk invariant — `candidate_idx + tail_limit ≤ + // concat.len()` since the candidate is within + // `[history_abs_start, abs_pos)` and `tail_limit ≤ + // current_abs_end - abs_pos`. + let match_len = unsafe { $cmf(concat, idx, candidate_idx, tail_limit, seed_len) }; + + if match_len > best_len { + best_len = match_len; + // `candidate_abs + match_len <= current_abs_end` by BT walk + // invariant — `match_len <= tail_limit = current_abs_end - + // abs_pos` and `candidate_abs < abs_pos`. + let candidate_end = candidate_abs + match_len; + if candidate_end > match_end_abs { + match_end_abs = candidate_end; + } + } + + if match_len >= tail_limit { + break; + } + + let candidate_next = candidate_idx + match_len; + let current_next = idx + match_len; + // SAFETY: first-differing positions after a match_len-long prefix; + // match_len < tail_limit (break above) + BT-walk bound + // idx/candidate_idx + tail_limit <= concat.len() keep both in range. + if unsafe { + *concat.get_unchecked(candidate_next) < *concat.get_unchecked(current_next) + } { + // SAFETY: `smaller_slot` holds a valid pair index (init + // `pair_idx`, updated to `next_pair_idx + 1`); the `usize::MAX` + // sentinel is set only just before `break`, never written here. + unsafe { *chain_ptr.add(smaller_slot) = match_stored }; + common_length_smaller = match_len; + if candidate_abs <= bt_low { + smaller_slot = usize::MAX; + break; + } + smaller_slot = next_pair_idx + 1; + match_stored = next_larger; + } else { + // SAFETY: as above for `larger_slot`. + unsafe { *chain_ptr.add(larger_slot) = match_stored }; + common_length_larger = match_len; + if candidate_abs <= bt_low { + larger_slot = usize::MAX; + break; + } + larger_slot = next_pair_idx; + match_stored = next_smaller; + } + } + + // SAFETY: both slots, when not the `usize::MAX` sentinel, hold valid + // pair indices into the hoisted `chain_table` base. + if smaller_slot != usize::MAX { + unsafe { + *chain_ptr.add(smaller_slot) = $crate::encoding::match_table::storage::HC_EMPTY + }; + } + if larger_slot != usize::MAX { + unsafe { + *chain_ptr.add(larger_slot) = $crate::encoding::match_table::storage::HC_EMPTY + }; + } + + let speed_positions = if best_len > 384 { + (best_len - 384).min(192) + } else { + 0 + }; + // `match_end_abs` is initialized to `abs_pos + 9` and is only + // reassigned inside the `candidate_end > match_end_abs` branch + // above. So even though an individual `candidate_end = + // candidate_abs + match_len` can land below `abs_pos` (the + // candidate sits earlier in history and the match runs short), + // the variable itself never drops below its initial value. + // That gives `match_end_abs ≥ abs_pos + 9 > abs_pos + 8` as a + // loop-wide invariant, so the raw subtraction below cannot + // underflow. + speed_positions.max(match_end_abs - ($abs_pos + 8)) + }}; +} +pub(crate) use bt_insert_step_no_rebase_body; + +/// `hash3_candidate` body parameterized over the per-CPU +/// `common_prefix_len_ptr` symbol. The hash3 probe checks one candidate per +/// position when invoked, so the per-call ABI savings compound across the +/// segment. Crate-private (see `bt_insert_step_no_rebase_body!`). +macro_rules! hash3_candidate_body { + ( + $table:expr, + $abs_pos:ident, + $current_abs_end:ident, + $min_match_len:ident, + $cpl:path $(,)? + ) => {{ + if $table.hash3_log == 0 { + return None; + } + let idx = $abs_pos.checked_sub($table.history_abs_start)?; + let concat = $table.live_history(); + if idx + 4 > concat.len() { + return None; + } + let hash3 = $crate::encoding::match_table::storage::MatchTable::hash_position_at( + concat, + idx, + $table.hash3_log, + 3, + ); + let entry = $table + .hash3_table + .get(hash3) + .copied() + .unwrap_or($crate::encoding::match_table::storage::HC_EMPTY); + let candidate_abs = + $crate::encoding::match_table::storage::MatchTable::stored_abs_position_fast( + entry, + $table.position_base, + $table.index_shift, + )?; + if candidate_abs < $table.history_abs_start || candidate_abs >= $abs_pos { + return None; + } + let offset = $abs_pos - candidate_abs; + if offset >= $crate::encoding::bt::HC3_MAX_OFFSET { + return None; + } + let candidate_idx = candidate_abs - $table.history_abs_start; + let tail_limit = $current_abs_end.saturating_sub($abs_pos); + let base = concat.as_ptr(); + // SAFETY: candidate/idx are within history range; tail_limit + // bounds the scan within `concat`. + let match_len = unsafe { $cpl(base.add(candidate_idx), base.add(idx), tail_limit) }; + (match_len >= $min_match_len).then_some($crate::encoding::opt::types::MatchCandidate { + start: $abs_pos, + offset, + match_len, + }) + }}; +} +pub(crate) use hash3_candidate_body; + +/// `for_each_repcode_candidate_with_reps` body parameterized over the per-CPU +/// `common_prefix_len_ptr` symbol so the per-rep prefix probe inlines under +/// the wrapper's `target_feature` umbrella instead of crossing the ABI +/// boundary through the dispatcher. Three rep probes per encoded position → +/// thousands per segment, so the per-call barrier was non-trivial. +/// +/// The callback `f` runs in the wrapper's umbrella context too, so closures +/// that capture mutable state still work (FnMut). Crate-private +/// (see `bt_insert_step_no_rebase_body!`). +macro_rules! for_each_repcode_candidate_body { + ( + $table:expr, + $abs_pos:ident, + $lit_len:ident, + $reps:ident, + $current_abs_end:ident, + $min_match_len:ident, + $f:ident, + $cpl:path $(,)? + ) => {{ + let rep_offsets: [Option; 3] = if $lit_len == 0 { + [ + Some($reps[1] as usize), + Some($reps[2] as usize), + ($reps[0] > 1).then_some(($reps[0] - 1) as usize), + ] + } else { + [ + Some($reps[0] as usize), + Some($reps[1] as usize), + Some($reps[2] as usize), + ] + }; + let concat = $table.live_history(); + let current_idx = $abs_pos - $table.history_abs_start; + if current_idx + 4 > concat.len() { + return; + } + let tail_limit = $current_abs_end.saturating_sub($abs_pos); + let base = concat.as_ptr(); + let concat_len = concat.len(); + for rep in rep_offsets.into_iter().flatten() { + if rep == 0 || rep > $abs_pos { + continue; + } + let candidate_pos = $abs_pos - rep; + if candidate_pos < $table.history_abs_start { + continue; + } + let candidate_idx = candidate_pos - $table.history_abs_start; + // Upstream zstd `ZSTD_readMINMATCH` gate (zstd_opt.c:657-674): a + // 4-byte (3-byte when min_match_len == 3) equality probe + // before the full prefix scan. Equivalent filtering — a + // mismatch here means `match_len < min_match_len`, which + // the post-scan check rejects anyway — but it skips the + // prefix-kernel call for the common no-match case (rep + // offsets rarely hit on low-redundancy input). + // + // SAFETY: `current_idx + 4 <= concat_len` (early return + // above) and `candidate_idx < current_idx` (rep >= 1), so + // both 4-byte reads stay inside `concat`. + let gate_matches = unsafe { + let cand = base.add(candidate_idx).cast::().read_unaligned(); + let cur = base.add(current_idx).cast::().read_unaligned(); + if $min_match_len == 3 { + // Compare the low-address 3 bytes regardless of + // endianness: byte-shift on LE, mask via to_le. + (cand.to_le() & 0x00FF_FFFF) == (cur.to_le() & 0x00FF_FFFF) + } else { + cand == cur + } + }; + if !gate_matches { + continue; + } + // SAFETY: `candidate_idx ≤ current_idx < concat_len` (since + // candidate_pos ≤ abs_pos and we early-returned on + // `current_idx + 4 > concat_len`). `max` clamps to the shorter + // remaining run so neither pointer overruns `concat`. + let max = (concat_len - candidate_idx) + .min(concat_len - current_idx) + .min(tail_limit); + let match_len = unsafe { $cpl(base.add(candidate_idx), base.add(current_idx), max) }; + if match_len < $min_match_len { + continue; + } + $f(MatchCandidate { + start: $abs_pos, + offset: rep, + match_len, + }); + } + }}; +} +pub(crate) use for_each_repcode_candidate_body; + +/// `bt_insert_and_collect_matches` body parameterized over the per-CPU +/// `count_match_from_indices` symbol. Same shape as +/// [`bt_insert_step_no_rebase_body`] — picks up the matching kernel through +/// `$cmf` so the per-iteration vector probe inlines under the wrapper's +/// `target_feature` umbrella. Returns nothing (matches the original method). +/// Crate-private (see `bt_insert_step_no_rebase_body!`). +macro_rules! bt_insert_and_collect_matches_body { + ( + $table:expr, + $search_depth:expr, + $abs_pos:ident, + $current_abs_end:ident, + $profile:ident, + $min_match_len:ident, + $best_len_for_skip:ident, + $out:ident, + $cmf:path $(,)? + ) => {{ + let idx = $abs_pos - $table.history_abs_start; + // Borrowed-aware live region (owned: `history[history_start..]`; + // borrowed: the in-place input `[0, block_end)`). Reborrow-then-raw-ptr + // so the slice holds NO borrow and coexists with the `&mut $table` + // binary-tree writes below. Owned is byte-identical (same bytes). + let concat: &[u8] = unsafe { + let lh = $table.live_history(); + core::slice::from_raw_parts(lh.as_ptr(), lh.len()) + }; + if idx + 8 > concat.len() { + return; + } + debug_assert!( + $abs_pos <= $current_abs_end, + "BT collect called past current block end" + ); + let tail_limit = $current_abs_end - $abs_pos; + let hash = $crate::encoding::match_table::storage::MatchTable::hash_position_at( + concat, + idx, + $table.hash_log, + $table.search_mls, + ); + // Prefetch the hash bucket now. For the large L16+ hash table over + // high-entropy input the bucket is L3/DRAM-cold, and unlike upstream's + // monolithic ZSTD_btGetAllMatches (which overlaps this miss with its + // inline rep/hash3 prologue) the read+write of `hash_table[hash]` + // below is reached with nothing to hide it behind — it stalled a large + // share of this function's cycles. Issuing the hint here lets the miss + // overlap the address setup that follows. + #[cfg(all( + target_feature = "sse", + any(target_arch = "x86", target_arch = "x86_64") + ))] + { + #[cfg(target_arch = "x86")] + use core::arch::x86::{_MM_HINT_T0, _mm_prefetch}; + #[cfg(target_arch = "x86_64")] + use core::arch::x86_64::{_MM_HINT_T0, _mm_prefetch}; + // SAFETY: prefetch is a hint that never faults; `hash` indexes + // `hash_table` directly below, so it is in bounds. + unsafe { + _mm_prefetch($table.hash_table.as_ptr().add(hash).cast(), _MM_HINT_T0); + } + // Prefetch the NEXT position's bucket too. The optimal-parser DP + // advances one position per iteration, so this miss is issued a + // full BT walk plus the next iteration's pre-collect work ahead of + // the collect that will read it — far more lead than the same-call + // hint above, enough to hide the full DRAM latency. + if idx + 1 + 8 <= concat.len() { + let hash_next = + $crate::encoding::match_table::storage::MatchTable::hash_position_at( + concat, + idx + 1, + $table.hash_log, + $table.search_mls, + ); + // SAFETY: prefetch never faults; an out-of-range index is a + // harmless no-op hint. + unsafe { + _mm_prefetch( + $table.hash_table.as_ptr().add(hash_next).cast(), + _MM_HINT_T0, + ); + } + } + } + let Some(relative_pos) = $table.relative_position($abs_pos) else { + return; + }; + let stored = relative_pos + 1; + let bt_mask = $table.bt_mask(); + // Hoist the BT pointer-pair table's base out of `self` once: every + // access below is `chain_table[computed_index]` through `&mut self`, + // which the optimizer cannot prove loop-invariant, so it reloads the + // Vec's (ptr,len) from the struct AND bounds-checks on every tree + // step (the upstream zstd walks a raw `U32* btable`, zstd_opt.c). The raw + // base carries no borrow, so the `&self` helper calls in the loop + // (`bt_pair_index_for_abs`, `window_low_abs_for_target`, + // `relative_position`) coexist — they read other fields, never + // `chain_table`. Indices are in bounds by the BT invariants: + // `bt_pair_index_for_abs` returns `2*(abs & bt_mask) (+1)` ≤ + // `chain_table.len()-1`, and the slots only ever hold those values. + let chain_ptr = $table.chain_table.as_mut_ptr(); + debug_assert_eq!($table.chain_table.len(), 2 << $table.bt_log()); + // See `bt_insert_step_no_rebase_body!`: saturating is needed for the + // first BT walk of a fresh frame where `abs_pos < bt_mask`. + let bt_low = $abs_pos.saturating_sub(bt_mask); + let window_low = $table.window_low_abs_for_target($abs_pos); + // Upstream zstd-style window bound in stored space so the BT-walk loop + // condition rejects out-of-window / HC_EMPTY candidates WITHOUT + // decoding them (mirrors upstream `while ... matchIndex >= matchLow`): + // one range check on `match_stored` instead of decode-then-break, + // dropping the wasted candidate_abs decode on every walk's terminating + // step. candidate_abs(s) = (position_base + s - 1) - index_shift = + // base + s (wrapping); in-window ⟺ candidate_abs - window_low < + // abs_pos - window_low ⟺ s.wrapping_add(win_off) < win_range. + // HC_EMPTY (s = 0) maps to base = (lowest representable abs) - 1 < + // window_low, so it falls out of range and ends the walk. + let win_off = $table + .position_base + .wrapping_sub(1) + .wrapping_sub($table.index_shift) + .wrapping_sub(window_low); + let win_range = $abs_pos - window_low; + // Raw `+ 9` is safe here — see `bt_insert_step_no_rebase_body!` + // for the full discussion of the upstream `STREAM_ABS_HEADROOM` + // cap in `MatchTable::add_data`. + let mut match_end_abs = $abs_pos + 9; + let mut compares_left = $profile.max_chain_depth.min($search_depth); + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + let pair_idx = $table.bt_pair_index_for_abs($abs_pos); + let mut smaller_slot = pair_idx; + let mut larger_slot = pair_idx + 1; + let mut match_stored = $table.hash_table[hash]; + $table.hash_table[hash] = stored; + // Upstream zstd semantics: `bestLength` starts at `lengthToBeat - 1`; rep/hash3 + // probing may raise it; BT then only reports strictly longer matches. + // `min_match_len >= HC_FORMAT_MINMATCH (3)` by configure invariant, + // so `min_match_len - 1 >= 2` cannot underflow. + debug_assert!( + $min_match_len >= $crate::encoding::cost_model::HC_FORMAT_MINMATCH, + "min_match_len must be at least HC_FORMAT_MINMATCH" + ); + let mut best_len = (*$best_len_for_skip).max($min_match_len - 1); + + // Upstream zstd-form loop condition: the stored-space window range check + // (`s.wrapping_add(win_off) < win_range`) rejects out-of-window and + // HC_EMPTY candidates here, so the terminating step never enters the + // body — no wasted candidate_abs decode, matching upstream's + // `while ... matchIndex >= matchLow`. + while compares_left > 0 && (match_stored as usize).wrapping_add(win_off) < win_range { + compares_left -= 1; + // The condition proved this candidate is in `[window_low, + // abs_pos)`, so `match_stored >= 1` (HC_EMPTY is out of range) and + // the `- 1` cannot underflow; candidate_abs == base + match_stored. + let candidate_abs = ($table.position_base + (match_stored as usize - 1)) + .wrapping_sub($table.index_shift); + + let next_pair_idx = $table.bt_pair_index_for_abs(candidate_abs); + // SAFETY: `next_pair_idx (+1)` = `2*(candidate_abs & bt_mask) (+1)` + // ≤ `chain_table.len()-1`; `chain_ptr` is the hoisted live base, + // table not realloc'd during the walk. + let next_smaller = unsafe { *chain_ptr.add(next_pair_idx) }; + let next_larger = unsafe { *chain_ptr.add(next_pair_idx + 1) }; + let seed_len = common_length_smaller.min(common_length_larger); + let candidate_idx = candidate_abs - $table.history_abs_start; + // SAFETY: BT walk invariant — `candidate_idx + tail_limit ≤ + // concat.len()`. + let match_len = unsafe { $cmf(concat, idx, candidate_idx, tail_limit, seed_len) }; + + if match_len > best_len { + let offset = $abs_pos - candidate_abs; + let accepted = $crate::encoding::bt::BtMatcher::push_candidate_ladder( + $out, + $best_len_for_skip, + $crate::encoding::opt::types::MatchCandidate { + start: $abs_pos, + offset, + match_len, + }, + $min_match_len, + ); + if accepted { + best_len = match_len; + // BT walker invariants: `candidate_abs < abs_pos` + // and `match_len <= tail_limit = current_abs_end - + // abs_pos`. So `candidate_abs + match_len < + // abs_pos + tail_limit = current_abs_end`, which + // fits in `usize` on every supported target (32-bit + // i686 included) — the addition stays within the + // current block. + let candidate_end = candidate_abs + match_len; + if candidate_end > match_end_abs { + match_end_abs = candidate_end; + } + if match_len >= tail_limit + || match_len > $crate::encoding::cost_model::HC_OPT_NUM + { + break; + } + } + } + + if match_len >= tail_limit { + break; + } + + let candidate_next = candidate_idx + match_len; + let current_next = idx + match_len; + // SAFETY: first-differing positions after a match_len-long prefix; + // match_len < tail_limit (break above) + BT-walk bound + // idx/candidate_idx + tail_limit <= concat.len() keep both in range. + if unsafe { + *concat.get_unchecked(candidate_next) < *concat.get_unchecked(current_next) + } { + // SAFETY: `smaller_slot` holds a valid pair index (init + // `pair_idx`, updated to `next_pair_idx + 1`); the `usize::MAX` + // sentinel is set only just before `break`, never written here. + unsafe { *chain_ptr.add(smaller_slot) = match_stored }; + common_length_smaller = match_len; + if candidate_abs <= bt_low { + smaller_slot = usize::MAX; + break; + } + smaller_slot = next_pair_idx + 1; + match_stored = next_larger; + } else { + // SAFETY: as above for `larger_slot`. + unsafe { *chain_ptr.add(larger_slot) = match_stored }; + common_length_larger = match_len; + if candidate_abs <= bt_low { + larger_slot = usize::MAX; + break; + } + larger_slot = next_pair_idx; + match_stored = next_smaller; + } + } + + // SAFETY: both slots, when not the `usize::MAX` sentinel, hold valid + // pair indices into the hoisted `chain_table` base. + if smaller_slot != usize::MAX { + unsafe { + *chain_ptr.add(smaller_slot) = $crate::encoding::match_table::storage::HC_EMPTY + }; + } + if larger_slot != usize::MAX { + unsafe { + *chain_ptr.add(larger_slot) = $crate::encoding::match_table::storage::HC_EMPTY + }; + } + + // Dict dual-probe (upstream zstd `ZSTD_dictMatchState`, zstd_opt.c:777-813): + // after the live tree, descend the immutable dictionary BINARY TREE + // (built in `prime_dms_bt`) with its OWN compare budget and push any + // dict match longer than the live best into the ladder. The DUBT + // descent reaches the longest dict match efficiently (a hash-chain + // surfaced only the few same-bucket candidates and left most of the + // dict savings unrealised at btlazy2 / btopt). Dict positions are + // dictionary-relative concat indices in `[0, region)`, pinned at the + // front of history, so a dict candidate at `dict_idx` sits at offset + // `idx - dict_idx` (no upstream zstd `dmsIndexDelta`). The optimal parser + // prices these (its DP lookahead values the repcode chain a dict match + // seeds); the greedy/lazy parser commits the longest. + if let Some(dms) = $table.dms.table() { + let region = $table.dms.region_len(); + let dh = $crate::encoding::match_table::storage::MatchTable::hash_position_at( + concat, + idx, + dms.hash_log, + dms.mls, + ); + let mut dcur = dms.hash_table[dh]; + // DUBT seed lengths: bytes already known common on each side, so + // `$cmf` resumes from there (upstream zstd commonLengthSmaller/Larger). + let mut common_smaller = 0usize; + let mut common_larger = 0usize; + let mut dms_compares = $profile.max_chain_depth.min($search_depth); + while dms_compares > 0 && dcur != $crate::encoding::match_table::storage::HC_EMPTY { + let dict_idx = (dcur - 1) as usize; + // The dict tree holds only dict positions (`< region <= idx`). + if dict_idx >= region || dict_idx >= idx { + break; + } + dms_compares -= 1; + let pair = 2 * dict_idx; + let seed = common_smaller.min(common_larger); + // SAFETY: `dict_idx < idx` and `idx + tail_limit <= + // concat.len()` (checked at entry); same umbrella as the live + // walk's `$cmf`. `seed <= prior match_len <= tail_limit`. + let match_len = unsafe { $cmf(concat, idx, dict_idx, tail_limit, seed) }; + if match_len > best_len { + let offset = idx - dict_idx; + let accepted = $crate::encoding::bt::BtMatcher::push_candidate_ladder( + $out, + $best_len_for_skip, + $crate::encoding::opt::types::MatchCandidate { + start: $abs_pos, + offset, + match_len, + }, + $min_match_len, + ); + if accepted { + best_len = match_len; + let candidate_end = $abs_pos + match_len; + if candidate_end > match_end_abs { + match_end_abs = candidate_end; + } + if match_len > $crate::encoding::cost_model::HC_OPT_NUM { + break; + } + } + } + // Match reached the block tail: can't order the pair (upstream zstd + // `ip+matchLength == iLimit`), and indexing `concat[idx + + // match_len]` below would step past the searchable region. + if match_len >= tail_limit { + break; + } + // Descend the DUBT (upstream zstd zstd_opt.c:806-811): dict candidate + // smaller than input → its larger child is closer to `idx`. + if concat[dict_idx + match_len] < concat[idx + match_len] { + common_smaller = match_len; + dcur = dms.chain_table[pair + 1]; + } else { + common_larger = match_len; + dcur = dms.chain_table[pair]; + } + } + } + + // `match_end_abs >= abs_pos + 9 >= 9` (initialized and monotonic), + // so `match_end_abs - 8 >= 1` cannot underflow. + $table.skip_insert_until_abs = match_end_abs - 8; + }}; +} +pub(crate) use bt_insert_and_collect_matches_body; + +impl HcMatchGenerator { + /// Heap bytes this generator owns: the shared match table plus the BT + /// backend's optimal-parser / LDM scratch (the HC knobs are inline). + pub(crate) fn heap_size(&self) -> usize { + self.table.heap_size() + self.backend.heap_size() + } + + pub(crate) fn should_run_btultra2_seed_pass( + &self, + current_len: usize, + ) -> bool { + // The in-block two-pass dynamic-stats seed (`initStats_ultra`) + // is btultra2-only. `TWO_PASS_SEED` is `false` for every other + // strategy — including btultra, which now shares the hash3 + // short-match probe but stays single-pass — so the seed call and + // its body drop at codegen time for all non-btultra2 kernels. + if !S::TWO_PASS_SEED { + return false; + } + let HcBackend::Bt(bt) = &self.backend else { + return false; + }; + bt.opt_state.lit_length_sum == 0 + && bt.opt_state.dictionary_seed.is_none() + && !self.table.dictionary_primed_for_frame + && bt.ldm_sequences.is_empty() + && self.table.window_size == current_len + && self.table.history_abs_start == 0 + && self.table.chunk_lens.len() == 1 + && current_len > HC_PREDEF_THRESHOLD + } + + pub(crate) fn new(max_window_size: usize) -> Self { + Self { + table: crate::encoding::match_table::storage::MatchTable::new(max_window_size), + hc: crate::encoding::hc::HcMatcher::new(2, HC_SEARCH_DEPTH, HC_TARGET_LEN), + // Default to the zero-sized HC backend; `configure()` swaps + // in a `BtMatcher` only when an optimal strategy lands. + backend: HcBackend::Hc, + // Lazy is the per-construct default — every production + // caller calls `configure()` before the first encode and + // overwrites this. Tests that drive `HcMatchGenerator` + // without calling `configure()` end up in the + // `start_matching_lazy` arm of the test dispatcher, which + // matches the previous default behaviour. + strategy_tag: crate::encoding::strategy::StrategyTag::Lazy, + } + } + + pub(crate) fn configure( + &mut self, + config: HcConfig, + tag: crate::encoding::strategy::StrategyTag, + window_log: u8, + ) { + use crate::encoding::strategy::StrategyTag; + // Mirror the driver-resolved strategy tag so the + // `#[cfg(test)] start_matching` dispatcher can route + // BtOpt / BtUltra / BtUltra2 to distinct monomorphisations. + self.strategy_tag = tag; + let is_btultra2 = tag == StrategyTag::BtUltra2; + let uses_bt = matches!( + tag, + StrategyTag::Btlazy2 + | StrategyTag::BtOpt + | StrategyTag::BtUltra + | StrategyTag::BtUltra2 + ); + // btultra and btultra2 both run the mls=3 hash3 short-match probe + // (clevels.h minMatch 3). The `is_btultra2` flag below stays + // exclusive to btultra2 because it tweaks the BT rebase boundary, + // not match finding. + let wants_hash3 = matches!(tag, StrategyTag::BtUltra | StrategyTag::BtUltra2); + let next_hash3_log = if wants_hash3 { + HC3_HASH_LOG.min(window_log as usize) + } else { + 0 + }; + let resize = self.table.hash_log != config.hash_log + || self.table.chain_log != config.chain_log + || self.table.hash3_log != next_hash3_log; + // Capture the layout flip BEFORE `uses_bt` is overwritten below — it + // feeds the dms invalidation (the dms is keyed by layout too). + let uses_bt_changed = self.table.uses_bt != uses_bt; + self.table.hash_log = config.hash_log; + self.table.chain_log = config.chain_log; + self.table.hash3_log = next_hash3_log; + self.hc.search_depth = if uses_bt { + config.search_depth + } else { + config.search_depth.min(MAX_HC_SEARCH_DEPTH) + }; + self.hc.target_len = config.target_len; + // Mirror strategy-derived flags + HC search depth onto MatchTable + // so the BT walker and rebase machinery can read them directly + // without dispatching back through HcMatchGenerator. + self.table.search_depth = self.hc.search_depth; + self.table.is_btultra2 = is_btultra2; + self.table.uses_bt = uses_bt; + // BT finder hash width, upstream zstd `mls = BOUNDED(4, cParams.minMatch, 6)`, + // carried explicitly in the level config so a `target_length` override + // cannot silently flip the finder between 5- and 4-byte hashing. Only + // the BT body reads it; HC/lazy levels leave it at 4. clevels.h + // (srcSize > 256 KiB tier): btlazy2 L13-15 + btopt L16 are minMatch=5, + // btopt L17 is minMatch=4, btultra/btultra2 are minMatch=3 (4-byte main + // hash + the hash3 short-match probe). + // The cached dms is keyed by the full (region, layout, mls, hash_log) + // shape that `build_dms!` validates on the normal prime path, but the + // reborrow fast path in `MatchTable::reset` reuses it on `dms.is_primed()` + // ALONE. A reused-compressor level switch can change the search mls (e.g. + // btlazy2 -> lazy), the table geometry (hash_log / chain_log / hash3, + // captured in `resize`), OR the HC<->BT layout (`uses_bt_changed`) + // independently of each other, and any of them leaves the dms hashed for + // a different shape. Invalidate on ANY so the next dict frame re-primes at + // the new shape (configure runs before reset) instead of probing a + // mismatched dms and silently degrading match quality. Over-invalidation + // only costs a re-prime, which a real shape change needs anyway. + let mls_changed = self.table.search_mls != config.search_mls; + if resize || mls_changed || uses_bt_changed { + self.table.dms.invalidate(); + } + self.table.search_mls = config.search_mls; + // Stage D: promote the backend discriminator. HC modes drop the + // BT scratch buffers entirely; switching back into a BT mode + // allocates a fresh `BtMatcher` on demand. + match (&self.backend, self.table.uses_bt) { + (HcBackend::Hc, true) => { + self.backend = + HcBackend::Bt(alloc::boxed::Box::new(crate::encoding::bt::BtMatcher::new())); + } + (HcBackend::Bt(_), false) => { + self.backend = HcBackend::Hc; + } + _ => {} + } + if resize && !self.table.hash_table.is_empty() { + // Force reallocation on next ensure_tables() call. + self.table.hash_table.clear(); + self.table.hash3_table.clear(); + self.table.chain_table.clear(); + } + } + + pub(crate) fn seed_dictionary_entropy( + &mut self, + huff: Option<&crate::huff0::huff0_encoder::HuffmanTable>, + ll: Option<&crate::fse::fse_encoder::FSETable>, + ml: Option<&crate::fse::fse_encoder::FSETable>, + of: Option<&crate::fse::fse_encoder::FSETable>, + ) { + if let HcBackend::Bt(bt) = &mut self.backend { + bt.opt_state.seed_dictionary_entropy(huff, ll, ml, of); + } + } + + /// Install (or clear) the long-distance-match producer (#27). Only + /// the BT backend owns an `ldm_producer` slot; on the HC (lazy) + /// backend the producer is dropped because there is no optimal-parser + /// candidate buffer to seed. Call after [`Self::reset`]. + #[cfg(feature = "hash")] + pub(crate) fn set_ldm_producer(&mut self, producer: Option) { + if let HcBackend::Bt(bt) = &mut self.backend { + bt.ldm_producer = producer; + } + } + + /// Move the LDM producer out of the BT backend, leaving `None`. Used by the + /// dictionary snapshot path: the producer carries no dictionary state (LDM + /// is not dict-primed; its hash table is empty at capture), so it is not + /// retained in the snapshot — the working frame's freshly-reset producer is + /// reinstated on restore instead. + #[cfg(feature = "hash")] + pub(crate) fn take_ldm_producer(&mut self) -> Option { + if let HcBackend::Bt(bt) = &mut self.backend { + bt.ldm_producer.take() + } else { + None + } + } + + pub(crate) fn reset(&mut self, reuse_space: impl FnMut(Vec)) { + self.table.reset(reuse_space); + if let HcBackend::Bt(bt) = &mut self.backend { + bt.reset(); + } + } + + /// Backfill positions from the tail of the previous slice that couldn't be + /// hashed at the time (insert_position needs 4 bytes of lookahead). + pub(crate) fn skip_matching(&mut self, incompressible_hint: Option) { + self.table.skip_matching(incompressible_hint); + } + + /// Runtime-dispatched entry kept only for in-crate tests. Production + /// callers reach the inner loops through + /// [`Self::start_matching_strategy`] / [`MatchGeneratorDriver::compress_block`] + /// which pick the lazy / optimal arm from `S::USE_BT` at + /// monomorphisation time. + #[cfg(test)] + pub(crate) fn start_matching(&mut self, mut handle_sequence: impl for<'a> FnMut(Sequence<'a>)) { + use crate::encoding::strategy::{self, StrategyTag}; + // Dispatch on the mirrored `strategy_tag` so each test runs + // under the same monomorphisation production would pick. + // `BtOpt` / `BtUltra` / `BtUltra2` remain distinct here even + // though `table.uses_bt` / `is_btultra2` alone can't separate + // BtOpt from BtUltra. + match self.strategy_tag { + StrategyTag::Fast | StrategyTag::Dfast | StrategyTag::Greedy | StrategyTag::Lazy => { + self.start_matching_lazy(&mut handle_sequence) + } + StrategyTag::Btlazy2 => self.start_matching_btlazy2(&mut handle_sequence), + StrategyTag::BtOpt => { + self.start_matching_optimal::(&mut handle_sequence) + } + StrategyTag::BtUltra => { + self.start_matching_optimal::(&mut handle_sequence) + } + StrategyTag::BtUltra2 => { + self.start_matching_optimal::(&mut handle_sequence) + } + } + } + + /// Strategy-aware entry point used by + /// [`MatchGeneratorDriver::compress_block`]. Branches on + /// `S::USE_BT` — a compile-time `const` — so each + /// monomorphisation keeps exactly one arm: `Lazy` / + /// `Fast` / `Dfast` / `Greedy` see only `start_matching_lazy`, + /// `BtOpt` / `BtUltra` / `BtUltra2` see only + /// `start_matching_optimal`. The inherent test-only + /// [`HcMatchGenerator::start_matching`] reaches the same arms by + /// runtime-matching on `self.strategy_tag` (the parse-mode field + /// has been removed); production never invokes that path. + pub(crate) fn start_matching_strategy( + &mut self, + handle_sequence: &mut impl for<'a> FnMut(Sequence<'a>), + ) { + debug_assert_eq!( + self.table.uses_bt, + S::USE_BT, + "Strategy::USE_BT disagrees with runtime table.uses_bt at HC dispatch" + ); + if S::USE_BT { + self.start_matching_optimal::(handle_sequence) + } else { + self.start_matching_lazy(handle_sequence) + } + } + + /// Dispatcher: pick the dict-aware monomorph when a separate dms is primed + /// (attach-mode dictionary), else the no-dict monomorph. Mirrors upstream's + /// compile-time `dictMode` split — the `DICT = false` body carries no dms + /// code at all, so the no-dict hot path is unaffected by the dict search. + pub(crate) fn start_matching_lazy( + &mut self, + handle_sequence: impl for<'a> FnMut(Sequence<'a>), + ) { + if self.table.dms.is_primed() { + self.start_matching_lazy_impl::(handle_sequence); + } else { + self.start_matching_lazy_impl::(handle_sequence); + } + } + + pub(crate) fn start_matching_lazy_impl( + &mut self, + mut handle_sequence: impl for<'a> FnMut(Sequence<'a>), + ) { + self.table.ensure_tables(); + + // `current_block_range()` is borrowed-aware: owned → last committed + // chunk; borrowed → the staged in-place block range. + let (current_abs_start, current_len) = self.table.current_block_range(); + if current_len == 0 { + return; + } + // The current block is the tail of `history` (owned) or the staged + // borrowed range (`get_last_space()` resolves both). Hoist it as a raw + // slice: the routine mutates the hash/chain tables + `offset_hist` but + // never reallocates `history`, so the slice stays valid and we avoid + // re-borrowing `self.table` (which would conflict with the + // `offset_hist` write). + let current_ptr = self.table.get_last_space().as_ptr(); + let current: &[u8] = unsafe { core::slice::from_raw_parts(current_ptr, current_len) }; + + // Full live history (dict + committed blocks + current block), hoisted + // ONCE for the whole position scan and threaded into every + // `find_best_match` / `pick_lazy_match` call. `live_history()` is + // loop-invariant here (the scan mutates the hash/chain tables + + // `offset_hist` but never the history bytes or length), so re-fetching + // it per find — inside `hash_chain_candidate` + the rep probe, plus + // again for each lazy lookahead at pos+1 / pos+2 — was pure + // per-position overhead. Same raw-slice detach as `current` so the + // loop's `&mut self.table` inserts coexist with this `&[u8]`. + let concat: &[u8] = { + let lh = self.table.live_history(); + unsafe { core::slice::from_raw_parts(lh.as_ptr(), lh.len()) } + }; + // Dict-match-state primed flag, hoisted ONCE for the scan: it is + // block-invariant (the dict is primed before the block) and lives on the + // cold `dms` cacheline, so the per-find `dms.is_primed()` load was a + // measurable hot-path cost (~8% of `hash_chain_candidate` on the + // dict-over-random fixture). The `DICT = false` monomorph ignores it. + let dms_primed = self.table.dms.is_primed(); + + let current_abs_end = current_abs_start + current_len; + self.table + .backfill_boundary_positions(current_abs_start, current_abs_end); + + let mut pos = 0usize; + let mut literals_start = 0usize; + while pos + HC_MIN_MATCH_LEN <= current_len { + let abs_pos = current_abs_start + pos; + let lit_len = pos - literals_start; + + // `find_best_match` returns the forward `(offset, length)` in + // registers (`HcMatch`, 16 bytes) — no 24-byte `MatchCandidate` / + // 32-byte `Option` spilled-and-copied per position. The backward + // extension that yields `start` runs ONCE here, after the lazy + // decision settles, exactly like upstream's lazy loop. + let best = + self.hc + .find_best_match::(concat, dms_primed, &self.table, abs_pos, lit_len); + if best.is_match() { + if self.hc.pick_lazy_match::( + concat, + dms_primed, + &self.table, + abs_pos, + lit_len, + best, + ) { + // Backward-extend over the literal run (upstream `zstd_lazy.c` + // after rep-vs-chain selection). The offset is preserved; + // `start` and `match_len` grow by the same amount, bounded by + // `literals_start` (the `min_abs` floor) so it never crosses + // an already-emitted sequence. + let history_abs_start = self.table.history_abs_start; + let min_abs = abs_pos - lit_len; + let mut start_abs = abs_pos; + let mut cand_abs = abs_pos - best.offset; + let mut match_len = best.match_len; + while start_abs > min_abs + && cand_abs > history_abs_start + && concat[cand_abs - history_abs_start - 1] + == concat[start_abs - history_abs_start - 1] + { + start_abs -= 1; + cand_abs -= 1; + match_len += 1; + } + self.table.insert_match_span(abs_pos, start_abs + match_len); + let start = start_abs - current_abs_start; + let literals = ¤t[literals_start..start]; + handle_sequence(Sequence::Triple { + literals, + offset: best.offset, + match_len, + }); + let _ = encode_offset_with_history( + best.offset as u32, + literals.len() as u32, + &mut self.table.offset_hist, + ); + pos = start + match_len; + literals_start = pos; + continue; + } + // Lazy lookahead found a better match at `abs_pos + 1` / `+ 2` + // (defer): advance exactly ONE byte (upstream + // `ZSTD_compressBlock_lazy_generic`) so the deferred candidate is + // re-evaluated at its own position; the no-match skip below could + // jump past it once the literal run reaches 256+ bytes. + self.table.insert_position(abs_pos); + pos += 1; + continue; + } + // No match found. + self.table.insert_position(abs_pos); + // Lazy skipping (upstream zstd `ZSTD_compressBlock_lazy_generic`, + // zstd_lazy.c:1614): advance faster over runs with no match. + // `step = ((ip - anchor) >> kSearchStrength) + 1` with + // kSearchStrength = 8, where `ip - anchor` is the current + // literal-run length. On compressible input the run stays short + // (step == 1, identical to a 1-byte advance); on incompressible + // / dict-over-random input the run grows so the parser skips + // ahead (one search per `step` positions) instead of searching + // every byte. Skipped positions are not inserted, mirroring + // upstream (it inserts only searched positions during a no-match + // run). Ratio follows upstream (not byte-identical). + let step = ((pos - literals_start) >> 8) + 1; + pos += step; + // No clamp needed before the tail loop: the search bound and the + // hashable bound are both `pos + HC_MIN_MATCH_LEN <= current_len` + // (HC_MIN_MATCH_LEN == 4 == the insert width), so there is no + // non-searchable-but-hashable anchor to miss. Positions the skip + // jumps over inside the searchable region are intentionally not + // inserted — same as upstream zstd, which advances past them via + // the identical `ip += step` and never hashes them either. + } + + // Insert remaining hashable positions in the tail (the matching loop + // stops at HC_MIN_MATCH_LEN but insert_position only needs 4 bytes). + while pos + 4 <= current_len { + self.table.insert_position(current_abs_start + pos); + pos += 1; + } + + if literals_start < current_len { + handle_sequence(Sequence::Literals { + literals: ¤t[literals_start..], + }); + } + } + + /// Register the borrowed input window for the no-copy one-shot path. + /// # Safety + /// `buffer` must outlive the borrowed scans (see `MatchTable`). + pub(crate) unsafe fn set_borrowed_window(&mut self, buffer: &[u8]) { + // SAFETY: forwarded liveness contract. + unsafe { self.table.set_borrowed_window(buffer) }; + } + + pub(crate) fn clear_borrowed_window(&mut self) { + self.table.clear_borrowed_window(); + } + + /// Borrowed (no-copy) equivalent of [`Self::start_matching_lazy`]: stage + /// the in-place block range, then run the same lazy chain parse. The + /// parse reads its range via `current_block_range()` and its bytes via + /// `get_last_space()` / `live_history()`, all borrowed-aware, so the block + /// is scanned in place with the per-position window_low offset cap. + pub(crate) fn start_matching_lazy_borrowed( + &mut self, + block_start: usize, + block_end: usize, + handle_sequence: impl for<'a> FnMut(Sequence<'a>), + ) { + self.table.stage_borrowed_block(block_start, block_end); + self.start_matching_lazy(handle_sequence); + } + + /// Borrowed (no-copy) equivalent of the lazy `skip_matching`: stage the + /// in-place block, then seed positions without an owned-history append. + pub(crate) fn skip_matching_borrowed( + &mut self, + block_start: usize, + block_end: usize, + incompressible_hint: Option, + ) { + self.table.stage_borrowed_block(block_start, block_end); + self.table.skip_matching(incompressible_hint); + } +} + +#[cfg(test)] +mod tests; diff --git a/zstd/src/encoding/hc/generator/tests.rs b/zstd/src/encoding/hc/generator/tests.rs new file mode 100644 index 000000000..2f7e34eab --- /dev/null +++ b/zstd/src/encoding/hc/generator/tests.rs @@ -0,0 +1,434 @@ +use super::*; + +#[cfg(test)] +use crate::encoding::CompressionLevel; +#[cfg(test)] +use crate::encoding::Matcher; +#[cfg(test)] +use crate::encoding::dfast::DfastMatchGenerator; +#[cfg(test)] +use crate::encoding::match_generator::MatchGeneratorDriver; + +#[cfg(any())] // disabled: tested legacy MatchGenerator/SuffixStore behavior removed in phase 1b +#[test] +fn matches() { + let mut matcher = MatchGenerator::new(1000); + let mut original_data = Vec::new(); + let mut reconstructed = Vec::new(); + + let replay_sequence = |seq: Sequence<'_>, reconstructed: &mut Vec| match seq { + Sequence::Literals { literals } => { + assert!(!literals.is_empty()); + reconstructed.extend_from_slice(literals); + } + Sequence::Triple { + literals, + offset, + match_len, + } => { + assert!(offset > 0); + assert!(match_len >= MIN_MATCH_LEN); + reconstructed.extend_from_slice(literals); + assert!(offset <= reconstructed.len()); + let start = reconstructed.len() - offset; + for i in 0..match_len { + let byte = reconstructed[start + i]; + reconstructed.push(byte); + } + } + }; + + matcher.add_data( + alloc::vec![0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + SuffixStore::with_capacity(100), + |_, _| {}, + ); + original_data.extend_from_slice(&[0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + + matcher.next_sequence(|seq| replay_sequence(seq, &mut reconstructed)); + + assert!(!matcher.next_sequence(|_| {})); + + matcher.add_data( + alloc::vec![ + 1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6, 0, 0, 0, 0, 0, + ], + SuffixStore::with_capacity(100), + |_, _| {}, + ); + original_data.extend_from_slice(&[ + 1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6, 0, 0, 0, 0, 0, + ]); + + matcher.next_sequence(|seq| replay_sequence(seq, &mut reconstructed)); + matcher.next_sequence(|seq| replay_sequence(seq, &mut reconstructed)); + matcher.next_sequence(|seq| replay_sequence(seq, &mut reconstructed)); + assert!(!matcher.next_sequence(|_| {})); + + matcher.add_data( + alloc::vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0, 0, 0, 0, 0], + SuffixStore::with_capacity(100), + |_, _| {}, + ); + original_data.extend_from_slice(&[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0, 0, 0, 0, 0]); + + matcher.next_sequence(|seq| replay_sequence(seq, &mut reconstructed)); + matcher.next_sequence(|seq| replay_sequence(seq, &mut reconstructed)); + assert!(!matcher.next_sequence(|_| {})); + + matcher.add_data( + alloc::vec![0, 0, 0, 0, 0], + SuffixStore::with_capacity(100), + |_, _| {}, + ); + original_data.extend_from_slice(&[0, 0, 0, 0, 0]); + + matcher.next_sequence(|seq| replay_sequence(seq, &mut reconstructed)); + assert!(!matcher.next_sequence(|_| {})); + + matcher.add_data( + alloc::vec![7, 8, 9, 10, 11], + SuffixStore::with_capacity(100), + |_, _| {}, + ); + original_data.extend_from_slice(&[7, 8, 9, 10, 11]); + + matcher.next_sequence(|seq| replay_sequence(seq, &mut reconstructed)); + assert!(!matcher.next_sequence(|_| {})); + + matcher.add_data( + alloc::vec![1, 3, 5, 7, 9], + SuffixStore::with_capacity(100), + |_, _| {}, + ); + matcher.skip_matching(); + original_data.extend_from_slice(&[1, 3, 5, 7, 9]); + reconstructed.extend_from_slice(&[1, 3, 5, 7, 9]); + assert!(!matcher.next_sequence(|_| {})); + + matcher.add_data( + alloc::vec![1, 3, 5, 7, 9], + SuffixStore::with_capacity(100), + |_, _| {}, + ); + original_data.extend_from_slice(&[1, 3, 5, 7, 9]); + + matcher.next_sequence(|seq| replay_sequence(seq, &mut reconstructed)); + assert!(!matcher.next_sequence(|_| {})); + + matcher.add_data( + alloc::vec![0, 0, 11, 13, 15, 17, 20, 11, 13, 15, 17, 20, 21, 23], + SuffixStore::with_capacity(100), + |_, _| {}, + ); + original_data.extend_from_slice(&[0, 0, 11, 13, 15, 17, 20, 11, 13, 15, 17, 20, 21, 23]); + + matcher.next_sequence(|seq| replay_sequence(seq, &mut reconstructed)); + matcher.next_sequence(|seq| replay_sequence(seq, &mut reconstructed)); + assert!(!matcher.next_sequence(|_| {})); + + assert_eq!(reconstructed, original_data); +} + +#[test] +fn dfast_matches_roundtrip_multi_block_pattern() { + let pattern = [9, 21, 44, 184, 19, 96, 171, 109, 141, 251]; + let first_block: Vec = pattern.iter().copied().cycle().take(128 * 1024).collect(); + let second_block: Vec = pattern.iter().copied().cycle().take(128 * 1024).collect(); + + let mut matcher = DfastMatchGenerator::new(1 << 22); + let replay_sequence = |decoded: &mut Vec, seq: Sequence<'_>| match seq { + Sequence::Literals { literals } => decoded.extend_from_slice(literals), + Sequence::Triple { + literals, + offset, + match_len, + } => { + decoded.extend_from_slice(literals); + let start = decoded.len() - offset; + for i in 0..match_len { + let byte = decoded[start + i]; + decoded.push(byte); + } + } + }; + + matcher.add_data(first_block.clone(), |_| {}); + let mut history = Vec::new(); + matcher.start_matching(|seq| replay_sequence(&mut history, seq)); + assert_eq!(history, first_block); + + matcher.add_data(second_block.clone(), |_| {}); + let prefix_len = history.len(); + matcher.start_matching(|seq| replay_sequence(&mut history, seq)); + + assert_eq!(&history[prefix_len..], second_block.as_slice()); +} + +/// Regression for the `DFAST_MIN_MATCH_LEN: 6 -> 5` drop. The fixture +/// is built so the longest available match is EXACTLY 5 bytes — a +/// matcher that still effectively requires a 6-byte floor would emit +/// only literals here and the assertion would catch the silent +/// 5-byte miss. +/// +/// Fixture layout (34 B): +/// bytes 0..5 `"ABCDE"` — match source +/// bytes 5..28 `'!'` × 23 — filler that does NOT start with 'A' +/// bytes 28..33 `"ABCDE"` — match site (repeats the prefix) +/// byte 33 `'F'` — terminator: differs from byte 5 (`'!'`), +/// so the forward extension at the match +/// site stops at exactly length 5. +/// +/// A 5-byte match at offset 28 must be emitted; a 6-byte+ match at the +/// same offset must NOT. +#[test] +fn dfast_accepts_exact_five_byte_match() { + // Layout the input so that: + // byte 0 = 'Z' (lead byte — keeps the match SOURCE off + // position 0, which the greedy loop never + // inserts: like the upstream zstd it starts the + // cursor at ip+1 and hashes only visited + // positions) + // bytes 1..6 = "ABCDE" (the match source — position 1 IS visited) + // bytes 6..29 = 23 filler bytes that do NOT start with 'A' + // bytes 29..34 = "ABCDE" (the 5-byte match site) + // byte 34 = 'F' (differs from byte 6 = '!') + // The longest available copy at position 29 is exactly 5 bytes: + // the byte at position 34 ('F') differs from the byte at position 6 + // ('!'), so the forward extension stops at length 5. + let mut data = Vec::new(); + data.push(b'Z'); // 0 + data.extend_from_slice(b"ABCDE"); // 1..6 + data.extend_from_slice(b"!!!!!!!!!!!!!!!!!!!!!!!"); // 6..29 (23 bytes) + data.extend_from_slice(b"ABCDE"); // 29..34 + data.push(b'F'); // 34: forces forward extension to stop at length 5 + // Trailing filler so the match site (29) sits at least HASH_READ_SIZE (8) + // bytes before the block end. The greedy double-fast — like the upstream zstd — + // stops probing at `ilimit = iend - HASH_READ_SIZE`, so a match in the + // final 8 bytes is never searched (upstream zstd parity, not a regression). + data.extend_from_slice(b"GHIJKLMNOPQRSTUVWXYZ"); // 35..55 + assert_eq!(data.len(), 55); + + let mut matcher = DfastMatchGenerator::new(1 << 22); + matcher.add_data(data.clone(), |_| {}); + + let mut saw_five_byte_match = false; + let mut saw_longer_match = false; + matcher.start_matching(|seq| { + if let Sequence::Triple { + offset, match_len, .. + } = seq + { + if offset == 28 && match_len == 5 { + saw_five_byte_match = true; + } else if offset == 28 && match_len > 5 { + saw_longer_match = true; + } + } + }); + + assert!( + saw_five_byte_match, + "dfast must accept the exact-5-byte match — a 6-byte floor would skip it" + ); + assert!( + !saw_longer_match, + "fixture pinned to length 5 — byte 33 ('F') must terminate the extension" + ); +} + +#[test] +fn driver_switches_backends_and_initializes_dfast_via_reset() { + let mut driver = MatchGeneratorDriver::new(32, 2); + + driver.reset(CompressionLevel::Default); + assert_eq!( + driver.active_backend(), + crate::encoding::strategy::BackendTag::Dfast + ); + assert_eq!(driver.window_size(), (1u64 << 21)); + + let mut first = driver.get_next_space(); + first[..12].copy_from_slice(b"abcabcabcabc"); + first.truncate(12); + driver.commit_space(first); + assert_eq!(driver.get_last_space(), b"abcabcabcabc"); + driver.skip_matching_with_hint(None); + + let mut second = driver.get_next_space(); + second[..12].copy_from_slice(b"abcabcabcabc"); + second.truncate(12); + driver.commit_space(second); + + let mut reconstructed = b"abcabcabcabc".to_vec(); + driver.start_matching(|seq| match seq { + Sequence::Literals { literals } => reconstructed.extend_from_slice(literals), + Sequence::Triple { + literals, + offset, + match_len, + } => { + reconstructed.extend_from_slice(literals); + let start = reconstructed.len() - offset; + for i in 0..match_len { + let byte = reconstructed[start + i]; + reconstructed.push(byte); + } + } + }); + assert_eq!(reconstructed, b"abcabcabcabcabcabcabcabc"); + + driver.reset(CompressionLevel::Fastest); + assert_eq!(driver.window_size(), (1u64 << 19)); +} + +#[test] +fn driver_level5_selects_row_backend() { + let mut driver = MatchGeneratorDriver::new(32, 2); + driver.reset(CompressionLevel::Level(5)); + assert_eq!( + driver.active_backend(), + crate::encoding::strategy::BackendTag::Row + ); + // Greedy-specific routing assertion: `MatchGeneratorDriver::start_matching` + // dispatches the Row backend into `start_matching_greedy` iff + // `self.parse == ParseMode::Greedy`, so assert that actual selector — + // round-trip alone passes on the lazy parser too. `row_matcher().lazy_depth` + // is a secondary corroboration of the same routing decision (a mirror of + // the parse mode); checking `parse` directly catches a regression even if + // the two ever drift apart. + assert_eq!( + driver.parse, + crate::encoding::strategy::ParseMode::Greedy, + "L5 must route to start_matching_greedy (parse == Greedy)", + ); + assert_eq!( + driver.row_matcher().lazy_depth, + 0, + "row matcher lazy_depth must mirror the greedy parse mode", + ); +} + +/// Level 4 maps to `StrategyTag::Dfast` (the greedy double-fast, upstream zstd +/// `ZSTD_dfast` — "greedy" is the parse discipline, not the Row/Greedy +/// strategy at Level 5). Round-trip alone doesn't pin match quality (a lazy +/// parser would also reconstruct the input correctly), so this test guards the +/// parse output itself: a small repeating pattern must produce at least one +/// `Sequence::Triple`, so a future regression that emits literals-only (e.g. a +/// `min_match` or rep-probe guard regression) is caught. +#[test] +fn driver_level4_greedy_round_trip_single_slice() { + let mut driver = MatchGeneratorDriver::new(64, 2); + driver.reset(CompressionLevel::Level(4)); + let input = b"abcdefgh_abcdefgh_abcdefgh_abcdefgh"; + let mut space = driver.get_next_space(); + space[..input.len()].copy_from_slice(input); + space.truncate(input.len()); + driver.commit_space(space); + + let mut reconstructed: Vec = Vec::new(); + let mut saw_triple = false; + driver.start_matching(|seq| match seq { + Sequence::Literals { literals } => reconstructed.extend_from_slice(literals), + Sequence::Triple { + literals, + offset, + match_len, + } => { + saw_triple = true; + reconstructed.extend_from_slice(literals); + let start = reconstructed.len() - offset; + for i in 0..match_len { + let byte = reconstructed[start + i]; + reconstructed.push(byte); + } + } + }); + assert_eq!( + reconstructed, + input.to_vec(), + "L4 greedy parse failed to reconstruct repeating-pattern input", + ); + assert!( + saw_triple, + "L4 greedy parse on a repeating pattern must emit at least one match (Triple)", + ); +} + +#[test] +fn driver_level4_greedy_round_trip_cross_slice() { + // Verifies that the greedy parse carries repcode / hash-table state + // across slice boundaries: the second slice repeats the first byte + // for byte, so the parse must pick up matches reaching back into + // the previous slice's history. + let mut driver = MatchGeneratorDriver::new(32, 4); + driver.reset(CompressionLevel::Level(4)); + let chunk = b"the quick brown fox jumps over!!"; + assert_eq!(chunk.len(), 32); + + let mut first = driver.get_next_space(); + first[..chunk.len()].copy_from_slice(chunk); + first.truncate(chunk.len()); + driver.commit_space(first); + + let mut first_recon: Vec = Vec::new(); + driver.start_matching(|seq| match seq { + Sequence::Literals { literals } => first_recon.extend_from_slice(literals), + Sequence::Triple { + literals, + offset, + match_len, + } => { + first_recon.extend_from_slice(literals); + let start = first_recon.len() - offset; + for i in 0..match_len { + let byte = first_recon[start + i]; + first_recon.push(byte); + } + } + }); + assert_eq!( + first_recon, + chunk.to_vec(), + "first slice failed to round-trip" + ); + + let mut second = driver.get_next_space(); + second[..chunk.len()].copy_from_slice(chunk); + second.truncate(chunk.len()); + driver.commit_space(second); + + let mut full = first_recon.clone(); + let mut saw_cross_slice_match = false; + driver.start_matching(|seq| match seq { + Sequence::Literals { literals } => full.extend_from_slice(literals), + Sequence::Triple { + literals, + offset, + match_len, + } => { + // A match whose offset reaches >= the current slice's literal + // run plus the second slice's index means we matched into the + // first slice — exactly the cross-slice behavior under test. + if offset >= chunk.len() { + saw_cross_slice_match = true; + } + full.extend_from_slice(literals); + let start = full.len() - offset; + for i in 0..match_len { + let byte = full[start + i]; + full.push(byte); + } + } + }); + let mut expected = chunk.to_vec(); + expected.extend_from_slice(chunk); + assert_eq!( + full, expected, + "cross-slice L4 greedy parse failed to reconstruct" + ); + assert!( + saw_cross_slice_match, + "L4 greedy parse must match across slice boundaries (history is shared)", + ); +} diff --git a/zstd/src/encoding/hc/hc_tests.rs b/zstd/src/encoding/hc/hc_tests.rs new file mode 100644 index 000000000..687396b83 --- /dev/null +++ b/zstd/src/encoding/hc/hc_tests.rs @@ -0,0 +1,185 @@ +//! Unit coverage for `HcMatcher` paths the encode-level suite +//! doesn't naturally hit: short-suffix early returns on probe +//! helpers, chain-walk self-loop branch, and the lazy-pick +//! "next match is better" decline paths. +use super::*; +use crate::encoding::match_table::storage::MatchTable; + +fn table_with_history(buf: &[u8]) -> MatchTable { + let mut t = MatchTable::new(buf.len().max(8)); + t.history = buf.to_vec(); + t.history_start = 0; + t.history_abs_start = 0; + t.window_size = buf.len(); + t.position_base = 0; + t.hash_log = 8; + t.chain_log = 8; + t.hash3_log = 0; + t.ensure_tables(); + // `history` is set directly above; record one live chunk. + t.chunk_lens.push_back(buf.len()); + t +} + +#[test] +fn chain_candidates_returns_sentinels_when_suffix_too_short() { + let hc = HcMatcher::new(2, 4, 32); + // History exactly at min-prefix - 1 → idx + 4 > concat.len() → + // early return with all-sentinel buffer. + let t = table_with_history(b"abc"); + let buf = hc.chain_candidates(&t, 0); + assert!(buf.iter().all(|&v| v == usize::MAX)); +} + +#[test] +fn chain_candidates_terminates_on_self_loop_with_in_range_pick() { + // Construct a self-loop in the chain: hash_table → cur, + // chain_table[cur_rel] = cur (points back to itself). The walker + // must pick the position (in-range) and stop. + let mut hc = HcMatcher::new(2, 4, 32); + hc.search_depth = 4; + let mut t = table_with_history(b"abcdef_abcdef_abcdef"); + let abs_pos = 10usize; + // The walker hashes the suffix at `abs_pos`, not the prefix at 0. + let concat = t.live_history(); + let hash = t.hash_position(&concat[abs_pos..]); + // Stored = relative + 1 → stored=6 means candidate_rel=5. + t.hash_table[hash] = 6; + let chain_mask = (1 << t.chain_log) - 1; + t.chain_table[5 & chain_mask] = 6; // self-loop + + let buf = hc.chain_candidates(&t, abs_pos); + assert_eq!( + buf[0], 5, + "self-loop pick must surface the in-range candidate" + ); + assert_eq!(buf[1], usize::MAX, "walker must stop after self-loop"); +} + +#[test] +fn repcode_candidate_returns_none_when_suffix_too_short() { + let mut t = table_with_history(b"abc"); + t.offset_hist = [1, 2, 3]; + // current_idx + HC_MIN_MATCH_LEN > concat.len() → early no-match. + assert!(!HcMatcher::repcode_candidate(&t, 0, 1).is_match()); +} + +#[test] +fn repcode_candidate_skips_rep_at_history_boundary() { + // rep=5 but abs_pos=4, so candidate_pos would underflow into + // pre-history bytes; the `rep > abs_pos` guard must skip it. + let mut t = table_with_history(b"abcdefgh"); + t.offset_hist = [5, 6, 7]; + // No match possible at abs_pos=4 because every rep aims past + // history start. + let result = HcMatcher::repcode_candidate(&t, 4, 1); + assert!(!result.is_match(), "no rep can land in-range"); +} + +#[test] +fn find_best_match_returns_none_for_short_suffix() { + let hc = HcMatcher::new(2, 4, 32); + let t = table_with_history(b"abc"); + assert!( + !hc.find_best_match::(t.live_history(), false, &t, 0, 1) + .is_match() + ); +} + +/// Forward-length selection (upstream `ZSTD_HcFindBestMatch`): the chain +/// walk keeps the longest FORWARD match (`currentMl > ml`) and applies the +/// single backward extension to THAT winner — a shorter-forward candidate +/// is NOT promoted just because it has more backward (`lit_len`) room. +/// Backward "catch up" happens once, on the forward winner, after the walk; +/// it never changes which candidate wins. +/// +/// Fixture (40 bytes): `"AAAabcdefZMQabcdefIJBAAAabcdefIJKKKKKKKK"`. +/// Probing `abs_pos = 24`: the 4-byte hash at `idx 24` ("abcd") collides +/// with `idx 3` and `idx 12`, so the walk visits `[12, 3]` (LIFO). +/// - `idx 12`: forward 8 (`"abcdefIJ"`), `concat[11] = 'Q'` != +/// `concat[23] = 'A'` so no backward room. Total 8, offset 12. +/// - `idx 3`: forward only 6 (`"abcdef"`), but `concat[0..3] = "AAA"` == +/// `concat[21..24]` so it could backward-extend 3 to a TOTAL of 9 — yet +/// its forward length (6) loses to candidate 12's (8), so it is never +/// selected. The forward winner (12) wins at both `lit_len`s. +#[test] +fn hash_chain_candidate_picks_longest_forward_over_shorter_with_backward_room() { + let mut t = MatchTable::new(64); + t.history = b"AAAabcdefZMQabcdefIJBAAAabcdefIJKKKKKKKK".to_vec(); + t.history_start = 0; + t.history_abs_start = 0; + t.window_size = t.history.len(); + t.position_base = 0; + t.hash_log = 8; + t.chain_log = 8; + t.hash3_log = 0; + t.ensure_tables(); + t.chunk_lens.push_back(t.history.len()); + t.insert_positions(0, 24); + + let hc = HcMatcher::new(2, 16, 64); + + // The forward winner (idx 12, forward 8) has no backward room. + let c0 = hc.hash_chain_candidate::(t.live_history(), false, &t, 24); + assert!(c0.is_match(), "forward match must be found"); + assert_eq!(c0.match_len, 8, "longest forward match is 8 (idx 12)"); + assert_eq!( + c0.offset, 12, + "winner is the forward-8 candidate at offset 12" + ); +} + +/// Forward-length ties keep the FIRST-visited candidate (upstream +/// `ZSTD_HcFindBestMatch` uses `currentMl > ml`, strictly-longer, so an +/// equal-length later candidate never displaces the earlier one). The walk +/// is newest-first, so in organic chains "first visited" is the closest +/// (smallest-offset) position anyway; this test hand-wires the chain into a +/// non-monotonic order to pin the tie-break rule itself. +/// +/// Fixture: four 8-byte `"abcdefgh"` chunks at `0 / 9 / 18 / 27`, each +/// followed by a unique terminator (`'A'/'B'/'C'/'D'`) capping cross-chunk +/// forward matches at exactly 8. Probing `abs_pos = 27` with the chain +/// hand-wired to visit pos 9 (offset 18) THEN pos 18 (offset 9): both have +/// forward length 8, so the first-visited (pos 9, offset 18) wins. The +/// self-tightening gate at `ml = 8` also rejects pos 18 (its tail byte is a +/// different chunk terminator), so the equal-length later candidate is +/// skipped before the count — consistent with ties-keep-first. +#[test] +fn hash_chain_candidate_forward_ties_keep_first_visited() { + let mut t = MatchTable::new(64); + t.history = b"abcdefghAabcdefghBabcdefghCabcdefghDZZZZ".to_vec(); + assert_eq!(t.history.len(), 40); + t.history_start = 0; + t.history_abs_start = 0; + t.window_size = t.history.len(); + t.position_base = 0; + t.hash_log = 8; + t.chain_log = 8; + t.hash3_log = 0; + t.ensure_tables(); + t.chunk_lens.push_back(t.history.len()); + + let abs_pos = 27usize; + let concat = t.live_history(); + let probe_hash = t.hash_position(&concat[abs_pos..]); + // Hand-wire the chain head + link so the walk surfaces pos 9 first + // (offset 18) then pos 18 (offset 9). `stored = pos + 1`. + t.hash_table[probe_hash] = 9 + 1; + let chain_mask = (1usize << t.chain_log) - 1; + t.chain_table[9 & chain_mask] = 18 + 1; + t.chain_table[18 & chain_mask] = HC_EMPTY; + + let hc = HcMatcher::new(2, 16, 64); + let cand = hc.hash_chain_candidate::(t.live_history(), false, &t, abs_pos); + assert!(cand.is_match(), "walk must still produce a match"); + assert_eq!( + cand.match_len, 8, + "both candidates have an 8-byte forward prefix" + ); + assert_eq!( + cand.offset, 18, + "forward-length ties keep the first-visited candidate (pos 9, \ + offset 18); a value of 9 would mean the equal-length later \ + candidate displaced it (non-upstream gain-based tie-break)" + ); +} diff --git a/zstd/src/encoding/hc/mod.rs b/zstd/src/encoding/hc/mod.rs index 9b2819557..7006b0655 100644 --- a/zstd/src/encoding/hc/mod.rs +++ b/zstd/src/encoding/hc/mod.rs @@ -15,6 +15,10 @@ #![allow(dead_code)] +pub(crate) mod generator; +pub(crate) mod optimal; +pub(crate) mod priceset; + use super::cost_model::{HC_FORMAT_MINMATCH, HC_OPT_NUM, HcOptimalCostProfile}; use super::match_table::helpers::common_prefix_len; use super::match_table::storage::{HC_EMPTY, MatchTable}; @@ -859,7 +863,7 @@ impl HcMatcher { mut f: impl FnMut(MatchCandidate), ) { let _ = self; - super::match_generator::for_each_repcode_candidate_body!( + super::hc::generator::for_each_repcode_candidate_body!( table, abs_pos, lit_len, @@ -889,7 +893,7 @@ impl HcMatcher { mut f: impl FnMut(MatchCandidate), ) { let _ = self; - super::match_generator::for_each_repcode_candidate_body!( + super::hc::generator::for_each_repcode_candidate_body!( table, abs_pos, lit_len, @@ -919,7 +923,7 @@ impl HcMatcher { mut f: impl FnMut(MatchCandidate), ) { let _ = self; - super::match_generator::for_each_repcode_candidate_body!( + super::hc::generator::for_each_repcode_candidate_body!( table, abs_pos, lit_len, @@ -945,7 +949,7 @@ impl HcMatcher { mut f: impl FnMut(MatchCandidate), ) { let _ = self; - super::match_generator::for_each_repcode_candidate_body!( + super::hc::generator::for_each_repcode_candidate_body!( table, abs_pos, lit_len, @@ -1006,190 +1010,4 @@ impl HcMatcher { } #[cfg(test)] -mod hc_tests { - //! Unit coverage for `HcMatcher` paths the encode-level suite - //! doesn't naturally hit: short-suffix early returns on probe - //! helpers, chain-walk self-loop branch, and the lazy-pick - //! "next match is better" decline paths. - use super::*; - use crate::encoding::match_table::storage::MatchTable; - - fn table_with_history(buf: &[u8]) -> MatchTable { - let mut t = MatchTable::new(buf.len().max(8)); - t.history = buf.to_vec(); - t.history_start = 0; - t.history_abs_start = 0; - t.window_size = buf.len(); - t.position_base = 0; - t.hash_log = 8; - t.chain_log = 8; - t.hash3_log = 0; - t.ensure_tables(); - // `history` is set directly above; record one live chunk. - t.chunk_lens.push_back(buf.len()); - t - } - - #[test] - fn chain_candidates_returns_sentinels_when_suffix_too_short() { - let hc = HcMatcher::new(2, 4, 32); - // History exactly at min-prefix - 1 → idx + 4 > concat.len() → - // early return with all-sentinel buffer. - let t = table_with_history(b"abc"); - let buf = hc.chain_candidates(&t, 0); - assert!(buf.iter().all(|&v| v == usize::MAX)); - } - - #[test] - fn chain_candidates_terminates_on_self_loop_with_in_range_pick() { - // Construct a self-loop in the chain: hash_table → cur, - // chain_table[cur_rel] = cur (points back to itself). The walker - // must pick the position (in-range) and stop. - let mut hc = HcMatcher::new(2, 4, 32); - hc.search_depth = 4; - let mut t = table_with_history(b"abcdef_abcdef_abcdef"); - let abs_pos = 10usize; - // The walker hashes the suffix at `abs_pos`, not the prefix at 0. - let concat = t.live_history(); - let hash = t.hash_position(&concat[abs_pos..]); - // Stored = relative + 1 → stored=6 means candidate_rel=5. - t.hash_table[hash] = 6; - let chain_mask = (1 << t.chain_log) - 1; - t.chain_table[5 & chain_mask] = 6; // self-loop - - let buf = hc.chain_candidates(&t, abs_pos); - assert_eq!( - buf[0], 5, - "self-loop pick must surface the in-range candidate" - ); - assert_eq!(buf[1], usize::MAX, "walker must stop after self-loop"); - } - - #[test] - fn repcode_candidate_returns_none_when_suffix_too_short() { - let mut t = table_with_history(b"abc"); - t.offset_hist = [1, 2, 3]; - // current_idx + HC_MIN_MATCH_LEN > concat.len() → early no-match. - assert!(!HcMatcher::repcode_candidate(&t, 0, 1).is_match()); - } - - #[test] - fn repcode_candidate_skips_rep_at_history_boundary() { - // rep=5 but abs_pos=4, so candidate_pos would underflow into - // pre-history bytes; the `rep > abs_pos` guard must skip it. - let mut t = table_with_history(b"abcdefgh"); - t.offset_hist = [5, 6, 7]; - // No match possible at abs_pos=4 because every rep aims past - // history start. - let result = HcMatcher::repcode_candidate(&t, 4, 1); - assert!(!result.is_match(), "no rep can land in-range"); - } - - #[test] - fn find_best_match_returns_none_for_short_suffix() { - let hc = HcMatcher::new(2, 4, 32); - let t = table_with_history(b"abc"); - assert!( - !hc.find_best_match::(t.live_history(), false, &t, 0, 1) - .is_match() - ); - } - - /// Forward-length selection (upstream `ZSTD_HcFindBestMatch`): the chain - /// walk keeps the longest FORWARD match (`currentMl > ml`) and applies the - /// single backward extension to THAT winner — a shorter-forward candidate - /// is NOT promoted just because it has more backward (`lit_len`) room. - /// Backward "catch up" happens once, on the forward winner, after the walk; - /// it never changes which candidate wins. - /// - /// Fixture (40 bytes): `"AAAabcdefZMQabcdefIJBAAAabcdefIJKKKKKKKK"`. - /// Probing `abs_pos = 24`: the 4-byte hash at `idx 24` ("abcd") collides - /// with `idx 3` and `idx 12`, so the walk visits `[12, 3]` (LIFO). - /// - `idx 12`: forward 8 (`"abcdefIJ"`), `concat[11] = 'Q'` != - /// `concat[23] = 'A'` so no backward room. Total 8, offset 12. - /// - `idx 3`: forward only 6 (`"abcdef"`), but `concat[0..3] = "AAA"` == - /// `concat[21..24]` so it could backward-extend 3 to a TOTAL of 9 — yet - /// its forward length (6) loses to candidate 12's (8), so it is never - /// selected. The forward winner (12) wins at both `lit_len`s. - #[test] - fn hash_chain_candidate_picks_longest_forward_over_shorter_with_backward_room() { - let mut t = MatchTable::new(64); - t.history = b"AAAabcdefZMQabcdefIJBAAAabcdefIJKKKKKKKK".to_vec(); - t.history_start = 0; - t.history_abs_start = 0; - t.window_size = t.history.len(); - t.position_base = 0; - t.hash_log = 8; - t.chain_log = 8; - t.hash3_log = 0; - t.ensure_tables(); - t.chunk_lens.push_back(t.history.len()); - t.insert_positions(0, 24); - - let hc = HcMatcher::new(2, 16, 64); - - // The forward winner (idx 12, forward 8) has no backward room. - let c0 = hc.hash_chain_candidate::(t.live_history(), false, &t, 24); - assert!(c0.is_match(), "forward match must be found"); - assert_eq!(c0.match_len, 8, "longest forward match is 8 (idx 12)"); - assert_eq!( - c0.offset, 12, - "winner is the forward-8 candidate at offset 12" - ); - } - - /// Forward-length ties keep the FIRST-visited candidate (upstream - /// `ZSTD_HcFindBestMatch` uses `currentMl > ml`, strictly-longer, so an - /// equal-length later candidate never displaces the earlier one). The walk - /// is newest-first, so in organic chains "first visited" is the closest - /// (smallest-offset) position anyway; this test hand-wires the chain into a - /// non-monotonic order to pin the tie-break rule itself. - /// - /// Fixture: four 8-byte `"abcdefgh"` chunks at `0 / 9 / 18 / 27`, each - /// followed by a unique terminator (`'A'/'B'/'C'/'D'`) capping cross-chunk - /// forward matches at exactly 8. Probing `abs_pos = 27` with the chain - /// hand-wired to visit pos 9 (offset 18) THEN pos 18 (offset 9): both have - /// forward length 8, so the first-visited (pos 9, offset 18) wins. The - /// self-tightening gate at `ml = 8` also rejects pos 18 (its tail byte is a - /// different chunk terminator), so the equal-length later candidate is - /// skipped before the count — consistent with ties-keep-first. - #[test] - fn hash_chain_candidate_forward_ties_keep_first_visited() { - let mut t = MatchTable::new(64); - t.history = b"abcdefghAabcdefghBabcdefghCabcdefghDZZZZ".to_vec(); - assert_eq!(t.history.len(), 40); - t.history_start = 0; - t.history_abs_start = 0; - t.window_size = t.history.len(); - t.position_base = 0; - t.hash_log = 8; - t.chain_log = 8; - t.hash3_log = 0; - t.ensure_tables(); - t.chunk_lens.push_back(t.history.len()); - - let abs_pos = 27usize; - let concat = t.live_history(); - let probe_hash = t.hash_position(&concat[abs_pos..]); - // Hand-wire the chain head + link so the walk surfaces pos 9 first - // (offset 18) then pos 18 (offset 9). `stored = pos + 1`. - t.hash_table[probe_hash] = 9 + 1; - let chain_mask = (1usize << t.chain_log) - 1; - t.chain_table[9 & chain_mask] = 18 + 1; - t.chain_table[18 & chain_mask] = HC_EMPTY; - - let hc = HcMatcher::new(2, 16, 64); - let cand = hc.hash_chain_candidate::(t.live_history(), false, &t, abs_pos); - assert!(cand.is_match(), "walk must still produce a match"); - assert_eq!( - cand.match_len, 8, - "both candidates have an 8-byte forward prefix" - ); - assert_eq!( - cand.offset, 18, - "forward-length ties keep the first-visited candidate (pos 9, \ - offset 18); a value of 9 would mean the equal-length later \ - candidate displaced it (non-upstream gain-based tie-break)" - ); - } -} +mod hc_tests; diff --git a/zstd/src/encoding/hc/optimal.rs b/zstd/src/encoding/hc/optimal.rs new file mode 100644 index 000000000..b86c84538 --- /dev/null +++ b/zstd/src/encoding/hc/optimal.rs @@ -0,0 +1,2246 @@ +//! Optimal-parse machinery for the binary-tree strategies (btopt / btultra / +//! btultra2): the `build_optimal_plan` DP + its per-CPU-tier kernels, the +//! candidate-collection + price-set body macros, and the `btlazy2` cost +//! helpers. Split out of `hc/generator.rs` (no behaviour change) as a second +//! `impl HcMatchGenerator` block over the same matcher, so the HC chain matcher +//! and the optimal parser live in separate files. + +// The DP body macros reference many opt-parser types / cost-model constants +// UNqualified inside their `macro_rules!` bodies; rustc's `unused_imports` lint +// does not count macro-body references, so it false-positives on every such +// import (gating or removing any one breaks the lib build — the expansions +// need them). This module is macro-dense enough that per-import suppression is +// noise, so the lint is disabled module-wide here. +#![allow(unused_imports)] + +use super::generator::HcMatchGenerator; +use crate::encoding::Sequence; +use crate::encoding::blocks::encode_offset_with_history; +use crate::encoding::hc::MAX_HC_SEARCH_DEPTH; +use alloc::vec::Vec; + +// The DP body macros reference these opt-parser types and cost-model constants +// UNqualified inside their `macro_rules!` bodies. rustc's `unused_imports` lint +// does not count macro-body references, so it reports every one of these as +// "unused" even though each macro expansion requires them (gating or removing +// any one breaks the lib build). Suppress the false positive on the group. +#[allow(unused_imports)] +use crate::encoding::{ + bt::BtMatcher, + cost_model::{ + HC_BITCOST_MULTIPLIER, HC_OPT_NODE_LEN, HC_OPT_NUM, HC_OPT_PRICE_ARENA_LEN, + HC_OPT_PRICE_STRIDE, HC_PREDEF_THRESHOLD, HcOptState, HcOptimalCostProfile, + }, + hc::HC_MIN_MATCH_LEN, + levels::config::HcConfig, + match_generator::{HC_OPT_MIN_MATCH_LEN, HC_SEARCH_DEPTH, HC_TARGET_LEN, HcBackend}, + match_table::storage::HC3_HASH_LOG, + opt::ldm::{HcOptLdmState, HcRawSeqStore}, + opt::types::{ + HcCandidateQuery, HcOptimalNode, HcOptimalPlanBuffers, HcOptimalPlanState, + HcOptimalSequence, MatchCandidate, + }, +}; + +/// Upstream zstd `offBase` for the btlazy2 lazy gain heuristic: a match whose +/// offset equals one of the three active repeat offsets prices as the cheap +/// repcode code (1/2/3); any other offset prices as `offset + 3`. So an +/// equal-length repeat-offset match always out-gains an explicit-offset one +/// (`zstd_lazy.c` `ZSTD_storeSeq` offBase convention). +#[inline] +fn btlazy2_offbase(offset: usize, reps: [u32; 3], ll0: bool) -> u32 { + let o = offset as u32; + // Upstream zstd repcode mapping shifts by `ll0` (zero-literal position): the cheap + // codes become rep1 / rep2 / (rep0 - 1) instead of rep0 / rep1 / rep2, + // because at ll0 an offset equal to rep0 is the special rep0-1 case, not + // repcode 1. Scoring offsets against the wrong code at ll0 over-rewards a + // rep0-distance match that does not actually encode as the cheapest code. + if ll0 { + if o == reps[1] { + 1 + } else if o == reps[2] { + 2 + } else if reps[0] > 1 && o == reps[0] - 1 { + 3 + } else { + // Offsets are < window (<= 2^27), so `+ 3` never overflows u32. + o + 3 + } + } else if o == reps[0] { + 1 + } else if o == reps[1] { + 2 + } else if o == reps[2] { + 3 + } else { + // Offsets are < window (<= 2^27), so `+ 3` never overflows u32. + o + 3 + } +} + +/// Upstream zstd lazy match gain (`matchLength * 4 - ZSTD_highbit32(offBase)`): the +/// selection metric that lets a shorter repeat-offset match beat a longer +/// explicit-offset one. `offBase >= 1`, so `highbit` is well-defined. +#[inline] +fn btlazy2_gain(match_len: usize, offset: usize, reps: [u32; 3], ll0: bool) -> i64 { + let offbase = btlazy2_offbase(offset, reps, ll0); + (match_len as i64) * 4 - (31 - offbase.leading_zeros()) as i64 +} + +/// Per-kernel body of the `btlazy2` (levels 13-15) greedy/lazy parse over +/// the binary-tree match finder. Mirrors `build_optimal_plan_impl_body!`'s +/// kernel-dispatch discipline: the wrapper carries the `#[target_feature]` +/// umbrella and passes its tier-specific `collect_optimal_candidates_initialized_` +/// as `$collect`, so the per-position BT collect (and its inlined cpl) +/// stays under one umbrella — the runtime `select_kernel()` dispatch happens +/// ONCE per block in the bare `start_matching_btlazy2`, never per position. +macro_rules! start_matching_btlazy2_body { + ($self:ident, $handle_sequence:ident, $collect:ident, $cmf:path $(,)?) => {{ + $self.table.ensure_tables(); + // Borrowed-aware: owned → last committed chunk; borrowed → staged block. + let (current_abs_start, current_len) = $self.table.current_block_range(); + if current_len == 0 { + return; + } + let current_ptr = $self.table.get_last_space().as_ptr(); + // Mutates tables but never reallocates `history`, so this tail slice + // stays valid for the routine's duration (same as the other parsers). + let current: &[u8] = unsafe { core::slice::from_raw_parts(current_ptr, current_len) }; + // Full contiguous live region (owned: dict + prior blocks + current + // block in `history`; borrowed: `[0, block_end)` of the in-place + // input) as a raw slice, for the explicit repcode probe: a rep offset + // can point before the current block, which `current` can't reach. + // `live_history()` is borrowed-aware; reborrow-then-raw-ptr so the + // slice holds NO borrow and coexists with the `&mut self` collector + // calls below. Same no-realloc validity contract as `current`. + let history_abs_start = $self.table.history_abs_start; + let concat_full: &[u8] = unsafe { + let lh = $self.table.live_history(); + core::slice::from_raw_parts(lh.as_ptr(), lh.len()) + }; + let current_abs_end = current_abs_start + current_len; + $self + .table + .apply_limited_update_after_long_match(current_abs_start); + $self + .table + .backfill_boundary_positions(current_abs_start, current_abs_end); + + let profile = + HcOptimalCostProfile::const_for_strategy::(); + let mut candidates = core::mem::take(&mut $self.backend.bt_mut().opt_candidates_scratch); + + let depth = $self.hc.lazy_depth as usize; + let mut pos = 0usize; + let mut literals_start = 0usize; + + // Collect + select the highest-GAIN match at a position (upstream zstd + // `ZSTD_searchMax` plus the explicit offset_1 repcode check): scan the + // length-sorted BT/dms ladder by gain, then probe rep0 directly since + // the ladder's strictly-increasing-length filter drops short cheap + // reps. Expands to `(match_len, offset)`; `match_len == 0` = no match. + macro_rules! bt_select { + ($p:expr) => {{ + let sel_pos: usize = $p; + // `ll0` (upstream zstd): zero literals pending before this position, so + // the repcode set is shifted (see `btlazy2_offbase`). + let ll0 = sel_pos == literals_start; + let sel_abs = current_abs_start + sel_pos; + candidates.clear(); + let query = HcCandidateQuery { + reps: $self.table.offset_hist, + lit_len: sel_pos - literals_start, + // No LDM seed: L13-15 run at windowLog 22, below upstream zstd's + // LDM auto-enable threshold (windowLog >= 27). + ldm_candidate: None, + }; + // SAFETY: called inside the wrapper's `#[target_feature]` + // umbrella (the scalar wrapper's `$collect` is a safe fn). + unsafe { + $self.$collect::( + sel_abs, + current_abs_end, + profile, + query, + &mut candidates, + ); + } + let reps = $self.table.offset_hist; + let mut sel_ml = 0usize; + let mut sel_off = 0usize; + let mut sel_gain = i64::MIN; + for c in candidates.iter() { + let ml = c.match_len.min(current_len - sel_pos); + if ml < HC_OPT_MIN_MATCH_LEN { + continue; + } + let g = btlazy2_gain(ml, c.offset, reps, ll0); + if g > sel_gain { + sel_gain = g; + sel_ml = ml; + sel_off = c.offset; + } + } + let sel_idx = sel_abs - history_abs_start; + // Upstream zstd probes `rep[0 + ll0]` directly (the length-sorted ladder + // drops short cheap reps): rep0 normally, rep1 at a zero-literal + // position where rep0 is not the cheapest code. + let probe_rep = if ll0 { + reps[1] as usize + } else { + reps[0] as usize + }; + if probe_rep != 0 && sel_idx >= probe_rep { + let tail = current_len - sel_pos; + // SAFETY: `sel_idx - probe_rep < sel_idx`, `sel_idx + tail <= + // concat_full.len()`; same overshoot slack the collector + // relies on for this block. + let rep_ml = + unsafe { $cmf(concat_full, sel_idx, sel_idx - probe_rep, tail, 0) }; + if rep_ml >= HC_OPT_MIN_MATCH_LEN + && btlazy2_gain(rep_ml, probe_rep, reps, ll0) > sel_gain + { + sel_ml = rep_ml; + sel_off = probe_rep; + } + } + (sel_ml, sel_off) + }}; + } + + while pos + HC_OPT_MIN_MATCH_LEN <= current_len { + let (mut best_ml, mut best_off) = bt_select!(pos); + if best_ml < HC_OPT_MIN_MATCH_LEN { + pos += 1; + continue; + } + // Lazy lookahead (upstream zstd depth 1/2): advance one byte and accept the + // later match only if it out-gains the current one by the upstream zstd + // margin (deferring costs an extra literal — `+4` at depth 1, `+7` + // at depth 2). `start` tracks where the chosen match begins. + let mut start = pos; + let mut d = 0usize; + while d < depth && start + 1 + HC_OPT_MIN_MATCH_LEN <= current_len { + let look = start + 1; + let (ml2, off2) = bt_select!(look); + if ml2 < HC_OPT_MIN_MATCH_LEN { + break; + } + let reps = $self.table.offset_hist; + let margin = if d == 0 { 4 } else { 7 }; + // `best` sits at `start` (ll0 iff no literals precede it); the + // lookahead match at `start + 1` always has a pending literal. + let gain1 = btlazy2_gain(best_ml, best_off, reps, start == literals_start) + margin; + let gain2 = btlazy2_gain(ml2, off2, reps, false); + if gain2 > gain1 { + best_ml = ml2; + best_off = off2; + start = look; + d += 1; + } else { + break; + } + } + // Commit the chosen match at `start`; [literals_start, start) is + // emitted as literals. `best_ml` was bounded to `current_len - + // start` at selection, so `start + best_ml <= current_len`. + let lit_len = start - literals_start; + let literals = ¤t[literals_start..start]; + $handle_sequence(Sequence::Triple { + literals, + offset: best_off, + match_len: best_ml, + }); + let _ = encode_offset_with_history( + best_off as u32, + lit_len as u32, + &mut $self.table.offset_hist, + ); + pos = start + best_ml; + literals_start = pos; + } + + if literals_start < current_len { + $handle_sequence(Sequence::Literals { + literals: ¤t[literals_start..], + }); + } + $self.backend.bt_mut().opt_candidates_scratch = candidates; + }}; +} + +macro_rules! build_optimal_plan_impl_body { + ( + $self:expr, + $strategy_ty:ty, + $current:ident, + $current_abs_start:ident, + $current_len:ident, + $initial_state:ident, + $stats:ident, + $out:ident, + $collect:ident, + $priceset:path $(,)? + ) => {{ + let current_abs_end = $current_abs_start + $current_len; + let min_match_len = HC_OPT_MIN_MATCH_LEN; + // `HC_OPT_NUM > 0` by const definition, so `HC_OPT_NUM - 1` is safe. + let frontier_limit = $current_len.min(HC_OPT_NUM - 1); + let initial_reps = $initial_state.reps; + let initial_litlen = $initial_state.litlen; + let ldm_block_offset = $initial_state.block_offset; + let mut profile = $initial_state.profile; + profile.sufficient_match_len = $self.hc.sufficient_match_len_for_pass(profile); + // Const-fold from the strategy's associated `OPT_LEVEL` + // (upstream zstd `optLevel`): BtOpt = 0, BtUltra / BtUltra2 = 2. + // The two flags below are the only places the inner DP loop + // used to consult `parse_mode`; lifting them into const + // expressions drops one indirect read + one branch on every + // candidate insertion and every traceback step. + // `let` (not `const`) — nested `const` items inside a + // generic fn cannot project through the outer fn's type + // parameter, but a `let` binding from a const expression + // does get folded by the optimiser per monomorphisation, + // which is what we actually want here. + debug_assert!( + <$strategy_ty as crate::encoding::strategy::Strategy>::USE_BT, + "build_optimal_plan_impl_body called on non-BT strategy" + ); + let abort_on_worse_match: bool = + <$strategy_ty as crate::encoding::strategy::Strategy>::OPT_LEVEL == 0; + let opt_level: bool = <$strategy_ty as crate::encoding::strategy::Strategy>::OPT_LEVEL >= 2; + let mut nodes = core::mem::take(&mut $self.backend.bt_mut().opt_nodes_scratch); + let mut node_prices = core::mem::take(&mut $self.backend.bt_mut().opt_node_prices_scratch); + // `frontier_limit + 2 <= HC_OPT_NODE_LEN` — bounded by const. + let frontier_buffer_size = frontier_limit + 2; + if nodes.len() < HC_OPT_NODE_LEN { + // First optimal-parse use (empty boxed slice) or an undersized + // buffer: allocate the fixed upstream-zstd-sized frontier once. The DP + // overwrites the active prefix before reading it. + nodes = alloc::vec![HcOptimalNode::default(); HC_OPT_NODE_LEN].into_boxed_slice(); + } + // The DP price array, same fixed length as `nodes`. This is the SOLE + // home of each position's price (the node struct carries no price), so + // the SIMD price-set vector-loads it directly. Initialised to u32::MAX + // so unwritten frontier cells compare as "unreachable". + if node_prices.len() < HC_OPT_NODE_LEN { + node_prices = alloc::vec![u32::MAX; HC_OPT_NODE_LEN].into_boxed_slice(); + } + let mut candidates = core::mem::take(&mut $self.backend.bt_mut().opt_candidates_scratch); + candidates.clear(); + if candidates.capacity() < MAX_HC_SEARCH_DEPTH { + candidates.reserve_exact(MAX_HC_SEARCH_DEPTH - candidates.capacity()); + } + let mut store = core::mem::take(&mut $self.backend.bt_mut().opt_store_scratch); + store.clear(); + let mut price_arena = core::mem::take(&mut $self.backend.bt_mut().opt_price_arena); + if price_arena.len() < HC_OPT_PRICE_ARENA_LEN { + price_arena = alloc::vec![[0u32; 2]; HC_OPT_PRICE_ARENA_LEN].into_boxed_slice(); + } + // Single arena → two disjoint fixed-stride regions of `[price, + // generation]` pairs (LL cache, ML cache): one base pointer + fixed + // offsets, mirroring upstream zstd's single opt workspace. Pairing + // price+generation per code keeps the optimal parser's cache probe + // on ONE line instead of two strided regions. + // SAFETY: `price_arena` is exactly `HC_OPT_PRICE_ARENA_LEN = + // 2 * HC_OPT_PRICE_STRIDE` pairs long (just ensured), so the two + // STRIDE-wide regions are in bounds and disjoint. The slices alias + // the heap buffer `price_arena` owns; that heap address is stable + // across the later move of the `price_arena` box into the result + // bundle (a `Box` move relocates only the pointer, not the heap + // data), and the slices are never used after the bundle is + // constructed. The fixed STRIDE (independent of `frontier_limit`) + // keeps every code's cell at a constant offset so the monotonic + // stamps stay valid across calls with different frontiers. + let arena_base = price_arena.as_mut_ptr(); + let mut ll_cache: &mut [[u32; 2]] = + unsafe { core::slice::from_raw_parts_mut(arena_base, HC_OPT_PRICE_STRIDE) }; + let mut ml_cache: &mut [[u32; 2]] = unsafe { + core::slice::from_raw_parts_mut(arena_base.add(HC_OPT_PRICE_STRIDE), HC_OPT_PRICE_STRIDE) + }; + $self.backend.bt_mut().opt_ll_price_stamp = $self + .backend + .bt_mut() + .opt_ll_price_stamp + .wrapping_add(1) + .max(1); + let ll_price_stamp = $self.backend.bt_mut().opt_ll_price_stamp; + $self.backend.bt_mut().opt_lit_price_stamp = $self + .backend + .bt_mut() + .opt_lit_price_stamp + .wrapping_add(1) + .max(1); + let lit_price_stamp = $self.backend.bt_mut().opt_lit_price_stamp; + $self.backend.bt_mut().opt_ml_price_stamp = $self + .backend + .bt_mut() + .opt_ml_price_stamp + .wrapping_add(1) + .max(1); + let ml_price_stamp = $self.backend.bt_mut().opt_ml_price_stamp; + let node0_price = BtMatcher::cached_lit_length_price( + profile, + $stats, + initial_litlen, + &mut ll_cache, + ll_price_stamp, + ); + nodes[0] = HcOptimalNode { + litlen: initial_litlen as u32, + reps: initial_reps, + ..HcOptimalNode::default() + }; + node_prices[0] = node0_price; + let sufficient_len = profile.sufficient_match_len; + let ll0_price = BtMatcher::cached_lit_length_price( + profile, + $stats, + 0, + &mut ll_cache, + ll_price_stamp, + ); + let ll1_price = BtMatcher::cached_lit_length_price( + profile, + $stats, + 1, + &mut ll_cache, + ll_price_stamp, + ); + let mut pos = 1usize; + let mut last_pos = 0usize; + let mut forced_end: Option = None; + let mut forced_end_state: Option = None; + // Price companion of `forced_end_state` (price no longer lives in the + // node struct; tracked alongside the forced-seed node). + let mut forced_end_price: Option = None; + let mut seed_forced_shortest_path = false; + let mut opt_ldm = HcOptLdmState { + seq_store: HcRawSeqStore { + pos: 0, + pos_in_sequence: 0, + size: $self.backend.bt_mut().ldm_sequences.len(), + }, + ..HcOptLdmState::default() + }; + let has_ldm = !$self.backend.bt_mut().ldm_sequences.is_empty(); + if has_ldm { + // `ldm_sequences` are emitted in BLOCK-relative coordinates, + // but this optimal-parser pass runs over a SEGMENT of the + // block starting at block-offset `$block_offset` and uses + // segment-relative positions throughout. Fast-forward the raw + // seq-store cursor past the bytes covered by earlier segments + // so the (segment-relative) LDM windows below land at the + // correct positions. Idempotent: `ldm_skip_raw_seq_store_bytes` + // recomputes from `pos = 0`, so re-running it per segment is + // safe. Without this, every segment after the first re-applied + // the block's leading LDM windows at the wrong offset, emitting + // matches that copy the wrong bytes (undecodable frame). + if ldm_block_offset > 0 { + $self + .backend + .bt_mut() + .ldm_skip_raw_seq_store_bytes(&mut opt_ldm.seq_store, ldm_block_offset); + } + $self + .backend + .bt_mut() + .ldm_get_next_match_and_update_seq_store(&mut opt_ldm, 0, $current_len); + } + + // Upstream zstd-like seed at rPos=0: initialize frontier with matches starting + // at current position before entering the generic forward DP loop. + if $current_len >= min_match_len { + let seed_ldm = if has_ldm { + $self.backend.bt_mut().ldm_process_match_candidate( + &mut opt_ldm, + 0, + $current_len, + min_match_len, + ) + } else { + None + }; + candidates.clear(); + // SAFETY: wrapper is in the same target_feature umbrella as the + // `$collect` kernel variant; the runtime kernel detector already + // gated entry into the wrapper. + unsafe { + $self.$collect::<$strategy_ty, true>( + $current_abs_start, + current_abs_end, + profile, + HcCandidateQuery { + reps: initial_reps, + lit_len: initial_litlen, + ldm_candidate: seed_ldm, + }, + &mut candidates, + ) + }; + if !candidates.is_empty() { + // `min_match_len >= HC_FORMAT_MINMATCH (3)` by invariant. + last_pos = (min_match_len - 1).min(frontier_limit); + for p in 1..min_match_len.min(frontier_buffer_size) { + BtMatcher::reset_opt_node(&mut nodes[p]); + // Reset the price (sole home; the node carries none). + node_prices[p] = u32::MAX; + // `initial_litlen` is the litlen carried from prior + // optimal-plan segments — its real bound is the + // current block length (the frame compressor caps + // block scan at `HC_BLOCKSIZE_MAX`), not the segment + // `current_len`. `p < min_match_len` (small constant), + // so the sum stays well within `u32::MAX`. Use + // `checked_add` FIRST so the `usize` addition itself + // cannot overflow on i686 (where `usize` is 32-bit + // and a wrapping `+` would slip past `try_from`). + let seed_litlen = initial_litlen + .checked_add(p) + .and_then(|s| u32::try_from(s).ok()) + .expect("optimal parser seed litlen out of u32 range"); + nodes[p].litlen = seed_litlen; + } + } + + if let Some(candidate) = candidates.last() { + let longest_len = candidate.match_len.min($current_len); + if longest_len > sufficient_len { + let off_base = BtMatcher::encode_offset_base_with_reps( + candidate.offset as u32, + initial_litlen, + initial_reps, + ); + let off_price = profile + .offset_price_for::($stats, off_base); + let ml_price = BtMatcher::cached_match_length_price( + profile, + $stats, + longest_len, + &mut ml_cache, + ml_price_stamp, + ); + let seq_cost = BtMatcher::add_prices( + ll0_price, + profile.match_price_from_parts(off_price, ml_price, $stats), + ); + let forced_price = BtMatcher::add_prices(node_prices[0], seq_cost); + let forced_state = HcOptimalNode { + off: candidate.offset as u32, + mlen: longest_len as u32, + litlen: 0, + reps: initial_reps, + }; + if longest_len < frontier_buffer_size && forced_price < node_prices[longest_len] { + nodes[longest_len] = forced_state; + node_prices[longest_len] = forced_price; + } + forced_end = Some(longest_len); + forced_end_state = Some(forced_state); + forced_end_price = Some(forced_price); + seed_forced_shortest_path = true; + } + } + if !seed_forced_shortest_path { + let mut prev_max_len = min_match_len - 1; + for candidate in candidates.iter() { + let max_match_len = candidate.match_len.min(frontier_limit); + if max_match_len < min_match_len { + continue; + } + let start_len = (prev_max_len + 1).max(min_match_len); + if start_len > max_match_len { + prev_max_len = prev_max_len.max(max_match_len); + continue; + } + if max_match_len > last_pos { + BtMatcher::reset_opt_nodes( + &mut nodes, + &mut node_prices, + last_pos + 1, + max_match_len, + ); + } + let off_base = BtMatcher::encode_offset_base_with_reps( + candidate.offset as u32, + initial_litlen, + initial_reps, + ); + let off_price = profile + .offset_price_for::($stats, off_base); + debug_assert!(max_match_len < frontier_buffer_size); + let nodes0_price = node_prices[0]; + for match_len in (start_len..=max_match_len).rev() { + let ml_price = BtMatcher::cached_match_length_price( + profile, + $stats, + match_len, + &mut ml_cache, + ml_price_stamp, + ); + let seq_cost = BtMatcher::add_prices( + ll0_price, + profile.match_price_from_parts(off_price, ml_price, $stats), + ); + let next_cost = BtMatcher::add_prices(nodes0_price, seq_cost); + let node_price = unsafe { *node_prices.get_unchecked(match_len) }; + if match_len > last_pos || next_cost < node_price { + let slot = unsafe { nodes.get_unchecked_mut(match_len) }; + *slot = HcOptimalNode { + off: candidate.offset as u32, + mlen: match_len as u32, + litlen: 0, + reps: initial_reps, + }; + unsafe { *node_prices.get_unchecked_mut(match_len) = next_cost }; + if match_len > last_pos { + last_pos = match_len; + } + } else if abort_on_worse_match { + break; + } + } + prev_max_len = prev_max_len.max(max_match_len); + } + if last_pos + 1 < frontier_buffer_size { + node_prices[last_pos + 1] = u32::MAX; + } + } + } + while !seed_forced_shortest_path && pos <= last_pos && pos <= frontier_limit { + debug_assert!(pos + 1 < frontier_buffer_size); + let prev_node = unsafe { *nodes.get_unchecked(pos - 1) }; + let prev_node_price = unsafe { *node_prices.get_unchecked(pos - 1) }; + if prev_node_price != u32::MAX { + let lit_len = prev_node.litlen as usize + 1; + let lit_price = { + let bt = $self.backend.bt_mut(); + BtMatcher::cached_literal_price( + profile, + $stats, + $current[pos - 1], + &mut bt.opt_lit_price_scratch, + &mut bt.opt_lit_price_generation, + lit_price_stamp, + ) + }; + let ll_delta = BtMatcher::cached_lit_length_delta_price( + profile, + $stats, + lit_len, + &mut ll_cache, + ll_price_stamp, + ); + let lit_cost = BtMatcher::add_price_delta(prev_node_price, lit_price, ll_delta); + // `node_pos_price` is the OLD price at `pos` (before the write + // below) — also the price of `prev_match`, the pre-overwrite copy. + let node_pos_price = unsafe { *node_prices.get_unchecked(pos) }; + if lit_cost <= node_pos_price { + let prev_match = unsafe { *nodes.get_unchecked(pos) }; + let slot = unsafe { nodes.get_unchecked_mut(pos) }; + *slot = prev_node; + slot.litlen = lit_len as u32; + node_prices[pos] = lit_cost; + #[allow(clippy::collapsible_if)] + if opt_level + && prev_match.mlen > 0 + && prev_match.litlen == 0 + && pos < $current_len + { + if ll1_price < ll0_price { + let next_lit_price = { + let bt = $self.backend.bt_mut(); + BtMatcher::cached_literal_price( + profile, + $stats, + $current[pos], + &mut bt.opt_lit_price_scratch, + &mut bt.opt_lit_price_generation, + lit_price_stamp, + ) + }; + let with1literal = BtMatcher::add_price_delta( + node_pos_price, + next_lit_price, + ll1_price as i32 - ll0_price as i32, + ); + let ll_delta_next = BtMatcher::cached_lit_length_delta_price( + profile, + $stats, + lit_len + 1, + &mut ll_cache, + ll_price_stamp, + ); + let with_more_literals = + BtMatcher::add_price_delta(lit_cost, next_lit_price, ll_delta_next); + let next = pos + 1; + let next_price = unsafe { *node_prices.get_unchecked(next) }; + if with1literal < with_more_literals && with1literal < next_price { + // Upstream zstd parity (zstd_opt.c:1232): `cur >= prevMatch.mlen`. + debug_assert!(pos >= prev_match.mlen as usize); + let prev_pos = pos - prev_match.mlen as usize; + { + let prev_state = unsafe { *nodes.get_unchecked(prev_pos) }; + let (_, reps_after_match) = BtMatcher::encode_offset_with_reps( + prev_match.off, + prev_state.litlen as usize, + prev_state.reps, + ); + let slot = unsafe { nodes.get_unchecked_mut(next) }; + *slot = prev_match; + slot.reps = reps_after_match; + slot.litlen = 1; + node_prices[next] = with1literal; + if next > last_pos { + last_pos = next; + } + } + } + } + } + } + } + + // Memory-resident DP (upstream zstd parity): read opt[cur] fields on + // demand instead of holding a 28-byte node copy live across the + // per-position `$collect` call below. The held copy forced LLVM + // to spill reps[3] + litlen around the (non-inlinable) call; + // reading the fields fresh on each side keeps them out of the + // cross-call live set. `nodes[pos]` is stable across `$collect` + // (it only fills `candidates`), so post-call reads are identical. + let base_cost = unsafe { *node_prices.get_unchecked(pos) }; + if base_cost == u32::MAX { + pos += 1; + continue; + } + { + let base_node = unsafe { *nodes.get_unchecked(pos) }; + if base_node.mlen > 0 && base_node.litlen == 0 { + // Upstream zstd parity (zstd_opt.c:1255): `cur >= opt[cur].mlen`. + debug_assert!(pos >= base_node.mlen as usize); + let prev_pos = pos - base_node.mlen as usize; + let prev_state = unsafe { *nodes.get_unchecked(prev_pos) }; + let (_, reps_after_match) = BtMatcher::encode_offset_with_reps( + base_node.off, + prev_state.litlen as usize, + prev_state.reps, + ); + unsafe { nodes.get_unchecked_mut(pos).reps = reps_after_match }; + } + } + + if pos + 8 > $current_len { + pos += 1; + continue; + } + + if pos == last_pos { + break; + } + + let next_price = unsafe { *node_prices.get_unchecked(pos + 1) }; + // `saturating_add` is REQUIRED here, not a masked bug: `base_cost` + // is a node price that can be the `u32::MAX` "unreachable" sentinel, + // and saturating keeps `base_cost + margin` pinned at MAX so the + // comparison stays correct. Plain `+` would wrap the sentinel and + // flip the abort decision (a ratio bug / debug overflow panic). + if abort_on_worse_match + && next_price <= base_cost.saturating_add(HC_BITCOST_MULTIPLIER / 2) + { + pos += 1; + continue; + } + + let abs_pos = $current_abs_start + pos; + let ldm_candidate = if has_ldm { + $self.backend.bt_mut().ldm_process_match_candidate( + &mut opt_ldm, + pos, + $current_len - pos, + min_match_len, + ) + } else { + None + }; + candidates.clear(); + // SAFETY: same umbrella as `$collect`. Query fields are read + // fresh here (consumed into the call's argument) so they do not + // stay live across the call; the post-call reads below are a + // separate, fresh load of the same stable `nodes[pos]`. + unsafe { + $self.$collect::<$strategy_ty, true>( + abs_pos, + current_abs_end, + profile, + HcCandidateQuery { + reps: nodes.get_unchecked(pos).reps, + lit_len: nodes.get_unchecked(pos).litlen as usize, + ldm_candidate, + }, + &mut candidates, + ) + }; + // Post-call reads of opt[cur]: fresh, born after `$collect`, so + // never part of the cross-call live set (see memory-resident note + // above). `nodes[pos]` is untouched by `$collect`. + let base_reps = unsafe { nodes.get_unchecked(pos).reps }; + let base_litlen = unsafe { nodes.get_unchecked(pos).litlen as usize }; + if let Some(candidate) = candidates.last() { + let longest_len = candidate.match_len.min($current_len - pos); + if longest_len > sufficient_len + || pos + longest_len >= HC_OPT_NUM + || pos + longest_len >= $current_len + { + let lit_len = base_litlen; + let off_base = BtMatcher::encode_offset_base_with_reps( + candidate.offset as u32, + lit_len, + base_reps, + ); + let off_price = profile + .offset_price_for::($stats, off_base); + let ml_price = BtMatcher::cached_match_length_price( + profile, + $stats, + longest_len, + &mut ml_cache, + ml_price_stamp, + ); + let seq_cost = BtMatcher::add_prices( + ll0_price, + profile.match_price_from_parts(off_price, ml_price, $stats), + ); + let forced_price = BtMatcher::add_prices(base_cost, seq_cost); + let end_pos = (pos + longest_len).min($current_len); + forced_end = Some(end_pos); + forced_end_state = Some(HcOptimalNode { + off: candidate.offset as u32, + mlen: longest_len as u32, + litlen: 0, + reps: base_reps, + }); + forced_end_price = Some(forced_price); + break; + } + } + let mut prev_max_len = min_match_len - 1; + for candidate in candidates.iter() { + // Outer loop guards `pos <= frontier_limit` (see the + // `while ... pos <= frontier_limit` condition); the + // subtraction below is therefore safe. + debug_assert!(pos <= frontier_limit); + let max_match_len = candidate + .match_len + .min($current_len - pos) + .min(frontier_limit - pos); + let min_len = min_match_len; + if max_match_len < min_len { + continue; + } + let start_len = (prev_max_len + 1).max(min_len); + if start_len > max_match_len { + prev_max_len = prev_max_len.max(max_match_len); + continue; + } + let max_next = pos + max_match_len; + if max_next > last_pos { + BtMatcher::reset_opt_nodes( + &mut nodes, + &mut node_prices, + last_pos + 1, + max_next, + ); + } + let lit_len = base_litlen; + let off_base = BtMatcher::encode_offset_base_with_reps( + candidate.offset as u32, + lit_len, + base_reps, + ); + let off_price = profile + .offset_price_for::($stats, off_base); + debug_assert!(pos + max_match_len < frontier_buffer_size); + if abort_on_worse_match { + // btopt (OPT_LEVEL == 0): reverse-iterate with early break — + // once a longer match stops improving, shorter ones are + // skipped. Order-dependent, stays scalar. + for match_len in (start_len..=max_match_len).rev() { + let next = pos + match_len; + let ml_price = BtMatcher::cached_match_length_price( + profile, + $stats, + match_len, + &mut ml_cache, + ml_price_stamp, + ); + let seq_cost = BtMatcher::add_prices( + ll0_price, + profile.match_price_from_parts(off_price, ml_price, $stats), + ); + let next_cost = BtMatcher::add_prices(base_cost, seq_cost); + let node_next_price = unsafe { *node_prices.get_unchecked(next) }; + if next > last_pos || next_cost < node_next_price { + let slot = unsafe { nodes.get_unchecked_mut(next) }; + *slot = HcOptimalNode { + off: candidate.offset as u32, + mlen: match_len as u32, + litlen: 0, + reps: base_reps, + }; + unsafe { *node_prices.get_unchecked_mut(next) = next_cost }; + if next > last_pos { + last_pos = next; + } + } else { + break; + } + } + } else { + // btultra / btultra2 (OPT_LEVEL >= 2): no abort, each + // match_len writes a distinct node => order-independent. + // Dispatch to the per-tier price-set ($priceset is the + // tier's fn: AVX2 SoA-vector compare for the avx2 wrapper, + // inline scalar otherwise) — it folds into this wrapper's + // monomorphisation, so no call ABI / runtime feature check. + #[allow(unused_unsafe)] + { + last_pos = last_pos.max(unsafe { + $priceset( + &mut node_prices, + &mut nodes, + ml_cache, + ml_price_stamp, + profile, + $stats, + pos, + start_len, + max_match_len, + ll0_price, + off_price, + base_cost, + candidate.offset as u32, + base_reps, + last_pos, + ) + }); + } + } + prev_max_len = prev_max_len.max(max_match_len); + } + + if last_pos + 1 < frontier_buffer_size { + unsafe { + *node_prices.get_unchecked_mut(last_pos + 1) = u32::MAX; + } + } + pos += 1; + } + + if last_pos == 0 { + if $current_len == 0 { + let price = node_prices[0]; + return $self.backend.bt_mut().finish_optimal_plan( + HcOptimalPlanBuffers { + nodes, + node_prices, + candidates, + store, + price_arena, + }, + (price, initial_reps, initial_litlen, 0), + ); + } + let lit_price = { + let bt = $self.backend.bt_mut(); + BtMatcher::cached_literal_price( + profile, + $stats, + $current[0], + &mut bt.opt_lit_price_scratch, + &mut bt.opt_lit_price_generation, + lit_price_stamp, + ) + }; + // `initial_litlen` is carried across optimal-plan segments; + // its real bound is the current block length, not + // `current_len`. On i686 (32-bit `usize`) `+ 1` could + // theoretically wrap if the invariant ever broke. Catch + // that explicitly via `checked_add` rather than letting a + // wrapping sum slip into the price lookup. + let next_litlen = initial_litlen + .checked_add(1) + .expect("optimal parser next litlen out of usize range"); + let ll_delta = BtMatcher::cached_lit_length_delta_price( + profile, + $stats, + next_litlen, + &mut ll_cache, + ll_price_stamp, + ); + let price = BtMatcher::add_price_delta(node_prices[0], lit_price, ll_delta); + return $self.backend.bt_mut().finish_optimal_plan( + HcOptimalPlanBuffers { + nodes, + node_prices, + candidates, + store, + price_arena, + }, + (price, initial_reps, next_litlen, 1), + ); + } + + let target_pos = forced_end.unwrap_or(last_pos.min(frontier_limit)); + // Price lives in `node_prices`, not the node struct, so carry the + // final-stretch price alongside its node (forced-seed companion or the + // frontier price at `target_pos`). + let (last_stretch, last_stretch_price) = if let Some(forced_state) = forced_end_state { + (forced_state, forced_end_price.expect("forced state has a price")) + } else { + (nodes[target_pos], node_prices[target_pos]) + }; + if last_stretch_price == u32::MAX { + return $self.backend.bt_mut().finish_optimal_plan( + HcOptimalPlanBuffers { + nodes, + node_prices, + candidates, + store, + price_arena, + }, + (u32::MAX, initial_reps, initial_litlen, $current_len), + ); + } + + if last_stretch.mlen == 0 { + return $self.backend.bt_mut().finish_optimal_plan( + HcOptimalPlanBuffers { + nodes, + node_prices, + candidates, + store, + price_arena, + }, + ( + last_stretch_price, + last_stretch.reps, + last_stretch.litlen as usize, + target_pos.min($current_len), + ), + ); + } + + let mut cur = target_pos.saturating_sub(last_stretch.mlen as usize); + let end_reps = if last_stretch.litlen == 0 { + let prev_state = nodes[cur]; + let (_, reps_after_match) = BtMatcher::encode_offset_with_reps( + last_stretch.off, + prev_state.litlen as usize, + prev_state.reps, + ); + reps_after_match + } else { + let tail_literals = last_stretch.litlen as usize; + if cur < tail_literals { + return $self.backend.bt_mut().finish_optimal_plan( + HcOptimalPlanBuffers { + nodes, + node_prices, + candidates, + store, + price_arena, + }, + ( + last_stretch_price, + last_stretch.reps, + tail_literals, + target_pos.min($current_len), + ), + ); + } + cur -= tail_literals; + last_stretch.reps + }; + let store_end = cur + 2; + if store.len() <= store_end { + store.resize(store_end + 1, HcOptimalNode::default()); + } + let mut store_start; + let mut stretch_pos = cur; + + if last_stretch.litlen > 0 { + store[store_end] = HcOptimalNode { + litlen: last_stretch.litlen, + mlen: 0, + ..HcOptimalNode::default() + }; + store_start = store_end.saturating_sub(1); + store[store_start] = last_stretch; + } + store[store_end] = last_stretch; + store_start = store_end; + + loop { + let next_stretch = nodes[stretch_pos]; + store[store_start].litlen = next_stretch.litlen; + if next_stretch.mlen == 0 { + break; + } + if store_start == 0 { + break; + } + store_start -= 1; + store[store_start] = next_stretch; + // Parser invariant: every emitted stretch is bounded by the + // current block, so `litlen + mlen <= current_len <= + // HC_BLOCKSIZE_MAX (128 KiB)`. The `as usize` widening + raw + // `+` is safe on 32-bit targets — two u32 values do NOT + // automatically fit in `usize` on i686, the block bound is + // what makes this addition safe. + let litlen = next_stretch.litlen as usize; + let mlen = next_stretch.mlen as usize; + debug_assert!(litlen + mlen <= $current_len); + let step = litlen + mlen; + if step == 0 || stretch_pos < step { + break; + } + stretch_pos -= step; + } + + let mut tail_literals = initial_litlen; + let mut store_pos = store_start; + while store_pos <= store_end { + let stretch = store[store_pos]; + let llen = stretch.litlen as usize; + let mlen = stretch.mlen as usize; + if mlen == 0 { + tail_literals = llen; + store_pos += 1; + continue; + } + $out.push(HcOptimalSequence { + offset: stretch.off, + match_len: mlen as u32, + lit_len: llen as u32, + }); + tail_literals = 0; + store_pos += 1; + } + let result = ( + last_stretch_price, + end_reps, + if last_stretch.litlen > 0 { + last_stretch.litlen as usize + } else { + tail_literals + }, + target_pos.min($current_len), + ); + $self.backend.bt_mut().finish_optimal_plan( + HcOptimalPlanBuffers { + nodes, + node_prices, + candidates, + store, + price_arena, + }, + result, + ) + }}; +} + +/// `collect_optimal_candidates_initialized` body parameterized over the per-CPU +/// kernel: the `$cpl` path is the kernel's `common_prefix_len_ptr` (used in +/// the HC chain walk fallback), and the four method-name substitutions +/// (`$bt_update`, `$bt_insert`, `$for_each_rep`, `$hash3`) route to the +/// kernel-specific wrappers of the inner helpers. With every helper under +/// the same `target_feature` umbrella, the entire per-position pipeline +/// (BT-tree fill + rep probing + hash3 probing + BT match collection / +/// HC chain walk) inlines without ABI barriers on the level22 hot path. +macro_rules! collect_optimal_candidates_initialized_body { + ( + $self:expr, + $strategy_ty:ty, + $abs_pos:ident, + $current_abs_end:ident, + $profile:ident, + $query:ident, + $out:ident, + $bt_matchfinder:ident, + $bt_update:ident, + $bt_insert:ident, + $for_each_rep:ident, + $hash3:ident, + $cpl:path $(,)? + ) => {{ + // Per-strategy compile-time const: only BtUltra2 drives the + // hash3 short-match table. All other monomorphisations drop + // the entire hash3 lookup block at codegen time. The relaxed + // implication enforces only the direction we depend on: + // if the strategy declares hash3, the table must be live. + // The reverse (`hash3_log != 0` without `USE_HASH3`) is OK — + // a future caller may pre-allocate hash3 storage without + // wiring the BtUltra2 path through. + let use_hash3: bool = <$strategy_ty as crate::encoding::strategy::Strategy>::USE_HASH3; + debug_assert!(!$self.table.hash_table.is_empty()); + debug_assert!($self.table.hash3_log == 0 || !$self.table.hash3_table.is_empty()); + debug_assert!( + !use_hash3 || $self.table.hash3_log != 0, + "Strategy::USE_HASH3 = true but runtime hash3_log is 0 — call configure() first", + ); + debug_assert!(!$self.table.chain_table.is_empty()); + let min_match_len = HC_OPT_MIN_MATCH_LEN; + let reps = $query.reps; + let lit_len = $query.lit_len; + let ldm_candidate = $query.ldm_candidate; + $out.clear(); + if $abs_pos < $self.table.skip_insert_until_abs { + if let Some(ldm) = ldm_candidate { + let mut best_len_for_skip = 0usize; + let _ = crate::encoding::bt::BtMatcher::push_candidate_ladder( + $out, + &mut best_len_for_skip, + ldm, + min_match_len, + ); + } + return; + } + if $bt_matchfinder { + // SAFETY: caller is in the same target_feature umbrella as + // `$bt_update`; the runtime kernel detector already gated entry. + unsafe { $self.table.$bt_update($abs_pos, $current_abs_end) }; + } + let current_idx = $abs_pos - $self.table.history_abs_start; + if current_idx + 4 > $self.table.live_history().len() { + if let Some(ldm) = ldm_candidate { + let mut best_len_for_skip = 0usize; + let _ = crate::encoding::bt::BtMatcher::push_candidate_ladder( + $out, + &mut best_len_for_skip, + ldm, + min_match_len, + ); + } + return; + } + let mut best_len_for_skip = 0usize; + let mut skip_further_match_search = false; + let mut rep_len_candidate_found = false; + // SAFETY: same umbrella; closure capture is monomorphized per call. + unsafe { + $self.hc.$for_each_rep( + &$self.table, + $abs_pos, + lit_len, + reps, + $current_abs_end, + min_match_len, + |rep| { + if rep.match_len >= min_match_len { + rep_len_candidate_found = true; + } + let _ = crate::encoding::bt::BtMatcher::push_candidate_ladder( + $out, + &mut best_len_for_skip, + rep, + min_match_len, + ); + if rep.match_len > $profile.sufficient_match_len { + skip_further_match_search = true; + } + // `for_each_repcode_candidate_with_reps` caps + // `rep.match_len` at the per-call `tail_limit = + // current_abs_end - abs_pos`, so `abs_pos + + // rep.match_len <= current_abs_end`. The raw sum + // therefore stays in `usize` on every supported + // target. + if $abs_pos + rep.match_len >= $current_abs_end { + skip_further_match_search = true; + } + }, + ) + }; + // Hash3 lookup runs only when the strategy enables it. The + // `use_hash3` binding above is a per-monomorphisation const, + // so non-BtUltra2 instances drop this entire block. + if use_hash3 && !skip_further_match_search && best_len_for_skip < min_match_len { + $self.table.update_hash3_until($abs_pos); + // SAFETY: same umbrella for hash3_candidate. + if let Some(h3) = unsafe { + $self + .table + .$hash3($abs_pos, $current_abs_end, min_match_len) + } { + let _ = crate::encoding::bt::BtMatcher::push_candidate_ladder( + $out, + &mut best_len_for_skip, + h3, + min_match_len, + ); + if !rep_len_candidate_found + && (h3.match_len > $profile.sufficient_match_len + || $abs_pos + h3.match_len >= $current_abs_end) + { + $self.table.skip_insert_until_abs = $abs_pos + 1; + skip_further_match_search = true; + } + } + } + if !skip_further_match_search && $bt_matchfinder { + // SAFETY: same umbrella for bt_insert_and_collect_matches. + unsafe { + $self.table.$bt_insert( + $abs_pos, + $current_abs_end, + $profile, + min_match_len, + &mut best_len_for_skip, + $out, + ) + }; + } else if !skip_further_match_search { + $self.table.insert_position($abs_pos); + let max_chain_depth = $profile.max_chain_depth.min($self.hc.search_depth); + let concat = $self.table.live_history(); + // Raw `+ 9` is safe here — see `bt_insert_step_no_rebase_body!` + // for the full discussion of the upstream `STREAM_ABS_HEADROOM` + // cap in `MatchTable::add_data`. + let mut match_end_abs = $abs_pos + 9; + if max_chain_depth > 0 { + for (visited, candidate_abs) in $self + .hc + .chain_candidates(&$self.table, $abs_pos) + .into_iter() + .enumerate() + { + if visited >= max_chain_depth { + break; + } + if candidate_abs == usize::MAX { + break; + } + if candidate_abs < $self.table.window_low_abs_for_target($abs_pos) + || candidate_abs >= $abs_pos + { + continue; + } + let candidate_idx = candidate_abs - $self.table.history_abs_start; + debug_assert!( + $abs_pos <= $current_abs_end, + "HC chain walker called past current block end" + ); + let tail_limit = $current_abs_end - $abs_pos; + let base = concat.as_ptr(); + // SAFETY: history-relative indices; `tail_limit` bounds + // the scan within `concat`. `$cpl` is the kernel-specific + // common_prefix_len_ptr — call inlines because the + // surrounding wrapper carries the same target_feature. + let match_len = + unsafe { $cpl(base.add(candidate_idx), base.add(current_idx), tail_limit) }; + if match_len < min_match_len { + continue; + } + let offset = $abs_pos - candidate_abs; + if crate::encoding::bt::BtMatcher::push_candidate_ladder( + $out, + &mut best_len_for_skip, + MatchCandidate { + start: $abs_pos, + offset, + match_len, + }, + min_match_len, + ) { + let candidate_end = candidate_abs + match_len; + if candidate_end > match_end_abs { + match_end_abs = candidate_end; + } + } + if match_len > HC_OPT_NUM || $abs_pos + match_len >= $current_abs_end { + break; + } + } + } + // `match_end_abs` initialized to `abs_pos + 9`; monotonic + // updates only ever extend it, so `match_end_abs - 8 >= 1`. + $self.table.skip_insert_until_abs = + $self.table.skip_insert_until_abs.max(match_end_abs - 8); + } + if let Some(ldm) = ldm_candidate { + let _ = crate::encoding::bt::BtMatcher::push_candidate_ladder( + $out, + &mut best_len_for_skip, + ldm, + min_match_len, + ); + } + }}; +} +impl HcMatchGenerator { + /// Upstream zstd `ZSTD_btlazy2` (levels 13-15): binary-tree match finder with a + /// greedy/lazy parse. Bare dispatcher — resolves the runtime tier ONCE + /// per block via `select_kernel()` and calls the matching + /// `start_matching_btlazy2_` wrapper, so the per-position BT + /// collect runs under a single `#[target_feature]` umbrella (mirrors + /// `build_optimal_plan_impl`). See `start_matching_btlazy2_body!` for the + /// shared loop. + pub(crate) fn start_matching_btlazy2( + &mut self, + mut handle_sequence: impl for<'a> FnMut(Sequence<'a>), + ) { + #[cfg(all(target_arch = "aarch64", target_endian = "little"))] + unsafe { + self.start_matching_btlazy2_neon(&mut handle_sequence) + } + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + use crate::encoding::fastpath::{FastpathKernel, select_kernel}; + match select_kernel() { + FastpathKernel::Avx2Bmi2 => unsafe { + self.start_matching_btlazy2_avx2_bmi2(&mut handle_sequence) + }, + FastpathKernel::Sse42 => unsafe { + self.start_matching_btlazy2_sse42(&mut handle_sequence) + }, + FastpathKernel::Scalar => self.start_matching_btlazy2_scalar(&mut handle_sequence), + } + } + #[cfg(not(any( + all(target_arch = "aarch64", target_endian = "little"), + target_arch = "x86", + target_arch = "x86_64" + )))] + { + self.start_matching_btlazy2_scalar(&mut handle_sequence) + } + } + + #[cfg(all(target_arch = "aarch64", target_endian = "little"))] + #[target_feature(enable = "neon")] + unsafe fn start_matching_btlazy2_neon( + &mut self, + mut handle_sequence: impl for<'a> FnMut(Sequence<'a>), + ) { + start_matching_btlazy2_body!( + self, + handle_sequence, + collect_optimal_candidates_initialized_neon, + crate::encoding::fastpath::neon::count_match_from_indices + ) + } + + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + #[target_feature(enable = "sse4.2")] + unsafe fn start_matching_btlazy2_sse42( + &mut self, + mut handle_sequence: impl for<'a> FnMut(Sequence<'a>), + ) { + start_matching_btlazy2_body!( + self, + handle_sequence, + collect_optimal_candidates_initialized_sse42, + crate::encoding::fastpath::sse42::count_match_from_indices + ) + } + + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + #[target_feature(enable = "avx2,bmi2")] + unsafe fn start_matching_btlazy2_avx2_bmi2( + &mut self, + mut handle_sequence: impl for<'a> FnMut(Sequence<'a>), + ) { + start_matching_btlazy2_body!( + self, + handle_sequence, + collect_optimal_candidates_initialized_avx2_bmi2, + crate::encoding::fastpath::avx2_bmi2::count_match_from_indices + ) + } + + // Scalar wrapper: no `#[target_feature]`; `$collect` (the scalar collect) + // is a safe fn, so the body macro's `unsafe` block is inert here. Same cfg + // as `collect_optimal_candidates_initialized_scalar` (absent on + // aarch64-little, where NEON is the baseline tier). + #[cfg(not(all(target_arch = "aarch64", target_endian = "little")))] + #[allow(unused_unsafe)] + pub(crate) fn start_matching_btlazy2_scalar( + &mut self, + mut handle_sequence: impl for<'a> FnMut(Sequence<'a>), + ) { + start_matching_btlazy2_body!( + self, + handle_sequence, + collect_optimal_candidates_initialized_scalar, + crate::encoding::fastpath::scalar::count_match_from_indices + ) + } + + pub(crate) fn start_matching_optimal( + &mut self, + mut handle_sequence: impl for<'a> FnMut(Sequence<'a>), + ) { + self.table.ensure_tables(); + // Borrowed-aware: owned → last committed chunk; borrowed → staged + // in-place block range. + let (current_abs_start, current_len) = self.table.current_block_range(); + if current_len == 0 { + return; + } + let current_ptr = self.table.get_last_space().as_ptr(); + // `start_matching_optimal()` mutates tables/state but never mutates or + // reallocates `self.table.history`, so this tail slice remains valid for + // the duration of the routine and avoids cloning the full block. + let current = unsafe { core::slice::from_raw_parts(current_ptr, current_len) }; + + let current_abs_end = current_abs_start + current_len; + self.table + .apply_limited_update_after_long_match(current_abs_start); + let hash3_start_cursor = self + .table + .skip_insert_until_abs + .max(self.table.history_abs_start); + self.table + .backfill_boundary_positions(current_abs_start, current_abs_end); + self.table.next_to_update3 = hash3_start_cursor; + // Borrow split: `prepare_ldm_candidates` needs immutable + // access to the live history (the post-`history_start` + // slice of `self.table.history`) while it mutates the LDM + // bucket table owned by `self.backend.bt_mut()`. Both live + // in disjoint fields of `Self`, so we capture the slice + + // its base before reaching for `bt_mut()`. + // + // The producer operates in absolute stream coordinates + // throughout; `live_history[0]` corresponds to absolute + // `history_abs_start` (upstream zstd `base + dictLimit`), and the + // abs→slice translation happens inside the producer at + // each `live_history[..]` access. Passing the full + // `history` Vec would index into the dead prefix (the + // bytes already retired past `history_start`). + let live_history = self.table.live_history(); + let history_abs_start = self.table.history_abs_start; + self.backend.bt_mut().prepare_ldm_candidates( + live_history, + history_abs_start, + current_abs_start, + current_len, + ); + + if self.should_run_btultra2_seed_pass::(current_len) { + self.run_btultra2_seed_pass(current, current_abs_start, current_len); + } + + // Const-generic profile selection: every field is folded from + // S's associated consts (MAX_CHAIN_DEPTH / + // SUFFICIENT_MATCH_LEN / ACCURATE_PRICE / FAVOR_SMALL_OFFSETS), + // so the optimiser produces the literal at codegen time + // without a runtime match. + let profile = HcOptimalCostProfile::const_for_strategy::(); + let mut opt_state = + core::mem::replace(&mut self.backend.bt_mut().opt_state, HcOptState::new()); + opt_state.rescale_freqs(current, profile); + let mut best_plan = core::mem::take(&mut self.backend.bt_mut().opt_segment_plan_scratch); + best_plan.clear(); + let mut plan_reps = self.table.offset_hist; + let (mut cursor, mut plan_litlen) = + self.table.opt_start_cursor_and_litlen(current_abs_start); + let mut plan_literals_cursor = 0usize; + let match_loop_limit = current_len.saturating_sub(8); + while cursor < match_loop_limit { + let remaining_len = current_len - cursor; + let segment_abs_start = current_abs_start + cursor; + let segment_start = best_plan.len(); + let (_, end_reps, end_litlen, consumed_len) = self.build_optimal_plan::( + ¤t[cursor..], + segment_abs_start, + remaining_len, + HcOptimalPlanState { + block_offset: cursor, + reps: plan_reps, + litlen: plan_litlen, + profile, + }, + &opt_state, + &mut best_plan, + ); + BtMatcher::update_plan_stats_segment( + current, + current_len, + &best_plan[segment_start..], + &mut plan_literals_cursor, + &mut plan_reps, + &mut opt_state, + profile.accurate, + ); + plan_reps = end_reps; + plan_litlen = end_litlen; + cursor += consumed_len; + } + + self.table + .emit_optimal_plan(current_len, &best_plan, &mut handle_sequence); + best_plan.clear(); + self.backend.bt_mut().opt_segment_plan_scratch = best_plan; + self.backend.bt_mut().opt_state = opt_state; + } + + fn run_btultra2_seed_pass( + &mut self, + current: &[u8], + current_abs_start: usize, + current_len: usize, + ) { + // The seed pass is BtUltra2-exclusive by name (the only + // caller is `should_run_btultra2_seed_pass`), so pin `S` to + // `BtUltra2` for both the cost-profile lookup and the + // `build_optimal_plan::` call below. + type S = crate::encoding::strategy::BtUltra2; + let seed_profile = HcOptimalCostProfile::const_for_strategy::(); + let mut opt_state = + core::mem::replace(&mut self.backend.bt_mut().opt_state, HcOptState::new()); + opt_state.rescale_freqs(current, seed_profile); + let mut seed_reps = self.table.offset_hist; + let (mut cursor, mut seed_litlen) = + self.table.opt_start_cursor_and_litlen(current_abs_start); + let mut seed_literals_cursor = 0usize; + let mut seed_plan = core::mem::take(&mut self.backend.bt_mut().opt_seed_plan_scratch); + seed_plan.clear(); + let match_loop_limit = current_len.saturating_sub(8); + while cursor < match_loop_limit { + let remaining_len = current_len - cursor; + let segment_abs_start = current_abs_start + cursor; + let segment_start = seed_plan.len(); + let (_, end_reps, end_litlen, consumed_len) = self.build_optimal_plan::( + ¤t[cursor..], + segment_abs_start, + remaining_len, + HcOptimalPlanState { + block_offset: cursor, + reps: seed_reps, + litlen: seed_litlen, + profile: seed_profile, + }, + &opt_state, + &mut seed_plan, + ); + BtMatcher::update_plan_stats_segment( + current, + current_len, + &seed_plan[segment_start..], + &mut seed_literals_cursor, + &mut seed_reps, + &mut opt_state, + seed_profile.accurate, + ); + seed_plan.truncate(segment_start); + seed_reps = end_reps; + seed_litlen = end_litlen; + cursor += consumed_len; + } + seed_plan.clear(); + self.backend.bt_mut().opt_seed_plan_scratch = seed_plan; + self.backend.bt_mut().opt_state = opt_state; + + // Upstream zstd initStats_ultra keeps the collected entropy statistics but + // invalidates the first-pass matchfinder history before the real pass. + self.table.position_base = self.table.history_abs_start; + self.table.index_shift = current_len; + self.table.next_to_update3 = current_abs_start; + self.table.skip_insert_until_abs = current_abs_start; + // Upstream zstd `ZSTD_initStats_ultra()` invalidates the first scan by moving + // `window.base` back by `srcSize`, making the real pass start at + // `curr == srcSize` instead of 0. Position 0 is therefore a valid + // table entry in the second pass even though raw C tables reserve + // value 0 as empty during an unshifted first pass. + self.table.allow_zero_relative_position = true; + } + + fn build_optimal_plan( + &mut self, + current: &[u8], + current_abs_start: usize, + current_len: usize, + initial_state: HcOptimalPlanState, + stats: &HcOptState, + out: &mut Vec, + ) -> (u32, [u32; 3], usize, usize) { + debug_assert!(S::USE_BT, "build_optimal_plan called on non-BT strategy"); + debug_assert_eq!(initial_state.profile.accurate, S::ACCURATE_PRICE); + debug_assert_eq!( + initial_state.profile.favor_small_offsets, + S::FAVOR_SMALL_OFFSETS + ); + // `S::ACCURATE_PRICE` / `S::FAVOR_SMALL_OFFSETS` cannot appear + // as const-generic arguments yet (`generic_const_exprs` is + // still unstable), so dispatch over a 4-arm match — but on the + // strategy's ASSOCIATED CONSTS, not the runtime profile (the + // `debug_assert_eq`s above pin the runtime profile to those + // consts). A const scrutinee folds the three dead arms at + // monomorphisation; matching the runtime profile instead kept + // all four `#[inline(always)]` DP bodies (~16 KB each) alive in + // EVERY `S` instantiation — ~360 KB of the wasm payload. + match (S::ACCURATE_PRICE, S::FAVOR_SMALL_OFFSETS) { + (true, false) => self.build_optimal_plan_impl::( + current, + current_abs_start, + current_len, + initial_state, + stats, + out, + ), + (true, true) => self.build_optimal_plan_impl::( + current, + current_abs_start, + current_len, + initial_state, + stats, + out, + ), + (false, false) => self.build_optimal_plan_impl::( + current, + current_abs_start, + current_len, + initial_state, + stats, + out, + ), + (false, true) => self.build_optimal_plan_impl::( + current, + current_abs_start, + current_len, + initial_state, + stats, + out, + ), + } + } + + /// Cross-platform DP entry. Picks the kernel-specific variant so the + /// entire optimal-parser DP body (per-position match gathering, price + /// updates, traceback) runs inside a single `target_feature` umbrella + /// alongside the per-position `collect_optimal_candidates_initialized_ + /// `. This eliminates the final ABI barrier on the hot per- + /// position match-collection call — the level22 critical path is now + /// one straight-line inline chain from DP body down through BT walk + /// and match-length probes. + #[inline(always)] + fn build_optimal_plan_impl< + S: crate::encoding::strategy::Strategy, + const ACCURATE_PRICE: bool, + const FAVOR_SMALL_OFFSETS: bool, + >( + &mut self, + current: &[u8], + current_abs_start: usize, + current_len: usize, + initial_state: HcOptimalPlanState, + stats: &HcOptState, + out: &mut Vec, + ) -> (u32, [u32; 3], usize, usize) { + #[cfg(all(target_arch = "aarch64", target_endian = "little"))] + unsafe { + self.build_optimal_plan_impl_neon::( + current, + current_abs_start, + current_len, + initial_state, + stats, + out, + ) + } + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + use crate::encoding::fastpath::{FastpathKernel, select_kernel}; + match select_kernel() { + FastpathKernel::Avx2Bmi2 => unsafe { + self.build_optimal_plan_impl_avx2_bmi2::( + current, + current_abs_start, + current_len, + initial_state, + stats, + out, + ) + }, + FastpathKernel::Sse42 => unsafe { + self.build_optimal_plan_impl_sse42::( + current, + current_abs_start, + current_len, + initial_state, + stats, + out, + ) + }, + FastpathKernel::Scalar => self + .build_optimal_plan_impl_scalar::( + current, + current_abs_start, + current_len, + initial_state, + stats, + out, + ), + } + } + // wasm with simd128: route through the simd128 DP body (4-lane price-set). + #[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] + unsafe { + self.build_optimal_plan_impl_simd128::( + current, + current_abs_start, + current_len, + initial_state, + stats, + out, + ) + } + #[cfg(not(any( + all(target_arch = "aarch64", target_endian = "little"), + target_arch = "x86", + target_arch = "x86_64", + all(target_arch = "wasm32", target_feature = "simd128") + )))] + { + self.build_optimal_plan_impl_scalar::( + current, + current_abs_start, + current_len, + initial_state, + stats, + out, + ) + } + } + + /// NEON-umbrella DP body. Inlines + /// `collect_optimal_candidates_initialized_neon` (and its entire + /// per-position pipeline) directly into the DP loop. + #[cfg(all(target_arch = "aarch64", target_endian = "little"))] + #[target_feature(enable = "neon")] + unsafe fn build_optimal_plan_impl_neon< + S: crate::encoding::strategy::Strategy, + const ACCURATE_PRICE: bool, + const FAVOR_SMALL_OFFSETS: bool, + >( + &mut self, + current: &[u8], + current_abs_start: usize, + current_len: usize, + initial_state: HcOptimalPlanState, + stats: &HcOptState, + out: &mut Vec, + ) -> (u32, [u32; 3], usize, usize) { + build_optimal_plan_impl_body!( + self, + S, + current, + current_abs_start, + current_len, + initial_state, + stats, + out, + collect_optimal_candidates_initialized_neon, + crate::encoding::hc::priceset::priceset_range_nonabort_neon, + ) + } + + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + #[target_feature(enable = "sse4.2")] + unsafe fn build_optimal_plan_impl_sse42< + S: crate::encoding::strategy::Strategy, + const ACCURATE_PRICE: bool, + const FAVOR_SMALL_OFFSETS: bool, + >( + &mut self, + current: &[u8], + current_abs_start: usize, + current_len: usize, + initial_state: HcOptimalPlanState, + stats: &HcOptState, + out: &mut Vec, + ) -> (u32, [u32; 3], usize, usize) { + build_optimal_plan_impl_body!( + self, + S, + current, + current_abs_start, + current_len, + initial_state, + stats, + out, + collect_optimal_candidates_initialized_sse42, + crate::encoding::hc::priceset::priceset_range_nonabort_sse41, + ) + } + + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + #[target_feature(enable = "avx2,bmi2")] + unsafe fn build_optimal_plan_impl_avx2_bmi2< + S: crate::encoding::strategy::Strategy, + const ACCURATE_PRICE: bool, + const FAVOR_SMALL_OFFSETS: bool, + >( + &mut self, + current: &[u8], + current_abs_start: usize, + current_len: usize, + initial_state: HcOptimalPlanState, + stats: &HcOptState, + out: &mut Vec, + ) -> (u32, [u32; 3], usize, usize) { + build_optimal_plan_impl_body!( + self, + S, + current, + current_abs_start, + current_len, + initial_state, + stats, + out, + collect_optimal_candidates_initialized_avx2_bmi2, + crate::encoding::hc::priceset::priceset_range_nonabort_avx2, + ) + } + + #[cfg(not(all(target_arch = "aarch64", target_endian = "little")))] + // Body macros wrap callees in `unsafe { }` for the NEON/AVX/SSE + // variants where callees are `unsafe fn`. The scalar wrappers route + // through safe fns, so those blocks are redundant on this path. + #[allow(unused_unsafe)] + // The dispatch reaches this only on non-SIMD x86 (Scalar tier) and the + // portable fallback; on wasm+simd128 the simd128 wrapper is selected, so + // this is cfg-dead there. + #[cfg_attr( + all(target_arch = "wasm32", target_feature = "simd128"), + allow(dead_code) + )] + fn build_optimal_plan_impl_scalar< + S: crate::encoding::strategy::Strategy, + const ACCURATE_PRICE: bool, + const FAVOR_SMALL_OFFSETS: bool, + >( + &mut self, + current: &[u8], + current_abs_start: usize, + current_len: usize, + initial_state: HcOptimalPlanState, + stats: &HcOptState, + out: &mut Vec, + ) -> (u32, [u32; 3], usize, usize) { + build_optimal_plan_impl_body!( + self, + S, + current, + current_abs_start, + current_len, + initial_state, + stats, + out, + collect_optimal_candidates_initialized_scalar, + crate::encoding::hc::priceset::priceset_range_nonabort_scalar, + ) + } + + /// wasm `simd128`-umbrella DP body: scalar candidate collection (no wasm + /// collect kernel) but the simd128 4-lane price-set. + #[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] + #[target_feature(enable = "simd128")] + // With `+simd128` in the wasm baseline the shared body macro's `unsafe` + // blocks (needed by the safe scalar wrapper) are redundant inside this + // target_feature fn. + #[allow(unused_unsafe)] + unsafe fn build_optimal_plan_impl_simd128< + S: crate::encoding::strategy::Strategy, + const ACCURATE_PRICE: bool, + const FAVOR_SMALL_OFFSETS: bool, + >( + &mut self, + current: &[u8], + current_abs_start: usize, + current_len: usize, + initial_state: HcOptimalPlanState, + stats: &HcOptState, + out: &mut Vec, + ) -> (u32, [u32; 3], usize, usize) { + build_optimal_plan_impl_body!( + self, + S, + current, + current_abs_start, + current_len, + initial_state, + stats, + out, + collect_optimal_candidates_initialized_scalar, + crate::encoding::hc::priceset::priceset_range_nonabort_simd128, + ) + } + + #[cfg(test)] + pub(crate) fn collect_optimal_candidates( + &mut self, + abs_pos: usize, + current_abs_end: usize, + profile: HcOptimalCostProfile, + query: HcCandidateQuery, + out: &mut Vec, + ) { + use crate::encoding::strategy::{self, StrategyTag}; + self.table.ensure_tables(); + // Dispatch purely from `self.strategy_tag` (set by + // `configure()`). Tests must configure the matcher the same + // way production does — wiring up `table.hash3_log` directly + // without setting a matching `strategy_tag` is no longer + // allowed. + match self.strategy_tag { + StrategyTag::BtUltra2 => self + .collect_optimal_candidates_initialized::( + abs_pos, + current_abs_end, + profile, + query, + out, + ), + StrategyTag::BtUltra => self + .collect_optimal_candidates_initialized::( + abs_pos, + current_abs_end, + profile, + query, + out, + ), + StrategyTag::Btlazy2 => self + .collect_optimal_candidates_initialized::( + abs_pos, + current_abs_end, + profile, + query, + out, + ), + StrategyTag::BtOpt => self + .collect_optimal_candidates_initialized::( + abs_pos, + current_abs_end, + profile, + query, + out, + ), + StrategyTag::Fast | StrategyTag::Dfast | StrategyTag::Greedy | StrategyTag::Lazy => { + self.collect_optimal_candidates_initialized::( + abs_pos, + current_abs_end, + profile, + query, + out, + ) + } + } + } + + /// Cross-platform entry. Picks the kernel-specific variant so the per- + /// position pipeline (BT-tree fill, rep probing, hash3 probing, BT + /// collect / HC chain walk) runs inside a single `target_feature` + /// umbrella — all inner SIMD probes inline without ABI barriers. + /// + /// The on-encode hot path bypasses this dispatcher: `build_optimal_plan_impl_` + /// calls the matching `_` variant directly. This entry is kept + /// for the cfg(test)-only `collect_optimal_candidates` shim and any + /// future caller that isn't already inside a kernel umbrella. + #[allow(dead_code)] + #[inline(always)] + pub(crate) fn collect_optimal_candidates_initialized< + S: crate::encoding::strategy::Strategy, + const USE_BT_MATCHFINDER: bool, + >( + &mut self, + abs_pos: usize, + current_abs_end: usize, + profile: HcOptimalCostProfile, + query: HcCandidateQuery, + out: &mut Vec, + ) { + #[cfg(all(target_arch = "aarch64", target_endian = "little"))] + unsafe { + self.collect_optimal_candidates_initialized_neon::( + abs_pos, + current_abs_end, + profile, + query, + out, + ) + } + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + use crate::encoding::fastpath::{FastpathKernel, select_kernel}; + match select_kernel() { + FastpathKernel::Avx2Bmi2 => unsafe { + self.collect_optimal_candidates_initialized_avx2_bmi2::( + abs_pos, + current_abs_end, + profile, + query, + out, + ) + }, + FastpathKernel::Sse42 => unsafe { + self.collect_optimal_candidates_initialized_sse42::( + abs_pos, + current_abs_end, + profile, + query, + out, + ) + }, + FastpathKernel::Scalar => self + .collect_optimal_candidates_initialized_scalar::( + abs_pos, + current_abs_end, + profile, + query, + out, + ), + } + } + #[cfg(not(any( + all(target_arch = "aarch64", target_endian = "little"), + target_arch = "x86", + target_arch = "x86_64" + )))] + { + self.collect_optimal_candidates_initialized_scalar::( + abs_pos, + current_abs_end, + profile, + query, + out, + ) + } + } + + /// NEON-umbrella variant. Every inner helper (`bt_update_tree_until_neon`, + /// `for_each_repcode_candidate_with_reps_neon`, `hash3_candidate_neon`, + /// `bt_insert_and_collect_matches_neon`, `fastpath::neon:: + /// common_prefix_len_ptr`) shares the NEON umbrella so the per-position + /// pipeline executes as a single straight-line inline sequence. + #[cfg(all(target_arch = "aarch64", target_endian = "little"))] + #[target_feature(enable = "neon")] + unsafe fn collect_optimal_candidates_initialized_neon< + S: crate::encoding::strategy::Strategy, + const USE_BT_MATCHFINDER: bool, + >( + &mut self, + abs_pos: usize, + current_abs_end: usize, + profile: HcOptimalCostProfile, + query: HcCandidateQuery, + out: &mut Vec, + ) { + collect_optimal_candidates_initialized_body!( + self, + S, + abs_pos, + current_abs_end, + profile, + query, + out, + USE_BT_MATCHFINDER, + bt_update_tree_until_neon, + bt_insert_and_collect_matches_neon, + for_each_repcode_candidate_with_reps_neon, + hash3_candidate_neon, + crate::encoding::fastpath::neon::common_prefix_len_ptr, + ) + } + + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + #[target_feature(enable = "sse4.2")] + unsafe fn collect_optimal_candidates_initialized_sse42< + S: crate::encoding::strategy::Strategy, + const USE_BT_MATCHFINDER: bool, + >( + &mut self, + abs_pos: usize, + current_abs_end: usize, + profile: HcOptimalCostProfile, + query: HcCandidateQuery, + out: &mut Vec, + ) { + collect_optimal_candidates_initialized_body!( + self, + S, + abs_pos, + current_abs_end, + profile, + query, + out, + USE_BT_MATCHFINDER, + bt_update_tree_until_sse42, + bt_insert_and_collect_matches_sse42, + for_each_repcode_candidate_with_reps_sse42, + hash3_candidate_sse42, + crate::encoding::fastpath::sse42::common_prefix_len_ptr, + ) + } + + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + #[target_feature(enable = "avx2,bmi2")] + unsafe fn collect_optimal_candidates_initialized_avx2_bmi2< + S: crate::encoding::strategy::Strategy, + const USE_BT_MATCHFINDER: bool, + >( + &mut self, + abs_pos: usize, + current_abs_end: usize, + profile: HcOptimalCostProfile, + query: HcCandidateQuery, + out: &mut Vec, + ) { + collect_optimal_candidates_initialized_body!( + self, + S, + abs_pos, + current_abs_end, + profile, + query, + out, + USE_BT_MATCHFINDER, + bt_update_tree_until_avx2_bmi2, + bt_insert_and_collect_matches_avx2_bmi2, + for_each_repcode_candidate_with_reps_avx2_bmi2, + hash3_candidate_avx2_bmi2, + crate::encoding::fastpath::avx2_bmi2::common_prefix_len_ptr, + ) + } + + #[cfg(not(all(target_arch = "aarch64", target_endian = "little")))] + // Macro emits `unsafe { }` wrappers for NEON/AVX/SSE variants; scalar + // callees are safe so the blocks are redundant here only. + #[allow(unused_unsafe)] + pub(crate) fn collect_optimal_candidates_initialized_scalar< + S: crate::encoding::strategy::Strategy, + const USE_BT_MATCHFINDER: bool, + >( + &mut self, + abs_pos: usize, + current_abs_end: usize, + profile: HcOptimalCostProfile, + query: HcCandidateQuery, + out: &mut Vec, + ) { + collect_optimal_candidates_initialized_body!( + self, + S, + abs_pos, + current_abs_end, + profile, + query, + out, + USE_BT_MATCHFINDER, + bt_update_tree_until_scalar, + bt_insert_and_collect_matches_scalar, + for_each_repcode_candidate_with_reps_scalar, + hash3_candidate_scalar, + crate::encoding::fastpath::scalar::common_prefix_len_ptr, + ) + } +} diff --git a/zstd/src/encoding/hc/priceset.rs b/zstd/src/encoding/hc/priceset.rs new file mode 100644 index 000000000..6d067b61e --- /dev/null +++ b/zstd/src/encoding/hc/priceset.rs @@ -0,0 +1,625 @@ +//! Per-CPU-tier SIMD kernels for the optimal parser's price-set update. +//! +//! The btopt / btultra parser refreshes a window of node prices each step. +//! `priceset_range_nonabort_*` are the per-tier kernels (scalar / AVX2 / NEON / +//! SSE4.1 / wasm `simd128`) that the optimal-plan driver dispatches by runtime +//! CPU-feature detection, with the scalar path as the universal fallback. +//! Moved verbatim from `match_generator.rs` (no behaviour change). The kernels +//! read the optimal-parser node / price types and update the BT matcher's +//! price-set scratch; the per-lane math itself is `core::arch` intrinsics. + +use super::super::bt::BtMatcher; +use super::super::cost_model::{HcOptState, HcOptimalCostProfile}; +use super::super::opt::types::HcOptimalNode; + +/// 8-lane `next_cost < node_price` mask for the optimal-parser price-set +/// loop. AVX2 lacks an unsigned `cmplt`, so derive `nc < np` from +/// `min_epu32`: `nc <= np` iff `min(nc,np) == nc`, then exclude equality. +/// Returns a bitmask (bit `k` set => lane `k` improves). Scalar fallback +/// for non-x86 / no-AVX2. +/// 8-lane `next_cost < node_price` mask for the optimal-parser price-set +/// loop. AVX2 lacks an unsigned `cmplt`, so derive `nc < np` from +/// `min_epu32`: `nc <= np` iff `min(nc,np) == nc`, then exclude equality. +/// Returns a bitmask (bit `k` set => lane `k` improves). Compiled on every +/// x86 target (same as the avx2 collect kernel); the cargo `kernel_avx2` +/// feature only gates the runtime dispatch, not compilation. +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +#[target_feature(enable = "avx2")] +unsafe fn priceset_improved_mask8_avx2(next_cost: &[u32; 8], node_price: &[u32]) -> u8 { + #[cfg(target_arch = "x86")] + use core::arch::x86::{ + __m256i, _mm256_andnot_si256, _mm256_castsi256_ps, _mm256_cmpeq_epi32, _mm256_loadu_si256, + _mm256_min_epu32, _mm256_movemask_ps, + }; + #[cfg(target_arch = "x86_64")] + use core::arch::x86_64::{ + __m256i, _mm256_andnot_si256, _mm256_castsi256_ps, _mm256_cmpeq_epi32, _mm256_loadu_si256, + _mm256_min_epu32, _mm256_movemask_ps, + }; + let nc = unsafe { _mm256_loadu_si256(next_cost.as_ptr() as *const __m256i) }; + let np = unsafe { _mm256_loadu_si256(node_price.as_ptr() as *const __m256i) }; + let min = _mm256_min_epu32(nc, np); + let le = _mm256_cmpeq_epi32(min, nc); // nc <= np + let eq = _mm256_cmpeq_epi32(nc, np); // nc == np + let lt = _mm256_andnot_si256(eq, le); // nc < np + _mm256_movemask_ps(_mm256_castsi256_ps(lt)) as u8 +} + +/// Inline `next_cost = base_cost + ll0_price + match_price_from_parts(off,ml)` +/// for one match length — the exact `add_prices` chain the scalar loop uses, +/// so the SoA vector path stays byte-identical. +#[inline(always)] +#[allow(clippy::too_many_arguments)] +fn priceset_next_cost( + profile: HcOptimalCostProfile, + stats: &HcOptState, + ml_cache: &mut [[u32; 2]], + ml_stamp: u32, + match_len: usize, + ll0_price: u32, + off_price: u32, + base_cost: u32, +) -> u32 { + let ml_price = + BtMatcher::cached_match_length_price(profile, stats, match_len, ml_cache, ml_stamp); + let seq_cost = BtMatcher::add_prices( + ll0_price, + profile.match_price_from_parts(off_price, ml_price, stats), + ); + BtMatcher::add_prices(base_cost, seq_cost) +} + +/// Scalar price-set over the match-length range `[start, max]` for the +/// NON-abort optimal modes (btultra / btultra2). Each `match_len` writes a +/// distinct node `pos + match_len`, so order is irrelevant; the improvement +/// test reduces to `next_cost < node_prices[next]` (`reset_opt_nodes` set +/// every beyond-frontier cell to `u32::MAX`, subsuming `next > last_pos`). +/// `#[inline]` so it folds into each per-tier optimal-parser monomorphisation +/// (no call overhead). Returns the highest written `next`. +#[inline] +#[allow(clippy::too_many_arguments)] +// Used by the scalar / sse42 DP wrappers; on aarch64 the dispatch only reaches +// the neon wrapper and on wasm+simd128 only the simd128 wrapper, so this is +// cfg-dead on those targets. +#[cfg_attr( + any( + all(target_arch = "aarch64", target_endian = "little"), + all(target_arch = "wasm32", target_feature = "simd128") + ), + allow(dead_code) +)] +pub(crate) fn priceset_range_nonabort_scalar( + node_prices: &mut [u32], + nodes: &mut [HcOptimalNode], + ml_cache: &mut [[u32; 2]], + ml_stamp: u32, + profile: HcOptimalCostProfile, + stats: &HcOptState, + pos: usize, + start: usize, + max: usize, + ll0_price: u32, + off_price: u32, + base_cost: u32, + off: u32, + reps: [u32; 3], + last_pos: usize, +) -> usize { + let mut new_last = last_pos; + for ml in start..=max { + let next_cost = priceset_next_cost( + profile, stats, ml_cache, ml_stamp, ml, ll0_price, off_price, base_cost, + ); + let next = pos + ml; + if next_cost < node_prices[next] { + node_prices[next] = next_cost; + nodes[next] = HcOptimalNode { + off, + mlen: ml as u32, + litlen: 0, + reps, + }; + if next > new_last { + new_last = next; + } + } + } + new_last +} + +/// Shared vectorised price-set loop body, generic over the SIMD width `W`. +/// The per-tier `deint` (vector-load plus deinterleave of `W` cached prices, +/// returning `Some` only on an all-warm chunk) and `mask` (per-tier +/// `next_cost` less-than `node_price` bitmask) are passed as zero-sized +/// `impl Fn`s. `#[inline(always)]` plus monomorphisation folds `deint` and +/// `mask` directly into each per-tier wrapper's `target_feature` umbrella, so +/// the intrinsics inline with no call ABI and no runtime feature detection. +/// Cold or out-of-cache chunks, and the sub-`W` remainder, fall back to the +/// scalar `priceset_next_cost` (which fills the cache); writes are +/// scalar-scatter on the improving lanes (1-8% of compares, per the +/// improve-ratio probe). Same signature tail as the scalar variant. +#[inline(always)] +#[allow(clippy::too_many_arguments)] +// Instantiated only by a vector tier wrapper (avx2/sse4.1 on x86, neon on +// aarch64, simd128 on wasm+simd128); a target with none of those (e.g. +// wasm without +simd128) uses only the scalar range, leaving this generic dead. +#[cfg_attr( + not(any( + target_arch = "x86", + target_arch = "x86_64", + all(target_arch = "aarch64", target_endian = "little"), + all(target_arch = "wasm32", target_feature = "simd128") + )), + allow(dead_code) +)] +fn priceset_range_vec( + node_prices: &mut [u32], + nodes: &mut [HcOptimalNode], + ml_cache: &mut [[u32; 2]], + ml_stamp: u32, + profile: HcOptimalCostProfile, + stats: &HcOptState, + pos: usize, + start: usize, + max: usize, + ll0_price: u32, + off_price: u32, + base_cost: u32, + off: u32, + reps: [u32; 3], + last_pos: usize, + deint: impl Fn(&[[u32; 2]], u32) -> Option<[u32; W]>, + mask: impl Fn(&[u32; W], &[u32]) -> u8, +) -> usize { + let mut new_last = last_pos; + let mut buf = [0u32; W]; + // Loop-invariant constant of the byte-identical next_cost chain: + // next_cost = add_prices(base_cost, add_prices(ll0_price, + // match_price_from_parts(off_price, ml_price))) = c_base + ml_price, + // c_base = base_cost + ll0_price + match_price_from_parts(off_price, 0). + // + // This stays bit-exact with the scalar `priceset_next_cost` because both + // helpers are affine in `ml_price`: `BtMatcher::add_prices(a, b) = a + b` + // and `match_price_from_parts(off, ml) = off + ml + bias` are plain integer + // additions, so `match_price_from_parts(off, ml) = match_price_from_parts( + // off, 0) + ml` and the whole chain collapses to `c_base + ml_price`. The + // `wrapping_add` here matches the scalar `+` under the cost model's + // no-overflow invariant (the `debug_assert`s in both helpers). Factoring the + // combine into one helper per the review suggestion would force a per-lane + // `match_price_from_parts(off, ml_price)` recompute instead of hoisting the + // ml-independent `c_base` once — a regression on this hot DP loop — so the + // hoist is kept and the equivalence documented here instead. + let c_base = base_cost + .wrapping_add(ll0_price) + .wrapping_add(profile.match_price_from_parts(off_price, 0, stats)); + let mut ml = start; + while ml + W <= max + 1 { + let vectorised = if ml + W <= ml_cache.len() { + deint(&ml_cache[ml..ml + W], ml_stamp) + } else { + None + }; + if let Some(prices) = vectorised { + for (k, slot) in buf.iter_mut().enumerate() { + *slot = c_base.wrapping_add(prices[k]); + } + } else { + for (k, slot) in buf.iter_mut().enumerate() { + *slot = priceset_next_cost( + profile, + stats, + ml_cache, + ml_stamp, + ml + k, + ll0_price, + off_price, + base_cost, + ); + } + } + let base_next = pos + ml; + let mut bits = mask(&buf, &node_prices[base_next..base_next + W]); + while bits != 0 { + let k = bits.trailing_zeros() as usize; + bits &= bits - 1; + let next = base_next + k; + node_prices[next] = buf[k]; + nodes[next] = HcOptimalNode { + off, + mlen: (ml + k) as u32, + litlen: 0, + reps, + }; + if next > new_last { + new_last = next; + } + } + ml += W; + } + while ml <= max { + let next_cost = priceset_next_cost( + profile, stats, ml_cache, ml_stamp, ml, ll0_price, off_price, base_cost, + ); + let next = pos + ml; + if next_cost < node_prices[next] { + node_prices[next] = next_cost; + nodes[next] = HcOptimalNode { + off, + mlen: ml as u32, + litlen: 0, + reps, + }; + if next > new_last { + new_last = next; + } + } + ml += 1; + } + new_last +} + +/// Vector-load 8 cached ml-prices for the optimal parser's price-set, given a +/// run of 8 contiguous `[price, generation]` cells. Returns `Some(prices)` +/// only when ALL eight cells are warm (`generation == stamp`) — the common +/// (~91-98%) case — so the caller can fold them with one broadcast constant; +/// any cold cell returns `None` to route the chunk through the scalar fill +/// (which recomputes + repopulates the misses). Deinterleaves with cheap +/// in-128-lane ops (`shuffle_epi32` + `unpack*_epi64`) and a single cross-lane +/// `permute4x64` for the ordered prices — avoiding the latency-bound chain of +/// cross-lane `permutevar8x32`s that lost to pipelined scalar loads on +/// high-chunk-count fixtures. +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +#[target_feature(enable = "avx2")] +#[inline] +unsafe fn priceset_cached_prices8_avx2(cells: &[[u32; 2]], stamp: u32) -> Option<[u32; 8]> { + #[cfg(target_arch = "x86")] + use core::arch::x86::{ + __m256i, _mm256_castsi256_ps, _mm256_cmpeq_epi32, _mm256_loadu_si256, _mm256_movemask_ps, + _mm256_permute4x64_epi64, _mm256_set1_epi32, _mm256_shuffle_epi32, _mm256_storeu_si256, + _mm256_unpackhi_epi64, _mm256_unpacklo_epi64, + }; + #[cfg(target_arch = "x86_64")] + use core::arch::x86_64::{ + __m256i, _mm256_castsi256_ps, _mm256_cmpeq_epi32, _mm256_loadu_si256, _mm256_movemask_ps, + _mm256_permute4x64_epi64, _mm256_set1_epi32, _mm256_shuffle_epi32, _mm256_storeu_si256, + _mm256_unpackhi_epi64, _mm256_unpacklo_epi64, + }; + debug_assert!(cells.len() >= 8); + let base = cells.as_ptr() as *const __m256i; + // v0 = [p0 g0 p1 g1 | p2 g2 p3 g3], v1 = [p4 g4 p5 g5 | p6 g6 p7 g7]. + let v0 = unsafe { _mm256_loadu_si256(base) }; + let v1 = unsafe { _mm256_loadu_si256(base.add(1)) }; + // In-128-lane group prices then gens: [p g p g] -> [p p g g] (control 0xD8). + let s0 = _mm256_shuffle_epi32(v0, 0xD8); // [p0 p1 g0 g1 | p2 p3 g2 g3] + let s1 = _mm256_shuffle_epi32(v1, 0xD8); // [p4 p5 g4 g5 | p6 p7 g6 g7] + // Gens (hi 64 of each 128-lane) — order irrelevant for the all-equal test. + let gens = _mm256_unpackhi_epi64(s0, s1); + let eq = _mm256_cmpeq_epi32(gens, _mm256_set1_epi32(stamp as i32)); + if _mm256_movemask_ps(_mm256_castsi256_ps(eq)) as u8 != 0xFF { + return None; + } + // Prices (lo 64 of each 128-lane): [p0 p1 p4 p5 | p2 p3 p6 p7] as 64-bit + // chunks [c0 c1 c2 c3] = [p0p1 p4p5 p2p3 p6p7]; reorder to [c0 c2 c1 c3] + // (control 0xD8) for in-order [p0..p7]. + let p_scrambled = _mm256_unpacklo_epi64(s0, s1); + let prices = _mm256_permute4x64_epi64(p_scrambled, 0xD8); + let mut out = [0u32; 8]; + unsafe { _mm256_storeu_si256(out.as_mut_ptr() as *mut __m256i, prices) }; + Some(out) +} + +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +#[target_feature(enable = "avx2")] +#[inline] +#[allow(clippy::too_many_arguments)] +pub(crate) unsafe fn priceset_range_nonabort_avx2( + node_prices: &mut [u32], + nodes: &mut [HcOptimalNode], + ml_cache: &mut [[u32; 2]], + ml_stamp: u32, + profile: HcOptimalCostProfile, + stats: &HcOptState, + pos: usize, + start: usize, + max: usize, + ll0_price: u32, + off_price: u32, + base_cost: u32, + off: u32, + reps: [u32; 3], + last_pos: usize, +) -> usize { + priceset_range_vec::<8>( + node_prices, + nodes, + ml_cache, + ml_stamp, + profile, + stats, + pos, + start, + max, + ll0_price, + off_price, + base_cost, + off, + reps, + last_pos, + // SAFETY: both closures run inside this fn's avx2 target_feature umbrella. + |cells, stamp| unsafe { priceset_cached_prices8_avx2(cells, stamp) }, + |nc, np| unsafe { priceset_improved_mask8_avx2(nc, np) }, + ) +} + +/// NEON 4-lane vector-load + deinterleave of cached ml-prices. `vld2q_u32` +/// deinterleaves the 4 contiguous `[price, generation]` pairs natively into +/// two registers (prices, gens) — no shuffle chain. `Some(prices)` only when +/// all 4 generations equal `stamp` (`vminvq` of the equality mask is all-ones). +#[cfg(all(target_arch = "aarch64", target_endian = "little"))] +#[target_feature(enable = "neon")] +#[inline] +unsafe fn priceset_cached_prices4_neon(cells: &[[u32; 2]], stamp: u32) -> Option<[u32; 4]> { + use core::arch::aarch64::{vceqq_u32, vdupq_n_u32, vld2q_u32, vminvq_u32, vst1q_u32}; + debug_assert!(cells.len() >= 4); + // SAFETY: caller's neon umbrella; `cells` is >= 4 pairs = 8 contiguous u32. + let pair = unsafe { vld2q_u32(cells.as_ptr() as *const u32) }; + let eq = vceqq_u32(pair.1, vdupq_n_u32(stamp)); + if vminvq_u32(eq) != u32::MAX { + return None; + } + let mut out = [0u32; 4]; + unsafe { vst1q_u32(out.as_mut_ptr(), pair.0) }; + Some(out) +} + +/// NEON 4-lane `next_cost < node_price` bitmask. NEON has an unsigned compare +/// (`vcltq_u32`) but no movemask; AND the all-ones lane mask with lane weights +/// `[1,2,4,8]` and horizontal-add (`vaddvq_u32`) to pack the 4 bits. +#[cfg(all(target_arch = "aarch64", target_endian = "little"))] +#[target_feature(enable = "neon")] +#[inline] +unsafe fn priceset_improved_mask4_neon(next_cost: &[u32; 4], node_price: &[u32]) -> u8 { + use core::arch::aarch64::{vaddvq_u32, vandq_u32, vcltq_u32, vld1q_u32, vst1q_u32}; + // SAFETY: neon umbrella; both spans are 4 u32 wide. + let nc = unsafe { vld1q_u32(next_cost.as_ptr()) }; + let np = unsafe { vld1q_u32(node_price.as_ptr()) }; + let lt = vcltq_u32(nc, np); + let weights: [u32; 4] = [1, 2, 4, 8]; + let w = unsafe { vld1q_u32(weights.as_ptr()) }; + let bits = vandq_u32(lt, w); + let _ = vst1q_u32; // silence unused import on some toolchains + vaddvq_u32(bits) as u8 +} + +#[cfg(all(target_arch = "aarch64", target_endian = "little"))] +#[target_feature(enable = "neon")] +#[inline] +#[allow(clippy::too_many_arguments)] +pub(crate) unsafe fn priceset_range_nonabort_neon( + node_prices: &mut [u32], + nodes: &mut [HcOptimalNode], + ml_cache: &mut [[u32; 2]], + ml_stamp: u32, + profile: HcOptimalCostProfile, + stats: &HcOptState, + pos: usize, + start: usize, + max: usize, + ll0_price: u32, + off_price: u32, + base_cost: u32, + off: u32, + reps: [u32; 3], + last_pos: usize, +) -> usize { + priceset_range_vec::<4>( + node_prices, + nodes, + ml_cache, + ml_stamp, + profile, + stats, + pos, + start, + max, + ll0_price, + off_price, + base_cost, + off, + reps, + last_pos, + // SAFETY: both closures run inside this fn's neon target_feature umbrella. + |cells, stamp| unsafe { priceset_cached_prices4_neon(cells, stamp) }, + |nc, np| unsafe { priceset_improved_mask4_neon(nc, np) }, + ) +} + +/// SSE4.1 4-lane vector-load + deinterleave of cached ml-prices. Two 128-bit +/// loads of `[price, gen]` pairs, `shuffle_epi32(0xD8)` groups prices then gens +/// within each, `unpacklo/hi_epi64` separates them. `Some(prices)` only when +/// all 4 generations equal `stamp`. +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +#[target_feature(enable = "sse4.2")] +#[inline] +unsafe fn priceset_cached_prices4_sse41(cells: &[[u32; 2]], stamp: u32) -> Option<[u32; 4]> { + #[cfg(target_arch = "x86")] + use core::arch::x86::{ + __m128i, _mm_castsi128_ps, _mm_cmpeq_epi32, _mm_loadu_si128, _mm_movemask_ps, + _mm_set1_epi32, _mm_shuffle_epi32, _mm_storeu_si128, _mm_unpackhi_epi64, + _mm_unpacklo_epi64, + }; + #[cfg(target_arch = "x86_64")] + use core::arch::x86_64::{ + __m128i, _mm_castsi128_ps, _mm_cmpeq_epi32, _mm_loadu_si128, _mm_movemask_ps, + _mm_set1_epi32, _mm_shuffle_epi32, _mm_storeu_si128, _mm_unpackhi_epi64, + _mm_unpacklo_epi64, + }; + debug_assert!(cells.len() >= 4); + let base = cells.as_ptr() as *const __m128i; + let v0 = unsafe { _mm_loadu_si128(base) }; // [p0 g0 p1 g1] + let v1 = unsafe { _mm_loadu_si128(base.add(1)) }; // [p2 g2 p3 g3] + let s0 = _mm_shuffle_epi32(v0, 0xD8); // [p0 p1 g0 g1] + let s1 = _mm_shuffle_epi32(v1, 0xD8); // [p2 p3 g2 g3] + let gens = _mm_unpackhi_epi64(s0, s1); // [g0 g1 g2 g3] + let eq = _mm_cmpeq_epi32(gens, _mm_set1_epi32(stamp as i32)); + if _mm_movemask_ps(_mm_castsi128_ps(eq)) as u8 & 0x0F != 0x0F { + return None; + } + let prices = _mm_unpacklo_epi64(s0, s1); // [p0 p1 p2 p3] + let mut out = [0u32; 4]; + unsafe { _mm_storeu_si128(out.as_mut_ptr() as *mut __m128i, prices) }; + Some(out) +} + +/// SSE4.1 4-lane `next_cost < node_price` bitmask (unsigned compare via +/// `min_epu32`, like the AVX2 path). +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +#[target_feature(enable = "sse4.2")] +#[inline] +unsafe fn priceset_improved_mask4_sse41(next_cost: &[u32; 4], node_price: &[u32]) -> u8 { + #[cfg(target_arch = "x86")] + use core::arch::x86::{ + __m128i, _mm_andnot_si128, _mm_castsi128_ps, _mm_cmpeq_epi32, _mm_loadu_si128, + _mm_min_epu32, _mm_movemask_ps, + }; + #[cfg(target_arch = "x86_64")] + use core::arch::x86_64::{ + __m128i, _mm_andnot_si128, _mm_castsi128_ps, _mm_cmpeq_epi32, _mm_loadu_si128, + _mm_min_epu32, _mm_movemask_ps, + }; + let nc = unsafe { _mm_loadu_si128(next_cost.as_ptr() as *const __m128i) }; + let np = unsafe { _mm_loadu_si128(node_price.as_ptr() as *const __m128i) }; + let min = _mm_min_epu32(nc, np); + let le = _mm_cmpeq_epi32(min, nc); + let eq = _mm_cmpeq_epi32(nc, np); + let lt = _mm_andnot_si128(eq, le); + (_mm_movemask_ps(_mm_castsi128_ps(lt)) as u8) & 0x0F +} + +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +#[target_feature(enable = "sse4.2")] +#[inline] +#[allow(clippy::too_many_arguments)] +pub(crate) unsafe fn priceset_range_nonabort_sse41( + node_prices: &mut [u32], + nodes: &mut [HcOptimalNode], + ml_cache: &mut [[u32; 2]], + ml_stamp: u32, + profile: HcOptimalCostProfile, + stats: &HcOptState, + pos: usize, + start: usize, + max: usize, + ll0_price: u32, + off_price: u32, + base_cost: u32, + off: u32, + reps: [u32; 3], + last_pos: usize, +) -> usize { + priceset_range_vec::<4>( + node_prices, + nodes, + ml_cache, + ml_stamp, + profile, + stats, + pos, + start, + max, + ll0_price, + off_price, + base_cost, + off, + reps, + last_pos, + // SAFETY: both closures run inside this fn's sse4.2 target_feature umbrella. + |cells, stamp| unsafe { priceset_cached_prices4_sse41(cells, stamp) }, + |nc, np| unsafe { priceset_improved_mask4_sse41(nc, np) }, + ) +} + +/// wasm `simd128` 4-lane vector-load + deinterleave of cached ml-prices. +/// `u32x4_shuffle` selects the price (even) and gen (odd) lanes across the two +/// loaded vectors natively. `Some(prices)` only when all 4 gens equal `stamp` +/// (`u32x4_all_true` of the equality vector). +#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] +#[target_feature(enable = "simd128")] +#[inline] +unsafe fn priceset_cached_prices4_simd128(cells: &[[u32; 2]], stamp: u32) -> Option<[u32; 4]> { + use core::arch::wasm32::{ + u32x4_all_true, u32x4_eq, u32x4_shuffle, u32x4_splat, v128, v128_load, v128_store, + }; + debug_assert!(cells.len() >= 4); + let base = cells.as_ptr() as *const v128; + let v0 = unsafe { v128_load(base) }; // [p0 g0 p1 g1] + let v1 = unsafe { v128_load(base.add(1)) }; // [p2 g2 p3 g3] + // Lanes 0..3 index v0, 4..7 index v1. + let gens = u32x4_shuffle::<1, 3, 5, 7>(v0, v1); // [g0 g1 g2 g3] + let eq = u32x4_eq(gens, u32x4_splat(stamp)); + if !u32x4_all_true(eq) { + return None; + } + let prices = u32x4_shuffle::<0, 2, 4, 6>(v0, v1); // [p0 p1 p2 p3] + let mut out = [0u32; 4]; + unsafe { v128_store(out.as_mut_ptr() as *mut v128, prices) }; + Some(out) +} + +/// wasm `simd128` 4-lane `next_cost < node_price` bitmask. wasm has a native +/// unsigned compare (`u32x4_lt`) and `u32x4_bitmask` to pack the lanes. +#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] +#[target_feature(enable = "simd128")] +#[inline] +unsafe fn priceset_improved_mask4_simd128(next_cost: &[u32; 4], node_price: &[u32]) -> u8 { + use core::arch::wasm32::{u32x4_bitmask, u32x4_lt, v128, v128_load}; + let nc = unsafe { v128_load(next_cost.as_ptr() as *const v128) }; + let np = unsafe { v128_load(node_price.as_ptr() as *const v128) }; + u32x4_bitmask(u32x4_lt(nc, np)) +} + +#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] +#[target_feature(enable = "simd128")] +#[inline] +#[allow(clippy::too_many_arguments)] +pub(crate) unsafe fn priceset_range_nonabort_simd128( + node_prices: &mut [u32], + nodes: &mut [HcOptimalNode], + ml_cache: &mut [[u32; 2]], + ml_stamp: u32, + profile: HcOptimalCostProfile, + stats: &HcOptState, + pos: usize, + start: usize, + max: usize, + ll0_price: u32, + off_price: u32, + base_cost: u32, + off: u32, + reps: [u32; 3], + last_pos: usize, +) -> usize { + priceset_range_vec::<4>( + node_prices, + nodes, + ml_cache, + ml_stamp, + profile, + stats, + pos, + start, + max, + ll0_price, + off_price, + base_cost, + off, + reps, + last_pos, + // SAFETY: both closures run inside this fn's simd128 target_feature umbrella. + |cells, stamp| unsafe { priceset_cached_prices4_simd128(cells, stamp) }, + |nc, np| unsafe { priceset_improved_mask4_simd128(nc, np) }, + ) +} + +#[cfg(test)] +mod tests; diff --git a/zstd/src/encoding/hc/priceset/tests.rs b/zstd/src/encoding/hc/priceset/tests.rs new file mode 100644 index 000000000..4a0158a50 --- /dev/null +++ b/zstd/src/encoding/hc/priceset/tests.rs @@ -0,0 +1,93 @@ +use super::*; + +/// Per-tier deinterleave + improve-mask correctness vs a scalar reference. +/// Each tier's dispatch only fires on matching hardware (i9 picks AVX2 over +/// SSE4.1, M1 picks NEON), so the non-dispatched tiers never run in the +/// roundtrip suite; this exercises the deinterleave/mask helpers directly on +/// whatever ISA the test host exposes (AVX2 + SSE4.1 on x86, NEON on aarch64). +#[cfg(test)] +#[test] +fn priceset_tier_helpers_match_scalar() { + // Reference: gen-stamped contiguous cells -> ordered prices on all-warm. + fn scalar_deint(cells: &[[u32; 2]], stamp: u32) -> Option<[u32; W]> { + let mut out = [0u32; W]; + for k in 0..W { + if cells[k][1] != stamp { + return None; + } + out[k] = cells[k][0]; + } + Some(out) + } + fn scalar_mask(nc: &[u32; W], np: &[u32]) -> u8 { + let mut m = 0u8; + for k in 0..W { + if nc[k] < np[k] { + m |= 1 << k; + } + } + m + } + const S: u32 = 0x55; + let warm: [[u32; 2]; 4] = [[11, S], [22, S], [33, S], [44, S]]; + let mut cold = warm; + cold[2][1] = S ^ 1; // one stale cell -> must yield None + let nc4: [u32; 4] = [10, 99, 30, 41]; + let np4: [u32; 4] = [20, 21, 30, 99]; // lt: lane0 (10<20), lane3 (41<99) + + #[cfg(all(target_arch = "aarch64", target_endian = "little"))] + unsafe { + assert_eq!( + priceset_cached_prices4_neon(&warm, S), + scalar_deint::<4>(&warm, S) + ); + assert_eq!(priceset_cached_prices4_neon(&cold, S), None); + assert_eq!( + priceset_improved_mask4_neon(&nc4, &np4), + scalar_mask::<4>(&nc4, &np4) + ); + } + #[cfg(all(feature = "std", any(target_arch = "x86", target_arch = "x86_64")))] + { + if std::is_x86_feature_detected!("sse4.2") { + unsafe { + assert_eq!( + priceset_cached_prices4_sse41(&warm, S), + scalar_deint::<4>(&warm, S) + ); + assert_eq!(priceset_cached_prices4_sse41(&cold, S), None); + assert_eq!( + priceset_improved_mask4_sse41(&nc4, &np4), + scalar_mask::<4>(&nc4, &np4) + ); + } + } + if std::is_x86_feature_detected!("avx2") { + let warm8: [[u32; 2]; 8] = [ + [11, S], + [22, S], + [33, S], + [44, S], + [55, S], + [66, S], + [77, S], + [88, S], + ]; + let mut cold8 = warm8; + cold8[5][1] = S ^ 1; + let nc8: [u32; 8] = [10, 99, 30, 41, 99, 60, 99, 80]; + let np8: [u32; 8] = [20, 21, 30, 99, 50, 99, 70, 99]; + unsafe { + assert_eq!( + priceset_cached_prices8_avx2(&warm8, S), + scalar_deint::<8>(&warm8, S) + ); + assert_eq!(priceset_cached_prices8_avx2(&cold8, S), None); + assert_eq!( + priceset_improved_mask8_avx2(&nc8, &np8), + scalar_mask::<8>(&nc8, &np8) + ); + } + } + } +} diff --git a/zstd/src/encoding/incompressible.rs b/zstd/src/encoding/incompressible.rs index df7dd26b0..cf5949a57 100644 --- a/zstd/src/encoding/incompressible.rs +++ b/zstd/src/encoding/incompressible.rs @@ -292,151 +292,4 @@ fn sample_looks_incompressible_capped(block: &[u8], max_sample_len: usize) -> bo } #[cfg(test)] -mod tests { - use super::*; - use crate::encoding::CompressionLevel; - use alloc::vec; - use alloc::vec::Vec; - - fn deterministic_bytes(seed: u64, len: usize) -> Vec { - let mut state = seed; - let mut out = vec![0u8; len]; - for byte in &mut out { - state ^= state << 13; - state ^= state >> 7; - state ^= state << 17; - *byte = state as u8; - } - out - } - - #[test] - fn sample_metrics_do_not_count_first_u32_max_as_repeat() { - let sample = [0xFF_u8; 4]; - let mut counts = [0u16; 256]; - let mut repeat_table = [u32::MAX; INCOMPRESSIBLE_REPEAT_TABLE_LEN]; - let mut repeat_occupied = [0_u64; INCOMPRESSIBLE_REPEAT_OCCUPANCY_WORDS]; - let mut repeats = 0usize; - - // Guard set high so the early-exit never fires: this exercises the - // repeat-table init, where `0xFFFFFFFF` matches the `u32::MAX` - // sentinel but the occupancy bit is still clear, so the first quad - // must NOT be counted as a repeat. - let bailed = scan_sample_region( - &sample, - &mut counts, - &mut repeat_table, - &mut repeat_occupied, - &mut repeats, - usize::MAX, - ); - - assert!(!bailed, "high guards must not trigger an early exit"); - assert_eq!(repeats, 0, "first quad must not be miscounted as a repeat"); - } - - #[test] - fn scan_sample_region_early_exits_on_repetitive_input() { - // 32 identical 4-byte quads: the repeat count climbs past any small - // guard, exercising the early-exit `true` path directly. - let sample = [0xAB_u8; 128]; - let mut counts = [0u16; 256]; - let mut repeat_table = [u32::MAX; INCOMPRESSIBLE_REPEAT_TABLE_LEN]; - let mut repeat_occupied = [0_u64; INCOMPRESSIBLE_REPEAT_OCCUPANCY_WORDS]; - let mut repeats = 0usize; - - // Guard of 1: the first quad seeds the table, the second is the first - // counted repeat (repeats == 1), the third pushes repeats past the - // guard and returns `true`. - let bailed = scan_sample_region( - &sample, - &mut counts, - &mut repeat_table, - &mut repeat_occupied, - &mut repeats, - 1, - ); - - assert!(bailed, "repetitive input must trigger the early exit"); - assert!(repeats > 1, "repeat count must have exceeded the guard"); - } - - #[test] - fn best_raw_fast_path_requires_better_sized_window() { - assert!(compression_level_allows_raw_fast_path( - CompressionLevel::Best, - RAW_FAST_PATH_MAX_WINDOW_SIZE_BYTES - )); - assert!(!compression_level_allows_raw_fast_path( - CompressionLevel::Best, - RAW_FAST_PATH_MAX_WINDOW_SIZE_BYTES + 1 - )); - } - - #[test] - fn level4_row_raw_fast_path_allowed_with_better_window_reach() { - assert!(compression_level_allows_raw_fast_path( - CompressionLevel::Level(4), - RAW_FAST_PATH_MAX_WINDOW_SIZE_BYTES - )); - // Over-cap numeric level is rejected, same boundary as `Best`, so the - // two branches can't drift apart. - assert!(!compression_level_allows_raw_fast_path( - CompressionLevel::Level(4), - RAW_FAST_PATH_MAX_WINDOW_SIZE_BYTES + 1 - )); - } - - #[test] - fn strict_incompressible_reuses_full_block_classification_for_min_block() { - let block = vec![0xA5; RAW_FAST_PATH_MIN_BLOCK_LEN]; - let probes = select_strict_probes(block.len()); - assert_eq!( - probes.tail_start, None, - "minimum-size strict blocks must reuse the full-block sample" - ); - assert_eq!( - block_looks_incompressible_strict(&block), - sample_looks_incompressible(&block), - "strict path should not re-score identical probes for minimum-size blocks" - ); - } - - #[test] - fn strict_probe_selector_avoids_overlap_on_small_non_min_blocks() { - let near_min = select_strict_probes(RAW_FAST_PATH_MIN_BLOCK_LEN + 1); - assert_eq!(near_min.tail_start, None); - assert_eq!(near_min.mid_start, None); - - let two_probe = select_strict_probes(RAW_FAST_PATH_MIN_BLOCK_LEN * 2); - assert_eq!(two_probe.tail_start, Some(RAW_FAST_PATH_MIN_BLOCK_LEN)); - assert_eq!(two_probe.mid_start, None); - - let three_probe = select_strict_probes(RAW_FAST_PATH_MIN_BLOCK_LEN * 3); - assert_eq!( - three_probe.tail_start, - Some(RAW_FAST_PATH_MIN_BLOCK_LEN * 2) - ); - assert_eq!(three_probe.mid_start, Some(RAW_FAST_PATH_MIN_BLOCK_LEN)); - } - - #[test] - fn capped_sample_probes_middle_and_blocks_raw_fast_path_for_mixed_entropy() { - let mut block = - deterministic_bytes(0x9E37_79B9_7F4A_7C15, RAW_FAST_PATH_MAX_SAMPLE_LEN * 2); - let mid_start = block.len() / 3; - let mid_end = block.len() - (block.len() / 3); - for byte in &mut block[mid_start..mid_end] { - *byte = 0; - } - - assert!( - !sample_looks_incompressible(&block), - "capped sampling must account for middle-region compressibility" - ); - assert!( - !block_looks_incompressible(&block), - "mixed-entropy block should not look incompressible for default fast-path gate" - ); - } -} +mod tests; diff --git a/zstd/src/encoding/incompressible/tests.rs b/zstd/src/encoding/incompressible/tests.rs new file mode 100644 index 000000000..cefc25ad1 --- /dev/null +++ b/zstd/src/encoding/incompressible/tests.rs @@ -0,0 +1,145 @@ +use super::*; +use crate::encoding::CompressionLevel; +use alloc::vec; +use alloc::vec::Vec; + +fn deterministic_bytes(seed: u64, len: usize) -> Vec { + let mut state = seed; + let mut out = vec![0u8; len]; + for byte in &mut out { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + *byte = state as u8; + } + out +} + +#[test] +fn sample_metrics_do_not_count_first_u32_max_as_repeat() { + let sample = [0xFF_u8; 4]; + let mut counts = [0u16; 256]; + let mut repeat_table = [u32::MAX; INCOMPRESSIBLE_REPEAT_TABLE_LEN]; + let mut repeat_occupied = [0_u64; INCOMPRESSIBLE_REPEAT_OCCUPANCY_WORDS]; + let mut repeats = 0usize; + + // Guard set high so the early-exit never fires: this exercises the + // repeat-table init, where `0xFFFFFFFF` matches the `u32::MAX` + // sentinel but the occupancy bit is still clear, so the first quad + // must NOT be counted as a repeat. + let bailed = scan_sample_region( + &sample, + &mut counts, + &mut repeat_table, + &mut repeat_occupied, + &mut repeats, + usize::MAX, + ); + + assert!(!bailed, "high guards must not trigger an early exit"); + assert_eq!(repeats, 0, "first quad must not be miscounted as a repeat"); +} + +#[test] +fn scan_sample_region_early_exits_on_repetitive_input() { + // 32 identical 4-byte quads: the repeat count climbs past any small + // guard, exercising the early-exit `true` path directly. + let sample = [0xAB_u8; 128]; + let mut counts = [0u16; 256]; + let mut repeat_table = [u32::MAX; INCOMPRESSIBLE_REPEAT_TABLE_LEN]; + let mut repeat_occupied = [0_u64; INCOMPRESSIBLE_REPEAT_OCCUPANCY_WORDS]; + let mut repeats = 0usize; + + // Guard of 1: the first quad seeds the table, the second is the first + // counted repeat (repeats == 1), the third pushes repeats past the + // guard and returns `true`. + let bailed = scan_sample_region( + &sample, + &mut counts, + &mut repeat_table, + &mut repeat_occupied, + &mut repeats, + 1, + ); + + assert!(bailed, "repetitive input must trigger the early exit"); + assert!(repeats > 1, "repeat count must have exceeded the guard"); +} + +#[test] +fn best_raw_fast_path_requires_better_sized_window() { + assert!(compression_level_allows_raw_fast_path( + CompressionLevel::Best, + RAW_FAST_PATH_MAX_WINDOW_SIZE_BYTES + )); + assert!(!compression_level_allows_raw_fast_path( + CompressionLevel::Best, + RAW_FAST_PATH_MAX_WINDOW_SIZE_BYTES + 1 + )); +} + +#[test] +fn level4_row_raw_fast_path_allowed_with_better_window_reach() { + assert!(compression_level_allows_raw_fast_path( + CompressionLevel::Level(4), + RAW_FAST_PATH_MAX_WINDOW_SIZE_BYTES + )); + // Over-cap numeric level is rejected, same boundary as `Best`, so the + // two branches can't drift apart. + assert!(!compression_level_allows_raw_fast_path( + CompressionLevel::Level(4), + RAW_FAST_PATH_MAX_WINDOW_SIZE_BYTES + 1 + )); +} + +#[test] +fn strict_incompressible_reuses_full_block_classification_for_min_block() { + let block = vec![0xA5; RAW_FAST_PATH_MIN_BLOCK_LEN]; + let probes = select_strict_probes(block.len()); + assert_eq!( + probes.tail_start, None, + "minimum-size strict blocks must reuse the full-block sample" + ); + assert_eq!( + block_looks_incompressible_strict(&block), + sample_looks_incompressible(&block), + "strict path should not re-score identical probes for minimum-size blocks" + ); +} + +#[test] +fn strict_probe_selector_avoids_overlap_on_small_non_min_blocks() { + let near_min = select_strict_probes(RAW_FAST_PATH_MIN_BLOCK_LEN + 1); + assert_eq!(near_min.tail_start, None); + assert_eq!(near_min.mid_start, None); + + let two_probe = select_strict_probes(RAW_FAST_PATH_MIN_BLOCK_LEN * 2); + assert_eq!(two_probe.tail_start, Some(RAW_FAST_PATH_MIN_BLOCK_LEN)); + assert_eq!(two_probe.mid_start, None); + + let three_probe = select_strict_probes(RAW_FAST_PATH_MIN_BLOCK_LEN * 3); + assert_eq!( + three_probe.tail_start, + Some(RAW_FAST_PATH_MIN_BLOCK_LEN * 2) + ); + assert_eq!(three_probe.mid_start, Some(RAW_FAST_PATH_MIN_BLOCK_LEN)); +} + +#[test] +fn capped_sample_probes_middle_and_blocks_raw_fast_path_for_mixed_entropy() { + let mut block = deterministic_bytes(0x9E37_79B9_7F4A_7C15, RAW_FAST_PATH_MAX_SAMPLE_LEN * 2); + let mid_start = block.len() / 3; + let mid_end = block.len() - (block.len() / 3); + for byte in &mut block[mid_start..mid_end] { + *byte = 0; + } + + assert!( + !sample_looks_incompressible(&block), + "capped sampling must account for middle-region compressibility" + ); + assert!( + !block_looks_incompressible(&block), + "mixed-entropy block should not look incompressible for default fast-path gate" + ); +} diff --git a/zstd/src/encoding/ldm/gear_hash.rs b/zstd/src/encoding/ldm/gear_hash.rs index 4ab7de6ad..5ff785750 100644 --- a/zstd/src/encoding/ldm/gear_hash.rs +++ b/zstd/src/encoding/ldm/gear_hash.rs @@ -351,193 +351,4 @@ pub(crate) const GEAR_TAB: [u64; 256] = [ ]; #[cfg(test)] -mod tests { - use super::*; - use alloc::vec; - - /// Sanity-check the verbatim port of `ZSTD_ldm_gearTab`. If these - /// anchors regress every downstream LDM split point drifts - /// relative to upstream output. - /// - /// Indices were cross-verified by parsing upstream zstd - /// `lib/compress/zstd_ldm_geartab.h` and computing each entry's - /// positional index (upstream zstd layout: 3 entries per source line - /// starting at line 18, except the final line which holds the - /// 256th element). The full 256-entry table was also - /// byte-compared against the parsed upstream zstd at build time during - /// the port (zero mismatches). - #[test] - fn gear_tab_anchor_entries_match_known_geartab_header() { - // First entry (upstream zstd zstd_ldm_geartab.h:18). - assert_eq!(GEAR_TAB[0], 0xf5b8f72c5f77775c, "GEAR_TAB[0]"); - // Mid-table anchor (upstream zstd row 14 col 0 → index 42). - assert_eq!(GEAR_TAB[42], 0x869cb54a8749c161, "GEAR_TAB[42]"); - // Three-quarter anchor (upstream zstd row 26 col 2 → index 80). - assert_eq!(GEAR_TAB[80], 0x3bed519cbcb4e1e1, "GEAR_TAB[80]"); - // Last entry: index 255 (upstream zstd zstd_ldm_geartab.h:103). - assert_eq!(GEAR_TAB[255], 0x2b4da14f2613d8f4, "GEAR_TAB[255]"); - } - - /// `stop_mask` derivation under the *common* parameter set - /// (`min_match_length = 64`, `hash_rate_log = 7`): upstream zstd - /// `zstd_ldm.c:52-53` produces - /// `(((1 << 7) - 1)) << (64 - 7) = 0x7F << 57 = 0xFE00_0000_0000_0000`. - #[test] - fn stop_mask_default_params_use_high_bit_window() { - let state = GearHashState::new(LDM_MIN_MATCH_LENGTH, LDM_HASH_RLOG); - assert_eq!( - state.stop_mask, 0xFE00_0000_0000_0000, - "default stop_mask must put the 7 active bits at the \ - top of the 64-bit window (upstream zstd zstd_ldm.c:52-53)" - ); - assert_eq!(state.rolling, GEAR_HASH_INIT); - } - - /// Degenerate path: when `hash_rate_log > min_match_length` - /// upstream zstd `zstd_ldm.c:55-56` falls back to the low-bit mask - /// `(1 << hash_rate_log) - 1` without trying to bias the bits - /// upward. Guards against silent drift if the params clamp ever - /// gets reworked. - #[test] - fn stop_mask_degenerate_path_returns_low_bit_mask() { - let state = GearHashState::new(4, 8); // hash_rate_log > min_match - assert_eq!( - state.stop_mask, - (1u64 << 8) - 1, - "fallback mask must equal (1 << hash_rate_log) - 1 \ - (upstream zstd zstd_ldm.c:56)" - ); - } - - /// `hash_rate_log == 0` also lands on the degenerate branch - /// (`> 0` precondition fails) → mask becomes `0`. Every byte - /// then qualifies as a split point. Matches upstream zstd behaviour. - #[test] - fn stop_mask_hash_rate_log_zero_disables_filter() { - let state = GearHashState::new(LDM_MIN_MATCH_LENGTH, 0); - assert_eq!(state.stop_mask, 0); - } - - /// Verify the rolling update against a hand-traced two-byte - /// stream. Upstream zstd recurrence is `hash = (hash << 1) + - /// GEAR_TAB[byte]`. Starting from `GEAR_HASH_INIT = 0xFFFF_FFFF`: - /// - /// after byte 0x00: - /// hash = (0xFFFF_FFFF << 1) + GEAR_TAB[0] - /// = 0x0000_0001_FFFF_FFFE + 0xf5b8f72c5f77775c - /// = 0xf5b8_f72e_5f77_775a - /// after byte 0x01: - /// hash = (0xf5b8_f72e_5f77_775a << 1) + GEAR_TAB[1] - /// = 0xeb71_ee5c_beee_eeb4 + 0x84935f266b7ac412 - /// = 0x7005_4d83_2a69_b2c6 - #[test] - fn reset_two_byte_stream_matches_hand_traced_recurrence() { - let mut state = GearHashState::new(LDM_MIN_MATCH_LENGTH, LDM_HASH_RLOG); - reset(&mut state, &[0x00, 0x01]); - let expected = (((GEAR_HASH_INIT.wrapping_shl(1)).wrapping_add(GEAR_TAB[0])) << 1) - .wrapping_add(GEAR_TAB[1]); - assert_eq!( - state.rolling, expected, - "rolling recurrence (hash << 1) + GEAR_TAB[byte] regressed" - ); - } - - /// `reset` must be order-equivalent to the unrolled `feed` body - /// modulo the split detection — feeding the same bytes through - /// both APIs leaves the same `rolling` value. Guards against - /// drift in the 4-wide unrolled head. - #[test] - fn reset_and_feed_produce_identical_rolling_state() { - let data: alloc::vec::Vec = (0u8..73).collect(); // crosses the 4-wide boundary - let mut s_reset = GearHashState::new(LDM_MIN_MATCH_LENGTH, LDM_HASH_RLOG); - reset(&mut s_reset, &data); - - let mut s_feed = GearHashState::new(LDM_MIN_MATCH_LENGTH, LDM_HASH_RLOG); - // `feed` with a mask that never matches → no early return, - // exercises exactly the same recurrence as `reset`. Use a - // mask of `u64::MAX` so the all-zeros condition is - // unreachable for the constructed stream. - s_feed.stop_mask = u64::MAX; - let mut splits = vec![0usize; LDM_BATCH_SIZE]; - let (consumed, num) = feed(&mut s_feed, &data, &mut splits); - assert_eq!(consumed, data.len()); - assert_eq!(num, 0, "u64::MAX mask must never trigger a split"); - assert_eq!( - s_feed.rolling, s_reset.rolling, - "reset and feed must agree on the rolling state after \ - identical input — guards the 4-wide unroll" - ); - } - - /// `feed` with a zero mask flags every byte as a split. The - /// number of recorded splits is therefore `min(data.len(), - /// LDM_BATCH_SIZE)`, the consumed-byte count matches, and each - /// `splits[i]` is `i + 1` (upstream zstd stores the 1-based post-byte - /// offset). - #[test] - fn feed_zero_mask_records_split_per_byte_up_to_batch_cap() { - let data: alloc::vec::Vec = vec![0u8; LDM_BATCH_SIZE * 2]; - let mut state = GearHashState::new(LDM_MIN_MATCH_LENGTH, 0); - // Defensive: `new` with hash_rate_log == 0 already sets - // stop_mask = 0, but re-assert here so the intent is obvious. - assert_eq!(state.stop_mask, 0); - - let mut splits = vec![0usize; LDM_BATCH_SIZE]; - let (consumed, num) = feed(&mut state, &data, &mut splits); - assert_eq!(num, LDM_BATCH_SIZE, "batch cap must be honoured"); - assert_eq!( - consumed, LDM_BATCH_SIZE, - "consumed must equal LDM_BATCH_SIZE" - ); - for (i, &s) in splits.iter().enumerate() { - assert_eq!( - s, - i + 1, - "upstream zstd records 1-based post-byte indices \ - (zstd_ldm.c:111); splits[{i}] should be {}", - i + 1 - ); - } - } - - /// Feeding in two consecutive chunks must produce the same - /// rolling state as feeding the concatenation. Guards against - /// state corruption between calls (the upstream zstd allows the caller - /// to drain its `splits` buffer and resume mid-stream). - #[test] - fn feed_concatenation_invariant() { - let part_a: alloc::vec::Vec = (0u8..30).collect(); - let part_b: alloc::vec::Vec = (30u8..73).collect(); - let mut joined = part_a.clone(); - joined.extend_from_slice(&part_b); - - let mut splits_a = vec![0usize; LDM_BATCH_SIZE]; - let mut splits_b = vec![0usize; LDM_BATCH_SIZE]; - let mut splits_joined = vec![0usize; LDM_BATCH_SIZE]; - - let mut s_chunked = GearHashState::new(LDM_MIN_MATCH_LENGTH, LDM_HASH_RLOG); - let _ = feed(&mut s_chunked, &part_a, &mut splits_a); - let _ = feed(&mut s_chunked, &part_b, &mut splits_b); - - let mut s_joined = GearHashState::new(LDM_MIN_MATCH_LENGTH, LDM_HASH_RLOG); - let _ = feed(&mut s_joined, &joined, &mut splits_joined); - - assert_eq!( - s_chunked.rolling, s_joined.rolling, - "chunked feed must leave the same rolling state as a single feed" - ); - } - - /// `splits` buffer too small must panic — upstream zstd pre-condition - /// enforced. Without this assertion a short buffer would - /// silently truncate at runtime once the inner branches start - /// indexing past the end. - #[test] - #[should_panic(expected = "LDM_BATCH_SIZE")] - fn feed_panics_on_undersized_splits_buffer() { - let data = [0u8; 8]; - let mut state = GearHashState::new(LDM_MIN_MATCH_LENGTH, LDM_HASH_RLOG); - let mut splits = vec![0usize; LDM_BATCH_SIZE - 1]; - let _ = feed(&mut state, &data, &mut splits); - } -} +mod tests; diff --git a/zstd/src/encoding/ldm/gear_hash/tests.rs b/zstd/src/encoding/ldm/gear_hash/tests.rs new file mode 100644 index 000000000..f9ba28e57 --- /dev/null +++ b/zstd/src/encoding/ldm/gear_hash/tests.rs @@ -0,0 +1,188 @@ +use super::*; +use alloc::vec; + +/// Sanity-check the verbatim port of `ZSTD_ldm_gearTab`. If these +/// anchors regress every downstream LDM split point drifts +/// relative to upstream output. +/// +/// Indices were cross-verified by parsing upstream zstd +/// `lib/compress/zstd_ldm_geartab.h` and computing each entry's +/// positional index (upstream zstd layout: 3 entries per source line +/// starting at line 18, except the final line which holds the +/// 256th element). The full 256-entry table was also +/// byte-compared against the parsed upstream zstd at build time during +/// the port (zero mismatches). +#[test] +fn gear_tab_anchor_entries_match_known_geartab_header() { + // First entry (upstream zstd zstd_ldm_geartab.h:18). + assert_eq!(GEAR_TAB[0], 0xf5b8f72c5f77775c, "GEAR_TAB[0]"); + // Mid-table anchor (upstream zstd row 14 col 0 → index 42). + assert_eq!(GEAR_TAB[42], 0x869cb54a8749c161, "GEAR_TAB[42]"); + // Three-quarter anchor (upstream zstd row 26 col 2 → index 80). + assert_eq!(GEAR_TAB[80], 0x3bed519cbcb4e1e1, "GEAR_TAB[80]"); + // Last entry: index 255 (upstream zstd zstd_ldm_geartab.h:103). + assert_eq!(GEAR_TAB[255], 0x2b4da14f2613d8f4, "GEAR_TAB[255]"); +} + +/// `stop_mask` derivation under the *common* parameter set +/// (`min_match_length = 64`, `hash_rate_log = 7`): upstream zstd +/// `zstd_ldm.c:52-53` produces +/// `(((1 << 7) - 1)) << (64 - 7) = 0x7F << 57 = 0xFE00_0000_0000_0000`. +#[test] +fn stop_mask_default_params_use_high_bit_window() { + let state = GearHashState::new(LDM_MIN_MATCH_LENGTH, LDM_HASH_RLOG); + assert_eq!( + state.stop_mask, 0xFE00_0000_0000_0000, + "default stop_mask must put the 7 active bits at the \ + top of the 64-bit window (upstream zstd zstd_ldm.c:52-53)" + ); + assert_eq!(state.rolling, GEAR_HASH_INIT); +} + +/// Degenerate path: when `hash_rate_log > min_match_length` +/// upstream zstd `zstd_ldm.c:55-56` falls back to the low-bit mask +/// `(1 << hash_rate_log) - 1` without trying to bias the bits +/// upward. Guards against silent drift if the params clamp ever +/// gets reworked. +#[test] +fn stop_mask_degenerate_path_returns_low_bit_mask() { + let state = GearHashState::new(4, 8); // hash_rate_log > min_match + assert_eq!( + state.stop_mask, + (1u64 << 8) - 1, + "fallback mask must equal (1 << hash_rate_log) - 1 \ + (upstream zstd zstd_ldm.c:56)" + ); +} + +/// `hash_rate_log == 0` also lands on the degenerate branch +/// (`> 0` precondition fails) → mask becomes `0`. Every byte +/// then qualifies as a split point. Matches upstream zstd behaviour. +#[test] +fn stop_mask_hash_rate_log_zero_disables_filter() { + let state = GearHashState::new(LDM_MIN_MATCH_LENGTH, 0); + assert_eq!(state.stop_mask, 0); +} + +/// Verify the rolling update against a hand-traced two-byte +/// stream. Upstream zstd recurrence is `hash = (hash << 1) + +/// GEAR_TAB[byte]`. Starting from `GEAR_HASH_INIT = 0xFFFF_FFFF`: +/// +/// after byte 0x00: +/// hash = (0xFFFF_FFFF << 1) + GEAR_TAB[0] +/// = 0x0000_0001_FFFF_FFFE + 0xf5b8f72c5f77775c +/// = 0xf5b8_f72e_5f77_775a +/// after byte 0x01: +/// hash = (0xf5b8_f72e_5f77_775a << 1) + GEAR_TAB[1] +/// = 0xeb71_ee5c_beee_eeb4 + 0x84935f266b7ac412 +/// = 0x7005_4d83_2a69_b2c6 +#[test] +fn reset_two_byte_stream_matches_hand_traced_recurrence() { + let mut state = GearHashState::new(LDM_MIN_MATCH_LENGTH, LDM_HASH_RLOG); + reset(&mut state, &[0x00, 0x01]); + let expected = (((GEAR_HASH_INIT.wrapping_shl(1)).wrapping_add(GEAR_TAB[0])) << 1) + .wrapping_add(GEAR_TAB[1]); + assert_eq!( + state.rolling, expected, + "rolling recurrence (hash << 1) + GEAR_TAB[byte] regressed" + ); +} + +/// `reset` must be order-equivalent to the unrolled `feed` body +/// modulo the split detection — feeding the same bytes through +/// both APIs leaves the same `rolling` value. Guards against +/// drift in the 4-wide unrolled head. +#[test] +fn reset_and_feed_produce_identical_rolling_state() { + let data: alloc::vec::Vec = (0u8..73).collect(); // crosses the 4-wide boundary + let mut s_reset = GearHashState::new(LDM_MIN_MATCH_LENGTH, LDM_HASH_RLOG); + reset(&mut s_reset, &data); + + let mut s_feed = GearHashState::new(LDM_MIN_MATCH_LENGTH, LDM_HASH_RLOG); + // `feed` with a mask that never matches → no early return, + // exercises exactly the same recurrence as `reset`. Use a + // mask of `u64::MAX` so the all-zeros condition is + // unreachable for the constructed stream. + s_feed.stop_mask = u64::MAX; + let mut splits = vec![0usize; LDM_BATCH_SIZE]; + let (consumed, num) = feed(&mut s_feed, &data, &mut splits); + assert_eq!(consumed, data.len()); + assert_eq!(num, 0, "u64::MAX mask must never trigger a split"); + assert_eq!( + s_feed.rolling, s_reset.rolling, + "reset and feed must agree on the rolling state after \ + identical input — guards the 4-wide unroll" + ); +} + +/// `feed` with a zero mask flags every byte as a split. The +/// number of recorded splits is therefore `min(data.len(), +/// LDM_BATCH_SIZE)`, the consumed-byte count matches, and each +/// `splits[i]` is `i + 1` (upstream zstd stores the 1-based post-byte +/// offset). +#[test] +fn feed_zero_mask_records_split_per_byte_up_to_batch_cap() { + let data: alloc::vec::Vec = vec![0u8; LDM_BATCH_SIZE * 2]; + let mut state = GearHashState::new(LDM_MIN_MATCH_LENGTH, 0); + // Defensive: `new` with hash_rate_log == 0 already sets + // stop_mask = 0, but re-assert here so the intent is obvious. + assert_eq!(state.stop_mask, 0); + + let mut splits = vec![0usize; LDM_BATCH_SIZE]; + let (consumed, num) = feed(&mut state, &data, &mut splits); + assert_eq!(num, LDM_BATCH_SIZE, "batch cap must be honoured"); + assert_eq!( + consumed, LDM_BATCH_SIZE, + "consumed must equal LDM_BATCH_SIZE" + ); + for (i, &s) in splits.iter().enumerate() { + assert_eq!( + s, + i + 1, + "upstream zstd records 1-based post-byte indices \ + (zstd_ldm.c:111); splits[{i}] should be {}", + i + 1 + ); + } +} + +/// Feeding in two consecutive chunks must produce the same +/// rolling state as feeding the concatenation. Guards against +/// state corruption between calls (the upstream zstd allows the caller +/// to drain its `splits` buffer and resume mid-stream). +#[test] +fn feed_concatenation_invariant() { + let part_a: alloc::vec::Vec = (0u8..30).collect(); + let part_b: alloc::vec::Vec = (30u8..73).collect(); + let mut joined = part_a.clone(); + joined.extend_from_slice(&part_b); + + let mut splits_a = vec![0usize; LDM_BATCH_SIZE]; + let mut splits_b = vec![0usize; LDM_BATCH_SIZE]; + let mut splits_joined = vec![0usize; LDM_BATCH_SIZE]; + + let mut s_chunked = GearHashState::new(LDM_MIN_MATCH_LENGTH, LDM_HASH_RLOG); + let _ = feed(&mut s_chunked, &part_a, &mut splits_a); + let _ = feed(&mut s_chunked, &part_b, &mut splits_b); + + let mut s_joined = GearHashState::new(LDM_MIN_MATCH_LENGTH, LDM_HASH_RLOG); + let _ = feed(&mut s_joined, &joined, &mut splits_joined); + + assert_eq!( + s_chunked.rolling, s_joined.rolling, + "chunked feed must leave the same rolling state as a single feed" + ); +} + +/// `splits` buffer too small must panic — upstream zstd pre-condition +/// enforced. Without this assertion a short buffer would +/// silently truncate at runtime once the inner branches start +/// indexing past the end. +#[test] +#[should_panic(expected = "LDM_BATCH_SIZE")] +fn feed_panics_on_undersized_splits_buffer() { + let data = [0u8; 8]; + let mut state = GearHashState::new(LDM_MIN_MATCH_LENGTH, LDM_HASH_RLOG); + let mut splits = vec![0usize; LDM_BATCH_SIZE - 1]; + let _ = feed(&mut state, &data, &mut splits); +} diff --git a/zstd/src/encoding/ldm/mod.rs b/zstd/src/encoding/ldm/mod.rs index d9e472a9c..a918ef04f 100644 --- a/zstd/src/encoding/ldm/mod.rs +++ b/zstd/src/encoding/ldm/mod.rs @@ -436,299 +436,4 @@ impl LdmProducer { } #[cfg(test)] -mod tests { - use super::*; - - /// `LdmParams::adjust_for` must derive a representative - /// upstream zstd-btultra2 parameter set at window_log=27. Checked via - /// the parameter struct alone — instantiating the producer - /// at these knobs would allocate `1 << 23 = 8M` table entries - /// (~64 MiB), which slows nextest under parallelism and risks - /// OOM on 32-bit CI shards. The producer-construction smoke - /// path is exercised by every other test in this module with - /// a smaller hand-tuned `LdmParams`. - #[test] - fn producer_constructs_with_default_params() { - let p = LdmParams::adjust_for(27, 9); - // Upstream zstd defaults at btultra2: minMatch halved, hash_rate_log - // = 4, bucket_size_log clamps to 8. See params::tests for - // the per-knob derivations. - assert_eq!(p.window_log, 27); - assert_eq!(p.min_match_length, 32); - assert_eq!(p.hash_rate_log, 4); - assert_eq!(p.bucket_size_log, 8); - } - - /// Compact `LdmParams` for unit-test producer construction — - /// keeps the table allocation at ~8 KiB instead of the - /// upstream zstd-btultra2 64 MiB so parallel nextest stays stable. - /// `min_match_length = 32` (half of upstream zstd floor) matches the - /// btultra2 derivation so engineered fixtures still trigger - /// long-range matches at 32-byte windows. - fn test_params() -> LdmParams { - LdmParams { - window_log: 27, - hash_log: 10, - hash_rate_log: 4, - min_match_length: 32, - bucket_size_log: 4, - } - } - - /// `clear` after `generate_into` rewinds the rolling hash to - /// the canonical init value — guards the frame-boundary - /// contract. - #[test] - fn clear_resets_rolling_hash_state() { - let mut producer = LdmProducer::new(test_params()); - let mut out = Vec::new(); - // Feed a non-empty chunk so the rolling hash advances. - let data = [0xAAu8; 256]; - producer.generate_into(&data, 0, 0, data.len(), &mut out); - let advanced = producer.hash_state.rolling; - assert_ne!( - advanced, - gear_hash::GEAR_HASH_INIT, - "rolling hash should have moved after generate_into" - ); - producer.clear(); - assert_eq!( - producer.hash_state.rolling, - gear_hash::GEAR_HASH_INIT, - "clear must rewind to GEAR_HASH_INIT" - ); - } - - /// End-to-end producer pipeline: a long-range repetition - /// (two copies of a 4 KiB random-like payload separated by - /// 64 KiB of unique filler) must emit at least one - /// [`HcRawSeq`] whose `offset` equals the distance between - /// the copies and whose `match_length` reaches at least the - /// upstream zstd `min_match_length` floor. Validates that - /// gear-hash → bucket-insert → checksum-filter → forward / - /// backward extend → emit all hold together. - #[test] - fn generate_into_emits_long_range_match_on_repeated_payload() { - // Use level 22 / btultra2 / windowLog 27 to get the - // halved min_match (32) — the default 64 would require a - // 128 KiB+ fixture to comfortably trigger a split inside - // the payload, which slows the test without adding - // coverage. - let mut producer = LdmProducer::new(test_params()); - let p = producer.params(); - assert_eq!(p.min_match_length, 32); - - // Build a fixture: payload, gap, payload, sized so the - // second payload sits comfortably past the gear hash's - // `2 ^ hash_rate_log` ≈ 16-byte expected split spacing. - // 4 KiB payload + 64 KiB gap + 4 KiB payload ⇒ ~72 KiB - // total, plenty of bytes for multiple split points inside - // each copy. - const PAYLOAD: usize = 4096; - const GAP: usize = 64 * 1024; - let mut history = Vec::with_capacity(2 * PAYLOAD + GAP); - // Deterministic non-trivial payload: a simple LCG so the - // bytes look random to the gear hash but the fixture - // remains reproducible across runs. - let mut prng: u32 = 0x1234_5678; - let payload: alloc::vec::Vec = (0..PAYLOAD) - .map(|_| { - prng = prng.wrapping_mul(1_103_515_245).wrapping_add(12_345); - (prng >> 16) as u8 - }) - .collect(); - history.extend_from_slice(&payload); - // Unique-byte gap: counter mod 251 keeps the gap - // statistically distinct from the payload (251 prime so - // it doesn't accidentally align with payload bytes). - history.extend((0..GAP).map(|i| (i % 251) as u8)); - history.extend_from_slice(&payload); - - let mut out = Vec::new(); - producer.generate_into(&history, 0, 0, history.len(), &mut out); - - assert!( - !out.is_empty(), - "long-range repetition must produce at least one LDM sequence" - ); - // Every emitted sequence must satisfy the upstream zstd floor. - for seq in &out { - assert!( - seq.match_length >= p.min_match_length as usize, - "every emitted match must reach min_match_length \ - (got match_length = {})", - seq.match_length - ); - } - // At least one emitted sequence must reach across the - // gap into the first payload copy — this is the - // long-range match LDM exists to capture. Short-distance - // matches found inside the first payload (statistically - // possible on random-like content) are allowed too, but - // the long-range one must show up. - let crossing_gap = out.iter().any(|s| s.offset >= GAP); - assert!( - crossing_gap, - "at least one emitted sequence must have offset >= GAP \ - ({GAP}); offsets observed: {:?}", - out.iter().map(|s| s.offset).collect::>() - ); - } - - /// Regression test for PR #139 round-2 review (CodeRabbit + - /// Copilot, Major): the producer must store **absolute stream - /// positions** in its bucket-table entries so that long-range - /// matches accumulated by one block remain valid after a - /// window eviction shifts `history_abs_start` forward. - /// - /// Setup: same 4 KiB payload appears twice in the per-frame - /// history. We invoke `generate_into` twice: - /// 1. First call covers absolute range `[0, payload_end_0)` - /// — the producer's bucket table accumulates entries - /// whose `offset` fields hold absolute positions inside - /// the first payload copy. - /// 2. Second call advances `history_abs_start` (simulates - /// window eviction) and covers the absolute range - /// containing the second payload copy. The bucket-table - /// entries from call 1 must remain reachable: their - /// absolute offsets still point at the SAME bytes - /// (now further left in the live slice), and the - /// inclusive lower-bound staleness check - /// `entry.offset < history_abs_start` keeps them - /// in-window (entries at exactly `history_abs_start` - /// survive). - /// - /// If the producer had stored slice-relative indices - /// instead, call 2 would either miss the long-range match - /// entirely (slice indices from call 1 would point into the - /// wrong bytes after the slide) or underflow `offset = - /// split_abs − best.match_pos` and emit a corrupt - /// back-reference. - #[test] - fn generate_into_preserves_bucket_entries_across_history_slide() { - let mut producer = LdmProducer::new(test_params()); - let p = producer.params(); - assert_eq!(p.min_match_length, 32); - - const PAYLOAD: usize = 4096; - const GAP_A: usize = 32 * 1024; - const GAP_B: usize = 32 * 1024; - - // Deterministic non-trivial payload. - let mut prng: u32 = 0xC0FFEEEE; - let payload: alloc::vec::Vec = (0..PAYLOAD) - .map(|_| { - prng = prng.wrapping_mul(1_103_515_245).wrapping_add(12_345); - (prng >> 16) as u8 - }) - .collect(); - - // Build the full frame in one Vec — represents the - // contiguous live history visible to the encoder before - // any eviction. - let mut frame = alloc::vec::Vec::with_capacity(2 * PAYLOAD + GAP_A + GAP_B); - frame.extend_from_slice(&payload); - frame.extend((0..GAP_A).map(|i| (i % 251) as u8)); - frame.extend_from_slice(&payload); - frame.extend((0..GAP_B).map(|i| ((i + 17) % 241) as u8)); - - // Call 1: cover the first half of the frame — - // history_abs_start = 0, walks the first payload copy - // and the start of the first gap. The bucket table - // populates with absolute offsets inside payload #1. - let mut out1 = Vec::new(); - let split_at = PAYLOAD + GAP_A; - producer.generate_into(&frame[..split_at], 0, 0, split_at, &mut out1); - - // Call 2: simulate a window slide. The encoder retired - // the leading `eviction` bytes from the live history; - // the surviving slice is `frame[eviction..]` and its - // byte 0 sits at absolute position `eviction`. The - // second payload copy is at absolute position `PAYLOAD - // + GAP_A`, which after the slide is at index - // `PAYLOAD + GAP_A − eviction` inside the slice. - let eviction = PAYLOAD / 2; // arbitrary — payload #1 partially evicted - let live = &frame[eviction..]; - let history_abs_start = eviction; - let block_start_abs = PAYLOAD + GAP_A; // start of payload #2 - let block_end_abs = block_start_abs + PAYLOAD; - - let mut out2 = Vec::new(); - producer.generate_into( - live, - history_abs_start, - block_start_abs, - block_end_abs, - &mut out2, - ); - - // The long-range match must still fire — entries from - // call 1 in the surviving tail of payload #1 - // (`absolute [eviction, PAYLOAD)`) are still in-window - // and still point at the right bytes. If the producer - // had stored slice-relative offsets, call 2 would - // either miss or emit corrupt offsets. - assert!( - !out2.is_empty(), - "cross-slide long-range match must survive a window eviction" - ); - // Every emitted match must (a) clear the min_match - // floor, and (b) point at bytes that still live inside - // the post-slide history. The latter is the actual - // round-2 review-fix invariant: if the producer had - // stored slice-relative offsets, entries inserted by - // call 1 would now reference the wrong absolute bytes - // and `offset = split_abs − stale_match_pos` could - // underflow `u32` and emit an offset > live_history - // length. - let live_len = live.len(); - for seq in &out2 { - assert!( - seq.match_length >= p.min_match_length as usize, - "every emitted match must reach min_match_length (got {})", - seq.match_length - ); - assert!( - seq.offset <= live_len, - "back-ref offset {} must stay within the live \ - history (= {} bytes); offsets larger than this \ - are the smoking gun for stale slice-relative \ - entries surviving the eviction", - seq.offset, - live_len - ); - } - - // At least one emitted sequence MUST hit a long-range - // back-reference (offset >= GAP_A, i.e. crossing from - // payload #2 back into payload #1's surviving tail). - // This is the long-range win LDM exists to capture and - // the round-2 fix has to preserve. - let crossed_gap = out2.iter().any(|s| s.offset >= GAP_A); - assert!( - crossed_gap, - "at least one emitted sequence must hit a back-ref of \ - >= GAP_A ({GAP_A}) — that's the long-range match \ - into the surviving tail of payload #1 the bucket \ - entries from call 1 are supposed to produce; \ - offsets observed: {:?}", - out2.iter() - .map(|s| s.offset) - .collect::>() - ); - } - - /// `generate_into` with an empty range is a no-op — emits - /// nothing and leaves the rolling hash untouched. Guards - /// against an off-by-one in the bounds check. - #[test] - fn generate_into_empty_range_is_noop() { - let mut producer = LdmProducer::new(test_params()); - let mut out = Vec::new(); - let data = [0u8; 128]; - let pre = producer.hash_state.rolling; - producer.generate_into(&data, 0, 64, 64, &mut out); - assert!(out.is_empty()); - assert_eq!(producer.hash_state.rolling, pre); - } -} +mod tests; diff --git a/zstd/src/encoding/ldm/params.rs b/zstd/src/encoding/ldm/params.rs index be2d85974..d2ae33ff4 100644 --- a/zstd/src/encoding/ldm/params.rs +++ b/zstd/src/encoding/ldm/params.rs @@ -170,90 +170,4 @@ const fn bounded(lo: u32, val: u32, hi: u32) -> u32 { } #[cfg(test)] -mod tests { - use super::*; - - /// Spot-check upstream zstd strategy → hash_rate_log mapping - /// (`zstd_ldm.c:151`): `7 - strategy/3`. - /// - /// strategy 1 (fast) → 7 - /// strategy 3 (greedy) → 6 - /// strategy 6 (btlazy2) → 5 - /// strategy 9 (btultra2) → 4 - /// `adjust_for` must panic in BOTH debug and release builds - /// when handed an out-of-range strategy (upstream zstd 1..=9). The - /// inner `LDM_HASH_RLOG - (strategy / 3)` would otherwise - /// underflow `u32` for `strategy >= 24` and produce - /// nonsensical params. Regression for PR #139 round-10 - /// review (Copilot). - #[test] - #[should_panic(expected = "strategy must be a upstream zstd 1..=9 ordinal")] - fn adjust_for_panics_on_out_of_range_strategy() { - let _ = LdmParams::adjust_for(27, 24); - } - - #[test] - fn adjust_strategy_to_hash_rate_log_matches_table() { - assert_eq!(LdmParams::adjust_for(27, 1).hash_rate_log, 7); - assert_eq!(LdmParams::adjust_for(27, 3).hash_rate_log, 6); - assert_eq!(LdmParams::adjust_for(27, 6).hash_rate_log, 5); - assert_eq!(LdmParams::adjust_for(27, 9).hash_rate_log, 4); - } - - /// `hash_log` clamping: `BOUNDED(6, window_log - hash_rate_log, 30)`. - /// - /// window_log = 27, strategy = 1 → hash_rate_log = 7 - /// → window_log - hash_rate_log = 20, in range → hash_log = 20 - /// window_log = 10, strategy = 1 → hash_rate_log = 7 - /// → window_log - hash_rate_log = 3 < 6 → clamps to 6 - /// window_log = 7, strategy = 1 → hash_rate_log = 7 - /// → window_log <= hash_rate_log → degenerate → hash_log = 6 - #[test] - fn adjust_hash_log_clamps_within_bounds() { - assert_eq!(LdmParams::adjust_for(27, 1).hash_log, 20); - assert_eq!(LdmParams::adjust_for(10, 1).hash_log, LDM_HASHLOG_MIN); - assert_eq!(LdmParams::adjust_for(7, 1).hash_log, LDM_HASHLOG_MIN); - } - - /// `min_match_length` halving at btultra (strategy ≥ 8). - /// Upstream zstd `zstd_ldm.c:163-164`. - #[test] - fn adjust_min_match_halved_for_btultra_and_above() { - assert_eq!( - LdmParams::adjust_for(27, 7).min_match_length, - LDM_MIN_MATCH_LENGTH as u32 - ); - assert_eq!( - LdmParams::adjust_for(27, 8).min_match_length, - (LDM_MIN_MATCH_LENGTH / 2) as u32 - ); - assert_eq!( - LdmParams::adjust_for(27, 9).min_match_length, - (LDM_MIN_MATCH_LENGTH / 2) as u32 - ); - } - - /// `bucket_size_log = BOUNDED(LDM_BUCKET_SIZE_LOG, strategy, - /// LDM_BUCKETSIZELOG_MAX)` — upstream zstd `zstd_ldm.c:168`. - /// `LDM_BUCKET_SIZE_LOG = 4`, `LDM_BUCKETSIZELOG_MAX = 8`. - #[test] - fn adjust_bucket_size_log_clamps_strategy_to_bounds() { - // strategy 1 < lower bound 4 → clamps up - assert_eq!(LdmParams::adjust_for(27, 1).bucket_size_log, 4); - // strategy 4 == lower bound → identity - assert_eq!(LdmParams::adjust_for(27, 4).bucket_size_log, 4); - // strategy 7 in range → identity - assert_eq!(LdmParams::adjust_for(27, 7).bucket_size_log, 7); - // strategy 9 > upper bound 8 → clamps down - assert_eq!(LdmParams::adjust_for(27, 9).bucket_size_log, 8); - } - - /// Derived hash-table size + bucket slots agree with the - /// `1 << log` definitions and with each other. - #[test] - fn derived_helpers_match_log_definitions() { - let p = LdmParams::adjust_for(27, 5); - assert_eq!(p.hash_table_entries(), 1usize << p.hash_log); - assert_eq!(p.bucket_slots(), 1usize << p.bucket_size_log); - } -} +mod tests; diff --git a/zstd/src/encoding/ldm/params/tests.rs b/zstd/src/encoding/ldm/params/tests.rs new file mode 100644 index 000000000..56ef68ee0 --- /dev/null +++ b/zstd/src/encoding/ldm/params/tests.rs @@ -0,0 +1,85 @@ +use super::*; + +/// Spot-check upstream zstd strategy → hash_rate_log mapping +/// (`zstd_ldm.c:151`): `7 - strategy/3`. +/// +/// strategy 1 (fast) → 7 +/// strategy 3 (greedy) → 6 +/// strategy 6 (btlazy2) → 5 +/// strategy 9 (btultra2) → 4 +/// `adjust_for` must panic in BOTH debug and release builds +/// when handed an out-of-range strategy (upstream zstd 1..=9). The +/// inner `LDM_HASH_RLOG - (strategy / 3)` would otherwise +/// underflow `u32` for `strategy >= 24` and produce +/// nonsensical params. Regression for PR #139 round-10 +/// review (Copilot). +#[test] +#[should_panic(expected = "strategy must be a upstream zstd 1..=9 ordinal")] +fn adjust_for_panics_on_out_of_range_strategy() { + let _ = LdmParams::adjust_for(27, 24); +} + +#[test] +fn adjust_strategy_to_hash_rate_log_matches_table() { + assert_eq!(LdmParams::adjust_for(27, 1).hash_rate_log, 7); + assert_eq!(LdmParams::adjust_for(27, 3).hash_rate_log, 6); + assert_eq!(LdmParams::adjust_for(27, 6).hash_rate_log, 5); + assert_eq!(LdmParams::adjust_for(27, 9).hash_rate_log, 4); +} + +/// `hash_log` clamping: `BOUNDED(6, window_log - hash_rate_log, 30)`. +/// +/// window_log = 27, strategy = 1 → hash_rate_log = 7 +/// → window_log - hash_rate_log = 20, in range → hash_log = 20 +/// window_log = 10, strategy = 1 → hash_rate_log = 7 +/// → window_log - hash_rate_log = 3 < 6 → clamps to 6 +/// window_log = 7, strategy = 1 → hash_rate_log = 7 +/// → window_log <= hash_rate_log → degenerate → hash_log = 6 +#[test] +fn adjust_hash_log_clamps_within_bounds() { + assert_eq!(LdmParams::adjust_for(27, 1).hash_log, 20); + assert_eq!(LdmParams::adjust_for(10, 1).hash_log, LDM_HASHLOG_MIN); + assert_eq!(LdmParams::adjust_for(7, 1).hash_log, LDM_HASHLOG_MIN); +} + +/// `min_match_length` halving at btultra (strategy ≥ 8). +/// Upstream zstd `zstd_ldm.c:163-164`. +#[test] +fn adjust_min_match_halved_for_btultra_and_above() { + assert_eq!( + LdmParams::adjust_for(27, 7).min_match_length, + LDM_MIN_MATCH_LENGTH as u32 + ); + assert_eq!( + LdmParams::adjust_for(27, 8).min_match_length, + (LDM_MIN_MATCH_LENGTH / 2) as u32 + ); + assert_eq!( + LdmParams::adjust_for(27, 9).min_match_length, + (LDM_MIN_MATCH_LENGTH / 2) as u32 + ); +} + +/// `bucket_size_log = BOUNDED(LDM_BUCKET_SIZE_LOG, strategy, +/// LDM_BUCKETSIZELOG_MAX)` — upstream zstd `zstd_ldm.c:168`. +/// `LDM_BUCKET_SIZE_LOG = 4`, `LDM_BUCKETSIZELOG_MAX = 8`. +#[test] +fn adjust_bucket_size_log_clamps_strategy_to_bounds() { + // strategy 1 < lower bound 4 → clamps up + assert_eq!(LdmParams::adjust_for(27, 1).bucket_size_log, 4); + // strategy 4 == lower bound → identity + assert_eq!(LdmParams::adjust_for(27, 4).bucket_size_log, 4); + // strategy 7 in range → identity + assert_eq!(LdmParams::adjust_for(27, 7).bucket_size_log, 7); + // strategy 9 > upper bound 8 → clamps down + assert_eq!(LdmParams::adjust_for(27, 9).bucket_size_log, 8); +} + +/// Derived hash-table size + bucket slots agree with the +/// `1 << log` definitions and with each other. +#[test] +fn derived_helpers_match_log_definitions() { + let p = LdmParams::adjust_for(27, 5); + assert_eq!(p.hash_table_entries(), 1usize << p.hash_log); + assert_eq!(p.bucket_slots(), 1usize << p.bucket_size_log); +} diff --git a/zstd/src/encoding/ldm/search.rs b/zstd/src/encoding/ldm/search.rs index 14729fb31..782a4c3d4 100644 --- a/zstd/src/encoding/ldm/search.rs +++ b/zstd/src/encoding/ldm/search.rs @@ -286,314 +286,4 @@ pub(crate) fn find_best_match( } #[cfg(test)] -mod tests { - use super::*; - use crate::encoding::ldm::table::LdmHashTable; - - fn fresh_table() -> LdmHashTable { - // 4-bucket × 4-slot table — small enough that we can - // hand-place candidates in known slots. - LdmHashTable::new(4, 2) - } - - /// `count_backwards_match` honours both lower bounds and - /// stops on the first mismatch. Fixture: "XXXabc" matches - /// "YYYabc" with 3 backward bytes from offset 3 in each. - #[test] - fn count_backwards_match_walks_until_mismatch_or_bound() { - // history = "abc__abc" positions: 0..3 = "abc", - // 3..5 = "__", - // 5..8 = "abc". - // Backwards walk from p_in=8 (after "abc") and p_match=3 - // (after the first "abc") should match the 3 bytes - // "abc". - let history = b"abc__abc"; - let len = count_backwards_match(history, 8, 0, 3, 0); - assert_eq!(len, 3); - - // Mismatch on the 4th byte back: '_' (history[4]) vs - // nothing in the first window — the walk hits the - // match_base_abs bound (0) earlier on the match side. - let len2 = count_backwards_match(history, 5, 0, 0, 0); - // p_match starts at 0 → match_base bound reached - // immediately → 0 bytes matched. - assert_eq!(len2, 0); - } - - /// `anchor_abs` caps the backward walk on the in-stream side. - #[test] - fn count_backwards_match_respects_anchor_bound() { - let history = b"aaaaaaaa"; - // anchor at position 5 → only 1 byte of leftward room. - let len = count_backwards_match(history, 6, 5, 4, 0); - assert_eq!(len, 1); - } - - /// Bucket lookup returns `None` when every slot mismatches - /// the checksum. - #[test] - fn find_best_match_returns_none_on_checksum_mismatch() { - let mut table = fresh_table(); - table.insert_absolute(1, 4, 0x1111_1111); - let history = b"abcdefghabcdefgh"; - let m = find_best_match( - &table, - 1, - 0xDEAD_BEEF, - FindBestMatchInputs { - live_history: history, - history_abs_start: 0, - split_abs: 8, - anchor_abs: 0, - lowest_index_abs: 0, - iend_abs: history.len(), - min_match_length: 4, - }, - ); - assert!(m.is_none(), "wrong checksum must be filtered out"); - } - - /// Bucket lookup returns `None` when the offset is strictly - /// below `lowest_index_abs` (inclusive lower bound — entries - /// at exactly `lowest_index_abs` survive, entries below are - /// stale). - #[test] - fn find_best_match_rejects_stale_entries() { - let mut table = fresh_table(); - table.insert_absolute(1, 4, 0xCAFE); - let history = b"abcdefghabcdefgh"; - // lowest_index_abs = 5 → entry offset 4 is strictly below - // → rejected. (At lowest_index_abs = 4 the entry would - // exactly meet the floor and survive — that's the edge - // case the inclusive bound is designed to preserve.) - let m = find_best_match( - &table, - 1, - 0xCAFE, - FindBestMatchInputs { - live_history: history, - history_abs_start: 0, - split_abs: 8, - anchor_abs: 0, - lowest_index_abs: 5, - iend_abs: history.len(), - min_match_length: 4, - }, - ); - assert!(m.is_none(), "stale entry must be filtered out"); - } - - /// `find_best_match` returns the longest combined - /// forward+backward match across the bucket. Engineered - /// fixture: a 4-byte preamble (so the upstream zstd `offset > 0` - /// staleness floor is satisfied — entry.offset == 0 is the - /// reserved "empty slot" sentinel) followed by two - /// repetitions of "abcdefgh". The single candidate at - /// offset 4 should produce forward 8 + backward 0 = 8. - #[test] - fn find_best_match_picks_longest_combined_match() { - let mut table = fresh_table(); - table.insert_absolute(1, 4, 0xCAFE); - let history = b"PPPPabcdefghabcdefgh"; - // split at position 12, anchor at 12 → no backward room. - // The forward count should match 8 bytes ("abcdefgh"). - let m = find_best_match( - &table, - 1, - 0xCAFE, - FindBestMatchInputs { - live_history: history, - history_abs_start: 0, - split_abs: 12, - anchor_abs: 12, - lowest_index_abs: 0, - iend_abs: history.len(), - min_match_length: 4, - }, - ) - .expect("a valid candidate must be found"); - assert_eq!(m.match_pos, 4); - assert_eq!(m.forward_len, 8); - assert_eq!(m.backward_len, 0); - assert_eq!(m.total_len(), 8); - } - - /// Backward extension picks up the bytes BEFORE `split` when - /// `anchor` allows. Fixture: "XYabcdefghXYabcdefgh" — split - /// at position 12 ('a'), anchor at 10 ('X') gives 2 bytes of - /// backward room ("XY"). Forward 8 + backward 2 = total 10. - #[test] - fn find_best_match_extends_backwards_into_pre_split_bytes() { - let mut table = fresh_table(); - table.insert_absolute(1, 2, 0xCAFE); - let history = b"XYabcdefghXYabcdefgh"; - // split at 12 (start of second "abcdefgh"), anchor at 10 - // → backward up to 2 bytes ("XY" at positions 10..12 vs - // 0..2). Forward count: 8 bytes ("abcdefgh"). - let m = find_best_match( - &table, - 1, - 0xCAFE, - FindBestMatchInputs { - live_history: history, - history_abs_start: 0, - split_abs: 12, - anchor_abs: 10, - lowest_index_abs: 0, - iend_abs: history.len(), - min_match_length: 4, - }, - ) - .expect("a valid candidate must be found"); - assert_eq!(m.match_pos, 2); - assert_eq!(m.forward_len, 8); - assert_eq!(m.backward_len, 2); - assert_eq!(m.total_len(), 10); - } - - /// When the bucket holds multiple valid candidates the longer - /// combined match wins, regardless of slot order. Preamble - /// bytes shift both candidate offsets above the upstream zstd `offset - /// > 0` floor. - #[test] - fn find_best_match_prefers_longer_total_across_slots() { - let mut table = fresh_table(); - // Slot 0: offset 4 (short forward match — only 4 bytes). - table.insert_absolute(1, 4, 0xCAFE); - // Slot 1: offset 8 (8-byte match — extends further forward). - table.insert_absolute(1, 8, 0xCAFE); - let history = b"PPPPabcdabcdefghabcdefgh"; - // split at position 16 ('a' of trailing block). Match at - // offset 8 ("abcdefgh") gives 8 bytes forward; match at - // offset 4 ("abcdabcd...") gives only 4 bytes forward - // because the 5th byte ('a' vs 'e') mismatches. - let m = find_best_match( - &table, - 1, - 0xCAFE, - FindBestMatchInputs { - live_history: history, - history_abs_start: 0, - split_abs: 16, - anchor_abs: 16, - lowest_index_abs: 0, - iend_abs: history.len(), - min_match_length: 4, - }, - ) - .expect("a valid candidate must be found"); - assert_eq!(m.match_pos, 8, "longer-forward winner must be picked"); - assert_eq!(m.forward_len, 8); - } - - /// `find_best_match` must reject entries at or past - /// `split_abs` so the producer's `offset = split_abs − - /// match_pos` never underflows / emits a zero back-reference. - /// Regression for PR #139 round-9 review. - #[test] - fn find_best_match_rejects_entries_at_or_past_split() { - let mut table = fresh_table(); - // Inject a stray entry at exactly `split_abs` — would be - // structurally impossible from `LdmProducer::generate_into` - // (inserts happen after the lookup) but possible from a - // direct caller / test fixture / future extDict variant. - table.insert_absolute(1, 12, 0xCAFE); - let history = b"PPPPabcdefghabcdefgh"; - let m = find_best_match( - &table, - 1, - 0xCAFE, - FindBestMatchInputs { - live_history: history, - history_abs_start: 0, - split_abs: 12, - anchor_abs: 12, - lowest_index_abs: 0, - iend_abs: history.len(), - min_match_length: 4, - }, - ); - assert!(m.is_none(), "entry at split_abs must be rejected"); - - // Same fixture but entry at strictly past split — also - // must be rejected. - let mut table_after = fresh_table(); - table_after.insert_absolute(1, 16, 0xCAFE); - let m_after = find_best_match( - &table_after, - 1, - 0xCAFE, - FindBestMatchInputs { - live_history: history, - history_abs_start: 0, - split_abs: 12, - anchor_abs: 12, - lowest_index_abs: 0, - iend_abs: history.len(), - min_match_length: 4, - }, - ); - assert!(m_after.is_none(), "entry past split_abs must be rejected"); - } - - /// Forward count must respect `iend_abs` — even when matching - /// bytes continue past the block end inside `live_history`, - /// `forward_len` is capped at `iend_abs - split_abs`. - /// Regression for PR #139 round-7 review. - #[test] - fn find_best_match_forward_count_is_bounded_by_iend_abs() { - let mut table = fresh_table(); - table.insert_absolute(1, 4, 0xCAFE); - // 4 preamble bytes + two 8-byte "abcdefgh" runs = 20 bytes. - // Without iend_abs cap the match at split=12 vs match=4 - // would return forward_len = 8 ("abcdefgh"). With - // iend_abs = 16 (4 bytes past the split) the match must - // cap at 4. - let history = b"PPPPabcdefghabcdefgh"; - let m = find_best_match( - &table, - 1, - 0xCAFE, - FindBestMatchInputs { - live_history: history, - history_abs_start: 0, - split_abs: 12, - anchor_abs: 12, - lowest_index_abs: 0, - iend_abs: 16, // cap: only 4 forward bytes allowed - min_match_length: 4, - }, - ) - .expect("a 4-byte forward match still passes the min_match floor"); - assert_eq!( - m.forward_len, 4, - "forward count must cap at iend_abs - split_abs" - ); - } - - /// Forward match below `min_match_length` is rejected even - /// when the checksum agrees (upstream zstd `zstd_ldm.c:444/452`). - #[test] - fn find_best_match_filters_short_forward_matches() { - let mut table = fresh_table(); - table.insert_absolute(1, 4, 0xCAFE); - let history = b"PPPPabXXXXXXab"; - // 2-byte forward match from split=12 vs match=4, but - // min_match_length = 4 → rejected. - let m = find_best_match( - &table, - 1, - 0xCAFE, - FindBestMatchInputs { - live_history: history, - history_abs_start: 0, - split_abs: 12, - anchor_abs: 12, - lowest_index_abs: 0, - iend_abs: history.len(), - min_match_length: 4, - }, - ); - assert!(m.is_none()); - } -} +mod tests; diff --git a/zstd/src/encoding/ldm/search/tests.rs b/zstd/src/encoding/ldm/search/tests.rs new file mode 100644 index 000000000..592de03ae --- /dev/null +++ b/zstd/src/encoding/ldm/search/tests.rs @@ -0,0 +1,309 @@ +use super::*; +use crate::encoding::ldm::table::LdmHashTable; + +fn fresh_table() -> LdmHashTable { + // 4-bucket × 4-slot table — small enough that we can + // hand-place candidates in known slots. + LdmHashTable::new(4, 2) +} + +/// `count_backwards_match` honours both lower bounds and +/// stops on the first mismatch. Fixture: "XXXabc" matches +/// "YYYabc" with 3 backward bytes from offset 3 in each. +#[test] +fn count_backwards_match_walks_until_mismatch_or_bound() { + // history = "abc__abc" positions: 0..3 = "abc", + // 3..5 = "__", + // 5..8 = "abc". + // Backwards walk from p_in=8 (after "abc") and p_match=3 + // (after the first "abc") should match the 3 bytes + // "abc". + let history = b"abc__abc"; + let len = count_backwards_match(history, 8, 0, 3, 0); + assert_eq!(len, 3); + + // Mismatch on the 4th byte back: '_' (history[4]) vs + // nothing in the first window — the walk hits the + // match_base_abs bound (0) earlier on the match side. + let len2 = count_backwards_match(history, 5, 0, 0, 0); + // p_match starts at 0 → match_base bound reached + // immediately → 0 bytes matched. + assert_eq!(len2, 0); +} + +/// `anchor_abs` caps the backward walk on the in-stream side. +#[test] +fn count_backwards_match_respects_anchor_bound() { + let history = b"aaaaaaaa"; + // anchor at position 5 → only 1 byte of leftward room. + let len = count_backwards_match(history, 6, 5, 4, 0); + assert_eq!(len, 1); +} + +/// Bucket lookup returns `None` when every slot mismatches +/// the checksum. +#[test] +fn find_best_match_returns_none_on_checksum_mismatch() { + let mut table = fresh_table(); + table.insert_absolute(1, 4, 0x1111_1111); + let history = b"abcdefghabcdefgh"; + let m = find_best_match( + &table, + 1, + 0xDEAD_BEEF, + FindBestMatchInputs { + live_history: history, + history_abs_start: 0, + split_abs: 8, + anchor_abs: 0, + lowest_index_abs: 0, + iend_abs: history.len(), + min_match_length: 4, + }, + ); + assert!(m.is_none(), "wrong checksum must be filtered out"); +} + +/// Bucket lookup returns `None` when the offset is strictly +/// below `lowest_index_abs` (inclusive lower bound — entries +/// at exactly `lowest_index_abs` survive, entries below are +/// stale). +#[test] +fn find_best_match_rejects_stale_entries() { + let mut table = fresh_table(); + table.insert_absolute(1, 4, 0xCAFE); + let history = b"abcdefghabcdefgh"; + // lowest_index_abs = 5 → entry offset 4 is strictly below + // → rejected. (At lowest_index_abs = 4 the entry would + // exactly meet the floor and survive — that's the edge + // case the inclusive bound is designed to preserve.) + let m = find_best_match( + &table, + 1, + 0xCAFE, + FindBestMatchInputs { + live_history: history, + history_abs_start: 0, + split_abs: 8, + anchor_abs: 0, + lowest_index_abs: 5, + iend_abs: history.len(), + min_match_length: 4, + }, + ); + assert!(m.is_none(), "stale entry must be filtered out"); +} + +/// `find_best_match` returns the longest combined +/// forward+backward match across the bucket. Engineered +/// fixture: a 4-byte preamble (so the upstream zstd `offset > 0` +/// staleness floor is satisfied — entry.offset == 0 is the +/// reserved "empty slot" sentinel) followed by two +/// repetitions of "abcdefgh". The single candidate at +/// offset 4 should produce forward 8 + backward 0 = 8. +#[test] +fn find_best_match_picks_longest_combined_match() { + let mut table = fresh_table(); + table.insert_absolute(1, 4, 0xCAFE); + let history = b"PPPPabcdefghabcdefgh"; + // split at position 12, anchor at 12 → no backward room. + // The forward count should match 8 bytes ("abcdefgh"). + let m = find_best_match( + &table, + 1, + 0xCAFE, + FindBestMatchInputs { + live_history: history, + history_abs_start: 0, + split_abs: 12, + anchor_abs: 12, + lowest_index_abs: 0, + iend_abs: history.len(), + min_match_length: 4, + }, + ) + .expect("a valid candidate must be found"); + assert_eq!(m.match_pos, 4); + assert_eq!(m.forward_len, 8); + assert_eq!(m.backward_len, 0); + assert_eq!(m.total_len(), 8); +} + +/// Backward extension picks up the bytes BEFORE `split` when +/// `anchor` allows. Fixture: "XYabcdefghXYabcdefgh" — split +/// at position 12 ('a'), anchor at 10 ('X') gives 2 bytes of +/// backward room ("XY"). Forward 8 + backward 2 = total 10. +#[test] +fn find_best_match_extends_backwards_into_pre_split_bytes() { + let mut table = fresh_table(); + table.insert_absolute(1, 2, 0xCAFE); + let history = b"XYabcdefghXYabcdefgh"; + // split at 12 (start of second "abcdefgh"), anchor at 10 + // → backward up to 2 bytes ("XY" at positions 10..12 vs + // 0..2). Forward count: 8 bytes ("abcdefgh"). + let m = find_best_match( + &table, + 1, + 0xCAFE, + FindBestMatchInputs { + live_history: history, + history_abs_start: 0, + split_abs: 12, + anchor_abs: 10, + lowest_index_abs: 0, + iend_abs: history.len(), + min_match_length: 4, + }, + ) + .expect("a valid candidate must be found"); + assert_eq!(m.match_pos, 2); + assert_eq!(m.forward_len, 8); + assert_eq!(m.backward_len, 2); + assert_eq!(m.total_len(), 10); +} + +/// When the bucket holds multiple valid candidates the longer +/// combined match wins, regardless of slot order. Preamble +/// bytes shift both candidate offsets above the upstream zstd `offset +/// > 0` floor. +#[test] +fn find_best_match_prefers_longer_total_across_slots() { + let mut table = fresh_table(); + // Slot 0: offset 4 (short forward match — only 4 bytes). + table.insert_absolute(1, 4, 0xCAFE); + // Slot 1: offset 8 (8-byte match — extends further forward). + table.insert_absolute(1, 8, 0xCAFE); + let history = b"PPPPabcdabcdefghabcdefgh"; + // split at position 16 ('a' of trailing block). Match at + // offset 8 ("abcdefgh") gives 8 bytes forward; match at + // offset 4 ("abcdabcd...") gives only 4 bytes forward + // because the 5th byte ('a' vs 'e') mismatches. + let m = find_best_match( + &table, + 1, + 0xCAFE, + FindBestMatchInputs { + live_history: history, + history_abs_start: 0, + split_abs: 16, + anchor_abs: 16, + lowest_index_abs: 0, + iend_abs: history.len(), + min_match_length: 4, + }, + ) + .expect("a valid candidate must be found"); + assert_eq!(m.match_pos, 8, "longer-forward winner must be picked"); + assert_eq!(m.forward_len, 8); +} + +/// `find_best_match` must reject entries at or past +/// `split_abs` so the producer's `offset = split_abs − +/// match_pos` never underflows / emits a zero back-reference. +/// Regression for PR #139 round-9 review. +#[test] +fn find_best_match_rejects_entries_at_or_past_split() { + let mut table = fresh_table(); + // Inject a stray entry at exactly `split_abs` — would be + // structurally impossible from `LdmProducer::generate_into` + // (inserts happen after the lookup) but possible from a + // direct caller / test fixture / future extDict variant. + table.insert_absolute(1, 12, 0xCAFE); + let history = b"PPPPabcdefghabcdefgh"; + let m = find_best_match( + &table, + 1, + 0xCAFE, + FindBestMatchInputs { + live_history: history, + history_abs_start: 0, + split_abs: 12, + anchor_abs: 12, + lowest_index_abs: 0, + iend_abs: history.len(), + min_match_length: 4, + }, + ); + assert!(m.is_none(), "entry at split_abs must be rejected"); + + // Same fixture but entry at strictly past split — also + // must be rejected. + let mut table_after = fresh_table(); + table_after.insert_absolute(1, 16, 0xCAFE); + let m_after = find_best_match( + &table_after, + 1, + 0xCAFE, + FindBestMatchInputs { + live_history: history, + history_abs_start: 0, + split_abs: 12, + anchor_abs: 12, + lowest_index_abs: 0, + iend_abs: history.len(), + min_match_length: 4, + }, + ); + assert!(m_after.is_none(), "entry past split_abs must be rejected"); +} + +/// Forward count must respect `iend_abs` — even when matching +/// bytes continue past the block end inside `live_history`, +/// `forward_len` is capped at `iend_abs - split_abs`. +/// Regression for PR #139 round-7 review. +#[test] +fn find_best_match_forward_count_is_bounded_by_iend_abs() { + let mut table = fresh_table(); + table.insert_absolute(1, 4, 0xCAFE); + // 4 preamble bytes + two 8-byte "abcdefgh" runs = 20 bytes. + // Without iend_abs cap the match at split=12 vs match=4 + // would return forward_len = 8 ("abcdefgh"). With + // iend_abs = 16 (4 bytes past the split) the match must + // cap at 4. + let history = b"PPPPabcdefghabcdefgh"; + let m = find_best_match( + &table, + 1, + 0xCAFE, + FindBestMatchInputs { + live_history: history, + history_abs_start: 0, + split_abs: 12, + anchor_abs: 12, + lowest_index_abs: 0, + iend_abs: 16, // cap: only 4 forward bytes allowed + min_match_length: 4, + }, + ) + .expect("a 4-byte forward match still passes the min_match floor"); + assert_eq!( + m.forward_len, 4, + "forward count must cap at iend_abs - split_abs" + ); +} + +/// Forward match below `min_match_length` is rejected even +/// when the checksum agrees (upstream zstd `zstd_ldm.c:444/452`). +#[test] +fn find_best_match_filters_short_forward_matches() { + let mut table = fresh_table(); + table.insert_absolute(1, 4, 0xCAFE); + let history = b"PPPPabXXXXXXab"; + // 2-byte forward match from split=12 vs match=4, but + // min_match_length = 4 → rejected. + let m = find_best_match( + &table, + 1, + 0xCAFE, + FindBestMatchInputs { + live_history: history, + history_abs_start: 0, + split_abs: 12, + anchor_abs: 12, + lowest_index_abs: 0, + iend_abs: history.len(), + min_match_length: 4, + }, + ); + assert!(m.is_none()); +} diff --git a/zstd/src/encoding/ldm/table.rs b/zstd/src/encoding/ldm/table.rs index 78277df35..c73553cc7 100644 --- a/zstd/src/encoding/ldm/table.rs +++ b/zstd/src/encoding/ldm/table.rs @@ -389,345 +389,4 @@ impl LdmHashTable { } #[cfg(test)] -mod tests { - use super::*; - - /// `hash_log = 8`, `bucket_size_log = 4` → 16 buckets × - /// 16 slots = 256 entries, matches upstream zstd sizing math. - #[test] - fn new_table_sizes_match_size_formulae() { - let t = LdmHashTable::new(8, 4); - assert_eq!(t.bucket_count(), 16); - assert_eq!(t.bucket_slots(), 16); - assert_eq!(t.entries.len(), 256); - assert_eq!(t.bucket_offsets.len(), 16); - assert_eq!(t.bucket_mask(), 15); - } - - /// Upstream zstd `MIN(bucketSizeLog, hashLog)` clamp must apply: when - /// caller requests `bucket_size_log > hash_log` the bucket - /// collapses to a single bucket covering all entries. - #[test] - fn new_clamps_bucket_size_log_to_hash_log() { - let t = LdmHashTable::new(6, 12); // bucket > hash → clamp - assert_eq!(t.bucket_count(), 1, "clamp must yield a single bucket"); - assert_eq!(t.bucket_slots(), 1usize << 6); - assert_eq!(t.entries.len(), 1usize << 6); - } - - /// Round-robin insertion fills the bucket then wraps. - /// Uses offsets `1..=6` (not `0..6`) so the test does not - /// rely on the sentinel value `0`, which - /// [`LdmHashTable::insert`] now rejects with a runtime assert. - #[test] - fn insert_round_robin_wraps_through_bucket_slots() { - let mut t = LdmHashTable::new(4, 2); // 4 buckets × 4 slots - for k in 1..=6u32 { - t.insert( - 1, - LdmEntry { - offset: k, - checksum: k * 7, - }, - ); - } - let b = t.bucket(1); - // After 6 inserts in a 4-slot bucket the round-robin - // cursor cycles 0→1→2→3→0→1, so the last write to each - // slot is k=5 (slot 0), k=6 (slot 1), k=3 (slot 2), - // k=4 (slot 3). - assert_eq!(b[0].offset, 5); - assert_eq!(b[1].offset, 6); - assert_eq!(b[2].offset, 3); - assert_eq!(b[3].offset, 4); - } - - /// `insert` must reject the empty-slot sentinel `offset == 0` - /// — otherwise the entry would survive in the bucket but be - /// invisible to [`LdmHashTable::resolve`] (which treats `0` - /// as the empty marker), silently dropping candidates. - /// Regression for PR #139 round-16 review (CodeRabbit Major). - #[test] - #[should_panic(expected = "offset 0 is reserved")] - fn insert_panics_on_sentinel_offset_zero() { - let mut t = LdmHashTable::new(4, 2); - t.insert( - 0, - LdmEntry { - offset: 0, - checksum: 0xDEAD, - }, - ); - } - - /// Inserts to one bucket must not bleed into adjacent buckets. - /// Guards against off-by-one in the `bucket_start` arithmetic. - #[test] - fn insert_does_not_contaminate_adjacent_bucket() { - let mut t = LdmHashTable::new(4, 2); - t.insert( - 2, - LdmEntry { - offset: 42, - checksum: 0xCAFE, - }, - ); - let b0 = t.bucket(0); - let b1 = t.bucket(1); - let b3 = t.bucket(3); - for e in b0.iter().chain(b1.iter()).chain(b3.iter()) { - assert_eq!( - *e, - LdmEntry::default(), - "neighbouring buckets must stay empty" - ); - } - assert_eq!(t.bucket(2)[0].offset, 42); - } - - /// `clear` rewinds bucket cursors and zeros entries. - #[test] - fn clear_zeros_entries_and_rewinds_cursors() { - let mut t = LdmHashTable::new(4, 2); - for k in 0..4u32 { - t.insert( - k % 4, - LdmEntry { - offset: k + 1, - checksum: k * 11, - }, - ); - } - t.clear(); - for e in t.bucket(0).iter().chain(t.bucket(3).iter()) { - assert_eq!(*e, LdmEntry::default()); - } - for c in &t.bucket_offsets { - assert_eq!(*c, 0); - } - // First insert after clear must land at slot 0. - t.insert( - 2, - LdmEntry { - offset: 99, - checksum: 0, - }, - ); - assert_eq!(t.bucket(2)[0].offset, 99); - } - - /// Boundary-arithmetic smoke test: a moderately large `hash_log` - /// must allocate without panic and produce a sane bucket count. - /// Doubles as a guard that the assertions don't accidentally - /// reject the upstream zstd-supported range. - /// - /// We deliberately do NOT use `hash_log = 30` (upstream zstd's max) - /// because that would allocate 8 GiB of entries; the bucket - /// arithmetic is the same at every log so 18 is sufficient. - /// Gated to 64-bit pointer widths to avoid the 32-bit CI shards - /// where the 2 MiB allocation would still succeed but the - /// `usize` × `u32` cast would over-restrict the integer types - /// we exercise elsewhere. - #[test] - #[cfg(target_pointer_width = "64")] - fn new_accepts_large_hash_log_smoke() { - // Use a small bucket_size_log so the entry count is bounded - // and we don't actually allocate 8 GiB. Upstream zstd itself never - // allocates the max at runtime either (window_log caps - // hash_log to 27 or so in practice). Test just the boundary - // arithmetic — request hash_log = 18 with bucket_size_log = - // 4 → 16384 buckets × 16 slots = 262144 entries × 8 bytes - // = ~2 MiB allocation, safe on every CI runner. - let t = LdmHashTable::new(18, 4); - assert_eq!(t.bucket_count(), 1usize << (18 - 4)); - assert_eq!(t.bucket_slots(), 1usize << 4); - } - - /// `effective_bucket_log > 8` (upstream zstd `LDM_BUCKETSIZELOG_MAX`) - /// must panic, not silently truncate the `u8` round-robin - /// cursor at 256 slots. Upstream zstd pre-condition mirrored from - /// `zstd_ldm.c:202` where `bucketOffsets` is a `BYTE`. - #[test] - #[should_panic(expected = "ZSTD_LDM_BUCKETSIZELOG_MAX")] - fn new_rejects_bucket_size_log_above_cap() { - // hash_log = 12, bucket_size_log = 9 → effective = 9 > 8 - // → assertion fires. - let _ = LdmHashTable::new(12, 9); - } - - /// `insert_absolute` + `resolve` round-trip preserves the - /// absolute position across the +1 empty-slot bias. - #[test] - fn insert_absolute_round_trips_through_resolve() { - let mut t = LdmHashTable::new(4, 2); - t.insert_absolute(1, 42, 0xCAFE); - let entry = t.bucket(1)[0]; - assert_eq!(entry.checksum, 0xCAFE); - assert_eq!( - t.resolve(&entry), - Some(42), - "stored relative offset must resolve back to the inserted absolute" - ); - } - - /// `resolve` returns `None` for the empty-slot sentinel - /// (`offset == 0`), distinguishing it from any real - /// inserted position. - #[test] - fn resolve_returns_none_for_empty_slot() { - let t = LdmHashTable::new(4, 2); - let empty = LdmEntry::default(); - assert_eq!(empty.offset, 0); - assert_eq!(t.resolve(&empty), None); - } - - /// `reduce` subtracts the reducer from every entry's relative - /// offset (saturating at 0 = empty sentinel) and advances - /// `position_base` so future `resolve` calls translate back - /// to the same absolute positions. Upstream zstd - /// `ZSTD_ldm_reduceTable` (`zstd_ldm.c:520`). - #[test] - fn reduce_preserves_resolved_absolute_positions() { - let mut t = LdmHashTable::new(4, 2); - t.insert_absolute(0, 100, 0xAAAA); - t.insert_absolute(1, 200, 0xBBBB); - t.insert_absolute(2, 300, 0xCCCC); - assert_eq!(t.position_base(), 0); - - // Shift the base forward by 150; positions 100 and 200 - // should still resolve to 100 and 200 (the relative - // offsets shift but absolute stays). - t.reduce(150); - assert_eq!(t.position_base(), 150); - // pos 100 had relative offset 101 → after reduce: max(101−150, 0) = 0 (sentinel) - let entry0 = t.bucket(0)[0]; - assert_eq!( - t.resolve(&entry0), - None, - "pos 100 < new_base 150 must be evicted" - ); - // pos 200 had relative 201 → 201−150 = 51 → resolved 150 + 51 − 1 = 200 - let entry1 = t.bucket(1)[0]; - assert_eq!(t.resolve(&entry1), Some(200)); - // pos 300 had relative 301 → 301−150 = 151 → resolved 150 + 151 − 1 = 300 - let entry2 = t.bucket(2)[0]; - assert_eq!(t.resolve(&entry2), Some(300)); - } - - /// `ensure_room_for` triggers a rebase when the relative - /// offset would exceed the guard band, keeping the `u32` - /// storage valid for streams past 4 GiB. - #[test] - fn ensure_room_for_rebases_above_guard_band() { - let mut t = LdmHashTable::new(4, 2); - // First insert at moderate offset — no rebase needed. - t.insert_absolute(0, 1024, 0xAAAA); - assert_eq!(t.position_base(), 0); - - // Probe a position that would overflow the guard band: - // u32::MAX - REBASE_GUARD_BAND + 1 = the smallest abs - // that triggers a rebase. - let trigger_pos = (u32::MAX as usize) - (REBASE_GUARD_BAND as usize) + 1; - t.ensure_room_for(trigger_pos); - assert_eq!( - t.position_base(), - REBASE_GUARD_BAND as usize, - "rebase must advance position_base by REBASE_GUARD_BAND" - ); - // The earlier insert at 1024 had relative offset 1025; - // after rebase by 2^30 it's clamped to 0 (empty) since - // 1025 < REBASE_GUARD_BAND. - assert_eq!(t.resolve(&t.bucket(0)[0]), None); - - // A fresh insert past the rebase boundary must still - // round-trip. - t.insert_absolute(2, trigger_pos, 0xCAFE); - assert_eq!(t.resolve(&t.bucket(2)[0]), Some(trigger_pos)); - } - - /// `ensure_room_for` must loop until `rel <= max_rel` even if - /// the caller jumps the position past several guard bands in - /// a single call. With the old single-shot `if`, a jump - /// larger than `2 * REBASE_GUARD_BAND` left `rel` above - /// `u32::MAX - REBASE_GUARD_BAND`, so the next - /// `insert_absolute` would panic on the `(rel + 1) as u32` - /// cast. Regression for PR #139 round-14 review (CodeRabbit - /// Major). - /// - /// Gated to 64-bit pointer widths: `5 * REBASE_GUARD_BAND` - /// (= 5 GiB) overflows `usize` on 32-bit targets, where the - /// scenario is unreachable anyway because `usize::MAX` < 4 - /// GiB caps the addressable stream below the rebase - /// threshold. - #[test] - #[cfg(target_pointer_width = "64")] - fn ensure_room_for_loops_across_multiple_guard_bands() { - let mut t = LdmHashTable::new(4, 2); - // Jump past two guard bands at once. With u32::MAX ≈ - // 4 * REBASE_GUARD_BAND and max_rel = u32::MAX - - // REBASE_GUARD_BAND ≈ 3 * REBASE_GUARD_BAND, an abs_pos - // of 5 * REBASE_GUARD_BAND yields rel = 5 * - // REBASE_GUARD_BAND > max_rel even after one reduce - // (rel = 4 * REBASE_GUARD_BAND > 3 * REBASE_GUARD_BAND). - // A second reduce brings rel = 3 * REBASE_GUARD_BAND ≤ - // max_rel and the loop exits. - let abs_pos = 5usize * (REBASE_GUARD_BAND as usize); - t.ensure_room_for(abs_pos); - let max_rel = u32::MAX as usize - REBASE_GUARD_BAND as usize; - assert!( - abs_pos - t.position_base() <= max_rel, - "ensure_room_for must rebase until rel ≤ max_rel \ - (got rel = {}, max_rel = {})", - abs_pos - t.position_base(), - max_rel - ); - // The subsequent insert must succeed (no u32 overflow). - t.insert_absolute(0, abs_pos, 0xFEED); - assert_eq!(t.resolve(&t.bucket(0)[0]), Some(abs_pos)); - } - - /// `insert_absolute` must panic in BOTH debug and release - /// builds when called with an `abs_pos` below the current - /// `position_base` — silently underflowing the subtraction - /// would store a wraparound relative offset and corrupt the - /// table far from the bug's source. Regression for PR #139 - /// round-6 review (Copilot + CodeRabbit, Major). - #[test] - #[should_panic(expected = "below position_base")] - fn insert_absolute_panics_below_position_base() { - let mut t = LdmHashTable::new(4, 2); - t.reduce(100); // shift position_base to 100 - t.insert_absolute(0, 50, 0xCAFE); // 50 < 100 → panic - } - - /// `clear()` must reset `position_base` to 0 so a subsequent - /// frame can insert at any absolute position (including - /// values below the previous frame's `position_base`) - /// without tripping the `abs_pos >= position_base` - /// assertion. Regression for PR #139 round-4 CodeRabbit - /// nitpick. - #[test] - fn clear_resets_position_base() { - let mut t = LdmHashTable::new(4, 2); - t.insert_absolute(0, 1024, 0xAAAA); - t.reduce(1 << 20); // shift base forward - assert!(t.position_base() > 0); - t.clear(); - assert_eq!(t.position_base(), 0, "clear must rewind position_base to 0"); - // Inserting at absolute 0 after clear must succeed — - // would panic on the assertion if position_base wasn't - // reset. - t.insert_absolute(0, 0, 0xCAFE); - let entry = t.bucket(0)[0]; - assert_eq!(t.resolve(&entry), Some(0)); - } - - /// `bucket_mask` returned by the table must agree with the - /// derived `bucket_count - 1`. Guard against drift if the - /// internal field is renamed. - #[test] - fn bucket_mask_matches_count_minus_one() { - let t = LdmHashTable::new(8, 3); - assert_eq!(t.bucket_mask() as usize + 1, t.bucket_count()); - } -} +mod tests; diff --git a/zstd/src/encoding/ldm/table/tests.rs b/zstd/src/encoding/ldm/table/tests.rs new file mode 100644 index 000000000..6479906c8 --- /dev/null +++ b/zstd/src/encoding/ldm/table/tests.rs @@ -0,0 +1,340 @@ +use super::*; + +/// `hash_log = 8`, `bucket_size_log = 4` → 16 buckets × +/// 16 slots = 256 entries, matches upstream zstd sizing math. +#[test] +fn new_table_sizes_match_size_formulae() { + let t = LdmHashTable::new(8, 4); + assert_eq!(t.bucket_count(), 16); + assert_eq!(t.bucket_slots(), 16); + assert_eq!(t.entries.len(), 256); + assert_eq!(t.bucket_offsets.len(), 16); + assert_eq!(t.bucket_mask(), 15); +} + +/// Upstream zstd `MIN(bucketSizeLog, hashLog)` clamp must apply: when +/// caller requests `bucket_size_log > hash_log` the bucket +/// collapses to a single bucket covering all entries. +#[test] +fn new_clamps_bucket_size_log_to_hash_log() { + let t = LdmHashTable::new(6, 12); // bucket > hash → clamp + assert_eq!(t.bucket_count(), 1, "clamp must yield a single bucket"); + assert_eq!(t.bucket_slots(), 1usize << 6); + assert_eq!(t.entries.len(), 1usize << 6); +} + +/// Round-robin insertion fills the bucket then wraps. +/// Uses offsets `1..=6` (not `0..6`) so the test does not +/// rely on the sentinel value `0`, which +/// [`LdmHashTable::insert`] now rejects with a runtime assert. +#[test] +fn insert_round_robin_wraps_through_bucket_slots() { + let mut t = LdmHashTable::new(4, 2); // 4 buckets × 4 slots + for k in 1..=6u32 { + t.insert( + 1, + LdmEntry { + offset: k, + checksum: k * 7, + }, + ); + } + let b = t.bucket(1); + // After 6 inserts in a 4-slot bucket the round-robin + // cursor cycles 0→1→2→3→0→1, so the last write to each + // slot is k=5 (slot 0), k=6 (slot 1), k=3 (slot 2), + // k=4 (slot 3). + assert_eq!(b[0].offset, 5); + assert_eq!(b[1].offset, 6); + assert_eq!(b[2].offset, 3); + assert_eq!(b[3].offset, 4); +} + +/// `insert` must reject the empty-slot sentinel `offset == 0` +/// — otherwise the entry would survive in the bucket but be +/// invisible to [`LdmHashTable::resolve`] (which treats `0` +/// as the empty marker), silently dropping candidates. +/// Regression for PR #139 round-16 review (CodeRabbit Major). +#[test] +#[should_panic(expected = "offset 0 is reserved")] +fn insert_panics_on_sentinel_offset_zero() { + let mut t = LdmHashTable::new(4, 2); + t.insert( + 0, + LdmEntry { + offset: 0, + checksum: 0xDEAD, + }, + ); +} + +/// Inserts to one bucket must not bleed into adjacent buckets. +/// Guards against off-by-one in the `bucket_start` arithmetic. +#[test] +fn insert_does_not_contaminate_adjacent_bucket() { + let mut t = LdmHashTable::new(4, 2); + t.insert( + 2, + LdmEntry { + offset: 42, + checksum: 0xCAFE, + }, + ); + let b0 = t.bucket(0); + let b1 = t.bucket(1); + let b3 = t.bucket(3); + for e in b0.iter().chain(b1.iter()).chain(b3.iter()) { + assert_eq!( + *e, + LdmEntry::default(), + "neighbouring buckets must stay empty" + ); + } + assert_eq!(t.bucket(2)[0].offset, 42); +} + +/// `clear` rewinds bucket cursors and zeros entries. +#[test] +fn clear_zeros_entries_and_rewinds_cursors() { + let mut t = LdmHashTable::new(4, 2); + for k in 0..4u32 { + t.insert( + k % 4, + LdmEntry { + offset: k + 1, + checksum: k * 11, + }, + ); + } + t.clear(); + for e in t.bucket(0).iter().chain(t.bucket(3).iter()) { + assert_eq!(*e, LdmEntry::default()); + } + for c in &t.bucket_offsets { + assert_eq!(*c, 0); + } + // First insert after clear must land at slot 0. + t.insert( + 2, + LdmEntry { + offset: 99, + checksum: 0, + }, + ); + assert_eq!(t.bucket(2)[0].offset, 99); +} + +/// Boundary-arithmetic smoke test: a moderately large `hash_log` +/// must allocate without panic and produce a sane bucket count. +/// Doubles as a guard that the assertions don't accidentally +/// reject the upstream zstd-supported range. +/// +/// We deliberately do NOT use `hash_log = 30` (upstream zstd's max) +/// because that would allocate 8 GiB of entries; the bucket +/// arithmetic is the same at every log so 18 is sufficient. +/// Gated to 64-bit pointer widths to avoid the 32-bit CI shards +/// where the 2 MiB allocation would still succeed but the +/// `usize` × `u32` cast would over-restrict the integer types +/// we exercise elsewhere. +#[test] +#[cfg(target_pointer_width = "64")] +fn new_accepts_large_hash_log_smoke() { + // Use a small bucket_size_log so the entry count is bounded + // and we don't actually allocate 8 GiB. Upstream zstd itself never + // allocates the max at runtime either (window_log caps + // hash_log to 27 or so in practice). Test just the boundary + // arithmetic — request hash_log = 18 with bucket_size_log = + // 4 → 16384 buckets × 16 slots = 262144 entries × 8 bytes + // = ~2 MiB allocation, safe on every CI runner. + let t = LdmHashTable::new(18, 4); + assert_eq!(t.bucket_count(), 1usize << (18 - 4)); + assert_eq!(t.bucket_slots(), 1usize << 4); +} + +/// `effective_bucket_log > 8` (upstream zstd `LDM_BUCKETSIZELOG_MAX`) +/// must panic, not silently truncate the `u8` round-robin +/// cursor at 256 slots. Upstream zstd pre-condition mirrored from +/// `zstd_ldm.c:202` where `bucketOffsets` is a `BYTE`. +#[test] +#[should_panic(expected = "ZSTD_LDM_BUCKETSIZELOG_MAX")] +fn new_rejects_bucket_size_log_above_cap() { + // hash_log = 12, bucket_size_log = 9 → effective = 9 > 8 + // → assertion fires. + let _ = LdmHashTable::new(12, 9); +} + +/// `insert_absolute` + `resolve` round-trip preserves the +/// absolute position across the +1 empty-slot bias. +#[test] +fn insert_absolute_round_trips_through_resolve() { + let mut t = LdmHashTable::new(4, 2); + t.insert_absolute(1, 42, 0xCAFE); + let entry = t.bucket(1)[0]; + assert_eq!(entry.checksum, 0xCAFE); + assert_eq!( + t.resolve(&entry), + Some(42), + "stored relative offset must resolve back to the inserted absolute" + ); +} + +/// `resolve` returns `None` for the empty-slot sentinel +/// (`offset == 0`), distinguishing it from any real +/// inserted position. +#[test] +fn resolve_returns_none_for_empty_slot() { + let t = LdmHashTable::new(4, 2); + let empty = LdmEntry::default(); + assert_eq!(empty.offset, 0); + assert_eq!(t.resolve(&empty), None); +} + +/// `reduce` subtracts the reducer from every entry's relative +/// offset (saturating at 0 = empty sentinel) and advances +/// `position_base` so future `resolve` calls translate back +/// to the same absolute positions. Upstream zstd +/// `ZSTD_ldm_reduceTable` (`zstd_ldm.c:520`). +#[test] +fn reduce_preserves_resolved_absolute_positions() { + let mut t = LdmHashTable::new(4, 2); + t.insert_absolute(0, 100, 0xAAAA); + t.insert_absolute(1, 200, 0xBBBB); + t.insert_absolute(2, 300, 0xCCCC); + assert_eq!(t.position_base(), 0); + + // Shift the base forward by 150; positions 100 and 200 + // should still resolve to 100 and 200 (the relative + // offsets shift but absolute stays). + t.reduce(150); + assert_eq!(t.position_base(), 150); + // pos 100 had relative offset 101 → after reduce: max(101−150, 0) = 0 (sentinel) + let entry0 = t.bucket(0)[0]; + assert_eq!( + t.resolve(&entry0), + None, + "pos 100 < new_base 150 must be evicted" + ); + // pos 200 had relative 201 → 201−150 = 51 → resolved 150 + 51 − 1 = 200 + let entry1 = t.bucket(1)[0]; + assert_eq!(t.resolve(&entry1), Some(200)); + // pos 300 had relative 301 → 301−150 = 151 → resolved 150 + 151 − 1 = 300 + let entry2 = t.bucket(2)[0]; + assert_eq!(t.resolve(&entry2), Some(300)); +} + +/// `ensure_room_for` triggers a rebase when the relative +/// offset would exceed the guard band, keeping the `u32` +/// storage valid for streams past 4 GiB. +#[test] +fn ensure_room_for_rebases_above_guard_band() { + let mut t = LdmHashTable::new(4, 2); + // First insert at moderate offset — no rebase needed. + t.insert_absolute(0, 1024, 0xAAAA); + assert_eq!(t.position_base(), 0); + + // Probe a position that would overflow the guard band: + // u32::MAX - REBASE_GUARD_BAND + 1 = the smallest abs + // that triggers a rebase. + let trigger_pos = (u32::MAX as usize) - (REBASE_GUARD_BAND as usize) + 1; + t.ensure_room_for(trigger_pos); + assert_eq!( + t.position_base(), + REBASE_GUARD_BAND as usize, + "rebase must advance position_base by REBASE_GUARD_BAND" + ); + // The earlier insert at 1024 had relative offset 1025; + // after rebase by 2^30 it's clamped to 0 (empty) since + // 1025 < REBASE_GUARD_BAND. + assert_eq!(t.resolve(&t.bucket(0)[0]), None); + + // A fresh insert past the rebase boundary must still + // round-trip. + t.insert_absolute(2, trigger_pos, 0xCAFE); + assert_eq!(t.resolve(&t.bucket(2)[0]), Some(trigger_pos)); +} + +/// `ensure_room_for` must loop until `rel <= max_rel` even if +/// the caller jumps the position past several guard bands in +/// a single call. With the old single-shot `if`, a jump +/// larger than `2 * REBASE_GUARD_BAND` left `rel` above +/// `u32::MAX - REBASE_GUARD_BAND`, so the next +/// `insert_absolute` would panic on the `(rel + 1) as u32` +/// cast. Regression for PR #139 round-14 review (CodeRabbit +/// Major). +/// +/// Gated to 64-bit pointer widths: `5 * REBASE_GUARD_BAND` +/// (= 5 GiB) overflows `usize` on 32-bit targets, where the +/// scenario is unreachable anyway because `usize::MAX` < 4 +/// GiB caps the addressable stream below the rebase +/// threshold. +#[test] +#[cfg(target_pointer_width = "64")] +fn ensure_room_for_loops_across_multiple_guard_bands() { + let mut t = LdmHashTable::new(4, 2); + // Jump past two guard bands at once. With u32::MAX ≈ + // 4 * REBASE_GUARD_BAND and max_rel = u32::MAX - + // REBASE_GUARD_BAND ≈ 3 * REBASE_GUARD_BAND, an abs_pos + // of 5 * REBASE_GUARD_BAND yields rel = 5 * + // REBASE_GUARD_BAND > max_rel even after one reduce + // (rel = 4 * REBASE_GUARD_BAND > 3 * REBASE_GUARD_BAND). + // A second reduce brings rel = 3 * REBASE_GUARD_BAND ≤ + // max_rel and the loop exits. + let abs_pos = 5usize * (REBASE_GUARD_BAND as usize); + t.ensure_room_for(abs_pos); + let max_rel = u32::MAX as usize - REBASE_GUARD_BAND as usize; + assert!( + abs_pos - t.position_base() <= max_rel, + "ensure_room_for must rebase until rel ≤ max_rel \ + (got rel = {}, max_rel = {})", + abs_pos - t.position_base(), + max_rel + ); + // The subsequent insert must succeed (no u32 overflow). + t.insert_absolute(0, abs_pos, 0xFEED); + assert_eq!(t.resolve(&t.bucket(0)[0]), Some(abs_pos)); +} + +/// `insert_absolute` must panic in BOTH debug and release +/// builds when called with an `abs_pos` below the current +/// `position_base` — silently underflowing the subtraction +/// would store a wraparound relative offset and corrupt the +/// table far from the bug's source. Regression for PR #139 +/// round-6 review (Copilot + CodeRabbit, Major). +#[test] +#[should_panic(expected = "below position_base")] +fn insert_absolute_panics_below_position_base() { + let mut t = LdmHashTable::new(4, 2); + t.reduce(100); // shift position_base to 100 + t.insert_absolute(0, 50, 0xCAFE); // 50 < 100 → panic +} + +/// `clear()` must reset `position_base` to 0 so a subsequent +/// frame can insert at any absolute position (including +/// values below the previous frame's `position_base`) +/// without tripping the `abs_pos >= position_base` +/// assertion. Regression for PR #139 round-4 CodeRabbit +/// nitpick. +#[test] +fn clear_resets_position_base() { + let mut t = LdmHashTable::new(4, 2); + t.insert_absolute(0, 1024, 0xAAAA); + t.reduce(1 << 20); // shift base forward + assert!(t.position_base() > 0); + t.clear(); + assert_eq!(t.position_base(), 0, "clear must rewind position_base to 0"); + // Inserting at absolute 0 after clear must succeed — + // would panic on the assertion if position_base wasn't + // reset. + t.insert_absolute(0, 0, 0xCAFE); + let entry = t.bucket(0)[0]; + assert_eq!(t.resolve(&entry), Some(0)); +} + +/// `bucket_mask` returned by the table must agree with the +/// derived `bucket_count - 1`. Guard against drift if the +/// internal field is renamed. +#[test] +fn bucket_mask_matches_count_minus_one() { + let t = LdmHashTable::new(8, 3); + assert_eq!(t.bucket_mask() as usize + 1, t.bucket_count()); +} diff --git a/zstd/src/encoding/ldm/tests.rs b/zstd/src/encoding/ldm/tests.rs new file mode 100644 index 000000000..cdeea4b8d --- /dev/null +++ b/zstd/src/encoding/ldm/tests.rs @@ -0,0 +1,294 @@ +use super::*; + +/// `LdmParams::adjust_for` must derive a representative +/// upstream zstd-btultra2 parameter set at window_log=27. Checked via +/// the parameter struct alone — instantiating the producer +/// at these knobs would allocate `1 << 23 = 8M` table entries +/// (~64 MiB), which slows nextest under parallelism and risks +/// OOM on 32-bit CI shards. The producer-construction smoke +/// path is exercised by every other test in this module with +/// a smaller hand-tuned `LdmParams`. +#[test] +fn producer_constructs_with_default_params() { + let p = LdmParams::adjust_for(27, 9); + // Upstream zstd defaults at btultra2: minMatch halved, hash_rate_log + // = 4, bucket_size_log clamps to 8. See params::tests for + // the per-knob derivations. + assert_eq!(p.window_log, 27); + assert_eq!(p.min_match_length, 32); + assert_eq!(p.hash_rate_log, 4); + assert_eq!(p.bucket_size_log, 8); +} + +/// Compact `LdmParams` for unit-test producer construction — +/// keeps the table allocation at ~8 KiB instead of the +/// upstream zstd-btultra2 64 MiB so parallel nextest stays stable. +/// `min_match_length = 32` (half of upstream zstd floor) matches the +/// btultra2 derivation so engineered fixtures still trigger +/// long-range matches at 32-byte windows. +fn test_params() -> LdmParams { + LdmParams { + window_log: 27, + hash_log: 10, + hash_rate_log: 4, + min_match_length: 32, + bucket_size_log: 4, + } +} + +/// `clear` after `generate_into` rewinds the rolling hash to +/// the canonical init value — guards the frame-boundary +/// contract. +#[test] +fn clear_resets_rolling_hash_state() { + let mut producer = LdmProducer::new(test_params()); + let mut out = Vec::new(); + // Feed a non-empty chunk so the rolling hash advances. + let data = [0xAAu8; 256]; + producer.generate_into(&data, 0, 0, data.len(), &mut out); + let advanced = producer.hash_state.rolling; + assert_ne!( + advanced, + gear_hash::GEAR_HASH_INIT, + "rolling hash should have moved after generate_into" + ); + producer.clear(); + assert_eq!( + producer.hash_state.rolling, + gear_hash::GEAR_HASH_INIT, + "clear must rewind to GEAR_HASH_INIT" + ); +} + +/// End-to-end producer pipeline: a long-range repetition +/// (two copies of a 4 KiB random-like payload separated by +/// 64 KiB of unique filler) must emit at least one +/// [`HcRawSeq`] whose `offset` equals the distance between +/// the copies and whose `match_length` reaches at least the +/// upstream zstd `min_match_length` floor. Validates that +/// gear-hash → bucket-insert → checksum-filter → forward / +/// backward extend → emit all hold together. +#[test] +fn generate_into_emits_long_range_match_on_repeated_payload() { + // Use level 22 / btultra2 / windowLog 27 to get the + // halved min_match (32) — the default 64 would require a + // 128 KiB+ fixture to comfortably trigger a split inside + // the payload, which slows the test without adding + // coverage. + let mut producer = LdmProducer::new(test_params()); + let p = producer.params(); + assert_eq!(p.min_match_length, 32); + + // Build a fixture: payload, gap, payload, sized so the + // second payload sits comfortably past the gear hash's + // `2 ^ hash_rate_log` ≈ 16-byte expected split spacing. + // 4 KiB payload + 64 KiB gap + 4 KiB payload ⇒ ~72 KiB + // total, plenty of bytes for multiple split points inside + // each copy. + const PAYLOAD: usize = 4096; + const GAP: usize = 64 * 1024; + let mut history = Vec::with_capacity(2 * PAYLOAD + GAP); + // Deterministic non-trivial payload: a simple LCG so the + // bytes look random to the gear hash but the fixture + // remains reproducible across runs. + let mut prng: u32 = 0x1234_5678; + let payload: alloc::vec::Vec = (0..PAYLOAD) + .map(|_| { + prng = prng.wrapping_mul(1_103_515_245).wrapping_add(12_345); + (prng >> 16) as u8 + }) + .collect(); + history.extend_from_slice(&payload); + // Unique-byte gap: counter mod 251 keeps the gap + // statistically distinct from the payload (251 prime so + // it doesn't accidentally align with payload bytes). + history.extend((0..GAP).map(|i| (i % 251) as u8)); + history.extend_from_slice(&payload); + + let mut out = Vec::new(); + producer.generate_into(&history, 0, 0, history.len(), &mut out); + + assert!( + !out.is_empty(), + "long-range repetition must produce at least one LDM sequence" + ); + // Every emitted sequence must satisfy the upstream zstd floor. + for seq in &out { + assert!( + seq.match_length >= p.min_match_length as usize, + "every emitted match must reach min_match_length \ + (got match_length = {})", + seq.match_length + ); + } + // At least one emitted sequence must reach across the + // gap into the first payload copy — this is the + // long-range match LDM exists to capture. Short-distance + // matches found inside the first payload (statistically + // possible on random-like content) are allowed too, but + // the long-range one must show up. + let crossing_gap = out.iter().any(|s| s.offset >= GAP); + assert!( + crossing_gap, + "at least one emitted sequence must have offset >= GAP \ + ({GAP}); offsets observed: {:?}", + out.iter().map(|s| s.offset).collect::>() + ); +} + +/// Regression test for PR #139 round-2 review (CodeRabbit + +/// Copilot, Major): the producer must store **absolute stream +/// positions** in its bucket-table entries so that long-range +/// matches accumulated by one block remain valid after a +/// window eviction shifts `history_abs_start` forward. +/// +/// Setup: same 4 KiB payload appears twice in the per-frame +/// history. We invoke `generate_into` twice: +/// 1. First call covers absolute range `[0, payload_end_0)` +/// — the producer's bucket table accumulates entries +/// whose `offset` fields hold absolute positions inside +/// the first payload copy. +/// 2. Second call advances `history_abs_start` (simulates +/// window eviction) and covers the absolute range +/// containing the second payload copy. The bucket-table +/// entries from call 1 must remain reachable: their +/// absolute offsets still point at the SAME bytes +/// (now further left in the live slice), and the +/// inclusive lower-bound staleness check +/// `entry.offset < history_abs_start` keeps them +/// in-window (entries at exactly `history_abs_start` +/// survive). +/// +/// If the producer had stored slice-relative indices +/// instead, call 2 would either miss the long-range match +/// entirely (slice indices from call 1 would point into the +/// wrong bytes after the slide) or underflow `offset = +/// split_abs − best.match_pos` and emit a corrupt +/// back-reference. +#[test] +fn generate_into_preserves_bucket_entries_across_history_slide() { + let mut producer = LdmProducer::new(test_params()); + let p = producer.params(); + assert_eq!(p.min_match_length, 32); + + const PAYLOAD: usize = 4096; + const GAP_A: usize = 32 * 1024; + const GAP_B: usize = 32 * 1024; + + // Deterministic non-trivial payload. + let mut prng: u32 = 0xC0FFEEEE; + let payload: alloc::vec::Vec = (0..PAYLOAD) + .map(|_| { + prng = prng.wrapping_mul(1_103_515_245).wrapping_add(12_345); + (prng >> 16) as u8 + }) + .collect(); + + // Build the full frame in one Vec — represents the + // contiguous live history visible to the encoder before + // any eviction. + let mut frame = alloc::vec::Vec::with_capacity(2 * PAYLOAD + GAP_A + GAP_B); + frame.extend_from_slice(&payload); + frame.extend((0..GAP_A).map(|i| (i % 251) as u8)); + frame.extend_from_slice(&payload); + frame.extend((0..GAP_B).map(|i| ((i + 17) % 241) as u8)); + + // Call 1: cover the first half of the frame — + // history_abs_start = 0, walks the first payload copy + // and the start of the first gap. The bucket table + // populates with absolute offsets inside payload #1. + let mut out1 = Vec::new(); + let split_at = PAYLOAD + GAP_A; + producer.generate_into(&frame[..split_at], 0, 0, split_at, &mut out1); + + // Call 2: simulate a window slide. The encoder retired + // the leading `eviction` bytes from the live history; + // the surviving slice is `frame[eviction..]` and its + // byte 0 sits at absolute position `eviction`. The + // second payload copy is at absolute position `PAYLOAD + // + GAP_A`, which after the slide is at index + // `PAYLOAD + GAP_A − eviction` inside the slice. + let eviction = PAYLOAD / 2; // arbitrary — payload #1 partially evicted + let live = &frame[eviction..]; + let history_abs_start = eviction; + let block_start_abs = PAYLOAD + GAP_A; // start of payload #2 + let block_end_abs = block_start_abs + PAYLOAD; + + let mut out2 = Vec::new(); + producer.generate_into( + live, + history_abs_start, + block_start_abs, + block_end_abs, + &mut out2, + ); + + // The long-range match must still fire — entries from + // call 1 in the surviving tail of payload #1 + // (`absolute [eviction, PAYLOAD)`) are still in-window + // and still point at the right bytes. If the producer + // had stored slice-relative offsets, call 2 would + // either miss or emit corrupt offsets. + assert!( + !out2.is_empty(), + "cross-slide long-range match must survive a window eviction" + ); + // Every emitted match must (a) clear the min_match + // floor, and (b) point at bytes that still live inside + // the post-slide history. The latter is the actual + // round-2 review-fix invariant: if the producer had + // stored slice-relative offsets, entries inserted by + // call 1 would now reference the wrong absolute bytes + // and `offset = split_abs − stale_match_pos` could + // underflow `u32` and emit an offset > live_history + // length. + let live_len = live.len(); + for seq in &out2 { + assert!( + seq.match_length >= p.min_match_length as usize, + "every emitted match must reach min_match_length (got {})", + seq.match_length + ); + assert!( + seq.offset <= live_len, + "back-ref offset {} must stay within the live \ + history (= {} bytes); offsets larger than this \ + are the smoking gun for stale slice-relative \ + entries surviving the eviction", + seq.offset, + live_len + ); + } + + // At least one emitted sequence MUST hit a long-range + // back-reference (offset >= GAP_A, i.e. crossing from + // payload #2 back into payload #1's surviving tail). + // This is the long-range win LDM exists to capture and + // the round-2 fix has to preserve. + let crossed_gap = out2.iter().any(|s| s.offset >= GAP_A); + assert!( + crossed_gap, + "at least one emitted sequence must hit a back-ref of \ + >= GAP_A ({GAP_A}) — that's the long-range match \ + into the surviving tail of payload #1 the bucket \ + entries from call 1 are supposed to produce; \ + offsets observed: {:?}", + out2.iter() + .map(|s| s.offset) + .collect::>() + ); +} + +/// `generate_into` with an empty range is a no-op — emits +/// nothing and leaves the rolling hash untouched. Guards +/// against an off-by-one in the bounds check. +#[test] +fn generate_into_empty_range_is_noop() { + let mut producer = LdmProducer::new(test_params()); + let mut out = Vec::new(); + let data = [0u8; 128]; + let pre = producer.hash_state.rolling; + producer.generate_into(&data, 0, 64, 64, &mut out); + assert!(out.is_empty()); + assert_eq!(producer.hash_state.rolling, pre); +} diff --git a/zstd/src/encoding/levels/config.rs b/zstd/src/encoding/levels/config.rs new file mode 100644 index 000000000..50ef7e958 --- /dev/null +++ b/zstd/src/encoding/levels/config.rs @@ -0,0 +1,1149 @@ +//! Per-level compression tuning: the matcher config structs, the level +//! parameter table, and the level → params resolution chain. +//! +//! Moved verbatim from `match_generator.rs` (no behaviour change): the +//! `HcConfig` / `RowConfig` / `DfastConfig` / `FastConfig` knobs, `LevelParams` +//! and `LEVEL_TABLE`, the public-parameter overrides, the source-size tiering, +//! and the workspace estimators. `match_generator` imports this resolution API +//! instead of carrying it inline. Encoding-level paths are written absolute +//! (`crate::encoding::…`) so the module can live under `levels/` unchanged. + +use crate::encoding::CompressionLevel; +use crate::encoding::match_generator::{HC_SEARCH_DEPTH, HC_TARGET_LEN, ROW_MIN_MATCH_LEN}; +#[cfg(test)] +use crate::encoding::match_generator::{ROW_HASH_BITS, ROW_LOG, ROW_SEARCH_DEPTH, ROW_TARGET_LEN}; +#[cfg(test)] +use crate::encoding::match_table::storage::{HC_CHAIN_LOG, HC_HASH_LOG}; +/// Bundled tuning knobs for the hash-chain matcher. Using a typed config +/// instead of positional `usize` args eliminates parameter-order hazards. +#[derive(Copy, Clone, PartialEq, Eq)] +pub(crate) struct HcConfig { + pub(crate) hash_log: usize, + pub(crate) chain_log: usize, + pub(crate) search_depth: usize, + pub(crate) target_len: usize, + /// Binary-tree finder hash width (upstream zstd `mls = BOUNDED(4, minMatch, 6)`), + /// carried explicitly per level so it is NOT inferred from `target_len` + /// (a `target_length` override must not silently flip the finder between + /// 5- and 4-byte hashing). Only the BT body reads it; HC/lazy levels keep + /// it at 4 (their `hash_position` is always 4-byte). 5 for the + /// minMatch=5 BT levels (btlazy2 + btopt L16), 4 elsewhere. + pub(crate) search_mls: usize, +} + +#[derive(Copy, Clone, PartialEq, Eq)] +pub(crate) struct RowConfig { + pub(crate) hash_bits: usize, + pub(crate) row_log: usize, + pub(crate) search_depth: usize, + pub(crate) target_len: usize, + /// Upstream zstd `cParams.minMatch` for the row matcher: the regular-search + /// acceptance floor (a row candidate must extend to >= `mls` bytes). + /// The C-like advanced API surfaces this as the row min-match knob. + /// `ROW_MIN_MATCH_LEN` (5) is the default; the row hash key width stays + /// 4 bytes (an internal detail), so this only tunes the acceptance + /// floor, not the candidate hash distribution. + pub(crate) mls: usize, +} + +// Only used as the default HashChain config when the test-only parse×search +// override pairs a level with a backend its native row doesn't populate. +#[cfg(test)] +pub(crate) const HC_CONFIG: HcConfig = HcConfig { + hash_log: HC_HASH_LOG, + chain_log: HC_CHAIN_LOG, + search_depth: HC_SEARCH_DEPTH, + target_len: HC_TARGET_LEN, + search_mls: 4, +}; + +/// Base HashChain config synthesized when a public-parameter strategy +/// override ([`crate::encoding::parameters`]) routes a level to the HC / BT +/// backend whose native level row didn't populate `hc` (e.g. forcing +/// `Strategy::Lazy2` onto a level the table resolves to Fast). Mirrors +/// the mid-band lazy defaults; the per-knob overrides then refine it. +pub(crate) const HC_OVERRIDE_DEFAULT: HcConfig = HcConfig { + hash_log: crate::encoding::match_table::storage::HC_HASH_LOG, + chain_log: crate::encoding::match_table::storage::HC_CHAIN_LOG, + search_depth: HC_SEARCH_DEPTH, + target_len: HC_TARGET_LEN, + search_mls: 4, +}; + +pub(crate) const BTULTRA2_HC_CONFIG: HcConfig = HcConfig { + hash_log: 24, + chain_log: 24, + search_depth: 512, + target_len: 256, + search_mls: 4, +}; + +pub(crate) const BTULTRA2_HC_CONFIG_L22: HcConfig = HcConfig { + hash_log: 25, + chain_log: 27, + search_depth: 512, + target_len: 999, + search_mls: 4, +}; + +pub(crate) const BTULTRA2_HC_CONFIG_L22_256K: HcConfig = HcConfig { + hash_log: 19, + chain_log: 19, + search_depth: 1 << 13, + target_len: 999, + search_mls: 4, +}; + +pub(crate) const BTULTRA2_HC_CONFIG_L22_128K: HcConfig = HcConfig { + hash_log: 17, + chain_log: 18, + search_depth: 1 << 11, + target_len: 999, + search_mls: 4, +}; + +pub(crate) const BTULTRA2_HC_CONFIG_L22_16K: HcConfig = HcConfig { + hash_log: 15, + chain_log: 15, + search_depth: 1 << 10, + target_len: 999, + search_mls: 4, +}; + +// Default Row config: only used by tests and the test-only parse×search +// override (production greedy L5 carries its own `ROW_L5`). +#[cfg(test)] +pub(crate) const ROW_CONFIG: RowConfig = RowConfig { + hash_bits: ROW_HASH_BITS, + row_log: ROW_LOG, + search_depth: ROW_SEARCH_DEPTH, + target_len: ROW_TARGET_LEN, + mls: ROW_MIN_MATCH_LEN, +}; + +// Level-5 greedy is the ONLY strategy routed to the Row backend +// (`StrategyTag::backend`: greedy -> Row; lazy / btopt / btultra* -> +// HashChain), so it is the only level whose `row:` field is read. The upstream zstd +// `clevels.h` default row (srcSize > 256 KB) for level 5 is searchLog=3, +// targetLength=2, from which the row matcher derives: +// rowLog = clamp(searchLog, 4, 6) = 4 +// search_depth = 1 << min(searchLog, rowLog) = 8 (= nbAttempts) +// target_len = targetLength = 2 (nice-match early-out) +// The shared `ROW_CONFIG` (row_log=5, search_depth=16, target_len=48) ran a +// level-12-grade search here: 16 slots per row, never early-exiting until a +// 48-byte match. That exhaustive walk was the dominant cost in greedy L5's +// encode-speed regression vs FFI. `hash_bits` matches upstream zstd's +// `ZSTD_getCParams(5, .., 0).hashLog` = 19 (verified via +// `cparams_check 5`), so the row table is the same width as upstream's +// (2^19 slots); the previous `ROW_HASH_BITS` (20) doubled both row tables vs +// upstream, the dominant peak-memory excess on the greedy band. +pub(crate) const ROW_L5: RowConfig = RowConfig { + hash_bits: 19, + row_log: 4, + search_depth: 8, + target_len: 2, + mls: ROW_MIN_MATCH_LEN, +}; + +// Upstream zstd `clevels.h` unbounded defaults for the lazy band, verified via +// `ZSTD_getCParams(level, 0, 0)`: +// L6 { w21 c18 h19 s3 mml5 t4 lazy } → rowLog 4, depth 1<<3 = 8 +// L7 { w21 c19 h20 s4 mml5 t8 lazy } → rowLog 4, depth 16 +// L8 { w21 c19 h20 s4 mml5 t16 lazy2 } → rowLog 4, depth 16 +// L9 { w22 c20 h21 s4 mml5 t16 lazy2 } → rowLog 4, depth 16 +// L10 { w22 c21 h22 s5 mml5 t16 lazy2 } → rowLog 5, depth 32 +// L11 { w22 c21 h22 s6 mml5 t16 lazy2 } → rowLog 6, depth 64 +// L12 { w22 c22 h23 s6 mml5 t32 lazy2 } → rowLog 6, depth 64 +// `rowLog = clamp(searchLog, 4, 6)`, `depth = 1 << min(searchLog, rowLog)` +// (same derivation as `ROW_L5` above). `hash_bits` carries the upstream zstd +// `hashLog`; the hinted-source clamp in `configure` caps it by the window +// exactly like the upstream zstd `ZSTD_adjustCParams` path. +pub(crate) const ROW_L6: RowConfig = RowConfig { + hash_bits: 19, + row_log: 4, + search_depth: 8, + target_len: 4, + mls: ROW_MIN_MATCH_LEN, +}; +pub(crate) const ROW_L7: RowConfig = RowConfig { + hash_bits: 20, + row_log: 4, + search_depth: 16, + target_len: 8, + mls: ROW_MIN_MATCH_LEN, +}; +pub(crate) const ROW_L8: RowConfig = RowConfig { + hash_bits: 20, + row_log: 4, + search_depth: 16, + target_len: 16, + mls: ROW_MIN_MATCH_LEN, +}; +pub(crate) const ROW_L9: RowConfig = RowConfig { + hash_bits: 21, + row_log: 4, + search_depth: 16, + target_len: 16, + mls: ROW_MIN_MATCH_LEN, +}; +pub(crate) const ROW_L10: RowConfig = RowConfig { + hash_bits: 22, + row_log: 5, + search_depth: 32, + target_len: 16, + mls: ROW_MIN_MATCH_LEN, +}; +pub(crate) const ROW_L11: RowConfig = RowConfig { + hash_bits: 22, + row_log: 6, + search_depth: 64, + target_len: 16, + mls: ROW_MIN_MATCH_LEN, +}; +pub(crate) const ROW_L12: RowConfig = RowConfig { + hash_bits: 23, + row_log: 6, + search_depth: 64, + target_len: 32, + mls: ROW_MIN_MATCH_LEN, +}; + +/// Per-level Double-Fast hash sizing, mirroring the upstream zstd `clevels.h` columns +/// (config-driven, not a hardcoded constant): `long_hash_log` = +/// `cParams.hashLog` (the long 8-byte hash table), `short_hash_log` = +/// `cParams.chainLog` (the short hash table dfast repurposes as its +/// secondary index). Only the Dfast backend reads it, so non-dfast level +/// rows carry `dfast: None`. `minMatch` stays the upstream zstd-fixed `5` +/// (`DFAST_MIN_MATCH_LEN`, used in const contexts). +#[derive(Copy, Clone, PartialEq, Eq)] +pub(crate) struct DfastConfig { + pub(crate) long_hash_log: u8, + pub(crate) short_hash_log: u8, +} + +// Upstream zstd clevels.h default row (srcSize > 256 KB): L3 {hashLog 17, chainLog 16}, +// L4 {hashLog 18, chainLog 18}. +pub(crate) const DFAST_L3: DfastConfig = DfastConfig { + long_hash_log: 17, + short_hash_log: 16, +}; +pub(crate) const DFAST_L4: DfastConfig = DfastConfig { + long_hash_log: 18, + short_hash_log: 18, +}; + +/// Per-level Fast-strategy tuning, only consumed by the `FastKernelMatcher` +/// (Simple backend): `hash_log` = upstream zstd `cParams.hashLog`, `mls` = upstream zstd +/// `cParams.minMatch` (4..=8), `step_size` = upstream zstd `stepSize`. Carried as +/// `LevelParams.fast` (`Some` only on Fast level rows; `None` elsewhere). +#[derive(Copy, Clone, PartialEq, Eq)] +pub(crate) struct FastConfig { + pub(crate) hash_log: u32, + pub(crate) mls: u32, + pub(crate) step_size: usize, +} + +pub(crate) const FAST_L1: FastConfig = FastConfig { + hash_log: 14, + // Tier-0 (srcSize > 256 KiB) `cParams.minMatch`. Upstream zstd selects the + // Level-1 row from a 4-way srcSize-tiered table (`ZSTD_getCParams_internal` + // → `ZSTD_defaultCParameters[tableID][1]`), and minMatch shrinks for + // smaller inputs: 7 (>256 KiB) / 6 (16..256 KiB) / 5 (<=16 KiB). The base + // here is the tier-0 value; `fast_l1_mls_for_source_size` lowers it per the + // tier in `adjust_params_for_source_size`. + mls: 7, + step_size: 2, +}; +pub(crate) const FAST_L2: FastConfig = FastConfig { + hash_log: 16, + mls: 6, + step_size: 2, +}; + +/// Resolved tuning parameters for a compression level. The +/// [`StrategyTag`] is the single source of truth for the backend +/// family and the compile-time strategy consts; the runtime +/// [`BackendTag`] used by the driver dispatcher is derived via +/// [`StrategyTag::backend`] so the two cannot drift. +#[derive(Copy, Clone, PartialEq, Eq)] +pub(crate) struct LevelParams { + pub(crate) strategy_tag: crate::encoding::strategy::StrategyTag, + /// Decoupled search-method axis. Independent of `strategy_tag`'s + /// parse half: a level can pair any parse (greedy / lazy depth via + /// `lazy_depth`) with any search backend here. Defaults to the + /// historical pairing (`strategy_tag.search()`) but is overridable + /// per level so the parse×search matrix can be swept and tuned. + pub(crate) search: crate::encoding::strategy::SearchMethod, + pub(crate) window_log: u8, + pub(crate) lazy_depth: u8, + /// Per-strategy tuning. Exactly one is `Some` on each level row, matching + /// `strategy_tag`'s backend, so the table self-documents which knobs a + /// level actually consumes (the others are `None`, not dead placeholders): + /// `fast` for the Fast/Simple backend, `dfast` for Double-Fast, `hc` for + /// the HashChain (lazy / btopt / btultra*) backend, `row` for the Row + /// (greedy L5) backend. + pub(crate) fast: Option, + pub(crate) dfast: Option, + pub(crate) hc: Option, + pub(crate) row: Option, +} + +impl LevelParams { + /// Backend family (storage variant) for the driver dispatcher. + /// Derived from the decoupled `search` axis so a level can route to + /// a different search backend than its `strategy_tag` historically + /// implied. + pub(crate) fn backend(&self) -> crate::encoding::strategy::BackendTag { + self.search.backend() + } + + /// Parse mode derived from the decoupled `search` axis: the binary-tree + /// search path carries `ParseMode::Optimal`; every other search backend + /// derives greedy/lazy/lazy2 from `lazy_depth`. Reading `search` (not the + /// strategy tag) keeps the parse×search decoupling complete even when a + /// level whose tag is `Bt*` is overridden to a non-BT search backend. + pub(crate) fn parse(&self) -> crate::encoding::strategy::ParseMode { + match self.search { + crate::encoding::strategy::SearchMethod::BinaryTree => { + crate::encoding::strategy::ParseMode::Optimal + } + _ => crate::encoding::strategy::ParseMode::from_lazy_depth(self.lazy_depth), + } + } + + /// Cheap fingerprint pre-splitter level (the C-like `blockSplitterLevel`): + /// the EFFECTIVE upstream `ZSTD_splitBlock` level that + /// `ZSTD_optimalBlockSize` dispatches, i.e. `splitLevels[strategy] - 2` + /// (clamped at 0), NOT the raw `splitLevels[]` value. `split_level == 0` + /// routes to the cheap from-borders heuristic; `1..=4` to byChunks with + /// internal sampling level `split_level - 1`. See the body for the + /// per-strategy tier table and why the raw-table mapping was wrong. + pub(crate) fn pre_split(&self) -> Option { + use crate::encoding::strategy::StrategyTag; + // Effective upstream `ZSTD_splitBlock` level = `splitLevels[strat] - 2` + // (clamped at 0). Upstream `splitLevels[] = {0,0,1,2,2,3,3,4,4,4}` then + // subtracts 2 before dispatch, so the byChunks sampling tier is two + // steps coarser than the raw table: greedy/lazy(d1)=0 (from-borders), + // lazy2/btlazy2=1 (byChunks rate 43), btopt+=2 (byChunks rate 11). + // An earlier version mirrored the RAW table AND bumped lazy2 to the + // rate-1 full scan (split 4) to dodge a periodic-input phantom-split — + // that ran the pre-splitter at up to 43x upstream's sampling cost + // (~87% of L9 encode time on the decode corpus). Per the drop-in + // contract ratio only needs to stay <= upstream, so matching upstream's + // sampling tier (and accepting upstream's identical over-split on + // periodic input) is the dominant large-input encode-speed win. + Some(match self.strategy_tag { + // splitLevels 0/1 -> 0: upstream does not pre-split fast/dfast at + // all; from-borders is the cheapest stand-in and rarely splits. + StrategyTag::Fast | StrategyTag::Dfast => 0, + // greedy / lazy(depth 1): splitLevels 2 -> 0 (from-borders). + StrategyTag::Greedy => 0, + StrategyTag::Lazy => { + if self.lazy_depth >= 2 { + 1 // lazy2: splitLevels 3 -> 1 (byChunks rate 43) + } else { + 0 // lazy depth 1: splitLevels 2 -> 0 (from-borders) + } + } + StrategyTag::Btlazy2 => 1, // splitLevels 3 -> 1 (byChunks rate 43) + StrategyTag::BtOpt | StrategyTag::BtUltra | StrategyTag::BtUltra2 => 2, + }) + } +} + +/// Apply the public-parameter per-knob overrides (#27) onto the +/// level-resolved [`LevelParams`], in place. Runs in [`Matcher::reset`] +/// after the level params are computed and before backend selection, so +/// a strategy override re-routes the backend uniformly. An all-`None` +/// override is a no-op the caller skips via +/// [`crate::encoding::parameters::ParamOverrides::is_empty`], keeping the default +/// level geometry byte-identical. +pub(crate) fn apply_param_overrides( + params: &mut LevelParams, + ov: &crate::encoding::parameters::ParamOverrides, +) { + use crate::encoding::strategy::SearchMethod; + + // 1. Strategy override re-derives tag / search / lazy depth. + if let Some(strategy) = ov.strategy { + let tag = strategy.tag(); + params.strategy_tag = tag; + params.search = tag.search(); + params.lazy_depth = strategy.lazy_depth(); + } + + // 2. Ensure the active backend's config row exists (synthesize a + // default when a strategy override moved off the native row). + match params.search { + SearchMethod::Fast => { + params.fast.get_or_insert(FAST_L1); + } + SearchMethod::DoubleFast => { + params.dfast.get_or_insert(DFAST_L3); + } + SearchMethod::RowHash => { + params.row.get_or_insert(ROW_L5); + } + SearchMethod::HashChain | SearchMethod::BinaryTree => { + // A `Btlazy2` strategy override moved off a non-HC row needs the + // BT 5-byte finder hash (upstream zstd minMatch 5); other synthesized HC + // rows keep the 4-byte default. An explicit `min_match` override + // below refines this further. + params.hc.get_or_insert(HcConfig { + search_mls: if matches!( + params.strategy_tag, + crate::encoding::strategy::StrategyTag::Btlazy2 + ) { + 5 + } else { + HC_OVERRIDE_DEFAULT.search_mls + }, + ..HC_OVERRIDE_DEFAULT + }); + } + } + + // 3. window_log (bounds-checked at <= 30 by the builder). + if let Some(window_log) = ov.window_log { + params.window_log = window_log; + } + + // 4. Per-backend numeric knobs map into the active config, mirroring + // the upstream zstd `cParams` -> matcher translation documented on each + // config struct. + match params.search { + SearchMethod::Fast => { + if let Some(fast) = params.fast.as_mut() { + if let Some(hash_log) = ov.hash_log { + fast.hash_log = hash_log; + } + if let Some(min_match) = ov.min_match { + fast.mls = min_match; + } + } + } + SearchMethod::DoubleFast => { + if let Some(dfast) = params.dfast.as_mut() { + // hashLog -> long table, chainLog -> short table (the + // dfast secondary index). Both bounds-checked <= 30, so + // the `u8` casts are lossless. + if let Some(hash_log) = ov.hash_log { + dfast.long_hash_log = hash_log as u8; + } + if let Some(chain_log) = ov.chain_log { + dfast.short_hash_log = chain_log as u8; + } + } + } + SearchMethod::RowHash => { + if let Some(row) = params.row.as_mut() { + // Row hash-table width override (mirrors dfast `long_hash_log` + // / hc `hash_log`). Row has no separate chain table — the + // per-row depth comes from `search_log` below — so only + // `hash_log` maps here; `chain_log` has no Row analogue. + if let Some(hash_log) = ov.hash_log { + row.hash_bits = hash_log as usize; + } + if let Some(search_log) = ov.search_log { + // Upstream zstd: rowLog = clamp(searchLog, 4, 6); + // nbAttempts = 1 << min(searchLog, rowLog). + let row_log = (search_log as usize).clamp(4, 6); + row.row_log = row_log; + row.search_depth = 1usize << (search_log as usize).min(row_log); + } + if let Some(target_length) = ov.target_length { + row.target_len = target_length as usize; + } + if let Some(min_match) = ov.min_match { + row.mls = min_match as usize; + } + } + } + SearchMethod::HashChain | SearchMethod::BinaryTree => { + if let Some(hc) = params.hc.as_mut() { + if let Some(hash_log) = ov.hash_log { + hc.hash_log = hash_log as usize; + } + if let Some(chain_log) = ov.chain_log { + hc.chain_log = chain_log as usize; + } + if let Some(search_log) = ov.search_log { + hc.search_depth = 1usize << search_log; + } + if let Some(target_length) = ov.target_length { + hc.target_len = target_length as usize; + } + if let Some(min_match) = ov.min_match { + // Upstream zstd `mls = BOUNDED(4, cParams.minMatch, 6)`: a BT + // min_match override maps into the finder hash width. Only + // the BT body reads `search_mls`; HC/lazy keep 4-byte + // hashing regardless, so this is a no-op for them. + hc.search_mls = (min_match as usize).clamp(4, 6); + } + } + } + } +} + +/// Map the resolved runtime strategy to the upstream zstd LDM strategy ordinal +/// (1..=9) that [`crate::encoding::ldm::params::LdmParams::adjust_for`] expects. +/// The collapsed `Lazy` tag splits on `lazy_depth` (lazy = 4, lazy2 = 5). +#[cfg(feature = "hash")] +pub(crate) fn ldm_strategy_ordinal( + tag: crate::encoding::strategy::StrategyTag, + lazy_depth: u8, +) -> u32 { + use crate::encoding::strategy::StrategyTag; + match tag { + StrategyTag::Fast => 1, + StrategyTag::Dfast => 2, + StrategyTag::Greedy => 3, + StrategyTag::Lazy => { + if lazy_depth >= 2 { + 5 + } else { + 4 + } + } + // Upstream zstd `ZSTD_btlazy2` ordinal. + StrategyTag::Btlazy2 => 6, + StrategyTag::BtOpt => 7, + StrategyTag::BtUltra => 8, + StrategyTag::BtUltra2 => 9, + } +} + +/// `ceil(log2(size))` of a source-size hint, with a zero hint floored to +/// [`MIN_WINDOW_LOG`]. This is the single quantization every hint-dependent +/// matcher parameter is derived from: the window-log cap, the HC / Fast hash +/// and chain widths, the Dfast / Row table widths, the L22 config buckets, and +/// the Fast attach-vs-copy cutoff. Two hints sharing this value resolve to the +/// identical matcher shape, which is why it (not the raw byte count) keys the +/// primed-dictionary snapshot — see [`PrimedKey`]. Operates on the full `u64` +/// so callers comparing a hint against a cutoff get the same bucketed decision +/// here and at the driver, with no `as usize` truncation on 32-bit targets. +pub(crate) fn source_size_ceil_log(size: u64) -> u8 { + if size == 0 { + MIN_WINDOW_LOG + } else { + (64 - (size - 1).leading_zeros()) as u8 + } +} + +/// Attach-vs-copy cutoff for the Fast strategy, as a ceil-log bucket: a hint at +/// or below `2^this` (or unknown, `None`) ATTACHES the dictionary (a separate +/// immutable table scanned in place via the borrowed dual-base kernel); a larger +/// hint would COPY it into the live table. +/// +/// We set this to `31` so every dictionary source up to 2 GiB attaches, +/// diverging from upstream zstd's 8 KiB `ZSTD_shouldAttachDict` cutoff ON +/// PURPOSE: upstream copy mode copies the small CDict TABLES into the cctx and +/// still scans the input in place, but our flat-history copy path memmoves the +/// whole INPUT into history every frame (profiled at 30% `__memmove` + 14% +/// `__memset` on a reused 1 MiB dict encode). Attach mode scans the caller's +/// input in place with the dict as a separate prefix base, so it is strictly +/// faster for every frame size here (measured: 1 MiB dict frame 167 us -> 52 us, +/// 0.42x of C; 10 KiB 20.4 us -> 4.4 us, 0.17x of C). The dual-base kernel +/// carries `window_low`, so over-window inputs stay in-window and C-decodable. +/// +/// `31` is also the largest bucket the borrowed kernel can attach: it stores +/// virtual positions as `u32` (`cur_abs as u32`), so the maximum attached source +/// `1 << 31` (plus the dict prefix) stays below `u32::MAX`; the next bucket `32` +/// (4 GiB) would wrap that arithmetic. Sources past 2 GiB therefore fall back to +/// copy mode — rare in practice, and the relative copy cost shrinks as the +/// source grows. Per the drop-in-not-binary-parity contract, we make this match +/// decision ourselves. +/// Shared by `reset` (records the mode in the primed-snapshot key) and +/// `prime_with_dictionary` (acts on it). +pub(crate) const FAST_ATTACH_DICT_CUTOFF_LOG: u8 = 31; + +/// Largest dictionary region (bytes) the Fast attach path can index. The tagged +/// dict table packs each position into `32 - DICT_TAG_BITS` (= 24) bits, so a +/// region past `2^24` (16 MiB) would overflow the packed position. Dictionaries +/// this large fall back to COPY mode, whose live table stores full `u32` +/// positions and handles them. The size hint set on dict load equals the actual +/// dict content length, so the attach-vs-copy decision (and the matching +/// snapshot-key / epoch bits) can gate on it consistently at reset time. +pub(crate) const MAX_FAST_ATTACH_DICT_REGION: usize = 1 << 24; + +/// Dfast counterpart of [`FAST_ATTACH_DICT_CUTOFF_LOG`]: upstream zstd +/// `ZSTD_dictMatchState` attach cutoff for the double-fast strategy is 16 KiB +/// (`2^14`), so small / unknown-size inputs ATTACH (separate immutable dict +/// long+short tables + dual-probe in `start_matching_fast_loop`) and larger +/// known-size inputs COPY (re-prime the dict into the live tables, where the +/// dense scan matches it as window history). The attach build also self-gates +/// on `use_fast_loop` inside `skip_matching_for_dict_attach` — only the +/// fast-loop levels (L3 / Default / L0) carry the dual-probe. +pub(crate) const DFAST_ATTACH_DICT_CUTOFF_LOG: u8 = 14; + +/// `ZSTD_dictMatchState` attach cutoff for the Row (greedy/lazy) strategy is +/// 32 KiB (`2^15`, upstream zstd `attachDictSizeCutoffs`): small / unknown-size inputs +/// ATTACH the dict into the separate immutable row index (bounded dual-probe in +/// `row_candidate_rl`), larger known-size inputs dense-COPY into the live rows. +pub(crate) const ROW_ATTACH_DICT_CUTOFF_LOG: u8 = 15; + +/// 32 KiB (`2^15`, upstream zstd `attachDictSizeCutoffs[ZSTD_lazy2]`): small / +/// unknown-size inputs ATTACH the dict as a separate hash-chain dms (the dual +/// search in `find_best_match` walks the live input chain + the dms), larger +/// known-size inputs dense-COPY (merge the dict into the live chain and search +/// the one combined chain). +pub(crate) const HC_ATTACH_DICT_CUTOFF_LOG: u8 = 15; + +/// BT/optimal attach cutoff for `btlazy2` + `btopt`: 32 KiB (`2^15`, upstream +/// zstd `attachDictSizeCutoffs[ZSTD_btlazy2]` == `[ZSTD_btopt]`). Small / +/// unknown-size inputs ATTACH the dict as a separate DUBT dms; larger known-size +/// inputs COPY the dict into the LIVE binary tree (upstream zstd +/// `ZSTD_resetCCtx_byCopyingCDict`). +pub(crate) const BT_OPT_ATTACH_DICT_CUTOFF_LOG: u8 = 15; + +/// BT/optimal attach cutoff for `btultra` + `btultra2`: 8 KiB (`2^13`, upstream +/// zstd `attachDictSizeCutoffs[ZSTD_btultra]` == `[ZSTD_btultra2]`). The deepest +/// parses copy the dict into the live tree past a much smaller source than the +/// `btopt` tier, matching upstream's per-strategy cutoff table. +pub(crate) const BT_ULTRA_ATTACH_DICT_CUTOFF_LOG: u8 = 13; + +// Source-size cap for the dfast hash bits when a size hint is present: a tiny +// input needs no larger hash than its window. The upstream zstd `cParams.hashLog` / +// `chainLog` (from `DfastConfig`) caps it from above at the call site. +pub(crate) fn dfast_hash_bits_for_window(max_window_size: usize) -> usize { + let window_log = (usize::BITS - 1 - max_window_size.leading_zeros()) as usize; + window_log.max(MIN_WINDOW_LOG as usize) +} + +pub(crate) fn row_hash_bits_for_window(max_window_size: usize) -> usize { + // Upstream zstd `ZSTD_adjustCParams_internal` cap: `hashLog <= windowLog + 1`. + // The `+ 1` is load-bearing for L12, whose upstream zstd hashLog (23) exceeds + // its windowLog (22) — a plain `windowLog` cap would shrink the L12 + // table on EVERY hinted reset and split primed snapshots between + // hinted and unhinted frames that resolve to the identical geometry. + // No constant upper clamp: the old `ROW_HASH_BITS` (20) ceiling + // predates the lazy band moving onto Row (L9-12 carry upstream zstd hashLog + // 21-23). + let window_log = (usize::BITS - 1 - max_window_size.leading_zeros()) as usize; + (window_log + 1).max(MIN_WINDOW_LOG as usize) +} + +/// `floor(log2(window))` for the HashChain table-log cap (upstream zstd +/// `ZSTD_adjustCParams_internal`). The caller clamps the level's `hash_log` / +/// `chain_log` from above with this so a small hinted input doesn't allocate the +/// full level's tables. +pub(crate) fn hc_hash_bits_for_window(max_window_size: usize) -> usize { + let window_log = (usize::BITS - 1 - max_window_size.leading_zeros()) as usize; + window_log.max(MIN_WINDOW_LOG as usize) +} + +/// Parameter table for numeric compression levels 1–22. +/// +/// Each entry maps a zstd compression level to the best-available matcher +/// backend and tuning knobs. High levels map to dedicated parse modes: +/// btopt (16-17), btultra (18), btultra2 (19-22) — matching upstream zstd +/// `clevels.h` (level 19 is `ZSTD_btultra2`, not plain btultra). +/// +/// Index 0 = level 1, index 21 = level 22. +#[rustfmt::skip] +pub(crate) const LEVEL_TABLE: [LevelParams; 22] = [ + // Exactly one of fast/dfast/hc/row is Some per row, matching the strategy + // backend; the rest are None (not dead placeholders). + // Lvl Strategy wlog lazy per-strategy config + // --- -------------- ---- ---- ------------------- + /* 1 */ LevelParams { strategy_tag: crate::encoding::strategy::StrategyTag::Fast, search: crate::encoding::strategy::SearchMethod::Fast, window_log: 19, lazy_depth: 0, fast: Some(FAST_L1), dfast: None, hc: None, row: None }, + /* 2 */ LevelParams { strategy_tag: crate::encoding::strategy::StrategyTag::Fast, search: crate::encoding::strategy::SearchMethod::Fast, window_log: 20, lazy_depth: 0, fast: Some(FAST_L2), dfast: None, hc: None, row: None }, + /* 3 */ LevelParams { strategy_tag: crate::encoding::strategy::StrategyTag::Dfast, search: crate::encoding::strategy::SearchMethod::DoubleFast, window_log: 21, lazy_depth: 1, fast: None, dfast: Some(DFAST_L3), hc: None, row: None }, + /* 4 */ LevelParams { strategy_tag: crate::encoding::strategy::StrategyTag::Dfast, search: crate::encoding::strategy::SearchMethod::DoubleFast, window_log: 21, lazy_depth: 1, fast: None, dfast: Some(DFAST_L4), hc: None, row: None }, + // target_len column for L5..=L15 matches upstream zstd cParams.targetLength + // from clevels.h table[0] (default — srcSize > 256 KB). Upstream zstd uses + // it as the lazy outer loop's `sufficient_len` (nice-match) threshold. + // Inflating it above upstream zstd forces the chain walk to complete + // search_depth iterations instead of breaking on the first + // long-enough match — the dominant cost in the L5..=L15 speed + // regression vs FFI (see lazy_band_target_len_matches_default_table). + /* 5 */ LevelParams { strategy_tag: crate::encoding::strategy::StrategyTag::Greedy, search: crate::encoding::strategy::SearchMethod::RowHash, window_log: 21, lazy_depth: 0, fast: None, dfast: None, hc: None, row: Some(ROW_L5) }, + // L6-12: the upstream zstd runs the lazy/lazy2 strategies on the ROW-based + // match finder by default (`ZSTD_resolveRowMatchFinderMode`: row mode + // is on for greedy..lazy2 whenever SIMD is available) — a bounded + // SIMD tag scan per row instead of a pointer-chasing hash-chain walk. + // Our HashChain walk on these levels was ~75% of L10 wall time on the + // 1 MiB corpus (dependent chain-table loads). Same `RowConfig` + // derivation as `ROW_L5` above, upstream zstd values per level in the + // `ROW_L6..ROW_L12` comment block. + /* 6 */ LevelParams { strategy_tag: crate::encoding::strategy::StrategyTag::Lazy, search: crate::encoding::strategy::SearchMethod::RowHash, window_log: 21, lazy_depth: 1, fast: None, dfast: None, hc: None, row: Some(ROW_L6) }, + /* 7 */ LevelParams { strategy_tag: crate::encoding::strategy::StrategyTag::Lazy, search: crate::encoding::strategy::SearchMethod::RowHash, window_log: 21, lazy_depth: 1, fast: None, dfast: None, hc: None, row: Some(ROW_L7) }, + /* 8 */ LevelParams { strategy_tag: crate::encoding::strategy::StrategyTag::Lazy, search: crate::encoding::strategy::SearchMethod::RowHash, window_log: 21, lazy_depth: 2, fast: None, dfast: None, hc: None, row: Some(ROW_L8) }, + /* 9 */ LevelParams { strategy_tag: crate::encoding::strategy::StrategyTag::Lazy, search: crate::encoding::strategy::SearchMethod::RowHash, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: None, row: Some(ROW_L9) }, + /*10 */ LevelParams { strategy_tag: crate::encoding::strategy::StrategyTag::Lazy, search: crate::encoding::strategy::SearchMethod::RowHash, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: None, row: Some(ROW_L10) }, + /*11 */ LevelParams { strategy_tag: crate::encoding::strategy::StrategyTag::Lazy, search: crate::encoding::strategy::SearchMethod::RowHash, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: None, row: Some(ROW_L11) }, + /*12 */ LevelParams { strategy_tag: crate::encoding::strategy::StrategyTag::Lazy, search: crate::encoding::strategy::SearchMethod::RowHash, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: None, row: Some(ROW_L12) }, + // L13-15: reference uses btlazy2 (binary-tree finder) with searchLog 4/5/6 + // (search_depth 16/32/64) and targetLength 32. We run the hash-chain Lazy + // parser here, so we mirror the reference search budget rather than inflate + // it: matching the table keeps speed near the reference and makes per-level + // perf divergences comparable. The binary-tree finder that would let a + // smaller searchLog find longer matches (and re-establish a strict ratio + // ladder above L12) is tracked separately; until it lands these levels sit + // close to L12 on hash-chain inputs by design. + /*13 */ LevelParams { strategy_tag: crate::encoding::strategy::StrategyTag::Btlazy2, search: crate::encoding::strategy::SearchMethod::BinaryTree, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 22, chain_log: 22, search_depth: 16, target_len: 32, search_mls: 5 }), row: None }, + /*14 */ LevelParams { strategy_tag: crate::encoding::strategy::StrategyTag::Btlazy2, search: crate::encoding::strategy::SearchMethod::BinaryTree, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 23, chain_log: 22, search_depth: 32, target_len: 32, search_mls: 5 }), row: None }, + /*15 */ LevelParams { strategy_tag: crate::encoding::strategy::StrategyTag::Btlazy2, search: crate::encoding::strategy::SearchMethod::BinaryTree, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 23, chain_log: 23, search_depth: 64, target_len: 32, search_mls: 5 }), row: None }, + /*16 */ LevelParams { strategy_tag: crate::encoding::strategy::StrategyTag::BtOpt, search: crate::encoding::strategy::SearchMethod::BinaryTree, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 22, chain_log: 22, search_depth: 32, target_len: 48, search_mls: 5 }), row: None }, + /*17 */ LevelParams { strategy_tag: crate::encoding::strategy::StrategyTag::BtOpt, search: crate::encoding::strategy::SearchMethod::BinaryTree, window_log: 23, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 22, chain_log: 23, search_depth: 32, target_len: 64, search_mls: 4 }), row: None }, + /*18 */ LevelParams { strategy_tag: crate::encoding::strategy::StrategyTag::BtUltra, search: crate::encoding::strategy::SearchMethod::BinaryTree, window_log: 23, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 22, chain_log: 23, search_depth: 64, target_len: 64, search_mls: 4 }), row: None }, + /*19 */ LevelParams { strategy_tag: crate::encoding::strategy::StrategyTag::BtUltra2, search: crate::encoding::strategy::SearchMethod::BinaryTree, window_log: 23, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 22, chain_log: 24, search_depth: 128, target_len: 256, search_mls: 4 }), row: None }, + /*20 */ LevelParams { strategy_tag: crate::encoding::strategy::StrategyTag::BtUltra2, search: crate::encoding::strategy::SearchMethod::BinaryTree, window_log: 25, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 23, chain_log: 25, search_depth: 128, target_len: 256, search_mls: 4 }), row: None }, + /*21 */ LevelParams { strategy_tag: crate::encoding::strategy::StrategyTag::BtUltra2, search: crate::encoding::strategy::SearchMethod::BinaryTree, window_log: 26, lazy_depth: 2, fast: None, dfast: None, hc: Some(BTULTRA2_HC_CONFIG), row: None }, + /*22 */ LevelParams { strategy_tag: crate::encoding::strategy::StrategyTag::BtUltra2, search: crate::encoding::strategy::SearchMethod::BinaryTree, window_log: 27, lazy_depth: 2, fast: None, dfast: None, hc: Some(BTULTRA2_HC_CONFIG_L22), row: None }, +]; + +/// Upstream `ZSTD_createCDict` table geometry: the `(hash_log, chain_log)` a +/// dictionary's prepared match-finder tables get. Thin adapter over the single +/// cParams source [`crate::encoding::cparams::create_cdict_table_logs`], which mirrors +/// `ZSTD_adjustCParams_internal` under `ZSTD_cpm_createCDict`. `window_log` is +/// the resolved compress window; `hash_log` / `chain_log` are the level's own +/// widths; `uses_bt` selects the binary-tree `cycleLog` (`chainLog - 1`). +pub(crate) fn cdict_table_logs( + window_log: u8, + hash_log: usize, + chain_log: usize, + uses_bt: bool, + dict_size: usize, +) -> (usize, usize) { + let (h, c) = crate::encoding::cparams::create_cdict_table_logs( + window_log, + hash_log as u32, + chain_log as u32, + uses_bt, + dict_size, + ); + (h as usize, c as usize) +} + +/// Smallest window_log the encoder will use regardless of source size. +pub(crate) const MIN_WINDOW_LOG: u8 = 10; +/// Conservative floor for source-size-hinted window tuning. +/// +/// Hinted windows below 16 KiB (`window_log < 14`) currently regress C-FFI +/// interoperability on certain compressed-block patterns. Keep hinted +/// windows at 16 KiB or larger until that compatibility gap is closed. +pub(crate) const MIN_HINTED_WINDOW_LOG: u8 = 14; + +/// Adjust level parameters for a known source size. +/// +/// This derives a cap from `ceil(log2(src_size))`, then clamps it to +/// [`MIN_HINTED_WINDOW_LOG`] (16 KiB). A zero-byte size hint is treated as +/// [`MIN_WINDOW_LOG`] for the raw ceil-log step and then promoted to the hinted +/// floor. This keeps tables bounded for small inputs while preserving the +/// encoder's baseline minimum supported window. +/// For the HC backend, `hash_log` and `chain_log` are reduced +/// proportionally. +/// Source-size tier index, matching upstream `ZSTD_getCParams_internal`'s +/// `tableID = (rSize<=256K)+(rSize<=128K)+(rSize<=16K)`: 0 = > 256 KiB or +/// unknown, 1 = 128..256 KiB, 2 = 16..128 KiB, 3 = <= 16 KiB. +fn cparams_tier(source_size: Option) -> usize { + match source_size { + Some(size) if size <= 16 * 1024 => 3, + Some(size) if size <= 128 * 1024 => 2, + Some(size) if size <= 256 * 1024 => 1, + _ => 0, + } +} + +/// Override a Fast (L1/L2) or Dfast (L3) level row's table-shaping cParams +/// (hashLog / chainLog / minMatch) by source-size tier, matching the +/// reference `ZSTD_defaultCParameters[tableID][level]`. L1 keeps its base +/// hashLog (the source-size window clamp in `adjust_params_for_source_size` +/// already lands on the reference value) and only tiers minMatch; L2 also +/// tiers hashLog (the tier-0 value 16 oversized the table on medium inputs, +/// the page-fault pathology); L3 tiers both dfast hash widths. Strategy +/// switches (L2 tier 1, L4) are intentionally not applied here. +fn apply_cparams_tier(level: i32, source_size: Option, p: &mut LevelParams) { + let tier = cparams_tier(source_size); + // Single source for the table data: the verbatim upstream + // `ZSTD_defaultCParameters[tier][level]` row (`cparams::default_cparams`). + // The encoder consumes only the table-shaping widths here; the window / + // `table_log` clamp lives in `adjust_params_for_source_size`. + match level { + // Fast, all tiers — minMatch only (hashLog handled by the window clamp). + 1 => { + if let Some(f) = p.fast.as_mut() { + f.mls = crate::encoding::cparams::default_cparams(tier, 1).min_match; + } + } + // Fast (base strategy; tier 1 is dfast upstream — not switched here). + 2 => { + if let Some(f) = p.fast.as_mut() { + let cp = crate::encoding::cparams::default_cparams(tier, 2); + f.hash_log = cp.hash_log; + f.mls = cp.min_match; + } + } + // Dfast, all tiers — long hashLog (`hash_log`) + short chainLog (`chain_log`). + 3 => { + if let Some(d) = p.dfast.as_mut() { + let cp = crate::encoding::cparams::default_cparams(tier, 3); + d.long_hash_log = cp.hash_log as u8; + d.short_hash_log = cp.chain_log as u8; + } + } + _ => {} + } +} + +pub(crate) fn adjust_params_for_source_size(mut params: LevelParams, src_size: u64) -> LevelParams { + // Derive a source-size-based cap from ceil(log2(src_size)), then + // clamp first to MIN_WINDOW_LOG (baseline encoder minimum) and then to + // MIN_HINTED_WINDOW_LOG (16 KiB hinted floor). For tiny or zero hints we + // therefore keep a 16 KiB effective minimum window in hinted mode. + // Raw ceil(log2(src_size)) drives the internal table sizes. The + // advertised `window_log` is separately floored at MIN_HINTED_WINDOW_LOG + // (a decoder-interop requirement on the wire format), but the hash / + // chain table widths are internal and never appear in the frame, so they + // can track the actual source size below that floor. + let raw_src_log = source_size_ceil_log(src_size); + let src_log = raw_src_log.max(MIN_WINDOW_LOG).max(MIN_HINTED_WINDOW_LOG); + if src_log < params.window_log { + params.window_log = src_log; + } + // Internal match-finder tables are sized from `table_log` — the RAW + // source log (floored only at the baseline `MIN_WINDOW_LOG`), NOT the + // wire `window_log` floor. The table widths never appear in the frame, so + // for small inputs they can track the actual source size and avoid + // zeroing a window-sized table per frame; large inputs keep the level's + // widths. The cap is applied with the same per-backend headroom the + // level table uses, so the load factor (and match quality) is unchanged. + // The Dfast backend derives its table widths from the source in `reset` + // (`set_hash_bits` recomputes there), so it is not adjusted here. The Row + // backend's width IS capped here, mirroring the upstream zstd (see the Row branch). + let table_log = raw_src_log.max(MIN_WINDOW_LOG); + let backend = params.backend(); + if backend == crate::encoding::strategy::BackendTag::HashChain { + let hc = params + .hc + .as_mut() + .expect("HashChain level row carries an HcConfig"); + if (table_log + 2) < hc.hash_log as u8 { + hc.hash_log = (table_log + 2) as usize; + } + if (table_log + 1) < hc.chain_log as u8 { + hc.chain_log = (table_log + 1) as usize; + } + } else if backend == crate::encoding::strategy::BackendTag::Row { + let row = params + .row + .as_mut() + .expect("Row level row carries a RowConfig"); + // Upstream zstd `ZSTD_adjustCParams_internal` (zstd_compress.c): once + // the window is source-capped, `hashLog <= windowLog + 1`. The row + // table is `2^hash_bits` slots, exactly upstream's row hashTable + // `2^hashLog` slots, so the same cap applies. Without it the row table + // stays at the level's unbounded width (e.g. L12 hash_bits 23 = 4x + // upstream's source-capped 21), the dominant peak-memory excess on the + // row band. + let row_cap = (table_log + 1) as usize; + if row_cap < row.hash_bits { + row.hash_bits = row_cap; + } + } else if backend == crate::encoding::strategy::BackendTag::Simple { + let fast = params + .fast + .as_mut() + .expect("Fast level row carries a FastConfig"); + let fast_cap = (table_log + 1) as u32; + if fast_cap < fast.hash_log { + fast.hash_log = fast_cap; + } + } + params +} + +fn level22_btultra2_params_for_source_size(source_size: Option) -> LevelParams { + let mut hc = match source_size { + Some(size) if size <= 16 * 1024 => BTULTRA2_HC_CONFIG_L22_16K, + Some(size) if size <= 128 * 1024 => BTULTRA2_HC_CONFIG_L22_128K, + Some(size) if size <= 256 * 1024 => BTULTRA2_HC_CONFIG_L22_256K, + _ => BTULTRA2_HC_CONFIG_L22, + }; + let mut window_log = match source_size { + Some(size) if size <= 16 * 1024 => 14, + Some(size) if size <= 128 * 1024 => 17, + Some(size) if size <= 256 * 1024 => 18, + _ => 27, + }; + if let Some(size) = source_size + && size > 256 * 1024 + { + let src_log = source_size_ceil_log(size); + window_log = window_log.min(src_log.max(MIN_WINDOW_LOG)); + let adjusted_table_log = window_log as usize + 1; + hc.hash_log = hc.hash_log.min(adjusted_table_log); + hc.chain_log = hc.chain_log.min(adjusted_table_log); + } + LevelParams { + strategy_tag: crate::encoding::strategy::StrategyTag::BtUltra2, + search: crate::encoding::strategy::SearchMethod::BinaryTree, + window_log, + lazy_depth: 2, + fast: None, + dfast: None, + hc: Some(hc), + row: None, + } +} + +/// Estimated steady-state heap footprint of a one-shot compression context +/// at `level` (window history + match-finder tables + block staging), in +/// bytes. Computed from the same per-level tuning table the encoder +/// resolves at frame start, so the estimate tracks the real allocations; +/// it is an upper-bound style budget figure, not an exact accounting. +pub fn estimated_compression_workspace_bytes(level: CompressionLevel) -> usize { + use crate::encoding::strategy::StrategyTag; + let params = resolve_level_params(level, None); + let window = 1usize << params.window_log; + // Mirror `configure()`: the HC3 short-match side table exists only on + // the btultra/btultra2 tags (minMatch 3), capped by the window log; the + // BT pointer-pair layout fits inside the `4 << chain_log` chain term + // (pairs over `chain_log - 1` nodes). + let wants_hash3 = matches!( + params.strategy_tag, + StrategyTag::BtUltra | StrategyTag::BtUltra2 + ); + let uses_bt = matches!( + params.strategy_tag, + StrategyTag::Btlazy2 | StrategyTag::BtOpt | StrategyTag::BtUltra | StrategyTag::BtUltra2 + ); + let tables = params.fast.map(|f| 4usize << f.hash_log).unwrap_or(0) + + params + .dfast + .map(|d| (4usize << d.long_hash_log) + (4usize << d.short_hash_log)) + .unwrap_or(0) + + params + .hc + .map(|h| { + let hash3 = if wants_hash3 { + 4usize + << crate::encoding::match_table::storage::HC3_HASH_LOG + .min(params.window_log as usize) + } else { + 0 + }; + (4usize << h.hash_log) + (4usize << h.chain_log) + hash3 + }) + .unwrap_or(0) + + params + .row + .map(|r| (4usize << r.hash_bits) + (2usize << r.hash_bits)) + .unwrap_or(0); + // BT modes box a `BtMatcher`; its retained scratch layout is budgeted + // next to the struct so estimator and allocator evolve together. + let bt = if uses_bt { + crate::encoding::bt::BtMatcher::estimated_workspace_bytes() + } else { + 0 + }; + // Block staging: literal + sequence buffers plus the compressed-block + // scratch, each bounded by the 128 KiB block size. + let staging = 3 * (128 * 1024); + window + tables + bt + staging +} + +/// Extra steady-state workspace the binary-tree strategies (ordinals 6..=9, +/// btlazy2..btultra2) retain beyond the hash/chain tables: the boxed matcher +/// plus its scratch arenas, and the HC3 short-match side table for +/// btultra/btultra2 (capped by the window log). 0 for non-BT ordinals. +pub fn estimated_bt_strategy_extra_bytes(strategy_ordinal: u32, window_log: u32) -> usize { + if !(6..=9).contains(&strategy_ordinal) { + return 0; + } + let hash3 = if matches!(strategy_ordinal, 8 | 9) { + 4usize << crate::encoding::match_table::storage::HC3_HASH_LOG.min(window_log as usize) + } else { + 0 + }; + crate::encoding::bt::BtMatcher::estimated_workspace_bytes() + hash3 +} + +/// Resolve a [`CompressionLevel`] (+ optional source-size hint) to the +/// concrete [`LevelParams`] the matcher runs: strategy tag, search method +/// (match-finder), window log, and per-backend config. +/// +/// ## CRITICAL: input size changes the match-finder (and can change strategy) +/// +/// The resolved geometry is a function of the SOURCE SIZE, not the level +/// alone. This is the easy-to-miss part (so read this before assuming a level +/// maps to one fixed match-finder). It mirrors three upstream zstd stages: +/// +/// 1. [`LEVEL_TABLE`] holds the tier-0 (source > 256 KiB) base row per level +/// (upstream `ZSTD_defaultCParameters[0]`). L6-L12 carry +/// `SearchMethod::RowHash` (the Row match-finder), like upstream's +/// greedy/lazy default. +/// 2. [`apply_cparams_tier`] overrides the table-shaping widths for the +/// smaller source tiers (upstream `ZSTD_getCParams_internal` tier table). +/// NOTE: upstream ALSO switches STRATEGY in some tiers (L2 → dfast, L4 → +/// greedy on small sources); those backend switches are NOT yet replicated, +/// so those levels keep their base strategy on small inputs. +/// 3. [`adjust_params_for_source_size`] caps `window_log` to +/// ~`ceil_log2(source_size)` (upstream `ZSTD_adjustCParams_internal`). +/// +/// THEN, in the matcher `reset`, the greedy/lazy band falls back from +/// `RowHash` to `SearchMethod::HashChain` when the resolved `window_log <= 14` +/// — exactly upstream's `ZSTD_resolveRowMatchFinderMode` (the Row match-finder +/// is used for greedy/lazy/lazy2 ONLY when `windowLog > 14`). Net effect for +/// the SAME level: +/// +/// * small input (e.g. a 10 KiB fixture → `window_log` 14) → **HashChain** +/// (`ZSTD_HcFindBestMatch`, scalar chain walk); +/// * large input (e.g. 1 MiB → `window_log` 20) → **RowHash** (the SIMD-tag +/// row match-finder). +/// +/// A dictionary does NOT change the match-finder: it only downsizes the +/// prepared tables (`cdict_table_logs`, mirroring `ZSTD_createCDict`'s +/// small-source assumption), while `window_log` stays source-derived. So +/// `(L6, 10 KiB, +dict)` is HashChain and `(L6, 1 MiB, +dict)` is RowHash, +/// both matching upstream. When comparing against C on a fixture, resolve the +/// match-finder from the fixture's size first, or you may optimise/benchmark a +/// path C does not even take for that input. +pub(crate) fn resolve_level_params( + level: CompressionLevel, + source_size: Option, +) -> LevelParams { + if matches!(level, CompressionLevel::Level(22)) { + return level22_btultra2_params_for_source_size(source_size); + } + let params = match level { + CompressionLevel::Uncompressed => LevelParams { + strategy_tag: crate::encoding::strategy::StrategyTag::Fast, + search: crate::encoding::strategy::SearchMethod::Fast, + // Uncompressed frames emit raw blocks and never reference + // history; advertising a larger window only inflates + // decoder-side buffer reservation. Stay at 17 (128 KiB). + window_log: 17, + lazy_depth: 0, + // Beyond-upstream zstd: hash_log=14 (vs upstream zstd's 13) for 2× fewer + // collisions on structured corpora. Upstream zstd's "base for negative" + // row has targetLength=1 → step_size = 1 + 0 + 1 = 2. + fast: Some(FastConfig { + hash_log: 14, + mls: 6, + step_size: 2, + }), + dfast: None, + hc: None, + row: None, + }, + CompressionLevel::Fastest => { + // Only the Fast-specific cParams + // (fast_hash_log / fast_mls / fast_step_size) align + // with Uncompressed / negative-base row. window_log + // stays at LEVEL_TABLE[0]'s value (19) — Fastest still + // does real compression on a full window, unlike + // Uncompressed which clamps to 17. + let mut p = LEVEL_TABLE[0]; + p.fast = Some(FastConfig { + hash_log: 14, + mls: 6, + step_size: 2, + }); + p + } + CompressionLevel::Default => { + // Default == Level(DEFAULT_LEVEL); tier it the same way an explicit + // positive level is, so hinted default compression shrinks its + // table widths on small / medium frames instead of keeping the + // tier-0 row (the oversized-table page-fault pathology). + let mut p = LEVEL_TABLE[CompressionLevel::DEFAULT_LEVEL as usize - 1]; + apply_cparams_tier(CompressionLevel::DEFAULT_LEVEL, source_size, &mut p); + p + } + CompressionLevel::Better => LEVEL_TABLE[6], + // Level 13: the first dominant point of the deep-lazy band. The + // mls-wide row key lifted the shallow band's ratio enough that + // level 11 no longer strictly beats level 7 on the ladder corpus; + // the `Best` alias belongs on a config that dominates everything + // below it rather than on a hair-thin margin. + CompressionLevel::Best => LEVEL_TABLE[12], + CompressionLevel::Level(n) => { + if n > 0 { + let idx = (n as usize).min(CompressionLevel::MAX_LEVEL as usize) - 1; + let mut p = LEVEL_TABLE[idx]; + // Upstream zstd selects the cParams row from a 4-way + // source-size-tiered table (`ZSTD_getCParams_internal` → + // `ZSTD_defaultCParameters[tableID][level]`), and the Fast / + // Dfast hashLog, chainLog and minMatch shrink for smaller + // inputs. The `LEVEL_TABLE` base is the tier-0 (> 256 KiB) row; + // override the table-shaping params per tier here so small and + // medium frames use the reference's table widths (the oversized + // tier-0 widths were a per-frame alloc / page-fault pathology on + // medium inputs) and minMatch (short matches the wide hash + // skips). NOTE: the reference also switches STRATEGY in some + // tiers (L2 → dfast at 128..256 KiB, L4 → greedy at <= 16 KiB + // and 128..256 KiB); those backend switches are not yet tiered, + // so those tiers keep the base strategy. + apply_cparams_tier(n, source_size, &mut p); + p + } else if n == 0 { + // Level 0 = default, matching C zstd semantics. Tier it like the + // `Default` alias so `Level(0)` and `Default` stay identical. + let mut p = LEVEL_TABLE[CompressionLevel::DEFAULT_LEVEL as usize - 1]; + apply_cparams_tier(CompressionLevel::DEFAULT_LEVEL, source_size, &mut p); + p + } else { + // Negative levels — upstream zstd sets + // targetLength = -level (clampedCompressionLevel), + // yielding step_size = (-level) + 1 since + // !(targetLength) = 0 when targetLength > 0. + // So L-1..L-7 get step_size 2..8. Acceleration + // gradient comes from larger step skipping more + // positions per iter (faster, worse ratio). + // Clamp to upstream zstd's MIN_LEVEL before negating so + // i32::MIN can't overflow on `-n`. + let clamped = n.max(CompressionLevel::MIN_LEVEL); + let target_length = (-clamped) as usize; + let step_size = target_length + 1; + // Upstream zstd row-0 ("base for negative", clevels.h srcSize>256KB): + // hashLog=13, minMatch=7. The 32 KiB hash table (2^13 * 4B) + // is L1d-resident on contemporary cores, so every probe is an + // L1 hit; hashLog=14 (64 KiB) overflows a 32 KiB L1d and turns + // each probe into an L2 access. minMatch=7 (vs 6) skips + // short-distance 6-byte matches: fewer sequences, less + // extension/emit work, and parity with the upstream zstd's negative + // ladder on both ratio and throughput. + LevelParams { + strategy_tag: crate::encoding::strategy::StrategyTag::Fast, + search: crate::encoding::strategy::SearchMethod::Fast, + window_log: 19, + lazy_depth: 0, + fast: Some(FastConfig { + hash_log: 13, + mls: 7, + step_size, + }), + dfast: None, + hc: None, + row: None, + } + } + } + }; + if let Some(size) = source_size { + adjust_params_for_source_size(params, size) + } else { + params + } +} + +/// The cheap fingerprint pre-splitter level for a compression level (the +/// C-like `blockSplitterLevel`), resolved through the same per-level +/// `LevelParams` table as every other tuning knob. `None` keeps the whole +/// 128 KiB block. The frame loop reads this instead of hardcoding the +/// level→split mapping at the call site. +pub(crate) fn level_pre_split(level: CompressionLevel) -> Option { + // Resolve through `resolve_level_params` directly — NOT via the legacy + // `numeric_level()` alias — so named presets read the SAME table row as + // every other tuning knob (`Best` maps to its own row there, which is + // not the row its numeric alias points at). `Uncompressed` (raw + // blocks) never splits. + if matches!(level, CompressionLevel::Uncompressed) { + return None; + } + resolve_level_params(level, None) + .pre_split() + .map(usize::from) +} diff --git a/zstd/src/encoding/levels/fastest.rs b/zstd/src/encoding/levels/fastest.rs index 1ba358590..1cb5f3354 100644 --- a/zstd/src/encoding/levels/fastest.rs +++ b/zstd/src/encoding/levels/fastest.rs @@ -422,167 +422,4 @@ fn should_emit_raw_fast_path( } #[cfg(test)] -mod tests { - use super::*; - use crate::encoding::{ - Matcher, Sequence, - frame_compressor::{CompressState, FseTables}, - }; - use alloc::vec; - - #[derive(Default)] - struct HintProbeMatcher { - last_space: Vec, - skip_hints: Vec>, - } - - impl Matcher for HintProbeMatcher { - fn get_next_space(&mut self) -> Vec { - vec![0; 1024] - } - - fn get_last_space(&mut self) -> &[u8] { - &self.last_space - } - - fn commit_space(&mut self, space: Vec) { - self.last_space = space; - } - - fn skip_matching(&mut self) { - self.skip_hints.push(None); - } - - fn skip_matching_with_hint(&mut self, incompressible_hint: Option) { - self.skip_hints.push(incompressible_hint); - } - - fn start_matching(&mut self, _handle_sequence: impl for<'a> FnMut(Sequence<'a>)) { - panic!("start_matching must not run for early-exit paths"); - } - - fn reset(&mut self, _level: CompressionLevel) {} - - fn window_size(&self) -> u64 { - 128 * 1024 - } - } - - #[test] - fn custom_matcher_dict_probe_defaults_to_false() { - // `Matcher::block_samples_match_dict` defaults to `false`: a CUSTOM matcher - // with no dict-table probe leaves the raw-fast-path on its content-only - // verdict. NOTE this is the TRAIT DEFAULT, not the production wrapper: the - // `MatchGeneratorDriver` overrides it to `true` for non-Simple backends so - // a dict frame stays on the scan (covered by - // `block_samples_match_dict_is_true_for_non_simple_backend`). Only the - // Simple/Fast backend with an attached dictionary runs the precise probe. - let m = HintProbeMatcher::default(); - assert!(!m.block_samples_match_dict(b"arbitrary block content, no dict probe")); - } - - #[test] - fn rle_branch_passes_compressible_hint_to_skip_matching() { - let mut state = CompressState { - matcher: HintProbeMatcher::default(), - last_huff_table: None, - huff_table_spare: None, - fse_tables: FseTables::new(), - block_scratch: crate::encoding::blocks::CompressedBlockScratch::new(), - offset_hist: [1, 4, 8], - strategy_tag: crate::encoding::strategy::StrategyTag::Fast, - huf_optimal_search: true, - }; - let mut output = Vec::new(); - - let emitted = compress_block_encoded( - &mut state, - CompressionLevel::Fastest, - true, - vec![0xAB; 1024], - &mut output, - false, - #[cfg(feature = "lsm")] - None, - #[cfg(all(feature = "lsm", feature = "hash"))] - None, - ); - assert_eq!(emitted, BlockType::RLE); - - assert_eq!( - state.matcher.skip_hints, - vec![Some(false)], - "RLE is already known compressible; skip_matching should bypass incompressible sampling" - ); - } - - #[test] - fn raw_fast_path_emits_raw_block_and_passes_incompressible_hint() { - let mut state = CompressState { - matcher: HintProbeMatcher::default(), - last_huff_table: None, - huff_table_spare: None, - fse_tables: FseTables::new(), - block_scratch: crate::encoding::blocks::CompressedBlockScratch::new(), - offset_hist: [1, 4, 8], - strategy_tag: crate::encoding::strategy::StrategyTag::Fast, - huf_optimal_search: true, - }; - let mut output = Vec::new(); - - let mut block = vec![0u8; 4096]; - let mut x = 0x1234_5678u32; - for byte in &mut block { - x ^= x << 13; - x ^= x >> 17; - x ^= x << 5; - *byte = x as u8; - } - assert!( - block_looks_incompressible(&block), - "fixture must look incompressible to hit raw fast-path success branch" - ); - - let emitted = compress_block_encoded( - &mut state, - CompressionLevel::Fastest, - true, - block.clone(), - &mut output, - false, - #[cfg(feature = "lsm")] - None, - #[cfg(all(feature = "lsm", feature = "hash"))] - None, - ); - assert_eq!(emitted, BlockType::Raw); - - assert_eq!(state.matcher.skip_hints, vec![Some(true)]); - assert_eq!(state.matcher.get_last_space(), block.as_slice()); - assert_eq!( - (output[0] >> 1) & 0b11, - 0, - "raw fast-path should emit BlockType::Raw header" - ); - } - - #[test] - fn best_raw_fast_path_disabled_when_window_exceeds_better_reach() { - let mut block = vec![0u8; 4096]; - let mut x = 0x1234_5678u32; - for byte in &mut block { - x ^= x << 13; - x ^= x >> 17; - x ^= x << 5; - *byte = x as u8; - } - assert!( - block_looks_incompressible_strict(&block), - "fixture must look incompressible to exercise Best window guard" - ); - assert!( - !should_emit_raw_fast_path(CompressionLevel::Best, 16 * 1024 * 1024, &block, false), - "Best should keep compressed path when large window can unlock long-distance matches" - ); - } -} +mod tests; diff --git a/zstd/src/encoding/levels/fastest/tests.rs b/zstd/src/encoding/levels/fastest/tests.rs new file mode 100644 index 000000000..8396ac8f6 --- /dev/null +++ b/zstd/src/encoding/levels/fastest/tests.rs @@ -0,0 +1,162 @@ +use super::*; +use crate::encoding::{ + Matcher, Sequence, + frame_compressor::{CompressState, FseTables}, +}; +use alloc::vec; + +#[derive(Default)] +struct HintProbeMatcher { + last_space: Vec, + skip_hints: Vec>, +} + +impl Matcher for HintProbeMatcher { + fn get_next_space(&mut self) -> Vec { + vec![0; 1024] + } + + fn get_last_space(&mut self) -> &[u8] { + &self.last_space + } + + fn commit_space(&mut self, space: Vec) { + self.last_space = space; + } + + fn skip_matching(&mut self) { + self.skip_hints.push(None); + } + + fn skip_matching_with_hint(&mut self, incompressible_hint: Option) { + self.skip_hints.push(incompressible_hint); + } + + fn start_matching(&mut self, _handle_sequence: impl for<'a> FnMut(Sequence<'a>)) { + panic!("start_matching must not run for early-exit paths"); + } + + fn reset(&mut self, _level: CompressionLevel) {} + + fn window_size(&self) -> u64 { + 128 * 1024 + } +} + +#[test] +fn custom_matcher_dict_probe_defaults_to_false() { + // `Matcher::block_samples_match_dict` defaults to `false`: a CUSTOM matcher + // with no dict-table probe leaves the raw-fast-path on its content-only + // verdict. NOTE this is the TRAIT DEFAULT, not the production wrapper: the + // `MatchGeneratorDriver` overrides it to `true` for non-Simple backends so + // a dict frame stays on the scan (covered by + // `block_samples_match_dict_is_true_for_non_simple_backend`). Only the + // Simple/Fast backend with an attached dictionary runs the precise probe. + let m = HintProbeMatcher::default(); + assert!(!m.block_samples_match_dict(b"arbitrary block content, no dict probe")); +} + +#[test] +fn rle_branch_passes_compressible_hint_to_skip_matching() { + let mut state = CompressState { + matcher: HintProbeMatcher::default(), + last_huff_table: None, + huff_table_spare: None, + fse_tables: FseTables::new(), + block_scratch: crate::encoding::blocks::CompressedBlockScratch::new(), + offset_hist: [1, 4, 8], + strategy_tag: crate::encoding::strategy::StrategyTag::Fast, + huf_optimal_search: true, + }; + let mut output = Vec::new(); + + let emitted = compress_block_encoded( + &mut state, + CompressionLevel::Fastest, + true, + vec![0xAB; 1024], + &mut output, + false, + #[cfg(feature = "lsm")] + None, + #[cfg(all(feature = "lsm", feature = "hash"))] + None, + ); + assert_eq!(emitted, BlockType::RLE); + + assert_eq!( + state.matcher.skip_hints, + vec![Some(false)], + "RLE is already known compressible; skip_matching should bypass incompressible sampling" + ); +} + +#[test] +fn raw_fast_path_emits_raw_block_and_passes_incompressible_hint() { + let mut state = CompressState { + matcher: HintProbeMatcher::default(), + last_huff_table: None, + huff_table_spare: None, + fse_tables: FseTables::new(), + block_scratch: crate::encoding::blocks::CompressedBlockScratch::new(), + offset_hist: [1, 4, 8], + strategy_tag: crate::encoding::strategy::StrategyTag::Fast, + huf_optimal_search: true, + }; + let mut output = Vec::new(); + + let mut block = vec![0u8; 4096]; + let mut x = 0x1234_5678u32; + for byte in &mut block { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + *byte = x as u8; + } + assert!( + block_looks_incompressible(&block), + "fixture must look incompressible to hit raw fast-path success branch" + ); + + let emitted = compress_block_encoded( + &mut state, + CompressionLevel::Fastest, + true, + block.clone(), + &mut output, + false, + #[cfg(feature = "lsm")] + None, + #[cfg(all(feature = "lsm", feature = "hash"))] + None, + ); + assert_eq!(emitted, BlockType::Raw); + + assert_eq!(state.matcher.skip_hints, vec![Some(true)]); + assert_eq!(state.matcher.get_last_space(), block.as_slice()); + assert_eq!( + (output[0] >> 1) & 0b11, + 0, + "raw fast-path should emit BlockType::Raw header" + ); +} + +#[test] +fn best_raw_fast_path_disabled_when_window_exceeds_better_reach() { + let mut block = vec![0u8; 4096]; + let mut x = 0x1234_5678u32; + for byte in &mut block { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + *byte = x as u8; + } + assert!( + block_looks_incompressible_strict(&block), + "fixture must look incompressible to exercise Best window guard" + ); + assert!( + !should_emit_raw_fast_path(CompressionLevel::Best, 16 * 1024 * 1024, &block, false), + "Best should keep compressed path when large window can unlock long-distance matches" + ); +} diff --git a/zstd/src/encoding/levels/mod.rs b/zstd/src/encoding/levels/mod.rs index e26aa6783..1aa828a72 100644 --- a/zstd/src/encoding/levels/mod.rs +++ b/zstd/src/encoding/levels/mod.rs @@ -1,2 +1,3 @@ +pub(crate) mod config; mod fastest; pub(crate) use fastest::{compress_block_encoded, compress_block_encoded_borrowed}; diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs deleted file mode 100644 index 62a14a3a4..000000000 --- a/zstd/src/encoding/match_generator.rs +++ /dev/null @@ -1,13125 +0,0 @@ -//! Matching algorithm used find repeated parts in the original data -//! -//! The Zstd format relies on finden repeated sequences of data and compressing these sequences as instructions to the decoder. -//! A sequence basically tells the decoder "Go back X bytes and copy Y bytes to the end of your decode buffer". -//! -//! The task here is to efficiently find matches in the already encoded data for the current suffix of the not yet encoded data. - -use alloc::vec::Vec; -// SIMD/CRC intrinsics now live in `crate::encoding::fastpath::*` where they -// sit under per-CPU `#[target_feature]` umbrellas; no architecture-specific -// intrinsic imports remain in this file. -use super::CompressionLevel; -use super::Matcher; -use super::Sequence; -use super::blocks::encode_offset_with_history; -use super::bt::BtMatcher; -#[cfg(test)] -use super::cost_model::HC_MAX_LIT; -use super::cost_model::{ - HC_BITCOST_MULTIPLIER, HC_FORMAT_MINMATCH, HC_OPT_NODE_LEN, HC_OPT_NUM, HC_OPT_PRICE_ARENA_LEN, - HC_OPT_PRICE_STRIDE, HC_PREDEF_THRESHOLD, HcOptState, HcOptimalCostProfile, -}; -#[cfg(test)] -use super::cost_model::{HC_BLOCKSIZE_MAX, HC_MAX_LL, HC_MAX_ML, HC_MAX_OFF, HcOptPriceType}; -use super::dfast::DfastMatchGenerator; -// FAST_HASH_FILL_STEP test-only re-export was tied to the legacy -// SuffixStore MatchGenerator's interleaved hash-fill stride. The -// upstream zstd-shape Fast kernel walks ip0 with kSearchStrength step-skip -// acceleration instead, so the constant has no consumer in the -// remaining live test set today. -#[cfg(test)] -use super::match_table::helpers::INCOMPRESSIBLE_SKIP_STEP; -use super::match_table::helpers::MIN_MATCH_LEN; -#[cfg(test)] -use super::match_table::helpers::common_prefix_len; -#[cfg(test)] -use super::opt::ldm::HcRawSeq; -use super::opt::ldm::{HcOptLdmState, HcRawSeqStore}; -use super::opt::types::{ - HcCandidateQuery, HcOptimalNode, HcOptimalPlanBuffers, HcOptimalPlanState, HcOptimalSequence, - MatchCandidate, -}; -use super::row::RowMatchGenerator; -use super::simple::fast_matcher::{FAST_LEVEL_1_HASH_LOG, FAST_LEVEL_1_MLS, FastKernelMatcher}; -#[cfg(all( - test, - feature = "std", - target_arch = "aarch64", - target_endian = "little" -))] -use std::arch::is_aarch64_feature_detected; -#[cfg(all(test, feature = "std", target_arch = "x86_64"))] -use std::arch::is_x86_feature_detected; - -pub(crate) const DFAST_MIN_MATCH_LEN: usize = 5; -// Bytes the dfast short hash reads (upstream zstd `mls = 5`). Seeding / lookahead -// guards use it so a position is only short-hashed once its full 5-byte key -// is in range. -pub(crate) const DFAST_SHORT_HASH_LOOKAHEAD: usize = 5; -pub(crate) const ROW_MIN_MATCH_LEN: usize = 5; -// Upstream zstd `clevels.h:31` at level 3 large-input bucket sets -// `hashLog = 17` (the long-hash table) and `chainLog = 16` (the -// short-hash table — upstream zstd names this `chainTable` even though for -// dfast it's used as a plain single-slot hash). Each table holds one -// `U32` per slot; the upstream zstd overwrites on collision and recovers -// compression quality via the inline `_search_next_long` retry -// (after a short-hash hit, probes `hashLong[hl1]` at `ip + 1` and -// keeps the longer match). -// -// We mirror that storage layout: single `u32` per bucket (no -// `[u32; N]` array), `long_hash` sized `1 << DFAST_HASH_BITS` and -// `short_hash` one bit smaller via `DFAST_SHORT_HASH_BITS_DELTA`. -// Two-table footprint at Level 3: `2^17 × 4 + 2^16 × 4 = 768 KiB`, -// exact upstream parity. The `_search_next_long` retry lives in -// `DfastMatchGenerator::hash_candidate` (called via -// `best_match`). Earlier revisions kept a -// 4-slot bucket per hash position; that paid 4× the upstream zstd memory -// without measurable ratio gain once the retry was in place. -// -// `dfast_hash_bits_for_window` still clamps the runtime long-hash -// value to `[MIN_WINDOW_LOG, DFAST_HASH_BITS]`, so this const is the -// upper bound rather than a fixed default. -pub(crate) const DFAST_HASH_BITS: usize = 17; -/// Difference between `long_hash_bits` and `short_hash_bits` — -/// upstream zstd `hashLog - chainLog` is 1 at every dfast level (`clevels.h` -/// level 2: 16-15=1; level 3: 17-16=1). The short hash is one bit -/// smaller than the long hash so the per-bucket footprint matches -/// upstream zstd sizing exactly. -pub(crate) const DFAST_SHORT_HASH_BITS_DELTA: usize = 1; -/// Sentinel value for an empty slot in the dfast hash tables. Real -/// positions are stored as `(abs_pos - position_base + 1) as u32`, so -/// `0` is reserved as the "empty" marker and a true relative offset -/// of `0` never appears in the table. Mirrors the LDM table's -/// `LdmEntry.offset == 0` convention (see `encoding/ldm/table.rs`) -/// so both rebasing structures share -/// one sentinel scheme. -pub(crate) const DFAST_EMPTY_SLOT: u32 = 0; - -/// Guard band reserved above the high-water mark before triggering a -/// rebase on the Dfast hash tables. When the next insert would push a -/// relative offset above `u32::MAX - DFAST_REBASE_GUARD_BAND`, the -/// table calls `reduce(GUARD_BAND)` to shift every slot down and -/// advance `position_base` so future inserts stay inside the `u32` -/// window. Same scheme as `encoding/ldm/table.rs`. -pub(crate) const DFAST_REBASE_GUARD_BAND: u32 = 1u32 << 30; -pub(crate) const DFAST_SKIP_SEARCH_STRENGTH: usize = 6; -pub(crate) const DFAST_SKIP_STEP_GROWTH_INTERVAL: usize = 1 << DFAST_SKIP_SEARCH_STRENGTH; -pub(crate) const DFAST_MAX_SKIP_STEP: usize = 8; -pub(crate) const DFAST_INCOMPRESSIBLE_SKIP_STEP: usize = 16; -pub(crate) const ROW_HASH_BITS: usize = 20; -pub(crate) const ROW_LOG: usize = 5; -pub(crate) const ROW_SEARCH_DEPTH: usize = 16; -pub(crate) const ROW_TARGET_LEN: usize = 48; -pub(crate) const ROW_TAG_BITS: usize = 8; -pub(crate) const ROW_EMPTY_SLOT: u32 = u32::MAX; -pub(crate) const ROW_HASH_KEY_LEN: usize = 4; -// HASH_MIX_PRIME now lives in `crate::encoding::fastpath::scalar`; the four -// per-CPU `hash_mix_u64` variants share it via that module. -// HC_PRIME3BYTES / HC_PRIME4BYTES moved to match_table::storage -// alongside the hash helpers in Phase 1e Stage A. Only the test -// module references the constants directly (production code goes -// through `MatchTable::hash_value_with_mls`). -#[cfg(test)] -use super::match_table::storage::{HC_PRIME3BYTES, HC_PRIME4BYTES}; - -// HC_HASH_LOG / HC_CHAIN_LOG / HC3_HASH_LOG / HC_EMPTY live on the -// shared storage module so MatchTable methods can reference them -// without pulling in this module. Re-imported here so existing -// macros / configs / tests keep their unqualified names. -#[cfg(test)] -use super::match_table::storage::HC_EMPTY; -use super::match_table::storage::HC3_HASH_LOG; -// HC_HASH_LOG / HC_CHAIN_LOG feed the test-only `HC_CONFIG` default. -#[cfg(test)] -use super::match_table::storage::{HC_CHAIN_LOG, HC_HASH_LOG}; -// HC3_MAX_OFFSET moved to encoding::bt alongside the hash3 candidate -// probe macro that consumes it; the macro references it via the -// fully-qualified `$crate::encoding::bt::HC3_MAX_OFFSET` path so this -// module no longer needs a local import. -const HC_SEARCH_DEPTH: usize = 16; -// HC_MIN_MATCH_LEN moved to encoding::hc; re-imported here so -// existing references compile unchanged. -use super::hc::HC_MIN_MATCH_LEN; -const HC_OPT_MIN_MATCH_LEN: usize = HC_FORMAT_MINMATCH; -const HC_TARGET_LEN: usize = 48; - -// MAX_HC_SEARCH_DEPTH moved to encoding::hc alongside chain_candidates. -use super::hc::MAX_HC_SEARCH_DEPTH; - -// `Strategy` and `StrategyTag` live in `crate::encoding::strategy`. -// The driver carries a `StrategyTag` field set at `reset()` and -// dispatches each block into a monomorphised `compress_block::` -// per concrete strategy. - -/// Bundled tuning knobs for the hash-chain matcher. Using a typed config -/// instead of positional `usize` args eliminates parameter-order hazards. -#[derive(Copy, Clone, PartialEq, Eq)] -struct HcConfig { - hash_log: usize, - chain_log: usize, - search_depth: usize, - target_len: usize, - /// Binary-tree finder hash width (upstream zstd `mls = BOUNDED(4, minMatch, 6)`), - /// carried explicitly per level so it is NOT inferred from `target_len` - /// (a `target_length` override must not silently flip the finder between - /// 5- and 4-byte hashing). Only the BT body reads it; HC/lazy levels keep - /// it at 4 (their `hash_position` is always 4-byte). 5 for the - /// minMatch=5 BT levels (btlazy2 + btopt L16), 4 elsewhere. - search_mls: usize, -} - -#[derive(Copy, Clone, PartialEq, Eq)] -pub(crate) struct RowConfig { - pub(crate) hash_bits: usize, - pub(crate) row_log: usize, - pub(crate) search_depth: usize, - pub(crate) target_len: usize, - /// Upstream zstd `cParams.minMatch` for the row matcher: the regular-search - /// acceptance floor (a row candidate must extend to >= `mls` bytes). - /// The C-like advanced API surfaces this as the row min-match knob. - /// `ROW_MIN_MATCH_LEN` (5) is the default; the row hash key width stays - /// 4 bytes (an internal detail), so this only tunes the acceptance - /// floor, not the candidate hash distribution. - pub(crate) mls: usize, -} - -// Only used as the default HashChain config when the test-only parse×search -// override pairs a level with a backend its native row doesn't populate. -#[cfg(test)] -const HC_CONFIG: HcConfig = HcConfig { - hash_log: HC_HASH_LOG, - chain_log: HC_CHAIN_LOG, - search_depth: HC_SEARCH_DEPTH, - target_len: HC_TARGET_LEN, - search_mls: 4, -}; - -/// Base HashChain config synthesized when a public-parameter strategy -/// override ([`super::parameters`]) routes a level to the HC / BT -/// backend whose native level row didn't populate `hc` (e.g. forcing -/// `Strategy::Lazy2` onto a level the table resolves to Fast). Mirrors -/// the mid-band lazy defaults; the per-knob overrides then refine it. -const HC_OVERRIDE_DEFAULT: HcConfig = HcConfig { - hash_log: super::match_table::storage::HC_HASH_LOG, - chain_log: super::match_table::storage::HC_CHAIN_LOG, - search_depth: HC_SEARCH_DEPTH, - target_len: HC_TARGET_LEN, - search_mls: 4, -}; - -const BTULTRA2_HC_CONFIG: HcConfig = HcConfig { - hash_log: 24, - chain_log: 24, - search_depth: 512, - target_len: 256, - search_mls: 4, -}; - -const BTULTRA2_HC_CONFIG_L22: HcConfig = HcConfig { - hash_log: 25, - chain_log: 27, - search_depth: 512, - target_len: 999, - search_mls: 4, -}; - -const BTULTRA2_HC_CONFIG_L22_256K: HcConfig = HcConfig { - hash_log: 19, - chain_log: 19, - search_depth: 1 << 13, - target_len: 999, - search_mls: 4, -}; - -const BTULTRA2_HC_CONFIG_L22_128K: HcConfig = HcConfig { - hash_log: 17, - chain_log: 18, - search_depth: 1 << 11, - target_len: 999, - search_mls: 4, -}; - -const BTULTRA2_HC_CONFIG_L22_16K: HcConfig = HcConfig { - hash_log: 15, - chain_log: 15, - search_depth: 1 << 10, - target_len: 999, - search_mls: 4, -}; - -// Default Row config: only used by tests and the test-only parse×search -// override (production greedy L5 carries its own `ROW_L5`). -#[cfg(test)] -const ROW_CONFIG: RowConfig = RowConfig { - hash_bits: ROW_HASH_BITS, - row_log: ROW_LOG, - search_depth: ROW_SEARCH_DEPTH, - target_len: ROW_TARGET_LEN, - mls: ROW_MIN_MATCH_LEN, -}; - -// Level-5 greedy is the ONLY strategy routed to the Row backend -// (`StrategyTag::backend`: greedy -> Row; lazy / btopt / btultra* -> -// HashChain), so it is the only level whose `row:` field is read. The upstream zstd -// `clevels.h` default row (srcSize > 256 KB) for level 5 is searchLog=3, -// targetLength=2, from which the row matcher derives: -// rowLog = clamp(searchLog, 4, 6) = 4 -// search_depth = 1 << min(searchLog, rowLog) = 8 (= nbAttempts) -// target_len = targetLength = 2 (nice-match early-out) -// The shared `ROW_CONFIG` (row_log=5, search_depth=16, target_len=48) ran a -// level-12-grade search here: 16 slots per row, never early-exiting until a -// 48-byte match. That exhaustive walk was the dominant cost in greedy L5's -// encode-speed regression vs FFI. `hash_bits` matches upstream zstd's -// `ZSTD_getCParams(5, .., 0).hashLog` = 19 (verified via -// `cparams_check 5`), so the row table is the same width as upstream's -// (2^19 slots); the previous `ROW_HASH_BITS` (20) doubled both row tables vs -// upstream, the dominant peak-memory excess on the greedy band. -const ROW_L5: RowConfig = RowConfig { - hash_bits: 19, - row_log: 4, - search_depth: 8, - target_len: 2, - mls: ROW_MIN_MATCH_LEN, -}; - -// Upstream zstd `clevels.h` unbounded defaults for the lazy band, verified via -// `ZSTD_getCParams(level, 0, 0)`: -// L6 { w21 c18 h19 s3 mml5 t4 lazy } → rowLog 4, depth 1<<3 = 8 -// L7 { w21 c19 h20 s4 mml5 t8 lazy } → rowLog 4, depth 16 -// L8 { w21 c19 h20 s4 mml5 t16 lazy2 } → rowLog 4, depth 16 -// L9 { w22 c20 h21 s4 mml5 t16 lazy2 } → rowLog 4, depth 16 -// L10 { w22 c21 h22 s5 mml5 t16 lazy2 } → rowLog 5, depth 32 -// L11 { w22 c21 h22 s6 mml5 t16 lazy2 } → rowLog 6, depth 64 -// L12 { w22 c22 h23 s6 mml5 t32 lazy2 } → rowLog 6, depth 64 -// `rowLog = clamp(searchLog, 4, 6)`, `depth = 1 << min(searchLog, rowLog)` -// (same derivation as `ROW_L5` above). `hash_bits` carries the upstream zstd -// `hashLog`; the hinted-source clamp in `configure` caps it by the window -// exactly like the upstream zstd `ZSTD_adjustCParams` path. -const ROW_L6: RowConfig = RowConfig { - hash_bits: 19, - row_log: 4, - search_depth: 8, - target_len: 4, - mls: ROW_MIN_MATCH_LEN, -}; -const ROW_L7: RowConfig = RowConfig { - hash_bits: 20, - row_log: 4, - search_depth: 16, - target_len: 8, - mls: ROW_MIN_MATCH_LEN, -}; -const ROW_L8: RowConfig = RowConfig { - hash_bits: 20, - row_log: 4, - search_depth: 16, - target_len: 16, - mls: ROW_MIN_MATCH_LEN, -}; -const ROW_L9: RowConfig = RowConfig { - hash_bits: 21, - row_log: 4, - search_depth: 16, - target_len: 16, - mls: ROW_MIN_MATCH_LEN, -}; -const ROW_L10: RowConfig = RowConfig { - hash_bits: 22, - row_log: 5, - search_depth: 32, - target_len: 16, - mls: ROW_MIN_MATCH_LEN, -}; -const ROW_L11: RowConfig = RowConfig { - hash_bits: 22, - row_log: 6, - search_depth: 64, - target_len: 16, - mls: ROW_MIN_MATCH_LEN, -}; -const ROW_L12: RowConfig = RowConfig { - hash_bits: 23, - row_log: 6, - search_depth: 64, - target_len: 32, - mls: ROW_MIN_MATCH_LEN, -}; - -/// Per-level Double-Fast hash sizing, mirroring the upstream zstd `clevels.h` columns -/// (config-driven, not a hardcoded constant): `long_hash_log` = -/// `cParams.hashLog` (the long 8-byte hash table), `short_hash_log` = -/// `cParams.chainLog` (the short hash table dfast repurposes as its -/// secondary index). Only the Dfast backend reads it, so non-dfast level -/// rows carry `dfast: None`. `minMatch` stays the upstream zstd-fixed `5` -/// (`DFAST_MIN_MATCH_LEN`, used in const contexts). -#[derive(Copy, Clone, PartialEq, Eq)] -struct DfastConfig { - long_hash_log: u8, - short_hash_log: u8, -} - -// Upstream zstd clevels.h default row (srcSize > 256 KB): L3 {hashLog 17, chainLog 16}, -// L4 {hashLog 18, chainLog 18}. -const DFAST_L3: DfastConfig = DfastConfig { - long_hash_log: 17, - short_hash_log: 16, -}; -const DFAST_L4: DfastConfig = DfastConfig { - long_hash_log: 18, - short_hash_log: 18, -}; - -/// Per-level Fast-strategy tuning, only consumed by the `FastKernelMatcher` -/// (Simple backend): `hash_log` = upstream zstd `cParams.hashLog`, `mls` = upstream zstd -/// `cParams.minMatch` (4..=8), `step_size` = upstream zstd `stepSize`. Carried as -/// `LevelParams.fast` (`Some` only on Fast level rows; `None` elsewhere). -#[derive(Copy, Clone, PartialEq, Eq)] -struct FastConfig { - hash_log: u32, - mls: u32, - step_size: usize, -} - -const FAST_L1: FastConfig = FastConfig { - hash_log: 14, - // Tier-0 (srcSize > 256 KiB) `cParams.minMatch`. Upstream zstd selects the - // Level-1 row from a 4-way srcSize-tiered table (`ZSTD_getCParams_internal` - // → `ZSTD_defaultCParameters[tableID][1]`), and minMatch shrinks for - // smaller inputs: 7 (>256 KiB) / 6 (16..256 KiB) / 5 (<=16 KiB). The base - // here is the tier-0 value; `fast_l1_mls_for_source_size` lowers it per the - // tier in `adjust_params_for_source_size`. - mls: 7, - step_size: 2, -}; -const FAST_L2: FastConfig = FastConfig { - hash_log: 16, - mls: 6, - step_size: 2, -}; - -/// Resolved tuning parameters for a compression level. The -/// [`StrategyTag`] is the single source of truth for the backend -/// family and the compile-time strategy consts; the runtime -/// [`BackendTag`] used by the driver dispatcher is derived via -/// [`StrategyTag::backend`] so the two cannot drift. -#[derive(Copy, Clone, PartialEq, Eq)] -struct LevelParams { - strategy_tag: super::strategy::StrategyTag, - /// Decoupled search-method axis. Independent of `strategy_tag`'s - /// parse half: a level can pair any parse (greedy / lazy depth via - /// `lazy_depth`) with any search backend here. Defaults to the - /// historical pairing (`strategy_tag.search()`) but is overridable - /// per level so the parse×search matrix can be swept and tuned. - search: super::strategy::SearchMethod, - window_log: u8, - lazy_depth: u8, - /// Per-strategy tuning. Exactly one is `Some` on each level row, matching - /// `strategy_tag`'s backend, so the table self-documents which knobs a - /// level actually consumes (the others are `None`, not dead placeholders): - /// `fast` for the Fast/Simple backend, `dfast` for Double-Fast, `hc` for - /// the HashChain (lazy / btopt / btultra*) backend, `row` for the Row - /// (greedy L5) backend. - fast: Option, - dfast: Option, - hc: Option, - row: Option, -} - -impl LevelParams { - /// Backend family (storage variant) for the driver dispatcher. - /// Derived from the decoupled `search` axis so a level can route to - /// a different search backend than its `strategy_tag` historically - /// implied. - fn backend(&self) -> super::strategy::BackendTag { - self.search.backend() - } - - /// Parse mode derived from the decoupled `search` axis: the binary-tree - /// search path carries `ParseMode::Optimal`; every other search backend - /// derives greedy/lazy/lazy2 from `lazy_depth`. Reading `search` (not the - /// strategy tag) keeps the parse×search decoupling complete even when a - /// level whose tag is `Bt*` is overridden to a non-BT search backend. - fn parse(&self) -> super::strategy::ParseMode { - match self.search { - super::strategy::SearchMethod::BinaryTree => super::strategy::ParseMode::Optimal, - _ => super::strategy::ParseMode::from_lazy_depth(self.lazy_depth), - } - } - - /// Cheap fingerprint pre-splitter level (the C-like `blockSplitterLevel`): - /// the EFFECTIVE upstream `ZSTD_splitBlock` level that - /// `ZSTD_optimalBlockSize` dispatches, i.e. `splitLevels[strategy] - 2` - /// (clamped at 0), NOT the raw `splitLevels[]` value. `split_level == 0` - /// routes to the cheap from-borders heuristic; `1..=4` to byChunks with - /// internal sampling level `split_level - 1`. See the body for the - /// per-strategy tier table and why the raw-table mapping was wrong. - fn pre_split(&self) -> Option { - use super::strategy::StrategyTag; - // Effective upstream `ZSTD_splitBlock` level = `splitLevels[strat] - 2` - // (clamped at 0). Upstream `splitLevels[] = {0,0,1,2,2,3,3,4,4,4}` then - // subtracts 2 before dispatch, so the byChunks sampling tier is two - // steps coarser than the raw table: greedy/lazy(d1)=0 (from-borders), - // lazy2/btlazy2=1 (byChunks rate 43), btopt+=2 (byChunks rate 11). - // An earlier version mirrored the RAW table AND bumped lazy2 to the - // rate-1 full scan (split 4) to dodge a periodic-input phantom-split — - // that ran the pre-splitter at up to 43x upstream's sampling cost - // (~87% of L9 encode time on the decode corpus). Per the drop-in - // contract ratio only needs to stay <= upstream, so matching upstream's - // sampling tier (and accepting upstream's identical over-split on - // periodic input) is the dominant large-input encode-speed win. - Some(match self.strategy_tag { - // splitLevels 0/1 -> 0: upstream does not pre-split fast/dfast at - // all; from-borders is the cheapest stand-in and rarely splits. - StrategyTag::Fast | StrategyTag::Dfast => 0, - // greedy / lazy(depth 1): splitLevels 2 -> 0 (from-borders). - StrategyTag::Greedy => 0, - StrategyTag::Lazy => { - if self.lazy_depth >= 2 { - 1 // lazy2: splitLevels 3 -> 1 (byChunks rate 43) - } else { - 0 // lazy depth 1: splitLevels 2 -> 0 (from-borders) - } - } - StrategyTag::Btlazy2 => 1, // splitLevels 3 -> 1 (byChunks rate 43) - StrategyTag::BtOpt | StrategyTag::BtUltra | StrategyTag::BtUltra2 => 2, - }) - } -} - -/// Apply the public-parameter per-knob overrides (#27) onto the -/// level-resolved [`LevelParams`], in place. Runs in [`Matcher::reset`] -/// after the level params are computed and before backend selection, so -/// a strategy override re-routes the backend uniformly. An all-`None` -/// override is a no-op the caller skips via -/// [`super::parameters::ParamOverrides::is_empty`], keeping the default -/// level geometry byte-identical. -fn apply_param_overrides(params: &mut LevelParams, ov: &super::parameters::ParamOverrides) { - use super::strategy::SearchMethod; - - // 1. Strategy override re-derives tag / search / lazy depth. - if let Some(strategy) = ov.strategy { - let tag = strategy.tag(); - params.strategy_tag = tag; - params.search = tag.search(); - params.lazy_depth = strategy.lazy_depth(); - } - - // 2. Ensure the active backend's config row exists (synthesize a - // default when a strategy override moved off the native row). - match params.search { - SearchMethod::Fast => { - params.fast.get_or_insert(FAST_L1); - } - SearchMethod::DoubleFast => { - params.dfast.get_or_insert(DFAST_L3); - } - SearchMethod::RowHash => { - params.row.get_or_insert(ROW_L5); - } - SearchMethod::HashChain | SearchMethod::BinaryTree => { - // A `Btlazy2` strategy override moved off a non-HC row needs the - // BT 5-byte finder hash (upstream zstd minMatch 5); other synthesized HC - // rows keep the 4-byte default. An explicit `min_match` override - // below refines this further. - params.hc.get_or_insert(HcConfig { - search_mls: if matches!(params.strategy_tag, super::strategy::StrategyTag::Btlazy2) - { - 5 - } else { - HC_OVERRIDE_DEFAULT.search_mls - }, - ..HC_OVERRIDE_DEFAULT - }); - } - } - - // 3. window_log (bounds-checked at <= 30 by the builder). - if let Some(window_log) = ov.window_log { - params.window_log = window_log; - } - - // 4. Per-backend numeric knobs map into the active config, mirroring - // the upstream zstd `cParams` -> matcher translation documented on each - // config struct. - match params.search { - SearchMethod::Fast => { - if let Some(fast) = params.fast.as_mut() { - if let Some(hash_log) = ov.hash_log { - fast.hash_log = hash_log; - } - if let Some(min_match) = ov.min_match { - fast.mls = min_match; - } - } - } - SearchMethod::DoubleFast => { - if let Some(dfast) = params.dfast.as_mut() { - // hashLog -> long table, chainLog -> short table (the - // dfast secondary index). Both bounds-checked <= 30, so - // the `u8` casts are lossless. - if let Some(hash_log) = ov.hash_log { - dfast.long_hash_log = hash_log as u8; - } - if let Some(chain_log) = ov.chain_log { - dfast.short_hash_log = chain_log as u8; - } - } - } - SearchMethod::RowHash => { - if let Some(row) = params.row.as_mut() { - // Row hash-table width override (mirrors dfast `long_hash_log` - // / hc `hash_log`). Row has no separate chain table — the - // per-row depth comes from `search_log` below — so only - // `hash_log` maps here; `chain_log` has no Row analogue. - if let Some(hash_log) = ov.hash_log { - row.hash_bits = hash_log as usize; - } - if let Some(search_log) = ov.search_log { - // Upstream zstd: rowLog = clamp(searchLog, 4, 6); - // nbAttempts = 1 << min(searchLog, rowLog). - let row_log = (search_log as usize).clamp(4, 6); - row.row_log = row_log; - row.search_depth = 1usize << (search_log as usize).min(row_log); - } - if let Some(target_length) = ov.target_length { - row.target_len = target_length as usize; - } - if let Some(min_match) = ov.min_match { - row.mls = min_match as usize; - } - } - } - SearchMethod::HashChain | SearchMethod::BinaryTree => { - if let Some(hc) = params.hc.as_mut() { - if let Some(hash_log) = ov.hash_log { - hc.hash_log = hash_log as usize; - } - if let Some(chain_log) = ov.chain_log { - hc.chain_log = chain_log as usize; - } - if let Some(search_log) = ov.search_log { - hc.search_depth = 1usize << search_log; - } - if let Some(target_length) = ov.target_length { - hc.target_len = target_length as usize; - } - if let Some(min_match) = ov.min_match { - // Upstream zstd `mls = BOUNDED(4, cParams.minMatch, 6)`: a BT - // min_match override maps into the finder hash width. Only - // the BT body reads `search_mls`; HC/lazy keep 4-byte - // hashing regardless, so this is a no-op for them. - hc.search_mls = (min_match as usize).clamp(4, 6); - } - } - } - } -} - -/// Map the resolved runtime strategy to the upstream zstd LDM strategy ordinal -/// (1..=9) that [`super::ldm::params::LdmParams::adjust_for`] expects. -/// The collapsed `Lazy` tag splits on `lazy_depth` (lazy = 4, lazy2 = 5). -#[cfg(feature = "hash")] -fn ldm_strategy_ordinal(tag: super::strategy::StrategyTag, lazy_depth: u8) -> u32 { - use super::strategy::StrategyTag; - match tag { - StrategyTag::Fast => 1, - StrategyTag::Dfast => 2, - StrategyTag::Greedy => 3, - StrategyTag::Lazy => { - if lazy_depth >= 2 { - 5 - } else { - 4 - } - } - // Upstream zstd `ZSTD_btlazy2` ordinal. - StrategyTag::Btlazy2 => 6, - StrategyTag::BtOpt => 7, - StrategyTag::BtUltra => 8, - StrategyTag::BtUltra2 => 9, - } -} - -/// `ceil(log2(size))` of a source-size hint, with a zero hint floored to -/// [`MIN_WINDOW_LOG`]. This is the single quantization every hint-dependent -/// matcher parameter is derived from: the window-log cap, the HC / Fast hash -/// and chain widths, the Dfast / Row table widths, the L22 config buckets, and -/// the Fast attach-vs-copy cutoff. Two hints sharing this value resolve to the -/// identical matcher shape, which is why it (not the raw byte count) keys the -/// primed-dictionary snapshot — see [`PrimedKey`]. Operates on the full `u64` -/// so callers comparing a hint against a cutoff get the same bucketed decision -/// here and at the driver, with no `as usize` truncation on 32-bit targets. -pub(crate) fn source_size_ceil_log(size: u64) -> u8 { - if size == 0 { - MIN_WINDOW_LOG - } else { - (64 - (size - 1).leading_zeros()) as u8 - } -} - -/// Attach-vs-copy cutoff for the Fast strategy, as a ceil-log bucket: a hint at -/// or below `2^this` (or unknown, `None`) ATTACHES the dictionary (a separate -/// immutable table scanned in place via the borrowed dual-base kernel); a larger -/// hint would COPY it into the live table. -/// -/// We set this to `31` so every dictionary source up to 2 GiB attaches, -/// diverging from upstream zstd's 8 KiB `ZSTD_shouldAttachDict` cutoff ON -/// PURPOSE: upstream copy mode copies the small CDict TABLES into the cctx and -/// still scans the input in place, but our flat-history copy path memmoves the -/// whole INPUT into history every frame (profiled at 30% `__memmove` + 14% -/// `__memset` on a reused 1 MiB dict encode). Attach mode scans the caller's -/// input in place with the dict as a separate prefix base, so it is strictly -/// faster for every frame size here (measured: 1 MiB dict frame 167 us -> 52 us, -/// 0.42x of C; 10 KiB 20.4 us -> 4.4 us, 0.17x of C). The dual-base kernel -/// carries `window_low`, so over-window inputs stay in-window and C-decodable. -/// -/// `31` is also the largest bucket the borrowed kernel can attach: it stores -/// virtual positions as `u32` (`cur_abs as u32`), so the maximum attached source -/// `1 << 31` (plus the dict prefix) stays below `u32::MAX`; the next bucket `32` -/// (4 GiB) would wrap that arithmetic. Sources past 2 GiB therefore fall back to -/// copy mode — rare in practice, and the relative copy cost shrinks as the -/// source grows. Per the drop-in-not-binary-parity contract, we make this match -/// decision ourselves. -/// Shared by `reset` (records the mode in the primed-snapshot key) and -/// `prime_with_dictionary` (acts on it). -pub(crate) const FAST_ATTACH_DICT_CUTOFF_LOG: u8 = 31; - -/// Largest dictionary region (bytes) the Fast attach path can index. The tagged -/// dict table packs each position into `32 - DICT_TAG_BITS` (= 24) bits, so a -/// region past `2^24` (16 MiB) would overflow the packed position. Dictionaries -/// this large fall back to COPY mode, whose live table stores full `u32` -/// positions and handles them. The size hint set on dict load equals the actual -/// dict content length, so the attach-vs-copy decision (and the matching -/// snapshot-key / epoch bits) can gate on it consistently at reset time. -pub(crate) const MAX_FAST_ATTACH_DICT_REGION: usize = 1 << 24; - -/// Dfast counterpart of [`FAST_ATTACH_DICT_CUTOFF_LOG`]: upstream zstd -/// `ZSTD_dictMatchState` attach cutoff for the double-fast strategy is 16 KiB -/// (`2^14`), so small / unknown-size inputs ATTACH (separate immutable dict -/// long+short tables + dual-probe in `start_matching_fast_loop`) and larger -/// known-size inputs COPY (re-prime the dict into the live tables, where the -/// dense scan matches it as window history). The attach build also self-gates -/// on `use_fast_loop` inside `skip_matching_for_dict_attach` — only the -/// fast-loop levels (L3 / Default / L0) carry the dual-probe. -const DFAST_ATTACH_DICT_CUTOFF_LOG: u8 = 14; - -/// `ZSTD_dictMatchState` attach cutoff for the Row (greedy/lazy) strategy is -/// 32 KiB (`2^15`, upstream zstd `attachDictSizeCutoffs`): small / unknown-size inputs -/// ATTACH the dict into the separate immutable row index (bounded dual-probe in -/// `row_candidate_rl`), larger known-size inputs dense-COPY into the live rows. -const ROW_ATTACH_DICT_CUTOFF_LOG: u8 = 15; - -/// 32 KiB (`2^15`, upstream zstd `attachDictSizeCutoffs[ZSTD_lazy2]`): small / -/// unknown-size inputs ATTACH the dict as a separate hash-chain dms (the dual -/// search in `find_best_match` walks the live input chain + the dms), larger -/// known-size inputs dense-COPY (merge the dict into the live chain and search -/// the one combined chain). -const HC_ATTACH_DICT_CUTOFF_LOG: u8 = 15; - -/// BT/optimal attach cutoff for `btlazy2` + `btopt`: 32 KiB (`2^15`, upstream -/// zstd `attachDictSizeCutoffs[ZSTD_btlazy2]` == `[ZSTD_btopt]`). Small / -/// unknown-size inputs ATTACH the dict as a separate DUBT dms; larger known-size -/// inputs COPY the dict into the LIVE binary tree (upstream zstd -/// `ZSTD_resetCCtx_byCopyingCDict`). -const BT_OPT_ATTACH_DICT_CUTOFF_LOG: u8 = 15; - -/// BT/optimal attach cutoff for `btultra` + `btultra2`: 8 KiB (`2^13`, upstream -/// zstd `attachDictSizeCutoffs[ZSTD_btultra]` == `[ZSTD_btultra2]`). The deepest -/// parses copy the dict into the live tree past a much smaller source than the -/// `btopt` tier, matching upstream's per-strategy cutoff table. -const BT_ULTRA_ATTACH_DICT_CUTOFF_LOG: u8 = 13; - -// Source-size cap for the dfast hash bits when a size hint is present: a tiny -// input needs no larger hash than its window. The upstream zstd `cParams.hashLog` / -// `chainLog` (from `DfastConfig`) caps it from above at the call site. -fn dfast_hash_bits_for_window(max_window_size: usize) -> usize { - let window_log = (usize::BITS - 1 - max_window_size.leading_zeros()) as usize; - window_log.max(MIN_WINDOW_LOG as usize) -} - -fn row_hash_bits_for_window(max_window_size: usize) -> usize { - // Upstream zstd `ZSTD_adjustCParams_internal` cap: `hashLog <= windowLog + 1`. - // The `+ 1` is load-bearing for L12, whose upstream zstd hashLog (23) exceeds - // its windowLog (22) — a plain `windowLog` cap would shrink the L12 - // table on EVERY hinted reset and split primed snapshots between - // hinted and unhinted frames that resolve to the identical geometry. - // No constant upper clamp: the old `ROW_HASH_BITS` (20) ceiling - // predates the lazy band moving onto Row (L9-12 carry upstream zstd hashLog - // 21-23). - let window_log = (usize::BITS - 1 - max_window_size.leading_zeros()) as usize; - (window_log + 1).max(MIN_WINDOW_LOG as usize) -} - -/// `floor(log2(window))` for the HashChain table-log cap (upstream zstd -/// `ZSTD_adjustCParams_internal`). The caller clamps the level's `hash_log` / -/// `chain_log` from above with this so a small hinted input doesn't allocate the -/// full level's tables. -fn hc_hash_bits_for_window(max_window_size: usize) -> usize { - let window_log = (usize::BITS - 1 - max_window_size.leading_zeros()) as usize; - window_log.max(MIN_WINDOW_LOG as usize) -} - -/// Parameter table for numeric compression levels 1–22. -/// -/// Each entry maps a zstd compression level to the best-available matcher -/// backend and tuning knobs. High levels map to dedicated parse modes: -/// btopt (16-17), btultra (18), btultra2 (19-22) — matching upstream zstd -/// `clevels.h` (level 19 is `ZSTD_btultra2`, not plain btultra). -/// -/// Index 0 = level 1, index 21 = level 22. -#[rustfmt::skip] -const LEVEL_TABLE: [LevelParams; 22] = [ - // Exactly one of fast/dfast/hc/row is Some per row, matching the strategy - // backend; the rest are None (not dead placeholders). - // Lvl Strategy wlog lazy per-strategy config - // --- -------------- ---- ---- ------------------- - /* 1 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Fast, search: super::strategy::SearchMethod::Fast, window_log: 19, lazy_depth: 0, fast: Some(FAST_L1), dfast: None, hc: None, row: None }, - /* 2 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Fast, search: super::strategy::SearchMethod::Fast, window_log: 20, lazy_depth: 0, fast: Some(FAST_L2), dfast: None, hc: None, row: None }, - /* 3 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Dfast, search: super::strategy::SearchMethod::DoubleFast, window_log: 21, lazy_depth: 1, fast: None, dfast: Some(DFAST_L3), hc: None, row: None }, - /* 4 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Dfast, search: super::strategy::SearchMethod::DoubleFast, window_log: 21, lazy_depth: 1, fast: None, dfast: Some(DFAST_L4), hc: None, row: None }, - // target_len column for L5..=L15 matches upstream zstd cParams.targetLength - // from clevels.h table[0] (default — srcSize > 256 KB). Upstream zstd uses - // it as the lazy outer loop's `sufficient_len` (nice-match) threshold. - // Inflating it above upstream zstd forces the chain walk to complete - // search_depth iterations instead of breaking on the first - // long-enough match — the dominant cost in the L5..=L15 speed - // regression vs FFI (see lazy_band_target_len_matches_default_table). - /* 5 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Greedy, search: super::strategy::SearchMethod::RowHash, window_log: 21, lazy_depth: 0, fast: None, dfast: None, hc: None, row: Some(ROW_L5) }, - // L6-12: the upstream zstd runs the lazy/lazy2 strategies on the ROW-based - // match finder by default (`ZSTD_resolveRowMatchFinderMode`: row mode - // is on for greedy..lazy2 whenever SIMD is available) — a bounded - // SIMD tag scan per row instead of a pointer-chasing hash-chain walk. - // Our HashChain walk on these levels was ~75% of L10 wall time on the - // 1 MiB corpus (dependent chain-table loads). Same `RowConfig` - // derivation as `ROW_L5` above, upstream zstd values per level in the - // `ROW_L6..ROW_L12` comment block. - /* 6 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::RowHash, window_log: 21, lazy_depth: 1, fast: None, dfast: None, hc: None, row: Some(ROW_L6) }, - /* 7 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::RowHash, window_log: 21, lazy_depth: 1, fast: None, dfast: None, hc: None, row: Some(ROW_L7) }, - /* 8 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::RowHash, window_log: 21, lazy_depth: 2, fast: None, dfast: None, hc: None, row: Some(ROW_L8) }, - /* 9 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::RowHash, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: None, row: Some(ROW_L9) }, - /*10 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::RowHash, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: None, row: Some(ROW_L10) }, - /*11 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::RowHash, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: None, row: Some(ROW_L11) }, - /*12 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, search: super::strategy::SearchMethod::RowHash, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: None, row: Some(ROW_L12) }, - // L13-15: reference uses btlazy2 (binary-tree finder) with searchLog 4/5/6 - // (search_depth 16/32/64) and targetLength 32. We run the hash-chain Lazy - // parser here, so we mirror the reference search budget rather than inflate - // it: matching the table keeps speed near the reference and makes per-level - // perf divergences comparable. The binary-tree finder that would let a - // smaller searchLog find longer matches (and re-establish a strict ratio - // ladder above L12) is tracked separately; until it lands these levels sit - // close to L12 on hash-chain inputs by design. - /*13 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Btlazy2, search: super::strategy::SearchMethod::BinaryTree, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 22, chain_log: 22, search_depth: 16, target_len: 32, search_mls: 5 }), row: None }, - /*14 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Btlazy2, search: super::strategy::SearchMethod::BinaryTree, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 23, chain_log: 22, search_depth: 32, target_len: 32, search_mls: 5 }), row: None }, - /*15 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Btlazy2, search: super::strategy::SearchMethod::BinaryTree, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 23, chain_log: 23, search_depth: 64, target_len: 32, search_mls: 5 }), row: None }, - /*16 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtOpt, search: super::strategy::SearchMethod::BinaryTree, window_log: 22, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 22, chain_log: 22, search_depth: 32, target_len: 48, search_mls: 5 }), row: None }, - /*17 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtOpt, search: super::strategy::SearchMethod::BinaryTree, window_log: 23, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 22, chain_log: 23, search_depth: 32, target_len: 64, search_mls: 4 }), row: None }, - /*18 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra, search: super::strategy::SearchMethod::BinaryTree, window_log: 23, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 22, chain_log: 23, search_depth: 64, target_len: 64, search_mls: 4 }), row: None }, - /*19 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log: 23, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 22, chain_log: 24, search_depth: 128, target_len: 256, search_mls: 4 }), row: None }, - /*20 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log: 25, lazy_depth: 2, fast: None, dfast: None, hc: Some(HcConfig { hash_log: 23, chain_log: 25, search_depth: 128, target_len: 256, search_mls: 4 }), row: None }, - /*21 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log: 26, lazy_depth: 2, fast: None, dfast: None, hc: Some(BTULTRA2_HC_CONFIG), row: None }, - /*22 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, search: super::strategy::SearchMethod::BinaryTree, window_log: 27, lazy_depth: 2, fast: None, dfast: None, hc: Some(BTULTRA2_HC_CONFIG_L22), row: None }, -]; - -/// Upstream `ZSTD_createCDict` table geometry: the `(hash_log, chain_log)` a -/// dictionary's prepared match-finder tables get. Thin adapter over the single -/// cParams source [`super::cparams::create_cdict_table_logs`], which mirrors -/// `ZSTD_adjustCParams_internal` under `ZSTD_cpm_createCDict`. `window_log` is -/// the resolved compress window; `hash_log` / `chain_log` are the level's own -/// widths; `uses_bt` selects the binary-tree `cycleLog` (`chainLog - 1`). -fn cdict_table_logs( - window_log: u8, - hash_log: usize, - chain_log: usize, - uses_bt: bool, - dict_size: usize, -) -> (usize, usize) { - let (h, c) = super::cparams::create_cdict_table_logs( - window_log, - hash_log as u32, - chain_log as u32, - uses_bt, - dict_size, - ); - (h as usize, c as usize) -} - -/// Smallest window_log the encoder will use regardless of source size. -pub(crate) const MIN_WINDOW_LOG: u8 = 10; -/// Conservative floor for source-size-hinted window tuning. -/// -/// Hinted windows below 16 KiB (`window_log < 14`) currently regress C-FFI -/// interoperability on certain compressed-block patterns. Keep hinted -/// windows at 16 KiB or larger until that compatibility gap is closed. -const MIN_HINTED_WINDOW_LOG: u8 = 14; - -/// Adjust level parameters for a known source size. -/// -/// This derives a cap from `ceil(log2(src_size))`, then clamps it to -/// [`MIN_HINTED_WINDOW_LOG`] (16 KiB). A zero-byte size hint is treated as -/// [`MIN_WINDOW_LOG`] for the raw ceil-log step and then promoted to the hinted -/// floor. This keeps tables bounded for small inputs while preserving the -/// encoder's baseline minimum supported window. -/// For the HC backend, `hash_log` and `chain_log` are reduced -/// proportionally. -/// Source-size tier index, matching upstream `ZSTD_getCParams_internal`'s -/// `tableID = (rSize<=256K)+(rSize<=128K)+(rSize<=16K)`: 0 = > 256 KiB or -/// unknown, 1 = 128..256 KiB, 2 = 16..128 KiB, 3 = <= 16 KiB. -fn cparams_tier(source_size: Option) -> usize { - match source_size { - Some(size) if size <= 16 * 1024 => 3, - Some(size) if size <= 128 * 1024 => 2, - Some(size) if size <= 256 * 1024 => 1, - _ => 0, - } -} - -/// Override a Fast (L1/L2) or Dfast (L3) level row's table-shaping cParams -/// (hashLog / chainLog / minMatch) by source-size tier, matching the -/// reference `ZSTD_defaultCParameters[tableID][level]`. L1 keeps its base -/// hashLog (the source-size window clamp in `adjust_params_for_source_size` -/// already lands on the reference value) and only tiers minMatch; L2 also -/// tiers hashLog (the tier-0 value 16 oversized the table on medium inputs, -/// the page-fault pathology); L3 tiers both dfast hash widths. Strategy -/// switches (L2 tier 1, L4) are intentionally not applied here. -fn apply_cparams_tier(level: i32, source_size: Option, p: &mut LevelParams) { - let tier = cparams_tier(source_size); - // Single source for the table data: the verbatim upstream - // `ZSTD_defaultCParameters[tier][level]` row (`cparams::default_cparams`). - // The encoder consumes only the table-shaping widths here; the window / - // `table_log` clamp lives in `adjust_params_for_source_size`. - match level { - // Fast, all tiers — minMatch only (hashLog handled by the window clamp). - 1 => { - if let Some(f) = p.fast.as_mut() { - f.mls = super::cparams::default_cparams(tier, 1).min_match; - } - } - // Fast (base strategy; tier 1 is dfast upstream — not switched here). - 2 => { - if let Some(f) = p.fast.as_mut() { - let cp = super::cparams::default_cparams(tier, 2); - f.hash_log = cp.hash_log; - f.mls = cp.min_match; - } - } - // Dfast, all tiers — long hashLog (`hash_log`) + short chainLog (`chain_log`). - 3 => { - if let Some(d) = p.dfast.as_mut() { - let cp = super::cparams::default_cparams(tier, 3); - d.long_hash_log = cp.hash_log as u8; - d.short_hash_log = cp.chain_log as u8; - } - } - _ => {} - } -} - -fn adjust_params_for_source_size(mut params: LevelParams, src_size: u64) -> LevelParams { - // Derive a source-size-based cap from ceil(log2(src_size)), then - // clamp first to MIN_WINDOW_LOG (baseline encoder minimum) and then to - // MIN_HINTED_WINDOW_LOG (16 KiB hinted floor). For tiny or zero hints we - // therefore keep a 16 KiB effective minimum window in hinted mode. - // Raw ceil(log2(src_size)) drives the internal table sizes. The - // advertised `window_log` is separately floored at MIN_HINTED_WINDOW_LOG - // (a decoder-interop requirement on the wire format), but the hash / - // chain table widths are internal and never appear in the frame, so they - // can track the actual source size below that floor. - let raw_src_log = source_size_ceil_log(src_size); - let src_log = raw_src_log.max(MIN_WINDOW_LOG).max(MIN_HINTED_WINDOW_LOG); - if src_log < params.window_log { - params.window_log = src_log; - } - // Internal match-finder tables are sized from `table_log` — the RAW - // source log (floored only at the baseline `MIN_WINDOW_LOG`), NOT the - // wire `window_log` floor. The table widths never appear in the frame, so - // for small inputs they can track the actual source size and avoid - // zeroing a window-sized table per frame; large inputs keep the level's - // widths. The cap is applied with the same per-backend headroom the - // level table uses, so the load factor (and match quality) is unchanged. - // The Dfast backend derives its table widths from the source in `reset` - // (`set_hash_bits` recomputes there), so it is not adjusted here. The Row - // backend's width IS capped here, mirroring the upstream zstd (see the Row branch). - let table_log = raw_src_log.max(MIN_WINDOW_LOG); - let backend = params.backend(); - if backend == super::strategy::BackendTag::HashChain { - let hc = params - .hc - .as_mut() - .expect("HashChain level row carries an HcConfig"); - if (table_log + 2) < hc.hash_log as u8 { - hc.hash_log = (table_log + 2) as usize; - } - if (table_log + 1) < hc.chain_log as u8 { - hc.chain_log = (table_log + 1) as usize; - } - } else if backend == super::strategy::BackendTag::Row { - let row = params - .row - .as_mut() - .expect("Row level row carries a RowConfig"); - // Upstream zstd `ZSTD_adjustCParams_internal` (zstd_compress.c): once - // the window is source-capped, `hashLog <= windowLog + 1`. The row - // table is `2^hash_bits` slots, exactly upstream's row hashTable - // `2^hashLog` slots, so the same cap applies. Without it the row table - // stays at the level's unbounded width (e.g. L12 hash_bits 23 = 4x - // upstream's source-capped 21), the dominant peak-memory excess on the - // row band. - let row_cap = (table_log + 1) as usize; - if row_cap < row.hash_bits { - row.hash_bits = row_cap; - } - } else if backend == super::strategy::BackendTag::Simple { - let fast = params - .fast - .as_mut() - .expect("Fast level row carries a FastConfig"); - let fast_cap = (table_log + 1) as u32; - if fast_cap < fast.hash_log { - fast.hash_log = fast_cap; - } - } - params -} - -fn level22_btultra2_params_for_source_size(source_size: Option) -> LevelParams { - let mut hc = match source_size { - Some(size) if size <= 16 * 1024 => BTULTRA2_HC_CONFIG_L22_16K, - Some(size) if size <= 128 * 1024 => BTULTRA2_HC_CONFIG_L22_128K, - Some(size) if size <= 256 * 1024 => BTULTRA2_HC_CONFIG_L22_256K, - _ => BTULTRA2_HC_CONFIG_L22, - }; - let mut window_log = match source_size { - Some(size) if size <= 16 * 1024 => 14, - Some(size) if size <= 128 * 1024 => 17, - Some(size) if size <= 256 * 1024 => 18, - _ => 27, - }; - if let Some(size) = source_size - && size > 256 * 1024 - { - let src_log = source_size_ceil_log(size); - window_log = window_log.min(src_log.max(MIN_WINDOW_LOG)); - let adjusted_table_log = window_log as usize + 1; - hc.hash_log = hc.hash_log.min(adjusted_table_log); - hc.chain_log = hc.chain_log.min(adjusted_table_log); - } - LevelParams { - strategy_tag: super::strategy::StrategyTag::BtUltra2, - search: super::strategy::SearchMethod::BinaryTree, - window_log, - lazy_depth: 2, - fast: None, - dfast: None, - hc: Some(hc), - row: None, - } -} - -/// Estimated steady-state heap footprint of a one-shot compression context -/// at `level` (window history + match-finder tables + block staging), in -/// bytes. Computed from the same per-level tuning table the encoder -/// resolves at frame start, so the estimate tracks the real allocations; -/// it is an upper-bound style budget figure, not an exact accounting. -pub fn estimated_compression_workspace_bytes(level: CompressionLevel) -> usize { - use super::strategy::StrategyTag; - let params = resolve_level_params(level, None); - let window = 1usize << params.window_log; - // Mirror `configure()`: the HC3 short-match side table exists only on - // the btultra/btultra2 tags (minMatch 3), capped by the window log; the - // BT pointer-pair layout fits inside the `4 << chain_log` chain term - // (pairs over `chain_log - 1` nodes). - let wants_hash3 = matches!( - params.strategy_tag, - StrategyTag::BtUltra | StrategyTag::BtUltra2 - ); - let uses_bt = matches!( - params.strategy_tag, - StrategyTag::Btlazy2 | StrategyTag::BtOpt | StrategyTag::BtUltra | StrategyTag::BtUltra2 - ); - let tables = params.fast.map(|f| 4usize << f.hash_log).unwrap_or(0) - + params - .dfast - .map(|d| (4usize << d.long_hash_log) + (4usize << d.short_hash_log)) - .unwrap_or(0) - + params - .hc - .map(|h| { - let hash3 = if wants_hash3 { - 4usize - << super::match_table::storage::HC3_HASH_LOG.min(params.window_log as usize) - } else { - 0 - }; - (4usize << h.hash_log) + (4usize << h.chain_log) + hash3 - }) - .unwrap_or(0) - + params - .row - .map(|r| (4usize << r.hash_bits) + (2usize << r.hash_bits)) - .unwrap_or(0); - // BT modes box a `BtMatcher`; its retained scratch layout is budgeted - // next to the struct so estimator and allocator evolve together. - let bt = if uses_bt { - super::bt::BtMatcher::estimated_workspace_bytes() - } else { - 0 - }; - // Block staging: literal + sequence buffers plus the compressed-block - // scratch, each bounded by the 128 KiB block size. - let staging = 3 * (128 * 1024); - window + tables + bt + staging -} - -/// Extra steady-state workspace the binary-tree strategies (ordinals 6..=9, -/// btlazy2..btultra2) retain beyond the hash/chain tables: the boxed matcher -/// plus its scratch arenas, and the HC3 short-match side table for -/// btultra/btultra2 (capped by the window log). 0 for non-BT ordinals. -pub fn estimated_bt_strategy_extra_bytes(strategy_ordinal: u32, window_log: u32) -> usize { - if !(6..=9).contains(&strategy_ordinal) { - return 0; - } - let hash3 = if matches!(strategy_ordinal, 8 | 9) { - 4usize << super::match_table::storage::HC3_HASH_LOG.min(window_log as usize) - } else { - 0 - }; - super::bt::BtMatcher::estimated_workspace_bytes() + hash3 -} - -/// Resolve a [`CompressionLevel`] (+ optional source-size hint) to the -/// concrete [`LevelParams`] the matcher runs: strategy tag, search method -/// (match-finder), window log, and per-backend config. -/// -/// ## CRITICAL: input size changes the match-finder (and can change strategy) -/// -/// The resolved geometry is a function of the SOURCE SIZE, not the level -/// alone. This is the easy-to-miss part (so read this before assuming a level -/// maps to one fixed match-finder). It mirrors three upstream zstd stages: -/// -/// 1. [`LEVEL_TABLE`] holds the tier-0 (source > 256 KiB) base row per level -/// (upstream `ZSTD_defaultCParameters[0]`). L6-L12 carry -/// `SearchMethod::RowHash` (the Row match-finder), like upstream's -/// greedy/lazy default. -/// 2. [`apply_cparams_tier`] overrides the table-shaping widths for the -/// smaller source tiers (upstream `ZSTD_getCParams_internal` tier table). -/// NOTE: upstream ALSO switches STRATEGY in some tiers (L2 → dfast, L4 → -/// greedy on small sources); those backend switches are NOT yet replicated, -/// so those levels keep their base strategy on small inputs. -/// 3. [`adjust_params_for_source_size`] caps `window_log` to -/// ~`ceil_log2(source_size)` (upstream `ZSTD_adjustCParams_internal`). -/// -/// THEN, in the matcher `reset`, the greedy/lazy band falls back from -/// `RowHash` to `SearchMethod::HashChain` when the resolved `window_log <= 14` -/// — exactly upstream's `ZSTD_resolveRowMatchFinderMode` (the Row match-finder -/// is used for greedy/lazy/lazy2 ONLY when `windowLog > 14`). Net effect for -/// the SAME level: -/// -/// * small input (e.g. a 10 KiB fixture → `window_log` 14) → **HashChain** -/// (`ZSTD_HcFindBestMatch`, scalar chain walk); -/// * large input (e.g. 1 MiB → `window_log` 20) → **RowHash** (the SIMD-tag -/// row match-finder). -/// -/// A dictionary does NOT change the match-finder: it only downsizes the -/// prepared tables (`cdict_table_logs`, mirroring `ZSTD_createCDict`'s -/// small-source assumption), while `window_log` stays source-derived. So -/// `(L6, 10 KiB, +dict)` is HashChain and `(L6, 1 MiB, +dict)` is RowHash, -/// both matching upstream. When comparing against C on a fixture, resolve the -/// match-finder from the fixture's size first, or you may optimise/benchmark a -/// path C does not even take for that input. -fn resolve_level_params(level: CompressionLevel, source_size: Option) -> LevelParams { - if matches!(level, CompressionLevel::Level(22)) { - return level22_btultra2_params_for_source_size(source_size); - } - let params = match level { - CompressionLevel::Uncompressed => LevelParams { - strategy_tag: super::strategy::StrategyTag::Fast, - search: super::strategy::SearchMethod::Fast, - // Uncompressed frames emit raw blocks and never reference - // history; advertising a larger window only inflates - // decoder-side buffer reservation. Stay at 17 (128 KiB). - window_log: 17, - lazy_depth: 0, - // Beyond-upstream zstd: hash_log=14 (vs upstream zstd's 13) for 2× fewer - // collisions on structured corpora. Upstream zstd's "base for negative" - // row has targetLength=1 → step_size = 1 + 0 + 1 = 2. - fast: Some(FastConfig { - hash_log: 14, - mls: 6, - step_size: 2, - }), - dfast: None, - hc: None, - row: None, - }, - CompressionLevel::Fastest => { - // Only the Fast-specific cParams - // (fast_hash_log / fast_mls / fast_step_size) align - // with Uncompressed / negative-base row. window_log - // stays at LEVEL_TABLE[0]'s value (19) — Fastest still - // does real compression on a full window, unlike - // Uncompressed which clamps to 17. - let mut p = LEVEL_TABLE[0]; - p.fast = Some(FastConfig { - hash_log: 14, - mls: 6, - step_size: 2, - }); - p - } - CompressionLevel::Default => { - // Default == Level(DEFAULT_LEVEL); tier it the same way an explicit - // positive level is, so hinted default compression shrinks its - // table widths on small / medium frames instead of keeping the - // tier-0 row (the oversized-table page-fault pathology). - let mut p = LEVEL_TABLE[CompressionLevel::DEFAULT_LEVEL as usize - 1]; - apply_cparams_tier(CompressionLevel::DEFAULT_LEVEL, source_size, &mut p); - p - } - CompressionLevel::Better => LEVEL_TABLE[6], - // Level 13: the first dominant point of the deep-lazy band. The - // mls-wide row key lifted the shallow band's ratio enough that - // level 11 no longer strictly beats level 7 on the ladder corpus; - // the `Best` alias belongs on a config that dominates everything - // below it rather than on a hair-thin margin. - CompressionLevel::Best => LEVEL_TABLE[12], - CompressionLevel::Level(n) => { - if n > 0 { - let idx = (n as usize).min(CompressionLevel::MAX_LEVEL as usize) - 1; - let mut p = LEVEL_TABLE[idx]; - // Upstream zstd selects the cParams row from a 4-way - // source-size-tiered table (`ZSTD_getCParams_internal` → - // `ZSTD_defaultCParameters[tableID][level]`), and the Fast / - // Dfast hashLog, chainLog and minMatch shrink for smaller - // inputs. The `LEVEL_TABLE` base is the tier-0 (> 256 KiB) row; - // override the table-shaping params per tier here so small and - // medium frames use the reference's table widths (the oversized - // tier-0 widths were a per-frame alloc / page-fault pathology on - // medium inputs) and minMatch (short matches the wide hash - // skips). NOTE: the reference also switches STRATEGY in some - // tiers (L2 → dfast at 128..256 KiB, L4 → greedy at <= 16 KiB - // and 128..256 KiB); those backend switches are not yet tiered, - // so those tiers keep the base strategy. - apply_cparams_tier(n, source_size, &mut p); - p - } else if n == 0 { - // Level 0 = default, matching C zstd semantics. Tier it like the - // `Default` alias so `Level(0)` and `Default` stay identical. - let mut p = LEVEL_TABLE[CompressionLevel::DEFAULT_LEVEL as usize - 1]; - apply_cparams_tier(CompressionLevel::DEFAULT_LEVEL, source_size, &mut p); - p - } else { - // Negative levels — upstream zstd sets - // targetLength = -level (clampedCompressionLevel), - // yielding step_size = (-level) + 1 since - // !(targetLength) = 0 when targetLength > 0. - // So L-1..L-7 get step_size 2..8. Acceleration - // gradient comes from larger step skipping more - // positions per iter (faster, worse ratio). - // Clamp to upstream zstd's MIN_LEVEL before negating so - // i32::MIN can't overflow on `-n`. - let clamped = n.max(CompressionLevel::MIN_LEVEL); - let target_length = (-clamped) as usize; - let step_size = target_length + 1; - // Upstream zstd row-0 ("base for negative", clevels.h srcSize>256KB): - // hashLog=13, minMatch=7. The 32 KiB hash table (2^13 * 4B) - // is L1d-resident on contemporary cores, so every probe is an - // L1 hit; hashLog=14 (64 KiB) overflows a 32 KiB L1d and turns - // each probe into an L2 access. minMatch=7 (vs 6) skips - // short-distance 6-byte matches: fewer sequences, less - // extension/emit work, and parity with the upstream zstd's negative - // ladder on both ratio and throughput. - LevelParams { - strategy_tag: super::strategy::StrategyTag::Fast, - search: super::strategy::SearchMethod::Fast, - window_log: 19, - lazy_depth: 0, - fast: Some(FastConfig { - hash_log: 13, - mls: 7, - step_size, - }), - dfast: None, - hc: None, - row: None, - } - } - } - }; - if let Some(size) = source_size { - adjust_params_for_source_size(params, size) - } else { - params - } -} - -/// The cheap fingerprint pre-splitter level for a compression level (the -/// C-like `blockSplitterLevel`), resolved through the same per-level -/// `LevelParams` table as every other tuning knob. `None` keeps the whole -/// 128 KiB block. The frame loop reads this instead of hardcoding the -/// level→split mapping at the call site. -pub(crate) fn level_pre_split(level: CompressionLevel) -> Option { - // Resolve through `resolve_level_params` directly — NOT via the legacy - // `numeric_level()` alias — so named presets read the SAME table row as - // every other tuning knob (`Best` maps to its own row there, which is - // not the row its numeric alias points at). `Uncompressed` (raw - // blocks) never splits. - if matches!(level, CompressionLevel::Uncompressed) { - return None; - } - resolve_level_params(level, None) - .pre_split() - .map(usize::from) -} - -/// Backend storage for [`MatchGeneratorDriver`]. Exactly one match-finder -/// state lives in the driver at a time — the active variant. Backend -/// transitions in [`Matcher::reset`] drain the current variant's allocations -/// into the shared `vec_pool` and then replace `storage` with a freshly -/// constructed variant for the new backend. -/// -/// Replaces the prior pattern of four parallel fields (`match_generator`, -/// `dfast_match_generator: Option<…>`, `row_match_generator: Option<…>`, -/// `hc_match_generator: Option<…>`) + an `active_backend: BackendTag` -/// discriminator: the parallel layout kept drained inner structures -/// allocated across backend switches, and every per-frame/per-slice -/// driver operation had to dispatch on `active_backend` to pick the -/// right field. A single enum collapses the storage and makes the -/// dispatcher pattern-match on the storage variant directly — same -/// number of arms, but `storage.backend()` is now the canonical source -/// of truth and dead variants are dropped when the active backend -/// changes. -#[derive(Clone)] -enum MatcherStorage { - /// Upstream zstd `ZSTD_fast` family. Constructed by - /// [`MatchGeneratorDriver::new`] as the initial variant and - /// re-selected by [`Matcher::reset`] for any [`CompressionLevel`] - /// that `resolve_level_params` maps to [`StrategyTag::Fast`] - /// (`Uncompressed`, `Fastest`, `Level(1)`, and any non-positive - /// `Level(n)` not equal to `0`). - Simple(FastKernelMatcher), - /// Upstream zstd `ZSTD_dfast` family — two-table hash chain. Selected for - /// any level that resolves to [`StrategyTag::Dfast`] in - /// `resolve_level_params` (`Default`, `Level(0)`, `Level(2)`, - /// `Level(3)`). - Dfast(DfastMatchGenerator), - /// Upstream zstd `ZSTD_greedy` family with row hashing. Selected for any - /// level that resolves to [`StrategyTag::Greedy`] (currently - /// `Level(4)` only). - Row(RowMatchGenerator), - /// Upstream zstd `ZSTD_lazy2` and the BT-based optimal modes - /// (`btopt` / `btultra` / `btultra2`). Selected for any level that - /// resolves to [`StrategyTag::Lazy`], [`StrategyTag::BtOpt`], - /// [`StrategyTag::BtUltra`], or [`StrategyTag::BtUltra2`] - /// (`Better`, `Best`, `Level(5..=22)`, and any `Level(n)` with - /// `n > MAX_LEVEL` — `resolve_level_params` clamps positive - /// numeric levels at `MAX_LEVEL = 22` via - /// `Level(n).clamp(1, MAX_LEVEL)`, so `Level(23..=i32::MAX)` all - /// land on `BtUltra2` here). The [`HcMatchGenerator`]'s internal - /// [`HcBackend`] discriminator decides whether BT scratch is - /// allocated. - HashChain(HcMatchGenerator), -} - -impl MatcherStorage { - /// Heap bytes the active backend variant holds (tables, history, scratch). - fn heap_size(&self) -> usize { - match self { - Self::Simple(m) => m.heap_size(), - Self::Dfast(m) => m.heap_size(), - Self::Row(m) => m.heap_size(), - Self::HashChain(m) => m.heap_size(), - } - } - - /// [`super::strategy::BackendTag`] family of the active variant. - fn backend(&self) -> super::strategy::BackendTag { - use super::strategy::BackendTag; - match self { - Self::Simple(_) => BackendTag::Simple, - Self::Dfast(_) => BackendTag::Dfast, - Self::Row(_) => BackendTag::Row, - Self::HashChain(_) => BackendTag::HashChain, - } - } -} - -/// This is the default implementation of the `Matcher` trait. It allocates and reuses the buffers when possible. -pub struct MatchGeneratorDriver { - vec_pool: Vec>, - /// Active match-finder state. Exactly one backend lives here at a - /// time; [`Matcher::reset`] drains the previous variant into - /// `vec_pool` before swapping in a freshly constructed variant for - /// the new backend. `storage.backend()` is the canonical source of - /// truth for the parse family; `strategy_tag` carries the - /// compile-time strategy chosen at the last `reset()`. - storage: MatcherStorage, - // Compile-time strategy tag resolved at `reset()` from the - // requested `CompressionLevel`'s `LevelParams`. The driver's - // hot-block dispatcher in `blocks/compressed.rs` matches on - // this tag to enter the corresponding `Strategy` - // monomorphisation (`compress_block::`). - strategy_tag: super::strategy::StrategyTag, - // Decoupled search-method axis resolved at `reset()` from - // `LevelParams.search`. The per-block dispatcher routes on this - // (not on `strategy_tag`) so a level's parse and search backend can - // be chosen independently. The `BinaryTree` arm still consults - // `strategy_tag` to pick the opt `Strategy` ZST. - search: super::strategy::SearchMethod, - // Decoupled parse-mode axis resolved at `reset()` from - // `LevelParams::parse()`. Independent of `search`: greedy / lazy / - // lazy2 can run on any non-opt search backend. The backends still - // read their own `lazy_depth` (kept in sync at `reset()`); this is - // the authoritative parse selector for the dispatcher. - parse: super::strategy::ParseMode, - /// Test-only per-level recipe override applied in `reset()` before - /// backend selection. Lets the parse×search matrix be exercised - /// without editing `LEVEL_TABLE`; never compiled into production. - #[cfg(test)] - config_override: Option<(super::strategy::SearchMethod, super::strategy::ParseMode)>, - /// Fine-grained per-knob overrides from the public - /// [`super::parameters::CompressionParameters`] surface (#27). - /// `None` (or an all-`None` [`super::parameters::ParamOverrides`]) - /// keeps the resolved level geometry byte-identical to plain - /// level-based compression. Applied in [`Matcher::reset`] after the - /// level params are resolved, before backend selection. Persists - /// across resets (it is frame configuration, not a one-shot) until - /// the caller changes it. - param_overrides: Option, - slice_size: usize, - base_slice_size: usize, - // Frame header window size must stay at the configured live-window budget. - // Dictionary retention expands internal matcher capacity only. - reported_window_size: usize, - // Tracks currently retained bytes that originated from primed dictionary - // history and have not been evicted yet. - dictionary_retained_budget: usize, - // Source size hint for next frame (set via set_source_size_hint, cleared on reset). - source_size_hint: Option, - // Dictionary content size for the next frame (set via set_dictionary_size_hint, - // consumed on reset). When present on a binary-tree / hash-chain backend, the - // match-finder hash/chain tables are sized from the DICTIONARY (upstream zstd CDict - // economics: a loaded dictionary supplies the long matches, so the live tables - // can shrink to the dict's size tier) while the eviction window stays - // source-sized. Mirrors upstream zstd `ZSTD_getCParamRowSize`, which picks the cParams - // table column from `dictSize` for a dictionary-bearing compress. - dictionary_size_hint: Option, - // Normalized `ceil_log2` bucket of the frame's source-size hint, captured at - // `reset` (where `source_size_hint` is consumed) via [`source_size_ceil_log`]. - // `None` means the frame was unhinted. Drives `prime_with_dictionary`'s upstream zstd - // `ZSTD_shouldAttachDict` mode for the Simple/Fast backend: `None` (unknown) - // or `<= FAST_ATTACH_DICT_CUTOFF_LOG` → attach (separate dict table, 2-cursor - // `compress_block_fast_dict`); larger → copy (dictionary primed into the live - // table, 4-cursor `compress_block_fast`). The primed-snapshot key is the - // resolved shape ([`reset_shape`](Self::reset_shape)), not this bucket. - reset_size_log: Option, - // Whether the loaded dictionary fits the Fast attach path's tagged position - // field (`<= MAX_FAST_ATTACH_DICT_REGION`). Captured at `reset` from the - // dict-size hint (which equals the actual dict length on load) so the Fast - // attach decision, the attach-epoch reset bit, and the primed-snapshot - // `fast_attach` bit all gate on it consistently. `true` when there is no - // dictionary (the attach path is then unused). A dict too large to tag falls - // back to copy mode instead of overflowing the packed position. - reset_dict_attach_ok: bool, - // Hint-resolved matcher shape from the last `reset`: the [`LevelParams`], the - // active backend's applied Dfast/Row hash-table width (`0` for HC/Fast), the - // Fast attach-vs-copy mode, and the active LDM override (#27). Combined with - // the frame's level into the [`PrimedKey`] that keys the primed snapshot, so - // it is only restored into a reset that resolved the identical matcher AND - // LDM configuration. `None` before the first `reset`. - reset_shape: Option<( - LevelParams, - usize, - bool, - Option, - )>, - // One-shot borrowed block range `[start, end)` staged by the borrowed - // Fast frame path (`set_borrowed_block`) for the NEXT - // `start_matching` / `skip_matching_with_hint`. `Some` routes that - // call to the Simple backend's borrowed scan instead of the owned - // committed-block path; consumed (reset to `None`) by the routed - // call. Always `None` on the owned streaming path. - borrowed_pending: Option<(usize, usize)>, - /// CDict-equivalent: snapshot of the post-prime matcher state taken - /// once after the first dictionary prime — the backend `storage` - /// (hash tables + dictionary history + offset history + window) plus - /// the driver-level `dictionary_retained_budget`, the only two pieces - /// `prime_with_dictionary` writes. Subsequent frames restore this - /// (a table memcpy) instead of re-hashing every dictionary position, - /// mirroring upstream zstd `ZSTD_compressBegin_usingCDict` copying the - /// precomputed `cdict->matchState`. Invalidated when the dictionary - /// changes; keyed by the [`PrimedKey`] resolved matcher shape so a snapshot - /// is only restored into a reset that produces the same matcher — see - /// `restore_primed_dictionary`. - primed: Option<(MatcherStorage, usize, PrimedKey)>, -} - -/// Identity of the matcher configuration a primed snapshot was captured under: -/// the FULLY RESOLVED matcher shape, not the raw source-size hint. -/// -/// `reset()` resolves the hint into a [`LevelParams`] (window_log cap, the -/// HC/Fast table and search geometry, the parse depth/target-length that get -/// baked into the restored `storage`) plus, for the Dfast/Row backends, a -/// table-width derived from the hint's ceil-log bucket. The mapping from hint -/// to resolved shape is many-to-one: the source-size adjustment is monotone in -/// `ceil_log2(hint)`, and Level 22 additionally collapses several buckets onto -/// one upstream zstd tier (its `<= 16/128/256 KiB` thresholds). Keying on the raw hint -/// (or even its ceil-log bucket) therefore over-keys — two hints that resolve -/// to the identical matcher would each force a full re-prime. Keying on the -/// resolved (`params`, `table_bits`) pair restores across them. -/// -/// `table_bits` is the hint-dependent hash-table width the ACTIVE backend -/// applied (`set_hash_bits` value for Dfast/Row; `0` for HC/Fast, whose widths -/// already live in `params`). The snapshot is only ever captured on the COPY -/// path (a hinted, above-cutoff frame), so `table_bits` is always the resolved -/// Dfast/Row value there, never the unhinted default. -/// -/// `level` is kept alongside the resolved `params` because some stored matcher -/// state is derived from the level DIRECTLY, not through `params`: e.g. Dfast's -/// `use_fast_loop` is true for L3 but false for L4, yet L3 and L4 resolve to -/// byte-identical `params`. Without `level` a snapshot captured at L3 could be -/// restored into an L4 reset, installing the wrong `use_fast_loop`. -/// -/// `fast_attach` records the Fast backend's attach-vs-copy mode -/// ([`FAST_ATTACH_DICT_CUTOFF_LOG`]) because that cutoff (8 KiB) falls INSIDE a -/// single resolved shape: an 8192- and an 8193-byte Level 1 hint both clamp to -/// window_log 14 with identical `params`/`table_bits`, yet 8192 attaches (a -/// separate dict table) while 8193 copies into the live table — two different -/// `storage` shapes. The frame compressor only captures/restores snapshots on -/// the copy path today, but keying on the mode keeps the snapshot identity -/// self-sufficient rather than relying on that external gate. -/// -/// Restoring a snapshot whose key differs would reinstate the old `storage` -/// (and its `max_window_size` / table dimensions / parse params / dict-table -/// shape) under a reset that resolved a different shape — the encoder could -/// then search past the frame header's window and emit an undecodable match. -/// All fields must match before a restore is allowed. -#[derive(Clone, Copy, PartialEq, Eq)] -struct PrimedKey { - level: super::CompressionLevel, - params: LevelParams, - table_bits: usize, - fast_attach: bool, - /// Fine-grained LDM override (#27) active at capture time. The - /// snapshot's cloned `storage` carries `BtMatcher::ldm_producer`, - /// which is configured from this override; restoring a snapshot - /// captured under a different LDM configuration (enable flip or - /// changed knobs) would reinstate a stale producer. `params` already - /// pins `window_log` / `strategy_tag` (the rest of the producer's - /// identity), so folding the override completes the LDM identity. - /// `None` = LDM off, matching `ParamOverrides::ldm`. - ldm: Option, -} - -impl MatchGeneratorDriver { - /// `slice_size` sets the base block allocation size used for matcher input chunks. - /// `max_slices_in_window` determines the initial window capacity at construction - /// time. Effective window sizing is recalculated on every [`reset`](Self::reset) - /// from the resolved compression level and optional source-size hint. - pub(crate) fn new(slice_size: usize, max_slices_in_window: usize) -> Self { - // Validate inputs before deriving window_log_init. Three - // failure modes need explicit guards: - // - // 1. Zero args → `max_window_size = 0` → silent 1-byte - // degenerate window (useless). - // 2. Multiplication overflow on `slice_size * - // max_slices_in_window` → wraps silently in release. - // 3. `next_power_of_two` overflow when the product is - // above `1 << (usize::BITS - 1)` → modern Rust PANICS - // on overflow (older Rust returned 0). - // - // Catch all three at construction with a clear domain- - // specific message via `assert!` + `checked_mul` + - // `checked_next_power_of_two`, rather than letting either - // mode produce a silent degenerate matcher OR a generic - // panic deep in `FastKernelMatcher::with_params`. - assert!( - slice_size > 0, - "MatchGeneratorDriver::new requires slice_size > 0 (got 0)", - ); - assert!( - max_slices_in_window > 0, - "MatchGeneratorDriver::new requires max_slices_in_window > 0 (got 0)", - ); - let max_window_size = max_slices_in_window - .checked_mul(slice_size) - .expect("MatchGeneratorDriver::new: slice_size * max_slices_in_window overflows usize"); - // Derive an effective window_log for the initial-state matcher. - // `MatchGeneratorDriver::new` runs BEFORE any reset, so it has - // no LevelParams to consult — we initialise to whatever - // window_log fits the caller's requested max_window_size - // (round up to the next power of two via `next_power_of_two`'s - // log). Reset() overwrites all three params from the resolved - // LevelParams. - // - // `checked_next_power_of_two` returns `None` if the next power - // of two would overflow `usize`. Modern Rust's - // `next_power_of_two` PANICS on overflow rather than returning - // 0 (the panic message is generic and unhelpful), so use the - // checked variant to surface the failure with a clear, - // domain-specific error. - let next_pow2 = max_window_size.checked_next_power_of_two().expect( - "MatchGeneratorDriver::new: max_window_size too large for \ - next_power_of_two without overflow", - ); - let window_log_init = next_pow2.trailing_zeros() as u8; - Self { - vec_pool: Vec::new(), - // Deferred table: `new` runs before any source size or resolved - // LevelParams exist, so allocating at the level-default hash_log - // here would be thrown away by the first frame's reset (which - // clamps the window to the input and reallocs at the resolved - // size). The deferral lets that first reset allocate exactly once. - storage: MatcherStorage::Simple(FastKernelMatcher::with_params_deferred( - window_log_init, - FAST_LEVEL_1_HASH_LOG, - FAST_LEVEL_1_MLS, - 2, // upstream zstd default step_size (targetLength=0 → step=2) - )), - strategy_tag: super::strategy::StrategyTag::Fast, - search: super::strategy::SearchMethod::Fast, - parse: super::strategy::ParseMode::Greedy, - #[cfg(test)] - config_override: None, - param_overrides: None, - slice_size, - base_slice_size: slice_size, - // Report the ROUNDED-UP window size that the matcher - // actually carries (via `window_log_init = log2(next_pow2)` - // → matcher's `max_window_size = 1 << window_log_init = - // next_pow2`). For non-power-of-two `slice_size * - // max_slices_in_window` inputs, the unrounded value - // would under-report the active backend's window until - // the first `reset()` overwrites both sides from the - // resolved LevelParams. - reported_window_size: next_pow2, - reset_size_log: None, - reset_dict_attach_ok: true, - reset_shape: None, - dictionary_retained_budget: 0, - source_size_hint: None, - dictionary_size_hint: None, - borrowed_pending: None, - primed: None, - } - } - - fn level_params(level: CompressionLevel, source_size: Option) -> LevelParams { - resolve_level_params(level, source_size) - } - - /// Install the public-parameter per-knob overrides (#27) applied at - /// the next [`Matcher::reset`]. `None` (or an all-`None` set) restores - /// plain level-based geometry. Persists across resets until changed. - pub(crate) fn set_param_overrides( - &mut self, - overrides: Option, - ) { - self.param_overrides = overrides; - } - - /// Active backend family derived from the storage variant. Single - /// source of truth — no separate runtime tag to drift against. - pub(crate) fn active_backend(&self) -> super::strategy::BackendTag { - self.storage.backend() - } - - /// Whether the borrowed (no-copy, in-place over-window) scan is - /// implemented for the current backend + search configuration. The - /// HashChain backend serves both the lazy CHAIN parser - /// (`SearchMethod::HashChain`) and the BT/optimal parsers - /// (`SearchMethod::BinaryTree`); only the lazy chain has a borrowed scan - /// so far, so BT/optimal stay on the owned path. - pub(crate) fn borrowed_supported(&self) -> bool { - use super::strategy::{BackendTag, SearchMethod, StrategyTag}; - match self.active_backend() { - BackendTag::Simple | BackendTag::Dfast | BackendTag::Row => true, - // The HashChain backend covers two searches: the lazy CHAIN parser - // (borrowed-capable) and the BINARY-TREE search (btlazy2 L13-15 + - // optimal BtOpt/BtUltra/BtUltra2 L16-22). btlazy2's BT-tree borrowed - // scan is byte-identical to owned (reads via live_history()), so it - // takes the in-place path. The OPTIMAL parsers stay owned: their - // cost-based DP is sensitive to candidate quality, and the borrowed - // continuous-index scan yields slightly different (ratio-worse) - // candidates than the owned evict+rehash scan — borrowed optimal - // both diverged from owned and fell outside the ffi ratio bound. - // Search-aware (not just strategy_tag) so optimal BT can never be - // staged on the borrowed path even via an internal caller. - BackendTag::HashChain => match self.search { - SearchMethod::HashChain => true, - SearchMethod::BinaryTree => matches!(self.strategy_tag, StrategyTag::Btlazy2), - _ => false, - }, - } - } - - /// Whether a DICTIONARY frame can take the borrowed (no input copy) path. - /// Only the Simple (Fast) backend with the dictionary ATTACHED (not the - /// copy/merge regime) has a borrowed dict scan — `start_matching_borrowed_dict` - /// reads live matches from the borrowed input in place and dict matches - /// from the committed dict prefix via the 2-segment counter. Every other - /// backend, and copy-mode (large-input) dict frames, stay on the owned - /// path. Checked AFTER priming, so `is_attached()` reflects the resolved - /// attach-vs-copy decision. - pub(crate) fn borrowed_dict_supported(&self) -> bool { - matches!( - &self.storage, - MatcherStorage::Simple(m) if m.dict_is_attached() - ) - } - - fn simple_mut(&mut self) -> &mut FastKernelMatcher { - match &mut self.storage { - MatcherStorage::Simple(m) => m, - _ => panic!("simple backend must be initialized by reset() before use"), - } - } - - /// Reclaim the per-block input buffer that the Simple backend - /// just spent inside `start_matching` / `skip_matching_with_hint`. - /// - /// `FastKernelMatcher::take_recycled_space` returns the cleared - /// (capacity-retained) `Vec` from the last - /// `extend_history_with_pending`. We push it onto `vec_pool` - /// as-is (with `len = 0`); `get_next_space()` is responsible for - /// resizing the buffer back to `slice_size` on its next pop. The - /// pushed length is irrelevant — only the capacity matters, and - /// `extend_history_with_pending` preserves it. Without this - /// recycle path, the Simple backend would allocate a new - /// `Vec` per block — a measurable hot-path cost when blocks - /// are small (~128 KiB) and processed at hundreds of MiB/s. - fn recycle_simple_space(&mut self) { - if let Some(space) = self.simple_mut().take_recycled_space() { - // `space` is already cleared (len = 0) by - // `extend_history_with_pending`; capacity is retained. - // Leaving `len = 0` here avoids the cost of zero-filling - // the entire allocation — `get_next_space()` resizes the - // popped buffer up to `slice_size` on demand, so the - // length the pool holds is irrelevant. This matters most - // after a small-source-size hint has shrunk `slice_size` - // mid-frame: the recycled buffer can be much larger than - // the current `slice_size`, and zero-filling 128 KiB+ on - // every block would erase the perf win the recycle path - // is meant to deliver. - self.vec_pool.push(space); - } - } - - /// Register a caller-owned input buffer as the Simple backend's - /// borrowed one-shot match window. Only valid on the Simple (Fast) - /// backend; the one-shot frame path gates on that before calling. - /// - /// # Safety - /// Same contract as [`FastKernelMatcher::set_borrowed_window`]: the - /// buffer must stay live and unmodified until the window is cleared, - /// and must be cleared before the buffer is dropped or the matcher is - /// reused for another frame. - pub(crate) unsafe fn set_borrowed_window(&mut self, buffer: &[u8]) { - // SAFETY: forwarded contract — caller upholds liveness/clear. - match self.active_backend() { - super::strategy::BackendTag::Simple => unsafe { - self.simple_mut().set_borrowed_window(buffer) - }, - super::strategy::BackendTag::Dfast => unsafe { - self.dfast_matcher_mut().set_borrowed_window(buffer) - }, - super::strategy::BackendTag::Row => unsafe { - self.row_matcher_mut().set_borrowed_window(buffer) - }, - super::strategy::BackendTag::HashChain => unsafe { - self.hc_matcher_mut().set_borrowed_window(buffer) - }, - } - } - - /// Clear the borrowed one-shot window, returning the active backend - /// to the owned `history` path. - pub(crate) fn clear_borrowed_window(&mut self) { - match self.active_backend() { - super::strategy::BackendTag::Simple => self.simple_mut().clear_borrowed_window(), - super::strategy::BackendTag::Dfast => self.dfast_matcher_mut().clear_borrowed_window(), - super::strategy::BackendTag::Row => self.row_matcher_mut().clear_borrowed_window(), - super::strategy::BackendTag::HashChain => self.hc_matcher_mut().clear_borrowed_window(), - #[allow(unreachable_patterns)] - _ => {} - } - self.borrowed_pending = None; - } - - /// Stage the borrowed block range `[block_start, block_end)` for the - /// NEXT `start_matching` / `skip_matching_with_hint`, which the - /// borrowed Fast frame path uses in place of `commit_space`. While - /// staged, those trait calls route to the Simple backend's borrowed - /// scan/skip (consuming the stage) instead of the owned committed - /// block. See [`Matcher::start_matching`] / - /// [`Matcher::skip_matching_with_hint`] on this type. - pub(crate) fn set_borrowed_block(&mut self, block_start: usize, block_end: usize) { - assert!( - self.borrowed_supported(), - "borrowed block staging is not supported for the active backend/search config", - ); - assert!( - block_start <= block_end, - "borrowed block range must satisfy start <= end (start={block_start} end={block_end})", - ); - self.borrowed_pending = Some((block_start, block_end)); - // Make the range visible to `get_last_space()` immediately: the - // emit pipeline reads `get_last_space().len()` in - // `collect_block_parts` BEFORE `start_matching` consumes the - // stage, so the staged block (not the whole borrowed window) must - // be reported now to keep the literal-buffer reservation right. - match self.active_backend() { - super::strategy::BackendTag::Simple => self - .simple_mut() - .stage_borrowed_block(block_start, block_end), - super::strategy::BackendTag::Dfast => self - .dfast_matcher_mut() - .stage_borrowed_block(block_start, block_end), - super::strategy::BackendTag::Row => self - .row_matcher_mut() - .stage_borrowed_block(block_start, block_end), - super::strategy::BackendTag::HashChain => self - .hc_matcher_mut() - .table - .stage_borrowed_block(block_start, block_end), - } - } - - #[cfg(test)] - fn dfast_matcher(&self) -> &DfastMatchGenerator { - match &self.storage { - MatcherStorage::Dfast(m) => m, - _ => panic!("dfast backend must be initialized by reset() before use"), - } - } - - fn dfast_matcher_mut(&mut self) -> &mut DfastMatchGenerator { - match &mut self.storage { - MatcherStorage::Dfast(m) => m, - _ => panic!("dfast backend must be initialized by reset() before use"), - } - } - - #[cfg(test)] - fn row_matcher(&self) -> &RowMatchGenerator { - match &self.storage { - MatcherStorage::Row(m) => m, - _ => panic!("row backend must be initialized by reset() before use"), - } - } - - fn row_matcher_mut(&mut self) -> &mut RowMatchGenerator { - match &mut self.storage { - MatcherStorage::Row(m) => m, - _ => panic!("row backend must be initialized by reset() before use"), - } - } - - #[cfg(test)] - fn hc_matcher(&self) -> &HcMatchGenerator { - match &self.storage { - MatcherStorage::HashChain(m) => m, - _ => panic!("hash chain backend must be initialized by reset() before use"), - } - } - - fn hc_matcher_mut(&mut self) -> &mut HcMatchGenerator { - match &mut self.storage { - MatcherStorage::HashChain(m) => m, - _ => panic!("hash chain backend must be initialized by reset() before use"), - } - } - - /// Shrink the active backend's `max_window_size` by the bytes - /// reclaimed from the dictionary-retention budget. Returns `true` - /// iff any reclamation happened — the caller uses that as the - /// gate for [`Self::trim_after_budget_retire`] (which is a no-op - /// otherwise: with `max_window_size` unchanged the backend's - /// `trim_to_window` cannot find anything to evict, so calling it - /// just runs an extra `match` ladder + a single early-out check - /// per slice commit). - #[must_use] - fn retire_dictionary_budget(&mut self, evicted_bytes: usize) -> bool { - let reclaimed = evicted_bytes.min(self.dictionary_retained_budget); - if reclaimed == 0 { - return false; - } - self.dictionary_retained_budget -= reclaimed; - match self.active_backend() { - super::strategy::BackendTag::Simple => { - let matcher = self.simple_mut(); - // `reclaimed` can exceed the CURRENT `max_window_size`: the - // retained dict budget is tracked independently and the - // window may already have been shrunk by a prior eviction, - // so the floor at 0 is the correct clamp, not a masked bug. - matcher.max_window_size = matcher.max_window_size.saturating_sub(reclaimed); - } - super::strategy::BackendTag::Dfast => { - let matcher = self.dfast_matcher_mut(); - // `reclaimed` can exceed the CURRENT `max_window_size`: the - // retained dict budget is tracked independently and the - // window may already have been shrunk by a prior eviction, - // so the floor at 0 is the correct clamp, not a masked bug. - matcher.max_window_size = matcher.max_window_size.saturating_sub(reclaimed); - } - super::strategy::BackendTag::Row => { - let matcher = self.row_matcher_mut(); - // `reclaimed` can exceed the CURRENT `max_window_size`: the - // retained dict budget is tracked independently and the - // window may already have been shrunk by a prior eviction, - // so the floor at 0 is the correct clamp, not a masked bug. - matcher.max_window_size = matcher.max_window_size.saturating_sub(reclaimed); - } - super::strategy::BackendTag::HashChain => { - let matcher = self.hc_matcher_mut(); - // See the Simple arm: `reclaimed` may exceed the current - // window, so saturating to 0 is the correct clamp. - matcher.table.max_window_size = - matcher.table.max_window_size.saturating_sub(reclaimed); - } - } - true - } - - fn trim_after_budget_retire(&mut self) { - loop { - let mut evicted_bytes = 0usize; - match self.active_backend() { - super::strategy::BackendTag::Simple => { - // FastKernelMatcher owns its history as a single - // flat `Vec` (upstream zstd's flat-buffer layout) - // rather than the legacy per-block `WindowEntry` - // stack. There are no per-block Vec allocations - // to recycle into `vec_pool` — `trim_to_window` - // drains the oldest bytes in-place and returns - // the count for the dictionary-budget loop's - // termination check. - let MatcherStorage::Simple(m) = &mut self.storage else { - unreachable!("active_backend() == Simple proven above"); - }; - evicted_bytes += m.trim_to_window(); - } - super::strategy::BackendTag::Dfast => { - // Dfast doesn't retain input Vecs — `history` is the - // only byte store, so there is no per-block buffer - // to push back through a callback. Eviction byte - // count is derived from the `window_size` delta - // before/after; the Dfast variant of - // `trim_to_window` takes no closure, sidestepping - // an unused-`impl FnMut` monomorphization that - // would otherwise contractually never fire. - let dfast = self.dfast_matcher_mut(); - let pre = dfast.window_size; - dfast.trim_to_window(); - evicted_bytes += pre - dfast.window_size; - } - super::strategy::BackendTag::Row => { - // Row keeps bytes only in the contiguous `history` mirror - // (block buffers are returned to the pool per block in - // `add_data`), so derive the eviction count from the - // `window_size` delta, mirroring the Dfast / HashChain arms. - let row = self.row_matcher_mut(); - let pre = row.window_size; - row.trim_to_window(); - evicted_bytes += pre - row.window_size; - } - super::strategy::BackendTag::HashChain => { - // HC keeps bytes only in the contiguous `history` mirror - // (no per-block Vecs to recycle since the window<->history - // dedup), so derive the eviction count from the - // `window_size` delta, mirroring the Dfast arm above. - let table = &mut self.hc_matcher_mut().table; - let pre = table.window_size; - table.trim_to_window(); - evicted_bytes += pre - table.window_size; - } - } - if evicted_bytes == 0 { - break; - } - // The loop's invariant is "the backend's previous - // `max_window_size` shrink had downstream bytes left to - // evict" — that's what `evicted_bytes != 0` proves at - // this point. `dictionary_retained_budget` is NOT - // guaranteed to be positive here: the outer - // `retire_dictionary_budget` call may have already - // drained it to zero by reclaiming the last retained - // bytes, while the backend still has bytes above the - // freshly-shrunk window cap waiting for this loop to - // evict. The return value of the retire call below is - // therefore intentionally discarded — the loop's - // termination is driven by `evicted_bytes == 0`, not by - // whether the budget has more bytes left to reclaim. - let _ = self.retire_dictionary_budget(evicted_bytes); - } - } - - /// ATTACH (`true`) vs COPY (`false`) decision for the dms-bearing HashChain - /// backend (lazy hash-chain AND binary-tree/optimal levels), mirroring - /// upstream `ZSTD_shouldAttachDict` and its per-strategy `attachDictSizeCutoffs`: - /// a small / unknown source ATTACHES the dict as a separate dms (hash-chain - /// dms for lazy, DUBT dms for BT); a large known source COPIES it into the - /// live chain / tree. The cutoff is the lazy/lazy2 value for HC, the - /// btlazy2/btopt value for Bt{Opt}, and the smaller btultra/btultra2 value for - /// the deepest parses. Both `skip_matching_for_dictionary_priming` (which - /// stages the dict) and `prime_with_dictionary` (which builds-or-drops the - /// dms) read this so the two stay in lock-step. - fn hc_dict_attach_mode(&self) -> bool { - // Only the HashChain backend (lazy hash-chain + BT/optimal) routes here; - // a non-HashChain storage has no dms decision, so default to attach. - let MatcherStorage::HashChain(hc) = &self.storage else { - return true; - }; - let cutoff = if hc.table.uses_bt { - match hc.strategy_tag { - super::strategy::StrategyTag::BtUltra | super::strategy::StrategyTag::BtUltra2 => { - BT_ULTRA_ATTACH_DICT_CUTOFF_LOG - } - _ => BT_OPT_ATTACH_DICT_CUTOFF_LOG, - } - } else { - HC_ATTACH_DICT_CUTOFF_LOG - }; - self.reset_size_log.is_none_or(|log| log <= cutoff) - } - - fn skip_matching_for_dictionary_priming(&mut self) { - match self.active_backend() { - super::strategy::BackendTag::Simple => { - // Upstream zstd `ZSTD_shouldAttachDict` mode selection for the Fast - // strategy (cutoff 8 KB): small / unknown-size inputs ATTACH - // (index dict positions into a SEPARATE immutable table; the - // dual-probe 2-cursor `compress_block_fast_dict` then prefers - // recent-input matches and falls back to the dict — the path - // that wins small/unknown). Large known-size inputs COPY (prime - // dict into the live table; the 4-cursor `compress_block_fast` - // matches against it as window history — the path that already - // matches/beats the upstream zstd on large corpora). The dispatch in - // `start_matching` keys off `dict_table.is_some()`, which only - // the attach path populates. See [`FAST_ATTACH_DICT_CUTOFF_LOG`]. - let attach = self.reset_dict_attach_ok - && self - .reset_size_log - .is_none_or(|log| log <= FAST_ATTACH_DICT_CUTOFF_LOG); - if attach { - self.simple_mut().skip_matching_for_dict_prime(); - } else { - self.simple_mut().skip_matching_with_hint(Some(false)); - } - self.recycle_simple_space(); - } - super::strategy::BackendTag::Dfast => { - // Upstream zstd `ZSTD_dictMatchState` mode selection for dfast (cutoff - // 16 KiB): small / unknown-size inputs ATTACH (build the - // separate immutable dict long+short tables; the dual-probe - // `start_matching_fast_loop` searches live + dict, the path that - // avoids the per-frame dict re-prime that dominates small - // `compress-dict`). Larger known-size inputs COPY (re-prime the - // dict into the live tables via `skip_matching_dense`, where the - // dense scan matches it as window history). `skip_matching_for_dict_attach` - // self-gates on `use_fast_loop` (only fast-loop levels carry the - // dual-probe; general-path levels fall back to the dense copy). - let attach = self - .reset_size_log - .is_none_or(|log| log <= DFAST_ATTACH_DICT_CUTOFF_LOG); - if attach { - self.dfast_matcher_mut().skip_matching_for_dict_attach(); - } else { - self.dfast_matcher_mut().invalidate_dict_cache(); - self.dfast_matcher_mut().skip_matching_dense(); - } - } - super::strategy::BackendTag::Row => { - // Upstream zstd `ZSTD_RowFindBestMatch` `dictMatchState`: small / - // unknown-size inputs ATTACH (build the separate immutable dict - // row index; the bounded dual-probe in `row_candidate_rl` - // searches live + dict, avoiding the per-frame dict re-index), - // larger known-size inputs COPY (dense re-prime into the live - // rows). - let attach = self - .reset_size_log - .is_none_or(|log| log <= ROW_ATTACH_DICT_CUTOFF_LOG); - if attach { - self.row_matcher_mut().prime_dict_attach_current_block(); - } else { - self.row_matcher_mut().invalidate_dict_cache(); - self.row_matcher_mut().skip_matching_with_hint(Some(false)); - } - } - super::strategy::BackendTag::HashChain => { - // Lazy-HC AND BT/optimal both follow upstream zstd `ZSTD_shouldAttachDict` - // per-strategy: ATTACH (a separate dms — hash-chain dms for lazy, - // DUBT dms for BT) for small / unknown inputs, COPY (merge the dict - // into the live chain/tree) for large known inputs. ATTACH keeps - // the dict in history but out of the live structure via - // `skip_matching_dict_bt` (the cursor advance is shared by both - // arms); COPY routes through the normal `skip_matching` (its - // `uses_bt` branch fills the live tree, the lazy branch the live - // chain). The dms is built-or-dropped to match in - // `prime_with_dictionary`. - if self.hc_dict_attach_mode() { - self.hc_matcher_mut().table.skip_matching_dict_bt(); - } else { - self.hc_matcher_mut().skip_matching(Some(false)); - } - } - } - } -} - -impl Matcher for MatchGeneratorDriver { - fn supports_dictionary_priming(&self) -> bool { - true - } - - fn set_source_size_hint(&mut self, size: u64) { - self.source_size_hint = Some(size); - } - - fn set_dictionary_size_hint(&mut self, size: usize) { - self.dictionary_size_hint = Some(size); - } - - /// Dict-relevance gate for the raw-fast-path. Reached only when a dictionary - /// is active (the caller short-circuits on `dict_active`), so this answers - /// "could the dict compress this otherwise-incompressible-looking block?". - /// The Simple (Fast) backend samples its dict table precisely - /// ([`FastKernelMatcher::block_samples_match_dict`]); the other backends - /// (Dfast / Row / HashChain / BT) have their own dict structures and no cheap - /// probe here, so they answer CONSERVATIVELY `true`: without a probe they - /// cannot tell whether the dict compresses an incompressible-LOOKING block, - /// and answering `false` would let the raw-fast-path emit such a block raw - /// and miss an embedded dict segment. `dictionary_segment_in_incompressible_input_is_matched` - /// pins this for Dfast/Row/BT — the 512-byte dict run inside high-entropy - /// filler is matched only because these backends stay on the scan. So they - /// keep the blanket scan the old `!dict_active` gate gave them; only the - /// Simple/Fast backend trades it for the precise probe. - fn block_samples_match_dict(&self, block: &[u8]) -> bool { - match &self.storage { - MatcherStorage::Simple(m) => m.block_samples_match_dict(block), - _ => true, - } - } - - /// Heap bytes this driver owns: the active backend's tables/history, the - /// recycled input-buffer pool, and the primed-dictionary snapshot (a cloned - /// backend kept for CDict-equivalent reuse). The inline struct itself is - /// accounted by the owner's `size_of`. - fn heap_size(&self) -> usize { - let pool: usize = self.vec_pool.capacity() * core::mem::size_of::>() - + self.vec_pool.iter().map(Vec::capacity).sum::(); - let snapshot = self - .primed - .as_ref() - .map_or(0, |(storage, _, _)| storage.heap_size()); - pool + self.storage.heap_size() + snapshot - } - - fn clear_param_overrides(&mut self) { - self.param_overrides = None; - } - - fn reset(&mut self, level: CompressionLevel) { - let hint = self.source_size_hint.take(); - let dict_hint = self.dictionary_size_hint.take(); - // Snapshot the hint's normalized ceil-log bucket for the primed-snapshot - // key and prime_with_dictionary's attach/copy mode decision (the hint is - // consumed here, but priming happens just after reset). Storing the - // bucket rather than the raw bytes means two hints that resolve to the - // same matcher shape share one snapshot instead of each re-priming. - self.reset_size_log = hint.map(source_size_ceil_log); - // A dictionary too large for the tagged attach position field falls back - // to copy mode. Captured here (from the load-set size hint = actual dict - // length) so the prime decision and the snapshot-key / epoch bits agree. - self.reset_dict_attach_ok = - dict_hint.is_none_or(|size| size <= MAX_FAST_ATTACH_DICT_REGION); - let hinted = hint.is_some(); - #[cfg_attr(not(test), allow(unused_mut))] - let mut params = Self::level_params(level, hint); - // Test-only: apply a parse×search override so the matrix can be - // exercised without editing `LEVEL_TABLE`. Mutating `params` here - // (before `next_backend`) flows the override through storage - // selection, `configure`, and the `self.search`/`self.parse` - // writes uniformly. Consumed with `take()` so it is one-shot: the - // synthetic pairing applies to exactly this `reset()`, and a later - // reset on the same driver falls back to the level's real config. - #[cfg(test)] - if let Some((search, parse)) = self.config_override.take() { - params.search = search; - params.lazy_depth = parse.lazy_depth(); - // The matrix sweep can pair a level with a backend its native - // row doesn't populate (e.g. greedy L5, which carries only `row`, - // run on HashChain). Synthesize a default config for the - // overridden backend so its `configure` arm has something to read. - use super::strategy::SearchMethod; - match search { - SearchMethod::Fast => { - params.fast.get_or_insert(FAST_L1); - } - SearchMethod::DoubleFast => { - params.dfast.get_or_insert(DFAST_L3); - } - SearchMethod::RowHash => { - params.row.get_or_insert(ROW_CONFIG); - } - SearchMethod::HashChain | SearchMethod::BinaryTree => { - params.hc.get_or_insert(HC_CONFIG); - } - } - } - // Public-parameter overrides (#27): apply the per-knob set on top - // of the level-resolved params. A strategy override re-routes the - // backend, so this must precede `next_backend` selection. The - // all-`None` case is skipped so default level geometry stays - // byte-identical to plain level-based compression. - if let Some(ov) = self.param_overrides - && !ov.is_empty() - { - apply_param_overrides(&mut params, &ov); - // `Self::level_params(level, hint)` applied the source-size cap - // for the LEVEL's native backend. If a strategy override moved - // the frame onto a different backend, `apply_param_overrides` - // synthesized that backend's DEFAULT config (FAST_L1 / - // HC_OVERRIDE_DEFAULT) with full-size table logs AFTER that cap - // ran. Re-apply the hint cap so a tiny hinted frame doesn't - // allocate the new backend's full-size tables. An explicit - // `window_log` override is the user's hard request and must - // survive the re-cap, so restore it afterwards. - if let Some(hint_size) = hint { - params = adjust_params_for_source_size(params, hint_size); - if let Some(window_log) = ov.window_log { - params.window_log = window_log; - } - } - } - // Dictionary-driven table sizing — parity with upstream zstd `ZSTD_createCDict` - // (`ZSTD_getCParams_internal(level, UNKNOWN, dictSize, ZSTD_cpm_createCDict)` - // → `ZSTD_adjustCParams_internal`). A loaded dictionary supplies the - // long-distance matches, so upstream zstd sizes the prepared match-finder tables - // to the DICTIONARY (assuming a `minSrcSize` source), not the live - // window: it downsizes `hashLog`/`chainLog` toward the dict-and-window - // log while leaving the frame's eviction `window_log` source-derived so - // the dictionary bytes stay referenceable (`ZSTD_resetCCtx_byCopyingCDict` - // copies the small CDict tables but keeps the source window). We apply - // the same downsizing to the level's own hc geometry and cap (min) so a - // dict never inflates the level tables. Only the binary-tree / hash-chain - // backend reads `hc.{hash,chain}_log`; Simple/Dfast/Row derive their - // widths from the source window in their `reset` arms. - // A zero-length dictionary is "no dictionary": running the CDict sizing - // path for `Some(0)` is not a no-op — `cdict_table_logs(.., 0)` still - // collapses the HC/BT tables toward the 513-byte upstream zstd tier via - // `DICT_MIN_SRC_SIZE`, tanking ratio/perf on the next frame. Priming - // already treats empty content as empty, so skip the downsizing here too. - if let Some(dict_size) = dict_hint.filter(|&size| size > 0) { - // Derive the dict-tier geometry from the level's FULL (un-source-capped) - // hc widths. `Self::level_params(level, hint)` already source-capped - // `params.hc`; feeding those capped widths into `cdict_table_logs` and - // then `.min()`-ing would double-cap, so on a small hinted source with a - // large dictionary the prepared tables collapse below what the dict needs - // — defeating the `ZSTD_createCDict` geometry this mirrors. Take the - // un-hinted base widths instead and assign the result directly: - // `cdict_table_logs` only ever downsizes, so it never exceeds the base - // level geometry, while the eviction `window_log` stays source-derived so - // the dictionary bytes remain referenceable. Active public-parameter - // overrides (#27) are applied to the base too, so a strategy override - // that routes onto HashChain/BinaryTree still gets dict-tier sizing and - // explicit hash/chain overrides feed through as the geometry ceiling. - let mut base_params = Self::level_params(level, None); - if let Some(ov) = self.param_overrides - && !ov.is_empty() - { - apply_param_overrides(&mut base_params, &ov); - } - if let (Some(hc), Some(base_hc)) = (params.hc.as_mut(), base_params.hc) { - let uses_bt = matches!( - params.strategy_tag, - super::strategy::StrategyTag::Btlazy2 - | super::strategy::StrategyTag::BtOpt - | super::strategy::StrategyTag::BtUltra - | super::strategy::StrategyTag::BtUltra2 - ); - let (dict_hash_log, dict_chain_log) = cdict_table_logs( - params.window_log, - base_hc.hash_log, - base_hc.chain_log, - uses_bt, - dict_size, - ); - hc.hash_log = dict_hash_log; - hc.chain_log = dict_chain_log; - } - } - // upstream zstd `ZSTD_resolveRowMatchFinderMode` (zstd_compress.c:238-245): - // the row matchfinder is used for greedy/lazy/lazy2 ONLY when - // `windowLog > 14`; at or below that upstream runs the hash-chain - // matcher (`ZSTD_HcFindBestMatch`). We previously hardcoded the Row - // backend for these strategies regardless of window, sending every - // small-window frame (hinted floor = windowLog 14, e.g. the small-4k/10k - // fixtures) through Row where upstream uses HC. Match it: fall back to - // the hash-chain matcher (lazy/greedy parse via `lazy_depth`) when the - // resolved window is <= 14. The HC config is synthesised from the - // level's RowConfig (HC and Row share the same cParams; only the - // matchfinder differs) — `hash_log` / `chain_log` are - // clamped to the (<= 14) window inside the HashChain reset arm, so the - // nominal width here only sets the clamp ceiling. - if params.search == super::strategy::SearchMethod::RowHash && params.window_log <= 14 { - let row = params - .row - .expect("a RowHash level row must carry a RowConfig"); - params.search = super::strategy::SearchMethod::HashChain; - // For a dict-bearing frame, downsize the synthesised HC logs to the - // dictionary's content tier via `cdict_table_logs` (the same - // correction the native HC dict-prime path applies above), so a dict - // much smaller than the window doesn't prime a needlessly sparse - // table. Row-finder levels are never BinaryTree, so `uses_bt = false`. - // - // Feed `cdict_table_logs` the UN-hinted base Row width, not the - // resolved `row.hash_bits`: the latter is already source-capped on a - // hinted reset (the `row_cap = table_log + 1` clamp), so passing it - // here would double-cap exactly as the native HC dict path warns - // above — a small hinted source with a large dictionary would - // collapse the prepared table below what the dict needs. - // `cdict_table_logs` only ever downsizes, so deriving the ceiling - // from the un-hinted base (plus active public overrides) keeps the - // dict-tier geometry intact. No source hint => `row.hash_bits` is - // already the level's full width, so reuse it directly. - let row_cdict_hash_bits = match dict_hint.filter(|&size| size > 0) { - Some(_) => { - let mut base_params = Self::level_params(level, None); - if let Some(ov) = self.param_overrides - && !ov.is_empty() - { - apply_param_overrides(&mut base_params, &ov); - } - base_params - .row - .map_or(row.hash_bits, |base_row| base_row.hash_bits) - } - None => row.hash_bits, - }; - // Row-backed levels carry only `hash_bits`; the HC chain table they - // fall back to follows the upstream zstd cParams relationship `chainLog = - // hashLog - 1` for every Row level (L6 c18 h19 .. L12 c22 h23, see - // the ROW_L* tables). Synthesise the chain width as `hash_bits - 1` - // so the dict path doesn't leave the chain table one bit too wide - // (cdict_table_logs only downsizes, so passing the full hash width - // for both would keep a 2x-too-large chain table on dict frames). - // Raw `- 1` is underflow-safe: `hash_bits` is either a predefined - // ROW_L* width (>= 19) or a public `hash_log` override, and the - // override is range-validated to `ZSTD_HASHLOG_MIN = 6` at the - // parameter API, so the value is always >= 6 here. - // - // A public `chain_log` override (#27) is dropped by the RowHash - // override arm (Row has no chain table), but once this frame falls - // back to HC the chain table is live and must honour it — mirror - // the native HC dict path, which feeds the override-applied - // `base_hc.chain_log` into `cdict_table_logs`. Use the explicit - // override (also API-validated to ZSTD_CHAINLOG_MIN = 6) when set, - // else the upstream zstd `hashLog - 1` relationship. - let explicit_chain_log = self - .param_overrides - .filter(|ov| !ov.is_empty()) - .and_then(|ov| ov.chain_log) - .map(|chain_log| chain_log as usize); - let row_cdict_chain_bits = explicit_chain_log.unwrap_or(row_cdict_hash_bits - 1); - let (mut hash_log, mut chain_log) = match dict_hint.filter(|&size| size > 0) { - Some(dict_size) => cdict_table_logs( - params.window_log, - row_cdict_hash_bits, - row_cdict_chain_bits, - false, - dict_size, - ), - None => ( - row.hash_bits, - explicit_chain_log.unwrap_or(row.hash_bits - 1), - ), - }; - // No-dict path: the HashChain reset arm only clamps the logs to the - // window when `hinted`, but a public `window_log` override can lower - // this level to <= 14 with no source hint — clamp the level's full - // Row `hash_bits` to the window here too (upstream zstd `ZSTD_adjustCParams`: - // hashLog <= windowLog + 1, chainLog <= windowLog) so a 16 KiB window - // doesn't allocate Row-sized HC tables. - if dict_hint.filter(|&size| size > 0).is_none() { - let wlog = params.window_log as usize; - hash_log = hash_log.min(wlog + 1); - chain_log = chain_log.min(wlog); - } - params.hc = Some(HcConfig { - hash_log, - chain_log, - search_depth: row.search_depth, - target_len: row.target_len, - search_mls: 4, - }); - params.row = None; - } - let next_backend = params.backend(); - let max_window_size = 1usize << params.window_log; - self.dictionary_retained_budget = 0; - // Drop any frame-local borrowed staging so it can't leak across a - // reset and misroute the next start/skip into borrowed dispatch. - self.borrowed_pending = None; - if self.active_backend() != next_backend { - // Drain the outgoing backend's allocations into the shared - // pool. The `match &mut self.storage { ... }` block runs to - // completion before the assignment below replaces the - // variant, so the inner state we just drained is dropped - // with the old variant. - match &mut self.storage { - MatcherStorage::Simple(_m) => { - // FastKernelMatcher owns a flat Vec history - // and a Vec hash table — both drop with the - // variant assignment below, no per-block buffers - // to recycle into the driver pools. The - // assignment-replace path collapses to a noop - // pre-pass for this backend. - } - MatcherStorage::Dfast(m) => { - // Drop the long / short hash table allocations - // before calling `m.reset`. Without this prepass, - // `DfastMatchGenerator::reset` would `fill` both - // tables with `DFAST_EMPTY_SLOT` sentinels — wasted - // work given the next assignment to `self.storage` - // is about to drop `m` entirely. `reset` itself - // short-circuits on `if !self.tables.is_empty()`, so - // handing it an empty `Vec` skips the fill loop. - // Mirrors the pre-drain pattern in the HashChain - // arm below (and serves the same peak-memory - // purpose: release the table-allocation footprint - // before constructing the replacement variant). - m.tables = Vec::new(); - m.reset(); - } - MatcherStorage::Row(m) => { - m.row_heads = Vec::new(); - m.row_positions = Vec::new(); - m.row_tags = Vec::new(); - m.reset(); - } - MatcherStorage::HashChain(m) => { - // Release oversized tables when switching away from - // HashChain so Best's larger allocations don't persist. - // hash3_table must be released alongside the other - // two: BtUltra2's `1 << HC3_HASH_LOG` entries would - // otherwise stay pinned across the backend switch, - // even though no future caller of this backend will - // touch them. - m.table.hash_table = Vec::new(); - m.table.chain_table = Vec::new(); - m.table.hash3_table = Vec::new(); - let vec_pool = &mut self.vec_pool; - m.reset(|mut data| { - data.resize(data.capacity(), 0); - vec_pool.push(data); - }); - } - } - // Swap in a fresh variant for the new backend. The previous - // `storage` is dropped here. - self.storage = match next_backend { - super::strategy::BackendTag::Simple => { - // Per-level Fast cParams from resolve_level_params: - // Level(1) gets (hash_log=14, mls=7); Level(-7..=-1) - // get upstream zstd row-0 (hash_log=13, mls=7); Fastest / - // Uncompressed keep (hash_log=14, mls=6). See - // resolve_level_params for rationale. - let fast = params.fast.expect("Fast level row carries a FastConfig"); - MatcherStorage::Simple(FastKernelMatcher::with_params( - params.window_log, - fast.hash_log, - fast.mls, - fast.step_size, - )) - } - super::strategy::BackendTag::Dfast => { - MatcherStorage::Dfast(DfastMatchGenerator::new(max_window_size)) - } - super::strategy::BackendTag::Row => { - MatcherStorage::Row(RowMatchGenerator::new(max_window_size)) - } - super::strategy::BackendTag::HashChain => { - MatcherStorage::HashChain(HcMatchGenerator::new(max_window_size)) - } - }; - } - - // Single source of truth: `LevelParams::strategy_tag` is the - // authoritative mapping from `CompressionLevel` to strategy. - // `storage.backend()` derives the parse family from the variant, - // so there is no separate runtime tag that could drift against - // `LEVEL_TABLE`. - self.strategy_tag = params.strategy_tag; - self.search = params.search; - self.parse = params.parse(); - self.slice_size = self.base_slice_size.min(max_window_size); - self.reported_window_size = max_window_size; - let strategy_tag = self.strategy_tag; - // Source-proportional table window for the backends whose hash-table - // widths are recomputed here (Dfast / Row). Like the HC / Fast caps - // in `adjust_params_for_source_size`, this sizes the internal tables - // from the RAW source log (not the wire `window_log` floor) so a - // small frame zeroes a small table; it never exceeds the real window. - let table_window_size = match hint { - Some(h) => { - let raw_log = source_size_ceil_log(h); - // Clamp the shift below the pointer width before `1usize <<`: - // an oversized hint (>= 2^63 + 1, and on 32-bit usize any hint - // >= 2^32) drives `raw_log` to 64 / >= 32, and the shift would - // overflow (panic in debug, wrap to 0 in release) before the - // `.min(max_window_size)` cap below could bound it. The min cap - // still provides the real semantic window bound. - let shift = raw_log.max(MIN_WINDOW_LOG).min(usize::BITS as u8 - 1); - (1usize << shift).min(max_window_size) - } - None => max_window_size, - }; - // The hint-dependent hash-table width the active backend applies, for - // the primed-snapshot key. Dfast/Row compute it from `table_window_size` - // below; HC/Fast leave it `0` because their widths live in `params` - // (`hc.{hash,chain}_log` / `fast_hash_log`) — already part of the key. - let mut resolved_table_bits: usize = 0; - match &mut self.storage { - MatcherStorage::Simple(m) => { - // Per-level Fast cParams threaded from - // resolve_level_params (see Simple-backend swap - // arm above for the (level → params) mapping). - let fast = params.fast.expect("Fast level row carries a FastConfig"); - // Same attach/copy split the dict-prime dispatch applies - // below (`prime_with_dictionary`): only attach-mode dict - // frames may keep the main table across the reset via an - // epoch advance — copy-mode and no-dict frames must memset - // it back to bias 0 for the raw-slice kernels. - // `Some(0)` is "no dictionary" (the dict-sizing path above - // filters it the same way): an empty dict primes nothing, so - // an epoch-advance reset would preserve stale attach state - // instead of clearing it. - let dict_attach_epoch = matches!(dict_hint, Some(size) if size > 0) - && self.reset_dict_attach_ok - && self - .reset_size_log - .is_none_or(|log| log <= FAST_ATTACH_DICT_CUTOFF_LOG); - // Copy-mode dictionary frame whose primed snapshot matches - // this exact resolved shape: `restore_primed_dictionary` - // (called right after this reset; the caller gates the - // restore on the same size bucket and the restore re-checks - // the same key) will `clone_from` the snapshot over this - // matcher, replacing the table contents and bias wholesale — - // the reset's full-table memset would be thrown away. The - // key components mirror `reset_shape` below: Simple leaves - // `resolved_table_bits` 0, never carries an LDM override, - // and `fast_attach` is false in copy mode by construction. - let table_overwritten_by_restore = matches!(dict_hint, Some(size) if size > 0) - && !dict_attach_epoch - && self.primed.as_ref().is_some_and(|(_, _, captured)| { - *captured - == PrimedKey { - level, - params, - table_bits: 0, - fast_attach: false, - ldm: None, - } - }); - // Cap `hash_log <= window_log + 1` (upstream zstd - // `ZSTD_adjustCParams_internal`): once `window_log` is resized - // down for a small source, a level-default `1 << hash_log` - // table is mostly wasted address space whose per-frame memset - // dominates the compress cost on tiny frames (a 4 KB frame at - // window_log 12 still zero-fills the 64 KiB hash_log-14 table). - // Gated to no-dict frames: the dict-attach path shares one - // hash_log between the main and dict tables (so one hash keys - // both), and shrinking only the main table would break that - // invariant and the small-frame dict ratio. - let hash_log = if dict_hint.is_some_and(|s| s > 0) { - fast.hash_log - } else { - fast.hash_log.min(params.window_log as u32 + 1) - }; - m.reset( - params.window_log, - hash_log, - fast.mls, - fast.step_size, - dict_attach_epoch, - table_overwritten_by_restore, - ); - } - MatcherStorage::Dfast(dfast) => { - dfast.max_window_size = max_window_size; - let dcfg = params - .dfast - .expect("Dfast level row must carry a DfastConfig"); - // Upstream zstd `cParams.hashLog`/`chainLog`, capped by the - // source-size window when hinted so tiny inputs don't - // over-allocate. - let long_bits = if hinted { - dfast_hash_bits_for_window(table_window_size).min(dcfg.long_hash_log as usize) - } else { - dcfg.long_hash_log as usize - }; - let short_bits = if hinted { - dfast_hash_bits_for_window(table_window_size).min(dcfg.short_hash_log as usize) - } else { - dcfg.short_hash_log as usize - }; - resolved_table_bits = long_bits; - dfast.set_hash_bits(long_bits, short_bits); - // Dfast holds no per-block input Vecs (history owns the - // bytes and `add_data` returns each Vec eagerly), so - // `reset` takes no `reuse_space` callback. - dfast.reset(); - } - MatcherStorage::Row(row) => { - row.max_window_size = max_window_size; - row.lazy_depth = params.lazy_depth; - let mut row_cfg = params.row.expect("Row level row carries a RowConfig"); - if hinted { - // Clamp the configured hash width by the hinted window - // (upstream zstd `ZSTD_adjustCParams` caps hashLog by windowLog) — - // `min`, not replace, so an explicit `hash_log` param - // override (`row_cfg.hash_bits`) survives the hinted path - // instead of being overwritten by the window value. - // - // Clamp BEFORE `configure` so the backend sees ONE width - // per frame. Configuring with the unclamped level width - // and then re-clamping made `row_hash_log` oscillate on - // every hinted frame, and each width change clears the - // row tables — `ensure_tables` then re-filled all three - // every frame in a reused compressor. - row_cfg.hash_bits = row_cfg - .hash_bits - .min(row_hash_bits_for_window(table_window_size)); - } - row.configure(row_cfg); - // Key the primed snapshot on the width the backend ACTUALLY - // applied (`set_hash_bits` clamps the request): recording the - // request — or the 0 default on the unhinted path — keys - // identical table geometries apart and forces needless - // dictionary re-primes. - resolved_table_bits = row.hash_bits(); - row.reset(); - } - MatcherStorage::HashChain(hc) => { - hc.table.max_window_size = max_window_size; - hc.hc.lazy_depth = params.lazy_depth; - let mut hc_cfg = params.hc.expect("HashChain level row carries an HcConfig"); - // Cap the hash / chain table logs by the hinted window so a small - // input doesn't allocate the full level's tables (the upstream zstd - // `ZSTD_adjustCParams_internal` clamp: `hashLog <= windowLog + 1`, - // and `cycleLog <= windowLog` — `cycleLog == chainLog` for the HC - // finder, `chainLog - 1` for the BT pair table, so `chainLog <= - // windowLog` (+1 for BT)). Ratio-neutral: a hinted window of - // `2^wlog` bytes holds at most `2^wlog` positions, so the slots - // beyond that are never populated — capping only sheds unused - // allocation. Was the source of L10-lazy peak-alloc ~2.15x the - // upstream zstd on a 1 MiB input. Only applied when hinted; an - // unknown-size stream keeps the full level tables. - // Skip for dict-bearing frames: their `hc_cfg.{hash,chain}_log` - // were already sized to the dictionary content tier via - // `cdict_table_logs` (the dict supplies the long-distance - // matches, so upstream `ZSTD_createCDict` sizes the prepared - // tables to the dict, not the source window). Re-applying the - // source-window cap here would collapse those dict-tier logs - // back to the small hinted source — the same double-cap the - // synthesis sites avoid by using the un-hinted base width. - if hinted && !matches!(dict_hint, Some(size) if size > 0) { - let wlog = hc_hash_bits_for_window(table_window_size); - let uses_bt = matches!( - strategy_tag, - super::strategy::StrategyTag::Btlazy2 - | super::strategy::StrategyTag::BtOpt - | super::strategy::StrategyTag::BtUltra - | super::strategy::StrategyTag::BtUltra2 - ); - hc_cfg.hash_log = hc_cfg.hash_log.min(wlog + 1); - hc_cfg.chain_log = hc_cfg.chain_log.min(if uses_bt { wlog + 1 } else { wlog }); - } - hc.configure(hc_cfg, strategy_tag, params.window_log); - let vec_pool = &mut self.vec_pool; - hc.reset(|mut data| { - data.resize(data.capacity(), 0); - vec_pool.push(data); - }); - // When the source size is known, pre-size the history mirror to - // the expected total (dictionary + payload) so per-block growth - // does not overshoot via Vec capacity doubling (upstream zstd sizes its - // window buffer exactly). Dominates peak once the match-finder - // tables are dictionary-tier-small. Unhinted streams skip this - // and keep doubling growth. - if let Some(src) = hint { - // `src` is a u64 hint and may be the u64::MAX "unknown - // size" sentinel, which truncates under `as usize` on - // 32-bit targets and overflows when the dict hint is - // added. Saturate the source size, then saturate the - // dict-hint addition; `reserve_history` applies the - // tighter window ceiling to the result. - let src_hint = usize::try_from(src).unwrap_or(usize::MAX); - let expected = src_hint.saturating_add(dict_hint.unwrap_or(0)); - hc.table.reserve_history(expected); - } - } - } - // LDM wiring (#27): attach (or clear) the long-distance-match - // producer on the optimal (BT) backend. LDM is the only - // back-reference path that crosses the regular window, so it - // only has a home on the `BtMatcher`; non-BT strategies drop the - // producer. Built AFTER `hc.reset()` because `BtMatcher::reset` - // clears an existing producer's table but does not null the - // slot — installing here gives the new frame a fresh producer. - #[cfg(feature = "hash")] - { - // Resolve the derived LDM params first (immutable borrow of the - // overrides), then reuse the existing producer's allocation below. - let derived_ldm = self - .param_overrides - .as_ref() - .and_then(|ov| ov.ldm) - .map(|ldm_ov| { - let strategy_ord = ldm_strategy_ordinal(params.strategy_tag, params.lazy_depth); - // Seed the caller-pinned knobs, then run the upstream zstd - // derivation over the seed so the remaining (zero) - // fields are filled with cross-field consistency - // (e.g. `hash_rate_log = window_log - hash_log`). - // Clobbering after `adjust_for` would break that and - // hand the producer an inconsistent set. - let seed = super::ldm::params::LdmParams { - window_log: params.window_log as u32, - hash_log: ldm_ov.hash_log.unwrap_or(0), - hash_rate_log: ldm_ov.hash_rate_log.unwrap_or(0), - min_match_length: ldm_ov.min_match.unwrap_or(0), - bucket_size_log: ldm_ov.bucket_size_log.unwrap_or(0), - }; - seed.derive(strategy_ord) - }); - if let MatcherStorage::HashChain(hc) = &mut self.storage { - // Reuse the existing producer's hash-table allocation when the - // derived params are unchanged: only `clear()` (re-zero the - // table + re-seed the rolling hash, no allocation) is needed for - // the new frame. A params change (or the first frame) forces a - // fresh `LdmProducer::new`. On the reused-encoder compress-dict - // path this avoids re-allocating the LDM hash table (large at - // btultra2) every frame — upstream zstd reuses its `ldmState_t` - // the same way. `clear()` is mandatory here for correctness - // regardless of what `BtMatcher::reset` did to the old table. - let producer = derived_ldm.map(|p| match hc.take_ldm_producer() { - Some(mut existing) if existing.params() == p => { - existing.clear(); - existing - } - _ => super::ldm::LdmProducer::new(p), - }); - hc.set_ldm_producer(producer); - } - } - // Record the resolved matcher shape for the primed-snapshot key. Captured - // here (post-resolution, after the test-only param override) so the key - // reflects exactly the geometry the restored `storage` must match. The - // Fast attach-vs-copy mode is part of the shape ONLY for the Simple - // backend (it decides the distinct dict-table shape that backend builds). - // Dfast/Row/HashChain have their OWN attach/copy regimes, but this bit - // models only the Fast table split; those backends are keyed by the - // resolved matcher geometry instead, so folding the Fast bit into their - // key would over-key identical resolved shapes. When it applies it - // matches the decision `prime_with_dictionary` makes from the same - // `reset_size_log`. - let fast_attach = matches!(next_backend, super::strategy::BackendTag::Simple) - && self.reset_dict_attach_ok - && self - .reset_size_log - .is_none_or(|log| log <= FAST_ATTACH_DICT_CUTOFF_LOG); - // The LDM override is part of the snapshot identity ONLY on the - // optimal (BinaryTree) path: that is the only backend whose cloned - // `storage` carries a `BtMatcher::ldm_producer`. On Fast / Dfast / - // Row and lazy-HashChain resets the producer slot does not exist, - // so folding the override there would over-key the snapshot and - // force needless re-primes when LDM is toggled. Gated like - // `fast_attach` (a key bit only participates where it changes the - // cloned matcher shape). - let active_ldm = if matches!(params.search, super::strategy::SearchMethod::BinaryTree) { - self.param_overrides.and_then(|ov| ov.ldm) - } else { - None - }; - self.reset_shape = Some((params, resolved_table_bits, fast_attach, active_ldm)); - } - - fn dictionary_is_resident(&self) -> bool { - match &self.storage { - MatcherStorage::HashChain(hc) => hc.table.dict_resident, - MatcherStorage::Simple(s) => s.dict_resident(), - MatcherStorage::Dfast(d) => d.dict_resident(), - _ => false, - } - } - - fn reapply_resident_dictionary(&mut self, offset_hist: [u32; 3]) { - // Same offset-history head as `prime_with_dictionary`, without the dict - // commit / re-index (resident dict bytes + cached dms already in place). - match self.active_backend() { - super::strategy::BackendTag::Simple => { - self.simple_mut().prime_offset_history(offset_hist) - } - super::strategy::BackendTag::Dfast => { - self.dfast_matcher_mut().offset_hist = offset_hist - } - super::strategy::BackendTag::Row => self.row_matcher_mut().offset_hist = offset_hist, - super::strategy::BackendTag::HashChain => { - let matcher = self.hc_matcher_mut(); - matcher.table.offset_hist = offset_hist; - matcher.table.mark_dictionary_primed(); - } - } - // Restore the retained-dictionary budget the per-frame `reset` cleared. - // The matcher's `reset` re-inflated `max_window_size` by the resident - // dict region (so the dict + next input both stay in the eviction band), - // exactly as `prime_with_dictionary` does — but the resident path skips - // that prime, so without this the driver-level budget stays 0 and - // `retire_dictionary_budget` never shrinks the inflated window as input - // evicts the dict. For HashChain (whose `window_low` is measured against - // `max_window_size`), a stuck-inflated window would let a post-eviction - // match exceed the frame header's base window and emit an over-window - // offset. The inflation equals `max_window_size - base`, and - // `reported_window_size` is the base `1 << window_log` set by `reset`. - let base = self.reported_window_size; - let inflated = match self.active_backend() { - super::strategy::BackendTag::Simple => self.simple_mut().max_window_size, - super::strategy::BackendTag::Dfast => self.dfast_matcher_mut().max_window_size, - super::strategy::BackendTag::Row => self.row_matcher_mut().max_window_size, - super::strategy::BackendTag::HashChain => self.hc_matcher_mut().table.max_window_size, - }; - self.dictionary_retained_budget = inflated.saturating_sub(base); - } - - fn prime_with_dictionary(&mut self, dict_content: &[u8], offset_hist: [u32; 3]) { - match self.active_backend() { - super::strategy::BackendTag::Simple => { - // Routes through prime_offset_history so BOTH - // offset_hist (wire encoder) and rep[0..2] (kernel) - // are updated atomically. Without this, the two - // tracks drift after dict priming — kernel emits - // repcode matches against stale FAST_INITIAL_REP - // while the wire encoder uses the primed history, - // producing divergent wire encoding (Copilot review - // #15 on #216). - self.simple_mut().prime_offset_history(offset_hist); - } - super::strategy::BackendTag::Dfast => { - self.dfast_matcher_mut().offset_hist = offset_hist - } - super::strategy::BackendTag::Row => self.row_matcher_mut().offset_hist = offset_hist, - super::strategy::BackendTag::HashChain => { - let matcher = self.hc_matcher_mut(); - // Clear the chain/hash tables (deferred from the dict-active - // `reset`): prime rebuilds them from the dict, so they must start - // empty. The reuse hot path skips prime and `clone_from`s a clean - // snapshot instead, so only the first-prime / key-mismatch frames - // pay the fill -- not every reused-CDict frame. - matcher.table.clear_chain_hash_tables(); - matcher.table.offset_hist = offset_hist; - matcher.table.mark_dictionary_primed(); - } - } - - if dict_content.is_empty() { - return; - } - - // Dictionary bytes should stay addressable until produced frame output - // itself exceeds the live window size. We bump `max_window_size` - // by the dictionary length so the eviction band keeps the - // primed bytes in `history`. - // - // Cap: `with_params`/`reset` enforce `window_log <= 30` so the - // eviction band `2 * max_window_size` stays below `u32::MAX` - // with headroom for one MAX_BLOCK_SIZE pending block — the - // kernel asserts `data.len() <= u32::MAX`. A large enough - // dictionary could otherwise push `max_window_size` past - // that ceiling via the `saturating_add` below and silently - // re-introduce the same overflow the `window_log` cap was - // designed to prevent. Clamp the post-priming size so the - // doubled-band-plus-block invariant survives. - use super::match_table::storage::MAX_PRIMED_WINDOW_SIZE; - - // `requested_dict_budget` is what the caller asked for; - // `base_max_window_size` snapshots the pre-priming cap so we - // can compute how much window the cap actually GRANTED below. - // The cap may clip the requested growth, in which case the - // bookkeeping (`dictionary_retained_budget` retire path) must - // track only the granted portion — otherwise - // `retire_dictionary_budget()` would later reclaim more than - // was actually added and shrink the matcher below its real - // base window (and `cap = 2 * max_window_size` would shrink - // with it, risking under-allocation on subsequent commits). - // The `granted_retained_budget` calculation further below is - // the load-bearing piece — see its block-level comment for - // the post-clip / post-uncommitted-tail math. - let requested_dict_budget = dict_content.len(); - let base_max_window_size = match self.active_backend() { - super::strategy::BackendTag::Simple => self.simple_mut().max_window_size, - super::strategy::BackendTag::Dfast => self.dfast_matcher_mut().max_window_size, - super::strategy::BackendTag::Row => self.row_matcher_mut().max_window_size, - super::strategy::BackendTag::HashChain => self.hc_matcher_mut().table.max_window_size, - }; - match self.active_backend() { - super::strategy::BackendTag::Simple => { - let matcher = self.simple_mut(); - matcher.max_window_size = matcher - .max_window_size - .saturating_add(requested_dict_budget) - .min(MAX_PRIMED_WINDOW_SIZE); - } - super::strategy::BackendTag::Dfast => { - let matcher = self.dfast_matcher_mut(); - matcher.max_window_size = matcher - .max_window_size - .saturating_add(requested_dict_budget) - .min(MAX_PRIMED_WINDOW_SIZE); - } - super::strategy::BackendTag::Row => { - let matcher = self.row_matcher_mut(); - matcher.max_window_size = matcher - .max_window_size - .saturating_add(requested_dict_budget) - .min(MAX_PRIMED_WINDOW_SIZE); - } - super::strategy::BackendTag::HashChain => { - let matcher = self.hc_matcher_mut(); - matcher.table.max_window_size = matcher - .table - .max_window_size - .saturating_add(requested_dict_budget) - .min(MAX_PRIMED_WINDOW_SIZE); - } - } - - let mut start = 0usize; - let mut committed_dict_budget = 0usize; - // insert_position needs 4 bytes of lookahead for hashing; - // backfill_boundary_positions re-visits tail positions once the - // next slice extends history, but cannot hash <4 byte fragments. - let min_primed_tail = match self.active_backend() { - super::strategy::BackendTag::Simple => MIN_MATCH_LEN, - super::strategy::BackendTag::Dfast - | super::strategy::BackendTag::Row - | super::strategy::BackendTag::HashChain => 4, - }; - while start < dict_content.len() { - let end = (start + self.slice_size).min(dict_content.len()); - if end - start < min_primed_tail { - break; - } - // Stage the dict chunk WITHOUT `get_next_space`'s - // `resize(slice_size, 0)` zero-fill: that memsets a full - // block-sized buffer (up to ~128 KiB) every frame only to have it - // `clear()`-ed and overwritten by the dict bytes on the very next - // lines — pure waste (measured ~10% of the small dict encode). - // Reuse a pooled buffer's capacity if one is free (the prime/skip - // cycle recycles them back), else allocate exactly the chunk. - // Mirrors upstream zstd, which references the CDict content rather - // than zero-filling a fresh window per frame. - let mut space = self.vec_pool.pop().unwrap_or_default(); - space.clear(); - space.extend_from_slice(&dict_content[start..end]); - self.commit_space(space); - self.skip_matching_for_dictionary_priming(); - committed_dict_budget += end - start; - start = end; - } - - // Derive `granted_retained_budget` directly from the two real - // bounds — bytes actually committed and bytes the cap allows - // — instead of doing a cap-clip pass followed by an - // uncommitted-tail subtract. Previous shape double-discounted - // when the cap clipped: clip lost `(requested - allowed)`, - // then tail-subtract lost ANOTHER `(requested - committed)`, - // leaving `max_window_size` shy of the dictionary that was - // actually retained (e.g. cap=900, committed=998, uncommitted=2 - // landed at granted=898 instead of the correct 900). - let capped_retained_budget = MAX_PRIMED_WINDOW_SIZE.saturating_sub(base_max_window_size); - let granted_retained_budget = committed_dict_budget.min(capped_retained_budget); - let final_max_window_size = base_max_window_size.saturating_add(granted_retained_budget); - match self.active_backend() { - super::strategy::BackendTag::Simple => { - self.simple_mut().max_window_size = final_max_window_size; - } - super::strategy::BackendTag::Dfast => { - self.dfast_matcher_mut().max_window_size = final_max_window_size; - } - super::strategy::BackendTag::Row => { - self.row_matcher_mut().max_window_size = final_max_window_size; - } - super::strategy::BackendTag::HashChain => { - self.hc_matcher_mut().table.max_window_size = final_max_window_size; - } - } - if granted_retained_budget > 0 { - self.dictionary_retained_budget = self - .dictionary_retained_budget - .saturating_add(granted_retained_budget); - } - if self.active_backend() == super::strategy::BackendTag::HashChain { - // Recompute the lazy-HC attach decision made per-chunk in - // `skip_matching_for_dictionary_priming` (stable across the prime — - // `reset_size_log` does not change here). - // - // The HC attach/copy mode is deliberately NOT folded into `PrimedKey` - // (unlike Fast `fast_attach`). Fast attach builds a separate dict - // table whose dimensions differ from the copy-mode live table, so a - // cross-mode restore would install mismatched table geometry and the - // encoder could search past the frame window (undecodable). The two - // HC modes share identical window geometry: `max_window_size` and the - // dictionary limit are both set ABOVE this branch (the same value in - // either mode), and the live chain table dimensions come from the - // resolved `params` the key already pins. The modes differ only in - // WHERE the committed dict lives — a single-link `dms` (attach) vs - // merged into the live chain (copy) — both producing valid matches at - // in-window offsets. Upstream zstd makes the same observation: attach - // (`ZSTD_resetCCtx_byAttachingCDict`) and copy - // (`ZSTD_resetCCtx_byCopyingCDict`) both keep the caller's - // `windowLog`; the choice is a memory/speed trade-off, not a wire - // contract. So restoring an attach snapshot where this frame would - // have copied (or vice versa) yields a decodable frame that may only - // differ in which matches are found (ratio) — algorithmic freedom, not - // a defect. Keying on the mode would instead force a re-prime across - // the cutoff, re-adding the per-frame cost this snapshot path removes. - // - // In practice the public reuse path (`compress_independent_frame`) - // only ever captures AND restores the COPY-mode snapshot — capture is - // gated on the above-cutoff source size, so a restored frame always - // matches the captured mode. `hc_dict_snapshot_reuse_roundtrips` pins - // that same-mode reuse decodes; the driver-level cross-mode restore is - // accepted (not refused) per - // `primed_snapshot_fast_attach_does_not_over_key_non_simple_backends`. - let attach = self.hc_dict_attach_mode(); - let table = &mut self.hc_matcher_mut().table; - table.set_dictionary_limit_from_primed_bytes(committed_dict_budget); - // Build the dictMatchState over the committed dict (front of history) - // so `find_best_match` dual-probes it with its own compare budget — - // but ONLY in ATTACH mode. BT/optimal attach → DUBT dms; lazy-HC - // attach → single-link hash-chain dms. COPY mode (large known source, - // both BT and lazy-HC) already merged the dict into the live tree / - // chain in `skip_matching_for_dictionary_priming`, so it carries no - // separate dms — drop any stale one. - if !attach { - table.dms.invalidate(); - } else if table.uses_bt { - table.prime_dms_bt(committed_dict_budget); - } else { - table.prime_dms_hc(committed_dict_budget); - } - } - // CDict-equivalent: now that every dict chunk is indexed, mark the - // Fast-backend dict table primed so the next frame's re-prime reuses - // it (skips the re-hash) while still re-committing the dict bytes to - // history. No-op when the attach path built no table (copy mode or a - // sub-8-byte dict) — `mark_dict_primed` self-guards on table presence. - match self.active_backend() { - super::strategy::BackendTag::Simple => self.simple_mut().mark_dict_primed(), - super::strategy::BackendTag::Dfast => self.dfast_matcher_mut().mark_dict_primed(), - super::strategy::BackendTag::Row => self.row_matcher_mut().mark_dict_primed(), - _ => {} - } - } - - fn restore_primed_dictionary(&mut self, level: super::CompressionLevel) -> bool { - // Only the (storage, dictionary_retained_budget) pair is what - // `prime_with_dictionary` writes; restoring them reproduces the - // post-prime state exactly. Gated on the FULL resolved key (level + the - // resolved `LevelParams` + the active backend's table width), not just - // the level: `reset` resolves the hint into a window/table geometry, so a - // same-level snapshot taken at a hint that resolved to a different shape - // carries a `storage.max_window_size` / table dimensions that no longer - // match this reset. Restoring it would let the encoder search past the - // frame header's window (an undecodable match), so on a key mismatch we - // refuse and the caller re-primes. - let Some((params, table_bits, fast_attach, ldm)) = self.reset_shape else { - return false; - }; - let key = PrimedKey { - level, - params, - table_bits, - fast_attach, - ldm, - }; - let Some((snapshot, budget, captured_key)) = &self.primed else { - return false; - }; - if *captured_key != key { - return false; - } - let budget = *budget; - match (&mut self.storage, snapshot) { - // Same-variant Fast restore: copy the snapshot into the retained - // live storage. `clone_from` reuses the history / hash-table / - // dict-table buffers, so this is the upstream zstd CDict table-copy - // regime's cost (pure copies) instead of a full per-frame - // allocation + copy + drop cycle. - (MatcherStorage::Simple(live), MatcherStorage::Simple(snap)) => { - live.clone_from(snap); - } - // Same-variant HC lazy/greedy restore (non-BT): the snapshot keeps - // the full primed hash/chain tables (capture's non-BT full clone), - // so `clone_from` reuses the live history/hash/chain/dms buffers in - // place — upstream zstd reuses the CDict tables rather than reallocating - // them. This is the per-frame allocate+copy+drop that dominated - // small `compress-dict` HC frames (5-7x vs C). BT (`uses_bt`) - // snapshots drop their live tables, so they stay on the realloc - // path below. - (MatcherStorage::HashChain(live), MatcherStorage::HashChain(snap)) - if !snap.table.uses_bt => - { - live.table.clone_from(&snap.table); - live.hc.clone_from(&snap.hc); - live.strategy_tag = snap.strategy_tag; - // backend is `HcBackend::Hc` (zero-sized) for non-BT levels; - // the live one is already correct for this resolved key. - } - (live, snapshot_storage) => { - let mut storage = snapshot_storage.clone(); - // This arm handles the binary-tree backend. In ATTACH mode the - // snapshot was stored WITHOUT its live hash / chain / hash3 - // tables (they hold no dictionary entries — the dict lives in - // `dms` + history; see `capture_primed_dictionary`), so - // `ensure_tables` re-allocates them zeroed to the snapshot's - // geometry, exactly reproducing the post-prime state (all - // `HC_EMPTY`). In COPY mode the snapshot retained its FULL live - // tree (the dict was merged into it, no `dms`), so the tables are - // already present at the right length and `ensure_tables` — which - // only allocates on a length mismatch — leaves them untouched. - // Either way this is a full storage replace, so no stale - // live-table entry from a prior frame can survive. - if let MatcherStorage::HashChain(hc) = &mut storage { - hc.table.ensure_tables(); - } - // The snapshot does not retain the LDM producer (it holds no - // dict state; see `capture_primed_dictionary`). Carry over the - // frame's freshly-reset producer — built this frame by `reset` - // with the same params the snapshot key pins, and empty (no - // input processed yet), so it is equivalent to the producer - // the snapshot was captured with. - #[cfg(feature = "hash")] - { - let fresh_ldm = if let MatcherStorage::HashChain(hc) = live { - hc.take_ldm_producer() - } else { - None - }; - if let MatcherStorage::HashChain(hc) = &mut storage { - hc.set_ldm_producer(fresh_ldm); - } - } - *live = storage; - } - } - self.dictionary_retained_budget = budget; - true - } - - fn capture_primed_dictionary(&mut self, level: super::CompressionLevel) { - // No resolved shape means `reset` has not run for this frame — nothing - // valid to key a snapshot on, so skip the capture. - let Some((params, table_bits, fast_attach, ldm)) = self.reset_shape else { - return; - }; - let key = PrimedKey { - level, - params, - table_bits, - fast_attach, - ldm, - }; - // CDict-equivalent retained state. A binary-tree level in ATTACH mode - // decouples the dictionary into `dms` (the upstream zstd `dictMatchState`); its - // live hash / chain / hash3 tables carry NO dict entries - // (`skip_matching_dict_bt` keeps the dict out of the live tree), so they - // are pure zeros. Storing them in the snapshot wastes the full table - // footprint (a second window-tier table set resident for the whole - // compress). Instead, move the live tables OUT of the working storage, - // clone only the dict-state (history + `dms` + window/offset/dict-limit), - // then move the live tables back — the snapshot keeps just what upstream zstd's - // CDict keeps, and `restore_primed_dictionary` re-allocates the zeroed - // live tables. Every other case keeps the dict reachable through the live - // structure, so the snapshot must retain the full tables (full clone): - // lazy-HC attach (it DOES prime a hash-chain `dms`, but the live chain is - // still the search structure, so the tables must travel) and COPY mode for - // BOTH BT and lazy-HC (`dms` invalidated, dict merged into the live tree / - // chain). `uses_bt && dms.is_primed()` is therefore the exact "decoupled" - // signal — true only for the BT attach prime; lazy-HC attach primes `dms` - // too but is intentionally NOT decoupled. - let bt_decoupled = matches!( - &self.storage, - MatcherStorage::HashChain(hc) if hc.table.uses_bt && hc.table.dms.is_primed() - ); - if bt_decoupled { - let MatcherStorage::HashChain(hc) = &mut self.storage else { - unreachable!("bt_decoupled implies HashChain storage"); - }; - let hash_table = core::mem::take(&mut hc.table.hash_table); - let chain_table = core::mem::take(&mut hc.table.chain_table); - let hash3_table = core::mem::take(&mut hc.table.hash3_table); - // The LDM producer carries no dictionary state (LDM is not - // dict-primed; its hash table is empty at capture), so it is not - // retained either — `restore` reinstates the frame's freshly-reset - // producer. Take it out so the clone does not duplicate its table. - #[cfg(feature = "hash")] - let ldm_producer = hc.take_ldm_producer(); - // Clone the dict-state-only storage (live tables now empty Vecs, - // LDM producer detached). - let snapshot = self.storage.clone(); - // Move the live tables (and LDM producer) back into the working storage. - let MatcherStorage::HashChain(hc) = &mut self.storage else { - unreachable!("storage variant is stable across the take/put"); - }; - hc.table.hash_table = hash_table; - hc.table.chain_table = chain_table; - hc.table.hash3_table = hash3_table; - #[cfg(feature = "hash")] - hc.set_ldm_producer(ldm_producer); - self.primed = Some((snapshot, self.dictionary_retained_budget, key)); - } else { - self.primed = Some((self.storage.clone(), self.dictionary_retained_budget, key)); - } - } - - fn invalidate_primed_dictionary(&mut self) { - self.primed = None; - // Drop the Fast-backend CDict-equivalent table cache too: it is keyed - // to the dictionary being removed / replaced. Left in place, the next - // same-params `reset` would retain it and the kernel would probe a - // dict region whose bytes are no longer re-committed to history. - match self.active_backend() { - super::strategy::BackendTag::Simple => self.simple_mut().invalidate_dict_cache(), - super::strategy::BackendTag::Dfast => self.dfast_matcher_mut().invalidate_dict_cache(), - // Row keeps its attach index across frames (like Simple/Dfast), - // so a dictionary swap must drop its cached dict rows too; - // otherwise the next small/unknown-size frame reuses stale - // attach state through `prime_dict_attach_current_block`. - super::strategy::BackendTag::Row => self.row_matcher_mut().invalidate_dict_cache(), - // The BT dms tree is keyed to the dict bytes; `prime_dms_bt` - // skips the rebuild while its shape matches, so a swapped - // dictionary of the same length would otherwise keep serving the - // OLD dictionary's tree. - super::strategy::BackendTag::HashChain => { - let table = &mut self.hc_matcher_mut().table; - table.dms.invalidate(); - // Deactivate the dictionary state so the next `reset` does not - // take the dict-active defer-the-table-clear branch. That branch - // rewinds the tables to the origin and hands the clear off to a - // following `prime_with_dictionary` / `restore_primed_dictionary`. - // After a dictionary is removed (or replaced), the very next - // frame may carry no dictionary, in which case neither hand-off - // runs and the deferred clear would never execute — leaving stale - // dict-region entries at the rewound base. Clearing the flag - // routes that reset down the no-dictionary path instead; a - // replacement dictionary re-arms the flag when it re-primes. - table.dictionary_active = false; - } - } - } - - fn seed_dictionary_entropy( - &mut self, - huff: Option<&crate::huff0::huff0_encoder::HuffmanTable>, - ll: Option<&crate::fse::fse_encoder::FSETable>, - ml: Option<&crate::fse::fse_encoder::FSETable>, - of: Option<&crate::fse::fse_encoder::FSETable>, - ) { - if self.active_backend() == super::strategy::BackendTag::HashChain { - self.hc_matcher_mut() - .seed_dictionary_entropy(huff, ll, ml, of); - } - } - - fn window_size(&self) -> u64 { - self.reported_window_size as u64 - } - - fn get_next_space(&mut self) -> Vec { - if let Some(mut space) = self.vec_pool.pop() { - if space.len() > self.slice_size { - space.truncate(self.slice_size); - } - if space.len() < self.slice_size { - space.resize(self.slice_size, 0); - } - return space; - } - alloc::vec![0; self.slice_size] - } - - fn get_last_space(&mut self) -> &[u8] { - match &self.storage { - MatcherStorage::Simple(m) => m.last_committed_space(), - MatcherStorage::Dfast(m) => m.get_last_space(), - MatcherStorage::Row(m) => m.get_last_space(), - MatcherStorage::HashChain(m) => m.table.get_last_space(), - } - } - - fn commit_space(&mut self, space: Vec) { - let mut evicted_bytes = 0usize; - // Split borrows manually so the `add_data` closures can write - // into `vec_pool` while the backend itself holds an exclusive - // borrow via `storage`. (Suffix-store recycling went away - // with the legacy `MatchGenerator`; the FastKernelMatcher - // arm below has no pool interaction.) - let vec_pool = &mut self.vec_pool; - match &mut self.storage { - MatcherStorage::Simple(m) => { - // FastKernelMatcher owns its history as a single - // flat Vec and the hash table as a Vec — - // neither recycles into the driver-side pools. The - // eager pre-commit eviction inside - // `FastKernelMatcher::accept_data` drops bytes when - // accepting this block would push history past 2× - // max_window_size; that delta is what feeds - // `evicted_bytes` here via the `pre / post` - // history-length comparison. - let pre = m.history_len_for_eviction_accounting(); - m.accept_data(space); - let post = m.history_len_for_eviction_accounting(); - // `accept_data` performs eager pre-commit window - // eviction (so this `pre - post` delta correctly - // feeds the dictionary-budget retire flow). See - // `FastKernelMatcher::accept_data` for the - // commit-time-visibility rationale (closes #216 - // CodeRabbit review #5 / Copilot review #1: without - // eager eviction, the delta was always 0 and the - // dict budget never retired, leaving max_window_size - // inflated post-dict-prime → matcher could emit - // offsets exceeding the frame header's window). - evicted_bytes += pre.saturating_sub(post); - } - MatcherStorage::Dfast(m) => { - // Dfast's `add_data` callback receives the INPUT - // `Vec` for pool recycling (Dfast stores its - // bytes in the contiguous `history` buffer, not in - // per-block Vecs — there is no per-block buffer to - // pop off and hand back). Counting `data.len()` as - // evicted bytes would conflate "new bytes ingested" - // with "old bytes evicted from window"; the two - // happen to coincide when the previous window was - // saturated and the new input fills it 1:1, but - // diverge when the eviction pop-loop drops blocks - // of a different size than the incoming input. The - // `dictionary_retained_budget` retire decision - // downstream then gets driven by inflated eviction - // counts and shrinks `max_window_size` prematurely. - // - // Derive the real eviction delta from `window_size` - // before/after the call. The pop loop inside - // `add_data` decrements `window_size` by each - // evicted block length and then the final - // `extend_from_slice + push_back` adds `space_len`, - // so `evicted = pre + space_len - post`. - let pre = m.window_size; - let space_len = space.len(); - m.add_data(space, |data| { - // Same per-block recycle as the HashChain arm: push - // the spent input buffer back as-is rather than - // zero-filling to capacity. `add_data` mirrors the - // bytes into `history` and calls this every block, so - // capacity-wide zeroing would be hot-path waste; - // `get_next_space` zeroes at most `slice_size` bytes - // when it later reuses the buffer. - vec_pool.push(data); - }); - // Plain `+` (the `saturating_sub` floors at 0): `pre` + one - // block are byte counts bounded by the window, no overflow. - evicted_bytes += (pre + space_len).saturating_sub(m.window_size); - } - MatcherStorage::Row(m) => { - // RowMatchGenerator::add_data recycles the *input* buffer - // through this callback every commit (its bytes are mirrored - // into `history`), not the evicted chunks. Derive the eviction - // delta from `window_size` before/after — `evicted = pre + - // space_len - post` — exactly like the Simple / HashChain arms. - // Counting the callback argument as evicted would charge the - // whole committed block as evicted and prematurely retire - // dictionary budget on a window that evicts nothing. - let pre = m.window_size; - let space_len = space.len(); - m.add_data(space, |data| { - // Recycle the spent buffer as-is; `add_data` runs this for - // every committed block, so zero-filling to capacity here - // would be hot-path waste (`get_next_space` zeroes at most - // `slice_size` on reuse). - vec_pool.push(data); - }); - // Plain `+` (the `saturating_sub` floors at 0): `pre` + one - // block are byte counts bounded by the window, no overflow. - evicted_bytes += (pre + space_len).saturating_sub(m.window_size); - } - MatcherStorage::HashChain(m) => { - // MatchTable::add_data now recycles the *incoming* buffer - // through `reuse_space` (its bytes are copied into the - // contiguous `history` mirror), so the callback no longer - // reports evicted chunks. Derive the eviction delta from - // `window_size` before/after, exactly like the Simple arm: - // `evicted = pre + space_len - post`. - let pre = m.table.window_size; - let space_len = space.len(); - m.table.add_data(space, |data| { - // Recycle the spent input buffer to the pool as-is. - // `add_data` runs this callback for every committed - // block (the bytes are mirrored into `history`), so - // growing the buffer to its full capacity here would - // zero the whole allocation on the hot path. - // `get_next_space` resizes a popped buffer to - // `slice_size` on demand, touching at most - // `slice_size` bytes — never the larger capacity the - // pool retains. - vec_pool.push(data); - }); - // Plain `+` (the `saturating_sub` floors at 0): byte counts - // bounded by the window, no overflow. - evicted_bytes += (pre + space_len).saturating_sub(m.table.window_size); - } - } - // Gate the second backend trim pass on actual budget - // reclamation. Without it, every slice commit on the - // no-dictionary / no-eviction path (the common case) would - // run a backend `match` ladder + `trim_to_window` early-out - // for no reason — `trim_after_budget_retire` only does - // meaningful work when `retire_dictionary_budget` shrank - // `max_window_size` enough to make the backend's - // `window_size > max_window_size` invariant trigger - // eviction. - if self.retire_dictionary_budget(evicted_bytes) { - self.trim_after_budget_retire(); - } - } - - fn start_matching(&mut self, mut handle_sequence: impl for<'a> FnMut(Sequence<'a>)) { - use super::strategy::{self, StrategyTag}; - // Borrowed one-shot Fast path: if the frame driver staged a - // block range via `set_borrowed_block`, scan it in place against - // the borrowed window instead of the owned committed block. Only - // the Simple backend is instrumented (the gate guarantees it), - // and the stage is consumed so the next block re-stages. - if let Some((block_start, block_end)) = self.borrowed_pending.take() { - match self.active_backend() { - super::strategy::BackendTag::Simple => { - let m = self.simple_mut(); - if m.dict_is_attached() { - // Dict-attach borrowed scan: live matches read the - // borrowed input in place, dict matches read the - // committed dict prefix via the 2-segment counter. - m.start_matching_borrowed_dict( - block_start, - block_end, - &mut handle_sequence, - ); - } else { - m.start_matching_borrowed(block_start, block_end, &mut handle_sequence); - } - } - super::strategy::BackendTag::Dfast => self - .dfast_matcher_mut() - .start_matching_borrowed(block_start, block_end, &mut handle_sequence), - super::strategy::BackendTag::Row => { - // Same greedy/lazy parse split as the owned RowHash arm. - let greedy = self.parse == super::strategy::ParseMode::Greedy; - self.row_matcher_mut().start_matching_borrowed( - block_start, - block_end, - greedy, - &mut handle_sequence, - ); - } - super::strategy::BackendTag::HashChain => match self.search { - super::strategy::SearchMethod::HashChain => self - .hc_matcher_mut() - .start_matching_lazy_borrowed(block_start, block_end, &mut handle_sequence), - super::strategy::SearchMethod::BinaryTree => { - // Run the SAME BT dispatch as the owned BinaryTree arm - // below — every BT body reads its range via - // current_block_range() and bytes via live_history() - // (borrowed-aware), so the staged block is scanned in - // place. The table was already staged by - // `set_borrowed_block` (the HashChain arm at the top of - // this file calls `table.stage_borrowed_block` with the - // same range, and `borrowed_pending` is set only there), - // so no re-stage is needed here. - // Only btlazy2 reaches the borrowed BinaryTree scan: - // `borrowed_supported()` keeps the optimal parsers - // (BtOpt/BtUltra/BtUltra2) on the owned path, and - // `set_borrowed_block` asserts that predicate before any - // range is staged, so an optimal strategy_tag can never - // arrive here. - match self.strategy_tag { - StrategyTag::Btlazy2 => self - .hc_matcher_mut() - .start_matching_btlazy2(&mut handle_sequence), - other => unreachable!( - "borrowed BinaryTree scan is only supported for Btlazy2, got {other:?}" - ), - } - } - other => { - unreachable!("HashChain backend with unexpected search {other:?}") - } - }, - } - return; - } - // Decoupled parse×search dispatch (fires once per block). The - // search axis (`self.search`) picks the candidate-finding backend; - // the parse axis (greedy vs lazy depth) is carried by the - // backend's runtime `lazy_depth`, set per level at `reset()`. - // The two are independent, so any parse can run on any search - // backend. The `BinaryTree` arm still selects the opt `Strategy` - // ZST off `strategy_tag` so `compress_block::` keeps its - // const-folded optimal-parser monomorphisation. - use super::strategy::SearchMethod; - match self.search { - SearchMethod::Fast => { - self.simple_mut().start_matching(&mut handle_sequence); - self.recycle_simple_space(); - } - SearchMethod::DoubleFast => { - self.dfast_matcher_mut() - .start_matching(&mut handle_sequence); - } - SearchMethod::RowHash => { - // Greedy parse (depth 0) = upstream zstd-greedy entry (default - // `ip + 1` start, greedy repcode commit); lazy / lazy2 use - // the `pick_lazy_match` lookahead entry (reads `lazy_depth`). - // Both bare entries dispatch on `row_log` internally into the - // const-`ROW_LOG` hot loop (upstream zstd per-rowLog variant table). - let greedy = self.parse == super::strategy::ParseMode::Greedy; - let row = self.row_matcher_mut(); - if greedy { - row.start_matching_greedy(&mut handle_sequence); - } else { - row.start_matching(&mut handle_sequence); - } - } - SearchMethod::HashChain => { - // Greedy/lazy/lazy2 all flow through the lazy parser; it - // reads `hc.lazy_depth` (0 = greedy commit). - self.hc_matcher_mut() - .start_matching_lazy(&mut handle_sequence); - } - SearchMethod::BinaryTree => match self.strategy_tag { - StrategyTag::Btlazy2 => self - .hc_matcher_mut() - .start_matching_btlazy2(&mut handle_sequence), - StrategyTag::BtOpt => self.compress_block::(&mut handle_sequence), - StrategyTag::BtUltra => { - self.compress_block::(&mut handle_sequence) - } - StrategyTag::BtUltra2 => { - self.compress_block::(&mut handle_sequence) - } - _ => unreachable!( - "SearchMethod::BinaryTree requires a BT strategy tag (Btlazy2/BtOpt/BtUltra/BtUltra2)" - ), - }, - } - } - - fn skip_matching(&mut self) { - self.skip_matching_with_hint(None); - } - - fn skip_matching_with_hint(&mut self, incompressible_hint: Option) { - // Borrowed one-shot Fast path: a staged block range routes to the - // borrowed skip (records the range for `get_last_space`, primes - // hashes on the dict-priming hint) with no owned-history append - // and nothing to recycle. Stage is consumed. - if let Some((block_start, block_end)) = self.borrowed_pending.take() { - match self.active_backend() { - super::strategy::BackendTag::Simple => self.simple_mut().skip_matching_borrowed( - block_start, - block_end, - incompressible_hint, - ), - super::strategy::BackendTag::Dfast => self - .dfast_matcher_mut() - .skip_matching_borrowed(block_start, block_end, incompressible_hint), - super::strategy::BackendTag::Row => self.row_matcher_mut().skip_matching_borrowed( - block_start, - block_end, - incompressible_hint, - ), - super::strategy::BackendTag::HashChain => self - .hc_matcher_mut() - .skip_matching_borrowed(block_start, block_end, incompressible_hint), - } - return; - } - match self.active_backend() { - super::strategy::BackendTag::Simple => { - self.simple_mut() - .skip_matching_with_hint(incompressible_hint); - self.recycle_simple_space(); - } - super::strategy::BackendTag::Dfast => { - self.dfast_matcher_mut().skip_matching(incompressible_hint) - } - super::strategy::BackendTag::Row => self - .row_matcher_mut() - .skip_matching_with_hint(incompressible_hint), - super::strategy::BackendTag::HashChain => { - self.hc_matcher_mut().skip_matching(incompressible_hint) - } - } - } -} - -impl MatchGeneratorDriver { - /// Monomorphised optimal-parser entry point. Only the `BinaryTree` - /// search arm of [`Matcher::start_matching`] routes here, selecting - /// the concrete opt `S: Strategy` (BtOpt / BtUltra / BtUltra2) off - /// `strategy_tag`, so the optimiser keeps the cost-model predicates - /// (`S::USE_BT` / `S::USE_HASH3` / `S::ACCURATE_PRICE` / - /// `S::TWO_PASS_SEED`) const-folded per strategy. The non-opt search - /// backends (Fast / DoubleFast / RowHash / HashChain) are dispatched - /// directly off the search axis and never reach this method, so all - /// strategies arriving here are HashChain-backed. - fn compress_block( - &mut self, - handle_sequence: &mut impl for<'a> FnMut(Sequence<'a>), - ) { - debug_assert_eq!(S::BACKEND, super::strategy::BackendTag::HashChain); - debug_assert!( - S::USE_BT, - "compress_block only handles the optimal (BT) path" - ); - self.hc_matcher_mut() - .start_matching_strategy::(handle_sequence); - } -} - -/// Stage D: backend storage discriminator. -/// -/// HC (lazy / lazy2) modes carry no extra per-frame state beyond the -/// shared `MatchTable` and `HcMatcher` runtime knobs, so the -/// [`HcBackend::Hc`] variant is zero-sized — no BT scratch is -/// allocated. BT-flavoured modes (`btopt` / `btultra` / `btultra2`) -/// hold the full [`super::bt::BtMatcher`] inside the -/// [`HcBackend::Bt`] variant (cost model, optimal-parser scratch -/// arenas, LDM candidate buffer). -/// -/// The discriminator lives next to `parse_mode` so `configure()` can -/// promote between the two on a level change without touching the -/// `MatchTable` storage. -#[derive(Clone)] -pub(crate) enum HcBackend { - /// Lazy / lazy2 modes — no per-frame backend state. - Hc, - /// BT-driven modes — owns the optimal parser's per-frame scratch. - /// Boxed so the enum stays pointer-sized: HC-only matchers pay - /// just the `Box`-niche, not the 4 KiB `BtMatcher` payload. - Bt(alloc::boxed::Box), -} - -impl HcBackend { - /// Heap bytes held by the backend. `Hc` is zero-sized; `Bt` boxes a - /// `BtMatcher`, so count the boxed payload plus its own scratch heap. - fn heap_size(&self) -> usize { - match self { - Self::Hc => 0, - Self::Bt(bt) => core::mem::size_of::() + bt.heap_size(), - } - } - - /// Mutable accessor on the BT matcher; panics if the active - /// backend is `Hc`. The HC-or-Bt branches in orchestrator code use - /// `let HcBackend::Bt(bt) = &self.backend` directly for readonly - /// access — this helper exists so macro bodies that already drive - /// a mutable BT update through the optimal parser can write - /// `$self.backend.bt_mut().X` without an outer `match` ladder. - #[inline(always)] - pub(crate) fn bt_mut(&mut self) -> &mut super::bt::BtMatcher { - match self { - Self::Bt(bt) => bt, - Self::Hc => unreachable!("BT-only accessor called in HC mode"), - } - } -} - -#[derive(Clone)] -struct HcMatchGenerator { - /// Shared match-finder storage (window, history, hash / chain / - /// hash3 tables, dictionary-priming flags). Used identically by HC - /// and BT modes; backend-specific table interpretation lives in the - /// matcher methods on this struct. - table: super::match_table::storage::MatchTable, - /// HC runtime knobs (lazy_depth, search_depth, target_len). Always - /// present — BT modes still consult `hc.search_depth` for repcode - /// probing and chain candidate enumeration. - hc: super::hc::HcMatcher, - /// Backend discriminator. [`HcBackend::Hc`] is zero-sized for the - /// lazy / lazy2 path so HC-only generators don't carry the BT - /// optimal-parser scratch buffers. [`HcBackend::Bt`] holds the - /// `BtMatcher` when an optimal mode is configured. - backend: HcBackend, - /// Compile-time strategy tag mirrored from - /// [`MatchGeneratorDriver::strategy_tag`] during `configure()`. - /// The driver hot path never reads this — it dispatches to - /// `compress_block::` from its own tag — but the - /// `#[cfg(test)] start_matching` helper consumes it so artificial - /// test setups still pick the correct concrete `S` for the - /// const-generic optimal parser (BtOpt vs BtUltra vs BtUltra2). - /// Without this field the test path would have to collapse - /// `BtOpt` and `BtUltra` onto the same monomorphisation since - /// `table.uses_bt` / `table.is_btultra2` alone can't tell them - /// apart. - strategy_tag: super::strategy::StrategyTag, -} - -// Plain-data types relocated to [`crate::encoding::opt::types`] and -// [`crate::encoding::opt::ldm`] by #111 Phase 1. The use statements at -// the top of this file bring them back into scope so the existing -// methods on `HcMatchGenerator` compile unchanged. - -/// `bt_insert_step_no_rebase` body parameterized over the per-CPU -/// `count_match_from_indices` symbol. Each kernel-specific wrapper invokes -/// the macro with its own `fastpath::::count_match_from_indices` -/// path so the call resolves inside the wrapper's `#[target_feature]` -/// umbrella and inlines instead of paying the function-call ABI per BT walk -/// iteration. Used only by `HcMatchGenerator` BT walk wrappers below. -/// -/// Crate-private: the macro body references private `encoding::*` -/// modules via `$crate::...`, so it is unusable downstream and is -/// re-exported only inside this crate via `pub(crate) use` below. -macro_rules! bt_insert_step_no_rebase_body { - ($table:expr, $search_depth:expr, $abs_pos:ident, $current_abs_end:ident, $target_abs:ident, $cmf:path) => {{ - let idx = $abs_pos - $table.history_abs_start; - // Borrowed-aware live region (owned: `history[history_start..]`; - // borrowed: the in-place input `[0, block_end)`). Reborrow-then-raw-ptr - // so the slice holds NO borrow and coexists with the `&mut $table` - // binary-tree writes below. Owned is byte-identical (same bytes). - let concat: &[u8] = unsafe { - let lh = $table.live_history(); - core::slice::from_raw_parts(lh.as_ptr(), lh.len()) - }; - if idx + 8 > concat.len() { - return 1; - } - debug_assert!( - $abs_pos <= $current_abs_end, - "BT walker called past current block end" - ); - let tail_limit = $current_abs_end - $abs_pos; - let hash = $crate::encoding::match_table::storage::MatchTable::hash_position_at( - concat, - idx, - $table.hash_log, - $table.search_mls, - ); - // Prefetch the hash bucket now. For the large L16+ hash table over - // high-entropy input the bucket is L3/DRAM-cold, and unlike upstream's - // monolithic ZSTD_btGetAllMatches (which overlaps this miss with its - // inline rep/hash3 prologue) the read+write of `hash_table[hash]` - // below is reached with nothing to hide it behind — it stalled a large - // share of this function's cycles. Issuing the hint here lets the miss - // overlap the address setup that follows. - #[cfg(all( - target_feature = "sse", - any(target_arch = "x86", target_arch = "x86_64") - ))] - { - #[cfg(target_arch = "x86")] - use core::arch::x86::{_MM_HINT_T0, _mm_prefetch}; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{_MM_HINT_T0, _mm_prefetch}; - // SAFETY: prefetch is a hint that never faults; `hash` indexes - // `hash_table` directly below, so it is in bounds. - unsafe { - _mm_prefetch($table.hash_table.as_ptr().add(hash).cast(), _MM_HINT_T0); - } - // Prefetch the NEXT position's bucket too. The optimal-parser DP - // advances one position per iteration, so this miss is issued a - // full BT walk plus the next iteration's pre-collect work ahead of - // the collect that will read it — far more lead than the same-call - // hint above, enough to hide the full DRAM latency. - if idx + 1 + 8 <= concat.len() { - let hash_next = - $crate::encoding::match_table::storage::MatchTable::hash_position_at( - concat, - idx + 1, - $table.hash_log, - $table.search_mls, - ); - // SAFETY: prefetch never faults; an out-of-range index is a - // harmless no-op hint. - unsafe { - _mm_prefetch( - $table.hash_table.as_ptr().add(hash_next).cast(), - _MM_HINT_T0, - ); - } - } - } - let Some(relative_pos) = $table.relative_position($abs_pos) else { - return 1; - }; - let stored = relative_pos + 1; - let bt_mask = $table.bt_mask(); - // `abs_pos < bt_mask` legitimately happens for the first BT walk of - // a fresh frame (bt_low effectively "no floor"). Saturating keeps - // the floor at 0 so the `candidate_abs <= bt_low` check never - // triggers early; raw subtraction would underflow into a huge - // sentinel that ALWAYS triggers. - let bt_low = $abs_pos.saturating_sub(bt_mask); - // Hoist the BT pointer-pair base out of `self` once — see the - // collect-matches body for the full rationale (per-step Vec reload + - // bounds check through `&mut self` vs the upstream zstd's raw `U32*` walk). - let chain_ptr = $table.chain_table.as_mut_ptr(); - debug_assert_eq!($table.chain_table.len(), 2 << $table.bt_log()); - let window_low = $table.window_low_abs_for_target($target_abs); - // `abs_pos + 9` is safe in raw form: `MatchTable::add_data` caps - // total input at `usize::MAX - STREAM_ABS_HEADROOM` (where - // `STREAM_ABS_HEADROOM = HC_OPT_NUM + 16`), so every - // frame-lifetime absolute cursor passed to the BT walker stays - // below `usize::MAX - 9` regardless of stream length or - // pointer width. The guard is hoisted to the data-ingest - // boundary so this per-position site pays zero arithmetic - // overhead in the hot loop. - let mut match_end_abs = $abs_pos + 9; - let mut best_len = 8usize; - let mut compares_left = $search_depth; - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - let pair_idx = $table.bt_pair_index_for_abs($abs_pos); - let mut smaller_slot = pair_idx; - let mut larger_slot = pair_idx + 1; - let mut match_stored = $table.hash_table[hash]; - $table.hash_table[hash] = stored; - - while compares_left > 0 { - if match_stored == $crate::encoding::match_table::storage::HC_EMPTY { - break; - } - // Reject stale post-rebase slots whose pre-shift position is below - // `index_shift` explicitly. A `wrapping_sub` maps such a slot to a - // near-`usize::MAX` value that the `>= abs_pos` test only rejects - // while `abs_pos` is far from the integer ceiling; on a - // long-running rebased stream (reachable on 32-bit) `abs_pos` can - // approach the ceiling and the wrapped value can land back inside - // `[window_low, abs_pos)`. `checked_sub` ends the walk on the - // underflow instead. `match_stored != HC_EMPTY` here, so the `- 1` - // cannot underflow. - let Some(candidate_abs) = ($table.position_base + (match_stored as usize - 1)) - .checked_sub($table.index_shift) - else { - break; - }; - if candidate_abs < window_low || candidate_abs >= $abs_pos { - break; - } - compares_left -= 1; - - let next_pair_idx = $table.bt_pair_index_for_abs(candidate_abs); - // SAFETY: `next_pair_idx (+1)` = `2*(candidate_abs & bt_mask) (+1)` - // ≤ `chain_table.len()-1`; `chain_ptr` is the hoisted live base, - // table not realloc'd during the walk. - let next_smaller = unsafe { *chain_ptr.add(next_pair_idx) }; - let next_larger = unsafe { *chain_ptr.add(next_pair_idx + 1) }; - let seed_len = common_length_smaller.min(common_length_larger); - let candidate_idx = candidate_abs - $table.history_abs_start; - // SAFETY: BT walk invariant — `candidate_idx + tail_limit ≤ - // concat.len()` since the candidate is within - // `[history_abs_start, abs_pos)` and `tail_limit ≤ - // current_abs_end - abs_pos`. - let match_len = unsafe { $cmf(concat, idx, candidate_idx, tail_limit, seed_len) }; - - if match_len > best_len { - best_len = match_len; - // `candidate_abs + match_len <= current_abs_end` by BT walk - // invariant — `match_len <= tail_limit = current_abs_end - - // abs_pos` and `candidate_abs < abs_pos`. - let candidate_end = candidate_abs + match_len; - if candidate_end > match_end_abs { - match_end_abs = candidate_end; - } - } - - if match_len >= tail_limit { - break; - } - - let candidate_next = candidate_idx + match_len; - let current_next = idx + match_len; - // SAFETY: first-differing positions after a match_len-long prefix; - // match_len < tail_limit (break above) + BT-walk bound - // idx/candidate_idx + tail_limit <= concat.len() keep both in range. - if unsafe { - *concat.get_unchecked(candidate_next) < *concat.get_unchecked(current_next) - } { - // SAFETY: `smaller_slot` holds a valid pair index (init - // `pair_idx`, updated to `next_pair_idx + 1`); the `usize::MAX` - // sentinel is set only just before `break`, never written here. - unsafe { *chain_ptr.add(smaller_slot) = match_stored }; - common_length_smaller = match_len; - if candidate_abs <= bt_low { - smaller_slot = usize::MAX; - break; - } - smaller_slot = next_pair_idx + 1; - match_stored = next_larger; - } else { - // SAFETY: as above for `larger_slot`. - unsafe { *chain_ptr.add(larger_slot) = match_stored }; - common_length_larger = match_len; - if candidate_abs <= bt_low { - larger_slot = usize::MAX; - break; - } - larger_slot = next_pair_idx; - match_stored = next_smaller; - } - } - - // SAFETY: both slots, when not the `usize::MAX` sentinel, hold valid - // pair indices into the hoisted `chain_table` base. - if smaller_slot != usize::MAX { - unsafe { - *chain_ptr.add(smaller_slot) = $crate::encoding::match_table::storage::HC_EMPTY - }; - } - if larger_slot != usize::MAX { - unsafe { - *chain_ptr.add(larger_slot) = $crate::encoding::match_table::storage::HC_EMPTY - }; - } - - let speed_positions = if best_len > 384 { - (best_len - 384).min(192) - } else { - 0 - }; - // `match_end_abs` is initialized to `abs_pos + 9` and is only - // reassigned inside the `candidate_end > match_end_abs` branch - // above. So even though an individual `candidate_end = - // candidate_abs + match_len` can land below `abs_pos` (the - // candidate sits earlier in history and the match runs short), - // the variable itself never drops below its initial value. - // That gives `match_end_abs ≥ abs_pos + 9 > abs_pos + 8` as a - // loop-wide invariant, so the raw subtraction below cannot - // underflow. - speed_positions.max(match_end_abs - ($abs_pos + 8)) - }}; -} -pub(crate) use bt_insert_step_no_rebase_body; - -/// `build_optimal_plan_impl` body parameterized over the per-CPU -/// `collect_optimal_candidates_initialized_` method name. Caller -/// passes its `&mut self`, the seven DP entry-point arguments, and the -/// kernel-specific collect method. Each per-kernel wrapper invokes this -/// macro inside its own `#[target_feature]` umbrella so the per-position -/// `$collect` call inlines and the entire DP loop runs as one straight-line -/// hot path without an ABI barrier between the DP and the match-gathering -/// pipeline. -/// -/// Body is ~730 lines but mechanically identical across kernels — the macro -/// keeps a single source of truth. The two const generics -/// (`ACCURATE_PRICE`, `FAVOR_SMALL_OFFSETS`) come from the wrapper's -/// generic parameter list and are referenced as bare identifiers; macro -/// hygiene resolves them at the expansion site. -/// Upstream zstd `offBase` for the btlazy2 lazy gain heuristic: a match whose offset -/// equals one of the three active repeat offsets prices as the cheap repcode -/// code (1/2/3); any other offset prices as `offset + 3`. So an equal-length -/// repeat-offset match always out-gains an explicit-offset one -/// (`zstd_lazy.c` `ZSTD_storeSeq` offBase convention). -#[inline] -fn btlazy2_offbase(offset: usize, reps: [u32; 3], ll0: bool) -> u32 { - let o = offset as u32; - // Upstream zstd repcode mapping shifts by `ll0` (zero-literal position): the cheap - // codes become rep1 / rep2 / (rep0 - 1) instead of rep0 / rep1 / rep2, - // because at ll0 an offset equal to rep0 is the special rep0-1 case, not - // repcode 1. Scoring offsets against the wrong code at ll0 over-rewards a - // rep0-distance match that does not actually encode as the cheapest code. - if ll0 { - if o == reps[1] { - 1 - } else if o == reps[2] { - 2 - } else if reps[0] > 1 && o == reps[0] - 1 { - 3 - } else { - // Offsets are < window (<= 2^27), so `+ 3` never overflows u32. - o + 3 - } - } else if o == reps[0] { - 1 - } else if o == reps[1] { - 2 - } else if o == reps[2] { - 3 - } else { - // Offsets are < window (<= 2^27), so `+ 3` never overflows u32. - o + 3 - } -} - -/// Upstream zstd lazy match gain (`matchLength * 4 - ZSTD_highbit32(offBase)`): the -/// selection metric that lets a shorter repeat-offset match beat a longer -/// explicit-offset one. `offBase >= 1`, so `highbit` is well-defined. -#[inline] -fn btlazy2_gain(match_len: usize, offset: usize, reps: [u32; 3], ll0: bool) -> i64 { - let offbase = btlazy2_offbase(offset, reps, ll0); - (match_len as i64) * 4 - (31 - offbase.leading_zeros()) as i64 -} - -/// Per-kernel body of the `btlazy2` (levels 13-15) greedy/lazy parse over -/// the binary-tree match finder. Mirrors `build_optimal_plan_impl_body!`'s -/// kernel-dispatch discipline: the wrapper carries the `#[target_feature]` -/// umbrella and passes its tier-specific `collect_optimal_candidates_initialized_` -/// as `$collect`, so the per-position BT collect (and its inlined cpl) -/// stays under one umbrella — the runtime `select_kernel()` dispatch happens -/// ONCE per block in the bare `start_matching_btlazy2`, never per position. -macro_rules! start_matching_btlazy2_body { - ($self:ident, $handle_sequence:ident, $collect:ident, $cmf:path $(,)?) => {{ - $self.table.ensure_tables(); - // Borrowed-aware: owned → last committed chunk; borrowed → staged block. - let (current_abs_start, current_len) = $self.table.current_block_range(); - if current_len == 0 { - return; - } - let current_ptr = $self.table.get_last_space().as_ptr(); - // Mutates tables but never reallocates `history`, so this tail slice - // stays valid for the routine's duration (same as the other parsers). - let current: &[u8] = unsafe { core::slice::from_raw_parts(current_ptr, current_len) }; - // Full contiguous live region (owned: dict + prior blocks + current - // block in `history`; borrowed: `[0, block_end)` of the in-place - // input) as a raw slice, for the explicit repcode probe: a rep offset - // can point before the current block, which `current` can't reach. - // `live_history()` is borrowed-aware; reborrow-then-raw-ptr so the - // slice holds NO borrow and coexists with the `&mut self` collector - // calls below. Same no-realloc validity contract as `current`. - let history_abs_start = $self.table.history_abs_start; - let concat_full: &[u8] = unsafe { - let lh = $self.table.live_history(); - core::slice::from_raw_parts(lh.as_ptr(), lh.len()) - }; - let current_abs_end = current_abs_start + current_len; - $self - .table - .apply_limited_update_after_long_match(current_abs_start); - $self - .table - .backfill_boundary_positions(current_abs_start, current_abs_end); - - let profile = HcOptimalCostProfile::const_for_strategy::(); - let mut candidates = core::mem::take(&mut $self.backend.bt_mut().opt_candidates_scratch); - - let depth = $self.hc.lazy_depth as usize; - let mut pos = 0usize; - let mut literals_start = 0usize; - - // Collect + select the highest-GAIN match at a position (upstream zstd - // `ZSTD_searchMax` plus the explicit offset_1 repcode check): scan the - // length-sorted BT/dms ladder by gain, then probe rep0 directly since - // the ladder's strictly-increasing-length filter drops short cheap - // reps. Expands to `(match_len, offset)`; `match_len == 0` = no match. - macro_rules! bt_select { - ($p:expr) => {{ - let sel_pos: usize = $p; - // `ll0` (upstream zstd): zero literals pending before this position, so - // the repcode set is shifted (see `btlazy2_offbase`). - let ll0 = sel_pos == literals_start; - let sel_abs = current_abs_start + sel_pos; - candidates.clear(); - let query = HcCandidateQuery { - reps: $self.table.offset_hist, - lit_len: sel_pos - literals_start, - // No LDM seed: L13-15 run at windowLog 22, below upstream zstd's - // LDM auto-enable threshold (windowLog >= 27). - ldm_candidate: None, - }; - // SAFETY: called inside the wrapper's `#[target_feature]` - // umbrella (the scalar wrapper's `$collect` is a safe fn). - unsafe { - $self.$collect::( - sel_abs, - current_abs_end, - profile, - query, - &mut candidates, - ); - } - let reps = $self.table.offset_hist; - let mut sel_ml = 0usize; - let mut sel_off = 0usize; - let mut sel_gain = i64::MIN; - for c in candidates.iter() { - let ml = c.match_len.min(current_len - sel_pos); - if ml < HC_OPT_MIN_MATCH_LEN { - continue; - } - let g = btlazy2_gain(ml, c.offset, reps, ll0); - if g > sel_gain { - sel_gain = g; - sel_ml = ml; - sel_off = c.offset; - } - } - let sel_idx = sel_abs - history_abs_start; - // Upstream zstd probes `rep[0 + ll0]` directly (the length-sorted ladder - // drops short cheap reps): rep0 normally, rep1 at a zero-literal - // position where rep0 is not the cheapest code. - let probe_rep = if ll0 { - reps[1] as usize - } else { - reps[0] as usize - }; - if probe_rep != 0 && sel_idx >= probe_rep { - let tail = current_len - sel_pos; - // SAFETY: `sel_idx - probe_rep < sel_idx`, `sel_idx + tail <= - // concat_full.len()`; same overshoot slack the collector - // relies on for this block. - let rep_ml = - unsafe { $cmf(concat_full, sel_idx, sel_idx - probe_rep, tail, 0) }; - if rep_ml >= HC_OPT_MIN_MATCH_LEN - && btlazy2_gain(rep_ml, probe_rep, reps, ll0) > sel_gain - { - sel_ml = rep_ml; - sel_off = probe_rep; - } - } - (sel_ml, sel_off) - }}; - } - - while pos + HC_OPT_MIN_MATCH_LEN <= current_len { - let (mut best_ml, mut best_off) = bt_select!(pos); - if best_ml < HC_OPT_MIN_MATCH_LEN { - pos += 1; - continue; - } - // Lazy lookahead (upstream zstd depth 1/2): advance one byte and accept the - // later match only if it out-gains the current one by the upstream zstd - // margin (deferring costs an extra literal — `+4` at depth 1, `+7` - // at depth 2). `start` tracks where the chosen match begins. - let mut start = pos; - let mut d = 0usize; - while d < depth && start + 1 + HC_OPT_MIN_MATCH_LEN <= current_len { - let look = start + 1; - let (ml2, off2) = bt_select!(look); - if ml2 < HC_OPT_MIN_MATCH_LEN { - break; - } - let reps = $self.table.offset_hist; - let margin = if d == 0 { 4 } else { 7 }; - // `best` sits at `start` (ll0 iff no literals precede it); the - // lookahead match at `start + 1` always has a pending literal. - let gain1 = btlazy2_gain(best_ml, best_off, reps, start == literals_start) + margin; - let gain2 = btlazy2_gain(ml2, off2, reps, false); - if gain2 > gain1 { - best_ml = ml2; - best_off = off2; - start = look; - d += 1; - } else { - break; - } - } - // Commit the chosen match at `start`; [literals_start, start) is - // emitted as literals. `best_ml` was bounded to `current_len - - // start` at selection, so `start + best_ml <= current_len`. - let lit_len = start - literals_start; - let literals = ¤t[literals_start..start]; - $handle_sequence(Sequence::Triple { - literals, - offset: best_off, - match_len: best_ml, - }); - let _ = encode_offset_with_history( - best_off as u32, - lit_len as u32, - &mut $self.table.offset_hist, - ); - pos = start + best_ml; - literals_start = pos; - } - - if literals_start < current_len { - $handle_sequence(Sequence::Literals { - literals: ¤t[literals_start..], - }); - } - $self.backend.bt_mut().opt_candidates_scratch = candidates; - }}; -} - -/// 8-lane `next_cost < node_price` mask for the optimal-parser price-set -/// loop. AVX2 lacks an unsigned `cmplt`, so derive `nc < np` from -/// `min_epu32`: `nc <= np` iff `min(nc,np) == nc`, then exclude equality. -/// Returns a bitmask (bit `k` set => lane `k` improves). Scalar fallback -/// for non-x86 / no-AVX2. -/// 8-lane `next_cost < node_price` mask for the optimal-parser price-set -/// loop. AVX2 lacks an unsigned `cmplt`, so derive `nc < np` from -/// `min_epu32`: `nc <= np` iff `min(nc,np) == nc`, then exclude equality. -/// Returns a bitmask (bit `k` set => lane `k` improves). Compiled on every -/// x86 target (same as the avx2 collect kernel); the cargo `kernel_avx2` -/// feature only gates the runtime dispatch, not compilation. -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -#[target_feature(enable = "avx2")] -unsafe fn priceset_improved_mask8_avx2(next_cost: &[u32; 8], node_price: &[u32]) -> u8 { - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m256i, _mm256_andnot_si256, _mm256_castsi256_ps, _mm256_cmpeq_epi32, _mm256_loadu_si256, - _mm256_min_epu32, _mm256_movemask_ps, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m256i, _mm256_andnot_si256, _mm256_castsi256_ps, _mm256_cmpeq_epi32, _mm256_loadu_si256, - _mm256_min_epu32, _mm256_movemask_ps, - }; - let nc = unsafe { _mm256_loadu_si256(next_cost.as_ptr() as *const __m256i) }; - let np = unsafe { _mm256_loadu_si256(node_price.as_ptr() as *const __m256i) }; - let min = _mm256_min_epu32(nc, np); - let le = _mm256_cmpeq_epi32(min, nc); // nc <= np - let eq = _mm256_cmpeq_epi32(nc, np); // nc == np - let lt = _mm256_andnot_si256(eq, le); // nc < np - _mm256_movemask_ps(_mm256_castsi256_ps(lt)) as u8 -} - -/// Inline `next_cost = base_cost + ll0_price + match_price_from_parts(off,ml)` -/// for one match length — the exact `add_prices` chain the scalar loop uses, -/// so the SoA vector path stays byte-identical. -#[inline(always)] -#[allow(clippy::too_many_arguments)] -fn priceset_next_cost( - profile: HcOptimalCostProfile, - stats: &HcOptState, - ml_cache: &mut [[u32; 2]], - ml_stamp: u32, - match_len: usize, - ll0_price: u32, - off_price: u32, - base_cost: u32, -) -> u32 { - let ml_price = - BtMatcher::cached_match_length_price(profile, stats, match_len, ml_cache, ml_stamp); - let seq_cost = BtMatcher::add_prices( - ll0_price, - profile.match_price_from_parts(off_price, ml_price, stats), - ); - BtMatcher::add_prices(base_cost, seq_cost) -} - -/// Scalar price-set over the match-length range `[start, max]` for the -/// NON-abort optimal modes (btultra / btultra2). Each `match_len` writes a -/// distinct node `pos + match_len`, so order is irrelevant; the improvement -/// test reduces to `next_cost < node_prices[next]` (`reset_opt_nodes` set -/// every beyond-frontier cell to `u32::MAX`, subsuming `next > last_pos`). -/// `#[inline]` so it folds into each per-tier optimal-parser monomorphisation -/// (no call overhead). Returns the highest written `next`. -#[inline] -#[allow(clippy::too_many_arguments)] -// Used by the scalar / sse42 DP wrappers; on aarch64 the dispatch only reaches -// the neon wrapper and on wasm+simd128 only the simd128 wrapper, so this is -// cfg-dead on those targets. -#[cfg_attr( - any( - all(target_arch = "aarch64", target_endian = "little"), - all(target_arch = "wasm32", target_feature = "simd128") - ), - allow(dead_code) -)] -fn priceset_range_nonabort_scalar( - node_prices: &mut [u32], - nodes: &mut [HcOptimalNode], - ml_cache: &mut [[u32; 2]], - ml_stamp: u32, - profile: HcOptimalCostProfile, - stats: &HcOptState, - pos: usize, - start: usize, - max: usize, - ll0_price: u32, - off_price: u32, - base_cost: u32, - off: u32, - reps: [u32; 3], - last_pos: usize, -) -> usize { - let mut new_last = last_pos; - for ml in start..=max { - let next_cost = priceset_next_cost( - profile, stats, ml_cache, ml_stamp, ml, ll0_price, off_price, base_cost, - ); - let next = pos + ml; - if next_cost < node_prices[next] { - node_prices[next] = next_cost; - nodes[next] = HcOptimalNode { - off, - mlen: ml as u32, - litlen: 0, - reps, - }; - if next > new_last { - new_last = next; - } - } - } - new_last -} - -/// Per-tier deinterleave + improve-mask correctness vs a scalar reference. -/// Each tier's dispatch only fires on matching hardware (i9 picks AVX2 over -/// SSE4.1, M1 picks NEON), so the non-dispatched tiers never run in the -/// roundtrip suite; this exercises the deinterleave/mask helpers directly on -/// whatever ISA the test host exposes (AVX2 + SSE4.1 on x86, NEON on aarch64). -#[cfg(test)] -#[test] -fn priceset_tier_helpers_match_scalar() { - // Reference: gen-stamped contiguous cells -> ordered prices on all-warm. - fn scalar_deint(cells: &[[u32; 2]], stamp: u32) -> Option<[u32; W]> { - let mut out = [0u32; W]; - for k in 0..W { - if cells[k][1] != stamp { - return None; - } - out[k] = cells[k][0]; - } - Some(out) - } - fn scalar_mask(nc: &[u32; W], np: &[u32]) -> u8 { - let mut m = 0u8; - for k in 0..W { - if nc[k] < np[k] { - m |= 1 << k; - } - } - m - } - const S: u32 = 0x55; - let warm: [[u32; 2]; 4] = [[11, S], [22, S], [33, S], [44, S]]; - let mut cold = warm; - cold[2][1] = S ^ 1; // one stale cell -> must yield None - let nc4: [u32; 4] = [10, 99, 30, 41]; - let np4: [u32; 4] = [20, 21, 30, 99]; // lt: lane0 (10<20), lane3 (41<99) - - #[cfg(all(target_arch = "aarch64", target_endian = "little"))] - unsafe { - assert_eq!( - priceset_cached_prices4_neon(&warm, S), - scalar_deint::<4>(&warm, S) - ); - assert_eq!(priceset_cached_prices4_neon(&cold, S), None); - assert_eq!( - priceset_improved_mask4_neon(&nc4, &np4), - scalar_mask::<4>(&nc4, &np4) - ); - } - #[cfg(all(feature = "std", any(target_arch = "x86", target_arch = "x86_64")))] - { - if std::is_x86_feature_detected!("sse4.2") { - unsafe { - assert_eq!( - priceset_cached_prices4_sse41(&warm, S), - scalar_deint::<4>(&warm, S) - ); - assert_eq!(priceset_cached_prices4_sse41(&cold, S), None); - assert_eq!( - priceset_improved_mask4_sse41(&nc4, &np4), - scalar_mask::<4>(&nc4, &np4) - ); - } - } - if std::is_x86_feature_detected!("avx2") { - let warm8: [[u32; 2]; 8] = [ - [11, S], - [22, S], - [33, S], - [44, S], - [55, S], - [66, S], - [77, S], - [88, S], - ]; - let mut cold8 = warm8; - cold8[5][1] = S ^ 1; - let nc8: [u32; 8] = [10, 99, 30, 41, 99, 60, 99, 80]; - let np8: [u32; 8] = [20, 21, 30, 99, 50, 99, 70, 99]; - unsafe { - assert_eq!( - priceset_cached_prices8_avx2(&warm8, S), - scalar_deint::<8>(&warm8, S) - ); - assert_eq!(priceset_cached_prices8_avx2(&cold8, S), None); - assert_eq!( - priceset_improved_mask8_avx2(&nc8, &np8), - scalar_mask::<8>(&nc8, &np8) - ); - } - } - } -} - -/// Shared vectorised price-set loop body, generic over the SIMD width `W`. -/// The per-tier `deint` (vector-load plus deinterleave of `W` cached prices, -/// returning `Some` only on an all-warm chunk) and `mask` (per-tier -/// `next_cost` less-than `node_price` bitmask) are passed as zero-sized -/// `impl Fn`s. `#[inline(always)]` plus monomorphisation folds `deint` and -/// `mask` directly into each per-tier wrapper's `target_feature` umbrella, so -/// the intrinsics inline with no call ABI and no runtime feature detection. -/// Cold or out-of-cache chunks, and the sub-`W` remainder, fall back to the -/// scalar `priceset_next_cost` (which fills the cache); writes are -/// scalar-scatter on the improving lanes (1-8% of compares, per the -/// improve-ratio probe). Same signature tail as the scalar variant. -#[inline(always)] -#[allow(clippy::too_many_arguments)] -// Instantiated only by a vector tier wrapper (avx2/sse4.1 on x86, neon on -// aarch64, simd128 on wasm+simd128); a target with none of those (e.g. -// wasm without +simd128) uses only the scalar range, leaving this generic dead. -#[cfg_attr( - not(any( - target_arch = "x86", - target_arch = "x86_64", - all(target_arch = "aarch64", target_endian = "little"), - all(target_arch = "wasm32", target_feature = "simd128") - )), - allow(dead_code) -)] -fn priceset_range_vec( - node_prices: &mut [u32], - nodes: &mut [HcOptimalNode], - ml_cache: &mut [[u32; 2]], - ml_stamp: u32, - profile: HcOptimalCostProfile, - stats: &HcOptState, - pos: usize, - start: usize, - max: usize, - ll0_price: u32, - off_price: u32, - base_cost: u32, - off: u32, - reps: [u32; 3], - last_pos: usize, - deint: impl Fn(&[[u32; 2]], u32) -> Option<[u32; W]>, - mask: impl Fn(&[u32; W], &[u32]) -> u8, -) -> usize { - let mut new_last = last_pos; - let mut buf = [0u32; W]; - // Loop-invariant constant of the byte-identical next_cost chain: - // next_cost = add_prices(base_cost, add_prices(ll0_price, - // match_price_from_parts(off_price, ml_price))) = c_base + ml_price, - // c_base = base_cost + ll0_price + match_price_from_parts(off_price, 0). - // - // This stays bit-exact with the scalar `priceset_next_cost` because both - // helpers are affine in `ml_price`: `BtMatcher::add_prices(a, b) = a + b` - // and `match_price_from_parts(off, ml) = off + ml + bias` are plain integer - // additions, so `match_price_from_parts(off, ml) = match_price_from_parts( - // off, 0) + ml` and the whole chain collapses to `c_base + ml_price`. The - // `wrapping_add` here matches the scalar `+` under the cost model's - // no-overflow invariant (the `debug_assert`s in both helpers). Factoring the - // combine into one helper per the review suggestion would force a per-lane - // `match_price_from_parts(off, ml_price)` recompute instead of hoisting the - // ml-independent `c_base` once — a regression on this hot DP loop — so the - // hoist is kept and the equivalence documented here instead. - let c_base = base_cost - .wrapping_add(ll0_price) - .wrapping_add(profile.match_price_from_parts(off_price, 0, stats)); - let mut ml = start; - while ml + W <= max + 1 { - let vectorised = if ml + W <= ml_cache.len() { - deint(&ml_cache[ml..ml + W], ml_stamp) - } else { - None - }; - if let Some(prices) = vectorised { - for (k, slot) in buf.iter_mut().enumerate() { - *slot = c_base.wrapping_add(prices[k]); - } - } else { - for (k, slot) in buf.iter_mut().enumerate() { - *slot = priceset_next_cost( - profile, - stats, - ml_cache, - ml_stamp, - ml + k, - ll0_price, - off_price, - base_cost, - ); - } - } - let base_next = pos + ml; - let mut bits = mask(&buf, &node_prices[base_next..base_next + W]); - while bits != 0 { - let k = bits.trailing_zeros() as usize; - bits &= bits - 1; - let next = base_next + k; - node_prices[next] = buf[k]; - nodes[next] = HcOptimalNode { - off, - mlen: (ml + k) as u32, - litlen: 0, - reps, - }; - if next > new_last { - new_last = next; - } - } - ml += W; - } - while ml <= max { - let next_cost = priceset_next_cost( - profile, stats, ml_cache, ml_stamp, ml, ll0_price, off_price, base_cost, - ); - let next = pos + ml; - if next_cost < node_prices[next] { - node_prices[next] = next_cost; - nodes[next] = HcOptimalNode { - off, - mlen: ml as u32, - litlen: 0, - reps, - }; - if next > new_last { - new_last = next; - } - } - ml += 1; - } - new_last -} - -/// Vector-load 8 cached ml-prices for the optimal parser's price-set, given a -/// run of 8 contiguous `[price, generation]` cells. Returns `Some(prices)` -/// only when ALL eight cells are warm (`generation == stamp`) — the common -/// (~91-98%) case — so the caller can fold them with one broadcast constant; -/// any cold cell returns `None` to route the chunk through the scalar fill -/// (which recomputes + repopulates the misses). Deinterleaves with cheap -/// in-128-lane ops (`shuffle_epi32` + `unpack*_epi64`) and a single cross-lane -/// `permute4x64` for the ordered prices — avoiding the latency-bound chain of -/// cross-lane `permutevar8x32`s that lost to pipelined scalar loads on -/// high-chunk-count fixtures. -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -#[target_feature(enable = "avx2")] -#[inline] -unsafe fn priceset_cached_prices8_avx2(cells: &[[u32; 2]], stamp: u32) -> Option<[u32; 8]> { - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m256i, _mm256_castsi256_ps, _mm256_cmpeq_epi32, _mm256_loadu_si256, _mm256_movemask_ps, - _mm256_permute4x64_epi64, _mm256_set1_epi32, _mm256_shuffle_epi32, _mm256_storeu_si256, - _mm256_unpackhi_epi64, _mm256_unpacklo_epi64, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m256i, _mm256_castsi256_ps, _mm256_cmpeq_epi32, _mm256_loadu_si256, _mm256_movemask_ps, - _mm256_permute4x64_epi64, _mm256_set1_epi32, _mm256_shuffle_epi32, _mm256_storeu_si256, - _mm256_unpackhi_epi64, _mm256_unpacklo_epi64, - }; - debug_assert!(cells.len() >= 8); - let base = cells.as_ptr() as *const __m256i; - // v0 = [p0 g0 p1 g1 | p2 g2 p3 g3], v1 = [p4 g4 p5 g5 | p6 g6 p7 g7]. - let v0 = unsafe { _mm256_loadu_si256(base) }; - let v1 = unsafe { _mm256_loadu_si256(base.add(1)) }; - // In-128-lane group prices then gens: [p g p g] -> [p p g g] (control 0xD8). - let s0 = _mm256_shuffle_epi32(v0, 0xD8); // [p0 p1 g0 g1 | p2 p3 g2 g3] - let s1 = _mm256_shuffle_epi32(v1, 0xD8); // [p4 p5 g4 g5 | p6 p7 g6 g7] - // Gens (hi 64 of each 128-lane) — order irrelevant for the all-equal test. - let gens = _mm256_unpackhi_epi64(s0, s1); - let eq = _mm256_cmpeq_epi32(gens, _mm256_set1_epi32(stamp as i32)); - if _mm256_movemask_ps(_mm256_castsi256_ps(eq)) as u8 != 0xFF { - return None; - } - // Prices (lo 64 of each 128-lane): [p0 p1 p4 p5 | p2 p3 p6 p7] as 64-bit - // chunks [c0 c1 c2 c3] = [p0p1 p4p5 p2p3 p6p7]; reorder to [c0 c2 c1 c3] - // (control 0xD8) for in-order [p0..p7]. - let p_scrambled = _mm256_unpacklo_epi64(s0, s1); - let prices = _mm256_permute4x64_epi64(p_scrambled, 0xD8); - let mut out = [0u32; 8]; - unsafe { _mm256_storeu_si256(out.as_mut_ptr() as *mut __m256i, prices) }; - Some(out) -} - -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -#[target_feature(enable = "avx2")] -#[inline] -#[allow(clippy::too_many_arguments)] -unsafe fn priceset_range_nonabort_avx2( - node_prices: &mut [u32], - nodes: &mut [HcOptimalNode], - ml_cache: &mut [[u32; 2]], - ml_stamp: u32, - profile: HcOptimalCostProfile, - stats: &HcOptState, - pos: usize, - start: usize, - max: usize, - ll0_price: u32, - off_price: u32, - base_cost: u32, - off: u32, - reps: [u32; 3], - last_pos: usize, -) -> usize { - priceset_range_vec::<8>( - node_prices, - nodes, - ml_cache, - ml_stamp, - profile, - stats, - pos, - start, - max, - ll0_price, - off_price, - base_cost, - off, - reps, - last_pos, - // SAFETY: both closures run inside this fn's avx2 target_feature umbrella. - |cells, stamp| unsafe { priceset_cached_prices8_avx2(cells, stamp) }, - |nc, np| unsafe { priceset_improved_mask8_avx2(nc, np) }, - ) -} - -/// NEON 4-lane vector-load + deinterleave of cached ml-prices. `vld2q_u32` -/// deinterleaves the 4 contiguous `[price, generation]` pairs natively into -/// two registers (prices, gens) — no shuffle chain. `Some(prices)` only when -/// all 4 generations equal `stamp` (`vminvq` of the equality mask is all-ones). -#[cfg(all(target_arch = "aarch64", target_endian = "little"))] -#[target_feature(enable = "neon")] -#[inline] -unsafe fn priceset_cached_prices4_neon(cells: &[[u32; 2]], stamp: u32) -> Option<[u32; 4]> { - use core::arch::aarch64::{vceqq_u32, vdupq_n_u32, vld2q_u32, vminvq_u32, vst1q_u32}; - debug_assert!(cells.len() >= 4); - // SAFETY: caller's neon umbrella; `cells` is >= 4 pairs = 8 contiguous u32. - let pair = unsafe { vld2q_u32(cells.as_ptr() as *const u32) }; - let eq = vceqq_u32(pair.1, vdupq_n_u32(stamp)); - if vminvq_u32(eq) != u32::MAX { - return None; - } - let mut out = [0u32; 4]; - unsafe { vst1q_u32(out.as_mut_ptr(), pair.0) }; - Some(out) -} - -/// NEON 4-lane `next_cost < node_price` bitmask. NEON has an unsigned compare -/// (`vcltq_u32`) but no movemask; AND the all-ones lane mask with lane weights -/// `[1,2,4,8]` and horizontal-add (`vaddvq_u32`) to pack the 4 bits. -#[cfg(all(target_arch = "aarch64", target_endian = "little"))] -#[target_feature(enable = "neon")] -#[inline] -unsafe fn priceset_improved_mask4_neon(next_cost: &[u32; 4], node_price: &[u32]) -> u8 { - use core::arch::aarch64::{vaddvq_u32, vandq_u32, vcltq_u32, vld1q_u32, vst1q_u32}; - // SAFETY: neon umbrella; both spans are 4 u32 wide. - let nc = unsafe { vld1q_u32(next_cost.as_ptr()) }; - let np = unsafe { vld1q_u32(node_price.as_ptr()) }; - let lt = vcltq_u32(nc, np); - let weights: [u32; 4] = [1, 2, 4, 8]; - let w = unsafe { vld1q_u32(weights.as_ptr()) }; - let bits = vandq_u32(lt, w); - let _ = vst1q_u32; // silence unused import on some toolchains - vaddvq_u32(bits) as u8 -} - -#[cfg(all(target_arch = "aarch64", target_endian = "little"))] -#[target_feature(enable = "neon")] -#[inline] -#[allow(clippy::too_many_arguments)] -unsafe fn priceset_range_nonabort_neon( - node_prices: &mut [u32], - nodes: &mut [HcOptimalNode], - ml_cache: &mut [[u32; 2]], - ml_stamp: u32, - profile: HcOptimalCostProfile, - stats: &HcOptState, - pos: usize, - start: usize, - max: usize, - ll0_price: u32, - off_price: u32, - base_cost: u32, - off: u32, - reps: [u32; 3], - last_pos: usize, -) -> usize { - priceset_range_vec::<4>( - node_prices, - nodes, - ml_cache, - ml_stamp, - profile, - stats, - pos, - start, - max, - ll0_price, - off_price, - base_cost, - off, - reps, - last_pos, - // SAFETY: both closures run inside this fn's neon target_feature umbrella. - |cells, stamp| unsafe { priceset_cached_prices4_neon(cells, stamp) }, - |nc, np| unsafe { priceset_improved_mask4_neon(nc, np) }, - ) -} - -/// SSE4.1 4-lane vector-load + deinterleave of cached ml-prices. Two 128-bit -/// loads of `[price, gen]` pairs, `shuffle_epi32(0xD8)` groups prices then gens -/// within each, `unpacklo/hi_epi64` separates them. `Some(prices)` only when -/// all 4 generations equal `stamp`. -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -#[target_feature(enable = "sse4.2")] -#[inline] -unsafe fn priceset_cached_prices4_sse41(cells: &[[u32; 2]], stamp: u32) -> Option<[u32; 4]> { - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_castsi128_ps, _mm_cmpeq_epi32, _mm_loadu_si128, _mm_movemask_ps, - _mm_set1_epi32, _mm_shuffle_epi32, _mm_storeu_si128, _mm_unpackhi_epi64, - _mm_unpacklo_epi64, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_castsi128_ps, _mm_cmpeq_epi32, _mm_loadu_si128, _mm_movemask_ps, - _mm_set1_epi32, _mm_shuffle_epi32, _mm_storeu_si128, _mm_unpackhi_epi64, - _mm_unpacklo_epi64, - }; - debug_assert!(cells.len() >= 4); - let base = cells.as_ptr() as *const __m128i; - let v0 = unsafe { _mm_loadu_si128(base) }; // [p0 g0 p1 g1] - let v1 = unsafe { _mm_loadu_si128(base.add(1)) }; // [p2 g2 p3 g3] - let s0 = _mm_shuffle_epi32(v0, 0xD8); // [p0 p1 g0 g1] - let s1 = _mm_shuffle_epi32(v1, 0xD8); // [p2 p3 g2 g3] - let gens = _mm_unpackhi_epi64(s0, s1); // [g0 g1 g2 g3] - let eq = _mm_cmpeq_epi32(gens, _mm_set1_epi32(stamp as i32)); - if _mm_movemask_ps(_mm_castsi128_ps(eq)) as u8 & 0x0F != 0x0F { - return None; - } - let prices = _mm_unpacklo_epi64(s0, s1); // [p0 p1 p2 p3] - let mut out = [0u32; 4]; - unsafe { _mm_storeu_si128(out.as_mut_ptr() as *mut __m128i, prices) }; - Some(out) -} - -/// SSE4.1 4-lane `next_cost < node_price` bitmask (unsigned compare via -/// `min_epu32`, like the AVX2 path). -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -#[target_feature(enable = "sse4.2")] -#[inline] -unsafe fn priceset_improved_mask4_sse41(next_cost: &[u32; 4], node_price: &[u32]) -> u8 { - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_andnot_si128, _mm_castsi128_ps, _mm_cmpeq_epi32, _mm_loadu_si128, - _mm_min_epu32, _mm_movemask_ps, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_andnot_si128, _mm_castsi128_ps, _mm_cmpeq_epi32, _mm_loadu_si128, - _mm_min_epu32, _mm_movemask_ps, - }; - let nc = unsafe { _mm_loadu_si128(next_cost.as_ptr() as *const __m128i) }; - let np = unsafe { _mm_loadu_si128(node_price.as_ptr() as *const __m128i) }; - let min = _mm_min_epu32(nc, np); - let le = _mm_cmpeq_epi32(min, nc); - let eq = _mm_cmpeq_epi32(nc, np); - let lt = _mm_andnot_si128(eq, le); - (_mm_movemask_ps(_mm_castsi128_ps(lt)) as u8) & 0x0F -} - -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -#[target_feature(enable = "sse4.2")] -#[inline] -#[allow(clippy::too_many_arguments)] -unsafe fn priceset_range_nonabort_sse41( - node_prices: &mut [u32], - nodes: &mut [HcOptimalNode], - ml_cache: &mut [[u32; 2]], - ml_stamp: u32, - profile: HcOptimalCostProfile, - stats: &HcOptState, - pos: usize, - start: usize, - max: usize, - ll0_price: u32, - off_price: u32, - base_cost: u32, - off: u32, - reps: [u32; 3], - last_pos: usize, -) -> usize { - priceset_range_vec::<4>( - node_prices, - nodes, - ml_cache, - ml_stamp, - profile, - stats, - pos, - start, - max, - ll0_price, - off_price, - base_cost, - off, - reps, - last_pos, - // SAFETY: both closures run inside this fn's sse4.2 target_feature umbrella. - |cells, stamp| unsafe { priceset_cached_prices4_sse41(cells, stamp) }, - |nc, np| unsafe { priceset_improved_mask4_sse41(nc, np) }, - ) -} - -/// wasm `simd128` 4-lane vector-load + deinterleave of cached ml-prices. -/// `u32x4_shuffle` selects the price (even) and gen (odd) lanes across the two -/// loaded vectors natively. `Some(prices)` only when all 4 gens equal `stamp` -/// (`u32x4_all_true` of the equality vector). -#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] -#[target_feature(enable = "simd128")] -#[inline] -unsafe fn priceset_cached_prices4_simd128(cells: &[[u32; 2]], stamp: u32) -> Option<[u32; 4]> { - use core::arch::wasm32::{ - u32x4_all_true, u32x4_eq, u32x4_shuffle, u32x4_splat, v128, v128_load, v128_store, - }; - debug_assert!(cells.len() >= 4); - let base = cells.as_ptr() as *const v128; - let v0 = unsafe { v128_load(base) }; // [p0 g0 p1 g1] - let v1 = unsafe { v128_load(base.add(1)) }; // [p2 g2 p3 g3] - // Lanes 0..3 index v0, 4..7 index v1. - let gens = u32x4_shuffle::<1, 3, 5, 7>(v0, v1); // [g0 g1 g2 g3] - let eq = u32x4_eq(gens, u32x4_splat(stamp)); - if !u32x4_all_true(eq) { - return None; - } - let prices = u32x4_shuffle::<0, 2, 4, 6>(v0, v1); // [p0 p1 p2 p3] - let mut out = [0u32; 4]; - unsafe { v128_store(out.as_mut_ptr() as *mut v128, prices) }; - Some(out) -} - -/// wasm `simd128` 4-lane `next_cost < node_price` bitmask. wasm has a native -/// unsigned compare (`u32x4_lt`) and `u32x4_bitmask` to pack the lanes. -#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] -#[target_feature(enable = "simd128")] -#[inline] -unsafe fn priceset_improved_mask4_simd128(next_cost: &[u32; 4], node_price: &[u32]) -> u8 { - use core::arch::wasm32::{u32x4_bitmask, u32x4_lt, v128, v128_load}; - let nc = unsafe { v128_load(next_cost.as_ptr() as *const v128) }; - let np = unsafe { v128_load(node_price.as_ptr() as *const v128) }; - u32x4_bitmask(u32x4_lt(nc, np)) -} - -#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] -#[target_feature(enable = "simd128")] -#[inline] -#[allow(clippy::too_many_arguments)] -unsafe fn priceset_range_nonabort_simd128( - node_prices: &mut [u32], - nodes: &mut [HcOptimalNode], - ml_cache: &mut [[u32; 2]], - ml_stamp: u32, - profile: HcOptimalCostProfile, - stats: &HcOptState, - pos: usize, - start: usize, - max: usize, - ll0_price: u32, - off_price: u32, - base_cost: u32, - off: u32, - reps: [u32; 3], - last_pos: usize, -) -> usize { - priceset_range_vec::<4>( - node_prices, - nodes, - ml_cache, - ml_stamp, - profile, - stats, - pos, - start, - max, - ll0_price, - off_price, - base_cost, - off, - reps, - last_pos, - // SAFETY: both closures run inside this fn's simd128 target_feature umbrella. - |cells, stamp| unsafe { priceset_cached_prices4_simd128(cells, stamp) }, - |nc, np| unsafe { priceset_improved_mask4_simd128(nc, np) }, - ) -} - -macro_rules! build_optimal_plan_impl_body { - ( - $self:expr, - $strategy_ty:ty, - $current:ident, - $current_abs_start:ident, - $current_len:ident, - $initial_state:ident, - $stats:ident, - $out:ident, - $collect:ident, - $priceset:path $(,)? - ) => {{ - let current_abs_end = $current_abs_start + $current_len; - let min_match_len = HC_OPT_MIN_MATCH_LEN; - // `HC_OPT_NUM > 0` by const definition, so `HC_OPT_NUM - 1` is safe. - let frontier_limit = $current_len.min(HC_OPT_NUM - 1); - let initial_reps = $initial_state.reps; - let initial_litlen = $initial_state.litlen; - let ldm_block_offset = $initial_state.block_offset; - let mut profile = $initial_state.profile; - profile.sufficient_match_len = $self.hc.sufficient_match_len_for_pass(profile); - // Const-fold from the strategy's associated `OPT_LEVEL` - // (upstream zstd `optLevel`): BtOpt = 0, BtUltra / BtUltra2 = 2. - // The two flags below are the only places the inner DP loop - // used to consult `parse_mode`; lifting them into const - // expressions drops one indirect read + one branch on every - // candidate insertion and every traceback step. - // `let` (not `const`) — nested `const` items inside a - // generic fn cannot project through the outer fn's type - // parameter, but a `let` binding from a const expression - // does get folded by the optimiser per monomorphisation, - // which is what we actually want here. - debug_assert!( - <$strategy_ty as super::strategy::Strategy>::USE_BT, - "build_optimal_plan_impl_body called on non-BT strategy" - ); - let abort_on_worse_match: bool = - <$strategy_ty as super::strategy::Strategy>::OPT_LEVEL == 0; - let opt_level: bool = <$strategy_ty as super::strategy::Strategy>::OPT_LEVEL >= 2; - let mut nodes = core::mem::take(&mut $self.backend.bt_mut().opt_nodes_scratch); - let mut node_prices = core::mem::take(&mut $self.backend.bt_mut().opt_node_prices_scratch); - // `frontier_limit + 2 <= HC_OPT_NODE_LEN` — bounded by const. - let frontier_buffer_size = frontier_limit + 2; - if nodes.len() < HC_OPT_NODE_LEN { - // First optimal-parse use (empty boxed slice) or an undersized - // buffer: allocate the fixed upstream-zstd-sized frontier once. The DP - // overwrites the active prefix before reading it. - nodes = alloc::vec![HcOptimalNode::default(); HC_OPT_NODE_LEN].into_boxed_slice(); - } - // The DP price array, same fixed length as `nodes`. This is the SOLE - // home of each position's price (the node struct carries no price), so - // the SIMD price-set vector-loads it directly. Initialised to u32::MAX - // so unwritten frontier cells compare as "unreachable". - if node_prices.len() < HC_OPT_NODE_LEN { - node_prices = alloc::vec![u32::MAX; HC_OPT_NODE_LEN].into_boxed_slice(); - } - let mut candidates = core::mem::take(&mut $self.backend.bt_mut().opt_candidates_scratch); - candidates.clear(); - if candidates.capacity() < MAX_HC_SEARCH_DEPTH { - candidates.reserve_exact(MAX_HC_SEARCH_DEPTH - candidates.capacity()); - } - let mut store = core::mem::take(&mut $self.backend.bt_mut().opt_store_scratch); - store.clear(); - let mut price_arena = core::mem::take(&mut $self.backend.bt_mut().opt_price_arena); - if price_arena.len() < HC_OPT_PRICE_ARENA_LEN { - price_arena = alloc::vec![[0u32; 2]; HC_OPT_PRICE_ARENA_LEN].into_boxed_slice(); - } - // Single arena → two disjoint fixed-stride regions of `[price, - // generation]` pairs (LL cache, ML cache): one base pointer + fixed - // offsets, mirroring upstream zstd's single opt workspace. Pairing - // price+generation per code keeps the optimal parser's cache probe - // on ONE line instead of two strided regions. - // SAFETY: `price_arena` is exactly `HC_OPT_PRICE_ARENA_LEN = - // 2 * HC_OPT_PRICE_STRIDE` pairs long (just ensured), so the two - // STRIDE-wide regions are in bounds and disjoint. The slices alias - // the heap buffer `price_arena` owns; that heap address is stable - // across the later move of the `price_arena` box into the result - // bundle (a `Box` move relocates only the pointer, not the heap - // data), and the slices are never used after the bundle is - // constructed. The fixed STRIDE (independent of `frontier_limit`) - // keeps every code's cell at a constant offset so the monotonic - // stamps stay valid across calls with different frontiers. - let arena_base = price_arena.as_mut_ptr(); - let mut ll_cache: &mut [[u32; 2]] = - unsafe { core::slice::from_raw_parts_mut(arena_base, HC_OPT_PRICE_STRIDE) }; - let mut ml_cache: &mut [[u32; 2]] = unsafe { - core::slice::from_raw_parts_mut(arena_base.add(HC_OPT_PRICE_STRIDE), HC_OPT_PRICE_STRIDE) - }; - $self.backend.bt_mut().opt_ll_price_stamp = $self - .backend - .bt_mut() - .opt_ll_price_stamp - .wrapping_add(1) - .max(1); - let ll_price_stamp = $self.backend.bt_mut().opt_ll_price_stamp; - $self.backend.bt_mut().opt_lit_price_stamp = $self - .backend - .bt_mut() - .opt_lit_price_stamp - .wrapping_add(1) - .max(1); - let lit_price_stamp = $self.backend.bt_mut().opt_lit_price_stamp; - $self.backend.bt_mut().opt_ml_price_stamp = $self - .backend - .bt_mut() - .opt_ml_price_stamp - .wrapping_add(1) - .max(1); - let ml_price_stamp = $self.backend.bt_mut().opt_ml_price_stamp; - let node0_price = BtMatcher::cached_lit_length_price( - profile, - $stats, - initial_litlen, - &mut ll_cache, - ll_price_stamp, - ); - nodes[0] = HcOptimalNode { - litlen: initial_litlen as u32, - reps: initial_reps, - ..HcOptimalNode::default() - }; - node_prices[0] = node0_price; - let sufficient_len = profile.sufficient_match_len; - let ll0_price = BtMatcher::cached_lit_length_price( - profile, - $stats, - 0, - &mut ll_cache, - ll_price_stamp, - ); - let ll1_price = BtMatcher::cached_lit_length_price( - profile, - $stats, - 1, - &mut ll_cache, - ll_price_stamp, - ); - let mut pos = 1usize; - let mut last_pos = 0usize; - let mut forced_end: Option = None; - let mut forced_end_state: Option = None; - // Price companion of `forced_end_state` (price no longer lives in the - // node struct; tracked alongside the forced-seed node). - let mut forced_end_price: Option = None; - let mut seed_forced_shortest_path = false; - let mut opt_ldm = HcOptLdmState { - seq_store: HcRawSeqStore { - pos: 0, - pos_in_sequence: 0, - size: $self.backend.bt_mut().ldm_sequences.len(), - }, - ..HcOptLdmState::default() - }; - let has_ldm = !$self.backend.bt_mut().ldm_sequences.is_empty(); - if has_ldm { - // `ldm_sequences` are emitted in BLOCK-relative coordinates, - // but this optimal-parser pass runs over a SEGMENT of the - // block starting at block-offset `$block_offset` and uses - // segment-relative positions throughout. Fast-forward the raw - // seq-store cursor past the bytes covered by earlier segments - // so the (segment-relative) LDM windows below land at the - // correct positions. Idempotent: `ldm_skip_raw_seq_store_bytes` - // recomputes from `pos = 0`, so re-running it per segment is - // safe. Without this, every segment after the first re-applied - // the block's leading LDM windows at the wrong offset, emitting - // matches that copy the wrong bytes (undecodable frame). - if ldm_block_offset > 0 { - $self - .backend - .bt_mut() - .ldm_skip_raw_seq_store_bytes(&mut opt_ldm.seq_store, ldm_block_offset); - } - $self - .backend - .bt_mut() - .ldm_get_next_match_and_update_seq_store(&mut opt_ldm, 0, $current_len); - } - - // Upstream zstd-like seed at rPos=0: initialize frontier with matches starting - // at current position before entering the generic forward DP loop. - if $current_len >= min_match_len { - let seed_ldm = if has_ldm { - $self.backend.bt_mut().ldm_process_match_candidate( - &mut opt_ldm, - 0, - $current_len, - min_match_len, - ) - } else { - None - }; - candidates.clear(); - // SAFETY: wrapper is in the same target_feature umbrella as the - // `$collect` kernel variant; the runtime kernel detector already - // gated entry into the wrapper. - unsafe { - $self.$collect::<$strategy_ty, true>( - $current_abs_start, - current_abs_end, - profile, - HcCandidateQuery { - reps: initial_reps, - lit_len: initial_litlen, - ldm_candidate: seed_ldm, - }, - &mut candidates, - ) - }; - if !candidates.is_empty() { - // `min_match_len >= HC_FORMAT_MINMATCH (3)` by invariant. - last_pos = (min_match_len - 1).min(frontier_limit); - for p in 1..min_match_len.min(frontier_buffer_size) { - BtMatcher::reset_opt_node(&mut nodes[p]); - // Reset the price (sole home; the node carries none). - node_prices[p] = u32::MAX; - // `initial_litlen` is the litlen carried from prior - // optimal-plan segments — its real bound is the - // current block length (the frame compressor caps - // block scan at `HC_BLOCKSIZE_MAX`), not the segment - // `current_len`. `p < min_match_len` (small constant), - // so the sum stays well within `u32::MAX`. Use - // `checked_add` FIRST so the `usize` addition itself - // cannot overflow on i686 (where `usize` is 32-bit - // and a wrapping `+` would slip past `try_from`). - let seed_litlen = initial_litlen - .checked_add(p) - .and_then(|s| u32::try_from(s).ok()) - .expect("optimal parser seed litlen out of u32 range"); - nodes[p].litlen = seed_litlen; - } - } - - if let Some(candidate) = candidates.last() { - let longest_len = candidate.match_len.min($current_len); - if longest_len > sufficient_len { - let off_base = BtMatcher::encode_offset_base_with_reps( - candidate.offset as u32, - initial_litlen, - initial_reps, - ); - let off_price = profile - .offset_price_for::($stats, off_base); - let ml_price = BtMatcher::cached_match_length_price( - profile, - $stats, - longest_len, - &mut ml_cache, - ml_price_stamp, - ); - let seq_cost = BtMatcher::add_prices( - ll0_price, - profile.match_price_from_parts(off_price, ml_price, $stats), - ); - let forced_price = BtMatcher::add_prices(node_prices[0], seq_cost); - let forced_state = HcOptimalNode { - off: candidate.offset as u32, - mlen: longest_len as u32, - litlen: 0, - reps: initial_reps, - }; - if longest_len < frontier_buffer_size && forced_price < node_prices[longest_len] { - nodes[longest_len] = forced_state; - node_prices[longest_len] = forced_price; - } - forced_end = Some(longest_len); - forced_end_state = Some(forced_state); - forced_end_price = Some(forced_price); - seed_forced_shortest_path = true; - } - } - if !seed_forced_shortest_path { - let mut prev_max_len = min_match_len - 1; - for candidate in candidates.iter() { - let max_match_len = candidate.match_len.min(frontier_limit); - if max_match_len < min_match_len { - continue; - } - let start_len = (prev_max_len + 1).max(min_match_len); - if start_len > max_match_len { - prev_max_len = prev_max_len.max(max_match_len); - continue; - } - if max_match_len > last_pos { - BtMatcher::reset_opt_nodes( - &mut nodes, - &mut node_prices, - last_pos + 1, - max_match_len, - ); - } - let off_base = BtMatcher::encode_offset_base_with_reps( - candidate.offset as u32, - initial_litlen, - initial_reps, - ); - let off_price = profile - .offset_price_for::($stats, off_base); - debug_assert!(max_match_len < frontier_buffer_size); - let nodes0_price = node_prices[0]; - for match_len in (start_len..=max_match_len).rev() { - let ml_price = BtMatcher::cached_match_length_price( - profile, - $stats, - match_len, - &mut ml_cache, - ml_price_stamp, - ); - let seq_cost = BtMatcher::add_prices( - ll0_price, - profile.match_price_from_parts(off_price, ml_price, $stats), - ); - let next_cost = BtMatcher::add_prices(nodes0_price, seq_cost); - let node_price = unsafe { *node_prices.get_unchecked(match_len) }; - if match_len > last_pos || next_cost < node_price { - let slot = unsafe { nodes.get_unchecked_mut(match_len) }; - *slot = HcOptimalNode { - off: candidate.offset as u32, - mlen: match_len as u32, - litlen: 0, - reps: initial_reps, - }; - unsafe { *node_prices.get_unchecked_mut(match_len) = next_cost }; - if match_len > last_pos { - last_pos = match_len; - } - } else if abort_on_worse_match { - break; - } - } - prev_max_len = prev_max_len.max(max_match_len); - } - if last_pos + 1 < frontier_buffer_size { - node_prices[last_pos + 1] = u32::MAX; - } - } - } - while !seed_forced_shortest_path && pos <= last_pos && pos <= frontier_limit { - debug_assert!(pos + 1 < frontier_buffer_size); - let prev_node = unsafe { *nodes.get_unchecked(pos - 1) }; - let prev_node_price = unsafe { *node_prices.get_unchecked(pos - 1) }; - if prev_node_price != u32::MAX { - let lit_len = prev_node.litlen as usize + 1; - let lit_price = { - let bt = $self.backend.bt_mut(); - BtMatcher::cached_literal_price( - profile, - $stats, - $current[pos - 1], - &mut bt.opt_lit_price_scratch, - &mut bt.opt_lit_price_generation, - lit_price_stamp, - ) - }; - let ll_delta = BtMatcher::cached_lit_length_delta_price( - profile, - $stats, - lit_len, - &mut ll_cache, - ll_price_stamp, - ); - let lit_cost = BtMatcher::add_price_delta(prev_node_price, lit_price, ll_delta); - // `node_pos_price` is the OLD price at `pos` (before the write - // below) — also the price of `prev_match`, the pre-overwrite copy. - let node_pos_price = unsafe { *node_prices.get_unchecked(pos) }; - if lit_cost <= node_pos_price { - let prev_match = unsafe { *nodes.get_unchecked(pos) }; - let slot = unsafe { nodes.get_unchecked_mut(pos) }; - *slot = prev_node; - slot.litlen = lit_len as u32; - node_prices[pos] = lit_cost; - #[allow(clippy::collapsible_if)] - if opt_level - && prev_match.mlen > 0 - && prev_match.litlen == 0 - && pos < $current_len - { - if ll1_price < ll0_price { - let next_lit_price = { - let bt = $self.backend.bt_mut(); - BtMatcher::cached_literal_price( - profile, - $stats, - $current[pos], - &mut bt.opt_lit_price_scratch, - &mut bt.opt_lit_price_generation, - lit_price_stamp, - ) - }; - let with1literal = BtMatcher::add_price_delta( - node_pos_price, - next_lit_price, - ll1_price as i32 - ll0_price as i32, - ); - let ll_delta_next = BtMatcher::cached_lit_length_delta_price( - profile, - $stats, - lit_len + 1, - &mut ll_cache, - ll_price_stamp, - ); - let with_more_literals = - BtMatcher::add_price_delta(lit_cost, next_lit_price, ll_delta_next); - let next = pos + 1; - let next_price = unsafe { *node_prices.get_unchecked(next) }; - if with1literal < with_more_literals && with1literal < next_price { - // Upstream zstd parity (zstd_opt.c:1232): `cur >= prevMatch.mlen`. - debug_assert!(pos >= prev_match.mlen as usize); - let prev_pos = pos - prev_match.mlen as usize; - { - let prev_state = unsafe { *nodes.get_unchecked(prev_pos) }; - let (_, reps_after_match) = BtMatcher::encode_offset_with_reps( - prev_match.off, - prev_state.litlen as usize, - prev_state.reps, - ); - let slot = unsafe { nodes.get_unchecked_mut(next) }; - *slot = prev_match; - slot.reps = reps_after_match; - slot.litlen = 1; - node_prices[next] = with1literal; - if next > last_pos { - last_pos = next; - } - } - } - } - } - } - } - - // Memory-resident DP (upstream zstd parity): read opt[cur] fields on - // demand instead of holding a 28-byte node copy live across the - // per-position `$collect` call below. The held copy forced LLVM - // to spill reps[3] + litlen around the (non-inlinable) call; - // reading the fields fresh on each side keeps them out of the - // cross-call live set. `nodes[pos]` is stable across `$collect` - // (it only fills `candidates`), so post-call reads are identical. - let base_cost = unsafe { *node_prices.get_unchecked(pos) }; - if base_cost == u32::MAX { - pos += 1; - continue; - } - { - let base_node = unsafe { *nodes.get_unchecked(pos) }; - if base_node.mlen > 0 && base_node.litlen == 0 { - // Upstream zstd parity (zstd_opt.c:1255): `cur >= opt[cur].mlen`. - debug_assert!(pos >= base_node.mlen as usize); - let prev_pos = pos - base_node.mlen as usize; - let prev_state = unsafe { *nodes.get_unchecked(prev_pos) }; - let (_, reps_after_match) = BtMatcher::encode_offset_with_reps( - base_node.off, - prev_state.litlen as usize, - prev_state.reps, - ); - unsafe { nodes.get_unchecked_mut(pos).reps = reps_after_match }; - } - } - - if pos + 8 > $current_len { - pos += 1; - continue; - } - - if pos == last_pos { - break; - } - - let next_price = unsafe { *node_prices.get_unchecked(pos + 1) }; - // `saturating_add` is REQUIRED here, not a masked bug: `base_cost` - // is a node price that can be the `u32::MAX` "unreachable" sentinel, - // and saturating keeps `base_cost + margin` pinned at MAX so the - // comparison stays correct. Plain `+` would wrap the sentinel and - // flip the abort decision (a ratio bug / debug overflow panic). - if abort_on_worse_match - && next_price <= base_cost.saturating_add(HC_BITCOST_MULTIPLIER / 2) - { - pos += 1; - continue; - } - - let abs_pos = $current_abs_start + pos; - let ldm_candidate = if has_ldm { - $self.backend.bt_mut().ldm_process_match_candidate( - &mut opt_ldm, - pos, - $current_len - pos, - min_match_len, - ) - } else { - None - }; - candidates.clear(); - // SAFETY: same umbrella as `$collect`. Query fields are read - // fresh here (consumed into the call's argument) so they do not - // stay live across the call; the post-call reads below are a - // separate, fresh load of the same stable `nodes[pos]`. - unsafe { - $self.$collect::<$strategy_ty, true>( - abs_pos, - current_abs_end, - profile, - HcCandidateQuery { - reps: nodes.get_unchecked(pos).reps, - lit_len: nodes.get_unchecked(pos).litlen as usize, - ldm_candidate, - }, - &mut candidates, - ) - }; - // Post-call reads of opt[cur]: fresh, born after `$collect`, so - // never part of the cross-call live set (see memory-resident note - // above). `nodes[pos]` is untouched by `$collect`. - let base_reps = unsafe { nodes.get_unchecked(pos).reps }; - let base_litlen = unsafe { nodes.get_unchecked(pos).litlen as usize }; - if let Some(candidate) = candidates.last() { - let longest_len = candidate.match_len.min($current_len - pos); - if longest_len > sufficient_len - || pos + longest_len >= HC_OPT_NUM - || pos + longest_len >= $current_len - { - let lit_len = base_litlen; - let off_base = BtMatcher::encode_offset_base_with_reps( - candidate.offset as u32, - lit_len, - base_reps, - ); - let off_price = profile - .offset_price_for::($stats, off_base); - let ml_price = BtMatcher::cached_match_length_price( - profile, - $stats, - longest_len, - &mut ml_cache, - ml_price_stamp, - ); - let seq_cost = BtMatcher::add_prices( - ll0_price, - profile.match_price_from_parts(off_price, ml_price, $stats), - ); - let forced_price = BtMatcher::add_prices(base_cost, seq_cost); - let end_pos = (pos + longest_len).min($current_len); - forced_end = Some(end_pos); - forced_end_state = Some(HcOptimalNode { - off: candidate.offset as u32, - mlen: longest_len as u32, - litlen: 0, - reps: base_reps, - }); - forced_end_price = Some(forced_price); - break; - } - } - let mut prev_max_len = min_match_len - 1; - for candidate in candidates.iter() { - // Outer loop guards `pos <= frontier_limit` (see the - // `while ... pos <= frontier_limit` condition); the - // subtraction below is therefore safe. - debug_assert!(pos <= frontier_limit); - let max_match_len = candidate - .match_len - .min($current_len - pos) - .min(frontier_limit - pos); - let min_len = min_match_len; - if max_match_len < min_len { - continue; - } - let start_len = (prev_max_len + 1).max(min_len); - if start_len > max_match_len { - prev_max_len = prev_max_len.max(max_match_len); - continue; - } - let max_next = pos + max_match_len; - if max_next > last_pos { - BtMatcher::reset_opt_nodes( - &mut nodes, - &mut node_prices, - last_pos + 1, - max_next, - ); - } - let lit_len = base_litlen; - let off_base = BtMatcher::encode_offset_base_with_reps( - candidate.offset as u32, - lit_len, - base_reps, - ); - let off_price = profile - .offset_price_for::($stats, off_base); - debug_assert!(pos + max_match_len < frontier_buffer_size); - if abort_on_worse_match { - // btopt (OPT_LEVEL == 0): reverse-iterate with early break — - // once a longer match stops improving, shorter ones are - // skipped. Order-dependent, stays scalar. - for match_len in (start_len..=max_match_len).rev() { - let next = pos + match_len; - let ml_price = BtMatcher::cached_match_length_price( - profile, - $stats, - match_len, - &mut ml_cache, - ml_price_stamp, - ); - let seq_cost = BtMatcher::add_prices( - ll0_price, - profile.match_price_from_parts(off_price, ml_price, $stats), - ); - let next_cost = BtMatcher::add_prices(base_cost, seq_cost); - let node_next_price = unsafe { *node_prices.get_unchecked(next) }; - if next > last_pos || next_cost < node_next_price { - let slot = unsafe { nodes.get_unchecked_mut(next) }; - *slot = HcOptimalNode { - off: candidate.offset as u32, - mlen: match_len as u32, - litlen: 0, - reps: base_reps, - }; - unsafe { *node_prices.get_unchecked_mut(next) = next_cost }; - if next > last_pos { - last_pos = next; - } - } else { - break; - } - } - } else { - // btultra / btultra2 (OPT_LEVEL >= 2): no abort, each - // match_len writes a distinct node => order-independent. - // Dispatch to the per-tier price-set ($priceset is the - // tier's fn: AVX2 SoA-vector compare for the avx2 wrapper, - // inline scalar otherwise) — it folds into this wrapper's - // monomorphisation, so no call ABI / runtime feature check. - #[allow(unused_unsafe)] - { - last_pos = last_pos.max(unsafe { - $priceset( - &mut node_prices, - &mut nodes, - ml_cache, - ml_price_stamp, - profile, - $stats, - pos, - start_len, - max_match_len, - ll0_price, - off_price, - base_cost, - candidate.offset as u32, - base_reps, - last_pos, - ) - }); - } - } - prev_max_len = prev_max_len.max(max_match_len); - } - - if last_pos + 1 < frontier_buffer_size { - unsafe { - *node_prices.get_unchecked_mut(last_pos + 1) = u32::MAX; - } - } - pos += 1; - } - - if last_pos == 0 { - if $current_len == 0 { - let price = node_prices[0]; - return $self.backend.bt_mut().finish_optimal_plan( - HcOptimalPlanBuffers { - nodes, - node_prices, - candidates, - store, - price_arena, - }, - (price, initial_reps, initial_litlen, 0), - ); - } - let lit_price = { - let bt = $self.backend.bt_mut(); - BtMatcher::cached_literal_price( - profile, - $stats, - $current[0], - &mut bt.opt_lit_price_scratch, - &mut bt.opt_lit_price_generation, - lit_price_stamp, - ) - }; - // `initial_litlen` is carried across optimal-plan segments; - // its real bound is the current block length, not - // `current_len`. On i686 (32-bit `usize`) `+ 1` could - // theoretically wrap if the invariant ever broke. Catch - // that explicitly via `checked_add` rather than letting a - // wrapping sum slip into the price lookup. - let next_litlen = initial_litlen - .checked_add(1) - .expect("optimal parser next litlen out of usize range"); - let ll_delta = BtMatcher::cached_lit_length_delta_price( - profile, - $stats, - next_litlen, - &mut ll_cache, - ll_price_stamp, - ); - let price = BtMatcher::add_price_delta(node_prices[0], lit_price, ll_delta); - return $self.backend.bt_mut().finish_optimal_plan( - HcOptimalPlanBuffers { - nodes, - node_prices, - candidates, - store, - price_arena, - }, - (price, initial_reps, next_litlen, 1), - ); - } - - let target_pos = forced_end.unwrap_or(last_pos.min(frontier_limit)); - // Price lives in `node_prices`, not the node struct, so carry the - // final-stretch price alongside its node (forced-seed companion or the - // frontier price at `target_pos`). - let (last_stretch, last_stretch_price) = if let Some(forced_state) = forced_end_state { - (forced_state, forced_end_price.expect("forced state has a price")) - } else { - (nodes[target_pos], node_prices[target_pos]) - }; - if last_stretch_price == u32::MAX { - return $self.backend.bt_mut().finish_optimal_plan( - HcOptimalPlanBuffers { - nodes, - node_prices, - candidates, - store, - price_arena, - }, - (u32::MAX, initial_reps, initial_litlen, $current_len), - ); - } - - if last_stretch.mlen == 0 { - return $self.backend.bt_mut().finish_optimal_plan( - HcOptimalPlanBuffers { - nodes, - node_prices, - candidates, - store, - price_arena, - }, - ( - last_stretch_price, - last_stretch.reps, - last_stretch.litlen as usize, - target_pos.min($current_len), - ), - ); - } - - let mut cur = target_pos.saturating_sub(last_stretch.mlen as usize); - let end_reps = if last_stretch.litlen == 0 { - let prev_state = nodes[cur]; - let (_, reps_after_match) = BtMatcher::encode_offset_with_reps( - last_stretch.off, - prev_state.litlen as usize, - prev_state.reps, - ); - reps_after_match - } else { - let tail_literals = last_stretch.litlen as usize; - if cur < tail_literals { - return $self.backend.bt_mut().finish_optimal_plan( - HcOptimalPlanBuffers { - nodes, - node_prices, - candidates, - store, - price_arena, - }, - ( - last_stretch_price, - last_stretch.reps, - tail_literals, - target_pos.min($current_len), - ), - ); - } - cur -= tail_literals; - last_stretch.reps - }; - let store_end = cur + 2; - if store.len() <= store_end { - store.resize(store_end + 1, HcOptimalNode::default()); - } - let mut store_start; - let mut stretch_pos = cur; - - if last_stretch.litlen > 0 { - store[store_end] = HcOptimalNode { - litlen: last_stretch.litlen, - mlen: 0, - ..HcOptimalNode::default() - }; - store_start = store_end.saturating_sub(1); - store[store_start] = last_stretch; - } - store[store_end] = last_stretch; - store_start = store_end; - - loop { - let next_stretch = nodes[stretch_pos]; - store[store_start].litlen = next_stretch.litlen; - if next_stretch.mlen == 0 { - break; - } - if store_start == 0 { - break; - } - store_start -= 1; - store[store_start] = next_stretch; - // Parser invariant: every emitted stretch is bounded by the - // current block, so `litlen + mlen <= current_len <= - // HC_BLOCKSIZE_MAX (128 KiB)`. The `as usize` widening + raw - // `+` is safe on 32-bit targets — two u32 values do NOT - // automatically fit in `usize` on i686, the block bound is - // what makes this addition safe. - let litlen = next_stretch.litlen as usize; - let mlen = next_stretch.mlen as usize; - debug_assert!(litlen + mlen <= $current_len); - let step = litlen + mlen; - if step == 0 || stretch_pos < step { - break; - } - stretch_pos -= step; - } - - let mut tail_literals = initial_litlen; - let mut store_pos = store_start; - while store_pos <= store_end { - let stretch = store[store_pos]; - let llen = stretch.litlen as usize; - let mlen = stretch.mlen as usize; - if mlen == 0 { - tail_literals = llen; - store_pos += 1; - continue; - } - $out.push(HcOptimalSequence { - offset: stretch.off, - match_len: mlen as u32, - lit_len: llen as u32, - }); - tail_literals = 0; - store_pos += 1; - } - let result = ( - last_stretch_price, - end_reps, - if last_stretch.litlen > 0 { - last_stretch.litlen as usize - } else { - tail_literals - }, - target_pos.min($current_len), - ); - $self.backend.bt_mut().finish_optimal_plan( - HcOptimalPlanBuffers { - nodes, - node_prices, - candidates, - store, - price_arena, - }, - result, - ) - }}; -} - -/// `collect_optimal_candidates_initialized` body parameterized over the per-CPU -/// kernel: the `$cpl` path is the kernel's `common_prefix_len_ptr` (used in -/// the HC chain walk fallback), and the four method-name substitutions -/// (`$bt_update`, `$bt_insert`, `$for_each_rep`, `$hash3`) route to the -/// kernel-specific wrappers of the inner helpers. With every helper under -/// the same `target_feature` umbrella, the entire per-position pipeline -/// (BT-tree fill + rep probing + hash3 probing + BT match collection / -/// HC chain walk) inlines without ABI barriers on the level22 hot path. -macro_rules! collect_optimal_candidates_initialized_body { - ( - $self:expr, - $strategy_ty:ty, - $abs_pos:ident, - $current_abs_end:ident, - $profile:ident, - $query:ident, - $out:ident, - $bt_matchfinder:ident, - $bt_update:ident, - $bt_insert:ident, - $for_each_rep:ident, - $hash3:ident, - $cpl:path $(,)? - ) => {{ - // Per-strategy compile-time const: only BtUltra2 drives the - // hash3 short-match table. All other monomorphisations drop - // the entire hash3 lookup block at codegen time. The relaxed - // implication enforces only the direction we depend on: - // if the strategy declares hash3, the table must be live. - // The reverse (`hash3_log != 0` without `USE_HASH3`) is OK — - // a future caller may pre-allocate hash3 storage without - // wiring the BtUltra2 path through. - let use_hash3: bool = <$strategy_ty as super::strategy::Strategy>::USE_HASH3; - debug_assert!(!$self.table.hash_table.is_empty()); - debug_assert!($self.table.hash3_log == 0 || !$self.table.hash3_table.is_empty()); - debug_assert!( - !use_hash3 || $self.table.hash3_log != 0, - "Strategy::USE_HASH3 = true but runtime hash3_log is 0 — call configure() first", - ); - debug_assert!(!$self.table.chain_table.is_empty()); - let min_match_len = HC_OPT_MIN_MATCH_LEN; - let reps = $query.reps; - let lit_len = $query.lit_len; - let ldm_candidate = $query.ldm_candidate; - $out.clear(); - if $abs_pos < $self.table.skip_insert_until_abs { - if let Some(ldm) = ldm_candidate { - let mut best_len_for_skip = 0usize; - let _ = super::bt::BtMatcher::push_candidate_ladder( - $out, - &mut best_len_for_skip, - ldm, - min_match_len, - ); - } - return; - } - if $bt_matchfinder { - // SAFETY: caller is in the same target_feature umbrella as - // `$bt_update`; the runtime kernel detector already gated entry. - unsafe { $self.table.$bt_update($abs_pos, $current_abs_end) }; - } - let current_idx = $abs_pos - $self.table.history_abs_start; - if current_idx + 4 > $self.table.live_history().len() { - if let Some(ldm) = ldm_candidate { - let mut best_len_for_skip = 0usize; - let _ = super::bt::BtMatcher::push_candidate_ladder( - $out, - &mut best_len_for_skip, - ldm, - min_match_len, - ); - } - return; - } - let mut best_len_for_skip = 0usize; - let mut skip_further_match_search = false; - let mut rep_len_candidate_found = false; - // SAFETY: same umbrella; closure capture is monomorphized per call. - unsafe { - $self.hc.$for_each_rep( - &$self.table, - $abs_pos, - lit_len, - reps, - $current_abs_end, - min_match_len, - |rep| { - if rep.match_len >= min_match_len { - rep_len_candidate_found = true; - } - let _ = super::bt::BtMatcher::push_candidate_ladder( - $out, - &mut best_len_for_skip, - rep, - min_match_len, - ); - if rep.match_len > $profile.sufficient_match_len { - skip_further_match_search = true; - } - // `for_each_repcode_candidate_with_reps` caps - // `rep.match_len` at the per-call `tail_limit = - // current_abs_end - abs_pos`, so `abs_pos + - // rep.match_len <= current_abs_end`. The raw sum - // therefore stays in `usize` on every supported - // target. - if $abs_pos + rep.match_len >= $current_abs_end { - skip_further_match_search = true; - } - }, - ) - }; - // Hash3 lookup runs only when the strategy enables it. The - // `use_hash3` binding above is a per-monomorphisation const, - // so non-BtUltra2 instances drop this entire block. - if use_hash3 && !skip_further_match_search && best_len_for_skip < min_match_len { - $self.table.update_hash3_until($abs_pos); - // SAFETY: same umbrella for hash3_candidate. - if let Some(h3) = unsafe { - $self - .table - .$hash3($abs_pos, $current_abs_end, min_match_len) - } { - let _ = super::bt::BtMatcher::push_candidate_ladder( - $out, - &mut best_len_for_skip, - h3, - min_match_len, - ); - if !rep_len_candidate_found - && (h3.match_len > $profile.sufficient_match_len - || $abs_pos + h3.match_len >= $current_abs_end) - { - $self.table.skip_insert_until_abs = $abs_pos + 1; - skip_further_match_search = true; - } - } - } - if !skip_further_match_search && $bt_matchfinder { - // SAFETY: same umbrella for bt_insert_and_collect_matches. - unsafe { - $self.table.$bt_insert( - $abs_pos, - $current_abs_end, - $profile, - min_match_len, - &mut best_len_for_skip, - $out, - ) - }; - } else if !skip_further_match_search { - $self.table.insert_position($abs_pos); - let max_chain_depth = $profile.max_chain_depth.min($self.hc.search_depth); - let concat = $self.table.live_history(); - // Raw `+ 9` is safe here — see `bt_insert_step_no_rebase_body!` - // for the full discussion of the upstream `STREAM_ABS_HEADROOM` - // cap in `MatchTable::add_data`. - let mut match_end_abs = $abs_pos + 9; - if max_chain_depth > 0 { - for (visited, candidate_abs) in $self - .hc - .chain_candidates(&$self.table, $abs_pos) - .into_iter() - .enumerate() - { - if visited >= max_chain_depth { - break; - } - if candidate_abs == usize::MAX { - break; - } - if candidate_abs < $self.table.window_low_abs_for_target($abs_pos) - || candidate_abs >= $abs_pos - { - continue; - } - let candidate_idx = candidate_abs - $self.table.history_abs_start; - debug_assert!( - $abs_pos <= $current_abs_end, - "HC chain walker called past current block end" - ); - let tail_limit = $current_abs_end - $abs_pos; - let base = concat.as_ptr(); - // SAFETY: history-relative indices; `tail_limit` bounds - // the scan within `concat`. `$cpl` is the kernel-specific - // common_prefix_len_ptr — call inlines because the - // surrounding wrapper carries the same target_feature. - let match_len = - unsafe { $cpl(base.add(candidate_idx), base.add(current_idx), tail_limit) }; - if match_len < min_match_len { - continue; - } - let offset = $abs_pos - candidate_abs; - if super::bt::BtMatcher::push_candidate_ladder( - $out, - &mut best_len_for_skip, - MatchCandidate { - start: $abs_pos, - offset, - match_len, - }, - min_match_len, - ) { - let candidate_end = candidate_abs + match_len; - if candidate_end > match_end_abs { - match_end_abs = candidate_end; - } - } - if match_len > HC_OPT_NUM || $abs_pos + match_len >= $current_abs_end { - break; - } - } - } - // `match_end_abs` initialized to `abs_pos + 9`; monotonic - // updates only ever extend it, so `match_end_abs - 8 >= 1`. - $self.table.skip_insert_until_abs = - $self.table.skip_insert_until_abs.max(match_end_abs - 8); - } - if let Some(ldm) = ldm_candidate { - let _ = super::bt::BtMatcher::push_candidate_ladder( - $out, - &mut best_len_for_skip, - ldm, - min_match_len, - ); - } - }}; -} - -/// `hash3_candidate` body parameterized over the per-CPU -/// `common_prefix_len_ptr` symbol. The hash3 probe checks one candidate per -/// position when invoked, so the per-call ABI savings compound across the -/// segment. Crate-private (see `bt_insert_step_no_rebase_body!`). -macro_rules! hash3_candidate_body { - ( - $table:expr, - $abs_pos:ident, - $current_abs_end:ident, - $min_match_len:ident, - $cpl:path $(,)? - ) => {{ - if $table.hash3_log == 0 { - return None; - } - let idx = $abs_pos.checked_sub($table.history_abs_start)?; - let concat = $table.live_history(); - if idx + 4 > concat.len() { - return None; - } - let hash3 = $crate::encoding::match_table::storage::MatchTable::hash_position_at( - concat, - idx, - $table.hash3_log, - 3, - ); - let entry = $table - .hash3_table - .get(hash3) - .copied() - .unwrap_or($crate::encoding::match_table::storage::HC_EMPTY); - let candidate_abs = - $crate::encoding::match_table::storage::MatchTable::stored_abs_position_fast( - entry, - $table.position_base, - $table.index_shift, - )?; - if candidate_abs < $table.history_abs_start || candidate_abs >= $abs_pos { - return None; - } - let offset = $abs_pos - candidate_abs; - if offset >= $crate::encoding::bt::HC3_MAX_OFFSET { - return None; - } - let candidate_idx = candidate_abs - $table.history_abs_start; - let tail_limit = $current_abs_end.saturating_sub($abs_pos); - let base = concat.as_ptr(); - // SAFETY: candidate/idx are within history range; tail_limit - // bounds the scan within `concat`. - let match_len = unsafe { $cpl(base.add(candidate_idx), base.add(idx), tail_limit) }; - (match_len >= $min_match_len).then_some($crate::encoding::opt::types::MatchCandidate { - start: $abs_pos, - offset, - match_len, - }) - }}; -} -pub(crate) use hash3_candidate_body; - -/// `for_each_repcode_candidate_with_reps` body parameterized over the per-CPU -/// `common_prefix_len_ptr` symbol so the per-rep prefix probe inlines under -/// the wrapper's `target_feature` umbrella instead of crossing the ABI -/// boundary through the dispatcher. Three rep probes per encoded position → -/// thousands per segment, so the per-call barrier was non-trivial. -/// -/// The callback `f` runs in the wrapper's umbrella context too, so closures -/// that capture mutable state still work (FnMut). Crate-private -/// (see `bt_insert_step_no_rebase_body!`). -macro_rules! for_each_repcode_candidate_body { - ( - $table:expr, - $abs_pos:ident, - $lit_len:ident, - $reps:ident, - $current_abs_end:ident, - $min_match_len:ident, - $f:ident, - $cpl:path $(,)? - ) => {{ - let rep_offsets: [Option; 3] = if $lit_len == 0 { - [ - Some($reps[1] as usize), - Some($reps[2] as usize), - ($reps[0] > 1).then_some(($reps[0] - 1) as usize), - ] - } else { - [ - Some($reps[0] as usize), - Some($reps[1] as usize), - Some($reps[2] as usize), - ] - }; - let concat = $table.live_history(); - let current_idx = $abs_pos - $table.history_abs_start; - if current_idx + 4 > concat.len() { - return; - } - let tail_limit = $current_abs_end.saturating_sub($abs_pos); - let base = concat.as_ptr(); - let concat_len = concat.len(); - for rep in rep_offsets.into_iter().flatten() { - if rep == 0 || rep > $abs_pos { - continue; - } - let candidate_pos = $abs_pos - rep; - if candidate_pos < $table.history_abs_start { - continue; - } - let candidate_idx = candidate_pos - $table.history_abs_start; - // Upstream zstd `ZSTD_readMINMATCH` gate (zstd_opt.c:657-674): a - // 4-byte (3-byte when min_match_len == 3) equality probe - // before the full prefix scan. Equivalent filtering — a - // mismatch here means `match_len < min_match_len`, which - // the post-scan check rejects anyway — but it skips the - // prefix-kernel call for the common no-match case (rep - // offsets rarely hit on low-redundancy input). - // - // SAFETY: `current_idx + 4 <= concat_len` (early return - // above) and `candidate_idx < current_idx` (rep >= 1), so - // both 4-byte reads stay inside `concat`. - let gate_matches = unsafe { - let cand = base.add(candidate_idx).cast::().read_unaligned(); - let cur = base.add(current_idx).cast::().read_unaligned(); - if $min_match_len == 3 { - // Compare the low-address 3 bytes regardless of - // endianness: byte-shift on LE, mask via to_le. - (cand.to_le() & 0x00FF_FFFF) == (cur.to_le() & 0x00FF_FFFF) - } else { - cand == cur - } - }; - if !gate_matches { - continue; - } - // SAFETY: `candidate_idx ≤ current_idx < concat_len` (since - // candidate_pos ≤ abs_pos and we early-returned on - // `current_idx + 4 > concat_len`). `max` clamps to the shorter - // remaining run so neither pointer overruns `concat`. - let max = (concat_len - candidate_idx) - .min(concat_len - current_idx) - .min(tail_limit); - let match_len = unsafe { $cpl(base.add(candidate_idx), base.add(current_idx), max) }; - if match_len < $min_match_len { - continue; - } - $f(MatchCandidate { - start: $abs_pos, - offset: rep, - match_len, - }); - } - }}; -} -pub(crate) use for_each_repcode_candidate_body; - -/// `bt_insert_and_collect_matches` body parameterized over the per-CPU -/// `count_match_from_indices` symbol. Same shape as -/// [`bt_insert_step_no_rebase_body`] — picks up the matching kernel through -/// `$cmf` so the per-iteration vector probe inlines under the wrapper's -/// `target_feature` umbrella. Returns nothing (matches the original method). -/// Crate-private (see `bt_insert_step_no_rebase_body!`). -macro_rules! bt_insert_and_collect_matches_body { - ( - $table:expr, - $search_depth:expr, - $abs_pos:ident, - $current_abs_end:ident, - $profile:ident, - $min_match_len:ident, - $best_len_for_skip:ident, - $out:ident, - $cmf:path $(,)? - ) => {{ - let idx = $abs_pos - $table.history_abs_start; - // Borrowed-aware live region (owned: `history[history_start..]`; - // borrowed: the in-place input `[0, block_end)`). Reborrow-then-raw-ptr - // so the slice holds NO borrow and coexists with the `&mut $table` - // binary-tree writes below. Owned is byte-identical (same bytes). - let concat: &[u8] = unsafe { - let lh = $table.live_history(); - core::slice::from_raw_parts(lh.as_ptr(), lh.len()) - }; - if idx + 8 > concat.len() { - return; - } - debug_assert!( - $abs_pos <= $current_abs_end, - "BT collect called past current block end" - ); - let tail_limit = $current_abs_end - $abs_pos; - let hash = $crate::encoding::match_table::storage::MatchTable::hash_position_at( - concat, - idx, - $table.hash_log, - $table.search_mls, - ); - // Prefetch the hash bucket now. For the large L16+ hash table over - // high-entropy input the bucket is L3/DRAM-cold, and unlike upstream's - // monolithic ZSTD_btGetAllMatches (which overlaps this miss with its - // inline rep/hash3 prologue) the read+write of `hash_table[hash]` - // below is reached with nothing to hide it behind — it stalled a large - // share of this function's cycles. Issuing the hint here lets the miss - // overlap the address setup that follows. - #[cfg(all( - target_feature = "sse", - any(target_arch = "x86", target_arch = "x86_64") - ))] - { - #[cfg(target_arch = "x86")] - use core::arch::x86::{_MM_HINT_T0, _mm_prefetch}; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{_MM_HINT_T0, _mm_prefetch}; - // SAFETY: prefetch is a hint that never faults; `hash` indexes - // `hash_table` directly below, so it is in bounds. - unsafe { - _mm_prefetch($table.hash_table.as_ptr().add(hash).cast(), _MM_HINT_T0); - } - // Prefetch the NEXT position's bucket too. The optimal-parser DP - // advances one position per iteration, so this miss is issued a - // full BT walk plus the next iteration's pre-collect work ahead of - // the collect that will read it — far more lead than the same-call - // hint above, enough to hide the full DRAM latency. - if idx + 1 + 8 <= concat.len() { - let hash_next = - $crate::encoding::match_table::storage::MatchTable::hash_position_at( - concat, - idx + 1, - $table.hash_log, - $table.search_mls, - ); - // SAFETY: prefetch never faults; an out-of-range index is a - // harmless no-op hint. - unsafe { - _mm_prefetch( - $table.hash_table.as_ptr().add(hash_next).cast(), - _MM_HINT_T0, - ); - } - } - } - let Some(relative_pos) = $table.relative_position($abs_pos) else { - return; - }; - let stored = relative_pos + 1; - let bt_mask = $table.bt_mask(); - // Hoist the BT pointer-pair table's base out of `self` once: every - // access below is `chain_table[computed_index]` through `&mut self`, - // which the optimizer cannot prove loop-invariant, so it reloads the - // Vec's (ptr,len) from the struct AND bounds-checks on every tree - // step (the upstream zstd walks a raw `U32* btable`, zstd_opt.c). The raw - // base carries no borrow, so the `&self` helper calls in the loop - // (`bt_pair_index_for_abs`, `window_low_abs_for_target`, - // `relative_position`) coexist — they read other fields, never - // `chain_table`. Indices are in bounds by the BT invariants: - // `bt_pair_index_for_abs` returns `2*(abs & bt_mask) (+1)` ≤ - // `chain_table.len()-1`, and the slots only ever hold those values. - let chain_ptr = $table.chain_table.as_mut_ptr(); - debug_assert_eq!($table.chain_table.len(), 2 << $table.bt_log()); - // See `bt_insert_step_no_rebase_body!`: saturating is needed for the - // first BT walk of a fresh frame where `abs_pos < bt_mask`. - let bt_low = $abs_pos.saturating_sub(bt_mask); - let window_low = $table.window_low_abs_for_target($abs_pos); - // Upstream zstd-style window bound in stored space so the BT-walk loop - // condition rejects out-of-window / HC_EMPTY candidates WITHOUT - // decoding them (mirrors upstream `while ... matchIndex >= matchLow`): - // one range check on `match_stored` instead of decode-then-break, - // dropping the wasted candidate_abs decode on every walk's terminating - // step. candidate_abs(s) = (position_base + s - 1) - index_shift = - // base + s (wrapping); in-window ⟺ candidate_abs - window_low < - // abs_pos - window_low ⟺ s.wrapping_add(win_off) < win_range. - // HC_EMPTY (s = 0) maps to base = (lowest representable abs) - 1 < - // window_low, so it falls out of range and ends the walk. - let win_off = $table - .position_base - .wrapping_sub(1) - .wrapping_sub($table.index_shift) - .wrapping_sub(window_low); - let win_range = $abs_pos - window_low; - // Raw `+ 9` is safe here — see `bt_insert_step_no_rebase_body!` - // for the full discussion of the upstream `STREAM_ABS_HEADROOM` - // cap in `MatchTable::add_data`. - let mut match_end_abs = $abs_pos + 9; - let mut compares_left = $profile.max_chain_depth.min($search_depth); - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - let pair_idx = $table.bt_pair_index_for_abs($abs_pos); - let mut smaller_slot = pair_idx; - let mut larger_slot = pair_idx + 1; - let mut match_stored = $table.hash_table[hash]; - $table.hash_table[hash] = stored; - // Upstream zstd semantics: `bestLength` starts at `lengthToBeat - 1`; rep/hash3 - // probing may raise it; BT then only reports strictly longer matches. - // `min_match_len >= HC_FORMAT_MINMATCH (3)` by configure invariant, - // so `min_match_len - 1 >= 2` cannot underflow. - debug_assert!( - $min_match_len >= $crate::encoding::cost_model::HC_FORMAT_MINMATCH, - "min_match_len must be at least HC_FORMAT_MINMATCH" - ); - let mut best_len = (*$best_len_for_skip).max($min_match_len - 1); - - // Upstream zstd-form loop condition: the stored-space window range check - // (`s.wrapping_add(win_off) < win_range`) rejects out-of-window and - // HC_EMPTY candidates here, so the terminating step never enters the - // body — no wasted candidate_abs decode, matching upstream's - // `while ... matchIndex >= matchLow`. - while compares_left > 0 && (match_stored as usize).wrapping_add(win_off) < win_range { - compares_left -= 1; - // The condition proved this candidate is in `[window_low, - // abs_pos)`, so `match_stored >= 1` (HC_EMPTY is out of range) and - // the `- 1` cannot underflow; candidate_abs == base + match_stored. - let candidate_abs = ($table.position_base + (match_stored as usize - 1)) - .wrapping_sub($table.index_shift); - - let next_pair_idx = $table.bt_pair_index_for_abs(candidate_abs); - // SAFETY: `next_pair_idx (+1)` = `2*(candidate_abs & bt_mask) (+1)` - // ≤ `chain_table.len()-1`; `chain_ptr` is the hoisted live base, - // table not realloc'd during the walk. - let next_smaller = unsafe { *chain_ptr.add(next_pair_idx) }; - let next_larger = unsafe { *chain_ptr.add(next_pair_idx + 1) }; - let seed_len = common_length_smaller.min(common_length_larger); - let candidate_idx = candidate_abs - $table.history_abs_start; - // SAFETY: BT walk invariant — `candidate_idx + tail_limit ≤ - // concat.len()`. - let match_len = unsafe { $cmf(concat, idx, candidate_idx, tail_limit, seed_len) }; - - if match_len > best_len { - let offset = $abs_pos - candidate_abs; - let accepted = $crate::encoding::bt::BtMatcher::push_candidate_ladder( - $out, - $best_len_for_skip, - $crate::encoding::opt::types::MatchCandidate { - start: $abs_pos, - offset, - match_len, - }, - $min_match_len, - ); - if accepted { - best_len = match_len; - // BT walker invariants: `candidate_abs < abs_pos` - // and `match_len <= tail_limit = current_abs_end - - // abs_pos`. So `candidate_abs + match_len < - // abs_pos + tail_limit = current_abs_end`, which - // fits in `usize` on every supported target (32-bit - // i686 included) — the addition stays within the - // current block. - let candidate_end = candidate_abs + match_len; - if candidate_end > match_end_abs { - match_end_abs = candidate_end; - } - if match_len >= tail_limit - || match_len > $crate::encoding::cost_model::HC_OPT_NUM - { - break; - } - } - } - - if match_len >= tail_limit { - break; - } - - let candidate_next = candidate_idx + match_len; - let current_next = idx + match_len; - // SAFETY: first-differing positions after a match_len-long prefix; - // match_len < tail_limit (break above) + BT-walk bound - // idx/candidate_idx + tail_limit <= concat.len() keep both in range. - if unsafe { - *concat.get_unchecked(candidate_next) < *concat.get_unchecked(current_next) - } { - // SAFETY: `smaller_slot` holds a valid pair index (init - // `pair_idx`, updated to `next_pair_idx + 1`); the `usize::MAX` - // sentinel is set only just before `break`, never written here. - unsafe { *chain_ptr.add(smaller_slot) = match_stored }; - common_length_smaller = match_len; - if candidate_abs <= bt_low { - smaller_slot = usize::MAX; - break; - } - smaller_slot = next_pair_idx + 1; - match_stored = next_larger; - } else { - // SAFETY: as above for `larger_slot`. - unsafe { *chain_ptr.add(larger_slot) = match_stored }; - common_length_larger = match_len; - if candidate_abs <= bt_low { - larger_slot = usize::MAX; - break; - } - larger_slot = next_pair_idx; - match_stored = next_smaller; - } - } - - // SAFETY: both slots, when not the `usize::MAX` sentinel, hold valid - // pair indices into the hoisted `chain_table` base. - if smaller_slot != usize::MAX { - unsafe { - *chain_ptr.add(smaller_slot) = $crate::encoding::match_table::storage::HC_EMPTY - }; - } - if larger_slot != usize::MAX { - unsafe { - *chain_ptr.add(larger_slot) = $crate::encoding::match_table::storage::HC_EMPTY - }; - } - - // Dict dual-probe (upstream zstd `ZSTD_dictMatchState`, zstd_opt.c:777-813): - // after the live tree, descend the immutable dictionary BINARY TREE - // (built in `prime_dms_bt`) with its OWN compare budget and push any - // dict match longer than the live best into the ladder. The DUBT - // descent reaches the longest dict match efficiently (a hash-chain - // surfaced only the few same-bucket candidates and left most of the - // dict savings unrealised at btlazy2 / btopt). Dict positions are - // dictionary-relative concat indices in `[0, region)`, pinned at the - // front of history, so a dict candidate at `dict_idx` sits at offset - // `idx - dict_idx` (no upstream zstd `dmsIndexDelta`). The optimal parser - // prices these (its DP lookahead values the repcode chain a dict match - // seeds); the greedy/lazy parser commits the longest. - if let Some(dms) = $table.dms.table() { - let region = $table.dms.region_len(); - let dh = $crate::encoding::match_table::storage::MatchTable::hash_position_at( - concat, - idx, - dms.hash_log, - dms.mls, - ); - let mut dcur = dms.hash_table[dh]; - // DUBT seed lengths: bytes already known common on each side, so - // `$cmf` resumes from there (upstream zstd commonLengthSmaller/Larger). - let mut common_smaller = 0usize; - let mut common_larger = 0usize; - let mut dms_compares = $profile.max_chain_depth.min($search_depth); - while dms_compares > 0 && dcur != $crate::encoding::match_table::storage::HC_EMPTY { - let dict_idx = (dcur - 1) as usize; - // The dict tree holds only dict positions (`< region <= idx`). - if dict_idx >= region || dict_idx >= idx { - break; - } - dms_compares -= 1; - let pair = 2 * dict_idx; - let seed = common_smaller.min(common_larger); - // SAFETY: `dict_idx < idx` and `idx + tail_limit <= - // concat.len()` (checked at entry); same umbrella as the live - // walk's `$cmf`. `seed <= prior match_len <= tail_limit`. - let match_len = unsafe { $cmf(concat, idx, dict_idx, tail_limit, seed) }; - if match_len > best_len { - let offset = idx - dict_idx; - let accepted = $crate::encoding::bt::BtMatcher::push_candidate_ladder( - $out, - $best_len_for_skip, - $crate::encoding::opt::types::MatchCandidate { - start: $abs_pos, - offset, - match_len, - }, - $min_match_len, - ); - if accepted { - best_len = match_len; - let candidate_end = $abs_pos + match_len; - if candidate_end > match_end_abs { - match_end_abs = candidate_end; - } - if match_len > $crate::encoding::cost_model::HC_OPT_NUM { - break; - } - } - } - // Match reached the block tail: can't order the pair (upstream zstd - // `ip+matchLength == iLimit`), and indexing `concat[idx + - // match_len]` below would step past the searchable region. - if match_len >= tail_limit { - break; - } - // Descend the DUBT (upstream zstd zstd_opt.c:806-811): dict candidate - // smaller than input → its larger child is closer to `idx`. - if concat[dict_idx + match_len] < concat[idx + match_len] { - common_smaller = match_len; - dcur = dms.chain_table[pair + 1]; - } else { - common_larger = match_len; - dcur = dms.chain_table[pair]; - } - } - } - - // `match_end_abs >= abs_pos + 9 >= 9` (initialized and monotonic), - // so `match_end_abs - 8 >= 1` cannot underflow. - $table.skip_insert_until_abs = match_end_abs - 8; - }}; -} -pub(crate) use bt_insert_and_collect_matches_body; - -impl HcMatchGenerator { - /// Heap bytes this generator owns: the shared match table plus the BT - /// backend's optimal-parser / LDM scratch (the HC knobs are inline). - fn heap_size(&self) -> usize { - self.table.heap_size() + self.backend.heap_size() - } - - fn should_run_btultra2_seed_pass( - &self, - current_len: usize, - ) -> bool { - // The in-block two-pass dynamic-stats seed (`initStats_ultra`) - // is btultra2-only. `TWO_PASS_SEED` is `false` for every other - // strategy — including btultra, which now shares the hash3 - // short-match probe but stays single-pass — so the seed call and - // its body drop at codegen time for all non-btultra2 kernels. - if !S::TWO_PASS_SEED { - return false; - } - let HcBackend::Bt(bt) = &self.backend else { - return false; - }; - bt.opt_state.lit_length_sum == 0 - && bt.opt_state.dictionary_seed.is_none() - && !self.table.dictionary_primed_for_frame - && bt.ldm_sequences.is_empty() - && self.table.window_size == current_len - && self.table.history_abs_start == 0 - && self.table.chunk_lens.len() == 1 - && current_len > HC_PREDEF_THRESHOLD - } - - fn new(max_window_size: usize) -> Self { - Self { - table: super::match_table::storage::MatchTable::new(max_window_size), - hc: super::hc::HcMatcher::new(2, HC_SEARCH_DEPTH, HC_TARGET_LEN), - // Default to the zero-sized HC backend; `configure()` swaps - // in a `BtMatcher` only when an optimal strategy lands. - backend: HcBackend::Hc, - // Lazy is the per-construct default — every production - // caller calls `configure()` before the first encode and - // overwrites this. Tests that drive `HcMatchGenerator` - // without calling `configure()` end up in the - // `start_matching_lazy` arm of the test dispatcher, which - // matches the previous default behaviour. - strategy_tag: super::strategy::StrategyTag::Lazy, - } - } - - fn configure(&mut self, config: HcConfig, tag: super::strategy::StrategyTag, window_log: u8) { - use super::strategy::StrategyTag; - // Mirror the driver-resolved strategy tag so the - // `#[cfg(test)] start_matching` dispatcher can route - // BtOpt / BtUltra / BtUltra2 to distinct monomorphisations. - self.strategy_tag = tag; - let is_btultra2 = tag == StrategyTag::BtUltra2; - let uses_bt = matches!( - tag, - StrategyTag::Btlazy2 - | StrategyTag::BtOpt - | StrategyTag::BtUltra - | StrategyTag::BtUltra2 - ); - // btultra and btultra2 both run the mls=3 hash3 short-match probe - // (clevels.h minMatch 3). The `is_btultra2` flag below stays - // exclusive to btultra2 because it tweaks the BT rebase boundary, - // not match finding. - let wants_hash3 = matches!(tag, StrategyTag::BtUltra | StrategyTag::BtUltra2); - let next_hash3_log = if wants_hash3 { - HC3_HASH_LOG.min(window_log as usize) - } else { - 0 - }; - let resize = self.table.hash_log != config.hash_log - || self.table.chain_log != config.chain_log - || self.table.hash3_log != next_hash3_log; - // Capture the layout flip BEFORE `uses_bt` is overwritten below — it - // feeds the dms invalidation (the dms is keyed by layout too). - let uses_bt_changed = self.table.uses_bt != uses_bt; - self.table.hash_log = config.hash_log; - self.table.chain_log = config.chain_log; - self.table.hash3_log = next_hash3_log; - self.hc.search_depth = if uses_bt { - config.search_depth - } else { - config.search_depth.min(MAX_HC_SEARCH_DEPTH) - }; - self.hc.target_len = config.target_len; - // Mirror strategy-derived flags + HC search depth onto MatchTable - // so the BT walker and rebase machinery can read them directly - // without dispatching back through HcMatchGenerator. - self.table.search_depth = self.hc.search_depth; - self.table.is_btultra2 = is_btultra2; - self.table.uses_bt = uses_bt; - // BT finder hash width, upstream zstd `mls = BOUNDED(4, cParams.minMatch, 6)`, - // carried explicitly in the level config so a `target_length` override - // cannot silently flip the finder between 5- and 4-byte hashing. Only - // the BT body reads it; HC/lazy levels leave it at 4. clevels.h - // (srcSize > 256 KiB tier): btlazy2 L13-15 + btopt L16 are minMatch=5, - // btopt L17 is minMatch=4, btultra/btultra2 are minMatch=3 (4-byte main - // hash + the hash3 short-match probe). - // The cached dms is keyed by the full (region, layout, mls, hash_log) - // shape that `build_dms!` validates on the normal prime path, but the - // reborrow fast path in `MatchTable::reset` reuses it on `dms.is_primed()` - // ALONE. A reused-compressor level switch can change the search mls (e.g. - // btlazy2 -> lazy), the table geometry (hash_log / chain_log / hash3, - // captured in `resize`), OR the HC<->BT layout (`uses_bt_changed`) - // independently of each other, and any of them leaves the dms hashed for - // a different shape. Invalidate on ANY so the next dict frame re-primes at - // the new shape (configure runs before reset) instead of probing a - // mismatched dms and silently degrading match quality. Over-invalidation - // only costs a re-prime, which a real shape change needs anyway. - let mls_changed = self.table.search_mls != config.search_mls; - if resize || mls_changed || uses_bt_changed { - self.table.dms.invalidate(); - } - self.table.search_mls = config.search_mls; - // Stage D: promote the backend discriminator. HC modes drop the - // BT scratch buffers entirely; switching back into a BT mode - // allocates a fresh `BtMatcher` on demand. - match (&self.backend, self.table.uses_bt) { - (HcBackend::Hc, true) => { - self.backend = HcBackend::Bt(alloc::boxed::Box::new(super::bt::BtMatcher::new())); - } - (HcBackend::Bt(_), false) => { - self.backend = HcBackend::Hc; - } - _ => {} - } - if resize && !self.table.hash_table.is_empty() { - // Force reallocation on next ensure_tables() call. - self.table.hash_table.clear(); - self.table.hash3_table.clear(); - self.table.chain_table.clear(); - } - } - - fn seed_dictionary_entropy( - &mut self, - huff: Option<&crate::huff0::huff0_encoder::HuffmanTable>, - ll: Option<&crate::fse::fse_encoder::FSETable>, - ml: Option<&crate::fse::fse_encoder::FSETable>, - of: Option<&crate::fse::fse_encoder::FSETable>, - ) { - if let HcBackend::Bt(bt) = &mut self.backend { - bt.opt_state.seed_dictionary_entropy(huff, ll, ml, of); - } - } - - /// Install (or clear) the long-distance-match producer (#27). Only - /// the BT backend owns an `ldm_producer` slot; on the HC (lazy) - /// backend the producer is dropped because there is no optimal-parser - /// candidate buffer to seed. Call after [`Self::reset`]. - #[cfg(feature = "hash")] - fn set_ldm_producer(&mut self, producer: Option) { - if let HcBackend::Bt(bt) = &mut self.backend { - bt.ldm_producer = producer; - } - } - - /// Move the LDM producer out of the BT backend, leaving `None`. Used by the - /// dictionary snapshot path: the producer carries no dictionary state (LDM - /// is not dict-primed; its hash table is empty at capture), so it is not - /// retained in the snapshot — the working frame's freshly-reset producer is - /// reinstated on restore instead. - #[cfg(feature = "hash")] - fn take_ldm_producer(&mut self) -> Option { - if let HcBackend::Bt(bt) = &mut self.backend { - bt.ldm_producer.take() - } else { - None - } - } - - fn reset(&mut self, reuse_space: impl FnMut(Vec)) { - self.table.reset(reuse_space); - if let HcBackend::Bt(bt) = &mut self.backend { - bt.reset(); - } - } - - /// Backfill positions from the tail of the previous slice that couldn't be - /// hashed at the time (insert_position needs 4 bytes of lookahead). - fn skip_matching(&mut self, incompressible_hint: Option) { - self.table.skip_matching(incompressible_hint); - } - - /// Runtime-dispatched entry kept only for in-crate tests. Production - /// callers reach the inner loops through - /// [`Self::start_matching_strategy`] / [`MatchGeneratorDriver::compress_block`] - /// which pick the lazy / optimal arm from `S::USE_BT` at - /// monomorphisation time. - #[cfg(test)] - fn start_matching(&mut self, mut handle_sequence: impl for<'a> FnMut(Sequence<'a>)) { - use super::strategy::{self, StrategyTag}; - // Dispatch on the mirrored `strategy_tag` so each test runs - // under the same monomorphisation production would pick. - // `BtOpt` / `BtUltra` / `BtUltra2` remain distinct here even - // though `table.uses_bt` / `is_btultra2` alone can't separate - // BtOpt from BtUltra. - match self.strategy_tag { - StrategyTag::Fast | StrategyTag::Dfast | StrategyTag::Greedy | StrategyTag::Lazy => { - self.start_matching_lazy(&mut handle_sequence) - } - StrategyTag::Btlazy2 => self.start_matching_btlazy2(&mut handle_sequence), - StrategyTag::BtOpt => { - self.start_matching_optimal::(&mut handle_sequence) - } - StrategyTag::BtUltra => { - self.start_matching_optimal::(&mut handle_sequence) - } - StrategyTag::BtUltra2 => { - self.start_matching_optimal::(&mut handle_sequence) - } - } - } - - /// Strategy-aware entry point used by - /// [`MatchGeneratorDriver::compress_block`]. Branches on - /// `S::USE_BT` — a compile-time `const` — so each - /// monomorphisation keeps exactly one arm: `Lazy` / - /// `Fast` / `Dfast` / `Greedy` see only `start_matching_lazy`, - /// `BtOpt` / `BtUltra` / `BtUltra2` see only - /// `start_matching_optimal`. The inherent test-only - /// [`HcMatchGenerator::start_matching`] reaches the same arms by - /// runtime-matching on `self.strategy_tag` (the parse-mode field - /// has been removed); production never invokes that path. - pub(crate) fn start_matching_strategy( - &mut self, - handle_sequence: &mut impl for<'a> FnMut(Sequence<'a>), - ) { - debug_assert_eq!( - self.table.uses_bt, - S::USE_BT, - "Strategy::USE_BT disagrees with runtime table.uses_bt at HC dispatch" - ); - if S::USE_BT { - self.start_matching_optimal::(handle_sequence) - } else { - self.start_matching_lazy(handle_sequence) - } - } - - /// Dispatcher: pick the dict-aware monomorph when a separate dms is primed - /// (attach-mode dictionary), else the no-dict monomorph. Mirrors upstream's - /// compile-time `dictMode` split — the `DICT = false` body carries no dms - /// code at all, so the no-dict hot path is unaffected by the dict search. - pub(crate) fn start_matching_lazy( - &mut self, - handle_sequence: impl for<'a> FnMut(Sequence<'a>), - ) { - if self.table.dms.is_primed() { - self.start_matching_lazy_impl::(handle_sequence); - } else { - self.start_matching_lazy_impl::(handle_sequence); - } - } - - fn start_matching_lazy_impl( - &mut self, - mut handle_sequence: impl for<'a> FnMut(Sequence<'a>), - ) { - self.table.ensure_tables(); - - // `current_block_range()` is borrowed-aware: owned → last committed - // chunk; borrowed → the staged in-place block range. - let (current_abs_start, current_len) = self.table.current_block_range(); - if current_len == 0 { - return; - } - // The current block is the tail of `history` (owned) or the staged - // borrowed range (`get_last_space()` resolves both). Hoist it as a raw - // slice: the routine mutates the hash/chain tables + `offset_hist` but - // never reallocates `history`, so the slice stays valid and we avoid - // re-borrowing `self.table` (which would conflict with the - // `offset_hist` write). - let current_ptr = self.table.get_last_space().as_ptr(); - let current: &[u8] = unsafe { core::slice::from_raw_parts(current_ptr, current_len) }; - - // Full live history (dict + committed blocks + current block), hoisted - // ONCE for the whole position scan and threaded into every - // `find_best_match` / `pick_lazy_match` call. `live_history()` is - // loop-invariant here (the scan mutates the hash/chain tables + - // `offset_hist` but never the history bytes or length), so re-fetching - // it per find — inside `hash_chain_candidate` + the rep probe, plus - // again for each lazy lookahead at pos+1 / pos+2 — was pure - // per-position overhead. Same raw-slice detach as `current` so the - // loop's `&mut self.table` inserts coexist with this `&[u8]`. - let concat: &[u8] = { - let lh = self.table.live_history(); - unsafe { core::slice::from_raw_parts(lh.as_ptr(), lh.len()) } - }; - // Dict-match-state primed flag, hoisted ONCE for the scan: it is - // block-invariant (the dict is primed before the block) and lives on the - // cold `dms` cacheline, so the per-find `dms.is_primed()` load was a - // measurable hot-path cost (~8% of `hash_chain_candidate` on the - // dict-over-random fixture). The `DICT = false` monomorph ignores it. - let dms_primed = self.table.dms.is_primed(); - - let current_abs_end = current_abs_start + current_len; - self.table - .backfill_boundary_positions(current_abs_start, current_abs_end); - - let mut pos = 0usize; - let mut literals_start = 0usize; - while pos + HC_MIN_MATCH_LEN <= current_len { - let abs_pos = current_abs_start + pos; - let lit_len = pos - literals_start; - - // `find_best_match` returns the forward `(offset, length)` in - // registers (`HcMatch`, 16 bytes) — no 24-byte `MatchCandidate` / - // 32-byte `Option` spilled-and-copied per position. The backward - // extension that yields `start` runs ONCE here, after the lazy - // decision settles, exactly like upstream's lazy loop. - let best = - self.hc - .find_best_match::(concat, dms_primed, &self.table, abs_pos, lit_len); - if best.is_match() { - if self.hc.pick_lazy_match::( - concat, - dms_primed, - &self.table, - abs_pos, - lit_len, - best, - ) { - // Backward-extend over the literal run (upstream `zstd_lazy.c` - // after rep-vs-chain selection). The offset is preserved; - // `start` and `match_len` grow by the same amount, bounded by - // `literals_start` (the `min_abs` floor) so it never crosses - // an already-emitted sequence. - let history_abs_start = self.table.history_abs_start; - let min_abs = abs_pos - lit_len; - let mut start_abs = abs_pos; - let mut cand_abs = abs_pos - best.offset; - let mut match_len = best.match_len; - while start_abs > min_abs - && cand_abs > history_abs_start - && concat[cand_abs - history_abs_start - 1] - == concat[start_abs - history_abs_start - 1] - { - start_abs -= 1; - cand_abs -= 1; - match_len += 1; - } - self.table.insert_match_span(abs_pos, start_abs + match_len); - let start = start_abs - current_abs_start; - let literals = ¤t[literals_start..start]; - handle_sequence(Sequence::Triple { - literals, - offset: best.offset, - match_len, - }); - let _ = encode_offset_with_history( - best.offset as u32, - literals.len() as u32, - &mut self.table.offset_hist, - ); - pos = start + match_len; - literals_start = pos; - continue; - } - // Lazy lookahead found a better match at `abs_pos + 1` / `+ 2` - // (defer): advance exactly ONE byte (upstream - // `ZSTD_compressBlock_lazy_generic`) so the deferred candidate is - // re-evaluated at its own position; the no-match skip below could - // jump past it once the literal run reaches 256+ bytes. - self.table.insert_position(abs_pos); - pos += 1; - continue; - } - // No match found. - self.table.insert_position(abs_pos); - // Lazy skipping (upstream zstd `ZSTD_compressBlock_lazy_generic`, - // zstd_lazy.c:1614): advance faster over runs with no match. - // `step = ((ip - anchor) >> kSearchStrength) + 1` with - // kSearchStrength = 8, where `ip - anchor` is the current - // literal-run length. On compressible input the run stays short - // (step == 1, identical to a 1-byte advance); on incompressible - // / dict-over-random input the run grows so the parser skips - // ahead (one search per `step` positions) instead of searching - // every byte. Skipped positions are not inserted, mirroring - // upstream (it inserts only searched positions during a no-match - // run). Ratio follows upstream (not byte-identical). - let step = ((pos - literals_start) >> 8) + 1; - pos += step; - // No clamp needed before the tail loop: the search bound and the - // hashable bound are both `pos + HC_MIN_MATCH_LEN <= current_len` - // (HC_MIN_MATCH_LEN == 4 == the insert width), so there is no - // non-searchable-but-hashable anchor to miss. Positions the skip - // jumps over inside the searchable region are intentionally not - // inserted — same as upstream zstd, which advances past them via - // the identical `ip += step` and never hashes them either. - } - - // Insert remaining hashable positions in the tail (the matching loop - // stops at HC_MIN_MATCH_LEN but insert_position only needs 4 bytes). - while pos + 4 <= current_len { - self.table.insert_position(current_abs_start + pos); - pos += 1; - } - - if literals_start < current_len { - handle_sequence(Sequence::Literals { - literals: ¤t[literals_start..], - }); - } - } - - /// Register the borrowed input window for the no-copy one-shot path. - /// # Safety - /// `buffer` must outlive the borrowed scans (see `MatchTable`). - pub(crate) unsafe fn set_borrowed_window(&mut self, buffer: &[u8]) { - // SAFETY: forwarded liveness contract. - unsafe { self.table.set_borrowed_window(buffer) }; - } - - pub(crate) fn clear_borrowed_window(&mut self) { - self.table.clear_borrowed_window(); - } - - /// Borrowed (no-copy) equivalent of [`Self::start_matching_lazy`]: stage - /// the in-place block range, then run the same lazy chain parse. The - /// parse reads its range via `current_block_range()` and its bytes via - /// `get_last_space()` / `live_history()`, all borrowed-aware, so the block - /// is scanned in place with the per-position window_low offset cap. - pub(crate) fn start_matching_lazy_borrowed( - &mut self, - block_start: usize, - block_end: usize, - handle_sequence: impl for<'a> FnMut(Sequence<'a>), - ) { - self.table.stage_borrowed_block(block_start, block_end); - self.start_matching_lazy(handle_sequence); - } - - /// Borrowed (no-copy) equivalent of the lazy `skip_matching`: stage the - /// in-place block, then seed positions without an owned-history append. - pub(crate) fn skip_matching_borrowed( - &mut self, - block_start: usize, - block_end: usize, - incompressible_hint: Option, - ) { - self.table.stage_borrowed_block(block_start, block_end); - self.table.skip_matching(incompressible_hint); - } - - /// Upstream zstd `ZSTD_btlazy2` (levels 13-15): binary-tree match finder with a - /// greedy/lazy parse. Bare dispatcher — resolves the runtime tier ONCE - /// per block via `select_kernel()` and calls the matching - /// `start_matching_btlazy2_` wrapper, so the per-position BT - /// collect runs under a single `#[target_feature]` umbrella (mirrors - /// `build_optimal_plan_impl`). See `start_matching_btlazy2_body!` for the - /// shared loop. - fn start_matching_btlazy2(&mut self, mut handle_sequence: impl for<'a> FnMut(Sequence<'a>)) { - #[cfg(all(target_arch = "aarch64", target_endian = "little"))] - unsafe { - self.start_matching_btlazy2_neon(&mut handle_sequence) - } - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - use crate::encoding::fastpath::{FastpathKernel, select_kernel}; - match select_kernel() { - FastpathKernel::Avx2Bmi2 => unsafe { - self.start_matching_btlazy2_avx2_bmi2(&mut handle_sequence) - }, - FastpathKernel::Sse42 => unsafe { - self.start_matching_btlazy2_sse42(&mut handle_sequence) - }, - FastpathKernel::Scalar => self.start_matching_btlazy2_scalar(&mut handle_sequence), - } - } - #[cfg(not(any( - all(target_arch = "aarch64", target_endian = "little"), - target_arch = "x86", - target_arch = "x86_64" - )))] - { - self.start_matching_btlazy2_scalar(&mut handle_sequence) - } - } - - #[cfg(all(target_arch = "aarch64", target_endian = "little"))] - #[target_feature(enable = "neon")] - unsafe fn start_matching_btlazy2_neon( - &mut self, - mut handle_sequence: impl for<'a> FnMut(Sequence<'a>), - ) { - start_matching_btlazy2_body!( - self, - handle_sequence, - collect_optimal_candidates_initialized_neon, - crate::encoding::fastpath::neon::count_match_from_indices - ) - } - - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - #[target_feature(enable = "sse4.2")] - unsafe fn start_matching_btlazy2_sse42( - &mut self, - mut handle_sequence: impl for<'a> FnMut(Sequence<'a>), - ) { - start_matching_btlazy2_body!( - self, - handle_sequence, - collect_optimal_candidates_initialized_sse42, - crate::encoding::fastpath::sse42::count_match_from_indices - ) - } - - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - #[target_feature(enable = "avx2,bmi2")] - unsafe fn start_matching_btlazy2_avx2_bmi2( - &mut self, - mut handle_sequence: impl for<'a> FnMut(Sequence<'a>), - ) { - start_matching_btlazy2_body!( - self, - handle_sequence, - collect_optimal_candidates_initialized_avx2_bmi2, - crate::encoding::fastpath::avx2_bmi2::count_match_from_indices - ) - } - - // Scalar wrapper: no `#[target_feature]`; `$collect` (the scalar collect) - // is a safe fn, so the body macro's `unsafe` block is inert here. Same cfg - // as `collect_optimal_candidates_initialized_scalar` (absent on - // aarch64-little, where NEON is the baseline tier). - #[cfg(not(all(target_arch = "aarch64", target_endian = "little")))] - #[allow(unused_unsafe)] - fn start_matching_btlazy2_scalar( - &mut self, - mut handle_sequence: impl for<'a> FnMut(Sequence<'a>), - ) { - start_matching_btlazy2_body!( - self, - handle_sequence, - collect_optimal_candidates_initialized_scalar, - crate::encoding::fastpath::scalar::count_match_from_indices - ) - } - - fn start_matching_optimal( - &mut self, - mut handle_sequence: impl for<'a> FnMut(Sequence<'a>), - ) { - self.table.ensure_tables(); - // Borrowed-aware: owned → last committed chunk; borrowed → staged - // in-place block range. - let (current_abs_start, current_len) = self.table.current_block_range(); - if current_len == 0 { - return; - } - let current_ptr = self.table.get_last_space().as_ptr(); - // `start_matching_optimal()` mutates tables/state but never mutates or - // reallocates `self.table.history`, so this tail slice remains valid for - // the duration of the routine and avoids cloning the full block. - let current = unsafe { core::slice::from_raw_parts(current_ptr, current_len) }; - - let current_abs_end = current_abs_start + current_len; - self.table - .apply_limited_update_after_long_match(current_abs_start); - let hash3_start_cursor = self - .table - .skip_insert_until_abs - .max(self.table.history_abs_start); - self.table - .backfill_boundary_positions(current_abs_start, current_abs_end); - self.table.next_to_update3 = hash3_start_cursor; - // Borrow split: `prepare_ldm_candidates` needs immutable - // access to the live history (the post-`history_start` - // slice of `self.table.history`) while it mutates the LDM - // bucket table owned by `self.backend.bt_mut()`. Both live - // in disjoint fields of `Self`, so we capture the slice + - // its base before reaching for `bt_mut()`. - // - // The producer operates in absolute stream coordinates - // throughout; `live_history[0]` corresponds to absolute - // `history_abs_start` (upstream zstd `base + dictLimit`), and the - // abs→slice translation happens inside the producer at - // each `live_history[..]` access. Passing the full - // `history` Vec would index into the dead prefix (the - // bytes already retired past `history_start`). - let live_history = self.table.live_history(); - let history_abs_start = self.table.history_abs_start; - self.backend.bt_mut().prepare_ldm_candidates( - live_history, - history_abs_start, - current_abs_start, - current_len, - ); - - if self.should_run_btultra2_seed_pass::(current_len) { - self.run_btultra2_seed_pass(current, current_abs_start, current_len); - } - - // Const-generic profile selection: every field is folded from - // S's associated consts (MAX_CHAIN_DEPTH / - // SUFFICIENT_MATCH_LEN / ACCURATE_PRICE / FAVOR_SMALL_OFFSETS), - // so the optimiser produces the literal at codegen time - // without a runtime match. - let profile = HcOptimalCostProfile::const_for_strategy::(); - let mut opt_state = - core::mem::replace(&mut self.backend.bt_mut().opt_state, HcOptState::new()); - opt_state.rescale_freqs(current, profile); - let mut best_plan = core::mem::take(&mut self.backend.bt_mut().opt_segment_plan_scratch); - best_plan.clear(); - let mut plan_reps = self.table.offset_hist; - let (mut cursor, mut plan_litlen) = - self.table.opt_start_cursor_and_litlen(current_abs_start); - let mut plan_literals_cursor = 0usize; - let match_loop_limit = current_len.saturating_sub(8); - while cursor < match_loop_limit { - let remaining_len = current_len - cursor; - let segment_abs_start = current_abs_start + cursor; - let segment_start = best_plan.len(); - let (_, end_reps, end_litlen, consumed_len) = self.build_optimal_plan::( - ¤t[cursor..], - segment_abs_start, - remaining_len, - HcOptimalPlanState { - block_offset: cursor, - reps: plan_reps, - litlen: plan_litlen, - profile, - }, - &opt_state, - &mut best_plan, - ); - BtMatcher::update_plan_stats_segment( - current, - current_len, - &best_plan[segment_start..], - &mut plan_literals_cursor, - &mut plan_reps, - &mut opt_state, - profile.accurate, - ); - plan_reps = end_reps; - plan_litlen = end_litlen; - cursor += consumed_len; - } - - self.table - .emit_optimal_plan(current_len, &best_plan, &mut handle_sequence); - best_plan.clear(); - self.backend.bt_mut().opt_segment_plan_scratch = best_plan; - self.backend.bt_mut().opt_state = opt_state; - } - - fn run_btultra2_seed_pass( - &mut self, - current: &[u8], - current_abs_start: usize, - current_len: usize, - ) { - // The seed pass is BtUltra2-exclusive by name (the only - // caller is `should_run_btultra2_seed_pass`), so pin `S` to - // `BtUltra2` for both the cost-profile lookup and the - // `build_optimal_plan::` call below. - type S = super::strategy::BtUltra2; - let seed_profile = HcOptimalCostProfile::const_for_strategy::(); - let mut opt_state = - core::mem::replace(&mut self.backend.bt_mut().opt_state, HcOptState::new()); - opt_state.rescale_freqs(current, seed_profile); - let mut seed_reps = self.table.offset_hist; - let (mut cursor, mut seed_litlen) = - self.table.opt_start_cursor_and_litlen(current_abs_start); - let mut seed_literals_cursor = 0usize; - let mut seed_plan = core::mem::take(&mut self.backend.bt_mut().opt_seed_plan_scratch); - seed_plan.clear(); - let match_loop_limit = current_len.saturating_sub(8); - while cursor < match_loop_limit { - let remaining_len = current_len - cursor; - let segment_abs_start = current_abs_start + cursor; - let segment_start = seed_plan.len(); - let (_, end_reps, end_litlen, consumed_len) = self.build_optimal_plan::( - ¤t[cursor..], - segment_abs_start, - remaining_len, - HcOptimalPlanState { - block_offset: cursor, - reps: seed_reps, - litlen: seed_litlen, - profile: seed_profile, - }, - &opt_state, - &mut seed_plan, - ); - BtMatcher::update_plan_stats_segment( - current, - current_len, - &seed_plan[segment_start..], - &mut seed_literals_cursor, - &mut seed_reps, - &mut opt_state, - seed_profile.accurate, - ); - seed_plan.truncate(segment_start); - seed_reps = end_reps; - seed_litlen = end_litlen; - cursor += consumed_len; - } - seed_plan.clear(); - self.backend.bt_mut().opt_seed_plan_scratch = seed_plan; - self.backend.bt_mut().opt_state = opt_state; - - // Upstream zstd initStats_ultra keeps the collected entropy statistics but - // invalidates the first-pass matchfinder history before the real pass. - self.table.position_base = self.table.history_abs_start; - self.table.index_shift = current_len; - self.table.next_to_update3 = current_abs_start; - self.table.skip_insert_until_abs = current_abs_start; - // Upstream zstd `ZSTD_initStats_ultra()` invalidates the first scan by moving - // `window.base` back by `srcSize`, making the real pass start at - // `curr == srcSize` instead of 0. Position 0 is therefore a valid - // table entry in the second pass even though raw C tables reserve - // value 0 as empty during an unshifted first pass. - self.table.allow_zero_relative_position = true; - } - - fn build_optimal_plan( - &mut self, - current: &[u8], - current_abs_start: usize, - current_len: usize, - initial_state: HcOptimalPlanState, - stats: &HcOptState, - out: &mut Vec, - ) -> (u32, [u32; 3], usize, usize) { - debug_assert!(S::USE_BT, "build_optimal_plan called on non-BT strategy"); - debug_assert_eq!(initial_state.profile.accurate, S::ACCURATE_PRICE); - debug_assert_eq!( - initial_state.profile.favor_small_offsets, - S::FAVOR_SMALL_OFFSETS - ); - // `S::ACCURATE_PRICE` / `S::FAVOR_SMALL_OFFSETS` cannot appear - // as const-generic arguments yet (`generic_const_exprs` is - // still unstable), so dispatch over a 4-arm match — but on the - // strategy's ASSOCIATED CONSTS, not the runtime profile (the - // `debug_assert_eq`s above pin the runtime profile to those - // consts). A const scrutinee folds the three dead arms at - // monomorphisation; matching the runtime profile instead kept - // all four `#[inline(always)]` DP bodies (~16 KB each) alive in - // EVERY `S` instantiation — ~360 KB of the wasm payload. - match (S::ACCURATE_PRICE, S::FAVOR_SMALL_OFFSETS) { - (true, false) => self.build_optimal_plan_impl::( - current, - current_abs_start, - current_len, - initial_state, - stats, - out, - ), - (true, true) => self.build_optimal_plan_impl::( - current, - current_abs_start, - current_len, - initial_state, - stats, - out, - ), - (false, false) => self.build_optimal_plan_impl::( - current, - current_abs_start, - current_len, - initial_state, - stats, - out, - ), - (false, true) => self.build_optimal_plan_impl::( - current, - current_abs_start, - current_len, - initial_state, - stats, - out, - ), - } - } - - /// Cross-platform DP entry. Picks the kernel-specific variant so the - /// entire optimal-parser DP body (per-position match gathering, price - /// updates, traceback) runs inside a single `target_feature` umbrella - /// alongside the per-position `collect_optimal_candidates_initialized_ - /// `. This eliminates the final ABI barrier on the hot per- - /// position match-collection call — the level22 critical path is now - /// one straight-line inline chain from DP body down through BT walk - /// and match-length probes. - #[inline(always)] - fn build_optimal_plan_impl< - S: super::strategy::Strategy, - const ACCURATE_PRICE: bool, - const FAVOR_SMALL_OFFSETS: bool, - >( - &mut self, - current: &[u8], - current_abs_start: usize, - current_len: usize, - initial_state: HcOptimalPlanState, - stats: &HcOptState, - out: &mut Vec, - ) -> (u32, [u32; 3], usize, usize) { - #[cfg(all(target_arch = "aarch64", target_endian = "little"))] - unsafe { - self.build_optimal_plan_impl_neon::( - current, - current_abs_start, - current_len, - initial_state, - stats, - out, - ) - } - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - use crate::encoding::fastpath::{FastpathKernel, select_kernel}; - match select_kernel() { - FastpathKernel::Avx2Bmi2 => unsafe { - self.build_optimal_plan_impl_avx2_bmi2::( - current, - current_abs_start, - current_len, - initial_state, - stats, - out, - ) - }, - FastpathKernel::Sse42 => unsafe { - self.build_optimal_plan_impl_sse42::( - current, - current_abs_start, - current_len, - initial_state, - stats, - out, - ) - }, - FastpathKernel::Scalar => self - .build_optimal_plan_impl_scalar::( - current, - current_abs_start, - current_len, - initial_state, - stats, - out, - ), - } - } - // wasm with simd128: route through the simd128 DP body (4-lane price-set). - #[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] - unsafe { - self.build_optimal_plan_impl_simd128::( - current, - current_abs_start, - current_len, - initial_state, - stats, - out, - ) - } - #[cfg(not(any( - all(target_arch = "aarch64", target_endian = "little"), - target_arch = "x86", - target_arch = "x86_64", - all(target_arch = "wasm32", target_feature = "simd128") - )))] - { - self.build_optimal_plan_impl_scalar::( - current, - current_abs_start, - current_len, - initial_state, - stats, - out, - ) - } - } - - /// NEON-umbrella DP body. Inlines - /// `collect_optimal_candidates_initialized_neon` (and its entire - /// per-position pipeline) directly into the DP loop. - #[cfg(all(target_arch = "aarch64", target_endian = "little"))] - #[target_feature(enable = "neon")] - unsafe fn build_optimal_plan_impl_neon< - S: super::strategy::Strategy, - const ACCURATE_PRICE: bool, - const FAVOR_SMALL_OFFSETS: bool, - >( - &mut self, - current: &[u8], - current_abs_start: usize, - current_len: usize, - initial_state: HcOptimalPlanState, - stats: &HcOptState, - out: &mut Vec, - ) -> (u32, [u32; 3], usize, usize) { - build_optimal_plan_impl_body!( - self, - S, - current, - current_abs_start, - current_len, - initial_state, - stats, - out, - collect_optimal_candidates_initialized_neon, - priceset_range_nonabort_neon, - ) - } - - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - #[target_feature(enable = "sse4.2")] - unsafe fn build_optimal_plan_impl_sse42< - S: super::strategy::Strategy, - const ACCURATE_PRICE: bool, - const FAVOR_SMALL_OFFSETS: bool, - >( - &mut self, - current: &[u8], - current_abs_start: usize, - current_len: usize, - initial_state: HcOptimalPlanState, - stats: &HcOptState, - out: &mut Vec, - ) -> (u32, [u32; 3], usize, usize) { - build_optimal_plan_impl_body!( - self, - S, - current, - current_abs_start, - current_len, - initial_state, - stats, - out, - collect_optimal_candidates_initialized_sse42, - priceset_range_nonabort_sse41, - ) - } - - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - #[target_feature(enable = "avx2,bmi2")] - unsafe fn build_optimal_plan_impl_avx2_bmi2< - S: super::strategy::Strategy, - const ACCURATE_PRICE: bool, - const FAVOR_SMALL_OFFSETS: bool, - >( - &mut self, - current: &[u8], - current_abs_start: usize, - current_len: usize, - initial_state: HcOptimalPlanState, - stats: &HcOptState, - out: &mut Vec, - ) -> (u32, [u32; 3], usize, usize) { - build_optimal_plan_impl_body!( - self, - S, - current, - current_abs_start, - current_len, - initial_state, - stats, - out, - collect_optimal_candidates_initialized_avx2_bmi2, - priceset_range_nonabort_avx2, - ) - } - - #[cfg(not(all(target_arch = "aarch64", target_endian = "little")))] - // Body macros wrap callees in `unsafe { }` for the NEON/AVX/SSE - // variants where callees are `unsafe fn`. The scalar wrappers route - // through safe fns, so those blocks are redundant on this path. - #[allow(unused_unsafe)] - // The dispatch reaches this only on non-SIMD x86 (Scalar tier) and the - // portable fallback; on wasm+simd128 the simd128 wrapper is selected, so - // this is cfg-dead there. - #[cfg_attr( - all(target_arch = "wasm32", target_feature = "simd128"), - allow(dead_code) - )] - fn build_optimal_plan_impl_scalar< - S: super::strategy::Strategy, - const ACCURATE_PRICE: bool, - const FAVOR_SMALL_OFFSETS: bool, - >( - &mut self, - current: &[u8], - current_abs_start: usize, - current_len: usize, - initial_state: HcOptimalPlanState, - stats: &HcOptState, - out: &mut Vec, - ) -> (u32, [u32; 3], usize, usize) { - build_optimal_plan_impl_body!( - self, - S, - current, - current_abs_start, - current_len, - initial_state, - stats, - out, - collect_optimal_candidates_initialized_scalar, - priceset_range_nonabort_scalar, - ) - } - - /// wasm `simd128`-umbrella DP body: scalar candidate collection (no wasm - /// collect kernel) but the simd128 4-lane price-set. - #[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] - #[target_feature(enable = "simd128")] - // With `+simd128` in the wasm baseline the shared body macro's `unsafe` - // blocks (needed by the safe scalar wrapper) are redundant inside this - // target_feature fn. - #[allow(unused_unsafe)] - unsafe fn build_optimal_plan_impl_simd128< - S: super::strategy::Strategy, - const ACCURATE_PRICE: bool, - const FAVOR_SMALL_OFFSETS: bool, - >( - &mut self, - current: &[u8], - current_abs_start: usize, - current_len: usize, - initial_state: HcOptimalPlanState, - stats: &HcOptState, - out: &mut Vec, - ) -> (u32, [u32; 3], usize, usize) { - build_optimal_plan_impl_body!( - self, - S, - current, - current_abs_start, - current_len, - initial_state, - stats, - out, - collect_optimal_candidates_initialized_scalar, - priceset_range_nonabort_simd128, - ) - } - - #[cfg(test)] - fn collect_optimal_candidates( - &mut self, - abs_pos: usize, - current_abs_end: usize, - profile: HcOptimalCostProfile, - query: HcCandidateQuery, - out: &mut Vec, - ) { - use super::strategy::{self, StrategyTag}; - self.table.ensure_tables(); - // Dispatch purely from `self.strategy_tag` (set by - // `configure()`). Tests must configure the matcher the same - // way production does — wiring up `table.hash3_log` directly - // without setting a matching `strategy_tag` is no longer - // allowed. - match self.strategy_tag { - StrategyTag::BtUltra2 => self - .collect_optimal_candidates_initialized::( - abs_pos, - current_abs_end, - profile, - query, - out, - ), - StrategyTag::BtUltra => self - .collect_optimal_candidates_initialized::( - abs_pos, - current_abs_end, - profile, - query, - out, - ), - StrategyTag::Btlazy2 => self - .collect_optimal_candidates_initialized::( - abs_pos, - current_abs_end, - profile, - query, - out, - ), - StrategyTag::BtOpt => self - .collect_optimal_candidates_initialized::( - abs_pos, - current_abs_end, - profile, - query, - out, - ), - StrategyTag::Fast | StrategyTag::Dfast | StrategyTag::Greedy | StrategyTag::Lazy => { - self.collect_optimal_candidates_initialized::( - abs_pos, - current_abs_end, - profile, - query, - out, - ) - } - } - } - - /// Cross-platform entry. Picks the kernel-specific variant so the per- - /// position pipeline (BT-tree fill, rep probing, hash3 probing, BT - /// collect / HC chain walk) runs inside a single `target_feature` - /// umbrella — all inner SIMD probes inline without ABI barriers. - /// - /// The on-encode hot path bypasses this dispatcher: `build_optimal_plan_impl_` - /// calls the matching `_` variant directly. This entry is kept - /// for the cfg(test)-only `collect_optimal_candidates` shim and any - /// future caller that isn't already inside a kernel umbrella. - #[allow(dead_code)] - #[inline(always)] - fn collect_optimal_candidates_initialized< - S: super::strategy::Strategy, - const USE_BT_MATCHFINDER: bool, - >( - &mut self, - abs_pos: usize, - current_abs_end: usize, - profile: HcOptimalCostProfile, - query: HcCandidateQuery, - out: &mut Vec, - ) { - #[cfg(all(target_arch = "aarch64", target_endian = "little"))] - unsafe { - self.collect_optimal_candidates_initialized_neon::( - abs_pos, - current_abs_end, - profile, - query, - out, - ) - } - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - use crate::encoding::fastpath::{FastpathKernel, select_kernel}; - match select_kernel() { - FastpathKernel::Avx2Bmi2 => unsafe { - self.collect_optimal_candidates_initialized_avx2_bmi2::( - abs_pos, - current_abs_end, - profile, - query, - out, - ) - }, - FastpathKernel::Sse42 => unsafe { - self.collect_optimal_candidates_initialized_sse42::( - abs_pos, - current_abs_end, - profile, - query, - out, - ) - }, - FastpathKernel::Scalar => self - .collect_optimal_candidates_initialized_scalar::( - abs_pos, - current_abs_end, - profile, - query, - out, - ), - } - } - #[cfg(not(any( - all(target_arch = "aarch64", target_endian = "little"), - target_arch = "x86", - target_arch = "x86_64" - )))] - { - self.collect_optimal_candidates_initialized_scalar::( - abs_pos, - current_abs_end, - profile, - query, - out, - ) - } - } - - /// NEON-umbrella variant. Every inner helper (`bt_update_tree_until_neon`, - /// `for_each_repcode_candidate_with_reps_neon`, `hash3_candidate_neon`, - /// `bt_insert_and_collect_matches_neon`, `fastpath::neon:: - /// common_prefix_len_ptr`) shares the NEON umbrella so the per-position - /// pipeline executes as a single straight-line inline sequence. - #[cfg(all(target_arch = "aarch64", target_endian = "little"))] - #[target_feature(enable = "neon")] - unsafe fn collect_optimal_candidates_initialized_neon< - S: super::strategy::Strategy, - const USE_BT_MATCHFINDER: bool, - >( - &mut self, - abs_pos: usize, - current_abs_end: usize, - profile: HcOptimalCostProfile, - query: HcCandidateQuery, - out: &mut Vec, - ) { - collect_optimal_candidates_initialized_body!( - self, - S, - abs_pos, - current_abs_end, - profile, - query, - out, - USE_BT_MATCHFINDER, - bt_update_tree_until_neon, - bt_insert_and_collect_matches_neon, - for_each_repcode_candidate_with_reps_neon, - hash3_candidate_neon, - crate::encoding::fastpath::neon::common_prefix_len_ptr, - ) - } - - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - #[target_feature(enable = "sse4.2")] - unsafe fn collect_optimal_candidates_initialized_sse42< - S: super::strategy::Strategy, - const USE_BT_MATCHFINDER: bool, - >( - &mut self, - abs_pos: usize, - current_abs_end: usize, - profile: HcOptimalCostProfile, - query: HcCandidateQuery, - out: &mut Vec, - ) { - collect_optimal_candidates_initialized_body!( - self, - S, - abs_pos, - current_abs_end, - profile, - query, - out, - USE_BT_MATCHFINDER, - bt_update_tree_until_sse42, - bt_insert_and_collect_matches_sse42, - for_each_repcode_candidate_with_reps_sse42, - hash3_candidate_sse42, - crate::encoding::fastpath::sse42::common_prefix_len_ptr, - ) - } - - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - #[target_feature(enable = "avx2,bmi2")] - unsafe fn collect_optimal_candidates_initialized_avx2_bmi2< - S: super::strategy::Strategy, - const USE_BT_MATCHFINDER: bool, - >( - &mut self, - abs_pos: usize, - current_abs_end: usize, - profile: HcOptimalCostProfile, - query: HcCandidateQuery, - out: &mut Vec, - ) { - collect_optimal_candidates_initialized_body!( - self, - S, - abs_pos, - current_abs_end, - profile, - query, - out, - USE_BT_MATCHFINDER, - bt_update_tree_until_avx2_bmi2, - bt_insert_and_collect_matches_avx2_bmi2, - for_each_repcode_candidate_with_reps_avx2_bmi2, - hash3_candidate_avx2_bmi2, - crate::encoding::fastpath::avx2_bmi2::common_prefix_len_ptr, - ) - } - - #[cfg(not(all(target_arch = "aarch64", target_endian = "little")))] - // Macro emits `unsafe { }` wrappers for NEON/AVX/SSE variants; scalar - // callees are safe so the blocks are redundant here only. - #[allow(unused_unsafe)] - fn collect_optimal_candidates_initialized_scalar< - S: super::strategy::Strategy, - const USE_BT_MATCHFINDER: bool, - >( - &mut self, - abs_pos: usize, - current_abs_end: usize, - profile: HcOptimalCostProfile, - query: HcCandidateQuery, - out: &mut Vec, - ) { - collect_optimal_candidates_initialized_body!( - self, - S, - abs_pos, - current_abs_end, - profile, - query, - out, - USE_BT_MATCHFINDER, - bt_update_tree_until_scalar, - bt_insert_and_collect_matches_scalar, - for_each_repcode_candidate_with_reps_scalar, - hash3_candidate_scalar, - crate::encoding::fastpath::scalar::common_prefix_len_ptr, - ) - } -} - -#[cfg(any())] // disabled: tested legacy MatchGenerator/SuffixStore behavior removed in phase 1b -#[test] -fn matches() { - let mut matcher = MatchGenerator::new(1000); - let mut original_data = Vec::new(); - let mut reconstructed = Vec::new(); - - let replay_sequence = |seq: Sequence<'_>, reconstructed: &mut Vec| match seq { - Sequence::Literals { literals } => { - assert!(!literals.is_empty()); - reconstructed.extend_from_slice(literals); - } - Sequence::Triple { - literals, - offset, - match_len, - } => { - assert!(offset > 0); - assert!(match_len >= MIN_MATCH_LEN); - reconstructed.extend_from_slice(literals); - assert!(offset <= reconstructed.len()); - let start = reconstructed.len() - offset; - for i in 0..match_len { - let byte = reconstructed[start + i]; - reconstructed.push(byte); - } - } - }; - - matcher.add_data( - alloc::vec![0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - SuffixStore::with_capacity(100), - |_, _| {}, - ); - original_data.extend_from_slice(&[0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); - - matcher.next_sequence(|seq| replay_sequence(seq, &mut reconstructed)); - - assert!(!matcher.next_sequence(|_| {})); - - matcher.add_data( - alloc::vec![ - 1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6, 0, 0, 0, 0, 0, - ], - SuffixStore::with_capacity(100), - |_, _| {}, - ); - original_data.extend_from_slice(&[ - 1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6, 0, 0, 0, 0, 0, - ]); - - matcher.next_sequence(|seq| replay_sequence(seq, &mut reconstructed)); - matcher.next_sequence(|seq| replay_sequence(seq, &mut reconstructed)); - matcher.next_sequence(|seq| replay_sequence(seq, &mut reconstructed)); - assert!(!matcher.next_sequence(|_| {})); - - matcher.add_data( - alloc::vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0, 0, 0, 0, 0], - SuffixStore::with_capacity(100), - |_, _| {}, - ); - original_data.extend_from_slice(&[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0, 0, 0, 0, 0]); - - matcher.next_sequence(|seq| replay_sequence(seq, &mut reconstructed)); - matcher.next_sequence(|seq| replay_sequence(seq, &mut reconstructed)); - assert!(!matcher.next_sequence(|_| {})); - - matcher.add_data( - alloc::vec![0, 0, 0, 0, 0], - SuffixStore::with_capacity(100), - |_, _| {}, - ); - original_data.extend_from_slice(&[0, 0, 0, 0, 0]); - - matcher.next_sequence(|seq| replay_sequence(seq, &mut reconstructed)); - assert!(!matcher.next_sequence(|_| {})); - - matcher.add_data( - alloc::vec![7, 8, 9, 10, 11], - SuffixStore::with_capacity(100), - |_, _| {}, - ); - original_data.extend_from_slice(&[7, 8, 9, 10, 11]); - - matcher.next_sequence(|seq| replay_sequence(seq, &mut reconstructed)); - assert!(!matcher.next_sequence(|_| {})); - - matcher.add_data( - alloc::vec![1, 3, 5, 7, 9], - SuffixStore::with_capacity(100), - |_, _| {}, - ); - matcher.skip_matching(); - original_data.extend_from_slice(&[1, 3, 5, 7, 9]); - reconstructed.extend_from_slice(&[1, 3, 5, 7, 9]); - assert!(!matcher.next_sequence(|_| {})); - - matcher.add_data( - alloc::vec![1, 3, 5, 7, 9], - SuffixStore::with_capacity(100), - |_, _| {}, - ); - original_data.extend_from_slice(&[1, 3, 5, 7, 9]); - - matcher.next_sequence(|seq| replay_sequence(seq, &mut reconstructed)); - assert!(!matcher.next_sequence(|_| {})); - - matcher.add_data( - alloc::vec![0, 0, 11, 13, 15, 17, 20, 11, 13, 15, 17, 20, 21, 23], - SuffixStore::with_capacity(100), - |_, _| {}, - ); - original_data.extend_from_slice(&[0, 0, 11, 13, 15, 17, 20, 11, 13, 15, 17, 20, 21, 23]); - - matcher.next_sequence(|seq| replay_sequence(seq, &mut reconstructed)); - matcher.next_sequence(|seq| replay_sequence(seq, &mut reconstructed)); - assert!(!matcher.next_sequence(|_| {})); - - assert_eq!(reconstructed, original_data); -} - -#[test] -fn dfast_matches_roundtrip_multi_block_pattern() { - let pattern = [9, 21, 44, 184, 19, 96, 171, 109, 141, 251]; - let first_block: Vec = pattern.iter().copied().cycle().take(128 * 1024).collect(); - let second_block: Vec = pattern.iter().copied().cycle().take(128 * 1024).collect(); - - let mut matcher = DfastMatchGenerator::new(1 << 22); - let replay_sequence = |decoded: &mut Vec, seq: Sequence<'_>| match seq { - Sequence::Literals { literals } => decoded.extend_from_slice(literals), - Sequence::Triple { - literals, - offset, - match_len, - } => { - decoded.extend_from_slice(literals); - let start = decoded.len() - offset; - for i in 0..match_len { - let byte = decoded[start + i]; - decoded.push(byte); - } - } - }; - - matcher.add_data(first_block.clone(), |_| {}); - let mut history = Vec::new(); - matcher.start_matching(|seq| replay_sequence(&mut history, seq)); - assert_eq!(history, first_block); - - matcher.add_data(second_block.clone(), |_| {}); - let prefix_len = history.len(); - matcher.start_matching(|seq| replay_sequence(&mut history, seq)); - - assert_eq!(&history[prefix_len..], second_block.as_slice()); -} - -/// Regression for the `DFAST_MIN_MATCH_LEN: 6 -> 5` drop. The fixture -/// is built so the longest available match is EXACTLY 5 bytes — a -/// matcher that still effectively requires a 6-byte floor would emit -/// only literals here and the assertion would catch the silent -/// 5-byte miss. -/// -/// Fixture layout (34 B): -/// bytes 0..5 `"ABCDE"` — match source -/// bytes 5..28 `'!'` × 23 — filler that does NOT start with 'A' -/// bytes 28..33 `"ABCDE"` — match site (repeats the prefix) -/// byte 33 `'F'` — terminator: differs from byte 5 (`'!'`), -/// so the forward extension at the match -/// site stops at exactly length 5. -/// -/// A 5-byte match at offset 28 must be emitted; a 6-byte+ match at the -/// same offset must NOT. -#[test] -fn dfast_accepts_exact_five_byte_match() { - // Layout the input so that: - // byte 0 = 'Z' (lead byte — keeps the match SOURCE off - // position 0, which the greedy loop never - // inserts: like the upstream zstd it starts the - // cursor at ip+1 and hashes only visited - // positions) - // bytes 1..6 = "ABCDE" (the match source — position 1 IS visited) - // bytes 6..29 = 23 filler bytes that do NOT start with 'A' - // bytes 29..34 = "ABCDE" (the 5-byte match site) - // byte 34 = 'F' (differs from byte 6 = '!') - // The longest available copy at position 29 is exactly 5 bytes: - // the byte at position 34 ('F') differs from the byte at position 6 - // ('!'), so the forward extension stops at length 5. - let mut data = Vec::new(); - data.push(b'Z'); // 0 - data.extend_from_slice(b"ABCDE"); // 1..6 - data.extend_from_slice(b"!!!!!!!!!!!!!!!!!!!!!!!"); // 6..29 (23 bytes) - data.extend_from_slice(b"ABCDE"); // 29..34 - data.push(b'F'); // 34: forces forward extension to stop at length 5 - // Trailing filler so the match site (29) sits at least HASH_READ_SIZE (8) - // bytes before the block end. The greedy double-fast — like the upstream zstd — - // stops probing at `ilimit = iend - HASH_READ_SIZE`, so a match in the - // final 8 bytes is never searched (upstream zstd parity, not a regression). - data.extend_from_slice(b"GHIJKLMNOPQRSTUVWXYZ"); // 35..55 - assert_eq!(data.len(), 55); - - let mut matcher = DfastMatchGenerator::new(1 << 22); - matcher.add_data(data.clone(), |_| {}); - - let mut saw_five_byte_match = false; - let mut saw_longer_match = false; - matcher.start_matching(|seq| { - if let Sequence::Triple { - offset, match_len, .. - } = seq - { - if offset == 28 && match_len == 5 { - saw_five_byte_match = true; - } else if offset == 28 && match_len > 5 { - saw_longer_match = true; - } - } - }); - - assert!( - saw_five_byte_match, - "dfast must accept the exact-5-byte match — a 6-byte floor would skip it" - ); - assert!( - !saw_longer_match, - "fixture pinned to length 5 — byte 33 ('F') must terminate the extension" - ); -} - -#[test] -fn driver_switches_backends_and_initializes_dfast_via_reset() { - let mut driver = MatchGeneratorDriver::new(32, 2); - - driver.reset(CompressionLevel::Default); - assert_eq!(driver.active_backend(), super::strategy::BackendTag::Dfast); - assert_eq!(driver.window_size(), (1u64 << 21)); - - let mut first = driver.get_next_space(); - first[..12].copy_from_slice(b"abcabcabcabc"); - first.truncate(12); - driver.commit_space(first); - assert_eq!(driver.get_last_space(), b"abcabcabcabc"); - driver.skip_matching_with_hint(None); - - let mut second = driver.get_next_space(); - second[..12].copy_from_slice(b"abcabcabcabc"); - second.truncate(12); - driver.commit_space(second); - - let mut reconstructed = b"abcabcabcabc".to_vec(); - driver.start_matching(|seq| match seq { - Sequence::Literals { literals } => reconstructed.extend_from_slice(literals), - Sequence::Triple { - literals, - offset, - match_len, - } => { - reconstructed.extend_from_slice(literals); - let start = reconstructed.len() - offset; - for i in 0..match_len { - let byte = reconstructed[start + i]; - reconstructed.push(byte); - } - } - }); - assert_eq!(reconstructed, b"abcabcabcabcabcabcabcabc"); - - driver.reset(CompressionLevel::Fastest); - assert_eq!(driver.window_size(), (1u64 << 19)); -} - -#[test] -fn driver_level5_selects_row_backend() { - let mut driver = MatchGeneratorDriver::new(32, 2); - driver.reset(CompressionLevel::Level(5)); - assert_eq!(driver.active_backend(), super::strategy::BackendTag::Row); - // Greedy-specific routing assertion: `MatchGeneratorDriver::start_matching` - // dispatches the Row backend into `start_matching_greedy` iff - // `self.parse == ParseMode::Greedy`, so assert that actual selector — - // round-trip alone passes on the lazy parser too. `row_matcher().lazy_depth` - // is a secondary corroboration of the same routing decision (a mirror of - // the parse mode); checking `parse` directly catches a regression even if - // the two ever drift apart. - assert_eq!( - driver.parse, - super::strategy::ParseMode::Greedy, - "L5 must route to start_matching_greedy (parse == Greedy)", - ); - assert_eq!( - driver.row_matcher().lazy_depth, - 0, - "row matcher lazy_depth must mirror the greedy parse mode", - ); -} - -/// Level 4 maps to `StrategyTag::Dfast` (the greedy double-fast, upstream zstd -/// `ZSTD_dfast` — "greedy" is the parse discipline, not the Row/Greedy -/// strategy at Level 5). Round-trip alone doesn't pin match quality (a lazy -/// parser would also reconstruct the input correctly), so this test guards the -/// parse output itself: a small repeating pattern must produce at least one -/// `Sequence::Triple`, so a future regression that emits literals-only (e.g. a -/// `min_match` or rep-probe guard regression) is caught. -#[test] -fn driver_level4_greedy_round_trip_single_slice() { - let mut driver = MatchGeneratorDriver::new(64, 2); - driver.reset(CompressionLevel::Level(4)); - let input = b"abcdefgh_abcdefgh_abcdefgh_abcdefgh"; - let mut space = driver.get_next_space(); - space[..input.len()].copy_from_slice(input); - space.truncate(input.len()); - driver.commit_space(space); - - let mut reconstructed: Vec = Vec::new(); - let mut saw_triple = false; - driver.start_matching(|seq| match seq { - Sequence::Literals { literals } => reconstructed.extend_from_slice(literals), - Sequence::Triple { - literals, - offset, - match_len, - } => { - saw_triple = true; - reconstructed.extend_from_slice(literals); - let start = reconstructed.len() - offset; - for i in 0..match_len { - let byte = reconstructed[start + i]; - reconstructed.push(byte); - } - } - }); - assert_eq!( - reconstructed, - input.to_vec(), - "L4 greedy parse failed to reconstruct repeating-pattern input", - ); - assert!( - saw_triple, - "L4 greedy parse on a repeating pattern must emit at least one match (Triple)", - ); -} - -#[test] -fn driver_level4_greedy_round_trip_cross_slice() { - // Verifies that the greedy parse carries repcode / hash-table state - // across slice boundaries: the second slice repeats the first byte - // for byte, so the parse must pick up matches reaching back into - // the previous slice's history. - let mut driver = MatchGeneratorDriver::new(32, 4); - driver.reset(CompressionLevel::Level(4)); - let chunk = b"the quick brown fox jumps over!!"; - assert_eq!(chunk.len(), 32); - - let mut first = driver.get_next_space(); - first[..chunk.len()].copy_from_slice(chunk); - first.truncate(chunk.len()); - driver.commit_space(first); - - let mut first_recon: Vec = Vec::new(); - driver.start_matching(|seq| match seq { - Sequence::Literals { literals } => first_recon.extend_from_slice(literals), - Sequence::Triple { - literals, - offset, - match_len, - } => { - first_recon.extend_from_slice(literals); - let start = first_recon.len() - offset; - for i in 0..match_len { - let byte = first_recon[start + i]; - first_recon.push(byte); - } - } - }); - assert_eq!( - first_recon, - chunk.to_vec(), - "first slice failed to round-trip" - ); - - let mut second = driver.get_next_space(); - second[..chunk.len()].copy_from_slice(chunk); - second.truncate(chunk.len()); - driver.commit_space(second); - - let mut full = first_recon.clone(); - let mut saw_cross_slice_match = false; - driver.start_matching(|seq| match seq { - Sequence::Literals { literals } => full.extend_from_slice(literals), - Sequence::Triple { - literals, - offset, - match_len, - } => { - // A match whose offset reaches >= the current slice's literal - // run plus the second slice's index means we matched into the - // first slice — exactly the cross-slice behavior under test. - if offset >= chunk.len() { - saw_cross_slice_match = true; - } - full.extend_from_slice(literals); - let start = full.len() - offset; - for i in 0..match_len { - let byte = full[start + i]; - full.push(byte); - } - } - }); - let mut expected = chunk.to_vec(); - expected.extend_from_slice(chunk); - assert_eq!( - full, expected, - "cross-slice L4 greedy parse failed to reconstruct" - ); - assert!( - saw_cross_slice_match, - "L4 greedy parse must match across slice boundaries (history is shared)", - ); -} - -/// Helper: round-trip `data` through the L4 greedy parse and assert -/// the reconstructed bytes match. Returns `(triple_count, max_offset)` -/// so callers can probe parse shape (matches emitted, max-offset). -#[cfg(test)] -impl MatchGeneratorDriver { - /// Test-only: stage a parse×search recipe override applied on the - /// next `reset()`. Routes a level through a non-default (parse, - /// search) pair so the decoupling can be exercised end-to-end. - pub(crate) fn set_config_override( - &mut self, - search: super::strategy::SearchMethod, - parse: super::strategy::ParseMode, - ) { - self.config_override = Some((search, parse)); - } - - /// Test-only: reset `level` routed onto the lazy HashChain pairing. - /// The lazy band runs on the Row backend in production, so HC-specific - /// behaviour (live-chain dict prime, eviction budget accounting, seed - /// pass gates) is exercised through this override-backed reset. - pub(crate) fn reset_on_hc_lazy(&mut self, level: CompressionLevel) { - self.set_config_override( - super::strategy::SearchMethod::HashChain, - super::strategy::ParseMode::Lazy2, - ); - self.reset(level); - } -} - -/// Drive a full compress parse for `data` at `level` (optionally with a -/// parse×search override) and reconstruct the bytes from the emitted -/// sequences. The returned buffer must equal `data` for a correct parse. -#[cfg(test)] -fn drive_roundtrip_with_override( - level: CompressionLevel, - over: Option<(super::strategy::SearchMethod, super::strategy::ParseMode)>, - data: &[u8], -) -> Vec { - let mut driver = MatchGeneratorDriver::new(1 << 17, 8); - if let Some((s, p)) = over { - driver.set_config_override(s, p); - } - driver.reset(level); - - let mut out: Vec = Vec::with_capacity(data.len()); - let mut offset_in_data = 0usize; - while offset_in_data < data.len() { - let mut space = driver.get_next_space(); - let take = (data.len() - offset_in_data).min(space.len()); - space[..take].copy_from_slice(&data[offset_in_data..offset_in_data + take]); - space.truncate(take); - driver.commit_space(space); - offset_in_data += take; - - driver.start_matching(|seq| match seq { - Sequence::Literals { literals } => out.extend_from_slice(literals), - Sequence::Triple { - literals, - offset, - match_len, - } => { - out.extend_from_slice(literals); - let start = out.len() - offset; - for i in 0..match_len { - let byte = out[start + i]; - out.push(byte); - } - } - }); - } - out -} - -/// Phase 1 capability proof: parse and search are decoupled, so a level -/// can run any parse mode on any non-opt search backend. Greedy-on- -/// HashChain and Lazy2-on-RowHash are pairings the legacy `strategy_tag` -/// could not express; both must reconstruct the input exactly. -#[test] -fn parse_search_matrix_decoupled_roundtrips() { - use super::strategy::{ParseMode, SearchMethod}; - // Mixed repetitive + literal payload that exercises matches and reps. - let mut data = Vec::new(); - for i in 0..4000u32 { - data.extend_from_slice(b"the quick brown fox "); - data.extend_from_slice(&i.to_le_bytes()); - } - - // Greedy parse on the HashChain search backend (legacy: Greedy was - // welded to RowHash). - let got = drive_roundtrip_with_override( - CompressionLevel::Level(5), - Some((SearchMethod::HashChain, ParseMode::Greedy)), - &data, - ); - assert_eq!(got, data, "greedy-on-hashchain diverged"); - - // Lazy2 parse on the RowHash search backend (legacy: Lazy was welded - // to HashChain). - let got = drive_roundtrip_with_override( - CompressionLevel::Level(8), - Some((SearchMethod::RowHash, ParseMode::Lazy2)), - &data, - ); - assert_eq!(got, data, "lazy2-on-rowhash diverged"); - - // Lazy on RowHash too (depth 1). - let got = drive_roundtrip_with_override( - CompressionLevel::Level(6), - Some((SearchMethod::RowHash, ParseMode::Lazy)), - &data, - ); - assert_eq!(got, data, "lazy-on-rowhash diverged"); -} - -/// The row `mls` knob (C-like `minMatch`) is respected: every accepted -/// match (regular row + repcode, on the lazy parse) is at least `mls` -/// bytes, and the stream still round-trips for the whole 4..=7 range. The -/// default (5) reproduces the historical `ROW_MIN_MATCH_LEN` behaviour. -#[test] -fn row_mls_knob_gates_matches_and_roundtrips() { - let data: Vec = (0..4000u32) - .flat_map(|i| { - let mut v = b"abcdefgh".to_vec(); - v.extend_from_slice(&i.to_le_bytes()); - v - }) - .collect(); - - for mls in [4usize, 5, 6, 7] { - let mut matcher = RowMatchGenerator::new(1 << 22); - let mut cfg = ROW_CONFIG; - cfg.mls = mls; - matcher.configure(cfg); - matcher.add_data(data.clone(), |_| {}); - - let mut out: Vec = Vec::with_capacity(data.len()); - let mut shortest_match = usize::MAX; - matcher.start_matching(|seq| match seq { - Sequence::Literals { literals } => out.extend_from_slice(literals), - Sequence::Triple { - literals, - offset, - match_len, - } => { - out.extend_from_slice(literals); - shortest_match = shortest_match.min(match_len); - let start = out.len() - offset; - for i in 0..match_len { - let byte = out[start + i]; - out.push(byte); - } - } - }); - - assert_eq!(out, data, "mls={mls} round-trip diverged"); - if shortest_match != usize::MAX { - assert!( - shortest_match >= mls, - "mls={mls}: emitted a {shortest_match}-byte match below the floor", - ); - } - } -} - -/// `LevelParams::parse()` derives the parse mode from the `search` axis, not -/// the strategy tag, so the decoupling holds even for a `Bt*`-tagged level -/// overridden to a non-BT search backend. Pre-fix the method matched on -/// `strategy_tag` and returned `Optimal` for any `Bt*` tag regardless of -/// `search`/`lazy_depth`. -#[test] -fn parse_mode_follows_search_axis_not_strategy_tag() { - use super::strategy::{ParseMode, SearchMethod}; - // LEVEL_TABLE[15] is level 16: BtOpt tag, BinaryTree search. - let mut p = LEVEL_TABLE[15]; - assert_eq!(p.parse(), ParseMode::Optimal, "BinaryTree search → Optimal"); - // Override the Bt-tagged level's search to a non-BT backend: parse must - // follow the search axis (derive from lazy_depth), not stay Optimal. - p.search = SearchMethod::RowHash; - p.lazy_depth = 0; - assert_eq!(p.parse(), ParseMode::Greedy, "RowHash + depth 0 → Greedy"); - p.lazy_depth = 2; - assert_eq!(p.parse(), ParseMode::Lazy2, "RowHash + depth 2 → Lazy2"); -} - -/// The test-only `config_override` is consumed by the first `reset()` (one -/// shot), so a reused driver does not silently keep the synthetic pairing -/// armed across later resets. Pre-fix `reset()` copied the override and left -/// it set. -#[test] -fn config_override_is_consumed_by_reset() { - use super::strategy::{ParseMode, SearchMethod}; - let mut driver = MatchGeneratorDriver::new(1 << 17, 8); - driver.set_config_override(SearchMethod::RowHash, ParseMode::Lazy2); - assert!(driver.config_override.is_some()); - driver.reset(CompressionLevel::Level(5)); - assert!( - driver.config_override.is_none(), - "override must be consumed after one reset", - ); -} - -// Level 4 maps to the greedy Dfast (double-fast) backend — "greedy" here is the -// parse discipline (no lazy lookahead, upstream zstd `ZSTD_dfast`), NOT the Row/Greedy -// strategy (which is Level 5). This roundtrip is intentional Dfast L4 coverage; -// the Row backend is exercised by the `Level(5)` fixtures elsewhere in this file. -#[cfg(test)] -fn l4_greedy_round_trip(slice_size: usize, max_slices: usize, data: &[u8]) -> (usize, usize) { - let mut driver = MatchGeneratorDriver::new(slice_size, max_slices); - driver.reset(CompressionLevel::Level(4)); - - let mut reconstructed: Vec = Vec::with_capacity(data.len()); - let mut triple_count = 0usize; - let mut max_offset = 0usize; - - // `start_matching` consumes the current pending slice; multi-slice - // payloads require commit + drive per slice so earlier slices' - // bytes actually round-trip out before they're displaced from the - // window. - let mut offset_in_data = 0usize; - while offset_in_data < data.len() { - let mut space = driver.get_next_space(); - let space_cap = space.len(); - let take = (data.len() - offset_in_data).min(space_cap); - space[..take].copy_from_slice(&data[offset_in_data..offset_in_data + take]); - space.truncate(take); - driver.commit_space(space); - offset_in_data += take; - - driver.start_matching(|seq| match seq { - Sequence::Literals { literals } => reconstructed.extend_from_slice(literals), - Sequence::Triple { - literals, - offset, - match_len, - } => { - triple_count += 1; - if offset > max_offset { - max_offset = offset; - } - reconstructed.extend_from_slice(literals); - let start = reconstructed.len() - offset; - for i in 0..match_len { - let byte = reconstructed[start + i]; - reconstructed.push(byte); - } - } - }); - } - - // Empty payload still needs one commit/drive round so the empty- - // input path of `start_matching_greedy` (the `current_len == 0` - // early-return guard) gets exercised. - if data.is_empty() { - let mut space = driver.get_next_space(); - space.truncate(0); - driver.commit_space(space); - driver.start_matching(|seq| match seq { - Sequence::Literals { literals } => reconstructed.extend_from_slice(literals), - Sequence::Triple { .. } => panic!("empty input must not emit any matches"), - }); - } - - assert_eq!(reconstructed, data, "L4 greedy round-trip diverged"); - (triple_count, max_offset) -} - -/// CodeRabbit-flagged tail rep-only case: the previous outer-loop -/// guard `pos + ROW_MIN_MATCH_LEN <= current_len` (6) meant the last -/// 5-byte position was unreachable. The rep probe at `abs_pos + 1` -/// only needs 4 bytes of lookahead beyond the probe point, so the -/// guard was relaxed to `pos + GREEDY_MIN_LOOKAHEAD <= current_len` -/// (5). This test drives the slices separately and asserts a match -/// is emitted **from the second slice's parse pass**, so a future -/// regression that re-tightens the guard or breaks the cross-slice -/// repcode lookup fails the test instead of being masked by -/// first-slice matches. -#[test] -fn driver_level5_greedy_tail_rep_only_reachable() { - // Period-4 first slice locks rep1 = 4 into `offset_hist` by the - // time the parse reaches the slice tail. Second slice is exactly - // 5 bytes ( = `GREEDY_MIN_LOOKAHEAD`) so the outer loop runs - // **once** at `pos = 0`; the regular `row_candidate` requires 6 - // bytes from `abs_pos`, which is past the live history, so the - // only viable hit is the `abs_pos + 1` rep probe. `second[0..]` - // is shaped so the rep probe at `abs_pos + 1` finds a 4-byte - // match at offset 4 (`second[1..5] == first[13..16] ++ second[0] - // == "BCDA"`), and `extend_backwards_shared` then absorbs - // `second[0]` into the match (extending one byte back into the - // implicit anchor, no further because anchor itself is the - // current `abs_pos`). - let first: &[u8] = b"ABCDABCDABCDABCD"; // 16 bytes — strict period 4 - let second: &[u8] = b"ABCDA"; // 5 bytes — exact GREEDY_MIN_LOOKAHEAD - let mut driver = MatchGeneratorDriver::new(16, 2); - driver.reset(CompressionLevel::Level(5)); - - let mut first_space = driver.get_next_space(); - first_space[..first.len()].copy_from_slice(first); - first_space.truncate(first.len()); - driver.commit_space(first_space); - driver.start_matching(|_| {}); - - let mut second_space = driver.get_next_space(); - second_space[..second.len()].copy_from_slice(second); - second_space.truncate(second.len()); - driver.commit_space(second_space); - - let mut second_slice_triples = 0usize; - driver.start_matching(|seq| { - if matches!(seq, Sequence::Triple { .. }) { - second_slice_triples += 1; - } - }); - - assert!( - second_slice_triples >= 1, - "tail rep-only position must produce a match in the second slice \ - (got {second_slice_triples} triples)", - ); -} - -#[test] -fn driver_level4_greedy_empty_input_emits_nothing() { - // Empty input: no slices committed → no sequences emitted, no - // panic. Exercises the `current_len == 0` early-return guard at - // the top of `start_matching_greedy`. - let mut driver = MatchGeneratorDriver::new(64, 2); - driver.reset(CompressionLevel::Level(4)); - // Commit an empty space so the matcher has SOMETHING to start - // matching on (otherwise `start_matching` panics on the - // `window.back()` unwrap — that's a separate path covered by - // existing reset tests). - let mut space = driver.get_next_space(); - space.truncate(0); - driver.commit_space(space); - let mut emitted_anything = false; - driver.start_matching(|_| emitted_anything = true); - assert!(!emitted_anything, "empty slice must not emit any sequences",); -} - -#[test] -fn driver_level4_greedy_sub_min_lookahead_input() { - // Input shorter than `GREEDY_MIN_LOOKAHEAD = 5` — the outer loop - // never executes a body iteration; the tail literal path must - // still emit the input bytes as a single `Sequence::Literals`. - let data: &[u8] = b"abcd"; // 4 bytes - let (triples, _) = l4_greedy_round_trip(64, 2, data); - assert_eq!( - triples, 0, - "sub-min-lookahead input must not emit any matches (got {triples})", - ); -} - -#[test] -fn driver_level4_greedy_incompressible_input() { - // Pseudo-random bytes with no exploitable structure — every - // position is a "miss" in both the rep probe and the row - // candidate. Exercises the miss branch + `SKIP_STRENGTH = 10` - // skip-step grow (irrelevant at this size, but the path runs). - let mut data = alloc::vec::Vec::with_capacity(256); - let mut x: u32 = 0xDEAD_BEEF; - for _ in 0..256 { - x = x.wrapping_mul(1_103_515_245).wrapping_add(12345); - data.push((x >> 16) as u8); - } - let (_triples, _) = l4_greedy_round_trip(64, 8, &data); - // No structural assertion — the test passes if round-trip is - // bit-exact and no panic / debug_assert fires. -} - -#[test] -fn driver_level4_greedy_long_literal_run_skip_step_growth() { - // 2 KiB of unstructured bytes drives the literal-run length past - // the `SKIP_STRENGTH = 10` threshold (~1 KiB), so the miss branch - // + per-miss step-grow path in `start_matching_greedy` is - // exercised. This test is a stress smoke — it only asserts - // bit-exact round-trip + no panic / `debug_assert!` fires; it - // does NOT pin the `SKIP_STRENGTH` constant or the per-iteration - // step count (round-trip would still pass on `SKIP_STRENGTH = 6` - // or `= 14` since both produce valid sequences). Pinning the - // exact step growth would require returning step / iteration - // metadata from the parse, which is invasive plumbing for a - // constant that hasn't been re-tuned in months. The value of - // this test is catching panics or correctness regressions on - // long incompressible runs, which is what its existing - // round-trip assertion checks. - let mut data = alloc::vec::Vec::with_capacity(2048); - let mut x: u32 = 0xC0FF_EE00; - for _ in 0..2048 { - x = x.wrapping_mul(0x9E37_79B9).wrapping_add(0xCAFEBABE); - data.push((x >> 24) as u8); - } - let (_triples, _) = l4_greedy_round_trip(512, 8, &data); -} - -#[test] -fn driver_level4_greedy_all_zeros_heavy_rep1() { - // All zeros: every position after the first byte has `byte[pos] - // == byte[pos - 1]`, so the rep1 probe at `abs_pos + 1` hits - // immediately and the parse collapses to a single long match. - // Exercises the `cheap rep at +1, full-match length` path. - let data: Vec = alloc::vec![0u8; 128]; - let (triples, max_offset) = l4_greedy_round_trip(64, 8, &data); - assert!( - triples >= 1, - "all-zeros input must produce at least one rep1 match", - ); - // The dominant match should reference rep1 (offset 1), since - // every byte at pos matches pos-1. A larger offset would - // indicate the rep1 probe was bypassed. - assert_eq!( - max_offset, 1, - "all-zeros L4 greedy parse should commit at offset 1 (got {max_offset})", - ); -} - -/// Periodic-pattern payload covers the steady-state rep-cascade path -/// of the greedy parse — the main-loop rep probe at `abs_pos + 1` -/// fires every iteration once the period is locked into -/// `offset_hist[0]`, and the parse emits a long chain of triples at -/// the same offset. -#[test] -fn driver_level4_greedy_periodic_pattern_rep_cascade() { - let unit: &[u8] = b"alpha_beta_gamma"; - assert_eq!(unit.len(), 16); - let mut data: Vec = Vec::with_capacity(unit.len() * 32); - for _ in 0..32 { - data.extend_from_slice(unit); - } - let (triples, max_offset) = l4_greedy_round_trip(64, 16, &data); - assert!( - triples >= 1, - "periodic 16-byte payload must emit matches (got {triples})", - ); - assert!( - max_offset >= 16, - "periodic 16-byte payload must produce at least one offset >= 16 \ - (got max_offset = {max_offset})", - ); -} - -#[test] -fn driver_reset_keeps_strategy_tag_in_sync_with_active_backend() { - use super::strategy::StrategyTag; - - fn check(level: CompressionLevel, expected: StrategyTag) { - let mut driver = MatchGeneratorDriver::new(32, 2); - driver.reset(level); - assert_eq!( - driver.strategy_tag, expected, - "strategy_tag wrong for {level:?}" - ); - assert_eq!( - driver.strategy_tag.backend(), - driver.active_backend(), - "strategy_tag backend disagrees with active_backend for {level:?}" - ); - } - - check(CompressionLevel::Level(1), StrategyTag::Fast); - check(CompressionLevel::Level(2), StrategyTag::Fast); - check(CompressionLevel::Level(3), StrategyTag::Dfast); - check(CompressionLevel::Level(4), StrategyTag::Dfast); - check(CompressionLevel::Level(5), StrategyTag::Greedy); - check(CompressionLevel::Level(7), StrategyTag::Lazy); - check(CompressionLevel::Level(12), StrategyTag::Lazy); - check(CompressionLevel::Level(13), StrategyTag::Btlazy2); - check(CompressionLevel::Level(14), StrategyTag::Btlazy2); - check(CompressionLevel::Level(15), StrategyTag::Btlazy2); - check(CompressionLevel::Level(16), StrategyTag::BtOpt); - check(CompressionLevel::Level(18), StrategyTag::BtUltra); - check(CompressionLevel::Level(22), StrategyTag::BtUltra2); - check(CompressionLevel::Fastest, StrategyTag::Fast); - check(CompressionLevel::Default, StrategyTag::Dfast); - check(CompressionLevel::Better, StrategyTag::Lazy); - // `Best` sits on level 13 (the first dominant point of the deep band). - check(CompressionLevel::Best, StrategyTag::Btlazy2); -} - -#[test] -fn level_16_17_map_to_btopt_strategy() { - use super::strategy::{BackendTag, StrategyTag}; - let p16 = resolve_level_params(CompressionLevel::Level(16), None); - let p17 = resolve_level_params(CompressionLevel::Level(17), None); - assert_eq!(p16.backend(), BackendTag::HashChain); - assert_eq!(p17.backend(), BackendTag::HashChain); - assert_eq!(StrategyTag::for_level(16), StrategyTag::BtOpt); - assert_eq!(StrategyTag::for_level(17), StrategyTag::BtOpt); -} - -#[test] -fn level_18_maps_to_btultra_level_19_to_btultra2_strategy() { - use super::strategy::{BackendTag, StrategyTag}; - // Upstream zstd `clevels.h` (srcSize > 256 KiB tier): level 18 = `ZSTD_btultra`, - // level 19 = `ZSTD_btultra2`. Level 19 was previously mapped to plain - // btultra, which under-searched (searchLog 6 vs 7) and lost ~3.7% ratio - // on the repo corpus. - let p18 = resolve_level_params(CompressionLevel::Level(18), None); - let p19 = resolve_level_params(CompressionLevel::Level(19), None); - assert_eq!(p18.backend(), BackendTag::HashChain); - assert_eq!(p19.backend(), BackendTag::HashChain); - assert_eq!(StrategyTag::for_level(18), StrategyTag::BtUltra); - assert_eq!(StrategyTag::for_level(19), StrategyTag::BtUltra2); -} - -#[test] -fn level_20_22_map_to_btultra2_strategy() { - use super::strategy::{BackendTag, StrategyTag}; - for level in 20..=22 { - let params = resolve_level_params(CompressionLevel::Level(level), None); - assert_eq!(params.backend(), BackendTag::HashChain); - assert_eq!(StrategyTag::for_level(level as u8), StrategyTag::BtUltra2); - } -} - -#[test] -fn level22_uses_target_length_and_large_input_tables() { - let params = resolve_level_params(CompressionLevel::Level(22), None); - assert_eq!(params.window_log, 27); - let hc = params.hc.unwrap(); - assert_eq!(hc.hash_log, 25); - assert_eq!(hc.chain_log, 27); - assert_eq!(hc.search_depth, 1 << 9); - assert_eq!(hc.target_len, 999); -} - -#[test] -fn bt_levels_16_to_21_pin_clevels_params() { - // Pins the BT-level (window_log, hash_log, chain_log, search_depth, - // target_len) tuples so the clevels.h alignment cannot silently drift. - // Levels 16-20 mirror upstream `clevels.h` (srcSize > 256 KiB tier, - // search_depth = 1 << searchLog); level 21 intentionally keeps a deeper - // search_depth (512 vs upstream's 128) — it beats C on ratio there and - // the deeper walk is a deliberate ratio-positive divergence. - let expected = [ - // (level, window_log, hash_log, chain_log, search_depth, target_len) - (16u8, 22u8, 22usize, 22usize, 32usize, 48usize), - (17, 23, 22, 23, 32, 64), - (18, 23, 22, 23, 64, 64), - (19, 23, 22, 24, 128, 256), - (20, 25, 23, 25, 128, 256), - (21, 26, 24, 24, 512, 256), - ]; - for (level, wlog, hlog, clog, sd, tl) in expected { - let p = resolve_level_params(CompressionLevel::Level(level as i32), None); - assert_eq!(p.window_log, wlog, "level {level} window_log"); - let hc = p.hc.unwrap(); - assert_eq!(hc.hash_log, hlog, "level {level} hash_log"); - assert_eq!(hc.chain_log, clog, "level {level} chain_log"); - assert_eq!(hc.search_depth, sd, "level {level} search_depth"); - assert_eq!(hc.target_len, tl, "level {level} target_len"); - } -} - -#[test] -fn level22_source_size_hint_uses_btultra2_tiers() { - let p16k = resolve_level_params(CompressionLevel::Level(22), Some(16 * 1024)); - assert_eq!(p16k.window_log, 14); - let hc16k = p16k.hc.unwrap(); - assert_eq!(hc16k.hash_log, 15); - assert_eq!(hc16k.chain_log, 15); - assert_eq!(hc16k.search_depth, 1 << 10); - assert_eq!(hc16k.target_len, 999); - - let p128k = resolve_level_params(CompressionLevel::Level(22), Some(128 * 1024)); - assert_eq!(p128k.window_log, 17); - let hc128k = p128k.hc.unwrap(); - assert_eq!(hc128k.hash_log, 17); - assert_eq!(hc128k.chain_log, 18); - assert_eq!(hc128k.search_depth, 1 << 11); - assert_eq!(hc128k.target_len, 999); - - let p256k = resolve_level_params(CompressionLevel::Level(22), Some(256 * 1024)); - assert_eq!(p256k.window_log, 18); - let hc256k = p256k.hc.unwrap(); - assert_eq!(hc256k.hash_log, 19); - assert_eq!(hc256k.chain_log, 19); - assert_eq!(hc256k.search_depth, 1 << 13); - assert_eq!(hc256k.target_len, 999); -} - -#[test] -fn level22_non_power_of_two_small_source_uses_tier3_params() { - // srcSize 15 027 (<= 16 KB) selects the table[3] btultra2 row; the - // source-size clamp gives windowLog 14 (ceil log2 15027). Pure-Rust - // assertion against the constant tier-3 geometry (no FFI). - let source_size = 15_027u64; - let params = resolve_level_params(CompressionLevel::Level(22), Some(source_size)); - - let hc = params.hc.unwrap(); - assert_eq!(params.window_log, 14); - assert_eq!(hc.chain_log, 15); - assert_eq!(hc.hash_log, 15); - assert_eq!(hc.search_depth, 1 << 10); - assert_eq!(HC_OPT_MIN_MATCH_LEN, 3); - assert_eq!(hc.target_len, 999); -} - -#[test] -fn level22_small_source_uses_window_bounded_hash3_log() { - let mut hc = HcMatchGenerator::new(1 << 14); - hc.configure( - BTULTRA2_HC_CONFIG_L22_16K, - super::strategy::StrategyTag::BtUltra2, - 14, - ); - assert_eq!(hc.table.hash3_log, 14); - - hc.configure( - BTULTRA2_HC_CONFIG_L22, - super::strategy::StrategyTag::BtUltra2, - 27, - ); - assert_eq!(hc.table.hash3_log, HC3_HASH_LOG); -} - -#[test] -fn btultra2_seed_pass_initializes_opt_state() { - let mut hc = HcMatchGenerator::new(1 << 20); - hc.configure( - BTULTRA2_HC_CONFIG, - super::strategy::StrategyTag::BtUltra2, - 26, - ); - let data: Vec = (0..32 * 1024).map(|i| (i % 251) as u8).collect(); - hc.table.add_data(data, |_| {}); - hc.start_matching(|_| {}); - assert!( - hc.backend.bt_mut().opt_state.lit_length_sum > 0, - "btultra2 first block should seed non-zero sequence statistics" - ); - assert!( - hc.backend.bt_mut().opt_state.off_code_sum > 0, - "btultra2 first block should seed offset-code statistics" - ); -} - -#[test] -fn btultra2_profile_disables_small_offset_handicap() { - // Pre-Phase-3 this test duplicated the profile build with - // `pass2=false` and `pass2=true` since `for_mode` differentiated - // them. With `const_for_strategy::()` there is only one - // profile — the upstream zstd `opt2` pricing — so a single binding - // captures the invariant the test is asserting. - let profile = HcOptimalCostProfile::const_for_strategy::(); - assert!( - !profile.favor_small_offsets, - "btultra2 should match upstream zstd opt2 offset pricing" - ); - assert!( - profile.accurate, - "btultra2 should use upstream zstd opt2 accurate pricing" - ); -} - -#[test] -fn btultra_profile_keeps_search_depth_budget() { - let p = HcOptimalCostProfile::const_for_strategy::(); - assert_eq!( - p.max_chain_depth, 64, - "btultra chain-depth budget must match clevels.h level 18 searchLog 6 (1 << 6 = 64)" - ); -} - -#[test] -fn btopt_profile_keeps_search_depth_budget() { - let p = HcOptimalCostProfile::const_for_strategy::(); - assert_eq!( - p.max_chain_depth, 32, - "btopt should not cap chain depth below upstream zstd btopt search budget" - ); -} - -#[test] -fn sufficient_match_len_is_clamped_by_target_len() { - let mut hc = HcMatchGenerator::new(1 << 20); - hc.configure( - BTULTRA2_HC_CONFIG, - super::strategy::StrategyTag::BtUltra2, - 26, - ); - hc.hc.target_len = 13; - let profile = HcOptimalCostProfile::const_for_strategy::(); - assert_eq!(hc.hc.sufficient_match_len_for_pass(profile), 13); -} - -#[test] -fn opt_modes_use_target_len_as_sufficient_len() { - use super::strategy; - let mut hc = HcMatchGenerator::new(1 << 20); - hc.hc.target_len = 57; - let profiles = [ - HcOptimalCostProfile::const_for_strategy::(), - HcOptimalCostProfile::const_for_strategy::(), - HcOptimalCostProfile::const_for_strategy::(), - ]; - for profile in profiles { - assert_eq!(hc.hc.sufficient_match_len_for_pass(profile), 57); - } -} - -#[test] -fn sufficient_match_len_is_capped_by_opt_num() { - let mut hc = HcMatchGenerator::new(1 << 20); - hc.hc.target_len = usize::MAX / 2; - let profile = HcOptimalCostProfile::const_for_strategy::(); - assert_eq!(hc.hc.sufficient_match_len_for_pass(profile), HC_OPT_NUM - 1); -} - -#[test] -#[allow(clippy::borrow_deref_ref)] -fn dictionary_entropy_seed_initializes_opt_state_from_tables() { - let mut hc = HcMatchGenerator::new(1 << 20); - hc.configure( - BTULTRA2_HC_CONFIG, - super::strategy::StrategyTag::BtUltra2, - 26, - ); - - let huff = crate::huff0::huff0_encoder::HuffmanTable::build_from_data( - b"aaabbbbccccddddeeeeefffffgggg", - ); - let ll = crate::fse::fse_encoder::default_ll_table(); - let ml = crate::fse::fse_encoder::default_ml_table(); - let of = crate::fse::fse_encoder::default_of_table(); - hc.seed_dictionary_entropy(Some(&huff), Some(&*ll), Some(&*ml), Some(&*of)); - - hc.backend.bt_mut().opt_state.rescale_freqs( - b"abcd", - HcOptimalCostProfile::const_for_strategy::(), - ); - - let base_ll_freqs: [u32; HC_MAX_LL + 1] = [ - 4, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, - ]; - - assert_ne!( - hc.backend.bt_mut().opt_state.lit_length_freq, - base_ll_freqs, - "dictionary entropy should override fallback LL bootstrap frequencies" - ); - assert!( - hc.backend - .bt_mut() - .opt_state - .match_length_freq - .iter() - .any(|&v| v != 1), - "dictionary entropy should seed non-uniform ML frequencies" - ); - assert_ne!( - hc.backend.bt_mut().opt_state.off_code_freq[0], - 6, - "dictionary entropy should override fallback OF bootstrap frequencies" - ); -} - -#[test] -#[allow(clippy::borrow_deref_ref)] -fn dictionary_fse_seed_applies_without_huffman_seed() { - let mut hc = HcMatchGenerator::new(1 << 20); - hc.configure( - BTULTRA2_HC_CONFIG, - super::strategy::StrategyTag::BtUltra2, - 26, - ); - - let ll = crate::fse::fse_encoder::default_ll_table(); - let ml = crate::fse::fse_encoder::default_ml_table(); - let of = crate::fse::fse_encoder::default_of_table(); - hc.seed_dictionary_entropy(None, Some(&*ll), Some(&*ml), Some(&*of)); - hc.backend.bt_mut().opt_state.rescale_freqs( - b"abcd", - HcOptimalCostProfile::const_for_strategy::(), - ); - - let base_ll_freqs: [u32; HC_MAX_LL + 1] = [ - 4, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, - ]; - assert_ne!( - hc.backend.bt_mut().opt_state.lit_length_freq, - base_ll_freqs, - "FSE seed should still override LL bootstrap frequencies without huffman seed" - ); - assert!( - hc.backend - .bt_mut() - .opt_state - .match_length_freq - .iter() - .any(|&v| v != 1), - "FSE seed should still seed non-uniform ML frequencies" - ); - assert_ne!( - hc.backend.bt_mut().opt_state.off_code_freq[0], - 6, - "FSE seed should still override OF bootstrap frequencies without huffman seed" - ); -} - -#[test] -#[allow(clippy::borrow_deref_ref)] -fn dictionary_seed_overrides_predef_price_mode_on_tiny_input() { - let mut hc = HcMatchGenerator::new(1 << 20); - hc.configure( - BTULTRA2_HC_CONFIG, - super::strategy::StrategyTag::BtUltra2, - 26, - ); - - let ll = crate::fse::fse_encoder::default_ll_table(); - let ml = crate::fse::fse_encoder::default_ml_table(); - let of = crate::fse::fse_encoder::default_of_table(); - hc.seed_dictionary_entropy(None, Some(&*ll), Some(&*ml), Some(&*of)); - hc.backend.bt_mut().opt_state.rescale_freqs( - b"abc", - HcOptimalCostProfile::const_for_strategy::(), - ); - assert!( - matches!( - hc.backend.bt_mut().opt_state.price_type, - HcOptPriceType::Dynamic - ), - "dictionary-seeded first block should stay in dynamic mode even for tiny src" - ); -} - -#[test] -fn lit_length_price_blocksize_max_costs_one_extra_bit() { - let profile_predef = HcOptimalCostProfile::const_for_strategy::(); - let mut stats_predef = HcOptState::new(); - stats_predef.price_type = HcOptPriceType::Predefined; - let predef_max = profile_predef.lit_length_price(&stats_predef, HC_BLOCKSIZE_MAX); - let predef_prev = - profile_predef.lit_length_price(&stats_predef, HC_BLOCKSIZE_MAX.saturating_sub(1)); - assert_eq!( - predef_max, - predef_prev + HC_BITCOST_MULTIPLIER, - "predefined litLength pricing at BLOCKSIZE_MAX must add exactly one bit" - ); - - let profile_dyn = HcOptimalCostProfile::const_for_strategy::(); - let mut stats_dyn = HcOptState::new(); - stats_dyn.price_type = HcOptPriceType::Dynamic; - stats_dyn.lit_length_freq.fill(1); - stats_dyn.lit_length_sum = (HC_MAX_LL + 1) as u32; - stats_dyn.match_length_freq.fill(1); - stats_dyn.match_length_sum = (HC_MAX_ML + 1) as u32; - stats_dyn.off_code_freq.fill(1); - stats_dyn.off_code_sum = (HC_MAX_OFF + 1) as u32; - stats_dyn.lit_freq.fill(1); - stats_dyn.lit_sum = (HC_MAX_LIT + 1) as u32; - stats_dyn.set_base_prices(true); - let dyn_max = profile_dyn.lit_length_price(&stats_dyn, HC_BLOCKSIZE_MAX); - let dyn_prev = profile_dyn.lit_length_price(&stats_dyn, HC_BLOCKSIZE_MAX.saturating_sub(1)); - assert_eq!( - dyn_max, - dyn_prev + HC_BITCOST_MULTIPLIER, - "dynamic litLength pricing at BLOCKSIZE_MAX must add exactly one bit" - ); -} - -#[test] -#[allow(clippy::borrow_deref_ref)] -fn btultra2_seed_pass_disabled_when_dictionary_entropy_seed_present() { - let mut hc = HcMatchGenerator::new(1 << 20); - hc.configure( - BTULTRA2_HC_CONFIG, - super::strategy::StrategyTag::BtUltra2, - 26, - ); - let ll = crate::fse::fse_encoder::default_ll_table(); - let ml = crate::fse::fse_encoder::default_ml_table(); - let of = crate::fse::fse_encoder::default_of_table(); - hc.seed_dictionary_entropy(None, Some(&*ll), Some(&*ml), Some(&*of)); - assert!( - !hc.should_run_btultra2_seed_pass::(HC_PREDEF_THRESHOLD + 1), - "dictionary-seeded first block should skip btultra2 warmup pass" - ); -} - -#[test] -fn btultra2_seed_pass_disabled_when_prefix_history_exists() { - let mut hc = HcMatchGenerator::new(1 << 20); - hc.configure( - BTULTRA2_HC_CONFIG, - super::strategy::StrategyTag::BtUltra2, - 26, - ); - hc.table.history_abs_start = 17; - hc.table.push_test_chunk(b"abcdefghijklmnop".to_vec()); - assert!( - !hc.should_run_btultra2_seed_pass::(HC_PREDEF_THRESHOLD + 9), - "btultra2 warmup must be first-block only (no prefix history)" - ); -} - -#[test] -fn btultra2_seed_pass_disabled_for_tiny_block() { - let mut hc = HcMatchGenerator::new(1 << 20); - hc.configure( - BTULTRA2_HC_CONFIG, - super::strategy::StrategyTag::BtUltra2, - 26, - ); - assert!( - !hc.should_run_btultra2_seed_pass::(HC_PREDEF_THRESHOLD), - "btultra2 warmup should not run at or below predefined threshold" - ); -} - -#[test] -fn btultra2_seed_pass_disabled_after_stats_initialized() { - let mut hc = HcMatchGenerator::new(1 << 20); - hc.configure( - BTULTRA2_HC_CONFIG, - super::strategy::StrategyTag::BtUltra2, - 26, - ); - hc.backend.bt_mut().opt_state.lit_length_sum = 1; - assert!( - !hc.should_run_btultra2_seed_pass::(HC_PREDEF_THRESHOLD + 32), - "btultra2 warmup should run only for first block before stats are initialized" - ); -} - -#[test] -fn btultra2_seed_pass_disabled_when_not_at_frame_start() { - let mut hc = HcMatchGenerator::new(1 << 20); - hc.configure( - BTULTRA2_HC_CONFIG, - super::strategy::StrategyTag::BtUltra2, - 26, - ); - // Simulate non-first block state: current block has no prefix in deque, - // but total produced window already includes prior output. - hc.table.window_size = HC_PREDEF_THRESHOLD + 64; - // window_size set manually above to simulate prior output; record the - // current block as one live chunk (seed-pass check reads lengths, not bytes). - hc.table.chunk_lens.push_back(HC_PREDEF_THRESHOLD + 32); - assert!( - !hc.should_run_btultra2_seed_pass::(HC_PREDEF_THRESHOLD + 32), - "btultra2 warmup must not run after frame start" - ); -} - -#[test] -fn btultra2_seed_pass_disabled_when_ldm_sequences_exist() { - let mut hc = HcMatchGenerator::new(1 << 20); - hc.configure( - BTULTRA2_HC_CONFIG, - super::strategy::StrategyTag::BtUltra2, - 26, - ); - hc.table.window_size = HC_PREDEF_THRESHOLD + 64; - hc.table.chunk_lens.push_back(HC_PREDEF_THRESHOLD + 64); - hc.backend.bt_mut().ldm_sequences.push(HcRawSeq { - lit_length: 8, - offset: 16, - match_length: 32, - }); - assert!( - !hc.should_run_btultra2_seed_pass::(HC_PREDEF_THRESHOLD + 32), - "btultra2 warmup must not run when LDM already produced sequences" - ); -} - -#[test] -fn literal_price_uses_eight_bits_when_literals_uncompressed() { - let profile = HcOptimalCostProfile::const_for_strategy::(); - let mut stats = HcOptState::new(); - stats.set_literals_compressed_for_tests(false); - stats.price_type = HcOptPriceType::Predefined; - assert_eq!( - profile.literal_price(&stats, b'a'), - 8 * HC_BITCOST_MULTIPLIER, - "uncompressed literals should cost 8 bits regardless of price mode" - ); -} - -#[test] -fn update_stats_skips_literal_frequencies_when_uncompressed() { - let mut stats = HcOptState::new(); - stats.set_literals_compressed_for_tests(false); - stats.update_stats(3, b"abc", 4, 8); - assert_eq!( - stats.lit_sum, 0, - "literal sum must remain unchanged when literal compression is disabled" - ); - assert_eq!( - stats.lit_freq.iter().copied().sum::(), - 0, - "literal frequencies must not be updated when literal compression is disabled" - ); - assert_eq!( - stats.lit_length_sum, 1, - "literal-length stats still update for sequence modeling" - ); - assert_eq!( - stats.match_length_sum, 1, - "match-length stats still update for sequence modeling" - ); - assert_eq!( - stats.off_code_sum, 1, - "offset-code stats still update for sequence modeling" - ); -} - -#[test] -#[allow(clippy::borrow_deref_ref)] -fn dictionary_huffman_seed_ignored_when_literals_uncompressed() { - let mut stats = HcOptState::new(); - stats.set_literals_compressed_for_tests(false); - let huff = crate::huff0::huff0_encoder::HuffmanTable::build_from_data( - b"aaaaabbbbcccddeeff00112233445566778899", - ); - let ll = crate::fse::fse_encoder::default_ll_table(); - let ml = crate::fse::fse_encoder::default_ml_table(); - let of = crate::fse::fse_encoder::default_of_table(); - stats.seed_dictionary_entropy(Some(&huff), Some(&*ll), Some(&*ml), Some(&*of)); - stats.rescale_freqs( - b"abcd", - HcOptimalCostProfile::const_for_strategy::(), - ); - assert_eq!( - stats.lit_sum, 0, - "literal sum must stay zero when literals are uncompressed" - ); - assert_eq!( - stats.lit_freq.iter().copied().sum::(), - 0, - "literal frequencies must ignore dictionary huffman seed when uncompressed" - ); -} - -#[test] -fn hc_repcode_candidates_respect_litlen_dependent_rep_order() { - let mut hc = HcMatchGenerator::new(64); - hc.table.history = b"xxxxxxABCDEFABCDEF".to_vec(); - hc.table.history_start = 0; - hc.table.history_abs_start = 0; - - let abs_pos = 12usize; // points at second "ABCDEF" - let current_abs_end = hc.table.history.len(); - let reps = [6u32, 3u32, 9u32]; - - let mut lit_pos_candidates = Vec::new(); - hc.hc.for_each_repcode_candidate_with_reps( - &hc.table, - abs_pos, - 1, - reps, - current_abs_end, - HC_OPT_MIN_MATCH_LEN, - |c| { - lit_pos_candidates.push(c.offset); - }, - ); - assert!( - lit_pos_candidates.contains(&6), - "when lit_len>0, rep0 should be considered and match" - ); - - let mut ll0_candidates = Vec::new(); - hc.hc.for_each_repcode_candidate_with_reps( - &hc.table, - abs_pos, - 0, - reps, - current_abs_end, - HC_OPT_MIN_MATCH_LEN, - |c| { - ll0_candidates.push(c.offset); - }, - ); - assert!( - !ll0_candidates.contains(&6), - "when lit_len==0, rep0 is not directly eligible (ll0 semantics)" - ); -} - -#[test] -fn hc_collect_optimal_candidates_keeps_reps_when_chain_depth_zero() { - let mut hc = HcMatchGenerator::new(64); - hc.hc.search_depth = 0; - hc.table.history = b"xyzxyzxyzxyz".to_vec(); - hc.table.history_start = 0; - hc.table.history_abs_start = 0; - - let abs_pos = 6usize; - let current_abs_end = hc.table.history.len(); - let profile = HcOptimalCostProfile { - max_chain_depth: 0, - sufficient_match_len: usize::MAX / 2, - accurate: false, - favor_small_offsets: false, - }; - let mut out = Vec::new(); - hc.collect_optimal_candidates( - abs_pos, - current_abs_end, - profile, - HcCandidateQuery { - reps: [3, 6, 9], - lit_len: 1, - ldm_candidate: None, - }, - &mut out, - ); - assert!( - !out.is_empty(), - "rep candidates should remain available even when chain depth is zero" - ); - assert!( - out.iter().any(|c| c.offset == 3), - "rep0 candidate should be retained" - ); -} - -#[test] -fn hc_collect_optimal_candidates_rep_tail_match_skips_chain_probe() { - let mut hc = HcMatchGenerator::new(64); - hc.table.history = b"aaaaaaaaaa".to_vec(); - hc.table.history_start = 0; - hc.table.history_abs_start = 0; - hc.table.position_base = 0; - hc.hc.search_depth = 32; - let abs_pos = 6usize; - hc.table.ensure_tables(); - hc.table.insert_positions(0, abs_pos); - - let profile = HcOptimalCostProfile { - max_chain_depth: 32, - sufficient_match_len: usize::MAX / 2, - accurate: true, - favor_small_offsets: false, - }; - let mut out = Vec::new(); - hc.collect_optimal_candidates( - abs_pos, - hc.table.history.len(), - profile, - HcCandidateQuery { - reps: [1, 4, 8], - lit_len: 1, - ldm_candidate: None, - }, - &mut out, - ); - - assert!( - out.iter() - .all(|candidate| matches!(candidate.offset, 1 | 4)), - "terminal rep match should return before chain probing adds non-rep offsets" - ); -} - -#[test] -fn hc_collect_optimal_candidates_long_chain_match_advances_skip_window() { - let mut hc = HcMatchGenerator::new(128); - hc.table.history = b"abcabcabcabcabcabcabcabc".to_vec(); - hc.table.history_start = 0; - hc.table.history_abs_start = 0; - hc.table.position_base = 0; - hc.hc.search_depth = 32; - let abs_pos = 9usize; - hc.table.ensure_tables(); - hc.table.insert_positions(0, abs_pos); - hc.table.skip_insert_until_abs = 0; - - let profile = HcOptimalCostProfile { - max_chain_depth: 32, - sufficient_match_len: usize::MAX / 2, - accurate: true, - favor_small_offsets: false, - }; - let mut out = Vec::new(); - hc.collect_optimal_candidates( - abs_pos, - hc.table.history.len(), - profile, - HcCandidateQuery { - reps: [1, 4, 8], - lit_len: 1, - ldm_candidate: None, - }, - &mut out, - ); - - assert!( - hc.table.skip_insert_until_abs > abs_pos, - "long chain match should advance skip window to avoid redundant immediate insertions" - ); -} - -#[test] -fn hc_collect_optimal_candidates_chain_fast_skip_uses_match_end_minus_8() { - let mut hc = HcMatchGenerator::new(128); - hc.table.history = b"abcabcabcabcabcabcabcabc".to_vec(); - hc.table.history_start = 0; - hc.table.history_abs_start = 0; - hc.table.position_base = 0; - hc.hc.search_depth = 32; - let abs_pos = 9usize; - hc.table.ensure_tables(); - hc.table.insert_positions(0, abs_pos); - hc.table.skip_insert_until_abs = 0; - - let profile = HcOptimalCostProfile { - max_chain_depth: 32, - sufficient_match_len: 10, - accurate: true, - favor_small_offsets: false, - }; - let mut out = Vec::new(); - hc.collect_optimal_candidates( - abs_pos, - hc.table.history.len(), - profile, - HcCandidateQuery { - reps: [1, 4, 8], - lit_len: 1, - ldm_candidate: None, - }, - &mut out, - ); - - let best_match_end = out - .iter() - .map(|candidate| candidate.start.saturating_add(candidate.match_len)) - .max() - .expect("expected at least one candidate"); - assert!( - hc.table.skip_insert_until_abs > abs_pos, - "chain fast-skip must advance past current position" - ); - assert!( - hc.table.skip_insert_until_abs <= best_match_end.saturating_sub(8), - "chain fast-skip must not exceed upstream zstd-style matchEndIdx - 8 bound" - ); -} - -#[test] -fn hc_collect_optimal_candidates_advances_skip_window_on_plain_bt_path() { - let mut hc = HcMatchGenerator::new(256); - hc.table.history = b"abcdefghijklmnop".to_vec(); - hc.table.history_start = 0; - hc.table.history_abs_start = 0; - hc.table.position_base = 0; - hc.hc.search_depth = 0; - hc.table.ensure_tables(); - - let abs_pos = 8usize; - hc.table.skip_insert_until_abs = 0; - - let profile = HcOptimalCostProfile { - max_chain_depth: 0, - sufficient_match_len: usize::MAX / 2, - accurate: true, - favor_small_offsets: false, - }; - let mut out = Vec::new(); - hc.collect_optimal_candidates( - abs_pos, - hc.table.history.len(), - profile, - HcCandidateQuery { - reps: [1, 4, 8], - lit_len: 1, - ldm_candidate: None, - }, - &mut out, - ); - - assert_eq!( - hc.table.skip_insert_until_abs, - abs_pos.saturating_add(1), - "plain BT path should advance skip window by 1 via upstream zstd matchEndIdx baseline" - ); -} - -// Removed: the three `hc_collect_optimal_candidates_*_hash3_*` / -// `hc_hash3_tail_match_*` tests forced `search_depth = 0` together -// with `hash3_log != 0`, an HC-chain-walker-only fixture state that -// production never reaches (hash3 is BtUltra2-only and BtUltra2 always -// runs `search_depth = 512`). They depended on the `has_hash3 => -// BtUltra2` escape hatch in the test dispatcher; with that hatch gone -// (CR review on PR #123) and the dispatcher routing purely from -// `self.strategy_tag`, there is no production-shaped configuration -// that reproduces what those tests asserted. The corresponding hash3 -// invariants are exercised end-to-end by the existing level22 roundtrip -// + upstream zstd-parity ratio gate. - -#[test] -fn hc_ldm_candidates_are_merged_into_optimal_candidates() { - let mut hc = HcMatchGenerator::new(512); - hc.table.history = (0..256).map(|i| (i % 251) as u8).collect(); - hc.table.history_start = 0; - hc.table.history_abs_start = 0; - - let abs_pos = 128usize; - let current_abs_end = 256usize; - let ldm = MatchCandidate { - start: abs_pos, - offset: 96, - match_len: 40, - }; - - let profile = HcOptimalCostProfile { - max_chain_depth: 0, - sufficient_match_len: usize::MAX / 2, - accurate: true, - favor_small_offsets: false, - }; - let mut out = Vec::new(); - hc.collect_optimal_candidates( - abs_pos, - current_abs_end, - profile, - HcCandidateQuery { - reps: [1, 4, 8], - lit_len: 1, - ldm_candidate: Some(ldm), - }, - &mut out, - ); - assert!( - out.iter().any( - |candidate| candidate.offset == ldm.offset && candidate.match_len == ldm.match_len - ), - "LDM candidate should be present in optimal candidate set" - ); -} - -#[test] -fn btultra_and_btultra2_both_keep_dictionary_candidates() { - // Routes the BtUltra2 / BtUltra fixture through the production - // `configure()` path so derived state (`hash3_log`, `is_btultra2`, - // `uses_bt`, `backend`) stays consistent — manually flipping the - // strategy flags here used to leave `hash3_log` / `hash3_table` in - // the previous mode's shape and trip the - // `Strategy::USE_HASH3 ⇒ hash3_log != 0` debug invariant inside - // `collect_optimal_candidates_initialized_body`. - use super::strategy::StrategyTag; - - let test_config = HcConfig { - hash_log: 23, - chain_log: 22, - search_depth: 32, - target_len: 256, - search_mls: 4, - }; - let window_log = 20u8; - - let prepare_history = |hc: &mut HcMatchGenerator, abs_pos: usize| { - hc.table.history = alloc::vec![0u8; 160]; - for i in 0..64 { - hc.table.history[i] = b'a' + (i % 7) as u8; - } - for i in 64..160 { - hc.table.history[i] = b'k' + (i % 5) as u8; - } - for i in 0..24 { - hc.table.history[abs_pos + i] = hc.table.history[16 + i]; - } - hc.table.history_start = 0; - hc.table.history_abs_start = 0; - hc.table.position_base = 0; - hc.table.ensure_tables(); - hc.table.insert_positions(0, abs_pos); - hc.table.dictionary_limit_abs = Some(64); - hc.table.skip_insert_until_abs = 0; - }; - - let profile = HcOptimalCostProfile { - max_chain_depth: 32, - sufficient_match_len: usize::MAX / 2, - accurate: true, - favor_small_offsets: false, - }; - let abs_pos = 96usize; - let mut out = Vec::new(); - - let mut hc = HcMatchGenerator::new(256); - hc.configure(test_config, StrategyTag::BtUltra2, window_log); - prepare_history(&mut hc, abs_pos); - hc.collect_optimal_candidates( - abs_pos, - 160, - profile, - HcCandidateQuery { - reps: [1, 4, 8], - lit_len: 1, - ldm_candidate: None, - }, - &mut out, - ); - assert!( - out.iter().any(|candidate| candidate.offset >= 32), - "btultra2 should retain dictionary candidates on upstream zstd-parity path" - ); - - let mut hc = HcMatchGenerator::new(256); - hc.configure(test_config, StrategyTag::BtUltra, window_log); - prepare_history(&mut hc, abs_pos); - hc.collect_optimal_candidates( - abs_pos, - 160, - profile, - HcCandidateQuery { - reps: [1, 4, 8], - lit_len: 1, - ldm_candidate: None, - }, - &mut out, - ); - assert!( - out.iter().any(|candidate| candidate.offset >= 32), - "btultra should retain dictionary candidates" - ); -} - -#[test] -fn driver_small_source_hint_shrinks_dfast_hash_tables() { - let mut driver = MatchGeneratorDriver::new(32, 2); - - driver.reset(CompressionLevel::Level(3)); - let mut space = driver.get_next_space(); - space[..12].copy_from_slice(b"abcabcabcabc"); - space.truncate(12); - driver.commit_space(space); - driver.skip_matching_with_hint(None); - // Upstream zstd-parity split sizes: long-hash = DFAST_HASH_BITS, - // short-hash = DFAST_HASH_BITS - DFAST_SHORT_HASH_BITS_DELTA. - let full_long = driver.dfast_matcher().long_len(); - let full_short = driver.dfast_matcher().short_len(); - assert_eq!(full_long, 1 << DFAST_HASH_BITS); - assert_eq!( - full_short, - 1 << (DFAST_HASH_BITS - DFAST_SHORT_HASH_BITS_DELTA) - ); - - driver.set_source_size_hint(1024); - driver.reset(CompressionLevel::Level(3)); - let mut space = driver.get_next_space(); - space[..12].copy_from_slice(b"xyzxyzxyzxyz"); - space.truncate(12); - driver.commit_space(space); - driver.skip_matching_with_hint(None); - let hinted_long = driver.dfast_matcher().long_len(); - let hinted_short = driver.dfast_matcher().short_len(); - - // The wire `window_log` stays at its floor (decoder-interop), but the - // internal dfast tables are sized from the RAW 1 KiB source, not the - // floored window: `table_window = 1 << ceil_log2(1024) = 1 << 10`, so - // both tables land at the `MIN_WINDOW_LOG` floor (the long table at - // `dfast_hash_bits_for_window(1 << 10) = 10`, the short table one - // `DFAST_SHORT_HASH_BITS_DELTA` step below but clamped back up to - // `MIN_WINDOW_LOG`). - assert_eq!(driver.window_size(), 1 << MIN_HINTED_WINDOW_LOG); - assert_eq!(hinted_long, 1 << MIN_WINDOW_LOG); - assert_eq!(hinted_short, 1 << MIN_WINDOW_LOG); - assert!( - hinted_long < full_long && hinted_short < full_short, - "tiny source hint should reduce both dfast tables" - ); -} - -#[test] -fn driver_huge_source_hint_does_not_overflow_table_window_shift() { - // Regression: the Dfast / Row table-window sizing in `reset` derives a - // shift from `ceil_log2(hint)`. A hint >= 2^63 + 1 makes that shift 64, - // and `1usize << 64` panics in debug / wraps to 0 in release before the - // `.min(max_window_size)` cap can apply. A `u64::MAX` pledged source size - // must size the table to the real window, never panic or wrap to zero. - let mut driver = MatchGeneratorDriver::new(32, 2); - driver.set_source_size_hint(u64::MAX); - driver.reset(CompressionLevel::Level(3)); - - let mut space = driver.get_next_space(); - space[..12].copy_from_slice(b"abcabcabcabc"); - space.truncate(12); - driver.commit_space(space); - driver.skip_matching_with_hint(None); - - assert!( - driver.dfast_matcher().long_len() >= 1 << MIN_WINDOW_LOG, - "huge hint must size the dfast table from the real window, not wrap to zero" - ); -} - -#[test] -fn driver_huge_source_hint_with_dict_does_not_overflow_hc_reserve() { - // Regression: the HC/BT history-mirror pre-size adds the dictionary - // hint to the source-size hint before `reserve_history` clamps to the - // window ceiling. A `u64::MAX` pledged source size (the "unknown size" - // sentinel) plus any positive dictionary hint overflows `usize` in - // `(src as usize) + dict_hint` — debug panic / release wrap on 64-bit, - // and `src as usize` truncation on 32-bit targets. Level 16 (BtOpt) - // routes through the HashChain/BT storage arm that owns this reserve. - // Must size the mirror to the real window, never panic, wrap, or - // truncate. - let mut driver = MatchGeneratorDriver::new(32, 2); - driver.set_source_size_hint(u64::MAX); - driver.set_dictionary_size_hint(64 * 1024); - driver.reset(CompressionLevel::Level(16)); - - // The saturated `usize::MAX` reserve target must be clamped to the HC - // history ceiling, not reserved literally (which would OOM/panic). Level 16 - // has window_log 22, so the ceiling is `window + window/4 + one block` - // (the `reserve_history` formula). Assert the reserve actually reached it — - // a no-panic-only check would also pass on an under-reserved mirror. - let window = 1usize << 22; - let expected_history_ceiling = window + (window >> 2) + crate::common::MAX_BLOCK_SIZE as usize; - assert!( - driver.hc_matcher().table.history.capacity() >= expected_history_ceiling, - "huge source + dict hint must reserve the clamped HC history ceiling, got {}", - driver.hc_matcher().table.history.capacity() - ); - - let mut space = driver.get_next_space(); - space[..12].copy_from_slice(b"abcabcabcabc"); - space.truncate(12); - driver.commit_space(space); - driver.skip_matching_with_hint(None); -} - -#[test] -fn driver_chain_log_override_survives_row_to_hc_fallback() { - // Regression: when a RowHash level is forced onto the HashChain backend - // (resolved window <= 14, upstream `ZSTD_resolveRowMatchFinderMode`), the - // synthesised HC chain table must honour an explicit `chain_log` override. - // The RowHash override arm drops `chain_log` (Row has no chain table), so - // the synthesis previously replaced the caller's `chain_log` with the upstream zstd - // `hashLog - 1`, silently ignoring it on small-window frames. - let chain_log_override = 10u32; - let ov = super::parameters::ParamOverrides { - chain_log: Some(chain_log_override), - ..Default::default() - }; - let mut driver = MatchGeneratorDriver::new(32, 2); - // Small source hint pins the window to the hinted floor (16 KiB = - // windowLog 14), so the Level 6 Row finder falls back to HashChain. - driver.set_source_size_hint(1 << 12); - driver.set_param_overrides(Some(ov)); - driver.reset(CompressionLevel::Level(6)); - let mut space = driver.get_next_space(); - space[..12].copy_from_slice(b"abcabcabcabc"); - space.truncate(12); - driver.commit_space(space); - driver.skip_matching_with_hint(None); - // The override (10) is below the window cap (14), so the resolved HC chain - // table must reflect it — NOT the upstream zstd `hashLog - 1` (18, clamped to the - // window 14). Pre-fix this resolved to 14. - assert_eq!( - driver.hc_matcher().table.chain_log, - chain_log_override as usize, - "explicit chain_log override must survive the Row->HC fallback, got {}", - driver.hc_matcher().table.chain_log - ); -} - -#[test] -fn driver_small_source_hint_shrinks_row_hash_tables() { - let mut driver = MatchGeneratorDriver::new(32, 2); - - driver.reset(CompressionLevel::Level(5)); - let mut space = driver.get_next_space(); - space[..12].copy_from_slice(b"abcabcabcabc"); - space.truncate(12); - driver.commit_space(space); - driver.skip_matching_with_hint(None); - let full_rows = driver.row_matcher().row_heads.len(); - // Level 5 uses the upstream row_log (clamp(searchLog=3, 4, 6) = 4) and the - // upstream L5 hashLog (`ZSTD_getCParams(5,..).hashLog` = 19), so the row - // count is 1 << (ROW_L5.hash_bits - ROW_L5.row_log). - assert_eq!(full_rows, 1 << (ROW_L5.hash_bits - ROW_L5.row_log)); - - // A hint that keeps the resolved window > 14 STILL uses the Row finder - // (upstream `ZSTD_resolveRowMatchFinderMode`: row mode on for windowLog > 14) - // and shrinks the row hash table to the source-derived width. 64 KiB → - // raw source log 16, so `row_hash_bits_for_window(1 << 16)` < the level's - // full hash_bits (19) and the row count drops. - driver.set_source_size_hint(1 << 16); - driver.reset(CompressionLevel::Level(5)); - let mut space = driver.get_next_space(); - space[..12].copy_from_slice(b"xyzxyzxyzxyz"); - space.truncate(12); - driver.commit_space(space); - driver.skip_matching_with_hint(None); - assert_eq!( - driver.active_backend(), - super::strategy::BackendTag::Row, - "windowLog > 14 keeps the upstream row matchfinder" - ); - let hinted_rows = driver.row_matcher().row_heads.len(); - assert!( - hinted_rows < full_rows, - "a window>14 source hint should reduce the row hash table footprint" - ); - - // A tiny hint floors the resolved window at MIN_HINTED_WINDOW_LOG = 14; - // upstream uses the HASH-CHAIN matcher (not Row) at windowLog <= 14, so the - // driver must route greedy/lazy/lazy2 to the HashChain backend there. - driver.set_source_size_hint(1024); - driver.reset(CompressionLevel::Level(5)); - assert_eq!(driver.window_size(), 1 << MIN_HINTED_WINDOW_LOG); - assert_eq!( - driver.active_backend(), - super::strategy::BackendTag::HashChain, - "windowLog <= 14 must fall back to the upstream zstd hash-chain matchfinder", - ); -} - -#[test] -fn row_matches_roundtrip_multi_block_pattern() { - let pattern = [7, 13, 44, 184, 19, 96, 171, 109, 141, 251]; - let first_block: Vec = pattern.iter().copied().cycle().take(128 * 1024).collect(); - let second_block: Vec = pattern.iter().copied().cycle().take(128 * 1024).collect(); - - let mut matcher = RowMatchGenerator::new(1 << 22); - matcher.configure(ROW_CONFIG); - matcher.ensure_tables(); - let replay_sequence = |decoded: &mut Vec, seq: Sequence<'_>| match seq { - Sequence::Literals { literals } => decoded.extend_from_slice(literals), - Sequence::Triple { - literals, - offset, - match_len, - } => { - decoded.extend_from_slice(literals); - let start = decoded.len() - offset; - for i in 0..match_len { - let byte = decoded[start + i]; - decoded.push(byte); - } - } - }; - - matcher.add_data(first_block.clone(), |_| {}); - let mut history = Vec::new(); - matcher.start_matching(|seq| replay_sequence(&mut history, seq)); - assert_eq!(history, first_block); - - matcher.add_data(second_block.clone(), |_| {}); - let prefix_len = history.len(); - matcher.start_matching(|seq| replay_sequence(&mut history, seq)); - - assert_eq!(&history[prefix_len..], second_block.as_slice()); - - // Force a literals-only pass so the Sequence::Literals arm is exercised. - let third_block: Vec = (0u8..=255).collect(); - matcher.add_data(third_block.clone(), |_| {}); - let third_prefix = history.len(); - matcher.start_matching(|seq| replay_sequence(&mut history, seq)); - assert_eq!(&history[third_prefix..], third_block.as_slice()); -} - -#[test] -fn row_short_block_emits_literals_only() { - let mut matcher = RowMatchGenerator::new(1 << 22); - matcher.configure(ROW_CONFIG); - - matcher.add_data(b"abcde".to_vec(), |_| {}); - - let mut saw_triple = false; - let mut reconstructed = Vec::new(); - matcher.start_matching(|seq| match seq { - Sequence::Literals { literals } => reconstructed.extend_from_slice(literals), - Sequence::Triple { .. } => saw_triple = true, - }); - - assert!( - !saw_triple, - "row backend must not emit triples for short blocks" - ); - assert_eq!(reconstructed, b"abcde"); - - // Then feed a clearly matchable block and ensure the Triple arm is reachable. - saw_triple = false; - matcher.add_data(b"abcdeabcde".to_vec(), |_| {}); - matcher.start_matching(|seq| { - if let Sequence::Triple { .. } = seq { - saw_triple = true; - } - }); - assert!( - saw_triple, - "row backend should emit triples on repeated data" - ); -} - -#[test] -fn row_pick_lazy_returns_best_when_lookahead_is_out_of_bounds() { - let mut matcher = RowMatchGenerator::new(1 << 22); - matcher.configure(ROW_CONFIG); - matcher.add_data(b"abcabc".to_vec(), |_| {}); - // Build the row tables before probing: the lookahead path reaches - // `row_candidate` -> `row_heads[..]` once the accept floor is small - // enough to pass the length gate, so the tables must be allocated - // (production always calls this before any candidate probe). - matcher.ensure_tables(); - - let best = MatchCandidate { - start: 0, - offset: 1, - match_len: ROW_MIN_MATCH_LEN, - }; - let picked = matcher - .pick_lazy_match(0, 0, Some(best)) - .expect("best candidate must survive"); - - assert_eq!(picked.start, best.start); - assert_eq!(picked.offset, best.offset); - assert_eq!(picked.match_len, best.match_len); -} - -#[test] -fn row_backfills_previous_block_tail_for_cross_boundary_match() { - let mut matcher = RowMatchGenerator::new(1 << 22); - matcher.configure(ROW_CONFIG); - - let mut first_block = alloc::vec![0xA5; 64]; - first_block.extend_from_slice(b"XYZ"); - let second_block = b"XYZXYZtail".to_vec(); - - let replay_sequence = |decoded: &mut Vec, seq: Sequence<'_>| match seq { - Sequence::Literals { literals } => decoded.extend_from_slice(literals), - Sequence::Triple { - literals, - offset, - match_len, - } => { - decoded.extend_from_slice(literals); - let start = decoded.len() - offset; - for i in 0..match_len { - let byte = decoded[start + i]; - decoded.push(byte); - } - } - }; - - matcher.add_data(first_block.clone(), |_| {}); - let mut reconstructed = Vec::new(); - matcher.start_matching(|seq| replay_sequence(&mut reconstructed, seq)); - assert_eq!(reconstructed, first_block); - - matcher.add_data(second_block.clone(), |_| {}); - let mut saw_cross_boundary = false; - let prefix_len = reconstructed.len(); - matcher.start_matching(|seq| { - if let Sequence::Triple { - literals, - offset, - match_len, - } = seq - && literals.is_empty() - && offset == 3 - && match_len >= ROW_MIN_MATCH_LEN - { - saw_cross_boundary = true; - } - replay_sequence(&mut reconstructed, seq); - }); - - assert!( - saw_cross_boundary, - "row matcher should reuse the 3-byte previous-block tail" - ); - assert_eq!(&reconstructed[prefix_len..], second_block.as_slice()); -} - -#[test] -fn row_skip_matching_with_incompressible_hint_uses_sparse_prefix() { - let data = deterministic_high_entropy_bytes(0xA713_9C5D_44E2_10B1, 4096); - - let mut dense = RowMatchGenerator::new(1 << 22); - dense.configure(ROW_CONFIG); - dense.add_data(data.clone(), |_| {}); - dense.skip_matching_with_hint(Some(false)); - let dense_slots = dense - .row_positions - .iter() - .filter(|&&pos| pos != ROW_EMPTY_SLOT) - .count(); - - let mut sparse = RowMatchGenerator::new(1 << 22); - sparse.configure(ROW_CONFIG); - sparse.add_data(data, |_| {}); - sparse.skip_matching_with_hint(Some(true)); - let sparse_slots = sparse - .row_positions - .iter() - .filter(|&&pos| pos != ROW_EMPTY_SLOT) - .count(); - - assert!( - sparse_slots < dense_slots, - "incompressible hint should seed fewer row slots (sparse={sparse_slots}, dense={dense_slots})" - ); -} - -/// Regression for the `None` arm of `skip_matching_with_hint`: the -/// row table must NOT receive dense inserts across the skipped range. -/// Upstream zstd parity (`ZSTD_row_fillHashCache` only pre-fills the next-scan -/// cache, not the skipped block's interior) trades cross-block -/// matches into the skipped interior for the per-block O(block_size) -/// insert cost. -/// -/// At input < 1 block (4096 B with default 128 KiB block boundary), -/// the only positions in the row table after the call should be those -/// produced by the `backfill_start` lookback at the block's start -/// (≤ `ROW_HASH_KEY_LEN - 1` positions when block_start < -/// ROW_HASH_KEY_LEN). For `current_abs_start == 0`, even that backfill -/// is empty — so the table stays fully empty. -#[test] -fn row_skip_matching_with_none_hint_leaves_interior_empty() { - let data = deterministic_high_entropy_bytes(0x9B47_F2A1_8C5E_3306, 4096); - - let mut none_hint = RowMatchGenerator::new(1 << 22); - none_hint.configure(ROW_CONFIG); - none_hint.add_data(data.clone(), |_| {}); - none_hint.skip_matching_with_hint(None); - let none_slots = none_hint - .row_positions - .iter() - .filter(|&&pos| pos != ROW_EMPTY_SLOT) - .count(); - - // Dense (Some(false), dict-priming path) for comparison — that - // path inserts every position in the skipped range. - let mut dense = RowMatchGenerator::new(1 << 22); - dense.configure(ROW_CONFIG); - dense.add_data(data, |_| {}); - dense.skip_matching_with_hint(Some(false)); - let dense_slots = dense - .row_positions - .iter() - .filter(|&&pos| pos != ROW_EMPTY_SLOT) - .count(); - - // Two assertions pin the contract: - // 1) None hint is dramatically sparser than dense (the whole point). - // 2) None hint at block-start==0 inserts ZERO positions (no - // backfill possible before position 0). - assert_eq!( - none_slots, 0, - "None hint at block_start=0 must leave row table fully empty \ - (upstream zstd parity — interior NOT inserted, no pre-block backfill possible)", - ); - assert!( - dense_slots > 0, - "Some(false) dict-priming path must still insert densely \ - (sanity check: control case for the `none_slots == 0` assertion)", - ); -} - -#[test] -fn driver_unhinted_level2_keeps_default_dfast_hash_table_size() { - let mut driver = MatchGeneratorDriver::new(32, 2); - - driver.reset(CompressionLevel::Level(3)); - let mut space = driver.get_next_space(); - space[..12].copy_from_slice(b"abcabcabcabc"); - space.truncate(12); - driver.commit_space(space); - driver.skip_matching_with_hint(None); - - // Upstream zstd-parity split: long-hash at DFAST_HASH_BITS, short-hash one - // bit smaller (DFAST_SHORT_HASH_BITS_DELTA = 1, matching upstream zstd - // `chainLog = hashLog - 1` for dfast levels). - let long_len = driver.dfast_matcher().long_len(); - let short_len = driver.dfast_matcher().short_len(); - assert_eq!( - long_len, - 1 << DFAST_HASH_BITS, - "unhinted Level(2) should keep default long-hash table size" - ); - assert_eq!( - short_len, - 1 << (DFAST_HASH_BITS - DFAST_SHORT_HASH_BITS_DELTA), - "unhinted Level(2) short-hash should be one bit smaller than long-hash" - ); -} - -#[cfg(any())] // disabled: tested legacy MatchGenerator/SuffixStore behavior removed in phase 1b -#[test] -fn simple_backend_rejects_undersized_pooled_suffix_store() { - let mut driver = MatchGeneratorDriver::new(128 * 1024, 2); - driver.reset(CompressionLevel::Fastest); - - driver.suffix_pool.push(SuffixStore::with_capacity(1024)); - - let mut space = driver.get_next_space(); - space.clear(); - space.resize(4096, 0xAB); - driver.commit_space(space); - - let last_suffix_slots = driver - .simple() - .window - .last() - .expect("window entry must exist after commit") - .suffixes - .slots - .len(); - assert!( - last_suffix_slots >= 4096, - "undersized pooled suffix store must not be reused for larger blocks" - ); -} - -#[test] -fn source_hint_clamps_driver_slice_size_to_window() { - let mut driver = MatchGeneratorDriver::new(128 * 1024, 2); - driver.set_source_size_hint(1024); - driver.reset(CompressionLevel::Default); - - let window = driver.window_size() as usize; - assert_eq!(window, 1 << MIN_HINTED_WINDOW_LOG); - assert_eq!(driver.slice_size, window); - - let space = driver.get_next_space(); - assert_eq!(space.len(), window); - driver.commit_space(space); -} - -#[test] -fn pooled_space_keeps_capacity_when_slice_size_shrinks() { - let mut driver = MatchGeneratorDriver::new(128 * 1024, 2); - driver.reset(CompressionLevel::Default); - - let large = driver.get_next_space(); - let large_capacity = large.capacity(); - assert!(large_capacity >= 128 * 1024); - driver.commit_space(large); - - driver.set_source_size_hint(1024); - driver.reset(CompressionLevel::Default); - - let small = driver.get_next_space(); - assert_eq!(small.len(), 1 << MIN_HINTED_WINDOW_LOG); - assert!( - small.capacity() >= large_capacity, - "pooled buffer capacity should be preserved to avoid shrink/grow churn" - ); -} - -#[test] -fn driver_best_to_fastest_releases_oversized_hc_tables() { - let mut driver = MatchGeneratorDriver::new(32, 2); - - // Initialize at Best routed onto HashChain via the test-only override - // (production `Best` sits on level 13, whose native backend differs) — - // allocates large HC tables (4M hash, 2M chain) so the swap below - // exercises the HC drain path this test pins. - driver.reset_on_hc_lazy(CompressionLevel::Best); - assert_eq!(driver.window_size(), (1u64 << 22)); - - // Feed data so tables are actually allocated via ensure_tables(). - let mut space = driver.get_next_space(); - space[..12].copy_from_slice(b"abcabcabcabc"); - space.truncate(12); - driver.commit_space(space); - driver.skip_matching_with_hint(None); - - // Switch to Fastest — the [`MatcherStorage`] enum swaps to the - // `Simple` variant and the `HashChain` variant is dropped. The - // drain block in `Matcher::reset` reassigns - // `m.table.hash_table` / `chain_table` / `hash3_table` to - // `Vec::new()` BEFORE constructing the replacement variant so the - // table backing allocations are released up front — this caps - // peak memory during the swap to "old data buffers being drained - // into `vec_pool` + new `MatchGenerator` skeleton" rather than - // "old tables still resident + new variant under construction". - // The eventual `Drop` on the old variant would release the tables - // anyway, but only after the new variant is built, so the early - // reassign shifts the peak. Post-switch the HC variant no longer - // exists; the assertion that storage is now `Simple` covers the - // invariant the old hash_table/chain_table checks were proxying. - driver.reset(CompressionLevel::Fastest); - assert_eq!(driver.window_size(), (1u64 << 19)); - assert_eq!(driver.active_backend(), super::strategy::BackendTag::Simple); -} - -#[test] -fn driver_better_to_best_resizes_hc_tables() { - let mut driver = MatchGeneratorDriver::new(32, 2); - - // The lazy band runs on the Row backend now, so the HC resize path is - // exercised across two BT levels whose native `HcConfig` widths differ: - // L13 (hash_log 22, chain_log 22) -> L15 (hash_log 23, chain_log 23). - driver.reset(CompressionLevel::Level(13)); - assert_eq!(driver.window_size(), (1u64 << 22)); - - let mut space = driver.get_next_space(); - space[..12].copy_from_slice(b"abcabcabcabc"); - space.truncate(12); - driver.commit_space(space); - driver.skip_matching_with_hint(None); - - let hc = driver.hc_matcher(); - let better_hash_len = hc.table.hash_table.len(); - let better_chain_len = hc.table.chain_table.len(); - - // Switch to L15 — must resize to larger tables. - driver.reset(CompressionLevel::Level(15)); - assert_eq!(driver.window_size(), (1u64 << 22)); - - // Feed data to trigger ensure_tables with new sizes. - let mut space = driver.get_next_space(); - space[..12].copy_from_slice(b"xyzxyzxyzxyz"); - space.truncate(12); - driver.commit_space(space); - driver.skip_matching_with_hint(None); - - let hc = driver.hc_matcher(); - assert!( - hc.table.hash_table.len() > better_hash_len, - "L15 hash_table ({}) should be larger than L13 ({})", - hc.table.hash_table.len(), - better_hash_len - ); - assert!( - hc.table.chain_table.len() > better_chain_len, - "L15 chain_table ({}) should be larger than L13 ({})", - hc.table.chain_table.len(), - better_chain_len - ); -} - -#[cfg(any())] -// disabled: tests legacy SuffixStore behavior incompatible with upstream zstd-shape kernel's HASH_READ_SIZE geometry -#[test] -fn prime_with_dictionary_preserves_history_for_first_full_block() { - let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset(CompressionLevel::Fastest); - - driver.prime_with_dictionary(b"abcdefgh", [1, 4, 8]); - - let mut space = driver.get_next_space(); - space.clear(); - space.extend_from_slice(b"abcdefgh"); - driver.commit_space(space); - - let mut saw_match = false; - driver.start_matching(|seq| { - if let Sequence::Triple { - literals, - offset, - match_len, - } = seq - && literals.is_empty() - && offset == 8 - && match_len >= MIN_MATCH_LEN - { - saw_match = true; - } - }); - - assert!( - saw_match, - "first full block should still match dictionary-primed history" - ); -} - -#[cfg(any())] -// disabled: tests legacy SuffixStore behavior incompatible with upstream zstd-shape kernel's HASH_READ_SIZE geometry -#[test] -fn prime_with_large_dictionary_preserves_early_history_until_first_block() { - let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset(CompressionLevel::Fastest); - - driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); - - let mut space = driver.get_next_space(); - space.clear(); - space.extend_from_slice(b"abcdefgh"); - driver.commit_space(space); - - let mut saw_match = false; - driver.start_matching(|seq| { - if let Sequence::Triple { - literals, - offset, - match_len, - } = seq - && literals.is_empty() - && offset == 24 - && match_len >= MIN_MATCH_LEN - { - saw_match = true; - } - }); - - assert!( - saw_match, - "dictionary bytes should remain addressable until frame output exceeds the live window" - ); -} - -#[test] -fn prime_with_dictionary_applies_offset_history_even_when_content_is_empty() { - let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset(CompressionLevel::Fastest); - - driver.prime_with_dictionary(&[], [11, 7, 3]); - - assert_eq!(driver.simple_mut().offset_hist, [11, 7, 3]); -} - -#[test] -fn hc_prime_with_empty_dictionary_disables_btultra2_seed_pass() { - let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset_on_hc_lazy(CompressionLevel::Better); - - driver.prime_with_dictionary(&[], [11, 7, 3]); - - assert_eq!(driver.hc_matcher().table.offset_hist, [11, 7, 3]); - assert!( - !driver - .hc_matcher() - .should_run_btultra2_seed_pass::(HC_PREDEF_THRESHOLD + 1), - "btultra2 warmup must stay disabled after dictionary priming, even when dict content is empty" - ); -} - -#[test] -fn primed_snapshot_not_restored_across_ldm_config_change() { - // The CDict-equivalent primed snapshot clones `storage`, which on the - // BT backend carries `BtMatcher::ldm_producer`. A snapshot captured - // under one LDM configuration must NOT be restored into a reset that - // resolved a different LDM configuration (else the restored producer - // is stale). `PrimedKey` must fold the LDM override into the key so - // such a restore is refused and the caller re-primes. - use super::parameters::CompressionParameters; - - let dict = b"abcdefghabcdefghabcdefgh"; - let ldm_on = CompressionParameters::builder(CompressionLevel::Level(19)) - .enable_long_distance_matching(true) - .build() - .unwrap() - .overrides(); - let ldm_off = CompressionParameters::builder(CompressionLevel::Level(19)) - .build() - .unwrap() - .overrides(); - - let mut driver = MatchGeneratorDriver::new(1024, 1); - - // Capture a snapshot primed under LDM-on at level 19. - driver.set_param_overrides(Some(ldm_on)); - driver.reset(CompressionLevel::Level(19)); - driver.prime_with_dictionary(dict, [1, 4, 8]); - driver.capture_primed_dictionary(CompressionLevel::Level(19)); - - // Same dictionary + level, but LDM now OFF: the snapshot's LDM state - // is stale, so restore must be refused. - driver.set_param_overrides(Some(ldm_off)); - driver.reset(CompressionLevel::Level(19)); - assert!( - !driver.restore_primed_dictionary(CompressionLevel::Level(19)), - "primed snapshot restored across an LDM config change (stale producer)", - ); - - // Sanity: re-priming + capturing under LDM-off, then restoring under - // the IDENTICAL LDM-off config DOES match (the key is not over-tight). - driver.prime_with_dictionary(dict, [1, 4, 8]); - driver.capture_primed_dictionary(CompressionLevel::Level(19)); - driver.reset(CompressionLevel::Level(19)); - assert!( - driver.restore_primed_dictionary(CompressionLevel::Level(19)), - "primed snapshot not restored under identical LDM config", - ); -} - -#[test] -fn hc_prime_with_dictionary_disables_btultra2_seed_pass() { - let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset_on_hc_lazy(CompressionLevel::Better); - - driver.prime_with_dictionary(b"abcdefgh", [1, 4, 8]); - - assert!( - !driver - .hc_matcher() - .should_run_btultra2_seed_pass::(HC_PREDEF_THRESHOLD + 1), - "btultra2 warmup must stay disabled after dictionary priming with content" - ); -} - -#[test] -fn dfast_prime_with_dictionary_preserves_history_for_first_full_block() { - let mut driver = MatchGeneratorDriver::new(8, 1); - // Level(4) is Dfast with the greedy double-fast loop (upstream zstd parity: - // clevels.h L3/L4 are both `ZSTD_dfast`, which has no lazy lookahead). - // The fast loop needs at least `HASH_READ_SIZE` (8) bytes ahead of the - // probe cursor, so this exercises a 16-byte dict + 16-byte block (the - // whole block matches the dict, offset = dict length = 16). - driver.reset(CompressionLevel::Level(4)); - - let payload = b"abcdefghijklmnop"; - driver.prime_with_dictionary(payload, [1, 4, 8]); - - let mut space = driver.get_next_space(); - space.clear(); - space.extend_from_slice(payload); - driver.commit_space(space); - - let mut saw_match = false; - driver.start_matching(|seq| { - if let Sequence::Triple { - literals, - offset, - match_len, - } = seq - && literals.is_empty() - && offset == payload.len() - && match_len >= DFAST_MIN_MATCH_LEN - { - saw_match = true; - } - }); - - assert!( - saw_match, - "dfast backend should match dictionary-primed history in first full block" - ); -} - -#[test] -fn prime_with_dictionary_does_not_inflate_reported_window_size() { - let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset(CompressionLevel::Fastest); - - let before = driver.window_size(); - driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); - let after = driver.window_size(); - - assert_eq!( - after, before, - "dictionary retention budget must not change reported frame window size" - ); -} - -#[test] -fn primed_snapshot_not_restored_when_window_hint_differs() { - // The copy-snapshot must be keyed on the resolved reset parameters, not - // just the CompressionLevel. `reset()` caps window_log by the source-size - // hint, so two same-level frames with different hints resolve to different - // windows. Restoring a snapshot captured at the larger hint into a reset - // for the smaller hint would advertise the smaller window in the frame - // header while the matcher's `max_window_size` (from the restored storage) - // still spans the larger window — the encoder could then emit a match - // (e.g. into the dictionary) past the advertised window, producing an - // undecodable frame. Restore must REFUSE when the resolved window differs. - let mut driver = MatchGeneratorDriver::new(8, 1); - let level = CompressionLevel::Best; - - // Frame A: large hint → larger resolved window. Prime + capture. - driver.set_source_size_hint(256 * 1024); - driver.reset(level); - let big_window = driver.window_size(); - driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); - driver.capture_primed_dictionary(level); - - // Frame B: smaller hint, SAME level → smaller resolved window. - driver.set_source_size_hint(48 * 1024); - driver.reset(level); - let small_window = driver.window_size(); - assert!( - small_window < big_window, - "precondition: the two hints must resolve to different windows \ - (small={small_window}, big={big_window})" - ); - - let restored = driver.restore_primed_dictionary(level); - assert!( - !restored, - "snapshot captured at window {big_window} must NOT be restored into a \ - reset advertising window {small_window} (level alone is an insufficient key)" - ); -} - -#[test] -fn primed_snapshot_restored_for_hints_in_same_window_bucket() { - // The snapshot key must normalize the source-size hint to the resolved - // matcher geometry, not the raw hinted byte count. `reset()` derives every - // hint-dependent parameter (window_log cap, HC/Fast/Dfast/Row table widths, - // the Fast attach-vs-copy cutoff) from `ceil_log2(hint)`, so two distinct - // hints that share a ceil-log bucket resolve to the *identical* matcher - // shape. Keying on the raw bytes over-keys: it forces a full re-prime on the - // second frame even though the cached snapshot is a perfect fit. Restore - // must SUCCEED across same-bucket hints. - let mut driver = MatchGeneratorDriver::new(8, 1); - let level = CompressionLevel::Best; - - // Both hints fall in ceil_log2 bucket 19 (2^18 < n <= 2^19): 300 KiB and - // 400 KiB resolve to the same window and table widths. - driver.set_source_size_hint(300 * 1024); - driver.reset(level); - let window_a = driver.window_size(); - driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); - driver.capture_primed_dictionary(level); - - driver.set_source_size_hint(400 * 1024); - driver.reset(level); - let window_b = driver.window_size(); - assert_eq!( - window_a, window_b, - "precondition: same-bucket hints must resolve to the same window \ - (a={window_a}, b={window_b})" - ); - - let restored = driver.restore_primed_dictionary(level); - assert!( - restored, - "snapshot captured at a 300 KiB hint must be restored into a 400 KiB \ - hint that resolves to the identical matcher shape (raw bytes over-key)" - ); -} - -#[test] -fn primed_snapshot_restored_across_level22_tier_hints() { - // Level 22 collapses several ceil-log buckets onto one upstream zstd source-size - // tier: `resolve_level_params(Level(22), ..)` selects the HC config and - // window_log by raw `<= 16 KiB / 128 KiB / 256 KiB` thresholds, so a 20 KiB - // and a 100 KiB hint (ceil-log buckets 15 and 17) both land in the - // `<= 128 KiB` tier and resolve to the IDENTICAL matcher (same window_log, - // same HC hash/chain/search geometry). Keying on the raw ceil-log bucket - // would still reject the restore here because the buckets differ; the key - // must compare the resolved matcher shape so these share one snapshot. - let mut driver = MatchGeneratorDriver::new(8, 1); - let level = CompressionLevel::Level(22); - - driver.set_source_size_hint(20 * 1024); - driver.reset(level); - let window_a = driver.window_size(); - driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); - driver.capture_primed_dictionary(level); - - driver.set_source_size_hint(100 * 1024); - driver.reset(level); - let window_b = driver.window_size(); - assert_eq!( - window_a, window_b, - "precondition: both hints must land in the same Level 22 upstream zstd tier \ - (a={window_a}, b={window_b})" - ); - - let restored = driver.restore_primed_dictionary(level); - assert!( - restored, - "Level 22 snapshot captured at a 20 KiB hint must be restored into a \ - 100 KiB hint that resolves to the same upstream zstd tier (different ceil-log \ - buckets, identical matcher shape)" - ); -} - -#[test] -fn fast_dict_attaches_within_cutoff_bounds() { - // Within the attach bounds, every Fast dict frame attaches (the copy-mode - // owned path memmoved the whole input into history each frame; attach scans - // the input in place via the borrowed dual-base kernel). All hints here sit - // far below `FAST_ATTACH_DICT_CUTOFF_LOG` (2 GiB source) and the dict is far - // below `MAX_FAST_ATTACH_DICT_REGION` (16 MiB), so a hint that used to cross - // the old 8 KiB cutoff (8193 B) and a small one (8192 B) BOTH resolve to - // attach, and the Simple backend reports a borrowed (in-place) dict scan for - // both. This guards `FAST_ATTACH_DICT_CUTOFF_LOG` staying high enough that no - // in-bounds Fast hint falls back to the input-copy path; the OUT-of-bounds - // fallbacks are covered by `fast_attach_cutoff_keeps_virtual_positions_within_u32` - // (source) and `oversized_dict_hint_routes_fast_to_copy_mode` (dict size). - let level = CompressionLevel::Level(1); - for hint in [8192u64, 8193, 1 << 20] { - let mut driver = MatchGeneratorDriver::new(8, 1); - driver.set_source_size_hint(hint); - driver.reset(level); - driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); - assert!( - driver.borrowed_dict_supported(), - "Fast dict frame with hint {hint} must attach (borrowed in-place \ - dict scan), never fall back to the copy-mode input-copy path" - ); - } -} - -#[test] -fn fast_attach_cutoff_keeps_virtual_positions_within_u32() { - // The cutoff is 31, NOT the full u64 source-size range, because the borrowed - // dict kernel stores virtual positions as u32 (`cur_abs as u32`). The largest - // attached source `1 << CUTOFF` (plus the dict prefix) must stay below - // u32::MAX or that arithmetic wraps; the next bucket (4 GiB) would. This pins - // the bound so a future "just raise it to attach everything" change cannot - // silently reintroduce the overflow — raising the cutoff requires widening - // the kernel's position type first. - let max_attached: u64 = 1u64 << FAST_ATTACH_DICT_CUTOFF_LOG; - assert!( - max_attached <= u32::MAX as u64, - "the largest attached source 2^{FAST_ATTACH_DICT_CUTOFF_LOG} must fit u32 \ - virtual positions", - ); - assert!( - (1u64 << (FAST_ATTACH_DICT_CUTOFF_LOG + 1)) > u32::MAX as u64, - "the next bucket 2^{} would overflow u32 virtual positions", - FAST_ATTACH_DICT_CUTOFF_LOG + 1, - ); -} - -#[test] -fn oversized_dict_hint_routes_fast_to_copy_mode() { - // A dict whose region exceeds the tagged attach position field - // (`MAX_FAST_ATTACH_DICT_REGION`, 16 MiB) must route the Fast prime to COPY - // mode instead of the tagged attach fill, which would overflow the packed - // position. The decision is keyed on the load-set size hint, so a hint past - // the limit suffices to exercise it without allocating a real 16 MiB dict. - // Copy mode leaves the borrowed in-place dict scan (attach-only) unavailable. - let mut driver = MatchGeneratorDriver::new(8, 1); - driver.set_dictionary_size_hint(MAX_FAST_ATTACH_DICT_REGION + 1); - driver.reset(CompressionLevel::Level(1)); - driver.prime_with_dictionary(b"small dict content with some padding here", [1, 4, 8]); - assert!( - !driver.borrowed_dict_supported(), - "an oversized dict must use copy mode, not the tagged attach fill" - ); -} - -#[test] -fn block_samples_match_dict_is_true_for_non_simple_backend() { - // Production fallback: a non-Simple backend (here Row, Level 6) has no dict - // probe, so the driver wrapper answers CONSERVATIVELY `true` for ANY block — - // keeping the dict frame on the scan rather than letting the raw-fast-path - // emit a block raw and miss an embedded dict segment (see - // `dictionary_segment_in_incompressible_input_is_matched`). Only the - // Simple/Fast backend trades the blanket scan for a precise probe. - let dict = b"the quick brown fox jumps over the lazy dog 0123456789abcdef"; - let mut row = MatchGeneratorDriver::new(8, 6); - row.set_dictionary_size_hint(dict.len()); - row.reset(CompressionLevel::Level(6)); - row.prime_with_dictionary(dict, [1, 4, 8]); - assert!( - row.block_samples_match_dict(&dict[..32]), - "non-Simple backend must stay on the scan (true) for a dict frame" - ); - let random: alloc::vec::Vec = (0..64u8) - .map(|i| i.wrapping_mul(37).wrapping_add(13)) - .collect(); - assert!( - row.block_samples_match_dict(&random), - "non-Simple backend reports true regardless of block content" - ); -} - -#[test] -fn primed_snapshot_fast_attach_does_not_over_key_non_simple_backends() { - // `fast_attach` is a Simple/Fast-backend concept (the 8 KiB attach-vs-copy - // table split). Dfast/Row/HashChain each have their OWN attach/copy regime - // (`DFAST_ATTACH_DICT_CUTOFF_LOG`, `ROW_ATTACH_DICT_CUTOFF_LOG`, - // `HC_ATTACH_DICT_CUTOFF_LOG`) but those are deliberately kept OUT of the - // `fast_attach` key, which only models the Fast table split. Their snapshots - // are keyed by the resolved matcher geometry instead, and the HC modes share - // one window geometry so an HC cross-mode restore stays decodable (see - // `prime_with_dictionary`). Either way the `fast_attach` - // bit must NOT enter a non-Simple snapshot key — otherwise an unhinted - // capture (which would record `fast_attach = true`) and a hinted reset that - // resolves to the IDENTICAL `LevelParams` would key differently and force a - // needless re-prime. `Best` is a Row-backend lazy - // level; this also pins the Row arm recording its RESOLVED hash width on - // the unhinted path (a 0 default there keyed unhinted-vs-hinted apart). - // An explicit Row-backend level: `Best` now sits on level 13 (Btlazy2), - // so the named alias no longer reaches the Row arm this test pins. - let mut driver = MatchGeneratorDriver::new(8, 1); - let level = CompressionLevel::Level(12); - - // Capture with no hint. - driver.reset(level); - let window_a = driver.window_size(); - driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); - driver.capture_primed_dictionary(level); - - // Reset with a hint large enough to resolve to the same window/params as - // the unhinted level (>= 2^window_log, so the source-size cap is a no-op). - driver.set_source_size_hint(64 * 1024 * 1024); - driver.reset(level); - let window_b = driver.window_size(); - assert_eq!( - window_a, window_b, - "precondition: the large hint must resolve to the same window as the \ - unhinted level (a={window_a}, b={window_b})" - ); - - let restored = driver.restore_primed_dictionary(level); - assert!( - restored, - "a Row snapshot must restore across an unhinted vs large-hinted \ - reset that resolves to the identical matcher — `fast_attach` is a Fast \ - backend concept and must not over-key non-Simple shapes" - ); -} - -#[cfg(any())] // disabled: tested SuffixStore-per-block tail-handling specific to legacy MatchGenerator -#[test] -fn prime_with_dictionary_does_not_reuse_tiny_suffix_store() { - let mut driver = MatchGeneratorDriver::new(8, 2); - driver.reset(CompressionLevel::Fastest); - - // This dictionary leaves a 1-byte tail chunk (capacity=1 suffix table), - // which should never be committed to the matcher window. - driver.prime_with_dictionary(b"abcdefghi", [1, 4, 8]); - - assert!( - driver - .simple() - .window - .iter() - .all(|entry| entry.data.len() >= MIN_MATCH_LEN), - "dictionary priming must not commit tails shorter than MIN_MATCH_LEN" - ); -} - -#[test] -fn prime_with_dictionary_counts_only_committed_tail_budget() { - let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset(CompressionLevel::Fastest); - - let before = driver.simple_mut().max_window_size; - // One full slice plus a 1-byte tail that cannot be committed. - driver.prime_with_dictionary(b"abcdefghi", [1, 4, 8]); - - assert_eq!( - driver.simple_mut().max_window_size, - before + 8, - "retention budget must account only for dictionary bytes actually committed to history" - ); -} - -#[test] -fn dfast_prime_with_dictionary_counts_four_byte_tail_budget() { - let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset(CompressionLevel::Level(3)); - - let before = driver.dfast_matcher().max_window_size; - // One full slice plus a 4-byte tail. Dfast can still use this tail through - // short-hash overlap into the next block, so it should stay retained. - driver.prime_with_dictionary(b"abcdefghijkl", [1, 4, 8]); - - assert_eq!( - driver.dfast_matcher().max_window_size, - before + 12, - "dfast retention budget should include 4-byte dictionary tails" - ); -} - -#[test] -fn row_prime_with_dictionary_preserves_history_for_first_full_block() { - let mut driver = MatchGeneratorDriver::new(8, 1); - // Level(5) is the greedy Row backend (LEVEL_TABLE row 5: Greedy / RowHash). - // Level(4) now routes to Dfast, so this test must use Level(5) to actually - // exercise `RowMatchGenerator`'s dictionary priming. The 16-byte dict + - // 16-byte block lets the whole block match the primed dict (offset = dict - // length = 16). - driver.reset(CompressionLevel::Level(5)); - - let payload = b"abcdefghijklmnop"; - driver.prime_with_dictionary(payload, [1, 4, 8]); - - let mut space = driver.get_next_space(); - space.clear(); - space.extend_from_slice(payload); - driver.commit_space(space); - - let mut saw_match = false; - driver.start_matching(|seq| { - if let Sequence::Triple { - literals, - offset, - match_len, - } = seq - && literals.is_empty() - && offset == payload.len() - && match_len >= ROW_MIN_MATCH_LEN - { - saw_match = true; - } - }); - - assert!( - saw_match, - "row backend should match dictionary-primed history in first full block" - ); -} - -#[test] -fn row_prime_with_dictionary_subtracts_uncommitted_tail_budget() { - let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset(CompressionLevel::Level(5)); - - let base_window = driver.row_matcher().max_window_size; - // Slice size is 8. The trailing byte cannot be committed (<4 tail), - // so it must be subtracted from retained budget. - driver.prime_with_dictionary(b"abcdefghi", [1, 4, 8]); - - assert_eq!( - driver.row_matcher().max_window_size, - base_window + 8, - "row retained window must exclude uncommitted 1-byte tail" - ); -} - -#[test] -fn prime_with_dictionary_budget_shrinks_after_row_eviction() { - let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset(CompressionLevel::Level(5)); - // Keep live window tiny so dictionary-primed slices are evicted quickly. - driver.row_matcher_mut().max_window_size = 8; - driver.reported_window_size = 8; - - let base_window = driver.row_matcher().max_window_size; - driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); - assert_eq!(driver.row_matcher().max_window_size, base_window + 24); - - for block in [b"AAAAAAAA", b"BBBBBBBB"] { - let mut space = driver.get_next_space(); - space.clear(); - space.extend_from_slice(block); - driver.commit_space(space); - driver.skip_matching_with_hint(None); - } - - assert_eq!( - driver.dictionary_retained_budget, 0, - "dictionary budget should be fully retired once primed dict slices are evicted" - ); - assert_eq!( - driver.row_matcher().max_window_size, - base_window, - "retired dictionary budget must not remain reusable for live history" - ); -} - -/// Row → Simple transition drops the Row variant and the -/// post-switch active backend is exactly Simple. The window-emptied -/// check from the pre-enum era (`driver.row_matcher().window.is_empty()`) -/// is intentionally gone — the `Row` variant no longer exists after -/// the swap, so there is nothing to inspect by accessor; the "window -/// cleared" invariant is replaced by "variant dropped", and a -/// subsequent `row_matcher()` call would panic by design. The -/// pool-recycling side of the row backend is covered by -/// [`driver_row_commit_recycles_block_buffer_into_pool`]. -#[test] -fn row_get_last_space_then_reset_to_fastest_drops_row_variant() { - let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset(CompressionLevel::Level(5)); - assert_eq!(driver.active_backend(), super::strategy::BackendTag::Row); - - let mut space = driver.get_next_space(); - space.clear(); - space.extend_from_slice(b"row-data"); - driver.commit_space(space); - - assert_eq!(driver.get_last_space(), b"row-data"); - - driver.reset(CompressionLevel::Fastest); - assert_eq!(driver.active_backend(), super::strategy::BackendTag::Simple); -} - -/// Committing a Row block must return the input buffer to `vec_pool` -/// immediately (the bytes are mirrored into the contiguous `history`, -/// so there is no reason to retain a second copy in the window). This -/// guards the chunk-length window: the previous `VecDeque>` -/// window retained a full `block_capacity` buffer per committed block, -/// which on a heavily pre-split frame ballooned peak memory to many -/// times the live byte count. With the buffer recycled at commit time -/// the pool grows by exactly one Vec per committed block. -#[test] -fn driver_row_commit_recycles_block_buffer_into_pool() { - let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset(CompressionLevel::Level(5)); - assert_eq!(driver.active_backend(), super::strategy::BackendTag::Row); - - let before_pool = driver.vec_pool.len(); - let mut space = driver.get_next_space(); - space.clear(); - space.extend_from_slice(b"row-data-to-recycle"); - driver.commit_space(space); - - // `>` not `>=`: a fresh driver starts with `before_pool == 0`, so the - // weaker bound passes even if the commit failed to recycle. Strict - // growth proves the buffer was returned to the pool at commit time - // rather than retained in the window (the pre-`chunk_lens` bug). - assert!( - driver.vec_pool.len() > before_pool, - "row commit must recycle the committed block buffer into vec_pool \ - (before_pool = {before_pool}, after = {})", - driver.vec_pool.len() - ); - // The bytes still resolve through the contiguous history mirror. - assert_eq!(driver.get_last_space(), b"row-data-to-recycle"); -} - -#[test] -fn adjust_params_for_zero_source_size_uses_min_hinted_window_floor() { - let mut params = resolve_level_params(CompressionLevel::Level(4), None); - params.window_log = 22; - let adjusted = adjust_params_for_source_size(params, 0); - assert_eq!(adjusted.window_log, MIN_HINTED_WINDOW_LOG); -} - -#[test] -fn common_prefix_len_matches_scalar_reference_across_offsets() { - fn scalar_reference(a: &[u8], b: &[u8]) -> usize { - a.iter() - .zip(b.iter()) - .take_while(|(lhs, rhs)| lhs == rhs) - .count() - } - - for total_len in [ - 0usize, 1, 5, 15, 16, 17, 31, 32, 33, 64, 65, 127, 191, 257, 320, - ] { - let base: Vec = (0..total_len) - .map(|i| ((i * 13 + 7) & 0xFF) as u8) - .collect(); - - for start in [0usize, 1, 3] { - if start > total_len { - continue; - } - let a = &base[start..]; - let b = a.to_vec(); - assert_eq!( - common_prefix_len(a, &b), - scalar_reference(a, &b), - "equal slices total_len={total_len} start={start}" - ); - - let len = a.len(); - for mismatch in [0usize, 1, 7, 15, 16, 31, 32, 47, 63, 95, 127, 128, 129, 191] { - if mismatch >= len { - continue; - } - let mut altered = b.clone(); - altered[mismatch] ^= 0x5A; - assert_eq!( - common_prefix_len(a, &altered), - scalar_reference(a, &altered), - "total_len={total_len} start={start} mismatch={mismatch}" - ); - } - - if len > 0 { - let mismatch = len - 1; - let mut altered = b.clone(); - altered[mismatch] ^= 0xA5; - assert_eq!( - common_prefix_len(a, &altered), - scalar_reference(a, &altered), - "tail mismatch total_len={total_len} start={start} mismatch={mismatch}" - ); - } - } - } - - let long = alloc::vec![0xAB; 320]; - let shorter = alloc::vec![0xAB; 137]; - assert_eq!( - common_prefix_len(&long, &shorter), - scalar_reference(&long, &shorter) - ); -} - -#[test] -fn row_pick_lazy_returns_none_when_next_is_better() { - let mut matcher = RowMatchGenerator::new(1 << 22); - matcher.configure(ROW_CONFIG); - matcher.add_data(alloc::vec![b'a'; 64], |_| {}); - matcher.ensure_tables(); - - let abs_pos = matcher.history_abs_start + 16; - let best = MatchCandidate { - start: abs_pos, - offset: 8, - match_len: ROW_MIN_MATCH_LEN, - }; - assert!( - matcher.pick_lazy_match(abs_pos, 0, Some(best)).is_none(), - "lazy picker should defer when next position is clearly better" - ); -} - -#[test] -fn row_pick_lazy_depth2_returns_none_when_next2_significantly_better() { - let mut matcher = RowMatchGenerator::new(1 << 22); - matcher.configure(ROW_CONFIG); - matcher.lazy_depth = 2; - matcher.search_depth = 0; - matcher.offset_hist = [6, 9, 1]; - - let mut data = alloc::vec![b'x'; 40]; - data[11..30].copy_from_slice(b"EFABCABCAEFABCAEFAB"); - matcher.add_data(data, |_| {}); - matcher.ensure_tables(); - - let abs_pos = matcher.history_abs_start + 20; - let best = matcher - .best_match(abs_pos, 0) - .expect("expected baseline repcode match"); - assert_eq!(best.offset, 9); - // Baseline match length is fixed by the fixture data (the offset-9 - // rep run is 6 bytes long), independent of the accept threshold. - assert_eq!(best.match_len, 6); - - if let Some(next) = matcher.best_match(abs_pos + 1, 1) { - assert!(next.match_len <= best.match_len); - } - - let next2 = matcher - .best_match(abs_pos + 2, 2) - .expect("expected +2 candidate"); - assert!( - next2.match_len > best.match_len + 1, - "+2 candidate must be significantly better for depth-2 lazy skip" - ); - assert!( - matcher.pick_lazy_match(abs_pos, 0, Some(best)).is_none(), - "lazy picker should defer when +2 candidate is significantly better" - ); -} - -#[test] -fn row_pick_lazy_depth2_keeps_best_when_next2_is_only_one_byte_better() { - let mut matcher = RowMatchGenerator::new(1 << 22); - matcher.configure(ROW_CONFIG); - matcher.lazy_depth = 2; - matcher.search_depth = 0; - matcher.offset_hist = [6, 9, 1]; - - let mut data = alloc::vec![b'x'; 40]; - data[11..30].copy_from_slice(b"EFABCABCAEFABCAEFAZ"); - matcher.add_data(data, |_| {}); - matcher.ensure_tables(); - - let abs_pos = matcher.history_abs_start + 20; - let best = matcher - .best_match(abs_pos, 0) - .expect("expected baseline repcode match"); - assert_eq!(best.offset, 9); - // Baseline match length is fixed by the fixture data (the offset-9 - // rep run is 6 bytes long), independent of the accept threshold. - assert_eq!(best.match_len, 6); - - let next2 = matcher - .best_match(abs_pos + 2, 2) - .expect("expected +2 candidate"); - assert_eq!(next2.match_len, best.match_len + 1); - let chosen = matcher - .pick_lazy_match(abs_pos, 0, Some(best)) - .expect("lazy picker should keep current best"); - assert_eq!(chosen.start, best.start); - assert_eq!(chosen.offset, best.offset); - assert_eq!(chosen.match_len, best.match_len); -} - -/// Verifies row/tag extraction uses the shared hash mix bit-splitting contract. -#[test] -fn row_hash_and_row_extracts_high_bits() { - let mut matcher = RowMatchGenerator::new(1 << 22); - matcher.configure(ROW_CONFIG); - matcher.add_data( - alloc::vec![ - 0xAA, 0xBB, 0xCC, 0x11, 0x10, 0x20, 0x30, 0x40, 0xAA, 0xBB, 0xCC, 0x22, 0x50, 0x60, - 0x70, 0x80, - ], - |_| {}, - ); - matcher.ensure_tables(); - - let pos = matcher.history_abs_start + 8; - let (row, tag) = matcher - .hash_and_row(pos) - .expect("row hash should be available"); - - let idx = pos - matcher.history_abs_start; - let concat = matcher.live_history(); - // Mirror `row_key_value`: an mls-wide masked key when 8 lookahead bytes - // exist, the 4-byte key in the tail. `idx = 8` on a 16-byte history has - // exactly 8 bytes left, so the wide arm applies here. - let key_len = matcher.mls.min(6); - let value = u64::from_le_bytes(concat[idx..idx + 8].try_into().unwrap()) - & ((1u64 << (key_len * 8)) - 1); - let hash = crate::encoding::fastpath::hash_mix_u64_with_kernel(matcher.hash_kernel, value); - let total_bits = matcher.row_hash_log + ROW_TAG_BITS; - let combined = hash >> (u64::BITS as usize - total_bits); - let expected_row = - ((combined >> ROW_TAG_BITS) as usize) & ((1usize << matcher.row_hash_log) - 1); - let expected_tag = combined as u8; - - assert_eq!(row, expected_row); - assert_eq!(tag, expected_tag); -} - -#[test] -fn row_repcode_skips_candidate_before_history_start() { - let mut matcher = RowMatchGenerator::new(1 << 22); - matcher.configure(ROW_CONFIG); - matcher.history = alloc::vec![b'a'; 20]; - matcher.history_start = 0; - matcher.history_abs_start = 10; - matcher.offset_hist = [3, 0, 0]; - - assert!(matcher.repcode_candidate(12, 1).is_none()); -} - -#[test] -fn row_repcode_returns_none_when_position_too_close_to_history_end() { - let mut matcher = RowMatchGenerator::new(1 << 22); - matcher.configure(ROW_CONFIG); - matcher.history = b"abcde".to_vec(); - matcher.history_start = 0; - matcher.history_abs_start = 0; - matcher.offset_hist = [1, 0, 0]; - - assert!(matcher.repcode_candidate(4, 1).is_none()); -} - -#[cfg(all(feature = "std", target_arch = "x86_64"))] -#[test] -fn hash_mix_sse42_path_is_available_and_matches_accelerated_impl_when_supported() { - use crate::encoding::fastpath::{self, FastpathKernel}; - if !is_x86_feature_detected!("sse4.2") { - return; - } - let v = 0x0123_4567_89AB_CDEFu64; - // SAFETY: feature check above guarantees SSE4.2 is available. - let accelerated = unsafe { fastpath::sse42::hash_mix_u64(v) }; - // Dispatcher must resolve to SSE4.2 (or better) and produce the same mix. - let dispatched = fastpath::dispatch_hash_mix_u64(v); - let kernel = fastpath::select_kernel(); - if kernel == FastpathKernel::Sse42 { - assert_eq!(dispatched, accelerated); - } else { - // AVX2 kernel uses the same CRC32 instruction under the hood. - assert_eq!(dispatched, accelerated, "AVX2/SSE4.2 share CRC32 mix"); - } -} - -#[cfg(all(feature = "std", target_arch = "aarch64", target_endian = "little"))] -#[test] -fn hash_mix_crc_path_is_available_and_matches_accelerated_impl_when_supported() { - use crate::encoding::fastpath; - if !is_aarch64_feature_detected!("crc") { - return; - } - let v = 0x0123_4567_89AB_CDEFu64; - // SAFETY: feature check above guarantees CRC32 is available. - let accelerated = unsafe { fastpath::neon::hash_mix_u64(v) }; - let dispatched = fastpath::dispatch_hash_mix_u64(v); - assert_eq!(dispatched, accelerated); -} - -#[test] -fn hc_hash3_position_matches_hash3_formula() { - let bytes = [b'a', b'b', b'c', b'd']; - let read32 = u32::from_le_bytes(bytes); - let expected = (((read32 << 8).wrapping_mul(HC_PRIME3BYTES)) >> (32 - HC3_HASH_LOG)) as usize; - assert_eq!( - super::match_table::storage::MatchTable::hash3_position(&bytes, HC3_HASH_LOG), - expected - ); -} - -#[test] -fn hc_hash_position_matches_hash4_formula() { - let mut hc = HcMatchGenerator::new(1 << 20); - hc.configure(HC_CONFIG, super::strategy::StrategyTag::Lazy, 22); - let bytes = [b'a', b'b', b'c', b'd']; - let read32 = u32::from_le_bytes(bytes); - let expected = ((read32.wrapping_mul(HC_PRIME4BYTES)) >> (32 - hc.table.hash_log)) as usize; - assert_eq!(hc.table.hash_position(&bytes), expected); -} - -#[test] -fn btultra2_main_hash_uses_hash4_formula() { - let mut hc = HcMatchGenerator::new(1 << 20); - hc.configure( - BTULTRA2_HC_CONFIG_L22, - super::strategy::StrategyTag::BtUltra2, - 27, - ); - let bytes = [b'a', b'b', b'c', b'd', b'e', b'f', b'g', b'h']; - let read32 = u32::from_le_bytes(bytes[..4].try_into().unwrap()); - let expected = ((read32.wrapping_mul(HC_PRIME4BYTES)) >> (32 - hc.table.hash_log)) as usize; - let actual = super::match_table::storage::MatchTable::hash_position_with_mls( - &bytes, - hc.table.hash_log, - super::bt::BtMatcher::HASH_MLS, - ); - assert_eq!(actual, expected); -} - -#[test] -fn row_candidate_returns_none_when_abs_pos_near_end_of_history() { - let mut matcher = RowMatchGenerator::new(1 << 22); - matcher.configure(ROW_CONFIG); - // One byte short of the accept floor: from abs_pos 0 there are fewer - // than `ROW_MIN_MATCH_LEN` bytes left, so the length gate in - // `row_candidate` must short-circuit to `None` before touching the - // (here unbuilt) row tables. - matcher.history = alloc::vec![b'a'; ROW_MIN_MATCH_LEN - 1]; - matcher.history_start = 0; - matcher.history_abs_start = 0; - - assert!(matcher.row_candidate(0, 0).is_none()); -} - -#[test] -fn hc_chain_candidates_returns_sentinels_for_short_suffix() { - let mut hc = HcMatchGenerator::new(32); - hc.table.history = b"abc".to_vec(); - hc.table.history_start = 0; - hc.table.history_abs_start = 0; - hc.table.ensure_tables(); - - let candidates = hc.hc.chain_candidates(&hc.table, 0); - assert!(candidates.iter().all(|&pos| pos == usize::MAX)); -} - -#[test] -fn hc_reset_advances_floor_past_prior_frame_entries() { - use super::match_table::storage::MatchTable; - let mut hc = HcMatchGenerator::new(32); - hc.table.add_data(b"abcdeabcde".to_vec(), |_| {}); - hc.table.ensure_tables(); - // Populate real hash / chain entries for the first frame's positions. - hc.table.insert_positions(0, 6); - let prev_end = hc.table.history_abs_end(); - assert_eq!(prev_end, 10); - assert!(hc.table.hash_table.iter().any(|&v| v != HC_EMPTY)); - - hc.reset(|_| {}); - - // Behavioural contract: the previous frame's entries are no longer - // matchable. `reset` advances the floor past every prior position - // instead of zeroing the tables, so each populated slot now decodes - // to an absolute position strictly below `history_abs_start` and is - // rejected by the `window_low` guard before any byte is read. - assert_eq!(hc.table.history_abs_start, prev_end); - for &slot in hc.table.hash_table.iter() { - if let Some(candidate_abs) = - MatchTable::stored_abs_position_fast(slot, hc.table.position_base, hc.table.index_shift) - { - assert!( - candidate_abs < hc.table.history_abs_start, - "a prior-frame entry must resolve below the advanced floor" - ); - } - } -} - -#[test] -fn hc_reset_full_zeroes_when_floor_would_cross_ceiling() { - use super::match_table::storage::REBASE_RESET_FLOOR_CEILING; - let mut hc = HcMatchGenerator::new(32); - hc.table.add_data(b"abcdeabcde".to_vec(), |_| {}); - hc.table.ensure_tables(); - hc.table.hash_table.fill(123); - hc.table.chain_table.fill(456); - // Push the would-be floor (`history_abs_end`) past the ceiling so - // `reset` takes the bounded fallback: rewind to the origin and zero - // the tables, keeping the absolute cursor from climbing toward - // `usize::MAX` on 32-bit targets. - hc.table.history_abs_start = REBASE_RESET_FLOOR_CEILING; - - hc.reset(|_| {}); - - assert_eq!(hc.table.history_abs_start, 0); - assert_eq!(hc.table.position_base, 0); - assert!(hc.table.hash_table.iter().all(|&v| v == HC_EMPTY)); - assert!(hc.table.chain_table.iter().all(|&v| v == HC_EMPTY)); -} - -#[test] -fn hc_start_matching_returns_early_for_empty_current_block() { - let mut hc = HcMatchGenerator::new(32); - hc.table.add_data(Vec::new(), |_| {}); - let mut called = false; - hc.start_matching(|_| called = true); - assert!(!called, "empty current block should not emit sequences"); -} - -#[cfg(test)] -fn deterministic_high_entropy_bytes(seed: u64, len: usize) -> Vec { - let mut out = Vec::with_capacity(len); - let mut state = seed; - for _ in 0..len { - state ^= state << 13; - state ^= state >> 7; - state ^= state << 17; - out.push((state >> 40) as u8); - } - out -} - -#[cfg(feature = "bench_internals")] -pub(crate) fn level22_block_ranges(data: &[u8]) -> Vec<(usize, usize)> { - let mut ranges = Vec::new(); - let mut cursor = 0usize; - let mut savings = 0i64; - while cursor < data.len() { - let remaining = data.len() - cursor; - let candidate_len = remaining.min(super::cost_model::HC_BLOCKSIZE_MAX); - let block_len = crate::encoding::frame_compressor::optimal_block_size( - CompressionLevel::Level(22), - &data[cursor..cursor + candidate_len], - remaining, - super::cost_model::HC_BLOCKSIZE_MAX, - savings, - ) - .min(candidate_len) - .max(1); - ranges.push((cursor, block_len)); - cursor += block_len; - // The exact upstream zstd gate uses compressed-size savings. For this corpus - // parity harness, after the first full block has compressed, savings is - // sufficient to authorize the same pre-block splitter path. - if cursor >= super::cost_model::HC_BLOCKSIZE_MAX { - savings = 3; - } - } - ranges -} - -#[cfg(feature = "bench_internals")] -fn merge_block_delimiters(sequences: Vec<(usize, usize, usize)>) -> Vec<(usize, usize, usize)> { - let mut out = Vec::with_capacity(sequences.len()); - let mut pending_lits = 0usize; - for (lit_len, offset, match_len) in sequences { - if offset == 0 && match_len == 0 { - pending_lits = pending_lits.saturating_add(lit_len); - continue; - } - out.push((lit_len.saturating_add(pending_lits), offset, match_len)); - pending_lits = 0; - } - if pending_lits > 0 { - out.push((pending_lits, 0, 0)); - } - out -} - -/// White-box capture of the level-22 sequence stream (literal-length, -/// offset, match-length triples) the match generator emits for `data`, -/// with block-delimiter pseudo-sequences merged into the following -/// triple's literal run. Pure Rust; the C-conformance comparison that -/// consumes it lives in the `ffi-bench` crate. -#[cfg(feature = "bench_internals")] -pub(crate) fn collect_level22_sequences(data: &[u8]) -> Vec<(usize, usize, usize)> { - merge_block_delimiters(collect_level22_sequences_with_delimiters(data)) - .into_iter() - .filter(|(_, offset, match_len)| *offset != 0 || *match_len != 0) - .collect() -} - -#[cfg(feature = "bench_internals")] -fn collect_level22_sequences_with_delimiters(data: &[u8]) -> Vec<(usize, usize, usize)> { - let mut driver = MatchGeneratorDriver::new(super::cost_model::HC_BLOCKSIZE_MAX, 1); - driver.set_source_size_hint(data.len() as u64); - driver.reset(CompressionLevel::Level(22)); - - let mut sequences = Vec::new(); - for (chunk_start, chunk_len) in level22_block_ranges(data) { - let chunk = &data[chunk_start..chunk_start + chunk_len]; - let mut space = driver.get_next_space(); - space[..chunk.len()].copy_from_slice(chunk); - space.truncate(chunk.len()); - driver.commit_space(space); - driver.start_matching(|seq| { - let entry = match seq { - Sequence::Literals { literals } => (literals.len(), 0usize, 0usize), - Sequence::Triple { - literals, - offset, - match_len, - } => (literals.len(), offset, match_len), - }; - sequences.push(entry); - }); - } - sequences -} - -#[test] -fn hc_sparse_skip_matching_preserves_tail_cross_block_match() { - let mut matcher = HcMatchGenerator::new(1 << 22); - let tail = b"Qz9kLm2Rp"; - let mut first = deterministic_high_entropy_bytes(0xD1B5_4A32_9C77_0E19, 4096); - let tail_start = first.len() - tail.len(); - first[tail_start..].copy_from_slice(tail); - matcher.table.add_data(first.clone(), |_| {}); - matcher.skip_matching(Some(true)); - - let mut second = tail.to_vec(); - second.extend_from_slice(b"after-tail-literals"); - matcher.table.add_data(second, |_| {}); - - let mut first_sequence = None; - matcher.start_matching(|seq| { - if first_sequence.is_some() { - return; - } - first_sequence = Some(match seq { - Sequence::Literals { literals } => (literals.len(), 0usize, 0usize), - Sequence::Triple { - literals, - offset, - match_len, - } => (literals.len(), offset, match_len), - }); - }); - - let (literals_len, offset, match_len) = - first_sequence.expect("expected at least one sequence after sparse skip"); - assert_eq!( - literals_len, 0, - "first sequence should start at block boundary" - ); - assert_eq!( - offset, - tail.len(), - "first match should reference previous tail" - ); - assert!( - match_len >= tail.len(), - "tail-aligned cross-block match must be preserved" - ); -} - -#[test] -fn btultra2_sparse_skip_matching_preserves_tail_cross_block_match() { - let mut matcher = HcMatchGenerator::new(1 << 20); - matcher.configure( - BTULTRA2_HC_CONFIG_L22, - super::strategy::StrategyTag::BtUltra2, - 20, - ); - let tail = b"Bt9kLm2Rp"; - let mut first = deterministic_high_entropy_bytes(0xA9C3_7F21_D4E8_510B, 4096); - let tail_start = first.len() - tail.len(); - first[tail_start..].copy_from_slice(tail); - matcher.table.add_data(first, |_| {}); - matcher.skip_matching(Some(true)); - - let mut second = tail.to_vec(); - second.extend_from_slice(b"after-tail-literals"); - matcher.table.add_data(second, |_| {}); - - let mut first_sequence = None; - matcher.start_matching(|seq| { - if first_sequence.is_some() { - return; - } - first_sequence = Some(match seq { - Sequence::Literals { literals } => (literals.len(), 0usize, 0usize), - Sequence::Triple { - literals, - offset, - match_len, - } => (literals.len(), offset, match_len), - }); - }); - - let (literals_len, offset, match_len) = - first_sequence.expect("expected at least one sequence after sparse BT skip"); - assert_eq!( - literals_len, 0, - "BT sparse skip should preserve an immediate boundary match" - ); - assert_eq!( - offset, - tail.len(), - "first BT match should reference previous tail" - ); - assert!( - match_len >= tail.len(), - "BT sparse skip must seed the dense tail for cross-block matching" - ); -} - -#[test] -fn hc_sparse_skip_matching_does_not_reinsert_sparse_tail_positions() { - let mut matcher = HcMatchGenerator::new(1 << 22); - let first = deterministic_high_entropy_bytes(0xC2B2_AE3D_27D4_EB4F, 4096); - matcher.table.add_data(first.clone(), |_| {}); - matcher.skip_matching(Some(true)); - - let current_len = first.len(); - let current_abs_start = - matcher.table.history_abs_start + matcher.table.window_size - current_len; - let current_abs_end = current_abs_start + current_len; - let dense_tail = HC_MIN_MATCH_LEN + INCOMPRESSIBLE_SKIP_STEP; - let tail_start = current_abs_end - .saturating_sub(dense_tail) - .max(matcher.table.history_abs_start) - .max(current_abs_start); - - let overlap_pos = (tail_start..current_abs_end) - .find(|&pos| (pos - current_abs_start).is_multiple_of(INCOMPRESSIBLE_SKIP_STEP)) - .expect("fixture should contain at least one sparse-grid overlap in dense tail"); - - let rel = matcher - .table - .relative_position(overlap_pos) - .expect("overlap position should be representable as relative position"); - let chain_idx = rel as usize & ((1 << matcher.table.chain_log) - 1); - assert_ne!( - matcher.table.chain_table[chain_idx], - rel + 1, - "sparse-grid tail positions must not be reinserted (self-loop chain entry)" - ); -} - -#[test] -fn hc_compact_history_drains_when_threshold_crossed() { - let mut hc = HcMatchGenerator::new(8); - hc.table.history = b"abcdefghijklmnopqrstuvwxyz".to_vec(); - hc.table.history_start = 16; - hc.table.compact_history(); - assert_eq!(hc.table.history_start, 0); - assert_eq!(hc.table.history, b"qrstuvwxyz"); -} - -#[test] -fn hc_insert_position_no_rebase_returns_when_relative_pos_unavailable() { - let mut hc = HcMatchGenerator::new(32); - hc.table.history = b"abcdefghijklmnop".to_vec(); - hc.table.history_abs_start = 0; - hc.table.position_base = 1; - hc.table.ensure_tables(); - let before_hash = hc.table.hash_table.clone(); - let before_chain = hc.table.chain_table.clone(); - - hc.table.insert_position_no_rebase(0); - - assert_eq!(hc.table.hash_table, before_hash); - assert_eq!(hc.table.chain_table, before_chain); -} - -#[test] -fn hc_insert_positions_advances_next_to_update3_for_contiguous_range() { - let mut hc = HcMatchGenerator::new(64); - hc.table.history = b"abcdefghijklmnopqrstuvwxyz".to_vec(); - hc.table.history_start = 0; - hc.table.history_abs_start = 0; - hc.table.position_base = 0; - hc.table.ensure_tables(); - hc.table.next_to_update3 = 0; - - hc.table.insert_positions(0, 9); - - assert_eq!( - hc.table.next_to_update3, 9, - "contiguous insert_positions should advance hash3 update cursor" - ); -} - -#[test] -fn hc_insert_positions_with_step_keeps_next_to_update3_cursor_for_sparse_ranges() { - let mut hc = HcMatchGenerator::new(64); - hc.table.history = b"abcdefghijklmnopqrstuvwxyz".to_vec(); - hc.table.history_start = 0; - hc.table.history_abs_start = 0; - hc.table.position_base = 0; - hc.table.ensure_tables(); - hc.table.next_to_update3 = 0; - - hc.table.insert_positions_with_step(0, 16, 4); - - assert_eq!( - hc.table.next_to_update3, 0, - "sparse insert_positions_with_step must not mark skipped positions as hash3-updated" - ); -} - -#[cfg(any())] -// disabled: tests legacy SuffixStore behavior incompatible with upstream zstd-shape kernel's HASH_READ_SIZE geometry -#[test] -fn prime_with_dictionary_budget_shrinks_after_simple_eviction() { - let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset(CompressionLevel::Fastest); - // Use a small live window so dictionary-primed slices are evicted - // quickly and budget retirement can be asserted deterministically. - driver.simple_mut().max_window_size = 8; - driver.reported_window_size = 8; - - let base_window = driver.simple_mut().max_window_size; - driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); - assert_eq!(driver.simple_mut().max_window_size, base_window + 24); - - for block in [b"AAAAAAAA", b"BBBBBBBB"] { - let mut space = driver.get_next_space(); - space.clear(); - space.extend_from_slice(block); - driver.commit_space(space); - driver.skip_matching_with_hint(None); - } - - assert_eq!( - driver.dictionary_retained_budget, 0, - "dictionary budget should be fully retired once primed dict slices are evicted" - ); - assert_eq!( - driver.simple_mut().max_window_size, - base_window, - "retired dictionary budget must not remain reusable for live history" - ); -} - -#[test] -fn prime_with_dictionary_budget_shrinks_after_dfast_eviction() { - let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset(CompressionLevel::Level(3)); - // Use a small live window in this regression so dictionary-primed slices are - // evicted quickly and budget retirement can be asserted deterministically. - driver.dfast_matcher_mut().max_window_size = 8; - driver.reported_window_size = 8; - - let base_window = driver.dfast_matcher().max_window_size; - driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); - assert_eq!(driver.dfast_matcher().max_window_size, base_window + 24); - - for block in [b"AAAAAAAA", b"BBBBBBBB"] { - let mut space = driver.get_next_space(); - space.clear(); - space.extend_from_slice(block); - driver.commit_space(space); - driver.skip_matching_with_hint(None); - } - - assert_eq!( - driver.dictionary_retained_budget, 0, - "dictionary budget should be fully retired once primed dict slices are evicted" - ); - assert_eq!( - driver.dfast_matcher().max_window_size, - base_window, - "retired dictionary budget must not remain reusable for live history" - ); -} - -#[test] -fn hc_prime_with_dictionary_preserves_history_for_first_full_block() { - let mut driver = MatchGeneratorDriver::new(8, 1); - // Route onto HashChain explicitly — `Better` resolves to the Row - // backend in production, and this test pins HC dict-prime behaviour. - driver.reset_on_hc_lazy(CompressionLevel::Better); - - driver.prime_with_dictionary(b"abcdefgh", [1, 4, 8]); - - let mut space = driver.get_next_space(); - space.clear(); - // Repeat the dictionary content so the HC matcher can find it. - // HC_MIN_MATCH_LEN is 5, so an 8-byte match is well above threshold. - space.extend_from_slice(b"abcdefgh"); - driver.commit_space(space); - - let mut saw_match = false; - driver.start_matching(|seq| { - if let Sequence::Triple { - literals, - offset, - match_len, - } = seq - && literals.is_empty() - && offset == 8 - && match_len >= HC_MIN_MATCH_LEN - { - saw_match = true; - } - }); - - assert!( - saw_match, - "hash-chain backend should match dictionary-primed history in first full block" - ); -} - -#[test] -fn prime_with_dictionary_budget_shrinks_after_hc_eviction() { - let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset_on_hc_lazy(CompressionLevel::Better); - // Use a small live window so dictionary-primed slices are evicted quickly. - driver.hc_matcher_mut().table.max_window_size = 8; - driver.reported_window_size = 8; - - let base_window = driver.hc_matcher().table.max_window_size; - driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); - assert_eq!(driver.hc_matcher().table.max_window_size, base_window + 24); - - for block in [b"AAAAAAAA", b"BBBBBBBB"] { - let mut space = driver.get_next_space(); - space.clear(); - space.extend_from_slice(block); - driver.commit_space(space); - driver.skip_matching_with_hint(None); - } - - assert_eq!( - driver.dictionary_retained_budget, 0, - "dictionary budget should be fully retired once primed dict slices are evicted" - ); - assert_eq!( - driver.hc_matcher().table.max_window_size, - base_window, - "retired dictionary budget must not remain reusable for live history" - ); -} - -#[test] -fn resident_reapply_restores_retained_dictionary_budget() { - // A reused-dict frame that re-borrows the resident dictionary (skips the - // re-prime) must restore the retained-dict budget the per-frame `reset` - // cleared. The matcher's `reset` re-inflates `max_window_size` by the dict - // region; without the restore the driver-level budget stays 0 and - // `retire_dictionary_budget` never shrinks that inflated window as the dict - // evicts. For the HashChain backend (whose `window_low` is measured against - // `max_window_size`) that lets a post-eviction match exceed the frame - // header's base window and emit an over-window offset. - let mut driver = MatchGeneratorDriver::new(1 << 16, 1); - let dict = b"abcdefghABCDEFGHijklmnopqrstuvwxyz0123456789"; - driver.set_dictionary_size_hint(dict.len()); - driver.reset_on_hc_lazy(CompressionLevel::Better); - driver.prime_with_dictionary(dict, [1, 4, 8]); - let base = driver.reported_window_size; - assert!( - driver.dictionary_retained_budget > 0, - "the priming frame must retain a non-zero dict budget" - ); - - // Second frame: the reset detects the resident dict and re-borrows it. - driver.set_dictionary_size_hint(dict.len()); - driver.reset_on_hc_lazy(CompressionLevel::Better); - assert!( - driver.dictionary_is_resident(), - "the second frame must re-borrow the resident dictionary" - ); - assert_eq!( - driver.dictionary_retained_budget, 0, - "reset clears the retained-dict budget" - ); - let inflated = driver.hc_matcher().table.max_window_size; - assert!( - inflated > base, - "reset re-inflates the window by the resident dict region \ - (inflated={inflated}, base={base})" - ); - - driver.reapply_resident_dictionary([1, 4, 8]); - assert_eq!( - driver.dictionary_retained_budget, - inflated - base, - "resident reapply must restore the retained-dict budget (= window \ - inflation) so the retire path can shrink the window as the dict evicts" - ); -} - -#[test] -fn hc_commit_without_eviction_retires_no_dictionary_budget() { - // Regression: after the window<->history dedup, MatchTable::add_data - // invokes its reuse_space callback for the *input* buffer (recycle), - // not for evicted chunks. The HC arm of commit_space must therefore - // derive eviction bytes from the window_size delta — counting the - // callback argument as evicted would charge the whole committed block - // as "evicted" and prematurely retire dictionary budget even when the - // window is nowhere near full. - let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset_on_hc_lazy(CompressionLevel::Better); - // A large live window so a small committed block evicts nothing. - driver.hc_matcher_mut().table.max_window_size = 1 << 20; - driver.reported_window_size = 1 << 20; - driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); - let budget_after_prime = driver.dictionary_retained_budget; - assert!( - budget_after_prime > 0, - "priming must retain a non-zero dictionary budget" - ); - - let mut space = driver.get_next_space(); - space.clear(); - space.extend_from_slice(b"AAAAAAAA"); - driver.commit_space(space); - driver.skip_matching_with_hint(None); - - assert_eq!( - driver.dictionary_retained_budget, budget_after_prime, - "a commit that evicts nothing must retire no dictionary budget" - ); -} - -#[test] -fn row_commit_without_eviction_retires_no_dictionary_budget() { - // Regression for the Row arm of commit_space after the window -> - // chunk_lens migration: RowMatchGenerator::add_data now invokes its - // reuse_space callback for the *input* buffer (per-commit recycle), - // not for evicted chunks. The Row arm must derive eviction bytes from - // the window_size delta like the Dfast / HashChain arms — counting the - // callback argument as evicted charges the whole committed block as - // "evicted" and prematurely retires dictionary budget even when the - // window is nowhere near full. - let mut driver = MatchGeneratorDriver::new(8, 1); - driver.reset(CompressionLevel::Level(5)); - assert!(matches!(driver.storage, MatcherStorage::Row(_))); - // A large live window so a small committed block evicts nothing. - driver.row_matcher_mut().max_window_size = 1 << 20; - driver.reported_window_size = 1 << 20; - driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); - let budget_after_prime = driver.dictionary_retained_budget; - assert!( - budget_after_prime > 0, - "priming must retain a non-zero dictionary budget" - ); - - let mut space = driver.get_next_space(); - space.clear(); - space.extend_from_slice(b"AAAAAAAA"); - driver.commit_space(space); - driver.skip_matching_with_hint(None); - - assert_eq!( - driver.dictionary_retained_budget, budget_after_prime, - "a Row commit that evicts nothing must retire no dictionary budget" - ); -} - -#[test] -fn hc_rebases_positions_after_u32_boundary() { - let mut matcher = HcMatchGenerator::new(64); - matcher.table.add_data(b"abcdeabcdeabcde".to_vec(), |_| {}); - matcher.table.ensure_tables(); - matcher.table.position_base = 0; - let history_abs_start: usize = match (u64::from(u32::MAX) + 64).try_into() { - Ok(value) => value, - Err(_) => return, - }; - // Simulate a long-running stream where absolute history positions crossed - // the u32 range. Before #51 this disabled HC inserts entirely. - matcher.table.history_abs_start = history_abs_start; - matcher.skip_matching(None); - assert_eq!( - matcher.table.position_base, matcher.table.history_abs_start, - "rebase should anchor to the oldest live absolute position" - ); - - assert!( - matcher - .table - .hash_table - .iter() - .any(|entry| *entry != HC_EMPTY), - "HC hash table should still be populated after crossing u32 boundary" - ); - - // Verify rebasing preserves candidate lookup, not just table population. - let abs_pos = matcher.table.history_abs_start + 10; - let candidates = matcher.hc.chain_candidates(&matcher.table, abs_pos); - assert!( - candidates.iter().any(|candidate| *candidate != usize::MAX), - "chain_candidates should return valid matches after rebase" - ); -} - -// 64-bit only: the >4 GiB absolute cursor this test fabricates cannot exist on -// a 32-bit target (usize == u32 can't address that much), and setting -// `history_abs_start` near `u32::MAX` there overflows `usize` in the -// `check_stream_abs_headroom` guard before the rebase path is reached. Mirrors -// the `try_into()` early-return guard on `hc_rebases_positions_after_u32_boundary`. -#[cfg(target_pointer_width = "64")] -#[test] -fn row_rebases_positions_after_u32_boundary() { - // Row stores absolute match positions as u32. On a long stream the - // cumulative absolute cursor crosses the u32 range even while the live - // window stays bounded; `add_data` must rebase the coordinate origin - // down to the oldest live byte instead of asserting. Before the rebase - // landed this panicked on the `< u32::MAX` assertion, dropping valid - // long Row-backed frames. - let mut m = RowMatchGenerator::new(64); - m.add_data(b"abcdeabcdeabcde".to_vec(), |_| {}); - - // Simulate ~4 GiB of stream behind a bounded window: the live bytes now - // sit just under the u32 absolute ceiling. - let near_ceiling = (u32::MAX as usize) - 16; - m.history_abs_start = near_ceiling; - - // The next commit would push a u32 position past the ceiling; add_data - // must rebase the origin rather than panic. - m.add_data(b"fghij".to_vec(), |_| {}); - - assert!( - m.history_abs_start < near_ceiling, - "add_data must rebase the absolute origin down when the cursor nears \ - u32::MAX (got {})", - m.history_abs_start - ); - assert!( - (m.history_abs_start + m.window_size) < u32::MAX as usize, - "after rebase the live window must fit below the u32 position ceiling" - ); -} - -#[test] -fn hc_rebase_rebuilds_only_inserted_prefix() { - let mut matcher = HcMatchGenerator::new(64); - matcher.table.add_data(b"abcdeabcdeabcde".to_vec(), |_| {}); - matcher.table.ensure_tables(); - matcher.table.position_base = 0; - let history_abs_start: usize = match (u64::from(u32::MAX) + 64).try_into() { - Ok(value) => value, - Err(_) => return, - }; - matcher.table.history_abs_start = history_abs_start; - let abs_pos = matcher.table.history_abs_start + 6; - - let mut expected = HcMatchGenerator::new(64); - expected.table.add_data(b"abcdeabcdeabcde".to_vec(), |_| {}); - expected.table.ensure_tables(); - expected.table.history_abs_start = history_abs_start; - expected.table.position_base = expected.table.history_abs_start; - expected.table.hash_table.fill(HC_EMPTY); - expected.table.chain_table.fill(HC_EMPTY); - for pos in expected.table.history_abs_start..abs_pos { - expected.table.insert_position_no_rebase(pos); - } - - matcher.table.maybe_rebase_positions(abs_pos); - - assert_eq!( - matcher.table.position_base, matcher.table.history_abs_start, - "rebase should still anchor to the oldest live absolute position" - ); - assert_eq!( - matcher.table.hash_table, expected.table.hash_table, - "rebase must rebuild only positions already inserted before abs_pos" - ); - assert_eq!( - matcher.table.chain_table, expected.table.chain_table, - "future positions must not be pre-seeded into HC chains during rebase" - ); -} - -#[cfg(any())] // disabled: tested legacy MatchGenerator/SuffixStore behavior removed in phase 1b -#[test] -fn suffix_store_with_single_slot_does_not_panic_on_keying() { - let mut suffixes = SuffixStore::with_capacity(1); - suffixes.insert(b"abcde", 0); - assert!(suffixes.contains_key(b"abcde")); - assert_eq!(suffixes.get(b"abcde"), Some(0)); -} - -#[cfg(any())] -// disabled: hash_fill_step is a legacy MatchGenerator field; FastKernelMatcher walks stride=1 today -#[test] -fn fastest_reset_uses_interleaved_hash_fill_step() { - let mut driver = MatchGeneratorDriver::new(32, 2); - - driver.reset(CompressionLevel::Uncompressed); - assert_eq!(driver.simple().hash_fill_step, 1); - - driver.reset(CompressionLevel::Fastest); - assert_eq!(driver.simple().hash_fill_step, FAST_HASH_FILL_STEP); - - // Better uses the HashChain backend with lazy2; verify that the backend switch - // happened and the lazy_depth is configured correctly. - driver.reset(CompressionLevel::Better); - assert_eq!( - driver.active_backend(), - super::strategy::BackendTag::HashChain - ); - assert_eq!(driver.window_size(), (1u64 << 23)); - assert_eq!(driver.hc_matcher().hc.lazy_depth, 2); -} - -#[cfg(any())] // disabled: tested legacy MatchGenerator/SuffixStore behavior removed in phase 1b -#[test] -fn simple_matcher_updates_offset_history_after_emitting_match() { - let mut matcher = MatchGenerator::new(64); - matcher.add_data( - b"abcdeabcdeabcde".to_vec(), - SuffixStore::with_capacity(64), - |_, _| {}, - ); - - assert!(matcher.next_sequence(|seq| { - assert_eq!( - seq, - Sequence::Triple { - literals: b"abcde", - offset: 5, - match_len: 10, - } - ); - })); - assert_eq!(matcher.offset_hist, [5, 1, 4]); -} - -#[cfg(any())] // disabled: tested legacy MatchGenerator/SuffixStore behavior removed in phase 1b -#[test] -fn simple_matcher_zero_literal_repcode_checks_rep1_before_hash_lookup() { - let mut matcher = MatchGenerator::new(64); - matcher.add_data( - b"abcdefghijabcdefghij".to_vec(), - SuffixStore::with_capacity(64), - |_, _| {}, - ); - - matcher.suffix_idx = 10; - matcher.last_idx_in_sequence = 10; - matcher.offset_hist = [99, 10, 4]; - - let candidate = matcher.repcode_candidate(&matcher.window.last().unwrap().data[10..], 0); - assert_eq!(candidate, Some((10, 10))); -} - -#[cfg(any())] // disabled: tested legacy MatchGenerator/SuffixStore behavior removed in phase 1b -#[test] -fn simple_matcher_repcode_can_target_previous_window_entry() { - let mut matcher = MatchGenerator::new(64); - matcher.add_data( - b"abcdefghij".to_vec(), - SuffixStore::with_capacity(64), - |_, _| {}, - ); - matcher.skip_matching(); - matcher.add_data( - b"abcdefghij".to_vec(), - SuffixStore::with_capacity(64), - |_, _| {}, - ); - - matcher.offset_hist = [99, 10, 4]; - - let candidate = matcher.repcode_candidate(&matcher.window.last().unwrap().data, 0); - assert_eq!(candidate, Some((10, 10))); -} - -#[cfg(any())] // disabled: tested legacy MatchGenerator/SuffixStore behavior removed in phase 1b -#[test] -fn simple_matcher_zero_literal_repcode_checks_rep2() { - let mut matcher = MatchGenerator::new(64); - matcher.add_data( - b"abcdefghijabcdefghij".to_vec(), - SuffixStore::with_capacity(64), - |_, _| {}, - ); - matcher.suffix_idx = 10; - matcher.last_idx_in_sequence = 10; - // rep1=4 does not match at idx 10, rep2=10 does. - matcher.offset_hist = [99, 4, 10]; - - let candidate = matcher.repcode_candidate(&matcher.window.last().unwrap().data[10..], 0); - assert_eq!(candidate, Some((10, 10))); -} - -#[cfg(any())] // disabled: tested legacy MatchGenerator/SuffixStore behavior removed in phase 1b -#[test] -fn simple_matcher_zero_literal_repcode_checks_rep0_minus1() { - let mut matcher = MatchGenerator::new(64); - matcher.add_data( - b"abcdefghijabcdefghij".to_vec(), - SuffixStore::with_capacity(64), - |_, _| {}, - ); - matcher.suffix_idx = 10; - matcher.last_idx_in_sequence = 10; - // rep1=4 and rep2=99 do not match; rep0-1 == 10 does. - matcher.offset_hist = [11, 4, 99]; - - let candidate = matcher.repcode_candidate(&matcher.window.last().unwrap().data[10..], 0); - assert_eq!(candidate, Some((10, 10))); -} - -#[cfg(any())] // disabled: tested legacy MatchGenerator/SuffixStore behavior removed in phase 1b -#[test] -fn simple_matcher_repcode_rejects_offsets_beyond_searchable_prefix() { - let mut matcher = MatchGenerator::new(64); - matcher.add_data( - b"abcdefghij".to_vec(), - SuffixStore::with_capacity(64), - |_, _| {}, - ); - matcher.skip_matching(); - matcher.add_data( - b"klmnopqrst".to_vec(), - SuffixStore::with_capacity(64), - |_, _| {}, - ); - matcher.suffix_idx = 3; - - let candidate = matcher.offset_match_len(14, &matcher.window.last().unwrap().data[3..]); - assert_eq!(candidate, None); -} - -#[cfg(any())] // disabled: tested legacy MatchGenerator/SuffixStore behavior removed in phase 1b -#[test] -fn simple_matcher_skip_matching_seeds_every_position_even_with_fast_step() { - let mut matcher = MatchGenerator::new(64); - matcher.hash_fill_step = FAST_HASH_FILL_STEP; - matcher.add_data( - b"abcdefghijklmnop".to_vec(), - SuffixStore::with_capacity(64), - |_, _| {}, - ); - matcher.skip_matching(); - matcher.add_data(b"bcdef".to_vec(), SuffixStore::with_capacity(64), |_, _| {}); - - assert!(matcher.next_sequence(|seq| { - assert_eq!( - seq, - Sequence::Triple { - literals: b"", - offset: 15, - match_len: 5, - } - ); - })); - assert!(!matcher.next_sequence(|_| {})); -} - -#[cfg(any())] // disabled: tested legacy MatchGenerator/SuffixStore behavior removed in phase 1b -#[test] -fn simple_matcher_skip_matching_with_incompressible_hint_uses_sparse_prefix() { - let mut matcher = MatchGenerator::new(128); - let first = b"abcdefghijklmnopqrstuvwxyz012345".to_vec(); - let sparse_probe = first[3..3 + MIN_MATCH_LEN].to_vec(); - let tail_start = first.len() - MIN_MATCH_LEN; - let tail_probe = first[tail_start..tail_start + MIN_MATCH_LEN].to_vec(); - matcher.add_data(first, SuffixStore::with_capacity(256), |_, _| {}); - - matcher.skip_matching_with_hint(Some(true)); - - // Observable behavior check: sparse-prefix probe should not immediately match. - matcher.add_data(sparse_probe, SuffixStore::with_capacity(256), |_, _| {}); - let mut sparse_first_is_literals = None; - assert!(matcher.next_sequence(|seq| { - if sparse_first_is_literals.is_none() { - sparse_first_is_literals = Some(matches!(seq, Sequence::Literals { .. })); - } - })); - assert!( - sparse_first_is_literals.unwrap_or(false), - "sparse-start probe should not produce an immediate match" - ); - - // Dense tail remains indexed for cross-block boundary matching. - let mut matcher = MatchGenerator::new(128); - matcher.add_data( - b"abcdefghijklmnopqrstuvwxyz012345".to_vec(), - SuffixStore::with_capacity(256), - |_, _| {}, - ); - matcher.skip_matching_with_hint(Some(true)); - matcher.add_data(tail_probe, SuffixStore::with_capacity(256), |_, _| {}); - let mut tail_first_is_immediate_match = None; - assert!(matcher.next_sequence(|seq| { - if tail_first_is_immediate_match.is_none() { - tail_first_is_immediate_match = - Some(matches!(seq, Sequence::Triple { literals, .. } if literals.is_empty())); - } - })); - assert!( - tail_first_is_immediate_match.unwrap_or(false), - "dense tail probe should match immediately at block start" - ); -} - -#[cfg(any())] // disabled: tested legacy MatchGenerator/SuffixStore behavior removed in phase 1b -#[test] -fn simple_matcher_add_suffixes_till_backfills_last_searchable_anchor() { - let mut matcher = MatchGenerator::new(64); - matcher.hash_fill_step = FAST_HASH_FILL_STEP; - matcher.add_data( - b"01234abcde".to_vec(), - SuffixStore::with_capacity(64), - |_, _| {}, - ); - matcher.add_suffixes_till(10, FAST_HASH_FILL_STEP); - - let last = matcher.window.last().unwrap(); - let tail = &last.data[5..10]; - assert_eq!(last.suffixes.get(tail), Some(5)); -} - -#[cfg(any())] // disabled: tested legacy MatchGenerator/SuffixStore behavior removed in phase 1b -#[test] -fn simple_matcher_add_suffixes_till_skips_when_idx_below_min_match_len() { - let mut matcher = MatchGenerator::new(128); - matcher.hash_fill_step = FAST_HASH_FILL_STEP; - matcher.add_data( - b"abcdefghijklmnopqrstuvwxyz".to_vec(), - SuffixStore::with_capacity(1 << 16), - |_, _| {}, - ); - - matcher.add_suffixes_till(MIN_MATCH_LEN - 1, FAST_HASH_FILL_STEP); - - let last = matcher.window.last().unwrap(); - let first_key = &last.data[..MIN_MATCH_LEN]; - assert_eq!(last.suffixes.get(first_key), None); -} - -#[cfg(any())] // disabled: tested legacy MatchGenerator/SuffixStore behavior removed in phase 1b -#[test] -fn simple_matcher_add_suffixes_till_fast_step_registers_interleaved_positions() { - let mut matcher = MatchGenerator::new(128); - matcher.hash_fill_step = FAST_HASH_FILL_STEP; - matcher.add_data( - b"abcdefghijklmnopqrstuvwxyz".to_vec(), - SuffixStore::with_capacity(1 << 16), - |_, _| {}, - ); - - matcher.add_suffixes_till(17, FAST_HASH_FILL_STEP); - - let last = matcher.window.last().unwrap(); - for pos in [0usize, 3, 6, 9, 12] { - let key = &last.data[pos..pos + MIN_MATCH_LEN]; - assert_eq!( - last.suffixes.get(key), - Some(pos), - "expected interleaved suffix registration at pos {pos}" - ); - } -} - -#[test] -fn dfast_skip_matching_handles_window_eviction() { - let mut matcher = DfastMatchGenerator::new(16); - - matcher.add_data(alloc::vec![1, 2, 3, 4, 5, 6], |_| {}); - matcher.skip_matching(None); - matcher.add_data(alloc::vec![7, 8, 9, 10, 11, 12], |_| {}); - matcher.skip_matching(None); - matcher.add_data(alloc::vec![7, 8, 9, 10, 11, 12], |_| {}); - - let mut reconstructed = alloc::vec![7, 8, 9, 10, 11, 12]; - matcher.start_matching(|seq| match seq { - Sequence::Literals { literals } => reconstructed.extend_from_slice(literals), - Sequence::Triple { - literals, - offset, - match_len, - } => { - reconstructed.extend_from_slice(literals); - let start = reconstructed.len() - offset; - for i in 0..match_len { - let byte = reconstructed[start + i]; - reconstructed.push(byte); - } - } - }); - - assert_eq!(reconstructed, [7, 8, 9, 10, 11, 12, 7, 8, 9, 10, 11, 12]); -} - -#[test] -fn dfast_add_data_callback_reports_evicted_len_not_capacity() { - let mut matcher = DfastMatchGenerator::new(8); - - let mut first = Vec::with_capacity(64); - first.extend_from_slice(b"abcdefgh"); - matcher.add_data(first, |_| {}); - - let mut second = Vec::with_capacity(64); - second.extend_from_slice(b"ijklmnop"); - - let mut observed_evicted_len = None; - matcher.add_data(second, |data| { - observed_evicted_len = Some(data.len()); - }); - - assert_eq!( - observed_evicted_len, - Some(8), - "eviction callback must report evicted byte length, not backing capacity" - ); -} - -/// Regression for the `commit_space` Dfast-branch eviction accounting bug -/// (CodeRabbit Critical on PR #146). Old code counted the INPUT buffer -/// length as `evicted_bytes` because Dfast's `add_data` callback receives -/// the input `Vec` for pool recycling (Dfast stores bytes in `history`, -/// not per-block Vecs). On the saturated-window 1:1 path the two coincide -/// so the previous test fixture passed by accident; this test forces the -/// divergent case where evicted != input by sequencing block lengths -/// `[4, 4, 5]` against `max_window_size = 10`: -/// -/// * after 1st commit: `window_blocks = [4]`, `window_size = 4` -/// * after 2nd commit: `window_blocks = [4, 4]`, `window_size = 8` -/// * 3rd commit (5 bytes): `8 + 5 > 10` → pop one 4-byte block (evict=4), -/// then push 5 (window_size=9). Bug counts `5`, fix counts `4`. -/// -/// The fix derives eviction from `window_size` delta + input length: -/// `evicted = pre + space_len - post`. Verified via the -/// `dictionary_retained_budget` observable: starting budget 100, after -/// the third commit (4 bytes actually evicted) the budget must read 96, -/// not 95. -/// Driver-path regression for the `commit_space` Dfast eviction accounting -/// bug. Exercises `MatchGeneratorDriver::commit_space` directly (not just -/// `DfastMatchGenerator::add_data`) so the assertion catches a future -/// regression that swaps the Dfast branch in `commit_space` back to -/// `evicted_bytes += data.len()` — the older draft of this regression -/// hand-recomputed the formula on the matcher and would pass either way. -/// -/// Fixture: `max_window_size = 10`, commit sequence `[4, 4, 5]`. The -/// divergent case where the popped block (4 bytes) and the new input -/// (5 bytes) have different sizes: -/// -/// * after commit `"abcd"` (4 B): window_blocks=[4], ws=4 -/// * after commit `"efgh"` (4 B): window_blocks=[4,4], ws=8 -/// * commit `"ijklm"` (5 B): 8+5>10 → pop front [4] (evict=4), -/// push 5 → window_blocks=[4,5], ws=9 -/// -/// `commit_space` then calls `retire_dictionary_budget(evicted)`. With -/// the fix `evicted=4`; with the bug it would be `evicted=5`. The -/// downstream `trim_after_budget_retire` cascade (which fires whenever -/// `retire_dictionary_budget` returns true) drives the budget further -/// down by trimming the now-oversize window; the final -/// `dictionary_retained_budget` differs between the two paths because -/// the cascade starting state differs (max_window_size after first -/// retire is `10 - evicted`). -/// -/// Tracing the fix path end-to-end with starting budget = 100: -/// 1st commit: evicted=0, no retire. -/// 2nd commit: evicted=0, no retire. -/// 3rd commit: evicted=4. retire(4) → budget=96, max_window=6. -/// trim_after_budget_retire: -/// iter1: ws=9 > max=6, pop [4] → ws=5, evicted=4. -/// retire(4) → budget=92, max_window=2. -/// iter2: ws=5 > max=2, pop [5] → ws=0, evicted=5. -/// retire(5) → budget=87, max_window=0. -/// iter3: ws=0, no trim, retire(0) → false, exit. -/// Final budget = 87. Final max_window_size = 0. -/// -/// In the buggy path the 3rd commit would compute `evicted=5`, retire -/// would reclaim 5 instead of 4, shrinking max_window_size to 5 -/// instead of 6 — and then the cascade arithmetic produces a -/// different final budget (and on the 2nd commit the cascade would -/// already have shrunk max_window_size to 0, causing the 3rd commit -/// to panic on `data.len() <= max_window_size`). Either way the -/// regression surfaces as a test failure. -#[test] -fn dfast_commit_space_eviction_uses_window_size_delta() { - use crate::encoding::CompressionLevel; - - let mut driver = MatchGeneratorDriver::new(10, 1); - driver.reset(CompressionLevel::Level(3)); - assert!(matches!(driver.storage, MatcherStorage::Dfast(_))); - - // Override the level-derived window with a tiny one so the - // 4 + 4 + 5 = 13 commit sequence below actually crosses the - // boundary. A 16 KiB+ default window would never evict on this - // little data and the bug would stay invisible. - driver.dfast_matcher_mut().max_window_size = 10; - driver.dictionary_retained_budget = 100; - - let mut space1 = Vec::with_capacity(64); - space1.extend_from_slice(b"abcd"); - driver.commit_space(space1); - assert_eq!( - driver.dictionary_retained_budget, 100, - "1st commit fills window 0 → 4, no eviction, no retire" - ); - - let mut space2 = Vec::with_capacity(64); - space2.extend_from_slice(b"efgh"); - driver.commit_space(space2); - assert_eq!( - driver.dictionary_retained_budget, 100, - "2nd commit fills window 4 → 8, no eviction, no retire" - ); - - let mut space3 = Vec::with_capacity(64); - space3.extend_from_slice(b"ijklm"); - driver.commit_space(space3); - assert_eq!( - driver.dictionary_retained_budget, 87, - "3rd commit + trim_after_budget_retire cascade. With the fix \ - (evicted=4 from window_size delta) the cascade reclaims 100 \ - → 96 → 92 → 87. With the bug (evicted=5 from data.len()) the \ - 3rd commit would panic on `data.len() <= max_window_size` \ - after the 2nd commit's cascade had already shrunk \ - max_window_size to 0." - ); - assert_eq!( - driver.dfast_matcher_mut().max_window_size, - 0, - "cascade drains max_window_size to 0 once budget reclaim \ - exceeds the initial window size" - ); -} - -#[test] -fn dfast_trim_to_window_evicts_oldest_block_by_length() { - // After the history-only storage refactor (#111 Phase 7c step 3), - // Dfast no longer retains input `Vec`s — the `history` - // contiguous buffer is the sole byte store, and `add_data` - // returns the input Vec to the caller's pool eagerly. So - // `trim_to_window` doesn't have anything to hand back to the - // closure (no Vec exists to give). The eviction is observable - // instead through `window_size` shrinking by the per-block - // length recorded in `window_blocks`. - let mut matcher = DfastMatchGenerator::new(16); - - let mut first = Vec::with_capacity(64); - first.extend_from_slice(b"abcdefgh"); - matcher.add_data(first, |_| {}); - - let mut second = Vec::with_capacity(64); - second.extend_from_slice(b"ijklmnop"); - matcher.add_data(second, |_| {}); - - assert_eq!(matcher.window_size, 16); - assert_eq!(matcher.window_blocks.len(), 2); - - matcher.max_window_size = 8; - - matcher.trim_to_window(); - - // No callback signature to assert on: the Dfast variant of - // `trim_to_window` takes none. That signature shape (vs HC/Row - // which accept `impl FnMut(Vec)`) is the property locking in - // the contract — there is no closure to invoke or skip, so no - // future change can "start invoking the callback" without a - // compile-time signature break that the dispatcher and this test - // would force the author to address. - assert_eq!( - matcher.window_size, 8, - "exactly one 8-byte block must remain" - ); - assert_eq!(matcher.window_blocks.len(), 1); - assert_eq!(matcher.history_abs_start, 8); -} - -#[test] -fn dfast_inserts_tail_positions_for_next_block_matching() { - let mut matcher = DfastMatchGenerator::new(1 << 22); - - matcher.add_data(b"012345bcdea".to_vec(), |_| {}); - let mut history = Vec::new(); - matcher.start_matching(|seq| match seq { - Sequence::Literals { literals } => history.extend_from_slice(literals), - Sequence::Triple { .. } => unreachable!("first block should not match history"), - }); - assert_eq!(history, b"012345bcdea"); - - matcher.add_data(b"bcdeabcdeab".to_vec(), |_| {}); - let mut saw_first_sequence = false; - matcher.start_matching(|seq| { - assert!(!saw_first_sequence, "expected a single cross-block match"); - saw_first_sequence = true; - match seq { - Sequence::Literals { .. } => { - panic!("expected tail-anchored cross-block match before any literals") - } - Sequence::Triple { - literals, - offset, - match_len, - } => { - assert_eq!(literals, b""); - assert_eq!(offset, 5); - assert_eq!(match_len, 11); - let start = history.len() - offset; - for i in 0..match_len { - let byte = history[start + i]; - history.push(byte); - } - } - } - }); - - assert!( - saw_first_sequence, - "expected tail-anchored cross-block match" - ); - assert_eq!(history, b"012345bcdeabcdeabcdeab"); -} - -/// Regression for #49 — locks down `MatchTable::backfill_boundary_positions` -/// for the [`HcMatchGenerator`] lazy path. `backfill_boundary_positions` -/// seeds ONLY the last `< 4` bytes of the previous slice (positions in -/// `[current_abs_start - 3, current_abs_start)`) — the bytes that -/// `insert_position` could not hash at the time because hashing needs -/// 4 bytes of lookahead. The existing 8 MiB window roundtrip test -/// exercises cross-slice behaviour end-to-end, but does not isolate -/// the backfill of those final 1-3 unhashable bytes. -/// -/// Fixture is built so the cross-block match's candidate position -/// MUST lie in `[block_1_end - 3, block_1_end)`: -/// -/// - Block 1 = `b"PQRSTBCD"` (8 bytes). Block 1's `start_matching` -/// hashes positions 0..=4 (each has 4 bytes of forward context); -/// positions 5/6/7 are the unhashable tail. -/// - Block 2 = `b"BCDBCDBCDB"` (10 bytes). At absolute position 8 -/// (block 2 start) the 4-byte window is `b"BCDB"`. The ONLY place -/// `b"BCDB"` was inserted in the hash + chain tables is position 5 -/// — via `backfill_boundary_positions` on the next-slice entry -/// (the 4-byte window at position 5 is `data[5..9] = b"BCD" + -/// block_2[0] = b"BCDB"`). -/// -/// If `backfill_boundary_positions` regresses, position 5 is never -/// hashed, position 8's lookup misses, and the lazy parser falls -/// through to a leading literals run — `offset == 3, match_len >= 4` -/// would no longer hold. -#[test] -fn hashchain_inserts_tail_positions_for_next_block_matching() { - let mut matcher = HcMatchGenerator::new(1 << 22); - matcher.configure(HC_CONFIG, super::strategy::StrategyTag::Lazy, 22); - - matcher.table.add_data(b"PQRSTBCD".to_vec(), |_| {}); - let mut history = alloc::vec::Vec::new(); - matcher.start_matching(|seq| match seq { - Sequence::Literals { literals } => history.extend_from_slice(literals), - Sequence::Triple { .. } => unreachable!("first block has no internal repeats"), - }); - assert_eq!(history, b"PQRSTBCD"); - - matcher.table.add_data(b"BCDBCDBCDB".to_vec(), |_| {}); - let mut first_sequence_offset: Option = None; - let mut first_sequence_match_len: Option = None; - matcher.start_matching(|seq| { - if first_sequence_offset.is_some() { - return; - } - match seq { - Sequence::Literals { .. } => { - panic!( - "expected tail-anchored cross-block match before any literals — \ - backfill_boundary_positions did not seed positions 5/6/7" - ) - } - Sequence::Triple { - literals, - offset, - match_len, - } => { - assert_eq!(literals, b"", "no leading literals on the boundary match"); - first_sequence_offset = Some(offset); - first_sequence_match_len = Some(match_len); - } - } - }); - - let offset = first_sequence_offset.expect( - "expected tail-anchored cross-block match emitted from backfill_boundary_positions", - ); - assert!( - (1..=3).contains(&offset), - "boundary match offset {offset} must point into the unhashable tail \ - (positions 5/6/7 of an 8-byte block 1) so the test specifically \ - locks down backfill_boundary_positions", - ); - assert_eq!( - offset, 3, - "candidate position must land at 5 (= block_1_len - 3) so the 4-byte \ - window `data[5..9] = b\"BCDB\"` matches block 2's first hash lookup", - ); - let match_len = first_sequence_match_len.unwrap(); - assert!( - match_len >= HC_MIN_MATCH_LEN, - "match_len {match_len} must clear the HC min-match floor", - ); -} - -#[test] -fn dfast_dense_skip_matching_backfills_previous_tail_for_next_block() { - let mut matcher = DfastMatchGenerator::new(1 << 22); - let tail = b"Qz9kLm2Rp"; - let mut first = b"0123456789abcdef".to_vec(); - first.extend_from_slice(tail); - matcher.add_data(first.clone(), |_| {}); - matcher.skip_matching(Some(false)); - - let mut second = tail.to_vec(); - second.extend_from_slice(b"after-tail-literals"); - matcher.add_data(second, |_| {}); - - let mut first_sequence = None; - matcher.start_matching(|seq| { - if first_sequence.is_some() { - return; - } - first_sequence = Some(match seq { - Sequence::Literals { literals } => (literals.len(), 0usize, 0usize), - Sequence::Triple { - literals, - offset, - match_len, - } => (literals.len(), offset, match_len), - }); - }); - - let (lit_len, offset, match_len) = first_sequence.expect("expected at least one sequence"); - assert_eq!( - lit_len, 0, - "expected immediate cross-block match at block start" - ); - assert_eq!( - offset, - tail.len(), - "expected dense skip to preserve cross-boundary tail match" - ); - assert!( - match_len >= DFAST_MIN_MATCH_LEN, - "match length should satisfy dfast minimum match length" - ); -} - -#[test] -fn dfast_sparse_skip_matching_preserves_tail_cross_block_match() { - let mut matcher = DfastMatchGenerator::new(1 << 22); - let tail = b"Qz9kLm2Rp"; - let mut first = deterministic_high_entropy_bytes(0x9E37_79B9_7F4A_7C15, 4096); - let tail_start = first.len() - tail.len(); - first[tail_start..].copy_from_slice(tail); - matcher.add_data(first.clone(), |_| {}); - - matcher.skip_matching(Some(true)); - - let mut second = tail.to_vec(); - second.extend_from_slice(b"after-tail-literals"); - matcher.add_data(second, |_| {}); - - let mut first_sequence = None; - matcher.start_matching(|seq| { - if first_sequence.is_some() { - return; - } - first_sequence = Some(match seq { - Sequence::Literals { literals } => (literals.len(), 0usize, 0usize), - Sequence::Triple { - literals, - offset, - match_len, - } => (literals.len(), offset, match_len), - }); - }); - - let (lit_len, offset, match_len) = first_sequence.expect("expected at least one sequence"); - assert_eq!( - lit_len, 0, - "expected immediate cross-block match at block start" - ); - assert_eq!( - offset, - tail.len(), - "expected match against densely seeded tail" - ); - assert!( - match_len >= DFAST_MIN_MATCH_LEN, - "match length should satisfy dfast minimum match length" - ); -} - -#[test] -fn dfast_skip_matching_dense_backfills_newly_hashable_long_tail_positions() { - let mut matcher = DfastMatchGenerator::new(1 << 22); - let first = deterministic_high_entropy_bytes(0x7A64_0315_D4E1_91C3, 4096); - let first_len = first.len(); - matcher.add_data(first, |_| {}); - matcher.skip_matching_dense(); - - // Appending one byte makes exactly the previous block's last 7 starts - // newly eligible for 8-byte long-hash insertion. - matcher.add_data(alloc::vec![0xAB], |_| {}); - matcher.skip_matching_dense(); - - let target_abs_pos = first_len - 7; - let target_rel = target_abs_pos - matcher.history_abs_start; - let live = matcher.live_history(); - assert!( - target_rel + 8 <= live.len(), - "fixture must make the boundary start long-hashable" - ); - let long_hash = matcher.long_hash_index(&live[target_rel..]); - let target_slot = matcher.pack_slot(target_abs_pos); - // Single-slot tables (upstream zstd parity): the bucket holds at most one - // u32; the assertion below is a direct equality (no `.contains`). - assert_ne!( - target_slot, DFAST_EMPTY_SLOT, - "pack_slot must never return the empty-slot sentinel for a real position" - ); - assert_eq!( - matcher.tables[long_hash], target_slot, - "dense skip must seed long-hash entry for newly hashable boundary start" - ); -} - -#[test] -fn dfast_seed_remaining_hashable_starts_seeds_last_short_hash_positions() { - let mut matcher = DfastMatchGenerator::new(1 << 20); - let block = deterministic_high_entropy_bytes(0x13F0_9A6D_55CE_7B21, 64); - matcher.add_data(block, |_| {}); - matcher.ensure_hash_tables(); - - let current_len = matcher.window_blocks.back().copied().unwrap_or(0); - let current_abs_start = matcher.history_abs_start + matcher.window_size - current_len; - let seed_start = current_len - DFAST_MIN_MATCH_LEN; - matcher.seed_remaining_hashable_starts(current_abs_start, current_len, seed_start); - - let target_abs_pos = current_abs_start + current_len - 5; - let target_rel = target_abs_pos - matcher.history_abs_start; - let live = matcher.live_history(); - assert!( - target_rel + 5 <= live.len(), - "fixture must leave the last short-hash start valid" - ); - let short_hash = matcher.short_hash_index(&live[target_rel..]); - let target_slot = matcher.pack_slot(target_abs_pos); - assert_ne!( - target_slot, DFAST_EMPTY_SLOT, - "pack_slot must never return the empty-slot sentinel for a real position" - ); - assert_eq!( - matcher.tables[matcher.long_len() + short_hash], - target_slot, - "tail seeding must include the last 5-byte-hashable start" - ); -} - -#[test] -fn dfast_seed_remaining_hashable_starts_handles_pos_at_block_end() { - let mut matcher = DfastMatchGenerator::new(1 << 20); - let block = deterministic_high_entropy_bytes(0x7BB2_DA91_441E_C0EF, 64); - matcher.add_data(block, |_| {}); - matcher.ensure_hash_tables(); - - let current_len = matcher.window_blocks.back().copied().unwrap_or(0); - let current_abs_start = matcher.history_abs_start + matcher.window_size - current_len; - matcher.seed_remaining_hashable_starts(current_abs_start, current_len, current_len); - - let target_abs_pos = current_abs_start + current_len - 5; - let target_rel = target_abs_pos - matcher.history_abs_start; - let live = matcher.live_history(); - assert!( - target_rel + 5 <= live.len(), - "fixture must leave the last short-hash start valid" - ); - let short_hash = matcher.short_hash_index(&live[target_rel..]); - let target_slot = matcher.pack_slot(target_abs_pos); - assert_ne!( - target_slot, DFAST_EMPTY_SLOT, - "pack_slot must never return the empty-slot sentinel for a real position" - ); - assert_eq!( - matcher.tables[matcher.long_len() + short_hash], - target_slot, - "tail seeding must still include the last 5-byte-hashable start when pos is at block end" - ); -} - -/// `ensure_room_for` must trigger `reduce()` when the requested -/// absolute position would push a relative offset past -/// `u32::MAX - DFAST_REBASE_GUARD_BAND`. After the rebase, the -/// pre-existing entry at a much-smaller absolute position falls -/// below `reducer` and gets cleared to `DFAST_EMPTY_SLOT`; a fresh -/// insert at the boundary position must `pack_slot` to a valid -/// non-sentinel value that `unpack_slot` resolves back to the same -/// absolute position. Mirrors `LdmHashTable::ensure_room_for_*` -/// from PR #139. -/// -/// Runs on every target — `trigger_abs = u32::MAX - -/// DFAST_REBASE_GUARD_BAND + 1 = 0xC0000000`, which fits in `usize` -/// on i686 (`usize::MAX = u32::MAX`) without overflow, so the -/// packed-slot boundary path + u32 ↔ usize round-trip is exercised -/// on every pointer width we ship. -#[test] -fn dfast_ensure_room_for_rebases_above_guard_band() { - let mut dfast = DfastMatchGenerator::new(1 << 22); - dfast.set_hash_bits(10, 10); - dfast.ensure_hash_tables(); - - // Seed an early insert near the current base in BOTH tables. - // `ensure_room_for` / `reduce` is a shared contract for both - // `short_hash` and `long_hash`; without seeding both, a - // regression that only cleared short_hash would still pass. - // Direct `pack_slot` + bucket write keeps the test focused on - // the rebase mechanics and avoids dragging in the full - // `insert_position` flow with its history/window setup. - let early_abs = 1024usize; - let early_packed = dfast.pack_slot(early_abs); - assert_ne!(early_packed, DFAST_EMPTY_SLOT); - let short0 = dfast.long_len(); - dfast.tables[short0] = early_packed; - dfast.tables[0] = early_packed; - - // Pick a trigger position that forces the first rebase. With - // `position_base = 0`, the smallest `abs_pos` that fails the - // `rel <= max_rel` test is `u32::MAX - DFAST_REBASE_GUARD_BAND - // + 1`. After one `reduce(DFAST_REBASE_GUARD_BAND)` the base - // advances by `DFAST_REBASE_GUARD_BAND`. - let trigger_abs = (u32::MAX as usize) - (DFAST_REBASE_GUARD_BAND as usize) + 1; - assert_eq!(dfast.position_base, 0); - dfast.ensure_room_for(trigger_abs); - assert_eq!( - dfast.position_base, DFAST_REBASE_GUARD_BAND as usize, - "rebase must advance position_base by DFAST_REBASE_GUARD_BAND" - ); - - // The early entry at abs=1024 had packed slot 1025; the rebase - // subtracts `DFAST_REBASE_GUARD_BAND` (= 2^30) from every slot. - // 1025 <= 2^30 so the slot drops to the empty sentinel — - // upstream zstd parity for `ZSTD_window_reduce`'s clamp-at-zero rule. - // Verify BOTH tables — `reduce()` walks them in sequence. - assert_eq!( - dfast.tables[dfast.long_len()], - DFAST_EMPTY_SLOT, - "pre-rebase short-hash entries below the reducer must become empty" - ); - assert_eq!( - dfast.tables[0], DFAST_EMPTY_SLOT, - "pre-rebase long-hash entries below the reducer must become empty" - ); - - // A fresh insert past the rebase boundary must round-trip: - // pack to a non-sentinel value, then unpack back to the same - // absolute position via `position_base + slot - 1`. - let post_packed = dfast.pack_slot(trigger_abs); - assert_ne!(post_packed, DFAST_EMPTY_SLOT); - let unpacked = dfast.position_base + (post_packed as usize) - 1; - assert_eq!( - unpacked, trigger_abs, - "post-rebase pack/unpack must round-trip the absolute position" - ); -} - -#[test] -fn dfast_sparse_skip_matching_backfills_previous_tail_for_consecutive_sparse_blocks() { - let mut matcher = DfastMatchGenerator::new(1 << 22); - let boundary_prefix = [0xFA, 0xFB, 0xFC]; - let boundary_suffix = [0xFD, 0xEE, 0xAD, 0xBE, 0xEF, 0x11, 0x22, 0x33]; - - let mut first = deterministic_high_entropy_bytes(0xA5A5_5A5A_C3C3_3C3C, 4096); - let first_tail_start = first.len() - boundary_prefix.len(); - first[first_tail_start..].copy_from_slice(&boundary_prefix); - matcher.add_data(first, |_| {}); - matcher.skip_matching(Some(true)); - - let mut second = deterministic_high_entropy_bytes(0xA5A5_5A5A_C3C3_3C3C, 4096); - second[..boundary_suffix.len()].copy_from_slice(&boundary_suffix); - matcher.add_data(second.clone(), |_| {}); - matcher.skip_matching(Some(true)); - - let mut third = boundary_prefix.to_vec(); - third.extend_from_slice(&boundary_suffix); - third.extend_from_slice(b"-trailing-literals"); - matcher.add_data(third, |_| {}); - - let mut first_sequence = None; - matcher.start_matching(|seq| { - if first_sequence.is_some() { - return; - } - first_sequence = Some(match seq { - Sequence::Literals { literals } => (literals.len(), 0usize, 0usize), - Sequence::Triple { - literals, - offset, - match_len, - } => (literals.len(), offset, match_len), - }); - }); - - let (lit_len, offset, match_len) = first_sequence.expect("expected at least one sequence"); - assert_eq!( - lit_len, 0, - "expected immediate match from the prior sparse-skip boundary" - ); - assert_eq!( - offset, - second.len() + boundary_prefix.len(), - "expected match against backfilled first→second boundary start" - ); - assert!( - match_len >= DFAST_MIN_MATCH_LEN, - "match length should satisfy dfast minimum match length" - ); -} - -#[test] -fn fastest_hint_iteration_23_sequences_reconstruct_source() { - fn generate_data(seed: u64, len: usize) -> Vec { - let mut state = seed; - let mut data = Vec::with_capacity(len); - for _ in 0..len { - state = state - .wrapping_mul(6364136223846793005) - .wrapping_add(1442695040888963407); - data.push((state >> 33) as u8); - } - data - } - - let i = 23u64; - let len = (i * 89 % 16384) as usize; - let mut data = generate_data(i, len); - // Append a repeated slice so the fixture deterministically exercises - // the match path (Sequence::Triple) instead of only literals. - let repeat = data[128..256].to_vec(); - data.extend_from_slice(&repeat); - data.extend_from_slice(&repeat); - - let mut driver = MatchGeneratorDriver::new(1024 * 128, 1); - driver.set_source_size_hint(data.len() as u64); - driver.reset(CompressionLevel::Fastest); - let mut space = driver.get_next_space(); - space[..data.len()].copy_from_slice(&data); - space.truncate(data.len()); - driver.commit_space(space); - - let mut rebuilt = Vec::with_capacity(data.len()); - let mut saw_triple = false; - driver.start_matching(|seq| match seq { - Sequence::Literals { literals } => rebuilt.extend_from_slice(literals), - Sequence::Triple { - literals, - offset, - match_len, - } => { - saw_triple = true; - rebuilt.extend_from_slice(literals); - assert!(offset > 0, "offset must be non-zero"); - assert!( - offset <= rebuilt.len(), - "offset must reference already-produced bytes: offset={} produced={}", - offset, - rebuilt.len() - ); - let start = rebuilt.len() - offset; - for idx in 0..match_len { - let b = rebuilt[start + idx]; - rebuilt.push(b); - } - } - }); - - // Whether THIS specific iteration produces a Triple depends on - // the matcher's step-skip schedule (upstream zstd-shape kernel walks ip0 - // with kSearchStrength-driven stride growth) — the legacy - // SuffixStore-based matcher iterated every position and always - // hit short repeats, but the upstream zstd-shape kernel may skip over - // them when the step has grown large by the time it reaches the - // repeat region. The substance of this test is the - // reconstruction assertion below; `saw_triple` was a legacy - // tuning preference, not a correctness invariant. - let _ = saw_triple; - assert_eq!(rebuilt, data); -} - -#[test] -fn fast_levels_dispatch_per_level_hash_log_and_mls() { - // Level 1 — upstream zstd `{ 19, 13, 14, 1, 7, 0, ZSTD_fast }` row: - // window_log=19, hash_log=14, mls=7. - let f1 = resolve_level_params(CompressionLevel::Level(1), None) - .fast - .unwrap(); - assert_eq!(f1.hash_log, 14); - assert_eq!(f1.mls, 7); - assert_eq!(f1.step_size, 2); - - // Negative levels — upstream zstd row-0 ("base for negative"): - // hash_log=13, mls=7. The 32 KiB table is L1d-resident (every - // probe an L1 hit, vs an L2 access for a 64 KiB hash_log=14 - // table), and minMatch=7 drops short-distance 6-byte matches — - // upstream zstd parity on both ratio and throughput. - // step_size follows upstream zstd's formula: targetLength = -level, - // step_size = (-level) + 1, giving 2..8 for L-1..L-7. - for n in -7..=-1 { - let f = resolve_level_params(CompressionLevel::Level(n), None) - .fast - .unwrap(); - assert_eq!(f.hash_log, 13, "Level({n}) fast_hash_log"); - assert_eq!(f.mls, 7, "Level({n}) fast_mls"); - let expected_step = ((-n) as usize) + 1; - assert_eq!(f.step_size, expected_step, "Level({n}) fast_step_size"); - } - - // Fastest + Uncompressed keep hash_log=14 / mls=6 (their own - // tuning; not part of the negative-level upstream zstd ladder). - let pf = resolve_level_params(CompressionLevel::Fastest, None); - let ff = pf.fast.unwrap(); - assert_eq!( - (pf.window_log, ff.hash_log, ff.mls, ff.step_size), - (19, 14, 6, 2), - ); - // Uncompressed keeps window_log=17 (no history references, smaller - // decoder reservation); fast cParams same as negative-base row. - let pu = resolve_level_params(CompressionLevel::Uncompressed, None); - let fu = pu.fast.unwrap(); - assert_eq!( - (pu.window_log, fu.hash_log, fu.mls, fu.step_size), - (17, 14, 6, 2), - ); -} - -/// Exercise the actual driver wiring: for every Fast level, reset a -/// `MatchGeneratorDriver` and assert the inner `FastKernelMatcher` -/// observed the same `(hash_log, mls, step_size)` tuple that -/// `resolve_level_params` reports. Catches plumbing bugs — argument -/// reordering, stale step_size carried from a prior frame, -/// stuck-on-default values — that the parameter-only test above -/// would miss. -#[test] -fn fast_levels_driver_wiring_threads_cparams_into_inner_matcher() { - let mut driver = MatchGeneratorDriver::new(64 * 1024, 1); - - let fast_levels = [ - CompressionLevel::Level(1), - CompressionLevel::Fastest, - CompressionLevel::Uncompressed, - CompressionLevel::Level(-1), - CompressionLevel::Level(-2), - CompressionLevel::Level(-3), - CompressionLevel::Level(-4), - CompressionLevel::Level(-5), - CompressionLevel::Level(-6), - CompressionLevel::Level(-7), - ]; - - for &level in &fast_levels { - let p = resolve_level_params(level, None); - // Sanity: every level in the table above must resolve to a - // Fast-strategy row — otherwise this test isn't testing what - // it claims to test. - assert_eq!( - p.strategy_tag, - super::strategy::StrategyTag::Fast, - "{level:?} must resolve to Fast strategy", - ); - - // Bounce through a non-Fast strategy first so the next - // reset actually goes through the backend-switch path - // (`MatchGeneratorDriver::new` / `simple_mut` recreate the - // Fast variant via `FastKernelMatcher::with_params`). Without - // this hop the loop would only ever stay in `BackendTag::Simple` - // and exercise `FastKernelMatcher::reset` — leaving the - // `with_params` wiring untested on the production path. - // `Default` resolves to Dfast strategy (a non-Fast row), - // which is enough to force the swap. - crate::encoding::Matcher::reset(&mut driver, CompressionLevel::Default); - - // Drive the production reset path (same code paths exercised - // by FrameCompressor / StreamingEncoder). - crate::encoding::Matcher::reset(&mut driver, level); - - let f = p.fast.unwrap(); - let m = driver.simple_mut(); - assert_eq!( - m.hash_log(), - f.hash_log, - "{level:?}: inner matcher hash_log mismatch — argument swap?", - ); - assert_eq!( - m.mls(), - f.mls, - "{level:?}: inner matcher mls mismatch — argument swap?", - ); - assert_eq!( - m.step_size(), - f.step_size, - "{level:?}: inner matcher step_size mismatch — stale value carried from prior reset?", - ); - } -} - -/// Pins `hc.target_len` to the reference `cParams.targetLength` from -/// `clevels.h` table[0] (default — `srcSize > 256 KB`) across levels -/// 5-15. The reference's lazy outer loop treats `targetLength` as -/// `sufficient_len` — the "nice match" threshold that breaks the chain -/// walk as soon as a candidate reaches that length. -/// -/// Levels 13-15 run btlazy2 in the reference and the hash-chain Lazy -/// parser here, but the reference `targetLength` (32) is the same nice-match -/// threshold for both finders, so we mirror it directly. -/// -/// Asserts against the constant `clevels.h` table[0] `targetLength` column -/// (transcribed inline) — a pure-Rust in-tree test, no FFI dependency. -#[test] -fn lazy_band_target_len_matches_default_table() { - // table[0] (srcSize > 256 KB) targetLength, levels 5..=15: the lazy - // outer loop's nice-match (`sufficient_len`) threshold. - let expected: [(i32, usize); 11] = [ - (5, 2), - (6, 4), - (7, 8), - (8, 16), - (9, 16), - (10, 16), - (11, 16), - (12, 32), - (13, 32), - (14, 32), - (15, 32), - ]; - for (level, want) in expected { - let params = resolve_level_params(CompressionLevel::Level(level), None); - // L5 = greedy (Row backend → `row`); L6-15 = lazy (HashChain → `hc`). - let target_len = params - .hc - .map(|hc| hc.target_len) - .or_else(|| params.row.map(|row| row.target_len)) - .expect("lazy/greedy level carries hc or row config"); - assert_eq!(target_len, want, "L{level}: target_len must match table[0]"); - } -} - -/// Levels 13-15 mirror the reference btlazy2 window/hash/chain/search -/// budget from `clevels.h` table[0]: `search_depth == 1 << cParams.searchLog` -/// (16 / 32 / 64) plus `window_log` / `hash_log` / `chain_log` equal to the -/// reference `windowLog` / `hashLog` / `chainLog`. We run them on the -/// hash-chain Lazy parser rather than a binary-tree finder, so they do not -/// re-establish a strict ratio ladder above L12 on window-fitting inputs; -/// asserting the full row (not just `search_depth`) keeps the whole budget -/// aligned and guards every field against silent drift. -#[test] -fn upper_lazy_band_params_match_default_table() { - // table[0] (srcSize > 256 KB), levels 13..=15 (btlazy2 budget): - // (level, windowLog, hashLog, chainLog, search_depth = 1 << searchLog). - let expected: [(i32, u8, usize, usize, usize); 3] = [ - (13, 22, 22, 22, 1 << 4), - (14, 22, 23, 22, 1 << 5), - (15, 22, 23, 23, 1 << 6), - ]; - for (level, wlog, hlog, clog, sd) in expected { - let params = resolve_level_params(CompressionLevel::Level(level), None); - let hc = params.hc.unwrap(); - assert_eq!(hc.search_depth, sd, "L{level}: search_depth"); - assert_eq!(params.window_log, wlog, "L{level}: window_log"); - assert_eq!(hc.hash_log, hlog, "L{level}: hash_log"); - assert_eq!(hc.chain_log, clog, "L{level}: chain_log"); - } -} diff --git a/zstd/src/encoding/match_generator/dict_prime.rs b/zstd/src/encoding/match_generator/dict_prime.rs new file mode 100644 index 000000000..656081d9e --- /dev/null +++ b/zstd/src/encoding/match_generator/dict_prime.rs @@ -0,0 +1,522 @@ +//! Dictionary priming and CDict-equivalent snapshot lifecycle for the +//! [`MatchGeneratorDriver`]. +//! +//! Holds the matcher's dictionary entry points (prime / restore / capture / +//! invalidate / resident reapply / entropy seed) as inherent `*_impl` helpers; +//! the parent module's trait surface forwards to them. Kept as a child module +//! so the snapshot path can touch the driver's private invariant-bearing +//! `primed` / `reset_shape` fields directly without widening them to the rest +//! of the encoding module. + +use super::{MIN_MATCH_LEN, MatchGeneratorDriver, Matcher, MatcherStorage, PrimedKey}; + +impl MatchGeneratorDriver { + pub(super) fn dictionary_is_resident_impl(&self) -> bool { + match &self.storage { + MatcherStorage::HashChain(hc) => hc.table.dict_resident, + MatcherStorage::Simple(s) => s.dict_resident(), + MatcherStorage::Dfast(d) => d.dict_resident(), + _ => false, + } + } + + pub(super) fn reapply_resident_dictionary_impl(&mut self, offset_hist: [u32; 3]) { + // Same offset-history head as `prime_with_dictionary`, without the dict + // commit / re-index (resident dict bytes + cached dms already in place). + match self.active_backend() { + super::super::strategy::BackendTag::Simple => { + self.simple_mut().prime_offset_history(offset_hist) + } + super::super::strategy::BackendTag::Dfast => { + self.dfast_matcher_mut().offset_hist = offset_hist + } + super::super::strategy::BackendTag::Row => { + self.row_matcher_mut().offset_hist = offset_hist + } + super::super::strategy::BackendTag::HashChain => { + let matcher = self.hc_matcher_mut(); + matcher.table.offset_hist = offset_hist; + matcher.table.mark_dictionary_primed(); + } + } + // Restore the retained-dictionary budget the per-frame `reset` cleared. + // The matcher's `reset` re-inflated `max_window_size` by the resident + // dict region (so the dict + next input both stay in the eviction band), + // exactly as `prime_with_dictionary` does — but the resident path skips + // that prime, so without this the driver-level budget stays 0 and + // `retire_dictionary_budget` never shrinks the inflated window as input + // evicts the dict. For HashChain (whose `window_low` is measured against + // `max_window_size`), a stuck-inflated window would let a post-eviction + // match exceed the frame header's base window and emit an over-window + // offset. The inflation equals `max_window_size - base`, and + // `reported_window_size` is the base `1 << window_log` set by `reset`. + let base = self.reported_window_size; + let inflated = match self.active_backend() { + super::super::strategy::BackendTag::Simple => self.simple_mut().max_window_size, + super::super::strategy::BackendTag::Dfast => self.dfast_matcher_mut().max_window_size, + super::super::strategy::BackendTag::Row => self.row_matcher_mut().max_window_size, + super::super::strategy::BackendTag::HashChain => { + self.hc_matcher_mut().table.max_window_size + } + }; + self.dictionary_retained_budget = inflated.saturating_sub(base); + } + + pub(super) fn prime_with_dictionary_impl( + &mut self, + dict_content: &[u8], + offset_hist: [u32; 3], + ) { + match self.active_backend() { + super::super::strategy::BackendTag::Simple => { + // Routes through prime_offset_history so BOTH + // offset_hist (wire encoder) and rep[0..2] (kernel) + // are updated atomically. Without this, the two + // tracks drift after dict priming — kernel emits + // repcode matches against stale FAST_INITIAL_REP + // while the wire encoder uses the primed history, + // producing divergent wire encoding (Copilot review + // #15 on #216). + self.simple_mut().prime_offset_history(offset_hist); + } + super::super::strategy::BackendTag::Dfast => { + self.dfast_matcher_mut().offset_hist = offset_hist + } + super::super::strategy::BackendTag::Row => { + self.row_matcher_mut().offset_hist = offset_hist + } + super::super::strategy::BackendTag::HashChain => { + let matcher = self.hc_matcher_mut(); + // Clear the chain/hash tables (deferred from the dict-active + // `reset`): prime rebuilds them from the dict, so they must start + // empty. The reuse hot path skips prime and `clone_from`s a clean + // snapshot instead, so only the first-prime / key-mismatch frames + // pay the fill -- not every reused-CDict frame. + matcher.table.clear_chain_hash_tables(); + matcher.table.offset_hist = offset_hist; + matcher.table.mark_dictionary_primed(); + } + } + + if dict_content.is_empty() { + return; + } + + // Dictionary bytes should stay addressable until produced frame output + // itself exceeds the live window size. We bump `max_window_size` + // by the dictionary length so the eviction band keeps the + // primed bytes in `history`. + // + // Cap: `with_params`/`reset` enforce `window_log <= 30` so the + // eviction band `2 * max_window_size` stays below `u32::MAX` + // with headroom for one MAX_BLOCK_SIZE pending block — the + // kernel asserts `data.len() <= u32::MAX`. A large enough + // dictionary could otherwise push `max_window_size` past + // that ceiling via the `saturating_add` below and silently + // re-introduce the same overflow the `window_log` cap was + // designed to prevent. Clamp the post-priming size so the + // doubled-band-plus-block invariant survives. + use super::super::match_table::storage::MAX_PRIMED_WINDOW_SIZE; + + // `requested_dict_budget` is what the caller asked for; + // `base_max_window_size` snapshots the pre-priming cap so we + // can compute how much window the cap actually GRANTED below. + // The cap may clip the requested growth, in which case the + // bookkeeping (`dictionary_retained_budget` retire path) must + // track only the granted portion — otherwise + // `retire_dictionary_budget()` would later reclaim more than + // was actually added and shrink the matcher below its real + // base window (and `cap = 2 * max_window_size` would shrink + // with it, risking under-allocation on subsequent commits). + // The `granted_retained_budget` calculation further below is + // the load-bearing piece — see its block-level comment for + // the post-clip / post-uncommitted-tail math. + let requested_dict_budget = dict_content.len(); + let base_max_window_size = match self.active_backend() { + super::super::strategy::BackendTag::Simple => self.simple_mut().max_window_size, + super::super::strategy::BackendTag::Dfast => self.dfast_matcher_mut().max_window_size, + super::super::strategy::BackendTag::Row => self.row_matcher_mut().max_window_size, + super::super::strategy::BackendTag::HashChain => { + self.hc_matcher_mut().table.max_window_size + } + }; + match self.active_backend() { + super::super::strategy::BackendTag::Simple => { + let matcher = self.simple_mut(); + matcher.max_window_size = matcher + .max_window_size + .saturating_add(requested_dict_budget) + .min(MAX_PRIMED_WINDOW_SIZE); + } + super::super::strategy::BackendTag::Dfast => { + let matcher = self.dfast_matcher_mut(); + matcher.max_window_size = matcher + .max_window_size + .saturating_add(requested_dict_budget) + .min(MAX_PRIMED_WINDOW_SIZE); + } + super::super::strategy::BackendTag::Row => { + let matcher = self.row_matcher_mut(); + matcher.max_window_size = matcher + .max_window_size + .saturating_add(requested_dict_budget) + .min(MAX_PRIMED_WINDOW_SIZE); + } + super::super::strategy::BackendTag::HashChain => { + let matcher = self.hc_matcher_mut(); + matcher.table.max_window_size = matcher + .table + .max_window_size + .saturating_add(requested_dict_budget) + .min(MAX_PRIMED_WINDOW_SIZE); + } + } + + let mut start = 0usize; + let mut committed_dict_budget = 0usize; + // insert_position needs 4 bytes of lookahead for hashing; + // backfill_boundary_positions re-visits tail positions once the + // next slice extends history, but cannot hash <4 byte fragments. + let min_primed_tail = match self.active_backend() { + super::super::strategy::BackendTag::Simple => MIN_MATCH_LEN, + super::super::strategy::BackendTag::Dfast + | super::super::strategy::BackendTag::Row + | super::super::strategy::BackendTag::HashChain => 4, + }; + while start < dict_content.len() { + let end = (start + self.slice_size).min(dict_content.len()); + if end - start < min_primed_tail { + break; + } + // Stage the dict chunk WITHOUT `get_next_space`'s + // `resize(slice_size, 0)` zero-fill: that memsets a full + // block-sized buffer (up to ~128 KiB) every frame only to have it + // `clear()`-ed and overwritten by the dict bytes on the very next + // lines — pure waste (measured ~10% of the small dict encode). + // Reuse a pooled buffer's capacity if one is free (the prime/skip + // cycle recycles them back), else allocate exactly the chunk. + // Mirrors upstream zstd, which references the CDict content rather + // than zero-filling a fresh window per frame. + let mut space = self.vec_pool.pop().unwrap_or_default(); + space.clear(); + space.extend_from_slice(&dict_content[start..end]); + self.commit_space(space); + self.skip_matching_for_dictionary_priming(); + committed_dict_budget += end - start; + start = end; + } + + // Derive `granted_retained_budget` directly from the two real + // bounds — bytes actually committed and bytes the cap allows + // — instead of doing a cap-clip pass followed by an + // uncommitted-tail subtract. Previous shape double-discounted + // when the cap clipped: clip lost `(requested - allowed)`, + // then tail-subtract lost ANOTHER `(requested - committed)`, + // leaving `max_window_size` shy of the dictionary that was + // actually retained (e.g. cap=900, committed=998, uncommitted=2 + // landed at granted=898 instead of the correct 900). + let capped_retained_budget = MAX_PRIMED_WINDOW_SIZE.saturating_sub(base_max_window_size); + let granted_retained_budget = committed_dict_budget.min(capped_retained_budget); + let final_max_window_size = base_max_window_size.saturating_add(granted_retained_budget); + match self.active_backend() { + super::super::strategy::BackendTag::Simple => { + self.simple_mut().max_window_size = final_max_window_size; + } + super::super::strategy::BackendTag::Dfast => { + self.dfast_matcher_mut().max_window_size = final_max_window_size; + } + super::super::strategy::BackendTag::Row => { + self.row_matcher_mut().max_window_size = final_max_window_size; + } + super::super::strategy::BackendTag::HashChain => { + self.hc_matcher_mut().table.max_window_size = final_max_window_size; + } + } + if granted_retained_budget > 0 { + self.dictionary_retained_budget = self + .dictionary_retained_budget + .saturating_add(granted_retained_budget); + } + if self.active_backend() == super::super::strategy::BackendTag::HashChain { + // Recompute the lazy-HC attach decision made per-chunk in + // `skip_matching_for_dictionary_priming` (stable across the prime — + // `reset_size_log` does not change here). + // + // The HC attach/copy mode is deliberately NOT folded into `PrimedKey` + // (unlike Fast `fast_attach`). Fast attach builds a separate dict + // table whose dimensions differ from the copy-mode live table, so a + // cross-mode restore would install mismatched table geometry and the + // encoder could search past the frame window (undecodable). The two + // HC modes share identical window geometry: `max_window_size` and the + // dictionary limit are both set ABOVE this branch (the same value in + // either mode), and the live chain table dimensions come from the + // resolved `params` the key already pins. The modes differ only in + // WHERE the committed dict lives — a single-link `dms` (attach) vs + // merged into the live chain (copy) — both producing valid matches at + // in-window offsets. Upstream zstd makes the same observation: attach + // (`ZSTD_resetCCtx_byAttachingCDict`) and copy + // (`ZSTD_resetCCtx_byCopyingCDict`) both keep the caller's + // `windowLog`; the choice is a memory/speed trade-off, not a wire + // contract. So restoring an attach snapshot where this frame would + // have copied (or vice versa) yields a decodable frame that may only + // differ in which matches are found (ratio) — algorithmic freedom, not + // a defect. Keying on the mode would instead force a re-prime across + // the cutoff, re-adding the per-frame cost this snapshot path removes. + // + // In practice the public reuse path (`compress_independent_frame`) + // only ever captures AND restores the COPY-mode snapshot — capture is + // gated on the above-cutoff source size, so a restored frame always + // matches the captured mode. `hc_dict_snapshot_reuse_roundtrips` pins + // that same-mode reuse decodes; the driver-level cross-mode restore is + // accepted (not refused) per + // `primed_snapshot_fast_attach_does_not_over_key_non_simple_backends`. + let attach = self.hc_dict_attach_mode(); + let table = &mut self.hc_matcher_mut().table; + table.set_dictionary_limit_from_primed_bytes(committed_dict_budget); + // Build the dictMatchState over the committed dict (front of history) + // so `find_best_match` dual-probes it with its own compare budget — + // but ONLY in ATTACH mode. BT/optimal attach → DUBT dms; lazy-HC + // attach → single-link hash-chain dms. COPY mode (large known source, + // both BT and lazy-HC) already merged the dict into the live tree / + // chain in `skip_matching_for_dictionary_priming`, so it carries no + // separate dms — drop any stale one. + if !attach { + table.dms.invalidate(); + } else if table.uses_bt { + table.prime_dms_bt(committed_dict_budget); + } else { + table.prime_dms_hc(committed_dict_budget); + } + } + // CDict-equivalent: now that every dict chunk is indexed, mark the + // Fast-backend dict table primed so the next frame's re-prime reuses + // it (skips the re-hash) while still re-committing the dict bytes to + // history. No-op when the attach path built no table (copy mode or a + // sub-8-byte dict) — `mark_dict_primed` self-guards on table presence. + match self.active_backend() { + super::super::strategy::BackendTag::Simple => self.simple_mut().mark_dict_primed(), + super::super::strategy::BackendTag::Dfast => { + self.dfast_matcher_mut().mark_dict_primed() + } + super::super::strategy::BackendTag::Row => self.row_matcher_mut().mark_dict_primed(), + _ => {} + } + } + + pub(super) fn restore_primed_dictionary_impl( + &mut self, + level: super::super::CompressionLevel, + ) -> bool { + // Only the (storage, dictionary_retained_budget) pair is what + // `prime_with_dictionary` writes; restoring them reproduces the + // post-prime state exactly. Gated on the FULL resolved key (level + the + // resolved `LevelParams` + the active backend's table width), not just + // the level: `reset` resolves the hint into a window/table geometry, so a + // same-level snapshot taken at a hint that resolved to a different shape + // carries a `storage.max_window_size` / table dimensions that no longer + // match this reset. Restoring it would let the encoder search past the + // frame header's window (an undecodable match), so on a key mismatch we + // refuse and the caller re-primes. + let Some((params, table_bits, fast_attach, ldm)) = self.reset_shape else { + return false; + }; + let key = PrimedKey { + level, + params, + table_bits, + fast_attach, + ldm, + }; + let Some((snapshot, budget, captured_key)) = &self.primed else { + return false; + }; + if *captured_key != key { + return false; + } + let budget = *budget; + match (&mut self.storage, snapshot) { + // Same-variant Fast restore: copy the snapshot into the retained + // live storage. `clone_from` reuses the history / hash-table / + // dict-table buffers, so this is the upstream zstd CDict table-copy + // regime's cost (pure copies) instead of a full per-frame + // allocation + copy + drop cycle. + (MatcherStorage::Simple(live), MatcherStorage::Simple(snap)) => { + live.clone_from(snap); + } + // Same-variant HC lazy/greedy restore (non-BT): the snapshot keeps + // the full primed hash/chain tables (capture's non-BT full clone), + // so `clone_from` reuses the live history/hash/chain/dms buffers in + // place — upstream zstd reuses the CDict tables rather than reallocating + // them. This is the per-frame allocate+copy+drop that dominated + // small `compress-dict` HC frames (5-7x vs C). BT (`uses_bt`) + // snapshots drop their live tables, so they stay on the realloc + // path below. + (MatcherStorage::HashChain(live), MatcherStorage::HashChain(snap)) + if !snap.table.uses_bt => + { + live.table.clone_from(&snap.table); + live.hc.clone_from(&snap.hc); + live.strategy_tag = snap.strategy_tag; + // backend is `HcBackend::Hc` (zero-sized) for non-BT levels; + // the live one is already correct for this resolved key. + } + (live, snapshot_storage) => { + let mut storage = snapshot_storage.clone(); + // This arm handles the binary-tree backend. In ATTACH mode the + // snapshot was stored WITHOUT its live hash / chain / hash3 + // tables (they hold no dictionary entries — the dict lives in + // `dms` + history; see `capture_primed_dictionary`), so + // `ensure_tables` re-allocates them zeroed to the snapshot's + // geometry, exactly reproducing the post-prime state (all + // `HC_EMPTY`). In COPY mode the snapshot retained its FULL live + // tree (the dict was merged into it, no `dms`), so the tables are + // already present at the right length and `ensure_tables` — which + // only allocates on a length mismatch — leaves them untouched. + // Either way this is a full storage replace, so no stale + // live-table entry from a prior frame can survive. + if let MatcherStorage::HashChain(hc) = &mut storage { + hc.table.ensure_tables(); + } + // The snapshot does not retain the LDM producer (it holds no + // dict state; see `capture_primed_dictionary`). Carry over the + // frame's freshly-reset producer — built this frame by `reset` + // with the same params the snapshot key pins, and empty (no + // input processed yet), so it is equivalent to the producer + // the snapshot was captured with. + #[cfg(feature = "hash")] + { + let fresh_ldm = if let MatcherStorage::HashChain(hc) = live { + hc.take_ldm_producer() + } else { + None + }; + if let MatcherStorage::HashChain(hc) = &mut storage { + hc.set_ldm_producer(fresh_ldm); + } + } + *live = storage; + } + } + self.dictionary_retained_budget = budget; + true + } + + pub(super) fn capture_primed_dictionary_impl(&mut self, level: super::super::CompressionLevel) { + // No resolved shape means `reset` has not run for this frame — nothing + // valid to key a snapshot on, so skip the capture. + let Some((params, table_bits, fast_attach, ldm)) = self.reset_shape else { + return; + }; + let key = PrimedKey { + level, + params, + table_bits, + fast_attach, + ldm, + }; + // CDict-equivalent retained state. A binary-tree level in ATTACH mode + // decouples the dictionary into `dms` (the upstream zstd `dictMatchState`); its + // live hash / chain / hash3 tables carry NO dict entries + // (`skip_matching_dict_bt` keeps the dict out of the live tree), so they + // are pure zeros. Storing them in the snapshot wastes the full table + // footprint (a second window-tier table set resident for the whole + // compress). Instead, move the live tables OUT of the working storage, + // clone only the dict-state (history + `dms` + window/offset/dict-limit), + // then move the live tables back — the snapshot keeps just what upstream zstd's + // CDict keeps, and `restore_primed_dictionary` re-allocates the zeroed + // live tables. Every other case keeps the dict reachable through the live + // structure, so the snapshot must retain the full tables (full clone): + // lazy-HC attach (it DOES prime a hash-chain `dms`, but the live chain is + // still the search structure, so the tables must travel) and COPY mode for + // BOTH BT and lazy-HC (`dms` invalidated, dict merged into the live tree / + // chain). `uses_bt && dms.is_primed()` is therefore the exact "decoupled" + // signal — true only for the BT attach prime; lazy-HC attach primes `dms` + // too but is intentionally NOT decoupled. + let bt_decoupled = matches!( + &self.storage, + MatcherStorage::HashChain(hc) if hc.table.uses_bt && hc.table.dms.is_primed() + ); + if bt_decoupled { + let MatcherStorage::HashChain(hc) = &mut self.storage else { + unreachable!("bt_decoupled implies HashChain storage"); + }; + let hash_table = core::mem::take(&mut hc.table.hash_table); + let chain_table = core::mem::take(&mut hc.table.chain_table); + let hash3_table = core::mem::take(&mut hc.table.hash3_table); + // The LDM producer carries no dictionary state (LDM is not + // dict-primed; its hash table is empty at capture), so it is not + // retained either — `restore` reinstates the frame's freshly-reset + // producer. Take it out so the clone does not duplicate its table. + #[cfg(feature = "hash")] + let ldm_producer = hc.take_ldm_producer(); + // Clone the dict-state-only storage (live tables now empty Vecs, + // LDM producer detached). + let snapshot = self.storage.clone(); + // Move the live tables (and LDM producer) back into the working storage. + let MatcherStorage::HashChain(hc) = &mut self.storage else { + unreachable!("storage variant is stable across the take/put"); + }; + hc.table.hash_table = hash_table; + hc.table.chain_table = chain_table; + hc.table.hash3_table = hash3_table; + #[cfg(feature = "hash")] + hc.set_ldm_producer(ldm_producer); + self.primed = Some((snapshot, self.dictionary_retained_budget, key)); + } else { + self.primed = Some((self.storage.clone(), self.dictionary_retained_budget, key)); + } + } + + pub(super) fn invalidate_primed_dictionary_impl(&mut self) { + self.primed = None; + // Drop the Fast-backend CDict-equivalent table cache too: it is keyed + // to the dictionary being removed / replaced. Left in place, the next + // same-params `reset` would retain it and the kernel would probe a + // dict region whose bytes are no longer re-committed to history. + match self.active_backend() { + super::super::strategy::BackendTag::Simple => self.simple_mut().invalidate_dict_cache(), + super::super::strategy::BackendTag::Dfast => { + self.dfast_matcher_mut().invalidate_dict_cache() + } + // Row keeps its attach index across frames (like Simple/Dfast), + // so a dictionary swap must drop its cached dict rows too; + // otherwise the next small/unknown-size frame reuses stale + // attach state through `prime_dict_attach_current_block`. + super::super::strategy::BackendTag::Row => { + self.row_matcher_mut().invalidate_dict_cache() + } + // The BT dms tree is keyed to the dict bytes; `prime_dms_bt` + // skips the rebuild while its shape matches, so a swapped + // dictionary of the same length would otherwise keep serving the + // OLD dictionary's tree. + super::super::strategy::BackendTag::HashChain => { + let table = &mut self.hc_matcher_mut().table; + table.dms.invalidate(); + // Deactivate the dictionary state so the next `reset` does not + // take the dict-active defer-the-table-clear branch. That branch + // rewinds the tables to the origin and hands the clear off to a + // following `prime_with_dictionary` / `restore_primed_dictionary`. + // After a dictionary is removed (or replaced), the very next + // frame may carry no dictionary, in which case neither hand-off + // runs and the deferred clear would never execute — leaving stale + // dict-region entries at the rewound base. Clearing the flag + // routes that reset down the no-dictionary path instead; a + // replacement dictionary re-arms the flag when it re-primes. + table.dictionary_active = false; + } + } + } + + pub(super) fn seed_dictionary_entropy_impl( + &mut self, + huff: Option<&crate::huff0::huff0_encoder::HuffmanTable>, + ll: Option<&crate::fse::fse_encoder::FSETable>, + ml: Option<&crate::fse::fse_encoder::FSETable>, + of: Option<&crate::fse::fse_encoder::FSETable>, + ) { + if self.active_backend() == super::super::strategy::BackendTag::HashChain { + self.hc_matcher_mut() + .seed_dictionary_entropy(huff, ll, ml, of); + } + } +} diff --git a/zstd/src/encoding/match_generator/mod.rs b/zstd/src/encoding/match_generator/mod.rs new file mode 100644 index 000000000..d992ffd42 --- /dev/null +++ b/zstd/src/encoding/match_generator/mod.rs @@ -0,0 +1,2153 @@ +//! Matching algorithm used find repeated parts in the original data +//! +//! The Zstd format relies on finden repeated sequences of data and compressing these sequences as instructions to the decoder. +//! A sequence basically tells the decoder "Go back X bytes and copy Y bytes to the end of your decode buffer". +//! +//! The task here is to efficiently find matches in the already encoded data for the current suffix of the not yet encoded data. + +use alloc::vec::Vec; +// SIMD/CRC intrinsics now live in `crate::encoding::fastpath::*` where they +// sit under per-CPU `#[target_feature]` umbrellas; no architecture-specific +// intrinsic imports remain in this file. +use super::CompressionLevel; +use super::Matcher; +use super::Sequence; +use super::cost_model::HC_FORMAT_MINMATCH; +#[cfg(test)] +use super::cost_model::HC_MAX_LIT; +#[cfg(test)] +use super::cost_model::{ + HC_BITCOST_MULTIPLIER, HC_OPT_NUM, HC_PREDEF_THRESHOLD, HcOptState, HcOptimalCostProfile, +}; +#[cfg(test)] +use super::cost_model::{HC_BLOCKSIZE_MAX, HC_MAX_LL, HC_MAX_ML, HC_MAX_OFF, HcOptPriceType}; +use super::dfast::DfastMatchGenerator; +#[cfg(test)] +use super::hc::HC_MIN_MATCH_LEN; +#[cfg(test)] +use super::match_table::storage::HC3_HASH_LOG; +// FAST_HASH_FILL_STEP test-only re-export was tied to the legacy +// SuffixStore MatchGenerator's interleaved hash-fill stride. The +// upstream zstd-shape Fast kernel walks ip0 with kSearchStrength step-skip +// acceleration instead, so the constant has no consumer in the +// remaining live test set today. +#[cfg(test)] +use super::match_table::helpers::INCOMPRESSIBLE_SKIP_STEP; +use super::match_table::helpers::MIN_MATCH_LEN; +#[cfg(test)] +use super::match_table::helpers::common_prefix_len; +#[cfg(test)] +use super::opt::ldm::HcRawSeq; +#[cfg(test)] +use super::opt::types::{HcCandidateQuery, MatchCandidate}; +use super::row::RowMatchGenerator; +use super::simple::fast_matcher::{FAST_LEVEL_1_HASH_LOG, FAST_LEVEL_1_MLS, FastKernelMatcher}; +#[cfg(all( + test, + feature = "std", + target_arch = "aarch64", + target_endian = "little" +))] +use std::arch::is_aarch64_feature_detected; +#[cfg(all(test, feature = "std", target_arch = "x86_64"))] +use std::arch::is_x86_feature_detected; + +pub(crate) const DFAST_MIN_MATCH_LEN: usize = 5; +// Bytes the dfast short hash reads (upstream zstd `mls = 5`). Seeding / lookahead +// guards use it so a position is only short-hashed once its full 5-byte key +// is in range. +pub(crate) const DFAST_SHORT_HASH_LOOKAHEAD: usize = 5; +pub(crate) const ROW_MIN_MATCH_LEN: usize = 5; +// Upstream zstd `clevels.h:31` at level 3 large-input bucket sets +// `hashLog = 17` (the long-hash table) and `chainLog = 16` (the +// short-hash table — upstream zstd names this `chainTable` even though for +// dfast it's used as a plain single-slot hash). Each table holds one +// `U32` per slot; the upstream zstd overwrites on collision and recovers +// compression quality via the inline `_search_next_long` retry +// (after a short-hash hit, probes `hashLong[hl1]` at `ip + 1` and +// keeps the longer match). +// +// We mirror that storage layout: single `u32` per bucket (no +// `[u32; N]` array), `long_hash` sized `1 << DFAST_HASH_BITS` and +// `short_hash` one bit smaller via `DFAST_SHORT_HASH_BITS_DELTA`. +// Two-table footprint at Level 3: `2^17 × 4 + 2^16 × 4 = 768 KiB`, +// exact upstream parity. The `_search_next_long` retry lives in +// `DfastMatchGenerator::hash_candidate` (called via +// `best_match`). Earlier revisions kept a +// 4-slot bucket per hash position; that paid 4× the upstream zstd memory +// without measurable ratio gain once the retry was in place. +// +// `dfast_hash_bits_for_window` still clamps the runtime long-hash +// value to `[MIN_WINDOW_LOG, DFAST_HASH_BITS]`, so this const is the +// upper bound rather than a fixed default. +pub(crate) const DFAST_HASH_BITS: usize = 17; +/// Difference between `long_hash_bits` and `short_hash_bits` — +/// upstream zstd `hashLog - chainLog` is 1 at every dfast level (`clevels.h` +/// level 2: 16-15=1; level 3: 17-16=1). The short hash is one bit +/// smaller than the long hash so the per-bucket footprint matches +/// upstream zstd sizing exactly. +pub(crate) const DFAST_SHORT_HASH_BITS_DELTA: usize = 1; +/// Sentinel value for an empty slot in the dfast hash tables. Real +/// positions are stored as `(abs_pos - position_base + 1) as u32`, so +/// `0` is reserved as the "empty" marker and a true relative offset +/// of `0` never appears in the table. Mirrors the LDM table's +/// `LdmEntry.offset == 0` convention (see `encoding/ldm/table.rs`) +/// so both rebasing structures share +/// one sentinel scheme. +pub(crate) const DFAST_EMPTY_SLOT: u32 = 0; + +/// Guard band reserved above the high-water mark before triggering a +/// rebase on the Dfast hash tables. When the next insert would push a +/// relative offset above `u32::MAX - DFAST_REBASE_GUARD_BAND`, the +/// table calls `reduce(GUARD_BAND)` to shift every slot down and +/// advance `position_base` so future inserts stay inside the `u32` +/// window. Same scheme as `encoding/ldm/table.rs`. +pub(crate) const DFAST_REBASE_GUARD_BAND: u32 = 1u32 << 30; +pub(crate) const DFAST_SKIP_SEARCH_STRENGTH: usize = 6; +pub(crate) const DFAST_SKIP_STEP_GROWTH_INTERVAL: usize = 1 << DFAST_SKIP_SEARCH_STRENGTH; +pub(crate) const DFAST_MAX_SKIP_STEP: usize = 8; +pub(crate) const DFAST_INCOMPRESSIBLE_SKIP_STEP: usize = 16; +pub(crate) const ROW_HASH_BITS: usize = 20; +pub(crate) const ROW_LOG: usize = 5; +pub(crate) const ROW_SEARCH_DEPTH: usize = 16; +pub(crate) const ROW_TARGET_LEN: usize = 48; +pub(crate) const ROW_TAG_BITS: usize = 8; +pub(crate) const ROW_EMPTY_SLOT: u32 = u32::MAX; +pub(crate) const ROW_HASH_KEY_LEN: usize = 4; +// HASH_MIX_PRIME now lives in `crate::encoding::fastpath::scalar`; the four +// per-CPU `hash_mix_u64` variants share it via that module. +// HC_PRIME3BYTES / HC_PRIME4BYTES moved to match_table::storage +// alongside the hash helpers in Phase 1e Stage A. Only the test +// module references the constants directly (production code goes +// through `MatchTable::hash_value_with_mls`). +#[cfg(test)] +use super::match_table::storage::{HC_PRIME3BYTES, HC_PRIME4BYTES}; + +// HC_HASH_LOG / HC_CHAIN_LOG / HC3_HASH_LOG / HC_EMPTY live on the +// shared storage module so MatchTable methods can reference them +// without pulling in this module. Re-imported here so existing +// macros / configs / tests keep their unqualified names. +#[cfg(test)] +use super::match_table::storage::HC_EMPTY; +// HC3_MAX_OFFSET moved to encoding::bt alongside the hash3 candidate +// probe macro that consumes it; the macro references it via the +// fully-qualified `$crate::encoding::bt::HC3_MAX_OFFSET` path so this +// module no longer needs a local import. +pub(crate) const HC_SEARCH_DEPTH: usize = 16; +// HC_MIN_MATCH_LEN moved to encoding::hc; re-imported here so +// existing references compile unchanged. +pub(crate) const HC_OPT_MIN_MATCH_LEN: usize = HC_FORMAT_MINMATCH; +pub(crate) const HC_TARGET_LEN: usize = 48; + +// MAX_HC_SEARCH_DEPTH moved to encoding::hc alongside chain_candidates. +// Per-level tuning config (the config structs + `LEVEL_TABLE` + the +// level→params resolution chain) lives in `levels::config`; the driver imports +// that resolution API here. +use super::levels::config::*; +// The HashChain / BT match generator + its optimal-parse machinery lives in +// `hc::generator`; the driver stores it in the `HashChain` storage variant. +use super::hc::generator::HcMatchGenerator; + +// Dictionary prime + CDict-equivalent snapshot lifecycle. A child module so it +// can reach the driver's private `primed` / `reset_shape` state directly; the +// `Matcher` trait's dict entry points forward to its inherent `*_impl` helpers. +mod dict_prime; + +// `Strategy` and `StrategyTag` live in `crate::encoding::strategy`. +// The driver carries a `StrategyTag` field set at `reset()` and +// dispatches each block into a monomorphised `compress_block::` +// per concrete strategy. + +/// Backend storage for [`MatchGeneratorDriver`]. Exactly one match-finder +/// state lives in the driver at a time — the active variant. Backend +/// transitions in [`Matcher::reset`] drain the current variant's allocations +/// into the shared `vec_pool` and then replace `storage` with a freshly +/// constructed variant for the new backend. +/// +/// Replaces the prior pattern of four parallel fields (`match_generator`, +/// `dfast_match_generator: Option<…>`, `row_match_generator: Option<…>`, +/// `hc_match_generator: Option<…>`) + an `active_backend: BackendTag` +/// discriminator: the parallel layout kept drained inner structures +/// allocated across backend switches, and every per-frame/per-slice +/// driver operation had to dispatch on `active_backend` to pick the +/// right field. A single enum collapses the storage and makes the +/// dispatcher pattern-match on the storage variant directly — same +/// number of arms, but `storage.backend()` is now the canonical source +/// of truth and dead variants are dropped when the active backend +/// changes. +#[derive(Clone)] +enum MatcherStorage { + /// Upstream zstd `ZSTD_fast` family. Constructed by + /// [`MatchGeneratorDriver::new`] as the initial variant and + /// re-selected by [`Matcher::reset`] for any [`CompressionLevel`] + /// that `resolve_level_params` maps to [`StrategyTag::Fast`] + /// (`Uncompressed`, `Fastest`, `Level(1)`, and any non-positive + /// `Level(n)` not equal to `0`). + Simple(FastKernelMatcher), + /// Upstream zstd `ZSTD_dfast` family — two-table hash chain. Selected for + /// any level that resolves to [`StrategyTag::Dfast`] in + /// `resolve_level_params` (`Default`, `Level(0)`, `Level(2)`, + /// `Level(3)`). + Dfast(DfastMatchGenerator), + /// Upstream zstd `ZSTD_greedy` family with row hashing. Selected for any + /// level that resolves to [`StrategyTag::Greedy`] (currently + /// `Level(4)` only). + Row(RowMatchGenerator), + /// Upstream zstd `ZSTD_lazy2` and the BT-based optimal modes + /// (`btopt` / `btultra` / `btultra2`). Selected for any level that + /// resolves to [`StrategyTag::Lazy`], [`StrategyTag::BtOpt`], + /// [`StrategyTag::BtUltra`], or [`StrategyTag::BtUltra2`] + /// (`Better`, `Best`, `Level(5..=22)`, and any `Level(n)` with + /// `n > MAX_LEVEL` — `resolve_level_params` clamps positive + /// numeric levels at `MAX_LEVEL = 22` via + /// `Level(n).clamp(1, MAX_LEVEL)`, so `Level(23..=i32::MAX)` all + /// land on `BtUltra2` here). The [`HcMatchGenerator`]'s internal + /// [`HcBackend`] discriminator decides whether BT scratch is + /// allocated. + HashChain(HcMatchGenerator), +} + +impl MatcherStorage { + /// Heap bytes the active backend variant holds (tables, history, scratch). + fn heap_size(&self) -> usize { + match self { + Self::Simple(m) => m.heap_size(), + Self::Dfast(m) => m.heap_size(), + Self::Row(m) => m.heap_size(), + Self::HashChain(m) => m.heap_size(), + } + } + + /// [`super::strategy::BackendTag`] family of the active variant. + fn backend(&self) -> super::strategy::BackendTag { + use super::strategy::BackendTag; + match self { + Self::Simple(_) => BackendTag::Simple, + Self::Dfast(_) => BackendTag::Dfast, + Self::Row(_) => BackendTag::Row, + Self::HashChain(_) => BackendTag::HashChain, + } + } +} + +/// This is the default implementation of the `Matcher` trait. It allocates and reuses the buffers when possible. +pub struct MatchGeneratorDriver { + vec_pool: Vec>, + /// Active match-finder state. Exactly one backend lives here at a + /// time; [`Matcher::reset`] drains the previous variant into + /// `vec_pool` before swapping in a freshly constructed variant for + /// the new backend. `storage.backend()` is the canonical source of + /// truth for the parse family; `strategy_tag` carries the + /// compile-time strategy chosen at the last `reset()`. + storage: MatcherStorage, + // Compile-time strategy tag resolved at `reset()` from the + // requested `CompressionLevel`'s `LevelParams`. The driver's + // hot-block dispatcher in `blocks/compressed.rs` matches on + // this tag to enter the corresponding `Strategy` + // monomorphisation (`compress_block::`). + strategy_tag: super::strategy::StrategyTag, + // Decoupled search-method axis resolved at `reset()` from + // `LevelParams.search`. The per-block dispatcher routes on this + // (not on `strategy_tag`) so a level's parse and search backend can + // be chosen independently. The `BinaryTree` arm still consults + // `strategy_tag` to pick the opt `Strategy` ZST. + search: super::strategy::SearchMethod, + // Decoupled parse-mode axis resolved at `reset()` from + // `LevelParams::parse()`. Independent of `search`: greedy / lazy / + // lazy2 can run on any non-opt search backend. The backends still + // read their own `lazy_depth` (kept in sync at `reset()`); this is + // the authoritative parse selector for the dispatcher. + pub(crate) parse: super::strategy::ParseMode, + /// Test-only per-level recipe override applied in `reset()` before + /// backend selection. Lets the parse×search matrix be exercised + /// without editing `LEVEL_TABLE`; never compiled into production. + #[cfg(test)] + config_override: Option<(super::strategy::SearchMethod, super::strategy::ParseMode)>, + /// Fine-grained per-knob overrides from the public + /// [`super::parameters::CompressionParameters`] surface (#27). + /// `None` (or an all-`None` [`super::parameters::ParamOverrides`]) + /// keeps the resolved level geometry byte-identical to plain + /// level-based compression. Applied in [`Matcher::reset`] after the + /// level params are resolved, before backend selection. Persists + /// across resets (it is frame configuration, not a one-shot) until + /// the caller changes it. + param_overrides: Option, + slice_size: usize, + base_slice_size: usize, + // Frame header window size must stay at the configured live-window budget. + // Dictionary retention expands internal matcher capacity only. + reported_window_size: usize, + // Tracks currently retained bytes that originated from primed dictionary + // history and have not been evicted yet. + dictionary_retained_budget: usize, + // Source size hint for next frame (set via set_source_size_hint, cleared on reset). + source_size_hint: Option, + // Dictionary content size for the next frame (set via set_dictionary_size_hint, + // consumed on reset). When present on a binary-tree / hash-chain backend, the + // match-finder hash/chain tables are sized from the DICTIONARY (upstream zstd CDict + // economics: a loaded dictionary supplies the long matches, so the live tables + // can shrink to the dict's size tier) while the eviction window stays + // source-sized. Mirrors upstream zstd `ZSTD_getCParamRowSize`, which picks the cParams + // table column from `dictSize` for a dictionary-bearing compress. + dictionary_size_hint: Option, + // Normalized `ceil_log2` bucket of the frame's source-size hint, captured at + // `reset` (where `source_size_hint` is consumed) via [`source_size_ceil_log`]. + // `None` means the frame was unhinted. Drives `prime_with_dictionary`'s upstream zstd + // `ZSTD_shouldAttachDict` mode for the Simple/Fast backend: `None` (unknown) + // or `<= FAST_ATTACH_DICT_CUTOFF_LOG` → attach (separate dict table, 2-cursor + // `compress_block_fast_dict`); larger → copy (dictionary primed into the live + // table, 4-cursor `compress_block_fast`). The primed-snapshot key is the + // resolved shape ([`reset_shape`](Self::reset_shape)), not this bucket. + reset_size_log: Option, + // Whether the loaded dictionary fits the Fast attach path's tagged position + // field (`<= MAX_FAST_ATTACH_DICT_REGION`). Captured at `reset` from the + // dict-size hint (which equals the actual dict length on load) so the Fast + // attach decision, the attach-epoch reset bit, and the primed-snapshot + // `fast_attach` bit all gate on it consistently. `true` when there is no + // dictionary (the attach path is then unused). A dict too large to tag falls + // back to copy mode instead of overflowing the packed position. + reset_dict_attach_ok: bool, + // Hint-resolved matcher shape from the last `reset`: the [`LevelParams`], the + // active backend's applied Dfast/Row hash-table width (`0` for HC/Fast), the + // Fast attach-vs-copy mode, and the active LDM override (#27). Combined with + // the frame's level into the [`PrimedKey`] that keys the primed snapshot, so + // it is only restored into a reset that resolved the identical matcher AND + // LDM configuration. `None` before the first `reset`. + reset_shape: Option<( + LevelParams, + usize, + bool, + Option, + )>, + // One-shot borrowed block range `[start, end)` staged by the borrowed + // Fast frame path (`set_borrowed_block`) for the NEXT + // `start_matching` / `skip_matching_with_hint`. `Some` routes that + // call to the Simple backend's borrowed scan instead of the owned + // committed-block path; consumed (reset to `None`) by the routed + // call. Always `None` on the owned streaming path. + borrowed_pending: Option<(usize, usize)>, + /// CDict-equivalent: snapshot of the post-prime matcher state taken + /// once after the first dictionary prime — the backend `storage` + /// (hash tables + dictionary history + offset history + window) plus + /// the driver-level `dictionary_retained_budget`, the only two pieces + /// `prime_with_dictionary` writes. Subsequent frames restore this + /// (a table memcpy) instead of re-hashing every dictionary position, + /// mirroring upstream zstd `ZSTD_compressBegin_usingCDict` copying the + /// precomputed `cdict->matchState`. Invalidated when the dictionary + /// changes; keyed by the [`PrimedKey`] resolved matcher shape so a snapshot + /// is only restored into a reset that produces the same matcher — see + /// `restore_primed_dictionary`. + primed: Option<(MatcherStorage, usize, PrimedKey)>, +} + +/// Identity of the matcher configuration a primed snapshot was captured under: +/// the FULLY RESOLVED matcher shape, not the raw source-size hint. +/// +/// `reset()` resolves the hint into a [`LevelParams`] (window_log cap, the +/// HC/Fast table and search geometry, the parse depth/target-length that get +/// baked into the restored `storage`) plus, for the Dfast/Row backends, a +/// table-width derived from the hint's ceil-log bucket. The mapping from hint +/// to resolved shape is many-to-one: the source-size adjustment is monotone in +/// `ceil_log2(hint)`, and Level 22 additionally collapses several buckets onto +/// one upstream zstd tier (its `<= 16/128/256 KiB` thresholds). Keying on the raw hint +/// (or even its ceil-log bucket) therefore over-keys — two hints that resolve +/// to the identical matcher would each force a full re-prime. Keying on the +/// resolved (`params`, `table_bits`) pair restores across them. +/// +/// `table_bits` is the hint-dependent hash-table width the ACTIVE backend +/// applied (`set_hash_bits` value for Dfast/Row; `0` for HC/Fast, whose widths +/// already live in `params`). The snapshot is only ever captured on the COPY +/// path (a hinted, above-cutoff frame), so `table_bits` is always the resolved +/// Dfast/Row value there, never the unhinted default. +/// +/// `level` is kept alongside the resolved `params` because some stored matcher +/// state is derived from the level DIRECTLY, not through `params`: e.g. Dfast's +/// `use_fast_loop` is true for L3 but false for L4, yet L3 and L4 resolve to +/// byte-identical `params`. Without `level` a snapshot captured at L3 could be +/// restored into an L4 reset, installing the wrong `use_fast_loop`. +/// +/// `fast_attach` records the Fast backend's attach-vs-copy mode +/// ([`FAST_ATTACH_DICT_CUTOFF_LOG`]) because that cutoff (8 KiB) falls INSIDE a +/// single resolved shape: an 8192- and an 8193-byte Level 1 hint both clamp to +/// window_log 14 with identical `params`/`table_bits`, yet 8192 attaches (a +/// separate dict table) while 8193 copies into the live table — two different +/// `storage` shapes. The frame compressor only captures/restores snapshots on +/// the copy path today, but keying on the mode keeps the snapshot identity +/// self-sufficient rather than relying on that external gate. +/// +/// Restoring a snapshot whose key differs would reinstate the old `storage` +/// (and its `max_window_size` / table dimensions / parse params / dict-table +/// shape) under a reset that resolved a different shape — the encoder could +/// then search past the frame header's window and emit an undecodable match. +/// All fields must match before a restore is allowed. +#[derive(Clone, Copy, PartialEq, Eq)] +struct PrimedKey { + level: super::CompressionLevel, + params: LevelParams, + table_bits: usize, + fast_attach: bool, + /// Fine-grained LDM override (#27) active at capture time. The + /// snapshot's cloned `storage` carries `BtMatcher::ldm_producer`, + /// which is configured from this override; restoring a snapshot + /// captured under a different LDM configuration (enable flip or + /// changed knobs) would reinstate a stale producer. `params` already + /// pins `window_log` / `strategy_tag` (the rest of the producer's + /// identity), so folding the override completes the LDM identity. + /// `None` = LDM off, matching `ParamOverrides::ldm`. + ldm: Option, +} + +impl MatchGeneratorDriver { + /// `slice_size` sets the base block allocation size used for matcher input chunks. + /// `max_slices_in_window` determines the initial window capacity at construction + /// time. Effective window sizing is recalculated on every [`reset`](Self::reset) + /// from the resolved compression level and optional source-size hint. + pub(crate) fn new(slice_size: usize, max_slices_in_window: usize) -> Self { + // Validate inputs before deriving window_log_init. Three + // failure modes need explicit guards: + // + // 1. Zero args → `max_window_size = 0` → silent 1-byte + // degenerate window (useless). + // 2. Multiplication overflow on `slice_size * + // max_slices_in_window` → wraps silently in release. + // 3. `next_power_of_two` overflow when the product is + // above `1 << (usize::BITS - 1)` → modern Rust PANICS + // on overflow (older Rust returned 0). + // + // Catch all three at construction with a clear domain- + // specific message via `assert!` + `checked_mul` + + // `checked_next_power_of_two`, rather than letting either + // mode produce a silent degenerate matcher OR a generic + // panic deep in `FastKernelMatcher::with_params`. + assert!( + slice_size > 0, + "MatchGeneratorDriver::new requires slice_size > 0 (got 0)", + ); + assert!( + max_slices_in_window > 0, + "MatchGeneratorDriver::new requires max_slices_in_window > 0 (got 0)", + ); + let max_window_size = max_slices_in_window + .checked_mul(slice_size) + .expect("MatchGeneratorDriver::new: slice_size * max_slices_in_window overflows usize"); + // Derive an effective window_log for the initial-state matcher. + // `MatchGeneratorDriver::new` runs BEFORE any reset, so it has + // no LevelParams to consult — we initialise to whatever + // window_log fits the caller's requested max_window_size + // (round up to the next power of two via `next_power_of_two`'s + // log). Reset() overwrites all three params from the resolved + // LevelParams. + // + // `checked_next_power_of_two` returns `None` if the next power + // of two would overflow `usize`. Modern Rust's + // `next_power_of_two` PANICS on overflow rather than returning + // 0 (the panic message is generic and unhelpful), so use the + // checked variant to surface the failure with a clear, + // domain-specific error. + let next_pow2 = max_window_size.checked_next_power_of_two().expect( + "MatchGeneratorDriver::new: max_window_size too large for \ + next_power_of_two without overflow", + ); + let window_log_init = next_pow2.trailing_zeros() as u8; + Self { + vec_pool: Vec::new(), + // Deferred table: `new` runs before any source size or resolved + // LevelParams exist, so allocating at the level-default hash_log + // here would be thrown away by the first frame's reset (which + // clamps the window to the input and reallocs at the resolved + // size). The deferral lets that first reset allocate exactly once. + storage: MatcherStorage::Simple(FastKernelMatcher::with_params_deferred( + window_log_init, + FAST_LEVEL_1_HASH_LOG, + FAST_LEVEL_1_MLS, + 2, // upstream zstd default step_size (targetLength=0 → step=2) + )), + strategy_tag: super::strategy::StrategyTag::Fast, + search: super::strategy::SearchMethod::Fast, + parse: super::strategy::ParseMode::Greedy, + #[cfg(test)] + config_override: None, + param_overrides: None, + slice_size, + base_slice_size: slice_size, + // Report the ROUNDED-UP window size that the matcher + // actually carries (via `window_log_init = log2(next_pow2)` + // → matcher's `max_window_size = 1 << window_log_init = + // next_pow2`). For non-power-of-two `slice_size * + // max_slices_in_window` inputs, the unrounded value + // would under-report the active backend's window until + // the first `reset()` overwrites both sides from the + // resolved LevelParams. + reported_window_size: next_pow2, + reset_size_log: None, + reset_dict_attach_ok: true, + reset_shape: None, + dictionary_retained_budget: 0, + source_size_hint: None, + dictionary_size_hint: None, + borrowed_pending: None, + primed: None, + } + } + + fn level_params(level: CompressionLevel, source_size: Option) -> LevelParams { + resolve_level_params(level, source_size) + } + + /// Install the public-parameter per-knob overrides (#27) applied at + /// the next [`Matcher::reset`]. `None` (or an all-`None` set) restores + /// plain level-based geometry. Persists across resets until changed. + pub(crate) fn set_param_overrides( + &mut self, + overrides: Option, + ) { + self.param_overrides = overrides; + } + + /// Active backend family derived from the storage variant. Single + /// source of truth — no separate runtime tag to drift against. + pub(crate) fn active_backend(&self) -> super::strategy::BackendTag { + self.storage.backend() + } + + /// Whether the borrowed (no-copy, in-place over-window) scan is + /// implemented for the current backend + search configuration. The + /// HashChain backend serves both the lazy CHAIN parser + /// (`SearchMethod::HashChain`) and the BT/optimal parsers + /// (`SearchMethod::BinaryTree`); only the lazy chain has a borrowed scan + /// so far, so BT/optimal stay on the owned path. + pub(crate) fn borrowed_supported(&self) -> bool { + use super::strategy::{BackendTag, SearchMethod, StrategyTag}; + match self.active_backend() { + BackendTag::Simple | BackendTag::Dfast | BackendTag::Row => true, + // The HashChain backend covers two searches: the lazy CHAIN parser + // (borrowed-capable) and the BINARY-TREE search (btlazy2 L13-15 + + // optimal BtOpt/BtUltra/BtUltra2 L16-22). btlazy2's BT-tree borrowed + // scan is byte-identical to owned (reads via live_history()), so it + // takes the in-place path. The OPTIMAL parsers stay owned: their + // cost-based DP is sensitive to candidate quality, and the borrowed + // continuous-index scan yields slightly different (ratio-worse) + // candidates than the owned evict+rehash scan — borrowed optimal + // both diverged from owned and fell outside the ffi ratio bound. + // Search-aware (not just strategy_tag) so optimal BT can never be + // staged on the borrowed path even via an internal caller. + BackendTag::HashChain => match self.search { + SearchMethod::HashChain => true, + SearchMethod::BinaryTree => matches!(self.strategy_tag, StrategyTag::Btlazy2), + _ => false, + }, + } + } + + /// Whether a DICTIONARY frame can take the borrowed (no input copy) path. + /// Only the Simple (Fast) backend with the dictionary ATTACHED (not the + /// copy/merge regime) has a borrowed dict scan — `start_matching_borrowed_dict` + /// reads live matches from the borrowed input in place and dict matches + /// from the committed dict prefix via the 2-segment counter. Every other + /// backend, and copy-mode (large-input) dict frames, stay on the owned + /// path. Checked AFTER priming, so `is_attached()` reflects the resolved + /// attach-vs-copy decision. + pub(crate) fn borrowed_dict_supported(&self) -> bool { + matches!( + &self.storage, + MatcherStorage::Simple(m) if m.dict_is_attached() + ) + } + + fn simple_mut(&mut self) -> &mut FastKernelMatcher { + match &mut self.storage { + MatcherStorage::Simple(m) => m, + _ => panic!("simple backend must be initialized by reset() before use"), + } + } + + /// Reclaim the per-block input buffer that the Simple backend + /// just spent inside `start_matching` / `skip_matching_with_hint`. + /// + /// `FastKernelMatcher::take_recycled_space` returns the cleared + /// (capacity-retained) `Vec` from the last + /// `extend_history_with_pending`. We push it onto `vec_pool` + /// as-is (with `len = 0`); `get_next_space()` is responsible for + /// resizing the buffer back to `slice_size` on its next pop. The + /// pushed length is irrelevant — only the capacity matters, and + /// `extend_history_with_pending` preserves it. Without this + /// recycle path, the Simple backend would allocate a new + /// `Vec` per block — a measurable hot-path cost when blocks + /// are small (~128 KiB) and processed at hundreds of MiB/s. + fn recycle_simple_space(&mut self) { + if let Some(space) = self.simple_mut().take_recycled_space() { + // `space` is already cleared (len = 0) by + // `extend_history_with_pending`; capacity is retained. + // Leaving `len = 0` here avoids the cost of zero-filling + // the entire allocation — `get_next_space()` resizes the + // popped buffer up to `slice_size` on demand, so the + // length the pool holds is irrelevant. This matters most + // after a small-source-size hint has shrunk `slice_size` + // mid-frame: the recycled buffer can be much larger than + // the current `slice_size`, and zero-filling 128 KiB+ on + // every block would erase the perf win the recycle path + // is meant to deliver. + self.vec_pool.push(space); + } + } + + /// Register a caller-owned input buffer as the Simple backend's + /// borrowed one-shot match window. Only valid on the Simple (Fast) + /// backend; the one-shot frame path gates on that before calling. + /// + /// # Safety + /// Same contract as [`FastKernelMatcher::set_borrowed_window`]: the + /// buffer must stay live and unmodified until the window is cleared, + /// and must be cleared before the buffer is dropped or the matcher is + /// reused for another frame. + pub(crate) unsafe fn set_borrowed_window(&mut self, buffer: &[u8]) { + // SAFETY: forwarded contract — caller upholds liveness/clear. + match self.active_backend() { + super::strategy::BackendTag::Simple => unsafe { + self.simple_mut().set_borrowed_window(buffer) + }, + super::strategy::BackendTag::Dfast => unsafe { + self.dfast_matcher_mut().set_borrowed_window(buffer) + }, + super::strategy::BackendTag::Row => unsafe { + self.row_matcher_mut().set_borrowed_window(buffer) + }, + super::strategy::BackendTag::HashChain => unsafe { + self.hc_matcher_mut().set_borrowed_window(buffer) + }, + } + } + + /// Clear the borrowed one-shot window, returning the active backend + /// to the owned `history` path. + pub(crate) fn clear_borrowed_window(&mut self) { + match self.active_backend() { + super::strategy::BackendTag::Simple => self.simple_mut().clear_borrowed_window(), + super::strategy::BackendTag::Dfast => self.dfast_matcher_mut().clear_borrowed_window(), + super::strategy::BackendTag::Row => self.row_matcher_mut().clear_borrowed_window(), + super::strategy::BackendTag::HashChain => self.hc_matcher_mut().clear_borrowed_window(), + #[allow(unreachable_patterns)] + _ => {} + } + self.borrowed_pending = None; + } + + /// Stage the borrowed block range `[block_start, block_end)` for the + /// NEXT `start_matching` / `skip_matching_with_hint`, which the + /// borrowed Fast frame path uses in place of `commit_space`. While + /// staged, those trait calls route to the Simple backend's borrowed + /// scan/skip (consuming the stage) instead of the owned committed + /// block. See [`Matcher::start_matching`] / + /// [`Matcher::skip_matching_with_hint`] on this type. + pub(crate) fn set_borrowed_block(&mut self, block_start: usize, block_end: usize) { + assert!( + self.borrowed_supported(), + "borrowed block staging is not supported for the active backend/search config", + ); + assert!( + block_start <= block_end, + "borrowed block range must satisfy start <= end (start={block_start} end={block_end})", + ); + self.borrowed_pending = Some((block_start, block_end)); + // Make the range visible to `get_last_space()` immediately: the + // emit pipeline reads `get_last_space().len()` in + // `collect_block_parts` BEFORE `start_matching` consumes the + // stage, so the staged block (not the whole borrowed window) must + // be reported now to keep the literal-buffer reservation right. + match self.active_backend() { + super::strategy::BackendTag::Simple => self + .simple_mut() + .stage_borrowed_block(block_start, block_end), + super::strategy::BackendTag::Dfast => self + .dfast_matcher_mut() + .stage_borrowed_block(block_start, block_end), + super::strategy::BackendTag::Row => self + .row_matcher_mut() + .stage_borrowed_block(block_start, block_end), + super::strategy::BackendTag::HashChain => self + .hc_matcher_mut() + .table + .stage_borrowed_block(block_start, block_end), + } + } + + #[cfg(test)] + fn dfast_matcher(&self) -> &DfastMatchGenerator { + match &self.storage { + MatcherStorage::Dfast(m) => m, + _ => panic!("dfast backend must be initialized by reset() before use"), + } + } + + fn dfast_matcher_mut(&mut self) -> &mut DfastMatchGenerator { + match &mut self.storage { + MatcherStorage::Dfast(m) => m, + _ => panic!("dfast backend must be initialized by reset() before use"), + } + } + + #[cfg(test)] + pub(crate) fn row_matcher(&self) -> &RowMatchGenerator { + match &self.storage { + MatcherStorage::Row(m) => m, + _ => panic!("row backend must be initialized by reset() before use"), + } + } + + pub(crate) fn row_matcher_mut(&mut self) -> &mut RowMatchGenerator { + match &mut self.storage { + MatcherStorage::Row(m) => m, + _ => panic!("row backend must be initialized by reset() before use"), + } + } + + #[cfg(test)] + fn hc_matcher(&self) -> &HcMatchGenerator { + match &self.storage { + MatcherStorage::HashChain(m) => m, + _ => panic!("hash chain backend must be initialized by reset() before use"), + } + } + + fn hc_matcher_mut(&mut self) -> &mut HcMatchGenerator { + match &mut self.storage { + MatcherStorage::HashChain(m) => m, + _ => panic!("hash chain backend must be initialized by reset() before use"), + } + } + + /// Shrink the active backend's `max_window_size` by the bytes + /// reclaimed from the dictionary-retention budget. Returns `true` + /// iff any reclamation happened — the caller uses that as the + /// gate for [`Self::trim_after_budget_retire`] (which is a no-op + /// otherwise: with `max_window_size` unchanged the backend's + /// `trim_to_window` cannot find anything to evict, so calling it + /// just runs an extra `match` ladder + a single early-out check + /// per slice commit). + #[must_use] + fn retire_dictionary_budget(&mut self, evicted_bytes: usize) -> bool { + let reclaimed = evicted_bytes.min(self.dictionary_retained_budget); + if reclaimed == 0 { + return false; + } + self.dictionary_retained_budget -= reclaimed; + match self.active_backend() { + super::strategy::BackendTag::Simple => { + let matcher = self.simple_mut(); + // `reclaimed` can exceed the CURRENT `max_window_size`: the + // retained dict budget is tracked independently and the + // window may already have been shrunk by a prior eviction, + // so the floor at 0 is the correct clamp, not a masked bug. + matcher.max_window_size = matcher.max_window_size.saturating_sub(reclaimed); + } + super::strategy::BackendTag::Dfast => { + let matcher = self.dfast_matcher_mut(); + // `reclaimed` can exceed the CURRENT `max_window_size`: the + // retained dict budget is tracked independently and the + // window may already have been shrunk by a prior eviction, + // so the floor at 0 is the correct clamp, not a masked bug. + matcher.max_window_size = matcher.max_window_size.saturating_sub(reclaimed); + } + super::strategy::BackendTag::Row => { + let matcher = self.row_matcher_mut(); + // `reclaimed` can exceed the CURRENT `max_window_size`: the + // retained dict budget is tracked independently and the + // window may already have been shrunk by a prior eviction, + // so the floor at 0 is the correct clamp, not a masked bug. + matcher.max_window_size = matcher.max_window_size.saturating_sub(reclaimed); + } + super::strategy::BackendTag::HashChain => { + let matcher = self.hc_matcher_mut(); + // See the Simple arm: `reclaimed` may exceed the current + // window, so saturating to 0 is the correct clamp. + matcher.table.max_window_size = + matcher.table.max_window_size.saturating_sub(reclaimed); + } + } + true + } + + fn trim_after_budget_retire(&mut self) { + loop { + let mut evicted_bytes = 0usize; + match self.active_backend() { + super::strategy::BackendTag::Simple => { + // FastKernelMatcher owns its history as a single + // flat `Vec` (upstream zstd's flat-buffer layout) + // rather than the legacy per-block `WindowEntry` + // stack. There are no per-block Vec allocations + // to recycle into `vec_pool` — `trim_to_window` + // drains the oldest bytes in-place and returns + // the count for the dictionary-budget loop's + // termination check. + let MatcherStorage::Simple(m) = &mut self.storage else { + unreachable!("active_backend() == Simple proven above"); + }; + evicted_bytes += m.trim_to_window(); + } + super::strategy::BackendTag::Dfast => { + // Dfast doesn't retain input Vecs — `history` is the + // only byte store, so there is no per-block buffer + // to push back through a callback. Eviction byte + // count is derived from the `window_size` delta + // before/after; the Dfast variant of + // `trim_to_window` takes no closure, sidestepping + // an unused-`impl FnMut` monomorphization that + // would otherwise contractually never fire. + let dfast = self.dfast_matcher_mut(); + let pre = dfast.window_size; + dfast.trim_to_window(); + evicted_bytes += pre - dfast.window_size; + } + super::strategy::BackendTag::Row => { + // Row keeps bytes only in the contiguous `history` mirror + // (block buffers are returned to the pool per block in + // `add_data`), so derive the eviction count from the + // `window_size` delta, mirroring the Dfast / HashChain arms. + let row = self.row_matcher_mut(); + let pre = row.window_size; + row.trim_to_window(); + evicted_bytes += pre - row.window_size; + } + super::strategy::BackendTag::HashChain => { + // HC keeps bytes only in the contiguous `history` mirror + // (no per-block Vecs to recycle since the window<->history + // dedup), so derive the eviction count from the + // `window_size` delta, mirroring the Dfast arm above. + let table = &mut self.hc_matcher_mut().table; + let pre = table.window_size; + table.trim_to_window(); + evicted_bytes += pre - table.window_size; + } + } + if evicted_bytes == 0 { + break; + } + // The loop's invariant is "the backend's previous + // `max_window_size` shrink had downstream bytes left to + // evict" — that's what `evicted_bytes != 0` proves at + // this point. `dictionary_retained_budget` is NOT + // guaranteed to be positive here: the outer + // `retire_dictionary_budget` call may have already + // drained it to zero by reclaiming the last retained + // bytes, while the backend still has bytes above the + // freshly-shrunk window cap waiting for this loop to + // evict. The return value of the retire call below is + // therefore intentionally discarded — the loop's + // termination is driven by `evicted_bytes == 0`, not by + // whether the budget has more bytes left to reclaim. + let _ = self.retire_dictionary_budget(evicted_bytes); + } + } + + /// ATTACH (`true`) vs COPY (`false`) decision for the dms-bearing HashChain + /// backend (lazy hash-chain AND binary-tree/optimal levels), mirroring + /// upstream `ZSTD_shouldAttachDict` and its per-strategy `attachDictSizeCutoffs`: + /// a small / unknown source ATTACHES the dict as a separate dms (hash-chain + /// dms for lazy, DUBT dms for BT); a large known source COPIES it into the + /// live chain / tree. The cutoff is the lazy/lazy2 value for HC, the + /// btlazy2/btopt value for Bt{Opt}, and the smaller btultra/btultra2 value for + /// the deepest parses. Both `skip_matching_for_dictionary_priming` (which + /// stages the dict) and `prime_with_dictionary` (which builds-or-drops the + /// dms) read this so the two stay in lock-step. + fn hc_dict_attach_mode(&self) -> bool { + // Only the HashChain backend (lazy hash-chain + BT/optimal) routes here; + // a non-HashChain storage has no dms decision, so default to attach. + let MatcherStorage::HashChain(hc) = &self.storage else { + return true; + }; + let cutoff = if hc.table.uses_bt { + match hc.strategy_tag { + super::strategy::StrategyTag::BtUltra | super::strategy::StrategyTag::BtUltra2 => { + BT_ULTRA_ATTACH_DICT_CUTOFF_LOG + } + _ => BT_OPT_ATTACH_DICT_CUTOFF_LOG, + } + } else { + HC_ATTACH_DICT_CUTOFF_LOG + }; + self.reset_size_log.is_none_or(|log| log <= cutoff) + } + + fn skip_matching_for_dictionary_priming(&mut self) { + match self.active_backend() { + super::strategy::BackendTag::Simple => { + // Upstream zstd `ZSTD_shouldAttachDict` mode selection for the Fast + // strategy (cutoff 8 KB): small / unknown-size inputs ATTACH + // (index dict positions into a SEPARATE immutable table; the + // dual-probe 2-cursor `compress_block_fast_dict` then prefers + // recent-input matches and falls back to the dict — the path + // that wins small/unknown). Large known-size inputs COPY (prime + // dict into the live table; the 4-cursor `compress_block_fast` + // matches against it as window history — the path that already + // matches/beats the upstream zstd on large corpora). The dispatch in + // `start_matching` keys off `dict_table.is_some()`, which only + // the attach path populates. See [`FAST_ATTACH_DICT_CUTOFF_LOG`]. + let attach = self.reset_dict_attach_ok + && self + .reset_size_log + .is_none_or(|log| log <= FAST_ATTACH_DICT_CUTOFF_LOG); + if attach { + self.simple_mut().skip_matching_for_dict_prime(); + } else { + self.simple_mut().skip_matching_with_hint(Some(false)); + } + self.recycle_simple_space(); + } + super::strategy::BackendTag::Dfast => { + // Upstream zstd `ZSTD_dictMatchState` mode selection for dfast (cutoff + // 16 KiB): small / unknown-size inputs ATTACH (build the + // separate immutable dict long+short tables; the dual-probe + // `start_matching_fast_loop` searches live + dict, the path that + // avoids the per-frame dict re-prime that dominates small + // `compress-dict`). Larger known-size inputs COPY (re-prime the + // dict into the live tables via `skip_matching_dense`, where the + // dense scan matches it as window history). `skip_matching_for_dict_attach` + // self-gates on `use_fast_loop` (only fast-loop levels carry the + // dual-probe; general-path levels fall back to the dense copy). + let attach = self + .reset_size_log + .is_none_or(|log| log <= DFAST_ATTACH_DICT_CUTOFF_LOG); + if attach { + self.dfast_matcher_mut().skip_matching_for_dict_attach(); + } else { + self.dfast_matcher_mut().invalidate_dict_cache(); + self.dfast_matcher_mut().skip_matching_dense(); + } + } + super::strategy::BackendTag::Row => { + // Upstream zstd `ZSTD_RowFindBestMatch` `dictMatchState`: small / + // unknown-size inputs ATTACH (build the separate immutable dict + // row index; the bounded dual-probe in `row_candidate_rl` + // searches live + dict, avoiding the per-frame dict re-index), + // larger known-size inputs COPY (dense re-prime into the live + // rows). + let attach = self + .reset_size_log + .is_none_or(|log| log <= ROW_ATTACH_DICT_CUTOFF_LOG); + if attach { + self.row_matcher_mut().prime_dict_attach_current_block(); + } else { + self.row_matcher_mut().invalidate_dict_cache(); + self.row_matcher_mut().skip_matching_with_hint(Some(false)); + } + } + super::strategy::BackendTag::HashChain => { + // Lazy-HC AND BT/optimal both follow upstream zstd `ZSTD_shouldAttachDict` + // per-strategy: ATTACH (a separate dms — hash-chain dms for lazy, + // DUBT dms for BT) for small / unknown inputs, COPY (merge the dict + // into the live chain/tree) for large known inputs. ATTACH keeps + // the dict in history but out of the live structure via + // `skip_matching_dict_bt` (the cursor advance is shared by both + // arms); COPY routes through the normal `skip_matching` (its + // `uses_bt` branch fills the live tree, the lazy branch the live + // chain). The dms is built-or-dropped to match in + // `prime_with_dictionary`. + if self.hc_dict_attach_mode() { + self.hc_matcher_mut().table.skip_matching_dict_bt(); + } else { + self.hc_matcher_mut().skip_matching(Some(false)); + } + } + } + } +} + +impl Matcher for MatchGeneratorDriver { + fn supports_dictionary_priming(&self) -> bool { + true + } + + fn set_source_size_hint(&mut self, size: u64) { + self.source_size_hint = Some(size); + } + + fn set_dictionary_size_hint(&mut self, size: usize) { + self.dictionary_size_hint = Some(size); + } + + /// Dict-relevance gate for the raw-fast-path. Reached only when a dictionary + /// is active (the caller short-circuits on `dict_active`), so this answers + /// "could the dict compress this otherwise-incompressible-looking block?". + /// The Simple (Fast) backend samples its dict table precisely + /// ([`FastKernelMatcher::block_samples_match_dict`]); the other backends + /// (Dfast / Row / HashChain / BT) have their own dict structures and no cheap + /// probe here, so they answer CONSERVATIVELY `true`: without a probe they + /// cannot tell whether the dict compresses an incompressible-LOOKING block, + /// and answering `false` would let the raw-fast-path emit such a block raw + /// and miss an embedded dict segment. `dictionary_segment_in_incompressible_input_is_matched` + /// pins this for Dfast/Row/BT — the 512-byte dict run inside high-entropy + /// filler is matched only because these backends stay on the scan. So they + /// keep the blanket scan the old `!dict_active` gate gave them; only the + /// Simple/Fast backend trades it for the precise probe. + fn block_samples_match_dict(&self, block: &[u8]) -> bool { + match &self.storage { + MatcherStorage::Simple(m) => m.block_samples_match_dict(block), + _ => true, + } + } + + /// Heap bytes this driver owns: the active backend's tables/history, the + /// recycled input-buffer pool, and the primed-dictionary snapshot (a cloned + /// backend kept for CDict-equivalent reuse). The inline struct itself is + /// accounted by the owner's `size_of`. + fn heap_size(&self) -> usize { + let pool: usize = self.vec_pool.capacity() * core::mem::size_of::>() + + self.vec_pool.iter().map(Vec::capacity).sum::(); + let snapshot = self + .primed + .as_ref() + .map_or(0, |(storage, _, _)| storage.heap_size()); + pool + self.storage.heap_size() + snapshot + } + + fn clear_param_overrides(&mut self) { + self.param_overrides = None; + } + + fn reset(&mut self, level: CompressionLevel) { + let hint = self.source_size_hint.take(); + let dict_hint = self.dictionary_size_hint.take(); + // Snapshot the hint's normalized ceil-log bucket for the primed-snapshot + // key and prime_with_dictionary's attach/copy mode decision (the hint is + // consumed here, but priming happens just after reset). Storing the + // bucket rather than the raw bytes means two hints that resolve to the + // same matcher shape share one snapshot instead of each re-priming. + self.reset_size_log = hint.map(source_size_ceil_log); + // A dictionary too large for the tagged attach position field falls back + // to copy mode. Captured here (from the load-set size hint = actual dict + // length) so the prime decision and the snapshot-key / epoch bits agree. + self.reset_dict_attach_ok = + dict_hint.is_none_or(|size| size <= MAX_FAST_ATTACH_DICT_REGION); + let hinted = hint.is_some(); + #[cfg_attr(not(test), allow(unused_mut))] + let mut params = Self::level_params(level, hint); + // Test-only: apply a parse×search override so the matrix can be + // exercised without editing `LEVEL_TABLE`. Mutating `params` here + // (before `next_backend`) flows the override through storage + // selection, `configure`, and the `self.search`/`self.parse` + // writes uniformly. Consumed with `take()` so it is one-shot: the + // synthetic pairing applies to exactly this `reset()`, and a later + // reset on the same driver falls back to the level's real config. + #[cfg(test)] + if let Some((search, parse)) = self.config_override.take() { + params.search = search; + params.lazy_depth = parse.lazy_depth(); + // The matrix sweep can pair a level with a backend its native + // row doesn't populate (e.g. greedy L5, which carries only `row`, + // run on HashChain). Synthesize a default config for the + // overridden backend so its `configure` arm has something to read. + use super::strategy::SearchMethod; + match search { + SearchMethod::Fast => { + params.fast.get_or_insert(FAST_L1); + } + SearchMethod::DoubleFast => { + params.dfast.get_or_insert(DFAST_L3); + } + SearchMethod::RowHash => { + params.row.get_or_insert(ROW_CONFIG); + } + SearchMethod::HashChain | SearchMethod::BinaryTree => { + params.hc.get_or_insert(HC_CONFIG); + } + } + } + // Public-parameter overrides (#27): apply the per-knob set on top + // of the level-resolved params. A strategy override re-routes the + // backend, so this must precede `next_backend` selection. The + // all-`None` case is skipped so default level geometry stays + // byte-identical to plain level-based compression. + if let Some(ov) = self.param_overrides + && !ov.is_empty() + { + apply_param_overrides(&mut params, &ov); + // `Self::level_params(level, hint)` applied the source-size cap + // for the LEVEL's native backend. If a strategy override moved + // the frame onto a different backend, `apply_param_overrides` + // synthesized that backend's DEFAULT config (FAST_L1 / + // HC_OVERRIDE_DEFAULT) with full-size table logs AFTER that cap + // ran. Re-apply the hint cap so a tiny hinted frame doesn't + // allocate the new backend's full-size tables. An explicit + // `window_log` override is the user's hard request and must + // survive the re-cap, so restore it afterwards. + if let Some(hint_size) = hint { + params = adjust_params_for_source_size(params, hint_size); + if let Some(window_log) = ov.window_log { + params.window_log = window_log; + } + } + } + // Dictionary-driven table sizing — parity with upstream zstd `ZSTD_createCDict` + // (`ZSTD_getCParams_internal(level, UNKNOWN, dictSize, ZSTD_cpm_createCDict)` + // → `ZSTD_adjustCParams_internal`). A loaded dictionary supplies the + // long-distance matches, so upstream zstd sizes the prepared match-finder tables + // to the DICTIONARY (assuming a `minSrcSize` source), not the live + // window: it downsizes `hashLog`/`chainLog` toward the dict-and-window + // log while leaving the frame's eviction `window_log` source-derived so + // the dictionary bytes stay referenceable (`ZSTD_resetCCtx_byCopyingCDict` + // copies the small CDict tables but keeps the source window). We apply + // the same downsizing to the level's own hc geometry and cap (min) so a + // dict never inflates the level tables. Only the binary-tree / hash-chain + // backend reads `hc.{hash,chain}_log`; Simple/Dfast/Row derive their + // widths from the source window in their `reset` arms. + // A zero-length dictionary is "no dictionary": running the CDict sizing + // path for `Some(0)` is not a no-op — `cdict_table_logs(.., 0)` still + // collapses the HC/BT tables toward the 513-byte upstream zstd tier via + // `DICT_MIN_SRC_SIZE`, tanking ratio/perf on the next frame. Priming + // already treats empty content as empty, so skip the downsizing here too. + if let Some(dict_size) = dict_hint.filter(|&size| size > 0) { + // Derive the dict-tier geometry from the level's FULL (un-source-capped) + // hc widths. `Self::level_params(level, hint)` already source-capped + // `params.hc`; feeding those capped widths into `cdict_table_logs` and + // then `.min()`-ing would double-cap, so on a small hinted source with a + // large dictionary the prepared tables collapse below what the dict needs + // — defeating the `ZSTD_createCDict` geometry this mirrors. Take the + // un-hinted base widths instead and assign the result directly: + // `cdict_table_logs` only ever downsizes, so it never exceeds the base + // level geometry, while the eviction `window_log` stays source-derived so + // the dictionary bytes remain referenceable. Active public-parameter + // overrides (#27) are applied to the base too, so a strategy override + // that routes onto HashChain/BinaryTree still gets dict-tier sizing and + // explicit hash/chain overrides feed through as the geometry ceiling. + let mut base_params = Self::level_params(level, None); + if let Some(ov) = self.param_overrides + && !ov.is_empty() + { + apply_param_overrides(&mut base_params, &ov); + } + if let (Some(hc), Some(base_hc)) = (params.hc.as_mut(), base_params.hc) { + let uses_bt = matches!( + params.strategy_tag, + super::strategy::StrategyTag::Btlazy2 + | super::strategy::StrategyTag::BtOpt + | super::strategy::StrategyTag::BtUltra + | super::strategy::StrategyTag::BtUltra2 + ); + let (dict_hash_log, dict_chain_log) = cdict_table_logs( + params.window_log, + base_hc.hash_log, + base_hc.chain_log, + uses_bt, + dict_size, + ); + hc.hash_log = dict_hash_log; + hc.chain_log = dict_chain_log; + } + } + // upstream zstd `ZSTD_resolveRowMatchFinderMode` (zstd_compress.c:238-245): + // the row matchfinder is used for greedy/lazy/lazy2 ONLY when + // `windowLog > 14`; at or below that upstream runs the hash-chain + // matcher (`ZSTD_HcFindBestMatch`). We previously hardcoded the Row + // backend for these strategies regardless of window, sending every + // small-window frame (hinted floor = windowLog 14, e.g. the small-4k/10k + // fixtures) through Row where upstream uses HC. Match it: fall back to + // the hash-chain matcher (lazy/greedy parse via `lazy_depth`) when the + // resolved window is <= 14. The HC config is synthesised from the + // level's RowConfig (HC and Row share the same cParams; only the + // matchfinder differs) — `hash_log` / `chain_log` are + // clamped to the (<= 14) window inside the HashChain reset arm, so the + // nominal width here only sets the clamp ceiling. + if params.search == super::strategy::SearchMethod::RowHash && params.window_log <= 14 { + let row = params + .row + .expect("a RowHash level row must carry a RowConfig"); + params.search = super::strategy::SearchMethod::HashChain; + // For a dict-bearing frame, downsize the synthesised HC logs to the + // dictionary's content tier via `cdict_table_logs` (the same + // correction the native HC dict-prime path applies above), so a dict + // much smaller than the window doesn't prime a needlessly sparse + // table. Row-finder levels are never BinaryTree, so `uses_bt = false`. + // + // Feed `cdict_table_logs` the UN-hinted base Row width, not the + // resolved `row.hash_bits`: the latter is already source-capped on a + // hinted reset (the `row_cap = table_log + 1` clamp), so passing it + // here would double-cap exactly as the native HC dict path warns + // above — a small hinted source with a large dictionary would + // collapse the prepared table below what the dict needs. + // `cdict_table_logs` only ever downsizes, so deriving the ceiling + // from the un-hinted base (plus active public overrides) keeps the + // dict-tier geometry intact. No source hint => `row.hash_bits` is + // already the level's full width, so reuse it directly. + let row_cdict_hash_bits = match dict_hint.filter(|&size| size > 0) { + Some(_) => { + let mut base_params = Self::level_params(level, None); + if let Some(ov) = self.param_overrides + && !ov.is_empty() + { + apply_param_overrides(&mut base_params, &ov); + } + base_params + .row + .map_or(row.hash_bits, |base_row| base_row.hash_bits) + } + None => row.hash_bits, + }; + // Row-backed levels carry only `hash_bits`; the HC chain table they + // fall back to follows the upstream zstd cParams relationship `chainLog = + // hashLog - 1` for every Row level (L6 c18 h19 .. L12 c22 h23, see + // the ROW_L* tables). Synthesise the chain width as `hash_bits - 1` + // so the dict path doesn't leave the chain table one bit too wide + // (cdict_table_logs only downsizes, so passing the full hash width + // for both would keep a 2x-too-large chain table on dict frames). + // Raw `- 1` is underflow-safe: `hash_bits` is either a predefined + // ROW_L* width (>= 19) or a public `hash_log` override, and the + // override is range-validated to `ZSTD_HASHLOG_MIN = 6` at the + // parameter API, so the value is always >= 6 here. + // + // A public `chain_log` override (#27) is dropped by the RowHash + // override arm (Row has no chain table), but once this frame falls + // back to HC the chain table is live and must honour it — mirror + // the native HC dict path, which feeds the override-applied + // `base_hc.chain_log` into `cdict_table_logs`. Use the explicit + // override (also API-validated to ZSTD_CHAINLOG_MIN = 6) when set, + // else the upstream zstd `hashLog - 1` relationship. + let explicit_chain_log = self + .param_overrides + .filter(|ov| !ov.is_empty()) + .and_then(|ov| ov.chain_log) + .map(|chain_log| chain_log as usize); + let row_cdict_chain_bits = explicit_chain_log.unwrap_or(row_cdict_hash_bits - 1); + let (mut hash_log, mut chain_log) = match dict_hint.filter(|&size| size > 0) { + Some(dict_size) => cdict_table_logs( + params.window_log, + row_cdict_hash_bits, + row_cdict_chain_bits, + false, + dict_size, + ), + None => ( + row.hash_bits, + explicit_chain_log.unwrap_or(row.hash_bits - 1), + ), + }; + // No-dict path: the HashChain reset arm only clamps the logs to the + // window when `hinted`, but a public `window_log` override can lower + // this level to <= 14 with no source hint — clamp the level's full + // Row `hash_bits` to the window here too (upstream zstd `ZSTD_adjustCParams`: + // hashLog <= windowLog + 1, chainLog <= windowLog) so a 16 KiB window + // doesn't allocate Row-sized HC tables. + if dict_hint.filter(|&size| size > 0).is_none() { + let wlog = params.window_log as usize; + hash_log = hash_log.min(wlog + 1); + chain_log = chain_log.min(wlog); + } + params.hc = Some(HcConfig { + hash_log, + chain_log, + search_depth: row.search_depth, + target_len: row.target_len, + search_mls: 4, + }); + params.row = None; + } + let next_backend = params.backend(); + let max_window_size = 1usize << params.window_log; + self.dictionary_retained_budget = 0; + // Drop any frame-local borrowed staging so it can't leak across a + // reset and misroute the next start/skip into borrowed dispatch. + self.borrowed_pending = None; + if self.active_backend() != next_backend { + // Drain the outgoing backend's allocations into the shared + // pool. The `match &mut self.storage { ... }` block runs to + // completion before the assignment below replaces the + // variant, so the inner state we just drained is dropped + // with the old variant. + match &mut self.storage { + MatcherStorage::Simple(_m) => { + // FastKernelMatcher owns a flat Vec history + // and a Vec hash table — both drop with the + // variant assignment below, no per-block buffers + // to recycle into the driver pools. The + // assignment-replace path collapses to a noop + // pre-pass for this backend. + } + MatcherStorage::Dfast(m) => { + // Drop the long / short hash table allocations + // before calling `m.reset`. Without this prepass, + // `DfastMatchGenerator::reset` would `fill` both + // tables with `DFAST_EMPTY_SLOT` sentinels — wasted + // work given the next assignment to `self.storage` + // is about to drop `m` entirely. `reset` itself + // short-circuits on `if !self.tables.is_empty()`, so + // handing it an empty `Vec` skips the fill loop. + // Mirrors the pre-drain pattern in the HashChain + // arm below (and serves the same peak-memory + // purpose: release the table-allocation footprint + // before constructing the replacement variant). + m.tables = Vec::new(); + m.reset(); + } + MatcherStorage::Row(m) => { + m.row_heads = Vec::new(); + m.row_positions = Vec::new(); + m.row_tags = Vec::new(); + m.reset(); + } + MatcherStorage::HashChain(m) => { + // Release oversized tables when switching away from + // HashChain so Best's larger allocations don't persist. + // hash3_table must be released alongside the other + // two: BtUltra2's `1 << HC3_HASH_LOG` entries would + // otherwise stay pinned across the backend switch, + // even though no future caller of this backend will + // touch them. + m.table.hash_table = Vec::new(); + m.table.chain_table = Vec::new(); + m.table.hash3_table = Vec::new(); + let vec_pool = &mut self.vec_pool; + m.reset(|mut data| { + data.resize(data.capacity(), 0); + vec_pool.push(data); + }); + } + } + // Swap in a fresh variant for the new backend. The previous + // `storage` is dropped here. + self.storage = match next_backend { + super::strategy::BackendTag::Simple => { + // Per-level Fast cParams from resolve_level_params: + // Level(1) gets (hash_log=14, mls=7); Level(-7..=-1) + // get upstream zstd row-0 (hash_log=13, mls=7); Fastest / + // Uncompressed keep (hash_log=14, mls=6). See + // resolve_level_params for rationale. + let fast = params.fast.expect("Fast level row carries a FastConfig"); + MatcherStorage::Simple(FastKernelMatcher::with_params( + params.window_log, + fast.hash_log, + fast.mls, + fast.step_size, + )) + } + super::strategy::BackendTag::Dfast => { + MatcherStorage::Dfast(DfastMatchGenerator::new(max_window_size)) + } + super::strategy::BackendTag::Row => { + MatcherStorage::Row(RowMatchGenerator::new(max_window_size)) + } + super::strategy::BackendTag::HashChain => { + MatcherStorage::HashChain(HcMatchGenerator::new(max_window_size)) + } + }; + } + + // Single source of truth: `LevelParams::strategy_tag` is the + // authoritative mapping from `CompressionLevel` to strategy. + // `storage.backend()` derives the parse family from the variant, + // so there is no separate runtime tag that could drift against + // `LEVEL_TABLE`. + self.strategy_tag = params.strategy_tag; + self.search = params.search; + self.parse = params.parse(); + self.slice_size = self.base_slice_size.min(max_window_size); + self.reported_window_size = max_window_size; + let strategy_tag = self.strategy_tag; + // Source-proportional table window for the backends whose hash-table + // widths are recomputed here (Dfast / Row). Like the HC / Fast caps + // in `adjust_params_for_source_size`, this sizes the internal tables + // from the RAW source log (not the wire `window_log` floor) so a + // small frame zeroes a small table; it never exceeds the real window. + let table_window_size = match hint { + Some(h) => { + let raw_log = source_size_ceil_log(h); + // Clamp the shift below the pointer width before `1usize <<`: + // an oversized hint (>= 2^63 + 1, and on 32-bit usize any hint + // >= 2^32) drives `raw_log` to 64 / >= 32, and the shift would + // overflow (panic in debug, wrap to 0 in release) before the + // `.min(max_window_size)` cap below could bound it. The min cap + // still provides the real semantic window bound. + let shift = raw_log.max(MIN_WINDOW_LOG).min(usize::BITS as u8 - 1); + (1usize << shift).min(max_window_size) + } + None => max_window_size, + }; + // The hint-dependent hash-table width the active backend applies, for + // the primed-snapshot key. Dfast/Row compute it from `table_window_size` + // below; HC/Fast leave it `0` because their widths live in `params` + // (`hc.{hash,chain}_log` / `fast_hash_log`) — already part of the key. + let mut resolved_table_bits: usize = 0; + match &mut self.storage { + MatcherStorage::Simple(m) => { + // Per-level Fast cParams threaded from + // resolve_level_params (see Simple-backend swap + // arm above for the (level → params) mapping). + let fast = params.fast.expect("Fast level row carries a FastConfig"); + // Same attach/copy split the dict-prime dispatch applies + // below (`prime_with_dictionary`): only attach-mode dict + // frames may keep the main table across the reset via an + // epoch advance — copy-mode and no-dict frames must memset + // it back to bias 0 for the raw-slice kernels. + // `Some(0)` is "no dictionary" (the dict-sizing path above + // filters it the same way): an empty dict primes nothing, so + // an epoch-advance reset would preserve stale attach state + // instead of clearing it. + let dict_attach_epoch = matches!(dict_hint, Some(size) if size > 0) + && self.reset_dict_attach_ok + && self + .reset_size_log + .is_none_or(|log| log <= FAST_ATTACH_DICT_CUTOFF_LOG); + // Copy-mode dictionary frame whose primed snapshot matches + // this exact resolved shape: `restore_primed_dictionary` + // (called right after this reset; the caller gates the + // restore on the same size bucket and the restore re-checks + // the same key) will `clone_from` the snapshot over this + // matcher, replacing the table contents and bias wholesale — + // the reset's full-table memset would be thrown away. The + // key components mirror `reset_shape` below: Simple leaves + // `resolved_table_bits` 0, never carries an LDM override, + // and `fast_attach` is false in copy mode by construction. + let table_overwritten_by_restore = matches!(dict_hint, Some(size) if size > 0) + && !dict_attach_epoch + && self.primed.as_ref().is_some_and(|(_, _, captured)| { + *captured + == PrimedKey { + level, + params, + table_bits: 0, + fast_attach: false, + ldm: None, + } + }); + // Cap `hash_log <= window_log + 1` (upstream zstd + // `ZSTD_adjustCParams_internal`): once `window_log` is resized + // down for a small source, a level-default `1 << hash_log` + // table is mostly wasted address space whose per-frame memset + // dominates the compress cost on tiny frames (a 4 KB frame at + // window_log 12 still zero-fills the 64 KiB hash_log-14 table). + // Gated to no-dict frames: the dict-attach path shares one + // hash_log between the main and dict tables (so one hash keys + // both), and shrinking only the main table would break that + // invariant and the small-frame dict ratio. + let hash_log = if dict_hint.is_some_and(|s| s > 0) { + fast.hash_log + } else { + fast.hash_log.min(params.window_log as u32 + 1) + }; + m.reset( + params.window_log, + hash_log, + fast.mls, + fast.step_size, + dict_attach_epoch, + table_overwritten_by_restore, + ); + } + MatcherStorage::Dfast(dfast) => { + dfast.max_window_size = max_window_size; + let dcfg = params + .dfast + .expect("Dfast level row must carry a DfastConfig"); + // Upstream zstd `cParams.hashLog`/`chainLog`, capped by the + // source-size window when hinted so tiny inputs don't + // over-allocate. + let long_bits = if hinted { + dfast_hash_bits_for_window(table_window_size).min(dcfg.long_hash_log as usize) + } else { + dcfg.long_hash_log as usize + }; + let short_bits = if hinted { + dfast_hash_bits_for_window(table_window_size).min(dcfg.short_hash_log as usize) + } else { + dcfg.short_hash_log as usize + }; + resolved_table_bits = long_bits; + dfast.set_hash_bits(long_bits, short_bits); + // Dfast holds no per-block input Vecs (history owns the + // bytes and `add_data` returns each Vec eagerly), so + // `reset` takes no `reuse_space` callback. + dfast.reset(); + } + MatcherStorage::Row(row) => { + row.max_window_size = max_window_size; + row.lazy_depth = params.lazy_depth; + let mut row_cfg = params.row.expect("Row level row carries a RowConfig"); + if hinted { + // Clamp the configured hash width by the hinted window + // (upstream zstd `ZSTD_adjustCParams` caps hashLog by windowLog) — + // `min`, not replace, so an explicit `hash_log` param + // override (`row_cfg.hash_bits`) survives the hinted path + // instead of being overwritten by the window value. + // + // Clamp BEFORE `configure` so the backend sees ONE width + // per frame. Configuring with the unclamped level width + // and then re-clamping made `row_hash_log` oscillate on + // every hinted frame, and each width change clears the + // row tables — `ensure_tables` then re-filled all three + // every frame in a reused compressor. + row_cfg.hash_bits = row_cfg + .hash_bits + .min(row_hash_bits_for_window(table_window_size)); + } + row.configure(row_cfg); + // Key the primed snapshot on the width the backend ACTUALLY + // applied (`set_hash_bits` clamps the request): recording the + // request — or the 0 default on the unhinted path — keys + // identical table geometries apart and forces needless + // dictionary re-primes. + resolved_table_bits = row.hash_bits(); + row.reset(); + } + MatcherStorage::HashChain(hc) => { + hc.table.max_window_size = max_window_size; + hc.hc.lazy_depth = params.lazy_depth; + let mut hc_cfg = params.hc.expect("HashChain level row carries an HcConfig"); + // Cap the hash / chain table logs by the hinted window so a small + // input doesn't allocate the full level's tables (the upstream zstd + // `ZSTD_adjustCParams_internal` clamp: `hashLog <= windowLog + 1`, + // and `cycleLog <= windowLog` — `cycleLog == chainLog` for the HC + // finder, `chainLog - 1` for the BT pair table, so `chainLog <= + // windowLog` (+1 for BT)). Ratio-neutral: a hinted window of + // `2^wlog` bytes holds at most `2^wlog` positions, so the slots + // beyond that are never populated — capping only sheds unused + // allocation. Was the source of L10-lazy peak-alloc ~2.15x the + // upstream zstd on a 1 MiB input. Only applied when hinted; an + // unknown-size stream keeps the full level tables. + // Skip for dict-bearing frames: their `hc_cfg.{hash,chain}_log` + // were already sized to the dictionary content tier via + // `cdict_table_logs` (the dict supplies the long-distance + // matches, so upstream `ZSTD_createCDict` sizes the prepared + // tables to the dict, not the source window). Re-applying the + // source-window cap here would collapse those dict-tier logs + // back to the small hinted source — the same double-cap the + // synthesis sites avoid by using the un-hinted base width. + if hinted && !matches!(dict_hint, Some(size) if size > 0) { + let wlog = hc_hash_bits_for_window(table_window_size); + let uses_bt = matches!( + strategy_tag, + super::strategy::StrategyTag::Btlazy2 + | super::strategy::StrategyTag::BtOpt + | super::strategy::StrategyTag::BtUltra + | super::strategy::StrategyTag::BtUltra2 + ); + hc_cfg.hash_log = hc_cfg.hash_log.min(wlog + 1); + hc_cfg.chain_log = hc_cfg.chain_log.min(if uses_bt { wlog + 1 } else { wlog }); + } + hc.configure(hc_cfg, strategy_tag, params.window_log); + let vec_pool = &mut self.vec_pool; + hc.reset(|mut data| { + data.resize(data.capacity(), 0); + vec_pool.push(data); + }); + // When the source size is known, pre-size the history mirror to + // the expected total (dictionary + payload) so per-block growth + // does not overshoot via Vec capacity doubling (upstream zstd sizes its + // window buffer exactly). Dominates peak once the match-finder + // tables are dictionary-tier-small. Unhinted streams skip this + // and keep doubling growth. + if let Some(src) = hint { + // `src` is a u64 hint and may be the u64::MAX "unknown + // size" sentinel, which truncates under `as usize` on + // 32-bit targets and overflows when the dict hint is + // added. Saturate the source size, then saturate the + // dict-hint addition; `reserve_history` applies the + // tighter window ceiling to the result. + let src_hint = usize::try_from(src).unwrap_or(usize::MAX); + let expected = src_hint.saturating_add(dict_hint.unwrap_or(0)); + hc.table.reserve_history(expected); + } + } + } + // LDM wiring (#27): attach (or clear) the long-distance-match + // producer on the optimal (BT) backend. LDM is the only + // back-reference path that crosses the regular window, so it + // only has a home on the `BtMatcher`; non-BT strategies drop the + // producer. Built AFTER `hc.reset()` because `BtMatcher::reset` + // clears an existing producer's table but does not null the + // slot — installing here gives the new frame a fresh producer. + #[cfg(feature = "hash")] + { + // Resolve the derived LDM params first (immutable borrow of the + // overrides), then reuse the existing producer's allocation below. + let derived_ldm = self + .param_overrides + .as_ref() + .and_then(|ov| ov.ldm) + .map(|ldm_ov| { + let strategy_ord = ldm_strategy_ordinal(params.strategy_tag, params.lazy_depth); + // Seed the caller-pinned knobs, then run the upstream zstd + // derivation over the seed so the remaining (zero) + // fields are filled with cross-field consistency + // (e.g. `hash_rate_log = window_log - hash_log`). + // Clobbering after `adjust_for` would break that and + // hand the producer an inconsistent set. + let seed = super::ldm::params::LdmParams { + window_log: params.window_log as u32, + hash_log: ldm_ov.hash_log.unwrap_or(0), + hash_rate_log: ldm_ov.hash_rate_log.unwrap_or(0), + min_match_length: ldm_ov.min_match.unwrap_or(0), + bucket_size_log: ldm_ov.bucket_size_log.unwrap_or(0), + }; + seed.derive(strategy_ord) + }); + if let MatcherStorage::HashChain(hc) = &mut self.storage { + // Reuse the existing producer's hash-table allocation when the + // derived params are unchanged: only `clear()` (re-zero the + // table + re-seed the rolling hash, no allocation) is needed for + // the new frame. A params change (or the first frame) forces a + // fresh `LdmProducer::new`. On the reused-encoder compress-dict + // path this avoids re-allocating the LDM hash table (large at + // btultra2) every frame — upstream zstd reuses its `ldmState_t` + // the same way. `clear()` is mandatory here for correctness + // regardless of what `BtMatcher::reset` did to the old table. + let producer = derived_ldm.map(|p| match hc.take_ldm_producer() { + Some(mut existing) if existing.params() == p => { + existing.clear(); + existing + } + _ => super::ldm::LdmProducer::new(p), + }); + hc.set_ldm_producer(producer); + } + } + // Record the resolved matcher shape for the primed-snapshot key. Captured + // here (post-resolution, after the test-only param override) so the key + // reflects exactly the geometry the restored `storage` must match. The + // Fast attach-vs-copy mode is part of the shape ONLY for the Simple + // backend (it decides the distinct dict-table shape that backend builds). + // Dfast/Row/HashChain have their OWN attach/copy regimes, but this bit + // models only the Fast table split; those backends are keyed by the + // resolved matcher geometry instead, so folding the Fast bit into their + // key would over-key identical resolved shapes. When it applies it + // matches the decision `prime_with_dictionary` makes from the same + // `reset_size_log`. + let fast_attach = matches!(next_backend, super::strategy::BackendTag::Simple) + && self.reset_dict_attach_ok + && self + .reset_size_log + .is_none_or(|log| log <= FAST_ATTACH_DICT_CUTOFF_LOG); + // The LDM override is part of the snapshot identity ONLY on the + // optimal (BinaryTree) path: that is the only backend whose cloned + // `storage` carries a `BtMatcher::ldm_producer`. On Fast / Dfast / + // Row and lazy-HashChain resets the producer slot does not exist, + // so folding the override there would over-key the snapshot and + // force needless re-primes when LDM is toggled. Gated like + // `fast_attach` (a key bit only participates where it changes the + // cloned matcher shape). + let active_ldm = if matches!(params.search, super::strategy::SearchMethod::BinaryTree) { + self.param_overrides.and_then(|ov| ov.ldm) + } else { + None + }; + self.reset_shape = Some((params, resolved_table_bits, fast_attach, active_ldm)); + } + + // Dictionary entry points forward to the `dict_prime` child module, which + // owns the prime / snapshot lifecycle (it reaches the driver's private + // `primed` / `reset_shape` state directly as a descendant module). + + #[inline] + fn dictionary_is_resident(&self) -> bool { + self.dictionary_is_resident_impl() + } + + #[inline] + fn reapply_resident_dictionary(&mut self, offset_hist: [u32; 3]) { + self.reapply_resident_dictionary_impl(offset_hist) + } + + #[inline] + fn prime_with_dictionary(&mut self, dict_content: &[u8], offset_hist: [u32; 3]) { + self.prime_with_dictionary_impl(dict_content, offset_hist) + } + + #[inline] + fn restore_primed_dictionary(&mut self, level: super::CompressionLevel) -> bool { + self.restore_primed_dictionary_impl(level) + } + + #[inline] + fn capture_primed_dictionary(&mut self, level: super::CompressionLevel) { + self.capture_primed_dictionary_impl(level) + } + + #[inline] + fn invalidate_primed_dictionary(&mut self) { + self.invalidate_primed_dictionary_impl() + } + + #[inline] + fn seed_dictionary_entropy( + &mut self, + huff: Option<&crate::huff0::huff0_encoder::HuffmanTable>, + ll: Option<&crate::fse::fse_encoder::FSETable>, + ml: Option<&crate::fse::fse_encoder::FSETable>, + of: Option<&crate::fse::fse_encoder::FSETable>, + ) { + self.seed_dictionary_entropy_impl(huff, ll, ml, of) + } + + fn window_size(&self) -> u64 { + self.reported_window_size as u64 + } + + fn get_next_space(&mut self) -> Vec { + if let Some(mut space) = self.vec_pool.pop() { + if space.len() > self.slice_size { + space.truncate(self.slice_size); + } + if space.len() < self.slice_size { + space.resize(self.slice_size, 0); + } + return space; + } + alloc::vec![0; self.slice_size] + } + + fn get_last_space(&mut self) -> &[u8] { + match &self.storage { + MatcherStorage::Simple(m) => m.last_committed_space(), + MatcherStorage::Dfast(m) => m.get_last_space(), + MatcherStorage::Row(m) => m.get_last_space(), + MatcherStorage::HashChain(m) => m.table.get_last_space(), + } + } + + fn commit_space(&mut self, space: Vec) { + let mut evicted_bytes = 0usize; + // Split borrows manually so the `add_data` closures can write + // into `vec_pool` while the backend itself holds an exclusive + // borrow via `storage`. (Suffix-store recycling went away + // with the legacy `MatchGenerator`; the FastKernelMatcher + // arm below has no pool interaction.) + let vec_pool = &mut self.vec_pool; + match &mut self.storage { + MatcherStorage::Simple(m) => { + // FastKernelMatcher owns its history as a single + // flat Vec and the hash table as a Vec — + // neither recycles into the driver-side pools. The + // eager pre-commit eviction inside + // `FastKernelMatcher::accept_data` drops bytes when + // accepting this block would push history past 2× + // max_window_size; that delta is what feeds + // `evicted_bytes` here via the `pre / post` + // history-length comparison. + let pre = m.history_len_for_eviction_accounting(); + m.accept_data(space); + let post = m.history_len_for_eviction_accounting(); + // `accept_data` performs eager pre-commit window + // eviction (so this `pre - post` delta correctly + // feeds the dictionary-budget retire flow). See + // `FastKernelMatcher::accept_data` for the + // commit-time-visibility rationale (closes #216 + // CodeRabbit review #5 / Copilot review #1: without + // eager eviction, the delta was always 0 and the + // dict budget never retired, leaving max_window_size + // inflated post-dict-prime → matcher could emit + // offsets exceeding the frame header's window). + evicted_bytes += pre.saturating_sub(post); + } + MatcherStorage::Dfast(m) => { + // Dfast's `add_data` callback receives the INPUT + // `Vec` for pool recycling (Dfast stores its + // bytes in the contiguous `history` buffer, not in + // per-block Vecs — there is no per-block buffer to + // pop off and hand back). Counting `data.len()` as + // evicted bytes would conflate "new bytes ingested" + // with "old bytes evicted from window"; the two + // happen to coincide when the previous window was + // saturated and the new input fills it 1:1, but + // diverge when the eviction pop-loop drops blocks + // of a different size than the incoming input. The + // `dictionary_retained_budget` retire decision + // downstream then gets driven by inflated eviction + // counts and shrinks `max_window_size` prematurely. + // + // Derive the real eviction delta from `window_size` + // before/after the call. The pop loop inside + // `add_data` decrements `window_size` by each + // evicted block length and then the final + // `extend_from_slice + push_back` adds `space_len`, + // so `evicted = pre + space_len - post`. + let pre = m.window_size; + let space_len = space.len(); + m.add_data(space, |data| { + // Same per-block recycle as the HashChain arm: push + // the spent input buffer back as-is rather than + // zero-filling to capacity. `add_data` mirrors the + // bytes into `history` and calls this every block, so + // capacity-wide zeroing would be hot-path waste; + // `get_next_space` zeroes at most `slice_size` bytes + // when it later reuses the buffer. + vec_pool.push(data); + }); + // Plain `+` (the `saturating_sub` floors at 0): `pre` + one + // block are byte counts bounded by the window, no overflow. + evicted_bytes += (pre + space_len).saturating_sub(m.window_size); + } + MatcherStorage::Row(m) => { + // RowMatchGenerator::add_data recycles the *input* buffer + // through this callback every commit (its bytes are mirrored + // into `history`), not the evicted chunks. Derive the eviction + // delta from `window_size` before/after — `evicted = pre + + // space_len - post` — exactly like the Simple / HashChain arms. + // Counting the callback argument as evicted would charge the + // whole committed block as evicted and prematurely retire + // dictionary budget on a window that evicts nothing. + let pre = m.window_size; + let space_len = space.len(); + m.add_data(space, |data| { + // Recycle the spent buffer as-is; `add_data` runs this for + // every committed block, so zero-filling to capacity here + // would be hot-path waste (`get_next_space` zeroes at most + // `slice_size` on reuse). + vec_pool.push(data); + }); + // Plain `+` (the `saturating_sub` floors at 0): `pre` + one + // block are byte counts bounded by the window, no overflow. + evicted_bytes += (pre + space_len).saturating_sub(m.window_size); + } + MatcherStorage::HashChain(m) => { + // MatchTable::add_data now recycles the *incoming* buffer + // through `reuse_space` (its bytes are copied into the + // contiguous `history` mirror), so the callback no longer + // reports evicted chunks. Derive the eviction delta from + // `window_size` before/after, exactly like the Simple arm: + // `evicted = pre + space_len - post`. + let pre = m.table.window_size; + let space_len = space.len(); + m.table.add_data(space, |data| { + // Recycle the spent input buffer to the pool as-is. + // `add_data` runs this callback for every committed + // block (the bytes are mirrored into `history`), so + // growing the buffer to its full capacity here would + // zero the whole allocation on the hot path. + // `get_next_space` resizes a popped buffer to + // `slice_size` on demand, touching at most + // `slice_size` bytes — never the larger capacity the + // pool retains. + vec_pool.push(data); + }); + // Plain `+` (the `saturating_sub` floors at 0): byte counts + // bounded by the window, no overflow. + evicted_bytes += (pre + space_len).saturating_sub(m.table.window_size); + } + } + // Gate the second backend trim pass on actual budget + // reclamation. Without it, every slice commit on the + // no-dictionary / no-eviction path (the common case) would + // run a backend `match` ladder + `trim_to_window` early-out + // for no reason — `trim_after_budget_retire` only does + // meaningful work when `retire_dictionary_budget` shrank + // `max_window_size` enough to make the backend's + // `window_size > max_window_size` invariant trigger + // eviction. + if self.retire_dictionary_budget(evicted_bytes) { + self.trim_after_budget_retire(); + } + } + + fn start_matching(&mut self, mut handle_sequence: impl for<'a> FnMut(Sequence<'a>)) { + use super::strategy::{self, StrategyTag}; + // Borrowed one-shot Fast path: if the frame driver staged a + // block range via `set_borrowed_block`, scan it in place against + // the borrowed window instead of the owned committed block. Only + // the Simple backend is instrumented (the gate guarantees it), + // and the stage is consumed so the next block re-stages. + if let Some((block_start, block_end)) = self.borrowed_pending.take() { + match self.active_backend() { + super::strategy::BackendTag::Simple => { + let m = self.simple_mut(); + if m.dict_is_attached() { + // Dict-attach borrowed scan: live matches read the + // borrowed input in place, dict matches read the + // committed dict prefix via the 2-segment counter. + m.start_matching_borrowed_dict( + block_start, + block_end, + &mut handle_sequence, + ); + } else { + m.start_matching_borrowed(block_start, block_end, &mut handle_sequence); + } + } + super::strategy::BackendTag::Dfast => self + .dfast_matcher_mut() + .start_matching_borrowed(block_start, block_end, &mut handle_sequence), + super::strategy::BackendTag::Row => { + // Same greedy/lazy parse split as the owned RowHash arm. + let greedy = self.parse == super::strategy::ParseMode::Greedy; + self.row_matcher_mut().start_matching_borrowed( + block_start, + block_end, + greedy, + &mut handle_sequence, + ); + } + super::strategy::BackendTag::HashChain => match self.search { + super::strategy::SearchMethod::HashChain => self + .hc_matcher_mut() + .start_matching_lazy_borrowed(block_start, block_end, &mut handle_sequence), + super::strategy::SearchMethod::BinaryTree => { + // Run the SAME BT dispatch as the owned BinaryTree arm + // below — every BT body reads its range via + // current_block_range() and bytes via live_history() + // (borrowed-aware), so the staged block is scanned in + // place. The table was already staged by + // `set_borrowed_block` (the HashChain arm at the top of + // this file calls `table.stage_borrowed_block` with the + // same range, and `borrowed_pending` is set only there), + // so no re-stage is needed here. + // Only btlazy2 reaches the borrowed BinaryTree scan: + // `borrowed_supported()` keeps the optimal parsers + // (BtOpt/BtUltra/BtUltra2) on the owned path, and + // `set_borrowed_block` asserts that predicate before any + // range is staged, so an optimal strategy_tag can never + // arrive here. + match self.strategy_tag { + StrategyTag::Btlazy2 => self + .hc_matcher_mut() + .start_matching_btlazy2(&mut handle_sequence), + other => unreachable!( + "borrowed BinaryTree scan is only supported for Btlazy2, got {other:?}" + ), + } + } + other => { + unreachable!("HashChain backend with unexpected search {other:?}") + } + }, + } + return; + } + // Decoupled parse×search dispatch (fires once per block). The + // search axis (`self.search`) picks the candidate-finding backend; + // the parse axis (greedy vs lazy depth) is carried by the + // backend's runtime `lazy_depth`, set per level at `reset()`. + // The two are independent, so any parse can run on any search + // backend. The `BinaryTree` arm still selects the opt `Strategy` + // ZST off `strategy_tag` so `compress_block::` keeps its + // const-folded optimal-parser monomorphisation. + use super::strategy::SearchMethod; + match self.search { + SearchMethod::Fast => { + self.simple_mut().start_matching(&mut handle_sequence); + self.recycle_simple_space(); + } + SearchMethod::DoubleFast => { + self.dfast_matcher_mut() + .start_matching(&mut handle_sequence); + } + SearchMethod::RowHash => { + // Greedy parse (depth 0) = upstream zstd-greedy entry (default + // `ip + 1` start, greedy repcode commit); lazy / lazy2 use + // the `pick_lazy_match` lookahead entry (reads `lazy_depth`). + // Both bare entries dispatch on `row_log` internally into the + // const-`ROW_LOG` hot loop (upstream zstd per-rowLog variant table). + let greedy = self.parse == super::strategy::ParseMode::Greedy; + let row = self.row_matcher_mut(); + if greedy { + row.start_matching_greedy(&mut handle_sequence); + } else { + row.start_matching(&mut handle_sequence); + } + } + SearchMethod::HashChain => { + // Greedy/lazy/lazy2 all flow through the lazy parser; it + // reads `hc.lazy_depth` (0 = greedy commit). + self.hc_matcher_mut() + .start_matching_lazy(&mut handle_sequence); + } + SearchMethod::BinaryTree => match self.strategy_tag { + StrategyTag::Btlazy2 => self + .hc_matcher_mut() + .start_matching_btlazy2(&mut handle_sequence), + StrategyTag::BtOpt => self.compress_block::(&mut handle_sequence), + StrategyTag::BtUltra => { + self.compress_block::(&mut handle_sequence) + } + StrategyTag::BtUltra2 => { + self.compress_block::(&mut handle_sequence) + } + _ => unreachable!( + "SearchMethod::BinaryTree requires a BT strategy tag (Btlazy2/BtOpt/BtUltra/BtUltra2)" + ), + }, + } + } + + fn skip_matching(&mut self) { + self.skip_matching_with_hint(None); + } + + fn skip_matching_with_hint(&mut self, incompressible_hint: Option) { + // Borrowed one-shot Fast path: a staged block range routes to the + // borrowed skip (records the range for `get_last_space`, primes + // hashes on the dict-priming hint) with no owned-history append + // and nothing to recycle. Stage is consumed. + if let Some((block_start, block_end)) = self.borrowed_pending.take() { + match self.active_backend() { + super::strategy::BackendTag::Simple => self.simple_mut().skip_matching_borrowed( + block_start, + block_end, + incompressible_hint, + ), + super::strategy::BackendTag::Dfast => self + .dfast_matcher_mut() + .skip_matching_borrowed(block_start, block_end, incompressible_hint), + super::strategy::BackendTag::Row => self.row_matcher_mut().skip_matching_borrowed( + block_start, + block_end, + incompressible_hint, + ), + super::strategy::BackendTag::HashChain => self + .hc_matcher_mut() + .skip_matching_borrowed(block_start, block_end, incompressible_hint), + } + return; + } + match self.active_backend() { + super::strategy::BackendTag::Simple => { + self.simple_mut() + .skip_matching_with_hint(incompressible_hint); + self.recycle_simple_space(); + } + super::strategy::BackendTag::Dfast => { + self.dfast_matcher_mut().skip_matching(incompressible_hint) + } + super::strategy::BackendTag::Row => self + .row_matcher_mut() + .skip_matching_with_hint(incompressible_hint), + super::strategy::BackendTag::HashChain => { + self.hc_matcher_mut().skip_matching(incompressible_hint) + } + } + } +} + +impl MatchGeneratorDriver { + /// Monomorphised optimal-parser entry point. Only the `BinaryTree` + /// search arm of [`Matcher::start_matching`] routes here, selecting + /// the concrete opt `S: Strategy` (BtOpt / BtUltra / BtUltra2) off + /// `strategy_tag`, so the optimiser keeps the cost-model predicates + /// (`S::USE_BT` / `S::USE_HASH3` / `S::ACCURATE_PRICE` / + /// `S::TWO_PASS_SEED`) const-folded per strategy. The non-opt search + /// backends (Fast / DoubleFast / RowHash / HashChain) are dispatched + /// directly off the search axis and never reach this method, so all + /// strategies arriving here are HashChain-backed. + fn compress_block( + &mut self, + handle_sequence: &mut impl for<'a> FnMut(Sequence<'a>), + ) { + debug_assert_eq!(S::BACKEND, super::strategy::BackendTag::HashChain); + debug_assert!( + S::USE_BT, + "compress_block only handles the optimal (BT) path" + ); + self.hc_matcher_mut() + .start_matching_strategy::(handle_sequence); + } +} + +/// Stage D: backend storage discriminator. +/// +/// HC (lazy / lazy2) modes carry no extra per-frame state beyond the +/// shared `MatchTable` and `HcMatcher` runtime knobs, so the +/// [`HcBackend::Hc`] variant is zero-sized — no BT scratch is +/// allocated. BT-flavoured modes (`btopt` / `btultra` / `btultra2`) +/// hold the full [`super::bt::BtMatcher`] inside the +/// [`HcBackend::Bt`] variant (cost model, optimal-parser scratch +/// arenas, LDM candidate buffer). +/// +/// The discriminator lives next to `parse_mode` so `configure()` can +/// promote between the two on a level change without touching the +/// `MatchTable` storage. +#[derive(Clone)] +pub(crate) enum HcBackend { + /// Lazy / lazy2 modes — no per-frame backend state. + Hc, + /// BT-driven modes — owns the optimal parser's per-frame scratch. + /// Boxed so the enum stays pointer-sized: HC-only matchers pay + /// just the `Box`-niche, not the 4 KiB `BtMatcher` payload. + Bt(alloc::boxed::Box), +} + +#[cfg(feature = "bench_internals")] +pub(crate) fn level22_block_ranges(data: &[u8]) -> Vec<(usize, usize)> { + let mut ranges = Vec::new(); + let mut cursor = 0usize; + let mut savings = 0i64; + while cursor < data.len() { + let remaining = data.len() - cursor; + let candidate_len = remaining.min(super::cost_model::HC_BLOCKSIZE_MAX); + let block_len = crate::encoding::frame_compressor::optimal_block_size( + CompressionLevel::Level(22), + &data[cursor..cursor + candidate_len], + remaining, + super::cost_model::HC_BLOCKSIZE_MAX, + savings, + ) + .min(candidate_len) + .max(1); + ranges.push((cursor, block_len)); + cursor += block_len; + // The exact upstream zstd gate uses compressed-size savings. For this corpus + // parity harness, after the first full block has compressed, savings is + // sufficient to authorize the same pre-block splitter path. + if cursor >= super::cost_model::HC_BLOCKSIZE_MAX { + savings = 3; + } + } + ranges +} + +#[cfg(feature = "bench_internals")] +fn merge_block_delimiters(sequences: Vec<(usize, usize, usize)>) -> Vec<(usize, usize, usize)> { + let mut out = Vec::with_capacity(sequences.len()); + let mut pending_lits = 0usize; + for (lit_len, offset, match_len) in sequences { + if offset == 0 && match_len == 0 { + pending_lits = pending_lits.saturating_add(lit_len); + continue; + } + out.push((lit_len.saturating_add(pending_lits), offset, match_len)); + pending_lits = 0; + } + if pending_lits > 0 { + out.push((pending_lits, 0, 0)); + } + out +} + +/// White-box capture of the level-22 sequence stream (literal-length, +/// offset, match-length triples) the match generator emits for `data`, +/// with block-delimiter pseudo-sequences merged into the following +/// triple's literal run. Pure Rust; the C-conformance comparison that +/// consumes it lives in the `ffi-bench` crate. +#[cfg(feature = "bench_internals")] +pub(crate) fn collect_level22_sequences(data: &[u8]) -> Vec<(usize, usize, usize)> { + merge_block_delimiters(collect_level22_sequences_with_delimiters(data)) + .into_iter() + .filter(|(_, offset, match_len)| *offset != 0 || *match_len != 0) + .collect() +} + +#[cfg(feature = "bench_internals")] +fn collect_level22_sequences_with_delimiters(data: &[u8]) -> Vec<(usize, usize, usize)> { + let mut driver = MatchGeneratorDriver::new(super::cost_model::HC_BLOCKSIZE_MAX, 1); + driver.set_source_size_hint(data.len() as u64); + driver.reset(CompressionLevel::Level(22)); + + let mut sequences = Vec::new(); + for (chunk_start, chunk_len) in level22_block_ranges(data) { + let chunk = &data[chunk_start..chunk_start + chunk_len]; + let mut space = driver.get_next_space(); + space[..chunk.len()].copy_from_slice(chunk); + space.truncate(chunk.len()); + driver.commit_space(space); + driver.start_matching(|seq| { + let entry = match seq { + Sequence::Literals { literals } => (literals.len(), 0usize, 0usize), + Sequence::Triple { + literals, + offset, + match_len, + } => (literals.len(), offset, match_len), + }; + sequences.push(entry); + }); + } + sequences +} + +#[cfg(test)] +mod tests; diff --git a/zstd/src/encoding/match_generator/tests.rs b/zstd/src/encoding/match_generator/tests.rs new file mode 100644 index 000000000..bde735842 --- /dev/null +++ b/zstd/src/encoding/match_generator/tests.rs @@ -0,0 +1,4886 @@ +//! Unit and round-trip tests for the match-generator driver: the +//! parse x search matrix, per-level parameter resolution, greedy / lazy / +//! optimal round-trips, and dictionary-priming behaviour. + +use super::*; + +#[cfg(test)] +impl MatchGeneratorDriver { + /// Test-only: stage a parse×search recipe override applied on the + /// next `reset()`. Routes a level through a non-default (parse, + /// search) pair so the decoupling can be exercised end-to-end. + pub(crate) fn set_config_override( + &mut self, + search: super::super::strategy::SearchMethod, + parse: super::super::strategy::ParseMode, + ) { + self.config_override = Some((search, parse)); + } + + /// Test-only: reset `level` routed onto the lazy HashChain pairing. + /// The lazy band runs on the Row backend in production, so HC-specific + /// behaviour (live-chain dict prime, eviction budget accounting, seed + /// pass gates) is exercised through this override-backed reset. + pub(crate) fn reset_on_hc_lazy(&mut self, level: CompressionLevel) { + self.set_config_override( + super::super::strategy::SearchMethod::HashChain, + super::super::strategy::ParseMode::Lazy2, + ); + self.reset(level); + } +} + +/// Drive a full compress parse for `data` at `level` (optionally with a +/// parse×search override) and reconstruct the bytes from the emitted +/// sequences. The returned buffer must equal `data` for a correct parse. +#[cfg(test)] +fn drive_roundtrip_with_override( + level: CompressionLevel, + over: Option<( + super::super::strategy::SearchMethod, + super::super::strategy::ParseMode, + )>, + data: &[u8], +) -> Vec { + let mut driver = MatchGeneratorDriver::new(1 << 17, 8); + if let Some((s, p)) = over { + driver.set_config_override(s, p); + } + driver.reset(level); + + let mut out: Vec = Vec::with_capacity(data.len()); + let mut offset_in_data = 0usize; + while offset_in_data < data.len() { + let mut space = driver.get_next_space(); + let take = (data.len() - offset_in_data).min(space.len()); + space[..take].copy_from_slice(&data[offset_in_data..offset_in_data + take]); + space.truncate(take); + driver.commit_space(space); + offset_in_data += take; + + driver.start_matching(|seq| match seq { + Sequence::Literals { literals } => out.extend_from_slice(literals), + Sequence::Triple { + literals, + offset, + match_len, + } => { + out.extend_from_slice(literals); + let start = out.len() - offset; + for i in 0..match_len { + let byte = out[start + i]; + out.push(byte); + } + } + }); + } + out +} + +/// Phase 1 capability proof: parse and search are decoupled, so a level +/// can run any parse mode on any non-opt search backend. Greedy-on- +/// HashChain and Lazy2-on-RowHash are pairings the legacy `strategy_tag` +/// could not express; both must reconstruct the input exactly. +#[test] +fn parse_search_matrix_decoupled_roundtrips() { + use super::super::strategy::{ParseMode, SearchMethod}; + // Mixed repetitive + literal payload that exercises matches and reps. + let mut data = Vec::new(); + for i in 0..4000u32 { + data.extend_from_slice(b"the quick brown fox "); + data.extend_from_slice(&i.to_le_bytes()); + } + + // Greedy parse on the HashChain search backend (legacy: Greedy was + // welded to RowHash). + let got = drive_roundtrip_with_override( + CompressionLevel::Level(5), + Some((SearchMethod::HashChain, ParseMode::Greedy)), + &data, + ); + assert_eq!(got, data, "greedy-on-hashchain diverged"); + + // Lazy2 parse on the RowHash search backend (legacy: Lazy was welded + // to HashChain). + let got = drive_roundtrip_with_override( + CompressionLevel::Level(8), + Some((SearchMethod::RowHash, ParseMode::Lazy2)), + &data, + ); + assert_eq!(got, data, "lazy2-on-rowhash diverged"); + + // Lazy on RowHash too (depth 1). + let got = drive_roundtrip_with_override( + CompressionLevel::Level(6), + Some((SearchMethod::RowHash, ParseMode::Lazy)), + &data, + ); + assert_eq!(got, data, "lazy-on-rowhash diverged"); +} + +/// The row `mls` knob (C-like `minMatch`) is respected: every accepted +/// match (regular row + repcode, on the lazy parse) is at least `mls` +/// bytes, and the stream still round-trips for the whole 4..=7 range. The +/// default (5) reproduces the historical `ROW_MIN_MATCH_LEN` behaviour. +#[test] +fn row_mls_knob_gates_matches_and_roundtrips() { + let data: Vec = (0..4000u32) + .flat_map(|i| { + let mut v = b"abcdefgh".to_vec(); + v.extend_from_slice(&i.to_le_bytes()); + v + }) + .collect(); + + for mls in [4usize, 5, 6, 7] { + let mut matcher = RowMatchGenerator::new(1 << 22); + let mut cfg = ROW_CONFIG; + cfg.mls = mls; + matcher.configure(cfg); + matcher.add_data(data.clone(), |_| {}); + + let mut out: Vec = Vec::with_capacity(data.len()); + let mut shortest_match = usize::MAX; + matcher.start_matching(|seq| match seq { + Sequence::Literals { literals } => out.extend_from_slice(literals), + Sequence::Triple { + literals, + offset, + match_len, + } => { + out.extend_from_slice(literals); + shortest_match = shortest_match.min(match_len); + let start = out.len() - offset; + for i in 0..match_len { + let byte = out[start + i]; + out.push(byte); + } + } + }); + + assert_eq!(out, data, "mls={mls} round-trip diverged"); + if shortest_match != usize::MAX { + assert!( + shortest_match >= mls, + "mls={mls}: emitted a {shortest_match}-byte match below the floor", + ); + } + } +} + +/// `LevelParams::parse()` derives the parse mode from the `search` axis, not +/// the strategy tag, so the decoupling holds even for a `Bt*`-tagged level +/// overridden to a non-BT search backend. Pre-fix the method matched on +/// `strategy_tag` and returned `Optimal` for any `Bt*` tag regardless of +/// `search`/`lazy_depth`. +#[test] +fn parse_mode_follows_search_axis_not_strategy_tag() { + use super::super::strategy::{ParseMode, SearchMethod}; + // LEVEL_TABLE[15] is level 16: BtOpt tag, BinaryTree search. + let mut p = LEVEL_TABLE[15]; + assert_eq!(p.parse(), ParseMode::Optimal, "BinaryTree search → Optimal"); + // Override the Bt-tagged level's search to a non-BT backend: parse must + // follow the search axis (derive from lazy_depth), not stay Optimal. + p.search = SearchMethod::RowHash; + p.lazy_depth = 0; + assert_eq!(p.parse(), ParseMode::Greedy, "RowHash + depth 0 → Greedy"); + p.lazy_depth = 2; + assert_eq!(p.parse(), ParseMode::Lazy2, "RowHash + depth 2 → Lazy2"); +} + +/// The test-only `config_override` is consumed by the first `reset()` (one +/// shot), so a reused driver does not silently keep the synthetic pairing +/// armed across later resets. Pre-fix `reset()` copied the override and left +/// it set. +#[test] +fn config_override_is_consumed_by_reset() { + use super::super::strategy::{ParseMode, SearchMethod}; + let mut driver = MatchGeneratorDriver::new(1 << 17, 8); + driver.set_config_override(SearchMethod::RowHash, ParseMode::Lazy2); + assert!(driver.config_override.is_some()); + driver.reset(CompressionLevel::Level(5)); + assert!( + driver.config_override.is_none(), + "override must be consumed after one reset", + ); +} + +// Level 4 maps to the greedy Dfast (double-fast) backend — "greedy" here is the +// parse discipline (no lazy lookahead, upstream zstd `ZSTD_dfast`), NOT the Row/Greedy +// strategy (which is Level 5). This roundtrip is intentional Dfast L4 coverage; +// the Row backend is exercised by the `Level(5)` fixtures elsewhere in this file. +#[cfg(test)] +fn l4_greedy_round_trip(slice_size: usize, max_slices: usize, data: &[u8]) -> (usize, usize) { + let mut driver = MatchGeneratorDriver::new(slice_size, max_slices); + driver.reset(CompressionLevel::Level(4)); + + let mut reconstructed: Vec = Vec::with_capacity(data.len()); + let mut triple_count = 0usize; + let mut max_offset = 0usize; + + // `start_matching` consumes the current pending slice; multi-slice + // payloads require commit + drive per slice so earlier slices' + // bytes actually round-trip out before they're displaced from the + // window. + let mut offset_in_data = 0usize; + while offset_in_data < data.len() { + let mut space = driver.get_next_space(); + let space_cap = space.len(); + let take = (data.len() - offset_in_data).min(space_cap); + space[..take].copy_from_slice(&data[offset_in_data..offset_in_data + take]); + space.truncate(take); + driver.commit_space(space); + offset_in_data += take; + + driver.start_matching(|seq| match seq { + Sequence::Literals { literals } => reconstructed.extend_from_slice(literals), + Sequence::Triple { + literals, + offset, + match_len, + } => { + triple_count += 1; + if offset > max_offset { + max_offset = offset; + } + reconstructed.extend_from_slice(literals); + let start = reconstructed.len() - offset; + for i in 0..match_len { + let byte = reconstructed[start + i]; + reconstructed.push(byte); + } + } + }); + } + + // Empty payload still needs one commit/drive round so the empty- + // input path of `start_matching_greedy` (the `current_len == 0` + // early-return guard) gets exercised. + if data.is_empty() { + let mut space = driver.get_next_space(); + space.truncate(0); + driver.commit_space(space); + driver.start_matching(|seq| match seq { + Sequence::Literals { literals } => reconstructed.extend_from_slice(literals), + Sequence::Triple { .. } => panic!("empty input must not emit any matches"), + }); + } + + assert_eq!(reconstructed, data, "L4 greedy round-trip diverged"); + (triple_count, max_offset) +} + +/// CodeRabbit-flagged tail rep-only case: the previous outer-loop +/// guard `pos + ROW_MIN_MATCH_LEN <= current_len` (6) meant the last +/// 5-byte position was unreachable. The rep probe at `abs_pos + 1` +/// only needs 4 bytes of lookahead beyond the probe point, so the +/// guard was relaxed to `pos + GREEDY_MIN_LOOKAHEAD <= current_len` +/// (5). This test drives the slices separately and asserts a match +/// is emitted **from the second slice's parse pass**, so a future +/// regression that re-tightens the guard or breaks the cross-slice +/// repcode lookup fails the test instead of being masked by +/// first-slice matches. +#[test] +fn driver_level5_greedy_tail_rep_only_reachable() { + // Period-4 first slice locks rep1 = 4 into `offset_hist` by the + // time the parse reaches the slice tail. Second slice is exactly + // 5 bytes ( = `GREEDY_MIN_LOOKAHEAD`) so the outer loop runs + // **once** at `pos = 0`; the regular `row_candidate` requires 6 + // bytes from `abs_pos`, which is past the live history, so the + // only viable hit is the `abs_pos + 1` rep probe. `second[0..]` + // is shaped so the rep probe at `abs_pos + 1` finds a 4-byte + // match at offset 4 (`second[1..5] == first[13..16] ++ second[0] + // == "BCDA"`), and `extend_backwards_shared` then absorbs + // `second[0]` into the match (extending one byte back into the + // implicit anchor, no further because anchor itself is the + // current `abs_pos`). + let first: &[u8] = b"ABCDABCDABCDABCD"; // 16 bytes — strict period 4 + let second: &[u8] = b"ABCDA"; // 5 bytes — exact GREEDY_MIN_LOOKAHEAD + let mut driver = MatchGeneratorDriver::new(16, 2); + driver.reset(CompressionLevel::Level(5)); + + let mut first_space = driver.get_next_space(); + first_space[..first.len()].copy_from_slice(first); + first_space.truncate(first.len()); + driver.commit_space(first_space); + driver.start_matching(|_| {}); + + let mut second_space = driver.get_next_space(); + second_space[..second.len()].copy_from_slice(second); + second_space.truncate(second.len()); + driver.commit_space(second_space); + + let mut second_slice_triples = 0usize; + driver.start_matching(|seq| { + if matches!(seq, Sequence::Triple { .. }) { + second_slice_triples += 1; + } + }); + + assert!( + second_slice_triples >= 1, + "tail rep-only position must produce a match in the second slice \ + (got {second_slice_triples} triples)", + ); +} + +#[test] +fn driver_level4_greedy_empty_input_emits_nothing() { + // Empty input: no slices committed → no sequences emitted, no + // panic. Exercises the `current_len == 0` early-return guard at + // the top of `start_matching_greedy`. + let mut driver = MatchGeneratorDriver::new(64, 2); + driver.reset(CompressionLevel::Level(4)); + // Commit an empty space so the matcher has SOMETHING to start + // matching on (otherwise `start_matching` panics on the + // `window.back()` unwrap — that's a separate path covered by + // existing reset tests). + let mut space = driver.get_next_space(); + space.truncate(0); + driver.commit_space(space); + let mut emitted_anything = false; + driver.start_matching(|_| emitted_anything = true); + assert!(!emitted_anything, "empty slice must not emit any sequences",); +} + +#[test] +fn driver_level4_greedy_sub_min_lookahead_input() { + // Input shorter than `GREEDY_MIN_LOOKAHEAD = 5` — the outer loop + // never executes a body iteration; the tail literal path must + // still emit the input bytes as a single `Sequence::Literals`. + let data: &[u8] = b"abcd"; // 4 bytes + let (triples, _) = l4_greedy_round_trip(64, 2, data); + assert_eq!( + triples, 0, + "sub-min-lookahead input must not emit any matches (got {triples})", + ); +} + +#[test] +fn driver_level4_greedy_incompressible_input() { + // Pseudo-random bytes with no exploitable structure — every + // position is a "miss" in both the rep probe and the row + // candidate. Exercises the miss branch + `SKIP_STRENGTH = 10` + // skip-step grow (irrelevant at this size, but the path runs). + let mut data = alloc::vec::Vec::with_capacity(256); + let mut x: u32 = 0xDEAD_BEEF; + for _ in 0..256 { + x = x.wrapping_mul(1_103_515_245).wrapping_add(12345); + data.push((x >> 16) as u8); + } + let (_triples, _) = l4_greedy_round_trip(64, 8, &data); + // No structural assertion — the test passes if round-trip is + // bit-exact and no panic / debug_assert fires. +} + +#[test] +fn driver_level4_greedy_long_literal_run_skip_step_growth() { + // 2 KiB of unstructured bytes drives the literal-run length past + // the `SKIP_STRENGTH = 10` threshold (~1 KiB), so the miss branch + // + per-miss step-grow path in `start_matching_greedy` is + // exercised. This test is a stress smoke — it only asserts + // bit-exact round-trip + no panic / `debug_assert!` fires; it + // does NOT pin the `SKIP_STRENGTH` constant or the per-iteration + // step count (round-trip would still pass on `SKIP_STRENGTH = 6` + // or `= 14` since both produce valid sequences). Pinning the + // exact step growth would require returning step / iteration + // metadata from the parse, which is invasive plumbing for a + // constant that hasn't been re-tuned in months. The value of + // this test is catching panics or correctness regressions on + // long incompressible runs, which is what its existing + // round-trip assertion checks. + let mut data = alloc::vec::Vec::with_capacity(2048); + let mut x: u32 = 0xC0FF_EE00; + for _ in 0..2048 { + x = x.wrapping_mul(0x9E37_79B9).wrapping_add(0xCAFEBABE); + data.push((x >> 24) as u8); + } + let (_triples, _) = l4_greedy_round_trip(512, 8, &data); +} + +#[test] +fn driver_level4_greedy_all_zeros_heavy_rep1() { + // All zeros: every position after the first byte has `byte[pos] + // == byte[pos - 1]`, so the rep1 probe at `abs_pos + 1` hits + // immediately and the parse collapses to a single long match. + // Exercises the `cheap rep at +1, full-match length` path. + let data: Vec = alloc::vec![0u8; 128]; + let (triples, max_offset) = l4_greedy_round_trip(64, 8, &data); + assert!( + triples >= 1, + "all-zeros input must produce at least one rep1 match", + ); + // The dominant match should reference rep1 (offset 1), since + // every byte at pos matches pos-1. A larger offset would + // indicate the rep1 probe was bypassed. + assert_eq!( + max_offset, 1, + "all-zeros L4 greedy parse should commit at offset 1 (got {max_offset})", + ); +} + +/// Periodic-pattern payload covers the steady-state rep-cascade path +/// of the greedy parse — the main-loop rep probe at `abs_pos + 1` +/// fires every iteration once the period is locked into +/// `offset_hist[0]`, and the parse emits a long chain of triples at +/// the same offset. +#[test] +fn driver_level4_greedy_periodic_pattern_rep_cascade() { + let unit: &[u8] = b"alpha_beta_gamma"; + assert_eq!(unit.len(), 16); + let mut data: Vec = Vec::with_capacity(unit.len() * 32); + for _ in 0..32 { + data.extend_from_slice(unit); + } + let (triples, max_offset) = l4_greedy_round_trip(64, 16, &data); + assert!( + triples >= 1, + "periodic 16-byte payload must emit matches (got {triples})", + ); + assert!( + max_offset >= 16, + "periodic 16-byte payload must produce at least one offset >= 16 \ + (got max_offset = {max_offset})", + ); +} + +#[test] +fn driver_reset_keeps_strategy_tag_in_sync_with_active_backend() { + use super::super::strategy::StrategyTag; + + fn check(level: CompressionLevel, expected: StrategyTag) { + let mut driver = MatchGeneratorDriver::new(32, 2); + driver.reset(level); + assert_eq!( + driver.strategy_tag, expected, + "strategy_tag wrong for {level:?}" + ); + assert_eq!( + driver.strategy_tag.backend(), + driver.active_backend(), + "strategy_tag backend disagrees with active_backend for {level:?}" + ); + } + + check(CompressionLevel::Level(1), StrategyTag::Fast); + check(CompressionLevel::Level(2), StrategyTag::Fast); + check(CompressionLevel::Level(3), StrategyTag::Dfast); + check(CompressionLevel::Level(4), StrategyTag::Dfast); + check(CompressionLevel::Level(5), StrategyTag::Greedy); + check(CompressionLevel::Level(7), StrategyTag::Lazy); + check(CompressionLevel::Level(12), StrategyTag::Lazy); + check(CompressionLevel::Level(13), StrategyTag::Btlazy2); + check(CompressionLevel::Level(14), StrategyTag::Btlazy2); + check(CompressionLevel::Level(15), StrategyTag::Btlazy2); + check(CompressionLevel::Level(16), StrategyTag::BtOpt); + check(CompressionLevel::Level(18), StrategyTag::BtUltra); + check(CompressionLevel::Level(22), StrategyTag::BtUltra2); + check(CompressionLevel::Fastest, StrategyTag::Fast); + check(CompressionLevel::Default, StrategyTag::Dfast); + check(CompressionLevel::Better, StrategyTag::Lazy); + // `Best` sits on level 13 (the first dominant point of the deep band). + check(CompressionLevel::Best, StrategyTag::Btlazy2); +} + +#[test] +fn level_16_17_map_to_btopt_strategy() { + use super::super::strategy::{BackendTag, StrategyTag}; + let p16 = resolve_level_params(CompressionLevel::Level(16), None); + let p17 = resolve_level_params(CompressionLevel::Level(17), None); + assert_eq!(p16.backend(), BackendTag::HashChain); + assert_eq!(p17.backend(), BackendTag::HashChain); + assert_eq!(StrategyTag::for_level(16), StrategyTag::BtOpt); + assert_eq!(StrategyTag::for_level(17), StrategyTag::BtOpt); +} + +#[test] +fn level_18_maps_to_btultra_level_19_to_btultra2_strategy() { + use super::super::strategy::{BackendTag, StrategyTag}; + // Upstream zstd `clevels.h` (srcSize > 256 KiB tier): level 18 = `ZSTD_btultra`, + // level 19 = `ZSTD_btultra2`. Level 19 was previously mapped to plain + // btultra, which under-searched (searchLog 6 vs 7) and lost ~3.7% ratio + // on the repo corpus. + let p18 = resolve_level_params(CompressionLevel::Level(18), None); + let p19 = resolve_level_params(CompressionLevel::Level(19), None); + assert_eq!(p18.backend(), BackendTag::HashChain); + assert_eq!(p19.backend(), BackendTag::HashChain); + assert_eq!(StrategyTag::for_level(18), StrategyTag::BtUltra); + assert_eq!(StrategyTag::for_level(19), StrategyTag::BtUltra2); +} + +#[test] +fn level_20_22_map_to_btultra2_strategy() { + use super::super::strategy::{BackendTag, StrategyTag}; + for level in 20..=22 { + let params = resolve_level_params(CompressionLevel::Level(level), None); + assert_eq!(params.backend(), BackendTag::HashChain); + assert_eq!(StrategyTag::for_level(level as u8), StrategyTag::BtUltra2); + } +} + +#[test] +fn level22_uses_target_length_and_large_input_tables() { + let params = resolve_level_params(CompressionLevel::Level(22), None); + assert_eq!(params.window_log, 27); + let hc = params.hc.unwrap(); + assert_eq!(hc.hash_log, 25); + assert_eq!(hc.chain_log, 27); + assert_eq!(hc.search_depth, 1 << 9); + assert_eq!(hc.target_len, 999); +} + +#[test] +fn bt_levels_16_to_21_pin_clevels_params() { + // Pins the BT-level (window_log, hash_log, chain_log, search_depth, + // target_len) tuples so the clevels.h alignment cannot silently drift. + // Levels 16-20 mirror upstream `clevels.h` (srcSize > 256 KiB tier, + // search_depth = 1 << searchLog); level 21 intentionally keeps a deeper + // search_depth (512 vs upstream's 128) — it beats C on ratio there and + // the deeper walk is a deliberate ratio-positive divergence. + let expected = [ + // (level, window_log, hash_log, chain_log, search_depth, target_len) + (16u8, 22u8, 22usize, 22usize, 32usize, 48usize), + (17, 23, 22, 23, 32, 64), + (18, 23, 22, 23, 64, 64), + (19, 23, 22, 24, 128, 256), + (20, 25, 23, 25, 128, 256), + (21, 26, 24, 24, 512, 256), + ]; + for (level, wlog, hlog, clog, sd, tl) in expected { + let p = resolve_level_params(CompressionLevel::Level(level as i32), None); + assert_eq!(p.window_log, wlog, "level {level} window_log"); + let hc = p.hc.unwrap(); + assert_eq!(hc.hash_log, hlog, "level {level} hash_log"); + assert_eq!(hc.chain_log, clog, "level {level} chain_log"); + assert_eq!(hc.search_depth, sd, "level {level} search_depth"); + assert_eq!(hc.target_len, tl, "level {level} target_len"); + } +} + +#[test] +fn level22_source_size_hint_uses_btultra2_tiers() { + let p16k = resolve_level_params(CompressionLevel::Level(22), Some(16 * 1024)); + assert_eq!(p16k.window_log, 14); + let hc16k = p16k.hc.unwrap(); + assert_eq!(hc16k.hash_log, 15); + assert_eq!(hc16k.chain_log, 15); + assert_eq!(hc16k.search_depth, 1 << 10); + assert_eq!(hc16k.target_len, 999); + + let p128k = resolve_level_params(CompressionLevel::Level(22), Some(128 * 1024)); + assert_eq!(p128k.window_log, 17); + let hc128k = p128k.hc.unwrap(); + assert_eq!(hc128k.hash_log, 17); + assert_eq!(hc128k.chain_log, 18); + assert_eq!(hc128k.search_depth, 1 << 11); + assert_eq!(hc128k.target_len, 999); + + let p256k = resolve_level_params(CompressionLevel::Level(22), Some(256 * 1024)); + assert_eq!(p256k.window_log, 18); + let hc256k = p256k.hc.unwrap(); + assert_eq!(hc256k.hash_log, 19); + assert_eq!(hc256k.chain_log, 19); + assert_eq!(hc256k.search_depth, 1 << 13); + assert_eq!(hc256k.target_len, 999); +} + +#[test] +fn level22_non_power_of_two_small_source_uses_tier3_params() { + // srcSize 15 027 (<= 16 KB) selects the table[3] btultra2 row; the + // source-size clamp gives windowLog 14 (ceil log2 15027). Pure-Rust + // assertion against the constant tier-3 geometry (no FFI). + let source_size = 15_027u64; + let params = resolve_level_params(CompressionLevel::Level(22), Some(source_size)); + + let hc = params.hc.unwrap(); + assert_eq!(params.window_log, 14); + assert_eq!(hc.chain_log, 15); + assert_eq!(hc.hash_log, 15); + assert_eq!(hc.search_depth, 1 << 10); + assert_eq!(HC_OPT_MIN_MATCH_LEN, 3); + assert_eq!(hc.target_len, 999); +} + +#[test] +fn level22_small_source_uses_window_bounded_hash3_log() { + let mut hc = HcMatchGenerator::new(1 << 14); + hc.configure( + BTULTRA2_HC_CONFIG_L22_16K, + super::super::strategy::StrategyTag::BtUltra2, + 14, + ); + assert_eq!(hc.table.hash3_log, 14); + + hc.configure( + BTULTRA2_HC_CONFIG_L22, + super::super::strategy::StrategyTag::BtUltra2, + 27, + ); + assert_eq!(hc.table.hash3_log, HC3_HASH_LOG); +} + +#[test] +fn btultra2_seed_pass_initializes_opt_state() { + let mut hc = HcMatchGenerator::new(1 << 20); + hc.configure( + BTULTRA2_HC_CONFIG, + super::super::strategy::StrategyTag::BtUltra2, + 26, + ); + let data: Vec = (0..32 * 1024).map(|i| (i % 251) as u8).collect(); + hc.table.add_data(data, |_| {}); + hc.start_matching(|_| {}); + assert!( + hc.backend.bt_mut().opt_state.lit_length_sum > 0, + "btultra2 first block should seed non-zero sequence statistics" + ); + assert!( + hc.backend.bt_mut().opt_state.off_code_sum > 0, + "btultra2 first block should seed offset-code statistics" + ); +} + +#[test] +fn btultra2_profile_disables_small_offset_handicap() { + // Pre-Phase-3 this test duplicated the profile build with + // `pass2=false` and `pass2=true` since `for_mode` differentiated + // them. With `const_for_strategy::()` there is only one + // profile — the upstream zstd `opt2` pricing — so a single binding + // captures the invariant the test is asserting. + let profile = HcOptimalCostProfile::const_for_strategy::(); + assert!( + !profile.favor_small_offsets, + "btultra2 should match upstream zstd opt2 offset pricing" + ); + assert!( + profile.accurate, + "btultra2 should use upstream zstd opt2 accurate pricing" + ); +} + +#[test] +fn btultra_profile_keeps_search_depth_budget() { + let p = HcOptimalCostProfile::const_for_strategy::(); + assert_eq!( + p.max_chain_depth, 64, + "btultra chain-depth budget must match clevels.h level 18 searchLog 6 (1 << 6 = 64)" + ); +} + +#[test] +fn btopt_profile_keeps_search_depth_budget() { + let p = HcOptimalCostProfile::const_for_strategy::(); + assert_eq!( + p.max_chain_depth, 32, + "btopt should not cap chain depth below upstream zstd btopt search budget" + ); +} + +#[test] +fn sufficient_match_len_is_clamped_by_target_len() { + let mut hc = HcMatchGenerator::new(1 << 20); + hc.configure( + BTULTRA2_HC_CONFIG, + super::super::strategy::StrategyTag::BtUltra2, + 26, + ); + hc.hc.target_len = 13; + let profile = HcOptimalCostProfile::const_for_strategy::(); + assert_eq!(hc.hc.sufficient_match_len_for_pass(profile), 13); +} + +#[test] +fn opt_modes_use_target_len_as_sufficient_len() { + use super::super::strategy; + let mut hc = HcMatchGenerator::new(1 << 20); + hc.hc.target_len = 57; + let profiles = [ + HcOptimalCostProfile::const_for_strategy::(), + HcOptimalCostProfile::const_for_strategy::(), + HcOptimalCostProfile::const_for_strategy::(), + ]; + for profile in profiles { + assert_eq!(hc.hc.sufficient_match_len_for_pass(profile), 57); + } +} + +#[test] +fn sufficient_match_len_is_capped_by_opt_num() { + let mut hc = HcMatchGenerator::new(1 << 20); + hc.hc.target_len = usize::MAX / 2; + let profile = HcOptimalCostProfile::const_for_strategy::(); + assert_eq!(hc.hc.sufficient_match_len_for_pass(profile), HC_OPT_NUM - 1); +} + +#[test] +#[allow(clippy::borrow_deref_ref)] +fn dictionary_entropy_seed_initializes_opt_state_from_tables() { + let mut hc = HcMatchGenerator::new(1 << 20); + hc.configure( + BTULTRA2_HC_CONFIG, + super::super::strategy::StrategyTag::BtUltra2, + 26, + ); + + let huff = crate::huff0::huff0_encoder::HuffmanTable::build_from_data( + b"aaabbbbccccddddeeeeefffffgggg", + ); + let ll = crate::fse::fse_encoder::default_ll_table(); + let ml = crate::fse::fse_encoder::default_ml_table(); + let of = crate::fse::fse_encoder::default_of_table(); + hc.seed_dictionary_entropy(Some(&huff), Some(&*ll), Some(&*ml), Some(&*of)); + + hc.backend.bt_mut().opt_state.rescale_freqs( + b"abcd", + HcOptimalCostProfile::const_for_strategy::(), + ); + + let base_ll_freqs: [u32; HC_MAX_LL + 1] = [ + 4, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, + ]; + + assert_ne!( + hc.backend.bt_mut().opt_state.lit_length_freq, + base_ll_freqs, + "dictionary entropy should override fallback LL bootstrap frequencies" + ); + assert!( + hc.backend + .bt_mut() + .opt_state + .match_length_freq + .iter() + .any(|&v| v != 1), + "dictionary entropy should seed non-uniform ML frequencies" + ); + assert_ne!( + hc.backend.bt_mut().opt_state.off_code_freq[0], + 6, + "dictionary entropy should override fallback OF bootstrap frequencies" + ); +} + +#[test] +#[allow(clippy::borrow_deref_ref)] +fn dictionary_fse_seed_applies_without_huffman_seed() { + let mut hc = HcMatchGenerator::new(1 << 20); + hc.configure( + BTULTRA2_HC_CONFIG, + super::super::strategy::StrategyTag::BtUltra2, + 26, + ); + + let ll = crate::fse::fse_encoder::default_ll_table(); + let ml = crate::fse::fse_encoder::default_ml_table(); + let of = crate::fse::fse_encoder::default_of_table(); + hc.seed_dictionary_entropy(None, Some(&*ll), Some(&*ml), Some(&*of)); + hc.backend.bt_mut().opt_state.rescale_freqs( + b"abcd", + HcOptimalCostProfile::const_for_strategy::(), + ); + + let base_ll_freqs: [u32; HC_MAX_LL + 1] = [ + 4, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, + ]; + assert_ne!( + hc.backend.bt_mut().opt_state.lit_length_freq, + base_ll_freqs, + "FSE seed should still override LL bootstrap frequencies without huffman seed" + ); + assert!( + hc.backend + .bt_mut() + .opt_state + .match_length_freq + .iter() + .any(|&v| v != 1), + "FSE seed should still seed non-uniform ML frequencies" + ); + assert_ne!( + hc.backend.bt_mut().opt_state.off_code_freq[0], + 6, + "FSE seed should still override OF bootstrap frequencies without huffman seed" + ); +} + +#[test] +#[allow(clippy::borrow_deref_ref)] +fn dictionary_seed_overrides_predef_price_mode_on_tiny_input() { + let mut hc = HcMatchGenerator::new(1 << 20); + hc.configure( + BTULTRA2_HC_CONFIG, + super::super::strategy::StrategyTag::BtUltra2, + 26, + ); + + let ll = crate::fse::fse_encoder::default_ll_table(); + let ml = crate::fse::fse_encoder::default_ml_table(); + let of = crate::fse::fse_encoder::default_of_table(); + hc.seed_dictionary_entropy(None, Some(&*ll), Some(&*ml), Some(&*of)); + hc.backend.bt_mut().opt_state.rescale_freqs( + b"abc", + HcOptimalCostProfile::const_for_strategy::(), + ); + assert!( + matches!( + hc.backend.bt_mut().opt_state.price_type, + HcOptPriceType::Dynamic + ), + "dictionary-seeded first block should stay in dynamic mode even for tiny src" + ); +} + +#[test] +fn lit_length_price_blocksize_max_costs_one_extra_bit() { + let profile_predef = + HcOptimalCostProfile::const_for_strategy::(); + let mut stats_predef = HcOptState::new(); + stats_predef.price_type = HcOptPriceType::Predefined; + let predef_max = profile_predef.lit_length_price(&stats_predef, HC_BLOCKSIZE_MAX); + let predef_prev = + profile_predef.lit_length_price(&stats_predef, HC_BLOCKSIZE_MAX.saturating_sub(1)); + assert_eq!( + predef_max, + predef_prev + HC_BITCOST_MULTIPLIER, + "predefined litLength pricing at BLOCKSIZE_MAX must add exactly one bit" + ); + + let profile_dyn = + HcOptimalCostProfile::const_for_strategy::(); + let mut stats_dyn = HcOptState::new(); + stats_dyn.price_type = HcOptPriceType::Dynamic; + stats_dyn.lit_length_freq.fill(1); + stats_dyn.lit_length_sum = (HC_MAX_LL + 1) as u32; + stats_dyn.match_length_freq.fill(1); + stats_dyn.match_length_sum = (HC_MAX_ML + 1) as u32; + stats_dyn.off_code_freq.fill(1); + stats_dyn.off_code_sum = (HC_MAX_OFF + 1) as u32; + stats_dyn.lit_freq.fill(1); + stats_dyn.lit_sum = (HC_MAX_LIT + 1) as u32; + stats_dyn.set_base_prices(true); + let dyn_max = profile_dyn.lit_length_price(&stats_dyn, HC_BLOCKSIZE_MAX); + let dyn_prev = profile_dyn.lit_length_price(&stats_dyn, HC_BLOCKSIZE_MAX.saturating_sub(1)); + assert_eq!( + dyn_max, + dyn_prev + HC_BITCOST_MULTIPLIER, + "dynamic litLength pricing at BLOCKSIZE_MAX must add exactly one bit" + ); +} + +#[test] +#[allow(clippy::borrow_deref_ref)] +fn btultra2_seed_pass_disabled_when_dictionary_entropy_seed_present() { + let mut hc = HcMatchGenerator::new(1 << 20); + hc.configure( + BTULTRA2_HC_CONFIG, + super::super::strategy::StrategyTag::BtUltra2, + 26, + ); + let ll = crate::fse::fse_encoder::default_ll_table(); + let ml = crate::fse::fse_encoder::default_ml_table(); + let of = crate::fse::fse_encoder::default_of_table(); + hc.seed_dictionary_entropy(None, Some(&*ll), Some(&*ml), Some(&*of)); + assert!( + !hc.should_run_btultra2_seed_pass::( + HC_PREDEF_THRESHOLD + 1 + ), + "dictionary-seeded first block should skip btultra2 warmup pass" + ); +} + +#[test] +fn btultra2_seed_pass_disabled_when_prefix_history_exists() { + let mut hc = HcMatchGenerator::new(1 << 20); + hc.configure( + BTULTRA2_HC_CONFIG, + super::super::strategy::StrategyTag::BtUltra2, + 26, + ); + hc.table.history_abs_start = 17; + hc.table.push_test_chunk(b"abcdefghijklmnop".to_vec()); + assert!( + !hc.should_run_btultra2_seed_pass::( + HC_PREDEF_THRESHOLD + 9 + ), + "btultra2 warmup must be first-block only (no prefix history)" + ); +} + +#[test] +fn btultra2_seed_pass_disabled_for_tiny_block() { + let mut hc = HcMatchGenerator::new(1 << 20); + hc.configure( + BTULTRA2_HC_CONFIG, + super::super::strategy::StrategyTag::BtUltra2, + 26, + ); + assert!( + !hc.should_run_btultra2_seed_pass::(HC_PREDEF_THRESHOLD), + "btultra2 warmup should not run at or below predefined threshold" + ); +} + +#[test] +fn btultra2_seed_pass_disabled_after_stats_initialized() { + let mut hc = HcMatchGenerator::new(1 << 20); + hc.configure( + BTULTRA2_HC_CONFIG, + super::super::strategy::StrategyTag::BtUltra2, + 26, + ); + hc.backend.bt_mut().opt_state.lit_length_sum = 1; + assert!( + !hc.should_run_btultra2_seed_pass::( + HC_PREDEF_THRESHOLD + 32 + ), + "btultra2 warmup should run only for first block before stats are initialized" + ); +} + +#[test] +fn btultra2_seed_pass_disabled_when_not_at_frame_start() { + let mut hc = HcMatchGenerator::new(1 << 20); + hc.configure( + BTULTRA2_HC_CONFIG, + super::super::strategy::StrategyTag::BtUltra2, + 26, + ); + // Simulate non-first block state: current block has no prefix in deque, + // but total produced window already includes prior output. + hc.table.window_size = HC_PREDEF_THRESHOLD + 64; + // window_size set manually above to simulate prior output; record the + // current block as one live chunk (seed-pass check reads lengths, not bytes). + hc.table.chunk_lens.push_back(HC_PREDEF_THRESHOLD + 32); + assert!( + !hc.should_run_btultra2_seed_pass::( + HC_PREDEF_THRESHOLD + 32 + ), + "btultra2 warmup must not run after frame start" + ); +} + +#[test] +fn btultra2_seed_pass_disabled_when_ldm_sequences_exist() { + let mut hc = HcMatchGenerator::new(1 << 20); + hc.configure( + BTULTRA2_HC_CONFIG, + super::super::strategy::StrategyTag::BtUltra2, + 26, + ); + hc.table.window_size = HC_PREDEF_THRESHOLD + 64; + hc.table.chunk_lens.push_back(HC_PREDEF_THRESHOLD + 64); + hc.backend.bt_mut().ldm_sequences.push(HcRawSeq { + lit_length: 8, + offset: 16, + match_length: 32, + }); + assert!( + !hc.should_run_btultra2_seed_pass::( + HC_PREDEF_THRESHOLD + 32 + ), + "btultra2 warmup must not run when LDM already produced sequences" + ); +} + +#[test] +fn literal_price_uses_eight_bits_when_literals_uncompressed() { + let profile = HcOptimalCostProfile::const_for_strategy::(); + let mut stats = HcOptState::new(); + stats.set_literals_compressed_for_tests(false); + stats.price_type = HcOptPriceType::Predefined; + assert_eq!( + profile.literal_price(&stats, b'a'), + 8 * HC_BITCOST_MULTIPLIER, + "uncompressed literals should cost 8 bits regardless of price mode" + ); +} + +#[test] +fn update_stats_skips_literal_frequencies_when_uncompressed() { + let mut stats = HcOptState::new(); + stats.set_literals_compressed_for_tests(false); + stats.update_stats(3, b"abc", 4, 8); + assert_eq!( + stats.lit_sum, 0, + "literal sum must remain unchanged when literal compression is disabled" + ); + assert_eq!( + stats.lit_freq.iter().copied().sum::(), + 0, + "literal frequencies must not be updated when literal compression is disabled" + ); + assert_eq!( + stats.lit_length_sum, 1, + "literal-length stats still update for sequence modeling" + ); + assert_eq!( + stats.match_length_sum, 1, + "match-length stats still update for sequence modeling" + ); + assert_eq!( + stats.off_code_sum, 1, + "offset-code stats still update for sequence modeling" + ); +} + +#[test] +#[allow(clippy::borrow_deref_ref)] +fn dictionary_huffman_seed_ignored_when_literals_uncompressed() { + let mut stats = HcOptState::new(); + stats.set_literals_compressed_for_tests(false); + let huff = crate::huff0::huff0_encoder::HuffmanTable::build_from_data( + b"aaaaabbbbcccddeeff00112233445566778899", + ); + let ll = crate::fse::fse_encoder::default_ll_table(); + let ml = crate::fse::fse_encoder::default_ml_table(); + let of = crate::fse::fse_encoder::default_of_table(); + stats.seed_dictionary_entropy(Some(&huff), Some(&*ll), Some(&*ml), Some(&*of)); + stats.rescale_freqs( + b"abcd", + HcOptimalCostProfile::const_for_strategy::(), + ); + assert_eq!( + stats.lit_sum, 0, + "literal sum must stay zero when literals are uncompressed" + ); + assert_eq!( + stats.lit_freq.iter().copied().sum::(), + 0, + "literal frequencies must ignore dictionary huffman seed when uncompressed" + ); +} + +#[test] +fn hc_repcode_candidates_respect_litlen_dependent_rep_order() { + let mut hc = HcMatchGenerator::new(64); + hc.table.history = b"xxxxxxABCDEFABCDEF".to_vec(); + hc.table.history_start = 0; + hc.table.history_abs_start = 0; + + let abs_pos = 12usize; // points at second "ABCDEF" + let current_abs_end = hc.table.history.len(); + let reps = [6u32, 3u32, 9u32]; + + let mut lit_pos_candidates = Vec::new(); + hc.hc.for_each_repcode_candidate_with_reps( + &hc.table, + abs_pos, + 1, + reps, + current_abs_end, + HC_OPT_MIN_MATCH_LEN, + |c| { + lit_pos_candidates.push(c.offset); + }, + ); + assert!( + lit_pos_candidates.contains(&6), + "when lit_len>0, rep0 should be considered and match" + ); + + let mut ll0_candidates = Vec::new(); + hc.hc.for_each_repcode_candidate_with_reps( + &hc.table, + abs_pos, + 0, + reps, + current_abs_end, + HC_OPT_MIN_MATCH_LEN, + |c| { + ll0_candidates.push(c.offset); + }, + ); + assert!( + !ll0_candidates.contains(&6), + "when lit_len==0, rep0 is not directly eligible (ll0 semantics)" + ); +} + +#[test] +fn hc_collect_optimal_candidates_keeps_reps_when_chain_depth_zero() { + let mut hc = HcMatchGenerator::new(64); + hc.hc.search_depth = 0; + hc.table.history = b"xyzxyzxyzxyz".to_vec(); + hc.table.history_start = 0; + hc.table.history_abs_start = 0; + + let abs_pos = 6usize; + let current_abs_end = hc.table.history.len(); + let profile = HcOptimalCostProfile { + max_chain_depth: 0, + sufficient_match_len: usize::MAX / 2, + accurate: false, + favor_small_offsets: false, + }; + let mut out = Vec::new(); + hc.collect_optimal_candidates( + abs_pos, + current_abs_end, + profile, + HcCandidateQuery { + reps: [3, 6, 9], + lit_len: 1, + ldm_candidate: None, + }, + &mut out, + ); + assert!( + !out.is_empty(), + "rep candidates should remain available even when chain depth is zero" + ); + assert!( + out.iter().any(|c| c.offset == 3), + "rep0 candidate should be retained" + ); +} + +#[test] +fn hc_collect_optimal_candidates_rep_tail_match_skips_chain_probe() { + let mut hc = HcMatchGenerator::new(64); + hc.table.history = b"aaaaaaaaaa".to_vec(); + hc.table.history_start = 0; + hc.table.history_abs_start = 0; + hc.table.position_base = 0; + hc.hc.search_depth = 32; + let abs_pos = 6usize; + hc.table.ensure_tables(); + hc.table.insert_positions(0, abs_pos); + + let profile = HcOptimalCostProfile { + max_chain_depth: 32, + sufficient_match_len: usize::MAX / 2, + accurate: true, + favor_small_offsets: false, + }; + let mut out = Vec::new(); + hc.collect_optimal_candidates( + abs_pos, + hc.table.history.len(), + profile, + HcCandidateQuery { + reps: [1, 4, 8], + lit_len: 1, + ldm_candidate: None, + }, + &mut out, + ); + + assert!( + out.iter() + .all(|candidate| matches!(candidate.offset, 1 | 4)), + "terminal rep match should return before chain probing adds non-rep offsets" + ); +} + +#[test] +fn hc_collect_optimal_candidates_long_chain_match_advances_skip_window() { + let mut hc = HcMatchGenerator::new(128); + hc.table.history = b"abcabcabcabcabcabcabcabc".to_vec(); + hc.table.history_start = 0; + hc.table.history_abs_start = 0; + hc.table.position_base = 0; + hc.hc.search_depth = 32; + let abs_pos = 9usize; + hc.table.ensure_tables(); + hc.table.insert_positions(0, abs_pos); + hc.table.skip_insert_until_abs = 0; + + let profile = HcOptimalCostProfile { + max_chain_depth: 32, + sufficient_match_len: usize::MAX / 2, + accurate: true, + favor_small_offsets: false, + }; + let mut out = Vec::new(); + hc.collect_optimal_candidates( + abs_pos, + hc.table.history.len(), + profile, + HcCandidateQuery { + reps: [1, 4, 8], + lit_len: 1, + ldm_candidate: None, + }, + &mut out, + ); + + assert!( + hc.table.skip_insert_until_abs > abs_pos, + "long chain match should advance skip window to avoid redundant immediate insertions" + ); +} + +#[test] +fn hc_collect_optimal_candidates_chain_fast_skip_uses_match_end_minus_8() { + let mut hc = HcMatchGenerator::new(128); + hc.table.history = b"abcabcabcabcabcabcabcabc".to_vec(); + hc.table.history_start = 0; + hc.table.history_abs_start = 0; + hc.table.position_base = 0; + hc.hc.search_depth = 32; + let abs_pos = 9usize; + hc.table.ensure_tables(); + hc.table.insert_positions(0, abs_pos); + hc.table.skip_insert_until_abs = 0; + + let profile = HcOptimalCostProfile { + max_chain_depth: 32, + sufficient_match_len: 10, + accurate: true, + favor_small_offsets: false, + }; + let mut out = Vec::new(); + hc.collect_optimal_candidates( + abs_pos, + hc.table.history.len(), + profile, + HcCandidateQuery { + reps: [1, 4, 8], + lit_len: 1, + ldm_candidate: None, + }, + &mut out, + ); + + let best_match_end = out + .iter() + .map(|candidate| candidate.start.saturating_add(candidate.match_len)) + .max() + .expect("expected at least one candidate"); + assert!( + hc.table.skip_insert_until_abs > abs_pos, + "chain fast-skip must advance past current position" + ); + assert!( + hc.table.skip_insert_until_abs <= best_match_end.saturating_sub(8), + "chain fast-skip must not exceed upstream zstd-style matchEndIdx - 8 bound" + ); +} + +#[test] +fn hc_collect_optimal_candidates_advances_skip_window_on_plain_bt_path() { + let mut hc = HcMatchGenerator::new(256); + hc.table.history = b"abcdefghijklmnop".to_vec(); + hc.table.history_start = 0; + hc.table.history_abs_start = 0; + hc.table.position_base = 0; + hc.hc.search_depth = 0; + hc.table.ensure_tables(); + + let abs_pos = 8usize; + hc.table.skip_insert_until_abs = 0; + + let profile = HcOptimalCostProfile { + max_chain_depth: 0, + sufficient_match_len: usize::MAX / 2, + accurate: true, + favor_small_offsets: false, + }; + let mut out = Vec::new(); + hc.collect_optimal_candidates( + abs_pos, + hc.table.history.len(), + profile, + HcCandidateQuery { + reps: [1, 4, 8], + lit_len: 1, + ldm_candidate: None, + }, + &mut out, + ); + + assert_eq!( + hc.table.skip_insert_until_abs, + abs_pos.saturating_add(1), + "plain BT path should advance skip window by 1 via upstream zstd matchEndIdx baseline" + ); +} + +// Removed: the three `hc_collect_optimal_candidates_*_hash3_*` / +// `hc_hash3_tail_match_*` tests forced `search_depth = 0` together +// with `hash3_log != 0`, an HC-chain-walker-only fixture state that +// production never reaches (hash3 is BtUltra2-only and BtUltra2 always +// runs `search_depth = 512`). They depended on the `has_hash3 => +// BtUltra2` escape hatch in the test dispatcher; with that hatch gone +// (CR review on PR #123) and the dispatcher routing purely from +// `self.strategy_tag`, there is no production-shaped configuration +// that reproduces what those tests asserted. The corresponding hash3 +// invariants are exercised end-to-end by the existing level22 roundtrip +// + upstream zstd-parity ratio gate. + +#[test] +fn hc_ldm_candidates_are_merged_into_optimal_candidates() { + let mut hc = HcMatchGenerator::new(512); + hc.table.history = (0..256).map(|i| (i % 251) as u8).collect(); + hc.table.history_start = 0; + hc.table.history_abs_start = 0; + + let abs_pos = 128usize; + let current_abs_end = 256usize; + let ldm = MatchCandidate { + start: abs_pos, + offset: 96, + match_len: 40, + }; + + let profile = HcOptimalCostProfile { + max_chain_depth: 0, + sufficient_match_len: usize::MAX / 2, + accurate: true, + favor_small_offsets: false, + }; + let mut out = Vec::new(); + hc.collect_optimal_candidates( + abs_pos, + current_abs_end, + profile, + HcCandidateQuery { + reps: [1, 4, 8], + lit_len: 1, + ldm_candidate: Some(ldm), + }, + &mut out, + ); + assert!( + out.iter().any( + |candidate| candidate.offset == ldm.offset && candidate.match_len == ldm.match_len + ), + "LDM candidate should be present in optimal candidate set" + ); +} + +#[test] +fn btultra_and_btultra2_both_keep_dictionary_candidates() { + // Routes the BtUltra2 / BtUltra fixture through the production + // `configure()` path so derived state (`hash3_log`, `is_btultra2`, + // `uses_bt`, `backend`) stays consistent — manually flipping the + // strategy flags here used to leave `hash3_log` / `hash3_table` in + // the previous mode's shape and trip the + // `Strategy::USE_HASH3 ⇒ hash3_log != 0` debug invariant inside + // `collect_optimal_candidates_initialized_body`. + use super::super::strategy::StrategyTag; + + let test_config = HcConfig { + hash_log: 23, + chain_log: 22, + search_depth: 32, + target_len: 256, + search_mls: 4, + }; + let window_log = 20u8; + + let prepare_history = |hc: &mut HcMatchGenerator, abs_pos: usize| { + hc.table.history = alloc::vec![0u8; 160]; + for i in 0..64 { + hc.table.history[i] = b'a' + (i % 7) as u8; + } + for i in 64..160 { + hc.table.history[i] = b'k' + (i % 5) as u8; + } + for i in 0..24 { + hc.table.history[abs_pos + i] = hc.table.history[16 + i]; + } + hc.table.history_start = 0; + hc.table.history_abs_start = 0; + hc.table.position_base = 0; + hc.table.ensure_tables(); + hc.table.insert_positions(0, abs_pos); + hc.table.dictionary_limit_abs = Some(64); + hc.table.skip_insert_until_abs = 0; + }; + + let profile = HcOptimalCostProfile { + max_chain_depth: 32, + sufficient_match_len: usize::MAX / 2, + accurate: true, + favor_small_offsets: false, + }; + let abs_pos = 96usize; + let mut out = Vec::new(); + + let mut hc = HcMatchGenerator::new(256); + hc.configure(test_config, StrategyTag::BtUltra2, window_log); + prepare_history(&mut hc, abs_pos); + hc.collect_optimal_candidates( + abs_pos, + 160, + profile, + HcCandidateQuery { + reps: [1, 4, 8], + lit_len: 1, + ldm_candidate: None, + }, + &mut out, + ); + assert!( + out.iter().any(|candidate| candidate.offset >= 32), + "btultra2 should retain dictionary candidates on upstream zstd-parity path" + ); + + let mut hc = HcMatchGenerator::new(256); + hc.configure(test_config, StrategyTag::BtUltra, window_log); + prepare_history(&mut hc, abs_pos); + hc.collect_optimal_candidates( + abs_pos, + 160, + profile, + HcCandidateQuery { + reps: [1, 4, 8], + lit_len: 1, + ldm_candidate: None, + }, + &mut out, + ); + assert!( + out.iter().any(|candidate| candidate.offset >= 32), + "btultra should retain dictionary candidates" + ); +} + +#[test] +fn driver_small_source_hint_shrinks_dfast_hash_tables() { + let mut driver = MatchGeneratorDriver::new(32, 2); + + driver.reset(CompressionLevel::Level(3)); + let mut space = driver.get_next_space(); + space[..12].copy_from_slice(b"abcabcabcabc"); + space.truncate(12); + driver.commit_space(space); + driver.skip_matching_with_hint(None); + // Upstream zstd-parity split sizes: long-hash = DFAST_HASH_BITS, + // short-hash = DFAST_HASH_BITS - DFAST_SHORT_HASH_BITS_DELTA. + let full_long = driver.dfast_matcher().long_len(); + let full_short = driver.dfast_matcher().short_len(); + assert_eq!(full_long, 1 << DFAST_HASH_BITS); + assert_eq!( + full_short, + 1 << (DFAST_HASH_BITS - DFAST_SHORT_HASH_BITS_DELTA) + ); + + driver.set_source_size_hint(1024); + driver.reset(CompressionLevel::Level(3)); + let mut space = driver.get_next_space(); + space[..12].copy_from_slice(b"xyzxyzxyzxyz"); + space.truncate(12); + driver.commit_space(space); + driver.skip_matching_with_hint(None); + let hinted_long = driver.dfast_matcher().long_len(); + let hinted_short = driver.dfast_matcher().short_len(); + + // The wire `window_log` stays at its floor (decoder-interop), but the + // internal dfast tables are sized from the RAW 1 KiB source, not the + // floored window: `table_window = 1 << ceil_log2(1024) = 1 << 10`, so + // both tables land at the `MIN_WINDOW_LOG` floor (the long table at + // `dfast_hash_bits_for_window(1 << 10) = 10`, the short table one + // `DFAST_SHORT_HASH_BITS_DELTA` step below but clamped back up to + // `MIN_WINDOW_LOG`). + assert_eq!(driver.window_size(), 1 << MIN_HINTED_WINDOW_LOG); + assert_eq!(hinted_long, 1 << MIN_WINDOW_LOG); + assert_eq!(hinted_short, 1 << MIN_WINDOW_LOG); + assert!( + hinted_long < full_long && hinted_short < full_short, + "tiny source hint should reduce both dfast tables" + ); +} + +#[test] +fn driver_huge_source_hint_does_not_overflow_table_window_shift() { + // Regression: the Dfast / Row table-window sizing in `reset` derives a + // shift from `ceil_log2(hint)`. A hint >= 2^63 + 1 makes that shift 64, + // and `1usize << 64` panics in debug / wraps to 0 in release before the + // `.min(max_window_size)` cap can apply. A `u64::MAX` pledged source size + // must size the table to the real window, never panic or wrap to zero. + let mut driver = MatchGeneratorDriver::new(32, 2); + driver.set_source_size_hint(u64::MAX); + driver.reset(CompressionLevel::Level(3)); + + let mut space = driver.get_next_space(); + space[..12].copy_from_slice(b"abcabcabcabc"); + space.truncate(12); + driver.commit_space(space); + driver.skip_matching_with_hint(None); + + assert!( + driver.dfast_matcher().long_len() >= 1 << MIN_WINDOW_LOG, + "huge hint must size the dfast table from the real window, not wrap to zero" + ); +} + +#[test] +fn driver_huge_source_hint_with_dict_does_not_overflow_hc_reserve() { + // Regression: the HC/BT history-mirror pre-size adds the dictionary + // hint to the source-size hint before `reserve_history` clamps to the + // window ceiling. A `u64::MAX` pledged source size (the "unknown size" + // sentinel) plus any positive dictionary hint overflows `usize` in + // `(src as usize) + dict_hint` — debug panic / release wrap on 64-bit, + // and `src as usize` truncation on 32-bit targets. Level 16 (BtOpt) + // routes through the HashChain/BT storage arm that owns this reserve. + // Must size the mirror to the real window, never panic, wrap, or + // truncate. + let mut driver = MatchGeneratorDriver::new(32, 2); + driver.set_source_size_hint(u64::MAX); + driver.set_dictionary_size_hint(64 * 1024); + driver.reset(CompressionLevel::Level(16)); + + // The saturated `usize::MAX` reserve target must be clamped to the HC + // history ceiling, not reserved literally (which would OOM/panic). Level 16 + // has window_log 22, so the ceiling is `window + window/4 + one block` + // (the `reserve_history` formula). Assert the reserve actually reached it — + // a no-panic-only check would also pass on an under-reserved mirror. + let window = 1usize << 22; + let expected_history_ceiling = window + (window >> 2) + crate::common::MAX_BLOCK_SIZE as usize; + assert!( + driver.hc_matcher().table.history.capacity() >= expected_history_ceiling, + "huge source + dict hint must reserve the clamped HC history ceiling, got {}", + driver.hc_matcher().table.history.capacity() + ); + + let mut space = driver.get_next_space(); + space[..12].copy_from_slice(b"abcabcabcabc"); + space.truncate(12); + driver.commit_space(space); + driver.skip_matching_with_hint(None); +} + +#[test] +fn driver_chain_log_override_survives_row_to_hc_fallback() { + // Regression: when a RowHash level is forced onto the HashChain backend + // (resolved window <= 14, upstream `ZSTD_resolveRowMatchFinderMode`), the + // synthesised HC chain table must honour an explicit `chain_log` override. + // The RowHash override arm drops `chain_log` (Row has no chain table), so + // the synthesis previously replaced the caller's `chain_log` with the upstream zstd + // `hashLog - 1`, silently ignoring it on small-window frames. + let chain_log_override = 10u32; + let ov = super::super::parameters::ParamOverrides { + chain_log: Some(chain_log_override), + ..Default::default() + }; + let mut driver = MatchGeneratorDriver::new(32, 2); + // Small source hint pins the window to the hinted floor (16 KiB = + // windowLog 14), so the Level 6 Row finder falls back to HashChain. + driver.set_source_size_hint(1 << 12); + driver.set_param_overrides(Some(ov)); + driver.reset(CompressionLevel::Level(6)); + let mut space = driver.get_next_space(); + space[..12].copy_from_slice(b"abcabcabcabc"); + space.truncate(12); + driver.commit_space(space); + driver.skip_matching_with_hint(None); + // The override (10) is below the window cap (14), so the resolved HC chain + // table must reflect it — NOT the upstream zstd `hashLog - 1` (18, clamped to the + // window 14). Pre-fix this resolved to 14. + assert_eq!( + driver.hc_matcher().table.chain_log, + chain_log_override as usize, + "explicit chain_log override must survive the Row->HC fallback, got {}", + driver.hc_matcher().table.chain_log + ); +} + +#[test] +fn driver_small_source_hint_shrinks_row_hash_tables() { + let mut driver = MatchGeneratorDriver::new(32, 2); + + driver.reset(CompressionLevel::Level(5)); + let mut space = driver.get_next_space(); + space[..12].copy_from_slice(b"abcabcabcabc"); + space.truncate(12); + driver.commit_space(space); + driver.skip_matching_with_hint(None); + let full_rows = driver.row_matcher().row_heads.len(); + // Level 5 uses the upstream row_log (clamp(searchLog=3, 4, 6) = 4) and the + // upstream L5 hashLog (`ZSTD_getCParams(5,..).hashLog` = 19), so the row + // count is 1 << (ROW_L5.hash_bits - ROW_L5.row_log). + assert_eq!(full_rows, 1 << (ROW_L5.hash_bits - ROW_L5.row_log)); + + // A hint that keeps the resolved window > 14 STILL uses the Row finder + // (upstream `ZSTD_resolveRowMatchFinderMode`: row mode on for windowLog > 14) + // and shrinks the row hash table to the source-derived width. 64 KiB → + // raw source log 16, so `row_hash_bits_for_window(1 << 16)` < the level's + // full hash_bits (19) and the row count drops. + driver.set_source_size_hint(1 << 16); + driver.reset(CompressionLevel::Level(5)); + let mut space = driver.get_next_space(); + space[..12].copy_from_slice(b"xyzxyzxyzxyz"); + space.truncate(12); + driver.commit_space(space); + driver.skip_matching_with_hint(None); + assert_eq!( + driver.active_backend(), + super::super::strategy::BackendTag::Row, + "windowLog > 14 keeps the upstream row matchfinder" + ); + let hinted_rows = driver.row_matcher().row_heads.len(); + assert!( + hinted_rows < full_rows, + "a window>14 source hint should reduce the row hash table footprint" + ); + + // A tiny hint floors the resolved window at MIN_HINTED_WINDOW_LOG = 14; + // upstream uses the HASH-CHAIN matcher (not Row) at windowLog <= 14, so the + // driver must route greedy/lazy/lazy2 to the HashChain backend there. + driver.set_source_size_hint(1024); + driver.reset(CompressionLevel::Level(5)); + assert_eq!(driver.window_size(), 1 << MIN_HINTED_WINDOW_LOG); + assert_eq!( + driver.active_backend(), + super::super::strategy::BackendTag::HashChain, + "windowLog <= 14 must fall back to the upstream zstd hash-chain matchfinder", + ); +} + +#[test] +fn row_matches_roundtrip_multi_block_pattern() { + let pattern = [7, 13, 44, 184, 19, 96, 171, 109, 141, 251]; + let first_block: Vec = pattern.iter().copied().cycle().take(128 * 1024).collect(); + let second_block: Vec = pattern.iter().copied().cycle().take(128 * 1024).collect(); + + let mut matcher = RowMatchGenerator::new(1 << 22); + matcher.configure(ROW_CONFIG); + matcher.ensure_tables(); + let replay_sequence = |decoded: &mut Vec, seq: Sequence<'_>| match seq { + Sequence::Literals { literals } => decoded.extend_from_slice(literals), + Sequence::Triple { + literals, + offset, + match_len, + } => { + decoded.extend_from_slice(literals); + let start = decoded.len() - offset; + for i in 0..match_len { + let byte = decoded[start + i]; + decoded.push(byte); + } + } + }; + + matcher.add_data(first_block.clone(), |_| {}); + let mut history = Vec::new(); + matcher.start_matching(|seq| replay_sequence(&mut history, seq)); + assert_eq!(history, first_block); + + matcher.add_data(second_block.clone(), |_| {}); + let prefix_len = history.len(); + matcher.start_matching(|seq| replay_sequence(&mut history, seq)); + + assert_eq!(&history[prefix_len..], second_block.as_slice()); + + // Force a literals-only pass so the Sequence::Literals arm is exercised. + let third_block: Vec = (0u8..=255).collect(); + matcher.add_data(third_block.clone(), |_| {}); + let third_prefix = history.len(); + matcher.start_matching(|seq| replay_sequence(&mut history, seq)); + assert_eq!(&history[third_prefix..], third_block.as_slice()); +} + +#[test] +fn row_short_block_emits_literals_only() { + let mut matcher = RowMatchGenerator::new(1 << 22); + matcher.configure(ROW_CONFIG); + + matcher.add_data(b"abcde".to_vec(), |_| {}); + + let mut saw_triple = false; + let mut reconstructed = Vec::new(); + matcher.start_matching(|seq| match seq { + Sequence::Literals { literals } => reconstructed.extend_from_slice(literals), + Sequence::Triple { .. } => saw_triple = true, + }); + + assert!( + !saw_triple, + "row backend must not emit triples for short blocks" + ); + assert_eq!(reconstructed, b"abcde"); + + // Then feed a clearly matchable block and ensure the Triple arm is reachable. + saw_triple = false; + matcher.add_data(b"abcdeabcde".to_vec(), |_| {}); + matcher.start_matching(|seq| { + if let Sequence::Triple { .. } = seq { + saw_triple = true; + } + }); + assert!( + saw_triple, + "row backend should emit triples on repeated data" + ); +} + +#[test] +fn row_pick_lazy_returns_best_when_lookahead_is_out_of_bounds() { + let mut matcher = RowMatchGenerator::new(1 << 22); + matcher.configure(ROW_CONFIG); + matcher.add_data(b"abcabc".to_vec(), |_| {}); + // Build the row tables before probing: the lookahead path reaches + // `row_candidate` -> `row_heads[..]` once the accept floor is small + // enough to pass the length gate, so the tables must be allocated + // (production always calls this before any candidate probe). + matcher.ensure_tables(); + + let best = MatchCandidate { + start: 0, + offset: 1, + match_len: ROW_MIN_MATCH_LEN, + }; + let picked = matcher + .pick_lazy_match(0, 0, Some(best)) + .expect("best candidate must survive"); + + assert_eq!(picked.start, best.start); + assert_eq!(picked.offset, best.offset); + assert_eq!(picked.match_len, best.match_len); +} + +#[test] +fn row_backfills_previous_block_tail_for_cross_boundary_match() { + let mut matcher = RowMatchGenerator::new(1 << 22); + matcher.configure(ROW_CONFIG); + + let mut first_block = alloc::vec![0xA5; 64]; + first_block.extend_from_slice(b"XYZ"); + let second_block = b"XYZXYZtail".to_vec(); + + let replay_sequence = |decoded: &mut Vec, seq: Sequence<'_>| match seq { + Sequence::Literals { literals } => decoded.extend_from_slice(literals), + Sequence::Triple { + literals, + offset, + match_len, + } => { + decoded.extend_from_slice(literals); + let start = decoded.len() - offset; + for i in 0..match_len { + let byte = decoded[start + i]; + decoded.push(byte); + } + } + }; + + matcher.add_data(first_block.clone(), |_| {}); + let mut reconstructed = Vec::new(); + matcher.start_matching(|seq| replay_sequence(&mut reconstructed, seq)); + assert_eq!(reconstructed, first_block); + + matcher.add_data(second_block.clone(), |_| {}); + let mut saw_cross_boundary = false; + let prefix_len = reconstructed.len(); + matcher.start_matching(|seq| { + if let Sequence::Triple { + literals, + offset, + match_len, + } = seq + && literals.is_empty() + && offset == 3 + && match_len >= ROW_MIN_MATCH_LEN + { + saw_cross_boundary = true; + } + replay_sequence(&mut reconstructed, seq); + }); + + assert!( + saw_cross_boundary, + "row matcher should reuse the 3-byte previous-block tail" + ); + assert_eq!(&reconstructed[prefix_len..], second_block.as_slice()); +} + +#[test] +fn row_skip_matching_with_incompressible_hint_uses_sparse_prefix() { + let data = deterministic_high_entropy_bytes(0xA713_9C5D_44E2_10B1, 4096); + + let mut dense = RowMatchGenerator::new(1 << 22); + dense.configure(ROW_CONFIG); + dense.add_data(data.clone(), |_| {}); + dense.skip_matching_with_hint(Some(false)); + let dense_slots = dense + .row_positions + .iter() + .filter(|&&pos| pos != ROW_EMPTY_SLOT) + .count(); + + let mut sparse = RowMatchGenerator::new(1 << 22); + sparse.configure(ROW_CONFIG); + sparse.add_data(data, |_| {}); + sparse.skip_matching_with_hint(Some(true)); + let sparse_slots = sparse + .row_positions + .iter() + .filter(|&&pos| pos != ROW_EMPTY_SLOT) + .count(); + + assert!( + sparse_slots < dense_slots, + "incompressible hint should seed fewer row slots (sparse={sparse_slots}, dense={dense_slots})" + ); +} + +/// Regression for the `None` arm of `skip_matching_with_hint`: the +/// row table must NOT receive dense inserts across the skipped range. +/// Upstream zstd parity (`ZSTD_row_fillHashCache` only pre-fills the next-scan +/// cache, not the skipped block's interior) trades cross-block +/// matches into the skipped interior for the per-block O(block_size) +/// insert cost. +/// +/// At input < 1 block (4096 B with default 128 KiB block boundary), +/// the only positions in the row table after the call should be those +/// produced by the `backfill_start` lookback at the block's start +/// (≤ `ROW_HASH_KEY_LEN - 1` positions when block_start < +/// ROW_HASH_KEY_LEN). For `current_abs_start == 0`, even that backfill +/// is empty — so the table stays fully empty. +#[test] +fn row_skip_matching_with_none_hint_leaves_interior_empty() { + let data = deterministic_high_entropy_bytes(0x9B47_F2A1_8C5E_3306, 4096); + + let mut none_hint = RowMatchGenerator::new(1 << 22); + none_hint.configure(ROW_CONFIG); + none_hint.add_data(data.clone(), |_| {}); + none_hint.skip_matching_with_hint(None); + let none_slots = none_hint + .row_positions + .iter() + .filter(|&&pos| pos != ROW_EMPTY_SLOT) + .count(); + + // Dense (Some(false), dict-priming path) for comparison — that + // path inserts every position in the skipped range. + let mut dense = RowMatchGenerator::new(1 << 22); + dense.configure(ROW_CONFIG); + dense.add_data(data, |_| {}); + dense.skip_matching_with_hint(Some(false)); + let dense_slots = dense + .row_positions + .iter() + .filter(|&&pos| pos != ROW_EMPTY_SLOT) + .count(); + + // Two assertions pin the contract: + // 1) None hint is dramatically sparser than dense (the whole point). + // 2) None hint at block-start==0 inserts ZERO positions (no + // backfill possible before position 0). + assert_eq!( + none_slots, 0, + "None hint at block_start=0 must leave row table fully empty \ + (upstream zstd parity — interior NOT inserted, no pre-block backfill possible)", + ); + assert!( + dense_slots > 0, + "Some(false) dict-priming path must still insert densely \ + (sanity check: control case for the `none_slots == 0` assertion)", + ); +} + +#[test] +fn driver_unhinted_level2_keeps_default_dfast_hash_table_size() { + let mut driver = MatchGeneratorDriver::new(32, 2); + + driver.reset(CompressionLevel::Level(3)); + let mut space = driver.get_next_space(); + space[..12].copy_from_slice(b"abcabcabcabc"); + space.truncate(12); + driver.commit_space(space); + driver.skip_matching_with_hint(None); + + // Upstream zstd-parity split: long-hash at DFAST_HASH_BITS, short-hash one + // bit smaller (DFAST_SHORT_HASH_BITS_DELTA = 1, matching upstream zstd + // `chainLog = hashLog - 1` for dfast levels). + let long_len = driver.dfast_matcher().long_len(); + let short_len = driver.dfast_matcher().short_len(); + assert_eq!( + long_len, + 1 << DFAST_HASH_BITS, + "unhinted Level(2) should keep default long-hash table size" + ); + assert_eq!( + short_len, + 1 << (DFAST_HASH_BITS - DFAST_SHORT_HASH_BITS_DELTA), + "unhinted Level(2) short-hash should be one bit smaller than long-hash" + ); +} + +#[cfg(any())] // disabled: tested legacy MatchGenerator/SuffixStore behavior removed in phase 1b +#[test] +fn simple_backend_rejects_undersized_pooled_suffix_store() { + let mut driver = MatchGeneratorDriver::new(128 * 1024, 2); + driver.reset(CompressionLevel::Fastest); + + driver.suffix_pool.push(SuffixStore::with_capacity(1024)); + + let mut space = driver.get_next_space(); + space.clear(); + space.resize(4096, 0xAB); + driver.commit_space(space); + + let last_suffix_slots = driver + .simple() + .window + .last() + .expect("window entry must exist after commit") + .suffixes + .slots + .len(); + assert!( + last_suffix_slots >= 4096, + "undersized pooled suffix store must not be reused for larger blocks" + ); +} + +#[test] +fn source_hint_clamps_driver_slice_size_to_window() { + let mut driver = MatchGeneratorDriver::new(128 * 1024, 2); + driver.set_source_size_hint(1024); + driver.reset(CompressionLevel::Default); + + let window = driver.window_size() as usize; + assert_eq!(window, 1 << MIN_HINTED_WINDOW_LOG); + assert_eq!(driver.slice_size, window); + + let space = driver.get_next_space(); + assert_eq!(space.len(), window); + driver.commit_space(space); +} + +#[test] +fn pooled_space_keeps_capacity_when_slice_size_shrinks() { + let mut driver = MatchGeneratorDriver::new(128 * 1024, 2); + driver.reset(CompressionLevel::Default); + + let large = driver.get_next_space(); + let large_capacity = large.capacity(); + assert!(large_capacity >= 128 * 1024); + driver.commit_space(large); + + driver.set_source_size_hint(1024); + driver.reset(CompressionLevel::Default); + + let small = driver.get_next_space(); + assert_eq!(small.len(), 1 << MIN_HINTED_WINDOW_LOG); + assert!( + small.capacity() >= large_capacity, + "pooled buffer capacity should be preserved to avoid shrink/grow churn" + ); +} + +#[test] +fn driver_best_to_fastest_releases_oversized_hc_tables() { + let mut driver = MatchGeneratorDriver::new(32, 2); + + // Initialize at Best routed onto HashChain via the test-only override + // (production `Best` sits on level 13, whose native backend differs) — + // allocates large HC tables (4M hash, 2M chain) so the swap below + // exercises the HC drain path this test pins. + driver.reset_on_hc_lazy(CompressionLevel::Best); + assert_eq!(driver.window_size(), (1u64 << 22)); + + // Feed data so tables are actually allocated via ensure_tables(). + let mut space = driver.get_next_space(); + space[..12].copy_from_slice(b"abcabcabcabc"); + space.truncate(12); + driver.commit_space(space); + driver.skip_matching_with_hint(None); + + // Switch to Fastest — the [`MatcherStorage`] enum swaps to the + // `Simple` variant and the `HashChain` variant is dropped. The + // drain block in `Matcher::reset` reassigns + // `m.table.hash_table` / `chain_table` / `hash3_table` to + // `Vec::new()` BEFORE constructing the replacement variant so the + // table backing allocations are released up front — this caps + // peak memory during the swap to "old data buffers being drained + // into `vec_pool` + new `MatchGenerator` skeleton" rather than + // "old tables still resident + new variant under construction". + // The eventual `Drop` on the old variant would release the tables + // anyway, but only after the new variant is built, so the early + // reassign shifts the peak. Post-switch the HC variant no longer + // exists; the assertion that storage is now `Simple` covers the + // invariant the old hash_table/chain_table checks were proxying. + driver.reset(CompressionLevel::Fastest); + assert_eq!(driver.window_size(), (1u64 << 19)); + assert_eq!( + driver.active_backend(), + super::super::strategy::BackendTag::Simple + ); +} + +#[test] +fn driver_better_to_best_resizes_hc_tables() { + let mut driver = MatchGeneratorDriver::new(32, 2); + + // The lazy band runs on the Row backend now, so the HC resize path is + // exercised across two BT levels whose native `HcConfig` widths differ: + // L13 (hash_log 22, chain_log 22) -> L15 (hash_log 23, chain_log 23). + driver.reset(CompressionLevel::Level(13)); + assert_eq!(driver.window_size(), (1u64 << 22)); + + let mut space = driver.get_next_space(); + space[..12].copy_from_slice(b"abcabcabcabc"); + space.truncate(12); + driver.commit_space(space); + driver.skip_matching_with_hint(None); + + let hc = driver.hc_matcher(); + let better_hash_len = hc.table.hash_table.len(); + let better_chain_len = hc.table.chain_table.len(); + + // Switch to L15 — must resize to larger tables. + driver.reset(CompressionLevel::Level(15)); + assert_eq!(driver.window_size(), (1u64 << 22)); + + // Feed data to trigger ensure_tables with new sizes. + let mut space = driver.get_next_space(); + space[..12].copy_from_slice(b"xyzxyzxyzxyz"); + space.truncate(12); + driver.commit_space(space); + driver.skip_matching_with_hint(None); + + let hc = driver.hc_matcher(); + assert!( + hc.table.hash_table.len() > better_hash_len, + "L15 hash_table ({}) should be larger than L13 ({})", + hc.table.hash_table.len(), + better_hash_len + ); + assert!( + hc.table.chain_table.len() > better_chain_len, + "L15 chain_table ({}) should be larger than L13 ({})", + hc.table.chain_table.len(), + better_chain_len + ); +} + +#[cfg(any())] +// disabled: tests legacy SuffixStore behavior incompatible with upstream zstd-shape kernel's HASH_READ_SIZE geometry +#[test] +fn prime_with_dictionary_preserves_history_for_first_full_block() { + let mut driver = MatchGeneratorDriver::new(8, 1); + driver.reset(CompressionLevel::Fastest); + + driver.prime_with_dictionary(b"abcdefgh", [1, 4, 8]); + + let mut space = driver.get_next_space(); + space.clear(); + space.extend_from_slice(b"abcdefgh"); + driver.commit_space(space); + + let mut saw_match = false; + driver.start_matching(|seq| { + if let Sequence::Triple { + literals, + offset, + match_len, + } = seq + && literals.is_empty() + && offset == 8 + && match_len >= MIN_MATCH_LEN + { + saw_match = true; + } + }); + + assert!( + saw_match, + "first full block should still match dictionary-primed history" + ); +} + +#[cfg(any())] +// disabled: tests legacy SuffixStore behavior incompatible with upstream zstd-shape kernel's HASH_READ_SIZE geometry +#[test] +fn prime_with_large_dictionary_preserves_early_history_until_first_block() { + let mut driver = MatchGeneratorDriver::new(8, 1); + driver.reset(CompressionLevel::Fastest); + + driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); + + let mut space = driver.get_next_space(); + space.clear(); + space.extend_from_slice(b"abcdefgh"); + driver.commit_space(space); + + let mut saw_match = false; + driver.start_matching(|seq| { + if let Sequence::Triple { + literals, + offset, + match_len, + } = seq + && literals.is_empty() + && offset == 24 + && match_len >= MIN_MATCH_LEN + { + saw_match = true; + } + }); + + assert!( + saw_match, + "dictionary bytes should remain addressable until frame output exceeds the live window" + ); +} + +#[test] +fn prime_with_dictionary_applies_offset_history_even_when_content_is_empty() { + let mut driver = MatchGeneratorDriver::new(8, 1); + driver.reset(CompressionLevel::Fastest); + + driver.prime_with_dictionary(&[], [11, 7, 3]); + + assert_eq!(driver.simple_mut().offset_hist, [11, 7, 3]); +} + +#[test] +fn hc_prime_with_empty_dictionary_disables_btultra2_seed_pass() { + let mut driver = MatchGeneratorDriver::new(8, 1); + driver.reset_on_hc_lazy(CompressionLevel::Better); + + driver.prime_with_dictionary(&[], [11, 7, 3]); + + assert_eq!(driver.hc_matcher().table.offset_hist, [11, 7, 3]); + assert!( + !driver + .hc_matcher() + .should_run_btultra2_seed_pass::( + HC_PREDEF_THRESHOLD + 1 + ), + "btultra2 warmup must stay disabled after dictionary priming, even when dict content is empty" + ); +} + +#[test] +fn primed_snapshot_not_restored_across_ldm_config_change() { + // The CDict-equivalent primed snapshot clones `storage`, which on the + // BT backend carries `BtMatcher::ldm_producer`. A snapshot captured + // under one LDM configuration must NOT be restored into a reset that + // resolved a different LDM configuration (else the restored producer + // is stale). `PrimedKey` must fold the LDM override into the key so + // such a restore is refused and the caller re-primes. + use super::super::parameters::CompressionParameters; + + let dict = b"abcdefghabcdefghabcdefgh"; + let ldm_on = CompressionParameters::builder(CompressionLevel::Level(19)) + .enable_long_distance_matching(true) + .build() + .unwrap() + .overrides(); + let ldm_off = CompressionParameters::builder(CompressionLevel::Level(19)) + .build() + .unwrap() + .overrides(); + + let mut driver = MatchGeneratorDriver::new(1024, 1); + + // Capture a snapshot primed under LDM-on at level 19. + driver.set_param_overrides(Some(ldm_on)); + driver.reset(CompressionLevel::Level(19)); + driver.prime_with_dictionary(dict, [1, 4, 8]); + driver.capture_primed_dictionary(CompressionLevel::Level(19)); + + // Same dictionary + level, but LDM now OFF: the snapshot's LDM state + // is stale, so restore must be refused. + driver.set_param_overrides(Some(ldm_off)); + driver.reset(CompressionLevel::Level(19)); + assert!( + !driver.restore_primed_dictionary(CompressionLevel::Level(19)), + "primed snapshot restored across an LDM config change (stale producer)", + ); + + // Sanity: re-priming + capturing under LDM-off, then restoring under + // the IDENTICAL LDM-off config DOES match (the key is not over-tight). + driver.prime_with_dictionary(dict, [1, 4, 8]); + driver.capture_primed_dictionary(CompressionLevel::Level(19)); + driver.reset(CompressionLevel::Level(19)); + assert!( + driver.restore_primed_dictionary(CompressionLevel::Level(19)), + "primed snapshot not restored under identical LDM config", + ); +} + +#[test] +fn hc_prime_with_dictionary_disables_btultra2_seed_pass() { + let mut driver = MatchGeneratorDriver::new(8, 1); + driver.reset_on_hc_lazy(CompressionLevel::Better); + + driver.prime_with_dictionary(b"abcdefgh", [1, 4, 8]); + + assert!( + !driver + .hc_matcher() + .should_run_btultra2_seed_pass::( + HC_PREDEF_THRESHOLD + 1 + ), + "btultra2 warmup must stay disabled after dictionary priming with content" + ); +} + +#[test] +fn dfast_prime_with_dictionary_preserves_history_for_first_full_block() { + let mut driver = MatchGeneratorDriver::new(8, 1); + // Level(4) is Dfast with the greedy double-fast loop (upstream zstd parity: + // clevels.h L3/L4 are both `ZSTD_dfast`, which has no lazy lookahead). + // The fast loop needs at least `HASH_READ_SIZE` (8) bytes ahead of the + // probe cursor, so this exercises a 16-byte dict + 16-byte block (the + // whole block matches the dict, offset = dict length = 16). + driver.reset(CompressionLevel::Level(4)); + + let payload = b"abcdefghijklmnop"; + driver.prime_with_dictionary(payload, [1, 4, 8]); + + let mut space = driver.get_next_space(); + space.clear(); + space.extend_from_slice(payload); + driver.commit_space(space); + + let mut saw_match = false; + driver.start_matching(|seq| { + if let Sequence::Triple { + literals, + offset, + match_len, + } = seq + && literals.is_empty() + && offset == payload.len() + && match_len >= DFAST_MIN_MATCH_LEN + { + saw_match = true; + } + }); + + assert!( + saw_match, + "dfast backend should match dictionary-primed history in first full block" + ); +} + +#[test] +fn prime_with_dictionary_does_not_inflate_reported_window_size() { + let mut driver = MatchGeneratorDriver::new(8, 1); + driver.reset(CompressionLevel::Fastest); + + let before = driver.window_size(); + driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); + let after = driver.window_size(); + + assert_eq!( + after, before, + "dictionary retention budget must not change reported frame window size" + ); +} + +#[test] +fn primed_snapshot_not_restored_when_window_hint_differs() { + // The copy-snapshot must be keyed on the resolved reset parameters, not + // just the CompressionLevel. `reset()` caps window_log by the source-size + // hint, so two same-level frames with different hints resolve to different + // windows. Restoring a snapshot captured at the larger hint into a reset + // for the smaller hint would advertise the smaller window in the frame + // header while the matcher's `max_window_size` (from the restored storage) + // still spans the larger window — the encoder could then emit a match + // (e.g. into the dictionary) past the advertised window, producing an + // undecodable frame. Restore must REFUSE when the resolved window differs. + let mut driver = MatchGeneratorDriver::new(8, 1); + let level = CompressionLevel::Best; + + // Frame A: large hint → larger resolved window. Prime + capture. + driver.set_source_size_hint(256 * 1024); + driver.reset(level); + let big_window = driver.window_size(); + driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); + driver.capture_primed_dictionary(level); + + // Frame B: smaller hint, SAME level → smaller resolved window. + driver.set_source_size_hint(48 * 1024); + driver.reset(level); + let small_window = driver.window_size(); + assert!( + small_window < big_window, + "precondition: the two hints must resolve to different windows \ + (small={small_window}, big={big_window})" + ); + + let restored = driver.restore_primed_dictionary(level); + assert!( + !restored, + "snapshot captured at window {big_window} must NOT be restored into a \ + reset advertising window {small_window} (level alone is an insufficient key)" + ); +} + +#[test] +fn primed_snapshot_restored_for_hints_in_same_window_bucket() { + // The snapshot key must normalize the source-size hint to the resolved + // matcher geometry, not the raw hinted byte count. `reset()` derives every + // hint-dependent parameter (window_log cap, HC/Fast/Dfast/Row table widths, + // the Fast attach-vs-copy cutoff) from `ceil_log2(hint)`, so two distinct + // hints that share a ceil-log bucket resolve to the *identical* matcher + // shape. Keying on the raw bytes over-keys: it forces a full re-prime on the + // second frame even though the cached snapshot is a perfect fit. Restore + // must SUCCEED across same-bucket hints. + let mut driver = MatchGeneratorDriver::new(8, 1); + let level = CompressionLevel::Best; + + // Both hints fall in ceil_log2 bucket 19 (2^18 < n <= 2^19): 300 KiB and + // 400 KiB resolve to the same window and table widths. + driver.set_source_size_hint(300 * 1024); + driver.reset(level); + let window_a = driver.window_size(); + driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); + driver.capture_primed_dictionary(level); + + driver.set_source_size_hint(400 * 1024); + driver.reset(level); + let window_b = driver.window_size(); + assert_eq!( + window_a, window_b, + "precondition: same-bucket hints must resolve to the same window \ + (a={window_a}, b={window_b})" + ); + + let restored = driver.restore_primed_dictionary(level); + assert!( + restored, + "snapshot captured at a 300 KiB hint must be restored into a 400 KiB \ + hint that resolves to the identical matcher shape (raw bytes over-key)" + ); +} + +#[test] +fn primed_snapshot_restored_across_level22_tier_hints() { + // Level 22 collapses several ceil-log buckets onto one upstream zstd source-size + // tier: `resolve_level_params(Level(22), ..)` selects the HC config and + // window_log by raw `<= 16 KiB / 128 KiB / 256 KiB` thresholds, so a 20 KiB + // and a 100 KiB hint (ceil-log buckets 15 and 17) both land in the + // `<= 128 KiB` tier and resolve to the IDENTICAL matcher (same window_log, + // same HC hash/chain/search geometry). Keying on the raw ceil-log bucket + // would still reject the restore here because the buckets differ; the key + // must compare the resolved matcher shape so these share one snapshot. + let mut driver = MatchGeneratorDriver::new(8, 1); + let level = CompressionLevel::Level(22); + + driver.set_source_size_hint(20 * 1024); + driver.reset(level); + let window_a = driver.window_size(); + driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); + driver.capture_primed_dictionary(level); + + driver.set_source_size_hint(100 * 1024); + driver.reset(level); + let window_b = driver.window_size(); + assert_eq!( + window_a, window_b, + "precondition: both hints must land in the same Level 22 upstream zstd tier \ + (a={window_a}, b={window_b})" + ); + + let restored = driver.restore_primed_dictionary(level); + assert!( + restored, + "Level 22 snapshot captured at a 20 KiB hint must be restored into a \ + 100 KiB hint that resolves to the same upstream zstd tier (different ceil-log \ + buckets, identical matcher shape)" + ); +} + +#[test] +fn fast_dict_attaches_within_cutoff_bounds() { + // Within the attach bounds, every Fast dict frame attaches (the copy-mode + // owned path memmoved the whole input into history each frame; attach scans + // the input in place via the borrowed dual-base kernel). All hints here sit + // far below `FAST_ATTACH_DICT_CUTOFF_LOG` (2 GiB source) and the dict is far + // below `MAX_FAST_ATTACH_DICT_REGION` (16 MiB), so a hint that used to cross + // the old 8 KiB cutoff (8193 B) and a small one (8192 B) BOTH resolve to + // attach, and the Simple backend reports a borrowed (in-place) dict scan for + // both. This guards `FAST_ATTACH_DICT_CUTOFF_LOG` staying high enough that no + // in-bounds Fast hint falls back to the input-copy path; the OUT-of-bounds + // fallbacks are covered by `fast_attach_cutoff_keeps_virtual_positions_within_u32` + // (source) and `oversized_dict_hint_routes_fast_to_copy_mode` (dict size). + let level = CompressionLevel::Level(1); + for hint in [8192u64, 8193, 1 << 20] { + let mut driver = MatchGeneratorDriver::new(8, 1); + driver.set_source_size_hint(hint); + driver.reset(level); + driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); + assert!( + driver.borrowed_dict_supported(), + "Fast dict frame with hint {hint} must attach (borrowed in-place \ + dict scan), never fall back to the copy-mode input-copy path" + ); + } +} + +#[test] +fn fast_attach_cutoff_keeps_virtual_positions_within_u32() { + // The cutoff is 31, NOT the full u64 source-size range, because the borrowed + // dict kernel stores virtual positions as u32 (`cur_abs as u32`). The largest + // attached source `1 << CUTOFF` (plus the dict prefix) must stay below + // u32::MAX or that arithmetic wraps; the next bucket (4 GiB) would. This pins + // the bound so a future "just raise it to attach everything" change cannot + // silently reintroduce the overflow — raising the cutoff requires widening + // the kernel's position type first. + let max_attached: u64 = 1u64 << FAST_ATTACH_DICT_CUTOFF_LOG; + assert!( + max_attached <= u32::MAX as u64, + "the largest attached source 2^{FAST_ATTACH_DICT_CUTOFF_LOG} must fit u32 \ + virtual positions", + ); + assert!( + (1u64 << (FAST_ATTACH_DICT_CUTOFF_LOG + 1)) > u32::MAX as u64, + "the next bucket 2^{} would overflow u32 virtual positions", + FAST_ATTACH_DICT_CUTOFF_LOG + 1, + ); +} + +#[test] +fn oversized_dict_hint_routes_fast_to_copy_mode() { + // A dict whose region exceeds the tagged attach position field + // (`MAX_FAST_ATTACH_DICT_REGION`, 16 MiB) must route the Fast prime to COPY + // mode instead of the tagged attach fill, which would overflow the packed + // position. The decision is keyed on the load-set size hint, so a hint past + // the limit suffices to exercise it without allocating a real 16 MiB dict. + // Copy mode leaves the borrowed in-place dict scan (attach-only) unavailable. + let mut driver = MatchGeneratorDriver::new(8, 1); + driver.set_dictionary_size_hint(MAX_FAST_ATTACH_DICT_REGION + 1); + driver.reset(CompressionLevel::Level(1)); + driver.prime_with_dictionary(b"small dict content with some padding here", [1, 4, 8]); + assert!( + !driver.borrowed_dict_supported(), + "an oversized dict must use copy mode, not the tagged attach fill" + ); +} + +#[test] +fn block_samples_match_dict_is_true_for_non_simple_backend() { + // Production fallback: a non-Simple backend (here Row, Level 6) has no dict + // probe, so the driver wrapper answers CONSERVATIVELY `true` for ANY block — + // keeping the dict frame on the scan rather than letting the raw-fast-path + // emit a block raw and miss an embedded dict segment (see + // `dictionary_segment_in_incompressible_input_is_matched`). Only the + // Simple/Fast backend trades the blanket scan for a precise probe. + let dict = b"the quick brown fox jumps over the lazy dog 0123456789abcdef"; + let mut row = MatchGeneratorDriver::new(8, 6); + row.set_dictionary_size_hint(dict.len()); + row.reset(CompressionLevel::Level(6)); + row.prime_with_dictionary(dict, [1, 4, 8]); + assert!( + row.block_samples_match_dict(&dict[..32]), + "non-Simple backend must stay on the scan (true) for a dict frame" + ); + let random: alloc::vec::Vec = (0..64u8) + .map(|i| i.wrapping_mul(37).wrapping_add(13)) + .collect(); + assert!( + row.block_samples_match_dict(&random), + "non-Simple backend reports true regardless of block content" + ); +} + +#[test] +fn primed_snapshot_fast_attach_does_not_over_key_non_simple_backends() { + // `fast_attach` is a Simple/Fast-backend concept (the 8 KiB attach-vs-copy + // table split). Dfast/Row/HashChain each have their OWN attach/copy regime + // (`DFAST_ATTACH_DICT_CUTOFF_LOG`, `ROW_ATTACH_DICT_CUTOFF_LOG`, + // `HC_ATTACH_DICT_CUTOFF_LOG`) but those are deliberately kept OUT of the + // `fast_attach` key, which only models the Fast table split. Their snapshots + // are keyed by the resolved matcher geometry instead, and the HC modes share + // one window geometry so an HC cross-mode restore stays decodable (see + // `prime_with_dictionary`). Either way the `fast_attach` + // bit must NOT enter a non-Simple snapshot key — otherwise an unhinted + // capture (which would record `fast_attach = true`) and a hinted reset that + // resolves to the IDENTICAL `LevelParams` would key differently and force a + // needless re-prime. `Best` is a Row-backend lazy + // level; this also pins the Row arm recording its RESOLVED hash width on + // the unhinted path (a 0 default there keyed unhinted-vs-hinted apart). + // An explicit Row-backend level: `Best` now sits on level 13 (Btlazy2), + // so the named alias no longer reaches the Row arm this test pins. + let mut driver = MatchGeneratorDriver::new(8, 1); + let level = CompressionLevel::Level(12); + + // Capture with no hint. + driver.reset(level); + let window_a = driver.window_size(); + driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); + driver.capture_primed_dictionary(level); + + // Reset with a hint large enough to resolve to the same window/params as + // the unhinted level (>= 2^window_log, so the source-size cap is a no-op). + driver.set_source_size_hint(64 * 1024 * 1024); + driver.reset(level); + let window_b = driver.window_size(); + assert_eq!( + window_a, window_b, + "precondition: the large hint must resolve to the same window as the \ + unhinted level (a={window_a}, b={window_b})" + ); + + let restored = driver.restore_primed_dictionary(level); + assert!( + restored, + "a Row snapshot must restore across an unhinted vs large-hinted \ + reset that resolves to the identical matcher — `fast_attach` is a Fast \ + backend concept and must not over-key non-Simple shapes" + ); +} + +#[cfg(any())] // disabled: tested SuffixStore-per-block tail-handling specific to legacy MatchGenerator +#[test] +fn prime_with_dictionary_does_not_reuse_tiny_suffix_store() { + let mut driver = MatchGeneratorDriver::new(8, 2); + driver.reset(CompressionLevel::Fastest); + + // This dictionary leaves a 1-byte tail chunk (capacity=1 suffix table), + // which should never be committed to the matcher window. + driver.prime_with_dictionary(b"abcdefghi", [1, 4, 8]); + + assert!( + driver + .simple() + .window + .iter() + .all(|entry| entry.data.len() >= MIN_MATCH_LEN), + "dictionary priming must not commit tails shorter than MIN_MATCH_LEN" + ); +} + +#[test] +fn prime_with_dictionary_counts_only_committed_tail_budget() { + let mut driver = MatchGeneratorDriver::new(8, 1); + driver.reset(CompressionLevel::Fastest); + + let before = driver.simple_mut().max_window_size; + // One full slice plus a 1-byte tail that cannot be committed. + driver.prime_with_dictionary(b"abcdefghi", [1, 4, 8]); + + assert_eq!( + driver.simple_mut().max_window_size, + before + 8, + "retention budget must account only for dictionary bytes actually committed to history" + ); +} + +#[test] +fn dfast_prime_with_dictionary_counts_four_byte_tail_budget() { + let mut driver = MatchGeneratorDriver::new(8, 1); + driver.reset(CompressionLevel::Level(3)); + + let before = driver.dfast_matcher().max_window_size; + // One full slice plus a 4-byte tail. Dfast can still use this tail through + // short-hash overlap into the next block, so it should stay retained. + driver.prime_with_dictionary(b"abcdefghijkl", [1, 4, 8]); + + assert_eq!( + driver.dfast_matcher().max_window_size, + before + 12, + "dfast retention budget should include 4-byte dictionary tails" + ); +} + +#[test] +fn row_prime_with_dictionary_preserves_history_for_first_full_block() { + let mut driver = MatchGeneratorDriver::new(8, 1); + // Level(5) is the greedy Row backend (LEVEL_TABLE row 5: Greedy / RowHash). + // Level(4) now routes to Dfast, so this test must use Level(5) to actually + // exercise `RowMatchGenerator`'s dictionary priming. The 16-byte dict + + // 16-byte block lets the whole block match the primed dict (offset = dict + // length = 16). + driver.reset(CompressionLevel::Level(5)); + + let payload = b"abcdefghijklmnop"; + driver.prime_with_dictionary(payload, [1, 4, 8]); + + let mut space = driver.get_next_space(); + space.clear(); + space.extend_from_slice(payload); + driver.commit_space(space); + + let mut saw_match = false; + driver.start_matching(|seq| { + if let Sequence::Triple { + literals, + offset, + match_len, + } = seq + && literals.is_empty() + && offset == payload.len() + && match_len >= ROW_MIN_MATCH_LEN + { + saw_match = true; + } + }); + + assert!( + saw_match, + "row backend should match dictionary-primed history in first full block" + ); +} + +#[test] +fn row_prime_with_dictionary_subtracts_uncommitted_tail_budget() { + let mut driver = MatchGeneratorDriver::new(8, 1); + driver.reset(CompressionLevel::Level(5)); + + let base_window = driver.row_matcher().max_window_size; + // Slice size is 8. The trailing byte cannot be committed (<4 tail), + // so it must be subtracted from retained budget. + driver.prime_with_dictionary(b"abcdefghi", [1, 4, 8]); + + assert_eq!( + driver.row_matcher().max_window_size, + base_window + 8, + "row retained window must exclude uncommitted 1-byte tail" + ); +} + +#[test] +fn prime_with_dictionary_budget_shrinks_after_row_eviction() { + let mut driver = MatchGeneratorDriver::new(8, 1); + driver.reset(CompressionLevel::Level(5)); + // Keep live window tiny so dictionary-primed slices are evicted quickly. + driver.row_matcher_mut().max_window_size = 8; + driver.reported_window_size = 8; + + let base_window = driver.row_matcher().max_window_size; + driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); + assert_eq!(driver.row_matcher().max_window_size, base_window + 24); + + for block in [b"AAAAAAAA", b"BBBBBBBB"] { + let mut space = driver.get_next_space(); + space.clear(); + space.extend_from_slice(block); + driver.commit_space(space); + driver.skip_matching_with_hint(None); + } + + assert_eq!( + driver.dictionary_retained_budget, 0, + "dictionary budget should be fully retired once primed dict slices are evicted" + ); + assert_eq!( + driver.row_matcher().max_window_size, + base_window, + "retired dictionary budget must not remain reusable for live history" + ); +} + +/// Row → Simple transition drops the Row variant and the +/// post-switch active backend is exactly Simple. The window-emptied +/// check from the pre-enum era (`driver.row_matcher().window.is_empty()`) +/// is intentionally gone — the `Row` variant no longer exists after +/// the swap, so there is nothing to inspect by accessor; the "window +/// cleared" invariant is replaced by "variant dropped", and a +/// subsequent `row_matcher()` call would panic by design. The +/// pool-recycling side of the row backend is covered by +/// [`driver_row_commit_recycles_block_buffer_into_pool`]. +#[test] +fn row_get_last_space_then_reset_to_fastest_drops_row_variant() { + let mut driver = MatchGeneratorDriver::new(8, 1); + driver.reset(CompressionLevel::Level(5)); + assert_eq!( + driver.active_backend(), + super::super::strategy::BackendTag::Row + ); + + let mut space = driver.get_next_space(); + space.clear(); + space.extend_from_slice(b"row-data"); + driver.commit_space(space); + + assert_eq!(driver.get_last_space(), b"row-data"); + + driver.reset(CompressionLevel::Fastest); + assert_eq!( + driver.active_backend(), + super::super::strategy::BackendTag::Simple + ); +} + +/// Committing a Row block must return the input buffer to `vec_pool` +/// immediately (the bytes are mirrored into the contiguous `history`, +/// so there is no reason to retain a second copy in the window). This +/// guards the chunk-length window: the previous `VecDeque>` +/// window retained a full `block_capacity` buffer per committed block, +/// which on a heavily pre-split frame ballooned peak memory to many +/// times the live byte count. With the buffer recycled at commit time +/// the pool grows by exactly one Vec per committed block. +#[test] +fn driver_row_commit_recycles_block_buffer_into_pool() { + let mut driver = MatchGeneratorDriver::new(8, 1); + driver.reset(CompressionLevel::Level(5)); + assert_eq!( + driver.active_backend(), + super::super::strategy::BackendTag::Row + ); + + let before_pool = driver.vec_pool.len(); + let mut space = driver.get_next_space(); + space.clear(); + space.extend_from_slice(b"row-data-to-recycle"); + driver.commit_space(space); + + // `>` not `>=`: a fresh driver starts with `before_pool == 0`, so the + // weaker bound passes even if the commit failed to recycle. Strict + // growth proves the buffer was returned to the pool at commit time + // rather than retained in the window (the pre-`chunk_lens` bug). + assert!( + driver.vec_pool.len() > before_pool, + "row commit must recycle the committed block buffer into vec_pool \ + (before_pool = {before_pool}, after = {})", + driver.vec_pool.len() + ); + // The bytes still resolve through the contiguous history mirror. + assert_eq!(driver.get_last_space(), b"row-data-to-recycle"); +} + +#[test] +fn adjust_params_for_zero_source_size_uses_min_hinted_window_floor() { + let mut params = resolve_level_params(CompressionLevel::Level(4), None); + params.window_log = 22; + let adjusted = adjust_params_for_source_size(params, 0); + assert_eq!(adjusted.window_log, MIN_HINTED_WINDOW_LOG); +} + +#[test] +fn common_prefix_len_matches_scalar_reference_across_offsets() { + fn scalar_reference(a: &[u8], b: &[u8]) -> usize { + a.iter() + .zip(b.iter()) + .take_while(|(lhs, rhs)| lhs == rhs) + .count() + } + + for total_len in [ + 0usize, 1, 5, 15, 16, 17, 31, 32, 33, 64, 65, 127, 191, 257, 320, + ] { + let base: Vec = (0..total_len) + .map(|i| ((i * 13 + 7) & 0xFF) as u8) + .collect(); + + for start in [0usize, 1, 3] { + if start > total_len { + continue; + } + let a = &base[start..]; + let b = a.to_vec(); + assert_eq!( + common_prefix_len(a, &b), + scalar_reference(a, &b), + "equal slices total_len={total_len} start={start}" + ); + + let len = a.len(); + for mismatch in [0usize, 1, 7, 15, 16, 31, 32, 47, 63, 95, 127, 128, 129, 191] { + if mismatch >= len { + continue; + } + let mut altered = b.clone(); + altered[mismatch] ^= 0x5A; + assert_eq!( + common_prefix_len(a, &altered), + scalar_reference(a, &altered), + "total_len={total_len} start={start} mismatch={mismatch}" + ); + } + + if len > 0 { + let mismatch = len - 1; + let mut altered = b.clone(); + altered[mismatch] ^= 0xA5; + assert_eq!( + common_prefix_len(a, &altered), + scalar_reference(a, &altered), + "tail mismatch total_len={total_len} start={start} mismatch={mismatch}" + ); + } + } + } + + let long = alloc::vec![0xAB; 320]; + let shorter = alloc::vec![0xAB; 137]; + assert_eq!( + common_prefix_len(&long, &shorter), + scalar_reference(&long, &shorter) + ); +} + +#[test] +fn row_pick_lazy_returns_none_when_next_is_better() { + let mut matcher = RowMatchGenerator::new(1 << 22); + matcher.configure(ROW_CONFIG); + matcher.add_data(alloc::vec![b'a'; 64], |_| {}); + matcher.ensure_tables(); + + let abs_pos = matcher.history_abs_start + 16; + let best = MatchCandidate { + start: abs_pos, + offset: 8, + match_len: ROW_MIN_MATCH_LEN, + }; + assert!( + matcher.pick_lazy_match(abs_pos, 0, Some(best)).is_none(), + "lazy picker should defer when next position is clearly better" + ); +} + +#[test] +fn row_pick_lazy_depth2_returns_none_when_next2_significantly_better() { + let mut matcher = RowMatchGenerator::new(1 << 22); + matcher.configure(ROW_CONFIG); + matcher.lazy_depth = 2; + matcher.search_depth = 0; + matcher.offset_hist = [6, 9, 1]; + + let mut data = alloc::vec![b'x'; 40]; + data[11..30].copy_from_slice(b"EFABCABCAEFABCAEFAB"); + matcher.add_data(data, |_| {}); + matcher.ensure_tables(); + + let abs_pos = matcher.history_abs_start + 20; + let best = matcher + .best_match(abs_pos, 0) + .expect("expected baseline repcode match"); + assert_eq!(best.offset, 9); + // Baseline match length is fixed by the fixture data (the offset-9 + // rep run is 6 bytes long), independent of the accept threshold. + assert_eq!(best.match_len, 6); + + if let Some(next) = matcher.best_match(abs_pos + 1, 1) { + assert!(next.match_len <= best.match_len); + } + + let next2 = matcher + .best_match(abs_pos + 2, 2) + .expect("expected +2 candidate"); + assert!( + next2.match_len > best.match_len + 1, + "+2 candidate must be significantly better for depth-2 lazy skip" + ); + assert!( + matcher.pick_lazy_match(abs_pos, 0, Some(best)).is_none(), + "lazy picker should defer when +2 candidate is significantly better" + ); +} + +#[test] +fn row_pick_lazy_depth2_keeps_best_when_next2_is_only_one_byte_better() { + let mut matcher = RowMatchGenerator::new(1 << 22); + matcher.configure(ROW_CONFIG); + matcher.lazy_depth = 2; + matcher.search_depth = 0; + matcher.offset_hist = [6, 9, 1]; + + let mut data = alloc::vec![b'x'; 40]; + data[11..30].copy_from_slice(b"EFABCABCAEFABCAEFAZ"); + matcher.add_data(data, |_| {}); + matcher.ensure_tables(); + + let abs_pos = matcher.history_abs_start + 20; + let best = matcher + .best_match(abs_pos, 0) + .expect("expected baseline repcode match"); + assert_eq!(best.offset, 9); + // Baseline match length is fixed by the fixture data (the offset-9 + // rep run is 6 bytes long), independent of the accept threshold. + assert_eq!(best.match_len, 6); + + let next2 = matcher + .best_match(abs_pos + 2, 2) + .expect("expected +2 candidate"); + assert_eq!(next2.match_len, best.match_len + 1); + let chosen = matcher + .pick_lazy_match(abs_pos, 0, Some(best)) + .expect("lazy picker should keep current best"); + assert_eq!(chosen.start, best.start); + assert_eq!(chosen.offset, best.offset); + assert_eq!(chosen.match_len, best.match_len); +} + +/// Verifies row/tag extraction uses the shared hash mix bit-splitting contract. +#[test] +fn row_hash_and_row_extracts_high_bits() { + let mut matcher = RowMatchGenerator::new(1 << 22); + matcher.configure(ROW_CONFIG); + matcher.add_data( + alloc::vec![ + 0xAA, 0xBB, 0xCC, 0x11, 0x10, 0x20, 0x30, 0x40, 0xAA, 0xBB, 0xCC, 0x22, 0x50, 0x60, + 0x70, 0x80, + ], + |_| {}, + ); + matcher.ensure_tables(); + + let pos = matcher.history_abs_start + 8; + let (row, tag) = matcher + .hash_and_row(pos) + .expect("row hash should be available"); + + let idx = pos - matcher.history_abs_start; + let concat = matcher.live_history(); + // Mirror `row_key_value`: an mls-wide masked key when 8 lookahead bytes + // exist, the 4-byte key in the tail. `idx = 8` on a 16-byte history has + // exactly 8 bytes left, so the wide arm applies here. + let key_len = matcher.mls.min(6); + let value = u64::from_le_bytes(concat[idx..idx + 8].try_into().unwrap()) + & ((1u64 << (key_len * 8)) - 1); + let hash = crate::encoding::fastpath::hash_mix_u64_with_kernel(matcher.hash_kernel, value); + let total_bits = matcher.row_hash_log + ROW_TAG_BITS; + let combined = hash >> (u64::BITS as usize - total_bits); + let expected_row = + ((combined >> ROW_TAG_BITS) as usize) & ((1usize << matcher.row_hash_log) - 1); + let expected_tag = combined as u8; + + assert_eq!(row, expected_row); + assert_eq!(tag, expected_tag); +} + +#[test] +fn row_repcode_skips_candidate_before_history_start() { + let mut matcher = RowMatchGenerator::new(1 << 22); + matcher.configure(ROW_CONFIG); + matcher.history = alloc::vec![b'a'; 20]; + matcher.history_start = 0; + matcher.history_abs_start = 10; + matcher.offset_hist = [3, 0, 0]; + + assert!(matcher.repcode_candidate(12, 1).is_none()); +} + +#[test] +fn row_repcode_returns_none_when_position_too_close_to_history_end() { + let mut matcher = RowMatchGenerator::new(1 << 22); + matcher.configure(ROW_CONFIG); + matcher.history = b"abcde".to_vec(); + matcher.history_start = 0; + matcher.history_abs_start = 0; + matcher.offset_hist = [1, 0, 0]; + + assert!(matcher.repcode_candidate(4, 1).is_none()); +} + +#[cfg(all(feature = "std", target_arch = "x86_64"))] +#[test] +fn hash_mix_sse42_path_is_available_and_matches_accelerated_impl_when_supported() { + use crate::encoding::fastpath::{self, FastpathKernel}; + if !is_x86_feature_detected!("sse4.2") { + return; + } + let v = 0x0123_4567_89AB_CDEFu64; + // SAFETY: feature check above guarantees SSE4.2 is available. + let accelerated = unsafe { fastpath::sse42::hash_mix_u64(v) }; + // Dispatcher must resolve to SSE4.2 (or better) and produce the same mix. + let dispatched = fastpath::dispatch_hash_mix_u64(v); + let kernel = fastpath::select_kernel(); + if kernel == FastpathKernel::Sse42 { + assert_eq!(dispatched, accelerated); + } else { + // AVX2 kernel uses the same CRC32 instruction under the hood. + assert_eq!(dispatched, accelerated, "AVX2/SSE4.2 share CRC32 mix"); + } +} + +#[cfg(all(feature = "std", target_arch = "aarch64", target_endian = "little"))] +#[test] +fn hash_mix_crc_path_is_available_and_matches_accelerated_impl_when_supported() { + use crate::encoding::fastpath; + if !is_aarch64_feature_detected!("crc") { + return; + } + let v = 0x0123_4567_89AB_CDEFu64; + // SAFETY: feature check above guarantees CRC32 is available. + let accelerated = unsafe { fastpath::neon::hash_mix_u64(v) }; + let dispatched = fastpath::dispatch_hash_mix_u64(v); + assert_eq!(dispatched, accelerated); +} + +#[test] +fn hc_hash3_position_matches_hash3_formula() { + let bytes = [b'a', b'b', b'c', b'd']; + let read32 = u32::from_le_bytes(bytes); + let expected = (((read32 << 8).wrapping_mul(HC_PRIME3BYTES)) >> (32 - HC3_HASH_LOG)) as usize; + assert_eq!( + super::super::match_table::storage::MatchTable::hash3_position(&bytes, HC3_HASH_LOG), + expected + ); +} + +#[test] +fn hc_hash_position_matches_hash4_formula() { + let mut hc = HcMatchGenerator::new(1 << 20); + hc.configure(HC_CONFIG, super::super::strategy::StrategyTag::Lazy, 22); + let bytes = [b'a', b'b', b'c', b'd']; + let read32 = u32::from_le_bytes(bytes); + let expected = ((read32.wrapping_mul(HC_PRIME4BYTES)) >> (32 - hc.table.hash_log)) as usize; + assert_eq!(hc.table.hash_position(&bytes), expected); +} + +#[test] +fn btultra2_main_hash_uses_hash4_formula() { + let mut hc = HcMatchGenerator::new(1 << 20); + hc.configure( + BTULTRA2_HC_CONFIG_L22, + super::super::strategy::StrategyTag::BtUltra2, + 27, + ); + let bytes = [b'a', b'b', b'c', b'd', b'e', b'f', b'g', b'h']; + let read32 = u32::from_le_bytes(bytes[..4].try_into().unwrap()); + let expected = ((read32.wrapping_mul(HC_PRIME4BYTES)) >> (32 - hc.table.hash_log)) as usize; + let actual = super::super::match_table::storage::MatchTable::hash_position_with_mls( + &bytes, + hc.table.hash_log, + super::super::bt::BtMatcher::HASH_MLS, + ); + assert_eq!(actual, expected); +} + +#[test] +fn row_candidate_returns_none_when_abs_pos_near_end_of_history() { + let mut matcher = RowMatchGenerator::new(1 << 22); + matcher.configure(ROW_CONFIG); + // One byte short of the accept floor: from abs_pos 0 there are fewer + // than `ROW_MIN_MATCH_LEN` bytes left, so the length gate in + // `row_candidate` must short-circuit to `None` before touching the + // (here unbuilt) row tables. + matcher.history = alloc::vec![b'a'; ROW_MIN_MATCH_LEN - 1]; + matcher.history_start = 0; + matcher.history_abs_start = 0; + + assert!(matcher.row_candidate(0, 0).is_none()); +} + +#[test] +fn hc_chain_candidates_returns_sentinels_for_short_suffix() { + let mut hc = HcMatchGenerator::new(32); + hc.table.history = b"abc".to_vec(); + hc.table.history_start = 0; + hc.table.history_abs_start = 0; + hc.table.ensure_tables(); + + let candidates = hc.hc.chain_candidates(&hc.table, 0); + assert!(candidates.iter().all(|&pos| pos == usize::MAX)); +} + +#[test] +fn hc_reset_advances_floor_past_prior_frame_entries() { + use super::super::match_table::storage::MatchTable; + let mut hc = HcMatchGenerator::new(32); + hc.table.add_data(b"abcdeabcde".to_vec(), |_| {}); + hc.table.ensure_tables(); + // Populate real hash / chain entries for the first frame's positions. + hc.table.insert_positions(0, 6); + let prev_end = hc.table.history_abs_end(); + assert_eq!(prev_end, 10); + assert!(hc.table.hash_table.iter().any(|&v| v != HC_EMPTY)); + + hc.reset(|_| {}); + + // Behavioural contract: the previous frame's entries are no longer + // matchable. `reset` advances the floor past every prior position + // instead of zeroing the tables, so each populated slot now decodes + // to an absolute position strictly below `history_abs_start` and is + // rejected by the `window_low` guard before any byte is read. + assert_eq!(hc.table.history_abs_start, prev_end); + for &slot in hc.table.hash_table.iter() { + if let Some(candidate_abs) = + MatchTable::stored_abs_position_fast(slot, hc.table.position_base, hc.table.index_shift) + { + assert!( + candidate_abs < hc.table.history_abs_start, + "a prior-frame entry must resolve below the advanced floor" + ); + } + } +} + +#[test] +fn hc_reset_full_zeroes_when_floor_would_cross_ceiling() { + use super::super::match_table::storage::REBASE_RESET_FLOOR_CEILING; + let mut hc = HcMatchGenerator::new(32); + hc.table.add_data(b"abcdeabcde".to_vec(), |_| {}); + hc.table.ensure_tables(); + hc.table.hash_table.fill(123); + hc.table.chain_table.fill(456); + // Push the would-be floor (`history_abs_end`) past the ceiling so + // `reset` takes the bounded fallback: rewind to the origin and zero + // the tables, keeping the absolute cursor from climbing toward + // `usize::MAX` on 32-bit targets. + hc.table.history_abs_start = REBASE_RESET_FLOOR_CEILING; + + hc.reset(|_| {}); + + assert_eq!(hc.table.history_abs_start, 0); + assert_eq!(hc.table.position_base, 0); + assert!(hc.table.hash_table.iter().all(|&v| v == HC_EMPTY)); + assert!(hc.table.chain_table.iter().all(|&v| v == HC_EMPTY)); +} + +#[test] +fn hc_start_matching_returns_early_for_empty_current_block() { + let mut hc = HcMatchGenerator::new(32); + hc.table.add_data(Vec::new(), |_| {}); + let mut called = false; + hc.start_matching(|_| called = true); + assert!(!called, "empty current block should not emit sequences"); +} + +#[cfg(test)] +fn deterministic_high_entropy_bytes(seed: u64, len: usize) -> Vec { + let mut out = Vec::with_capacity(len); + let mut state = seed; + for _ in 0..len { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + out.push((state >> 40) as u8); + } + out +} + +#[test] +fn hc_sparse_skip_matching_preserves_tail_cross_block_match() { + let mut matcher = HcMatchGenerator::new(1 << 22); + let tail = b"Qz9kLm2Rp"; + let mut first = deterministic_high_entropy_bytes(0xD1B5_4A32_9C77_0E19, 4096); + let tail_start = first.len() - tail.len(); + first[tail_start..].copy_from_slice(tail); + matcher.table.add_data(first.clone(), |_| {}); + matcher.skip_matching(Some(true)); + + let mut second = tail.to_vec(); + second.extend_from_slice(b"after-tail-literals"); + matcher.table.add_data(second, |_| {}); + + let mut first_sequence = None; + matcher.start_matching(|seq| { + if first_sequence.is_some() { + return; + } + first_sequence = Some(match seq { + Sequence::Literals { literals } => (literals.len(), 0usize, 0usize), + Sequence::Triple { + literals, + offset, + match_len, + } => (literals.len(), offset, match_len), + }); + }); + + let (literals_len, offset, match_len) = + first_sequence.expect("expected at least one sequence after sparse skip"); + assert_eq!( + literals_len, 0, + "first sequence should start at block boundary" + ); + assert_eq!( + offset, + tail.len(), + "first match should reference previous tail" + ); + assert!( + match_len >= tail.len(), + "tail-aligned cross-block match must be preserved" + ); +} + +#[test] +fn btultra2_sparse_skip_matching_preserves_tail_cross_block_match() { + let mut matcher = HcMatchGenerator::new(1 << 20); + matcher.configure( + BTULTRA2_HC_CONFIG_L22, + super::super::strategy::StrategyTag::BtUltra2, + 20, + ); + let tail = b"Bt9kLm2Rp"; + let mut first = deterministic_high_entropy_bytes(0xA9C3_7F21_D4E8_510B, 4096); + let tail_start = first.len() - tail.len(); + first[tail_start..].copy_from_slice(tail); + matcher.table.add_data(first, |_| {}); + matcher.skip_matching(Some(true)); + + let mut second = tail.to_vec(); + second.extend_from_slice(b"after-tail-literals"); + matcher.table.add_data(second, |_| {}); + + let mut first_sequence = None; + matcher.start_matching(|seq| { + if first_sequence.is_some() { + return; + } + first_sequence = Some(match seq { + Sequence::Literals { literals } => (literals.len(), 0usize, 0usize), + Sequence::Triple { + literals, + offset, + match_len, + } => (literals.len(), offset, match_len), + }); + }); + + let (literals_len, offset, match_len) = + first_sequence.expect("expected at least one sequence after sparse BT skip"); + assert_eq!( + literals_len, 0, + "BT sparse skip should preserve an immediate boundary match" + ); + assert_eq!( + offset, + tail.len(), + "first BT match should reference previous tail" + ); + assert!( + match_len >= tail.len(), + "BT sparse skip must seed the dense tail for cross-block matching" + ); +} + +#[test] +fn hc_sparse_skip_matching_does_not_reinsert_sparse_tail_positions() { + let mut matcher = HcMatchGenerator::new(1 << 22); + let first = deterministic_high_entropy_bytes(0xC2B2_AE3D_27D4_EB4F, 4096); + matcher.table.add_data(first.clone(), |_| {}); + matcher.skip_matching(Some(true)); + + let current_len = first.len(); + let current_abs_start = + matcher.table.history_abs_start + matcher.table.window_size - current_len; + let current_abs_end = current_abs_start + current_len; + let dense_tail = HC_MIN_MATCH_LEN + INCOMPRESSIBLE_SKIP_STEP; + let tail_start = current_abs_end + .saturating_sub(dense_tail) + .max(matcher.table.history_abs_start) + .max(current_abs_start); + + let overlap_pos = (tail_start..current_abs_end) + .find(|&pos| (pos - current_abs_start).is_multiple_of(INCOMPRESSIBLE_SKIP_STEP)) + .expect("fixture should contain at least one sparse-grid overlap in dense tail"); + + let rel = matcher + .table + .relative_position(overlap_pos) + .expect("overlap position should be representable as relative position"); + let chain_idx = rel as usize & ((1 << matcher.table.chain_log) - 1); + assert_ne!( + matcher.table.chain_table[chain_idx], + rel + 1, + "sparse-grid tail positions must not be reinserted (self-loop chain entry)" + ); +} + +#[test] +fn hc_compact_history_drains_when_threshold_crossed() { + let mut hc = HcMatchGenerator::new(8); + hc.table.history = b"abcdefghijklmnopqrstuvwxyz".to_vec(); + hc.table.history_start = 16; + hc.table.compact_history(); + assert_eq!(hc.table.history_start, 0); + assert_eq!(hc.table.history, b"qrstuvwxyz"); +} + +#[test] +fn hc_insert_position_no_rebase_returns_when_relative_pos_unavailable() { + let mut hc = HcMatchGenerator::new(32); + hc.table.history = b"abcdefghijklmnop".to_vec(); + hc.table.history_abs_start = 0; + hc.table.position_base = 1; + hc.table.ensure_tables(); + let before_hash = hc.table.hash_table.clone(); + let before_chain = hc.table.chain_table.clone(); + + hc.table.insert_position_no_rebase(0); + + assert_eq!(hc.table.hash_table, before_hash); + assert_eq!(hc.table.chain_table, before_chain); +} + +#[test] +fn hc_insert_positions_advances_next_to_update3_for_contiguous_range() { + let mut hc = HcMatchGenerator::new(64); + hc.table.history = b"abcdefghijklmnopqrstuvwxyz".to_vec(); + hc.table.history_start = 0; + hc.table.history_abs_start = 0; + hc.table.position_base = 0; + hc.table.ensure_tables(); + hc.table.next_to_update3 = 0; + + hc.table.insert_positions(0, 9); + + assert_eq!( + hc.table.next_to_update3, 9, + "contiguous insert_positions should advance hash3 update cursor" + ); +} + +#[test] +fn hc_insert_positions_with_step_keeps_next_to_update3_cursor_for_sparse_ranges() { + let mut hc = HcMatchGenerator::new(64); + hc.table.history = b"abcdefghijklmnopqrstuvwxyz".to_vec(); + hc.table.history_start = 0; + hc.table.history_abs_start = 0; + hc.table.position_base = 0; + hc.table.ensure_tables(); + hc.table.next_to_update3 = 0; + + hc.table.insert_positions_with_step(0, 16, 4); + + assert_eq!( + hc.table.next_to_update3, 0, + "sparse insert_positions_with_step must not mark skipped positions as hash3-updated" + ); +} + +#[cfg(any())] +// disabled: tests legacy SuffixStore behavior incompatible with upstream zstd-shape kernel's HASH_READ_SIZE geometry +#[test] +fn prime_with_dictionary_budget_shrinks_after_simple_eviction() { + let mut driver = MatchGeneratorDriver::new(8, 1); + driver.reset(CompressionLevel::Fastest); + // Use a small live window so dictionary-primed slices are evicted + // quickly and budget retirement can be asserted deterministically. + driver.simple_mut().max_window_size = 8; + driver.reported_window_size = 8; + + let base_window = driver.simple_mut().max_window_size; + driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); + assert_eq!(driver.simple_mut().max_window_size, base_window + 24); + + for block in [b"AAAAAAAA", b"BBBBBBBB"] { + let mut space = driver.get_next_space(); + space.clear(); + space.extend_from_slice(block); + driver.commit_space(space); + driver.skip_matching_with_hint(None); + } + + assert_eq!( + driver.dictionary_retained_budget, 0, + "dictionary budget should be fully retired once primed dict slices are evicted" + ); + assert_eq!( + driver.simple_mut().max_window_size, + base_window, + "retired dictionary budget must not remain reusable for live history" + ); +} + +#[test] +fn prime_with_dictionary_budget_shrinks_after_dfast_eviction() { + let mut driver = MatchGeneratorDriver::new(8, 1); + driver.reset(CompressionLevel::Level(3)); + // Use a small live window in this regression so dictionary-primed slices are + // evicted quickly and budget retirement can be asserted deterministically. + driver.dfast_matcher_mut().max_window_size = 8; + driver.reported_window_size = 8; + + let base_window = driver.dfast_matcher().max_window_size; + driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); + assert_eq!(driver.dfast_matcher().max_window_size, base_window + 24); + + for block in [b"AAAAAAAA", b"BBBBBBBB"] { + let mut space = driver.get_next_space(); + space.clear(); + space.extend_from_slice(block); + driver.commit_space(space); + driver.skip_matching_with_hint(None); + } + + assert_eq!( + driver.dictionary_retained_budget, 0, + "dictionary budget should be fully retired once primed dict slices are evicted" + ); + assert_eq!( + driver.dfast_matcher().max_window_size, + base_window, + "retired dictionary budget must not remain reusable for live history" + ); +} + +#[test] +fn hc_prime_with_dictionary_preserves_history_for_first_full_block() { + let mut driver = MatchGeneratorDriver::new(8, 1); + // Route onto HashChain explicitly — `Better` resolves to the Row + // backend in production, and this test pins HC dict-prime behaviour. + driver.reset_on_hc_lazy(CompressionLevel::Better); + + driver.prime_with_dictionary(b"abcdefgh", [1, 4, 8]); + + let mut space = driver.get_next_space(); + space.clear(); + // Repeat the dictionary content so the HC matcher can find it. + // HC_MIN_MATCH_LEN is 5, so an 8-byte match is well above threshold. + space.extend_from_slice(b"abcdefgh"); + driver.commit_space(space); + + let mut saw_match = false; + driver.start_matching(|seq| { + if let Sequence::Triple { + literals, + offset, + match_len, + } = seq + && literals.is_empty() + && offset == 8 + && match_len >= HC_MIN_MATCH_LEN + { + saw_match = true; + } + }); + + assert!( + saw_match, + "hash-chain backend should match dictionary-primed history in first full block" + ); +} + +#[test] +fn prime_with_dictionary_budget_shrinks_after_hc_eviction() { + let mut driver = MatchGeneratorDriver::new(8, 1); + driver.reset_on_hc_lazy(CompressionLevel::Better); + // Use a small live window so dictionary-primed slices are evicted quickly. + driver.hc_matcher_mut().table.max_window_size = 8; + driver.reported_window_size = 8; + + let base_window = driver.hc_matcher().table.max_window_size; + driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); + assert_eq!(driver.hc_matcher().table.max_window_size, base_window + 24); + + for block in [b"AAAAAAAA", b"BBBBBBBB"] { + let mut space = driver.get_next_space(); + space.clear(); + space.extend_from_slice(block); + driver.commit_space(space); + driver.skip_matching_with_hint(None); + } + + assert_eq!( + driver.dictionary_retained_budget, 0, + "dictionary budget should be fully retired once primed dict slices are evicted" + ); + assert_eq!( + driver.hc_matcher().table.max_window_size, + base_window, + "retired dictionary budget must not remain reusable for live history" + ); +} + +#[test] +fn resident_reapply_restores_retained_dictionary_budget() { + // A reused-dict frame that re-borrows the resident dictionary (skips the + // re-prime) must restore the retained-dict budget the per-frame `reset` + // cleared. The matcher's `reset` re-inflates `max_window_size` by the dict + // region; without the restore the driver-level budget stays 0 and + // `retire_dictionary_budget` never shrinks that inflated window as the dict + // evicts. For the HashChain backend (whose `window_low` is measured against + // `max_window_size`) that lets a post-eviction match exceed the frame + // header's base window and emit an over-window offset. + let mut driver = MatchGeneratorDriver::new(1 << 16, 1); + let dict = b"abcdefghABCDEFGHijklmnopqrstuvwxyz0123456789"; + driver.set_dictionary_size_hint(dict.len()); + driver.reset_on_hc_lazy(CompressionLevel::Better); + driver.prime_with_dictionary(dict, [1, 4, 8]); + let base = driver.reported_window_size; + assert!( + driver.dictionary_retained_budget > 0, + "the priming frame must retain a non-zero dict budget" + ); + + // Second frame: the reset detects the resident dict and re-borrows it. + driver.set_dictionary_size_hint(dict.len()); + driver.reset_on_hc_lazy(CompressionLevel::Better); + assert!( + driver.dictionary_is_resident(), + "the second frame must re-borrow the resident dictionary" + ); + assert_eq!( + driver.dictionary_retained_budget, 0, + "reset clears the retained-dict budget" + ); + let inflated = driver.hc_matcher().table.max_window_size; + assert!( + inflated > base, + "reset re-inflates the window by the resident dict region \ + (inflated={inflated}, base={base})" + ); + + driver.reapply_resident_dictionary([1, 4, 8]); + assert_eq!( + driver.dictionary_retained_budget, + inflated - base, + "resident reapply must restore the retained-dict budget (= window \ + inflation) so the retire path can shrink the window as the dict evicts" + ); +} + +#[test] +fn hc_commit_without_eviction_retires_no_dictionary_budget() { + // Regression: after the window<->history dedup, MatchTable::add_data + // invokes its reuse_space callback for the *input* buffer (recycle), + // not for evicted chunks. The HC arm of commit_space must therefore + // derive eviction bytes from the window_size delta — counting the + // callback argument as evicted would charge the whole committed block + // as "evicted" and prematurely retire dictionary budget even when the + // window is nowhere near full. + let mut driver = MatchGeneratorDriver::new(8, 1); + driver.reset_on_hc_lazy(CompressionLevel::Better); + // A large live window so a small committed block evicts nothing. + driver.hc_matcher_mut().table.max_window_size = 1 << 20; + driver.reported_window_size = 1 << 20; + driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); + let budget_after_prime = driver.dictionary_retained_budget; + assert!( + budget_after_prime > 0, + "priming must retain a non-zero dictionary budget" + ); + + let mut space = driver.get_next_space(); + space.clear(); + space.extend_from_slice(b"AAAAAAAA"); + driver.commit_space(space); + driver.skip_matching_with_hint(None); + + assert_eq!( + driver.dictionary_retained_budget, budget_after_prime, + "a commit that evicts nothing must retire no dictionary budget" + ); +} + +#[test] +fn row_commit_without_eviction_retires_no_dictionary_budget() { + // Regression for the Row arm of commit_space after the window -> + // chunk_lens migration: RowMatchGenerator::add_data now invokes its + // reuse_space callback for the *input* buffer (per-commit recycle), + // not for evicted chunks. The Row arm must derive eviction bytes from + // the window_size delta like the Dfast / HashChain arms — counting the + // callback argument as evicted charges the whole committed block as + // "evicted" and prematurely retires dictionary budget even when the + // window is nowhere near full. + let mut driver = MatchGeneratorDriver::new(8, 1); + driver.reset(CompressionLevel::Level(5)); + assert!(matches!(driver.storage, MatcherStorage::Row(_))); + // A large live window so a small committed block evicts nothing. + driver.row_matcher_mut().max_window_size = 1 << 20; + driver.reported_window_size = 1 << 20; + driver.prime_with_dictionary(b"abcdefghABCDEFGHijklmnop", [1, 4, 8]); + let budget_after_prime = driver.dictionary_retained_budget; + assert!( + budget_after_prime > 0, + "priming must retain a non-zero dictionary budget" + ); + + let mut space = driver.get_next_space(); + space.clear(); + space.extend_from_slice(b"AAAAAAAA"); + driver.commit_space(space); + driver.skip_matching_with_hint(None); + + assert_eq!( + driver.dictionary_retained_budget, budget_after_prime, + "a Row commit that evicts nothing must retire no dictionary budget" + ); +} + +#[test] +fn hc_rebases_positions_after_u32_boundary() { + let mut matcher = HcMatchGenerator::new(64); + matcher.table.add_data(b"abcdeabcdeabcde".to_vec(), |_| {}); + matcher.table.ensure_tables(); + matcher.table.position_base = 0; + let history_abs_start: usize = match (u64::from(u32::MAX) + 64).try_into() { + Ok(value) => value, + Err(_) => return, + }; + // Simulate a long-running stream where absolute history positions crossed + // the u32 range. Before #51 this disabled HC inserts entirely. + matcher.table.history_abs_start = history_abs_start; + matcher.skip_matching(None); + assert_eq!( + matcher.table.position_base, matcher.table.history_abs_start, + "rebase should anchor to the oldest live absolute position" + ); + + assert!( + matcher + .table + .hash_table + .iter() + .any(|entry| *entry != HC_EMPTY), + "HC hash table should still be populated after crossing u32 boundary" + ); + + // Verify rebasing preserves candidate lookup, not just table population. + let abs_pos = matcher.table.history_abs_start + 10; + let candidates = matcher.hc.chain_candidates(&matcher.table, abs_pos); + assert!( + candidates.iter().any(|candidate| *candidate != usize::MAX), + "chain_candidates should return valid matches after rebase" + ); +} + +// 64-bit only: the >4 GiB absolute cursor this test fabricates cannot exist on +// a 32-bit target (usize == u32 can't address that much), and setting +// `history_abs_start` near `u32::MAX` there overflows `usize` in the +// `check_stream_abs_headroom` guard before the rebase path is reached. Mirrors +// the `try_into()` early-return guard on `hc_rebases_positions_after_u32_boundary`. +#[cfg(target_pointer_width = "64")] +#[test] +fn row_rebases_positions_after_u32_boundary() { + // Row stores absolute match positions as u32. On a long stream the + // cumulative absolute cursor crosses the u32 range even while the live + // window stays bounded; `add_data` must rebase the coordinate origin + // down to the oldest live byte instead of asserting. Before the rebase + // landed this panicked on the `< u32::MAX` assertion, dropping valid + // long Row-backed frames. + let mut m = RowMatchGenerator::new(64); + m.add_data(b"abcdeabcdeabcde".to_vec(), |_| {}); + + // Simulate ~4 GiB of stream behind a bounded window: the live bytes now + // sit just under the u32 absolute ceiling. + let near_ceiling = (u32::MAX as usize) - 16; + m.history_abs_start = near_ceiling; + + // The next commit would push a u32 position past the ceiling; add_data + // must rebase the origin rather than panic. + m.add_data(b"fghij".to_vec(), |_| {}); + + assert!( + m.history_abs_start < near_ceiling, + "add_data must rebase the absolute origin down when the cursor nears \ + u32::MAX (got {})", + m.history_abs_start + ); + assert!( + (m.history_abs_start + m.window_size) < u32::MAX as usize, + "after rebase the live window must fit below the u32 position ceiling" + ); +} + +#[test] +fn hc_rebase_rebuilds_only_inserted_prefix() { + let mut matcher = HcMatchGenerator::new(64); + matcher.table.add_data(b"abcdeabcdeabcde".to_vec(), |_| {}); + matcher.table.ensure_tables(); + matcher.table.position_base = 0; + let history_abs_start: usize = match (u64::from(u32::MAX) + 64).try_into() { + Ok(value) => value, + Err(_) => return, + }; + matcher.table.history_abs_start = history_abs_start; + let abs_pos = matcher.table.history_abs_start + 6; + + let mut expected = HcMatchGenerator::new(64); + expected.table.add_data(b"abcdeabcdeabcde".to_vec(), |_| {}); + expected.table.ensure_tables(); + expected.table.history_abs_start = history_abs_start; + expected.table.position_base = expected.table.history_abs_start; + expected.table.hash_table.fill(HC_EMPTY); + expected.table.chain_table.fill(HC_EMPTY); + for pos in expected.table.history_abs_start..abs_pos { + expected.table.insert_position_no_rebase(pos); + } + + matcher.table.maybe_rebase_positions(abs_pos); + + assert_eq!( + matcher.table.position_base, matcher.table.history_abs_start, + "rebase should still anchor to the oldest live absolute position" + ); + assert_eq!( + matcher.table.hash_table, expected.table.hash_table, + "rebase must rebuild only positions already inserted before abs_pos" + ); + assert_eq!( + matcher.table.chain_table, expected.table.chain_table, + "future positions must not be pre-seeded into HC chains during rebase" + ); +} + +#[cfg(any())] // disabled: tested legacy MatchGenerator/SuffixStore behavior removed in phase 1b +#[test] +fn suffix_store_with_single_slot_does_not_panic_on_keying() { + let mut suffixes = SuffixStore::with_capacity(1); + suffixes.insert(b"abcde", 0); + assert!(suffixes.contains_key(b"abcde")); + assert_eq!(suffixes.get(b"abcde"), Some(0)); +} + +#[cfg(any())] +// disabled: hash_fill_step is a legacy MatchGenerator field; FastKernelMatcher walks stride=1 today +#[test] +fn fastest_reset_uses_interleaved_hash_fill_step() { + let mut driver = MatchGeneratorDriver::new(32, 2); + + driver.reset(CompressionLevel::Uncompressed); + assert_eq!(driver.simple().hash_fill_step, 1); + + driver.reset(CompressionLevel::Fastest); + assert_eq!(driver.simple().hash_fill_step, FAST_HASH_FILL_STEP); + + // Better uses the HashChain backend with lazy2; verify that the backend switch + // happened and the lazy_depth is configured correctly. + driver.reset(CompressionLevel::Better); + assert_eq!( + driver.active_backend(), + super::super::strategy::BackendTag::HashChain + ); + assert_eq!(driver.window_size(), (1u64 << 23)); + assert_eq!(driver.hc_matcher().hc.lazy_depth, 2); +} + +#[cfg(any())] // disabled: tested legacy MatchGenerator/SuffixStore behavior removed in phase 1b +#[test] +fn simple_matcher_updates_offset_history_after_emitting_match() { + let mut matcher = MatchGenerator::new(64); + matcher.add_data( + b"abcdeabcdeabcde".to_vec(), + SuffixStore::with_capacity(64), + |_, _| {}, + ); + + assert!(matcher.next_sequence(|seq| { + assert_eq!( + seq, + Sequence::Triple { + literals: b"abcde", + offset: 5, + match_len: 10, + } + ); + })); + assert_eq!(matcher.offset_hist, [5, 1, 4]); +} + +#[cfg(any())] // disabled: tested legacy MatchGenerator/SuffixStore behavior removed in phase 1b +#[test] +fn simple_matcher_zero_literal_repcode_checks_rep1_before_hash_lookup() { + let mut matcher = MatchGenerator::new(64); + matcher.add_data( + b"abcdefghijabcdefghij".to_vec(), + SuffixStore::with_capacity(64), + |_, _| {}, + ); + + matcher.suffix_idx = 10; + matcher.last_idx_in_sequence = 10; + matcher.offset_hist = [99, 10, 4]; + + let candidate = matcher.repcode_candidate(&matcher.window.last().unwrap().data[10..], 0); + assert_eq!(candidate, Some((10, 10))); +} + +#[cfg(any())] // disabled: tested legacy MatchGenerator/SuffixStore behavior removed in phase 1b +#[test] +fn simple_matcher_repcode_can_target_previous_window_entry() { + let mut matcher = MatchGenerator::new(64); + matcher.add_data( + b"abcdefghij".to_vec(), + SuffixStore::with_capacity(64), + |_, _| {}, + ); + matcher.skip_matching(); + matcher.add_data( + b"abcdefghij".to_vec(), + SuffixStore::with_capacity(64), + |_, _| {}, + ); + + matcher.offset_hist = [99, 10, 4]; + + let candidate = matcher.repcode_candidate(&matcher.window.last().unwrap().data, 0); + assert_eq!(candidate, Some((10, 10))); +} + +#[cfg(any())] // disabled: tested legacy MatchGenerator/SuffixStore behavior removed in phase 1b +#[test] +fn simple_matcher_zero_literal_repcode_checks_rep2() { + let mut matcher = MatchGenerator::new(64); + matcher.add_data( + b"abcdefghijabcdefghij".to_vec(), + SuffixStore::with_capacity(64), + |_, _| {}, + ); + matcher.suffix_idx = 10; + matcher.last_idx_in_sequence = 10; + // rep1=4 does not match at idx 10, rep2=10 does. + matcher.offset_hist = [99, 4, 10]; + + let candidate = matcher.repcode_candidate(&matcher.window.last().unwrap().data[10..], 0); + assert_eq!(candidate, Some((10, 10))); +} + +#[cfg(any())] // disabled: tested legacy MatchGenerator/SuffixStore behavior removed in phase 1b +#[test] +fn simple_matcher_zero_literal_repcode_checks_rep0_minus1() { + let mut matcher = MatchGenerator::new(64); + matcher.add_data( + b"abcdefghijabcdefghij".to_vec(), + SuffixStore::with_capacity(64), + |_, _| {}, + ); + matcher.suffix_idx = 10; + matcher.last_idx_in_sequence = 10; + // rep1=4 and rep2=99 do not match; rep0-1 == 10 does. + matcher.offset_hist = [11, 4, 99]; + + let candidate = matcher.repcode_candidate(&matcher.window.last().unwrap().data[10..], 0); + assert_eq!(candidate, Some((10, 10))); +} + +#[cfg(any())] // disabled: tested legacy MatchGenerator/SuffixStore behavior removed in phase 1b +#[test] +fn simple_matcher_repcode_rejects_offsets_beyond_searchable_prefix() { + let mut matcher = MatchGenerator::new(64); + matcher.add_data( + b"abcdefghij".to_vec(), + SuffixStore::with_capacity(64), + |_, _| {}, + ); + matcher.skip_matching(); + matcher.add_data( + b"klmnopqrst".to_vec(), + SuffixStore::with_capacity(64), + |_, _| {}, + ); + matcher.suffix_idx = 3; + + let candidate = matcher.offset_match_len(14, &matcher.window.last().unwrap().data[3..]); + assert_eq!(candidate, None); +} + +#[cfg(any())] // disabled: tested legacy MatchGenerator/SuffixStore behavior removed in phase 1b +#[test] +fn simple_matcher_skip_matching_seeds_every_position_even_with_fast_step() { + let mut matcher = MatchGenerator::new(64); + matcher.hash_fill_step = FAST_HASH_FILL_STEP; + matcher.add_data( + b"abcdefghijklmnop".to_vec(), + SuffixStore::with_capacity(64), + |_, _| {}, + ); + matcher.skip_matching(); + matcher.add_data(b"bcdef".to_vec(), SuffixStore::with_capacity(64), |_, _| {}); + + assert!(matcher.next_sequence(|seq| { + assert_eq!( + seq, + Sequence::Triple { + literals: b"", + offset: 15, + match_len: 5, + } + ); + })); + assert!(!matcher.next_sequence(|_| {})); +} + +#[cfg(any())] // disabled: tested legacy MatchGenerator/SuffixStore behavior removed in phase 1b +#[test] +fn simple_matcher_skip_matching_with_incompressible_hint_uses_sparse_prefix() { + let mut matcher = MatchGenerator::new(128); + let first = b"abcdefghijklmnopqrstuvwxyz012345".to_vec(); + let sparse_probe = first[3..3 + MIN_MATCH_LEN].to_vec(); + let tail_start = first.len() - MIN_MATCH_LEN; + let tail_probe = first[tail_start..tail_start + MIN_MATCH_LEN].to_vec(); + matcher.add_data(first, SuffixStore::with_capacity(256), |_, _| {}); + + matcher.skip_matching_with_hint(Some(true)); + + // Observable behavior check: sparse-prefix probe should not immediately match. + matcher.add_data(sparse_probe, SuffixStore::with_capacity(256), |_, _| {}); + let mut sparse_first_is_literals = None; + assert!(matcher.next_sequence(|seq| { + if sparse_first_is_literals.is_none() { + sparse_first_is_literals = Some(matches!(seq, Sequence::Literals { .. })); + } + })); + assert!( + sparse_first_is_literals.unwrap_or(false), + "sparse-start probe should not produce an immediate match" + ); + + // Dense tail remains indexed for cross-block boundary matching. + let mut matcher = MatchGenerator::new(128); + matcher.add_data( + b"abcdefghijklmnopqrstuvwxyz012345".to_vec(), + SuffixStore::with_capacity(256), + |_, _| {}, + ); + matcher.skip_matching_with_hint(Some(true)); + matcher.add_data(tail_probe, SuffixStore::with_capacity(256), |_, _| {}); + let mut tail_first_is_immediate_match = None; + assert!(matcher.next_sequence(|seq| { + if tail_first_is_immediate_match.is_none() { + tail_first_is_immediate_match = + Some(matches!(seq, Sequence::Triple { literals, .. } if literals.is_empty())); + } + })); + assert!( + tail_first_is_immediate_match.unwrap_or(false), + "dense tail probe should match immediately at block start" + ); +} + +#[cfg(any())] // disabled: tested legacy MatchGenerator/SuffixStore behavior removed in phase 1b +#[test] +fn simple_matcher_add_suffixes_till_backfills_last_searchable_anchor() { + let mut matcher = MatchGenerator::new(64); + matcher.hash_fill_step = FAST_HASH_FILL_STEP; + matcher.add_data( + b"01234abcde".to_vec(), + SuffixStore::with_capacity(64), + |_, _| {}, + ); + matcher.add_suffixes_till(10, FAST_HASH_FILL_STEP); + + let last = matcher.window.last().unwrap(); + let tail = &last.data[5..10]; + assert_eq!(last.suffixes.get(tail), Some(5)); +} + +#[cfg(any())] // disabled: tested legacy MatchGenerator/SuffixStore behavior removed in phase 1b +#[test] +fn simple_matcher_add_suffixes_till_skips_when_idx_below_min_match_len() { + let mut matcher = MatchGenerator::new(128); + matcher.hash_fill_step = FAST_HASH_FILL_STEP; + matcher.add_data( + b"abcdefghijklmnopqrstuvwxyz".to_vec(), + SuffixStore::with_capacity(1 << 16), + |_, _| {}, + ); + + matcher.add_suffixes_till(MIN_MATCH_LEN - 1, FAST_HASH_FILL_STEP); + + let last = matcher.window.last().unwrap(); + let first_key = &last.data[..MIN_MATCH_LEN]; + assert_eq!(last.suffixes.get(first_key), None); +} + +#[cfg(any())] // disabled: tested legacy MatchGenerator/SuffixStore behavior removed in phase 1b +#[test] +fn simple_matcher_add_suffixes_till_fast_step_registers_interleaved_positions() { + let mut matcher = MatchGenerator::new(128); + matcher.hash_fill_step = FAST_HASH_FILL_STEP; + matcher.add_data( + b"abcdefghijklmnopqrstuvwxyz".to_vec(), + SuffixStore::with_capacity(1 << 16), + |_, _| {}, + ); + + matcher.add_suffixes_till(17, FAST_HASH_FILL_STEP); + + let last = matcher.window.last().unwrap(); + for pos in [0usize, 3, 6, 9, 12] { + let key = &last.data[pos..pos + MIN_MATCH_LEN]; + assert_eq!( + last.suffixes.get(key), + Some(pos), + "expected interleaved suffix registration at pos {pos}" + ); + } +} + +#[test] +fn dfast_skip_matching_handles_window_eviction() { + let mut matcher = DfastMatchGenerator::new(16); + + matcher.add_data(alloc::vec![1, 2, 3, 4, 5, 6], |_| {}); + matcher.skip_matching(None); + matcher.add_data(alloc::vec![7, 8, 9, 10, 11, 12], |_| {}); + matcher.skip_matching(None); + matcher.add_data(alloc::vec![7, 8, 9, 10, 11, 12], |_| {}); + + let mut reconstructed = alloc::vec![7, 8, 9, 10, 11, 12]; + matcher.start_matching(|seq| match seq { + Sequence::Literals { literals } => reconstructed.extend_from_slice(literals), + Sequence::Triple { + literals, + offset, + match_len, + } => { + reconstructed.extend_from_slice(literals); + let start = reconstructed.len() - offset; + for i in 0..match_len { + let byte = reconstructed[start + i]; + reconstructed.push(byte); + } + } + }); + + assert_eq!(reconstructed, [7, 8, 9, 10, 11, 12, 7, 8, 9, 10, 11, 12]); +} + +#[test] +fn dfast_add_data_callback_reports_evicted_len_not_capacity() { + let mut matcher = DfastMatchGenerator::new(8); + + let mut first = Vec::with_capacity(64); + first.extend_from_slice(b"abcdefgh"); + matcher.add_data(first, |_| {}); + + let mut second = Vec::with_capacity(64); + second.extend_from_slice(b"ijklmnop"); + + let mut observed_evicted_len = None; + matcher.add_data(second, |data| { + observed_evicted_len = Some(data.len()); + }); + + assert_eq!( + observed_evicted_len, + Some(8), + "eviction callback must report evicted byte length, not backing capacity" + ); +} + +/// Regression for the `commit_space` Dfast-branch eviction accounting bug +/// (CodeRabbit Critical on PR #146). Old code counted the INPUT buffer +/// length as `evicted_bytes` because Dfast's `add_data` callback receives +/// the input `Vec` for pool recycling (Dfast stores bytes in `history`, +/// not per-block Vecs). On the saturated-window 1:1 path the two coincide +/// so the previous test fixture passed by accident; this test forces the +/// divergent case where evicted != input by sequencing block lengths +/// `[4, 4, 5]` against `max_window_size = 10`: +/// +/// * after 1st commit: `window_blocks = [4]`, `window_size = 4` +/// * after 2nd commit: `window_blocks = [4, 4]`, `window_size = 8` +/// * 3rd commit (5 bytes): `8 + 5 > 10` → pop one 4-byte block (evict=4), +/// then push 5 (window_size=9). Bug counts `5`, fix counts `4`. +/// +/// The fix derives eviction from `window_size` delta + input length: +/// `evicted = pre + space_len - post`. Verified via the +/// `dictionary_retained_budget` observable: starting budget 100, after +/// the third commit (4 bytes actually evicted) the budget must read 96, +/// not 95. +/// Driver-path regression for the `commit_space` Dfast eviction accounting +/// bug. Exercises `MatchGeneratorDriver::commit_space` directly (not just +/// `DfastMatchGenerator::add_data`) so the assertion catches a future +/// regression that swaps the Dfast branch in `commit_space` back to +/// `evicted_bytes += data.len()` — the older draft of this regression +/// hand-recomputed the formula on the matcher and would pass either way. +/// +/// Fixture: `max_window_size = 10`, commit sequence `[4, 4, 5]`. The +/// divergent case where the popped block (4 bytes) and the new input +/// (5 bytes) have different sizes: +/// +/// * after commit `"abcd"` (4 B): window_blocks=[4], ws=4 +/// * after commit `"efgh"` (4 B): window_blocks=[4,4], ws=8 +/// * commit `"ijklm"` (5 B): 8+5>10 → pop front [4] (evict=4), +/// push 5 → window_blocks=[4,5], ws=9 +/// +/// `commit_space` then calls `retire_dictionary_budget(evicted)`. With +/// the fix `evicted=4`; with the bug it would be `evicted=5`. The +/// downstream `trim_after_budget_retire` cascade (which fires whenever +/// `retire_dictionary_budget` returns true) drives the budget further +/// down by trimming the now-oversize window; the final +/// `dictionary_retained_budget` differs between the two paths because +/// the cascade starting state differs (max_window_size after first +/// retire is `10 - evicted`). +/// +/// Tracing the fix path end-to-end with starting budget = 100: +/// 1st commit: evicted=0, no retire. +/// 2nd commit: evicted=0, no retire. +/// 3rd commit: evicted=4. retire(4) → budget=96, max_window=6. +/// trim_after_budget_retire: +/// iter1: ws=9 > max=6, pop [4] → ws=5, evicted=4. +/// retire(4) → budget=92, max_window=2. +/// iter2: ws=5 > max=2, pop [5] → ws=0, evicted=5. +/// retire(5) → budget=87, max_window=0. +/// iter3: ws=0, no trim, retire(0) → false, exit. +/// Final budget = 87. Final max_window_size = 0. +/// +/// In the buggy path the 3rd commit would compute `evicted=5`, retire +/// would reclaim 5 instead of 4, shrinking max_window_size to 5 +/// instead of 6 — and then the cascade arithmetic produces a +/// different final budget (and on the 2nd commit the cascade would +/// already have shrunk max_window_size to 0, causing the 3rd commit +/// to panic on `data.len() <= max_window_size`). Either way the +/// regression surfaces as a test failure. +#[test] +fn dfast_commit_space_eviction_uses_window_size_delta() { + use crate::encoding::CompressionLevel; + + let mut driver = MatchGeneratorDriver::new(10, 1); + driver.reset(CompressionLevel::Level(3)); + assert!(matches!(driver.storage, MatcherStorage::Dfast(_))); + + // Override the level-derived window with a tiny one so the + // 4 + 4 + 5 = 13 commit sequence below actually crosses the + // boundary. A 16 KiB+ default window would never evict on this + // little data and the bug would stay invisible. + driver.dfast_matcher_mut().max_window_size = 10; + driver.dictionary_retained_budget = 100; + + let mut space1 = Vec::with_capacity(64); + space1.extend_from_slice(b"abcd"); + driver.commit_space(space1); + assert_eq!( + driver.dictionary_retained_budget, 100, + "1st commit fills window 0 → 4, no eviction, no retire" + ); + + let mut space2 = Vec::with_capacity(64); + space2.extend_from_slice(b"efgh"); + driver.commit_space(space2); + assert_eq!( + driver.dictionary_retained_budget, 100, + "2nd commit fills window 4 → 8, no eviction, no retire" + ); + + let mut space3 = Vec::with_capacity(64); + space3.extend_from_slice(b"ijklm"); + driver.commit_space(space3); + assert_eq!( + driver.dictionary_retained_budget, 87, + "3rd commit + trim_after_budget_retire cascade. With the fix \ + (evicted=4 from window_size delta) the cascade reclaims 100 \ + → 96 → 92 → 87. With the bug (evicted=5 from data.len()) the \ + 3rd commit would panic on `data.len() <= max_window_size` \ + after the 2nd commit's cascade had already shrunk \ + max_window_size to 0." + ); + assert_eq!( + driver.dfast_matcher_mut().max_window_size, + 0, + "cascade drains max_window_size to 0 once budget reclaim \ + exceeds the initial window size" + ); +} + +#[test] +fn dfast_trim_to_window_evicts_oldest_block_by_length() { + // After the history-only storage refactor (#111 Phase 7c step 3), + // Dfast no longer retains input `Vec`s — the `history` + // contiguous buffer is the sole byte store, and `add_data` + // returns the input Vec to the caller's pool eagerly. So + // `trim_to_window` doesn't have anything to hand back to the + // closure (no Vec exists to give). The eviction is observable + // instead through `window_size` shrinking by the per-block + // length recorded in `window_blocks`. + let mut matcher = DfastMatchGenerator::new(16); + + let mut first = Vec::with_capacity(64); + first.extend_from_slice(b"abcdefgh"); + matcher.add_data(first, |_| {}); + + let mut second = Vec::with_capacity(64); + second.extend_from_slice(b"ijklmnop"); + matcher.add_data(second, |_| {}); + + assert_eq!(matcher.window_size, 16); + assert_eq!(matcher.window_blocks.len(), 2); + + matcher.max_window_size = 8; + + matcher.trim_to_window(); + + // No callback signature to assert on: the Dfast variant of + // `trim_to_window` takes none. That signature shape (vs HC/Row + // which accept `impl FnMut(Vec)`) is the property locking in + // the contract — there is no closure to invoke or skip, so no + // future change can "start invoking the callback" without a + // compile-time signature break that the dispatcher and this test + // would force the author to address. + assert_eq!( + matcher.window_size, 8, + "exactly one 8-byte block must remain" + ); + assert_eq!(matcher.window_blocks.len(), 1); + assert_eq!(matcher.history_abs_start, 8); +} + +#[test] +fn dfast_inserts_tail_positions_for_next_block_matching() { + let mut matcher = DfastMatchGenerator::new(1 << 22); + + matcher.add_data(b"012345bcdea".to_vec(), |_| {}); + let mut history = Vec::new(); + matcher.start_matching(|seq| match seq { + Sequence::Literals { literals } => history.extend_from_slice(literals), + Sequence::Triple { .. } => unreachable!("first block should not match history"), + }); + assert_eq!(history, b"012345bcdea"); + + matcher.add_data(b"bcdeabcdeab".to_vec(), |_| {}); + let mut saw_first_sequence = false; + matcher.start_matching(|seq| { + assert!(!saw_first_sequence, "expected a single cross-block match"); + saw_first_sequence = true; + match seq { + Sequence::Literals { .. } => { + panic!("expected tail-anchored cross-block match before any literals") + } + Sequence::Triple { + literals, + offset, + match_len, + } => { + assert_eq!(literals, b""); + assert_eq!(offset, 5); + assert_eq!(match_len, 11); + let start = history.len() - offset; + for i in 0..match_len { + let byte = history[start + i]; + history.push(byte); + } + } + } + }); + + assert!( + saw_first_sequence, + "expected tail-anchored cross-block match" + ); + assert_eq!(history, b"012345bcdeabcdeabcdeab"); +} + +/// Regression for #49 — locks down `MatchTable::backfill_boundary_positions` +/// for the [`HcMatchGenerator`] lazy path. `backfill_boundary_positions` +/// seeds ONLY the last `< 4` bytes of the previous slice (positions in +/// `[current_abs_start - 3, current_abs_start)`) — the bytes that +/// `insert_position` could not hash at the time because hashing needs +/// 4 bytes of lookahead. The existing 8 MiB window roundtrip test +/// exercises cross-slice behaviour end-to-end, but does not isolate +/// the backfill of those final 1-3 unhashable bytes. +/// +/// Fixture is built so the cross-block match's candidate position +/// MUST lie in `[block_1_end - 3, block_1_end)`: +/// +/// - Block 1 = `b"PQRSTBCD"` (8 bytes). Block 1's `start_matching` +/// hashes positions 0..=4 (each has 4 bytes of forward context); +/// positions 5/6/7 are the unhashable tail. +/// - Block 2 = `b"BCDBCDBCDB"` (10 bytes). At absolute position 8 +/// (block 2 start) the 4-byte window is `b"BCDB"`. The ONLY place +/// `b"BCDB"` was inserted in the hash + chain tables is position 5 +/// — via `backfill_boundary_positions` on the next-slice entry +/// (the 4-byte window at position 5 is `data[5..9] = b"BCD" + +/// block_2[0] = b"BCDB"`). +/// +/// If `backfill_boundary_positions` regresses, position 5 is never +/// hashed, position 8's lookup misses, and the lazy parser falls +/// through to a leading literals run — `offset == 3, match_len >= 4` +/// would no longer hold. +#[test] +fn hashchain_inserts_tail_positions_for_next_block_matching() { + let mut matcher = HcMatchGenerator::new(1 << 22); + matcher.configure(HC_CONFIG, super::super::strategy::StrategyTag::Lazy, 22); + + matcher.table.add_data(b"PQRSTBCD".to_vec(), |_| {}); + let mut history = alloc::vec::Vec::new(); + matcher.start_matching(|seq| match seq { + Sequence::Literals { literals } => history.extend_from_slice(literals), + Sequence::Triple { .. } => unreachable!("first block has no internal repeats"), + }); + assert_eq!(history, b"PQRSTBCD"); + + matcher.table.add_data(b"BCDBCDBCDB".to_vec(), |_| {}); + let mut first_sequence_offset: Option = None; + let mut first_sequence_match_len: Option = None; + matcher.start_matching(|seq| { + if first_sequence_offset.is_some() { + return; + } + match seq { + Sequence::Literals { .. } => { + panic!( + "expected tail-anchored cross-block match before any literals — \ + backfill_boundary_positions did not seed positions 5/6/7" + ) + } + Sequence::Triple { + literals, + offset, + match_len, + } => { + assert_eq!(literals, b"", "no leading literals on the boundary match"); + first_sequence_offset = Some(offset); + first_sequence_match_len = Some(match_len); + } + } + }); + + let offset = first_sequence_offset.expect( + "expected tail-anchored cross-block match emitted from backfill_boundary_positions", + ); + assert!( + (1..=3).contains(&offset), + "boundary match offset {offset} must point into the unhashable tail \ + (positions 5/6/7 of an 8-byte block 1) so the test specifically \ + locks down backfill_boundary_positions", + ); + assert_eq!( + offset, 3, + "candidate position must land at 5 (= block_1_len - 3) so the 4-byte \ + window `data[5..9] = b\"BCDB\"` matches block 2's first hash lookup", + ); + let match_len = first_sequence_match_len.unwrap(); + assert!( + match_len >= HC_MIN_MATCH_LEN, + "match_len {match_len} must clear the HC min-match floor", + ); +} + +#[test] +fn dfast_dense_skip_matching_backfills_previous_tail_for_next_block() { + let mut matcher = DfastMatchGenerator::new(1 << 22); + let tail = b"Qz9kLm2Rp"; + let mut first = b"0123456789abcdef".to_vec(); + first.extend_from_slice(tail); + matcher.add_data(first.clone(), |_| {}); + matcher.skip_matching(Some(false)); + + let mut second = tail.to_vec(); + second.extend_from_slice(b"after-tail-literals"); + matcher.add_data(second, |_| {}); + + let mut first_sequence = None; + matcher.start_matching(|seq| { + if first_sequence.is_some() { + return; + } + first_sequence = Some(match seq { + Sequence::Literals { literals } => (literals.len(), 0usize, 0usize), + Sequence::Triple { + literals, + offset, + match_len, + } => (literals.len(), offset, match_len), + }); + }); + + let (lit_len, offset, match_len) = first_sequence.expect("expected at least one sequence"); + assert_eq!( + lit_len, 0, + "expected immediate cross-block match at block start" + ); + assert_eq!( + offset, + tail.len(), + "expected dense skip to preserve cross-boundary tail match" + ); + assert!( + match_len >= DFAST_MIN_MATCH_LEN, + "match length should satisfy dfast minimum match length" + ); +} + +#[test] +fn dfast_sparse_skip_matching_preserves_tail_cross_block_match() { + let mut matcher = DfastMatchGenerator::new(1 << 22); + let tail = b"Qz9kLm2Rp"; + let mut first = deterministic_high_entropy_bytes(0x9E37_79B9_7F4A_7C15, 4096); + let tail_start = first.len() - tail.len(); + first[tail_start..].copy_from_slice(tail); + matcher.add_data(first.clone(), |_| {}); + + matcher.skip_matching(Some(true)); + + let mut second = tail.to_vec(); + second.extend_from_slice(b"after-tail-literals"); + matcher.add_data(second, |_| {}); + + let mut first_sequence = None; + matcher.start_matching(|seq| { + if first_sequence.is_some() { + return; + } + first_sequence = Some(match seq { + Sequence::Literals { literals } => (literals.len(), 0usize, 0usize), + Sequence::Triple { + literals, + offset, + match_len, + } => (literals.len(), offset, match_len), + }); + }); + + let (lit_len, offset, match_len) = first_sequence.expect("expected at least one sequence"); + assert_eq!( + lit_len, 0, + "expected immediate cross-block match at block start" + ); + assert_eq!( + offset, + tail.len(), + "expected match against densely seeded tail" + ); + assert!( + match_len >= DFAST_MIN_MATCH_LEN, + "match length should satisfy dfast minimum match length" + ); +} + +#[test] +fn dfast_skip_matching_dense_backfills_newly_hashable_long_tail_positions() { + let mut matcher = DfastMatchGenerator::new(1 << 22); + let first = deterministic_high_entropy_bytes(0x7A64_0315_D4E1_91C3, 4096); + let first_len = first.len(); + matcher.add_data(first, |_| {}); + matcher.skip_matching_dense(); + + // Appending one byte makes exactly the previous block's last 7 starts + // newly eligible for 8-byte long-hash insertion. + matcher.add_data(alloc::vec![0xAB], |_| {}); + matcher.skip_matching_dense(); + + let target_abs_pos = first_len - 7; + let target_rel = target_abs_pos - matcher.history_abs_start; + let live = matcher.live_history(); + assert!( + target_rel + 8 <= live.len(), + "fixture must make the boundary start long-hashable" + ); + let long_hash = matcher.long_hash_index(&live[target_rel..]); + let target_slot = matcher.pack_slot(target_abs_pos); + // Single-slot tables (upstream zstd parity): the bucket holds at most one + // u32; the assertion below is a direct equality (no `.contains`). + assert_ne!( + target_slot, DFAST_EMPTY_SLOT, + "pack_slot must never return the empty-slot sentinel for a real position" + ); + assert_eq!( + matcher.tables[long_hash], target_slot, + "dense skip must seed long-hash entry for newly hashable boundary start" + ); +} + +#[test] +fn dfast_seed_remaining_hashable_starts_seeds_last_short_hash_positions() { + let mut matcher = DfastMatchGenerator::new(1 << 20); + let block = deterministic_high_entropy_bytes(0x13F0_9A6D_55CE_7B21, 64); + matcher.add_data(block, |_| {}); + matcher.ensure_hash_tables(); + + let current_len = matcher.window_blocks.back().copied().unwrap_or(0); + let current_abs_start = matcher.history_abs_start + matcher.window_size - current_len; + let seed_start = current_len - DFAST_MIN_MATCH_LEN; + matcher.seed_remaining_hashable_starts(current_abs_start, current_len, seed_start); + + let target_abs_pos = current_abs_start + current_len - 5; + let target_rel = target_abs_pos - matcher.history_abs_start; + let live = matcher.live_history(); + assert!( + target_rel + 5 <= live.len(), + "fixture must leave the last short-hash start valid" + ); + let short_hash = matcher.short_hash_index(&live[target_rel..]); + let target_slot = matcher.pack_slot(target_abs_pos); + assert_ne!( + target_slot, DFAST_EMPTY_SLOT, + "pack_slot must never return the empty-slot sentinel for a real position" + ); + assert_eq!( + matcher.tables[matcher.long_len() + short_hash], + target_slot, + "tail seeding must include the last 5-byte-hashable start" + ); +} + +#[test] +fn dfast_seed_remaining_hashable_starts_handles_pos_at_block_end() { + let mut matcher = DfastMatchGenerator::new(1 << 20); + let block = deterministic_high_entropy_bytes(0x7BB2_DA91_441E_C0EF, 64); + matcher.add_data(block, |_| {}); + matcher.ensure_hash_tables(); + + let current_len = matcher.window_blocks.back().copied().unwrap_or(0); + let current_abs_start = matcher.history_abs_start + matcher.window_size - current_len; + matcher.seed_remaining_hashable_starts(current_abs_start, current_len, current_len); + + let target_abs_pos = current_abs_start + current_len - 5; + let target_rel = target_abs_pos - matcher.history_abs_start; + let live = matcher.live_history(); + assert!( + target_rel + 5 <= live.len(), + "fixture must leave the last short-hash start valid" + ); + let short_hash = matcher.short_hash_index(&live[target_rel..]); + let target_slot = matcher.pack_slot(target_abs_pos); + assert_ne!( + target_slot, DFAST_EMPTY_SLOT, + "pack_slot must never return the empty-slot sentinel for a real position" + ); + assert_eq!( + matcher.tables[matcher.long_len() + short_hash], + target_slot, + "tail seeding must still include the last 5-byte-hashable start when pos is at block end" + ); +} + +/// `ensure_room_for` must trigger `reduce()` when the requested +/// absolute position would push a relative offset past +/// `u32::MAX - DFAST_REBASE_GUARD_BAND`. After the rebase, the +/// pre-existing entry at a much-smaller absolute position falls +/// below `reducer` and gets cleared to `DFAST_EMPTY_SLOT`; a fresh +/// insert at the boundary position must `pack_slot` to a valid +/// non-sentinel value that `unpack_slot` resolves back to the same +/// absolute position. Mirrors `LdmHashTable::ensure_room_for_*` +/// from PR #139. +/// +/// Runs on every target — `trigger_abs = u32::MAX - +/// DFAST_REBASE_GUARD_BAND + 1 = 0xC0000000`, which fits in `usize` +/// on i686 (`usize::MAX = u32::MAX`) without overflow, so the +/// packed-slot boundary path + u32 ↔ usize round-trip is exercised +/// on every pointer width we ship. +#[test] +fn dfast_ensure_room_for_rebases_above_guard_band() { + let mut dfast = DfastMatchGenerator::new(1 << 22); + dfast.set_hash_bits(10, 10); + dfast.ensure_hash_tables(); + + // Seed an early insert near the current base in BOTH tables. + // `ensure_room_for` / `reduce` is a shared contract for both + // `short_hash` and `long_hash`; without seeding both, a + // regression that only cleared short_hash would still pass. + // Direct `pack_slot` + bucket write keeps the test focused on + // the rebase mechanics and avoids dragging in the full + // `insert_position` flow with its history/window setup. + let early_abs = 1024usize; + let early_packed = dfast.pack_slot(early_abs); + assert_ne!(early_packed, DFAST_EMPTY_SLOT); + let short0 = dfast.long_len(); + dfast.tables[short0] = early_packed; + dfast.tables[0] = early_packed; + + // Pick a trigger position that forces the first rebase. With + // `position_base = 0`, the smallest `abs_pos` that fails the + // `rel <= max_rel` test is `u32::MAX - DFAST_REBASE_GUARD_BAND + // + 1`. After one `reduce(DFAST_REBASE_GUARD_BAND)` the base + // advances by `DFAST_REBASE_GUARD_BAND`. + let trigger_abs = (u32::MAX as usize) - (DFAST_REBASE_GUARD_BAND as usize) + 1; + assert_eq!(dfast.position_base, 0); + dfast.ensure_room_for(trigger_abs); + assert_eq!( + dfast.position_base, DFAST_REBASE_GUARD_BAND as usize, + "rebase must advance position_base by DFAST_REBASE_GUARD_BAND" + ); + + // The early entry at abs=1024 had packed slot 1025; the rebase + // subtracts `DFAST_REBASE_GUARD_BAND` (= 2^30) from every slot. + // 1025 <= 2^30 so the slot drops to the empty sentinel — + // upstream zstd parity for `ZSTD_window_reduce`'s clamp-at-zero rule. + // Verify BOTH tables — `reduce()` walks them in sequence. + assert_eq!( + dfast.tables[dfast.long_len()], + DFAST_EMPTY_SLOT, + "pre-rebase short-hash entries below the reducer must become empty" + ); + assert_eq!( + dfast.tables[0], DFAST_EMPTY_SLOT, + "pre-rebase long-hash entries below the reducer must become empty" + ); + + // A fresh insert past the rebase boundary must round-trip: + // pack to a non-sentinel value, then unpack back to the same + // absolute position via `position_base + slot - 1`. + let post_packed = dfast.pack_slot(trigger_abs); + assert_ne!(post_packed, DFAST_EMPTY_SLOT); + let unpacked = dfast.position_base + (post_packed as usize) - 1; + assert_eq!( + unpacked, trigger_abs, + "post-rebase pack/unpack must round-trip the absolute position" + ); +} + +#[test] +fn dfast_sparse_skip_matching_backfills_previous_tail_for_consecutive_sparse_blocks() { + let mut matcher = DfastMatchGenerator::new(1 << 22); + let boundary_prefix = [0xFA, 0xFB, 0xFC]; + let boundary_suffix = [0xFD, 0xEE, 0xAD, 0xBE, 0xEF, 0x11, 0x22, 0x33]; + + let mut first = deterministic_high_entropy_bytes(0xA5A5_5A5A_C3C3_3C3C, 4096); + let first_tail_start = first.len() - boundary_prefix.len(); + first[first_tail_start..].copy_from_slice(&boundary_prefix); + matcher.add_data(first, |_| {}); + matcher.skip_matching(Some(true)); + + let mut second = deterministic_high_entropy_bytes(0xA5A5_5A5A_C3C3_3C3C, 4096); + second[..boundary_suffix.len()].copy_from_slice(&boundary_suffix); + matcher.add_data(second.clone(), |_| {}); + matcher.skip_matching(Some(true)); + + let mut third = boundary_prefix.to_vec(); + third.extend_from_slice(&boundary_suffix); + third.extend_from_slice(b"-trailing-literals"); + matcher.add_data(third, |_| {}); + + let mut first_sequence = None; + matcher.start_matching(|seq| { + if first_sequence.is_some() { + return; + } + first_sequence = Some(match seq { + Sequence::Literals { literals } => (literals.len(), 0usize, 0usize), + Sequence::Triple { + literals, + offset, + match_len, + } => (literals.len(), offset, match_len), + }); + }); + + let (lit_len, offset, match_len) = first_sequence.expect("expected at least one sequence"); + assert_eq!( + lit_len, 0, + "expected immediate match from the prior sparse-skip boundary" + ); + assert_eq!( + offset, + second.len() + boundary_prefix.len(), + "expected match against backfilled first→second boundary start" + ); + assert!( + match_len >= DFAST_MIN_MATCH_LEN, + "match length should satisfy dfast minimum match length" + ); +} + +#[test] +fn fastest_hint_iteration_23_sequences_reconstruct_source() { + fn generate_data(seed: u64, len: usize) -> Vec { + let mut state = seed; + let mut data = Vec::with_capacity(len); + for _ in 0..len { + state = state + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + data.push((state >> 33) as u8); + } + data + } + + let i = 23u64; + let len = (i * 89 % 16384) as usize; + let mut data = generate_data(i, len); + // Append a repeated slice so the fixture deterministically exercises + // the match path (Sequence::Triple) instead of only literals. + let repeat = data[128..256].to_vec(); + data.extend_from_slice(&repeat); + data.extend_from_slice(&repeat); + + let mut driver = MatchGeneratorDriver::new(1024 * 128, 1); + driver.set_source_size_hint(data.len() as u64); + driver.reset(CompressionLevel::Fastest); + let mut space = driver.get_next_space(); + space[..data.len()].copy_from_slice(&data); + space.truncate(data.len()); + driver.commit_space(space); + + let mut rebuilt = Vec::with_capacity(data.len()); + let mut saw_triple = false; + driver.start_matching(|seq| match seq { + Sequence::Literals { literals } => rebuilt.extend_from_slice(literals), + Sequence::Triple { + literals, + offset, + match_len, + } => { + saw_triple = true; + rebuilt.extend_from_slice(literals); + assert!(offset > 0, "offset must be non-zero"); + assert!( + offset <= rebuilt.len(), + "offset must reference already-produced bytes: offset={} produced={}", + offset, + rebuilt.len() + ); + let start = rebuilt.len() - offset; + for idx in 0..match_len { + let b = rebuilt[start + idx]; + rebuilt.push(b); + } + } + }); + + // Whether THIS specific iteration produces a Triple depends on + // the matcher's step-skip schedule (upstream zstd-shape kernel walks ip0 + // with kSearchStrength-driven stride growth) — the legacy + // SuffixStore-based matcher iterated every position and always + // hit short repeats, but the upstream zstd-shape kernel may skip over + // them when the step has grown large by the time it reaches the + // repeat region. The substance of this test is the + // reconstruction assertion below; `saw_triple` was a legacy + // tuning preference, not a correctness invariant. + let _ = saw_triple; + assert_eq!(rebuilt, data); +} + +#[test] +fn fast_levels_dispatch_per_level_hash_log_and_mls() { + // Level 1 — upstream zstd `{ 19, 13, 14, 1, 7, 0, ZSTD_fast }` row: + // window_log=19, hash_log=14, mls=7. + let f1 = resolve_level_params(CompressionLevel::Level(1), None) + .fast + .unwrap(); + assert_eq!(f1.hash_log, 14); + assert_eq!(f1.mls, 7); + assert_eq!(f1.step_size, 2); + + // Negative levels — upstream zstd row-0 ("base for negative"): + // hash_log=13, mls=7. The 32 KiB table is L1d-resident (every + // probe an L1 hit, vs an L2 access for a 64 KiB hash_log=14 + // table), and minMatch=7 drops short-distance 6-byte matches — + // upstream zstd parity on both ratio and throughput. + // step_size follows upstream zstd's formula: targetLength = -level, + // step_size = (-level) + 1, giving 2..8 for L-1..L-7. + for n in -7..=-1 { + let f = resolve_level_params(CompressionLevel::Level(n), None) + .fast + .unwrap(); + assert_eq!(f.hash_log, 13, "Level({n}) fast_hash_log"); + assert_eq!(f.mls, 7, "Level({n}) fast_mls"); + let expected_step = ((-n) as usize) + 1; + assert_eq!(f.step_size, expected_step, "Level({n}) fast_step_size"); + } + + // Fastest + Uncompressed keep hash_log=14 / mls=6 (their own + // tuning; not part of the negative-level upstream zstd ladder). + let pf = resolve_level_params(CompressionLevel::Fastest, None); + let ff = pf.fast.unwrap(); + assert_eq!( + (pf.window_log, ff.hash_log, ff.mls, ff.step_size), + (19, 14, 6, 2), + ); + // Uncompressed keeps window_log=17 (no history references, smaller + // decoder reservation); fast cParams same as negative-base row. + let pu = resolve_level_params(CompressionLevel::Uncompressed, None); + let fu = pu.fast.unwrap(); + assert_eq!( + (pu.window_log, fu.hash_log, fu.mls, fu.step_size), + (17, 14, 6, 2), + ); +} + +/// Exercise the actual driver wiring: for every Fast level, reset a +/// `MatchGeneratorDriver` and assert the inner `FastKernelMatcher` +/// observed the same `(hash_log, mls, step_size)` tuple that +/// `resolve_level_params` reports. Catches plumbing bugs — argument +/// reordering, stale step_size carried from a prior frame, +/// stuck-on-default values — that the parameter-only test above +/// would miss. +#[test] +fn fast_levels_driver_wiring_threads_cparams_into_inner_matcher() { + let mut driver = MatchGeneratorDriver::new(64 * 1024, 1); + + let fast_levels = [ + CompressionLevel::Level(1), + CompressionLevel::Fastest, + CompressionLevel::Uncompressed, + CompressionLevel::Level(-1), + CompressionLevel::Level(-2), + CompressionLevel::Level(-3), + CompressionLevel::Level(-4), + CompressionLevel::Level(-5), + CompressionLevel::Level(-6), + CompressionLevel::Level(-7), + ]; + + for &level in &fast_levels { + let p = resolve_level_params(level, None); + // Sanity: every level in the table above must resolve to a + // Fast-strategy row — otherwise this test isn't testing what + // it claims to test. + assert_eq!( + p.strategy_tag, + super::super::strategy::StrategyTag::Fast, + "{level:?} must resolve to Fast strategy", + ); + + // Bounce through a non-Fast strategy first so the next + // reset actually goes through the backend-switch path + // (`MatchGeneratorDriver::new` / `simple_mut` recreate the + // Fast variant via `FastKernelMatcher::with_params`). Without + // this hop the loop would only ever stay in `BackendTag::Simple` + // and exercise `FastKernelMatcher::reset` — leaving the + // `with_params` wiring untested on the production path. + // `Default` resolves to Dfast strategy (a non-Fast row), + // which is enough to force the swap. + crate::encoding::Matcher::reset(&mut driver, CompressionLevel::Default); + + // Drive the production reset path (same code paths exercised + // by FrameCompressor / StreamingEncoder). + crate::encoding::Matcher::reset(&mut driver, level); + + let f = p.fast.unwrap(); + let m = driver.simple_mut(); + assert_eq!( + m.hash_log(), + f.hash_log, + "{level:?}: inner matcher hash_log mismatch — argument swap?", + ); + assert_eq!( + m.mls(), + f.mls, + "{level:?}: inner matcher mls mismatch — argument swap?", + ); + assert_eq!( + m.step_size(), + f.step_size, + "{level:?}: inner matcher step_size mismatch — stale value carried from prior reset?", + ); + } +} + +/// Pins `hc.target_len` to the reference `cParams.targetLength` from +/// `clevels.h` table[0] (default — `srcSize > 256 KB`) across levels +/// 5-15. The reference's lazy outer loop treats `targetLength` as +/// `sufficient_len` — the "nice match" threshold that breaks the chain +/// walk as soon as a candidate reaches that length. +/// +/// Levels 13-15 run btlazy2 in the reference and the hash-chain Lazy +/// parser here, but the reference `targetLength` (32) is the same nice-match +/// threshold for both finders, so we mirror it directly. +/// +/// Asserts against the constant `clevels.h` table[0] `targetLength` column +/// (transcribed inline) — a pure-Rust in-tree test, no FFI dependency. +#[test] +fn lazy_band_target_len_matches_default_table() { + // table[0] (srcSize > 256 KB) targetLength, levels 5..=15: the lazy + // outer loop's nice-match (`sufficient_len`) threshold. + let expected: [(i32, usize); 11] = [ + (5, 2), + (6, 4), + (7, 8), + (8, 16), + (9, 16), + (10, 16), + (11, 16), + (12, 32), + (13, 32), + (14, 32), + (15, 32), + ]; + for (level, want) in expected { + let params = resolve_level_params(CompressionLevel::Level(level), None); + // L5 = greedy (Row backend → `row`); L6-15 = lazy (HashChain → `hc`). + let target_len = params + .hc + .map(|hc| hc.target_len) + .or_else(|| params.row.map(|row| row.target_len)) + .expect("lazy/greedy level carries hc or row config"); + assert_eq!(target_len, want, "L{level}: target_len must match table[0]"); + } +} + +/// Levels 13-15 mirror the reference btlazy2 window/hash/chain/search +/// budget from `clevels.h` table[0]: `search_depth == 1 << cParams.searchLog` +/// (16 / 32 / 64) plus `window_log` / `hash_log` / `chain_log` equal to the +/// reference `windowLog` / `hashLog` / `chainLog`. We run them on the +/// hash-chain Lazy parser rather than a binary-tree finder, so they do not +/// re-establish a strict ratio ladder above L12 on window-fitting inputs; +/// asserting the full row (not just `search_depth`) keeps the whole budget +/// aligned and guards every field against silent drift. +#[test] +fn upper_lazy_band_params_match_default_table() { + // table[0] (srcSize > 256 KB), levels 13..=15 (btlazy2 budget): + // (level, windowLog, hashLog, chainLog, search_depth = 1 << searchLog). + let expected: [(i32, u8, usize, usize, usize); 3] = [ + (13, 22, 22, 22, 1 << 4), + (14, 22, 23, 22, 1 << 5), + (15, 22, 23, 23, 1 << 6), + ]; + for (level, wlog, hlog, clog, sd) in expected { + let params = resolve_level_params(CompressionLevel::Level(level), None); + let hc = params.hc.unwrap(); + assert_eq!(hc.search_depth, sd, "L{level}: search_depth"); + assert_eq!(params.window_log, wlog, "L{level}: window_log"); + assert_eq!(hc.hash_log, hlog, "L{level}: hash_log"); + assert_eq!(hc.chain_log, clog, "L{level}: chain_log"); + } +} diff --git a/zstd/src/encoding/match_table/storage.rs b/zstd/src/encoding/match_table/storage.rs index 7aa46a3dc..29d7ca53e 100644 --- a/zstd/src/encoding/match_table/storage.rs +++ b/zstd/src/encoding/match_table/storage.rs @@ -92,94 +92,10 @@ pub(crate) fn check_stream_abs_headroom( } #[cfg(test)] -mod bt_pair_index_wrap_tests { - use super::MatchTable; - - /// `bt_pair_index_for_abs` switched from `+` to `wrapping_add` so the - /// release-mode overflow branch and the debug-mode panic stay off - /// the hot path on rare 32-bit streams where `abs_pos + index_shift` - /// overflows `usize`. This test forces that overflow and verifies - /// the returned BT slot still equals what the modulo-ring identity - /// promises: `(abs_pos + index_shift) mod 2^bt_log`, doubled - /// because the table stores pointer pairs. - #[test] - fn bt_pair_index_matches_modulo_ring_after_wraparound() { - let mut table = MatchTable::new(1 << 20); - // Small BT ring (`bt_log = chain_log - 1 = 3` → mask = 0b0111) so - // the modular identity is easy to read at a glance. - table.chain_log = 4; - // Use a wide `index_shift` so the addition wraps for many of - // the `abs_pos` values we probe below. - table.index_shift = usize::MAX - 5; - - let bt_mask = table.bt_mask(); - assert_eq!(bt_mask, 0b0111); - - for abs_pos in [0usize, 1, 4, 5, 6, 7, 8, 12, 17] { - let got = table.bt_pair_index_for_abs(abs_pos); - let expected = 2 * (abs_pos.wrapping_add(table.index_shift) & bt_mask); - assert_eq!( - got, expected, - "abs_pos={abs_pos}: wrapping_add ring slot must match the masked sum" - ); - } - - // Spot-check one value where overflow is certain (abs_pos > 5) - // and the ring index has a stable closed form. - // abs_pos=7, index_shift=usize::MAX-5: sum wraps to 1, mask -> 1, doubled -> 2. - assert_eq!(table.bt_pair_index_for_abs(7), 2); - // abs_pos=14: sum wraps to 8, masked -> 0, doubled -> 0. - assert_eq!(table.bt_pair_index_for_abs(14), 0); - } - - /// Sanity check the non-overflow path keeps the same identity so a - /// future refactor cannot regress the common case while leaving the - /// overflow case green. - #[test] - fn bt_pair_index_matches_modulo_ring_without_overflow() { - let mut table = MatchTable::new(1 << 20); - table.chain_log = 8; // bt_log = 7, mask = 0x7f - table.index_shift = 17; - - let bt_mask = table.bt_mask(); - for abs_pos in [0usize, 1, 16, 32, 64, 127, 128, 255, 1 << 20] { - let got = table.bt_pair_index_for_abs(abs_pos); - let expected = 2 * ((abs_pos + table.index_shift) & bt_mask); - assert_eq!(got, expected, "abs_pos={abs_pos}"); - } - } -} +mod bt_pair_index_wrap_tests; #[cfg(test)] -mod stream_abs_headroom_tests { - use super::{STREAM_ABS_HEADROOM, check_stream_abs_headroom}; - - #[test] - fn accepts_exactly_at_the_boundary() { - // `history_abs_start + window_size + data_len + STREAM_ABS_HEADROOM == usize::MAX`. - let history_abs_start = usize::MAX - STREAM_ABS_HEADROOM - 2; - check_stream_abs_headroom(history_abs_start, 1, 1); - } - - #[test] - fn accepts_well_below_the_boundary() { - check_stream_abs_headroom(0, 1 << 20, 1 << 20); - } - - #[test] - #[should_panic(expected = "STREAM_ABS_HEADROOM")] - fn rejects_one_byte_past_the_boundary() { - // One byte over: sum = usize::MAX + 1 → checked_add returns None. - let history_abs_start = usize::MAX - STREAM_ABS_HEADROOM - 1; - check_stream_abs_headroom(history_abs_start, 1, 1); - } - - #[test] - #[should_panic(expected = "STREAM_ABS_HEADROOM")] - fn rejects_history_abs_start_already_too_high() { - check_stream_abs_headroom(usize::MAX - 10, 0, 0); - } -} +mod stream_abs_headroom_tests; /// Knuth-style 3-byte hash multiplier. Upstream zstd parity: /// `ZSTD_HASH3PRIME` in `lib/compress/zstd_compress_internal.h`. Used @@ -1473,7 +1389,7 @@ impl MatchTable { target_abs: usize, ) -> usize { let search_depth = self.search_depth; - super::super::match_generator::bt_insert_step_no_rebase_body!( + super::super::hc::generator::bt_insert_step_no_rebase_body!( self, search_depth, abs_pos, @@ -1496,7 +1412,7 @@ impl MatchTable { target_abs: usize, ) -> usize { let search_depth = self.search_depth; - super::super::match_generator::bt_insert_step_no_rebase_body!( + super::super::hc::generator::bt_insert_step_no_rebase_body!( self, search_depth, abs_pos, @@ -1519,7 +1435,7 @@ impl MatchTable { target_abs: usize, ) -> usize { let search_depth = self.search_depth; - super::super::match_generator::bt_insert_step_no_rebase_body!( + super::super::hc::generator::bt_insert_step_no_rebase_body!( self, search_depth, abs_pos, @@ -1538,7 +1454,7 @@ impl MatchTable { target_abs: usize, ) -> usize { let search_depth = self.search_depth; - super::super::match_generator::bt_insert_step_no_rebase_body!( + super::super::hc::generator::bt_insert_step_no_rebase_body!( self, search_depth, abs_pos, @@ -1643,7 +1559,7 @@ impl MatchTable { out: &mut Vec, ) { let search_depth = self.search_depth; - super::super::match_generator::bt_insert_and_collect_matches_body!( + super::super::hc::generator::bt_insert_and_collect_matches_body!( self, search_depth, abs_pos, @@ -1673,7 +1589,7 @@ impl MatchTable { out: &mut Vec, ) { let search_depth = self.search_depth; - super::super::match_generator::bt_insert_and_collect_matches_body!( + super::super::hc::generator::bt_insert_and_collect_matches_body!( self, search_depth, abs_pos, @@ -1703,7 +1619,7 @@ impl MatchTable { out: &mut Vec, ) { let search_depth = self.search_depth; - super::super::match_generator::bt_insert_and_collect_matches_body!( + super::super::hc::generator::bt_insert_and_collect_matches_body!( self, search_depth, abs_pos, @@ -1729,7 +1645,7 @@ impl MatchTable { out: &mut Vec, ) { let search_depth = self.search_depth; - super::super::match_generator::bt_insert_and_collect_matches_body!( + super::super::hc::generator::bt_insert_and_collect_matches_body!( self, search_depth, abs_pos, @@ -2293,7 +2209,7 @@ impl MatchTable { current_abs_end: usize, min_match_len: usize, ) -> Option { - super::super::match_generator::hash3_candidate_body!( + super::super::hc::generator::hash3_candidate_body!( self, abs_pos, current_abs_end, @@ -2314,7 +2230,7 @@ impl MatchTable { current_abs_end: usize, min_match_len: usize, ) -> Option { - super::super::match_generator::hash3_candidate_body!( + super::super::hc::generator::hash3_candidate_body!( self, abs_pos, current_abs_end, @@ -2335,7 +2251,7 @@ impl MatchTable { current_abs_end: usize, min_match_len: usize, ) -> Option { - super::super::match_generator::hash3_candidate_body!( + super::super::hc::generator::hash3_candidate_body!( self, abs_pos, current_abs_end, @@ -2352,7 +2268,7 @@ impl MatchTable { current_abs_end: usize, min_match_len: usize, ) -> Option { - super::super::match_generator::hash3_candidate_body!( + super::super::hc::generator::hash3_candidate_body!( self, abs_pos, current_abs_end, @@ -2452,307 +2368,4 @@ impl MatchTable { } #[cfg(test)] -mod storage_tests { - //! Stage D coverage for `MatchTable` entry points that the - //! end-to-end compression path doesn't naturally hit on CI: - //! * `set_dictionary_limit_from_primed_bytes(0)` — the "clear" - //! branch is only entered when a dictionary frame is reset. - //! * BT-mode incompressible-block skip path on `skip_matching`. - //! * `replay_history_for_rebase_bt` — exercised only when the - //! BT cursor crosses the rolling-rebase threshold (~`u32::MAX`), - //! so we drive it directly here. - use alloc::vec; - - use super::*; - - fn new_table(window: usize) -> MatchTable { - let mut t = MatchTable::new(window); - // window_size is driven by `push_test_chunk` (= sum of live chunk - // lengths), so it is not preset here. - t.hash_log = 8; - t.chain_log = 8; - t.hash3_log = 0; - t - } - - #[test] - fn set_dictionary_limit_from_primed_bytes_zero_clears_limit() { - let mut t = new_table(64); - t.history_abs_start = 100; - t.dictionary_limit_abs = Some(123); - t.set_dictionary_limit_from_primed_bytes(0); - assert_eq!(t.dictionary_limit_abs, None); - } - - #[test] - fn set_dictionary_limit_from_primed_bytes_offsets_from_history_start() { - let mut t = new_table(64); - t.history_abs_start = 100; - t.set_dictionary_limit_from_primed_bytes(40); - assert_eq!(t.dictionary_limit_abs, Some(140)); - } - - #[test] - fn dms_cache_rebuilds_across_hc_bt_layout_switch() { - // A reused compressor that changes level across the HC↔BT boundary lands - // back in prime_dms_* with the SAME (region, mls, hash_log) but the OTHER - // builder. Without a layout discriminator in the cache key, the second - // prime would reuse the first builder's tables verbatim (HC single-link - // chain reinterpreted as a BT DUBT, or vice versa) — a silent corruption. - // The cache key must include the layout so the switch forces a rebuild. - let mut t = new_table(64); - // dms_hash_log clamps to [10, hash_log]; keep hash_log above the floor. - t.hash_log = 12; - // Match HC's fixed mls (HC_MIN_MATCH_LEN == 4) so the BT prime resolves - // the SAME (region, mls, hash_log) as the HC prime — then the LAYOUT field - // is the only differing key, which is exactly what this test guards. With - // search_mls != 4 the rebuild would happen via the mls mismatch and the - // test would pass even without the layout discriminator. - t.search_mls = 4; - t.push_test_chunk(vec![7u8; 48]); - t.ensure_tables(); - let region = 48; - - t.prime_dms_hc(region); - assert_eq!(t.dms.table().unwrap().layout, DmsDictLayout::Hc); - // HC chain has one `next` per dict position. - assert_eq!(t.dms.table().unwrap().chain_table.len(), region); - - // Same region/mls/hash_log, but the BT builder must NOT reuse the HC - // tables: it rebuilds to the BT layout (2 children per dict position). - t.prime_dms_bt(region); - assert_eq!(t.dms.table().unwrap().layout, DmsDictLayout::Bt); - assert_eq!(t.dms.table().unwrap().chain_table.len(), 2 * region); - - // And back to HC rebuilds again. - t.prime_dms_hc(region); - assert_eq!(t.dms.table().unwrap().layout, DmsDictLayout::Hc); - assert_eq!(t.dms.table().unwrap().chain_table.len(), region); - } - - #[test] - fn skip_matching_bt_incompressible_routes_through_sparse_block() { - let mut t = new_table(32); - t.push_test_chunk(vec![0u8; 32]); - t.ensure_tables(); - t.uses_bt = true; - t.is_btultra2 = false; - t.search_depth = 4; - let before_skip_until = t.skip_insert_until_abs; - t.skip_matching(Some(true)); - // BT + incompressible path must take the - // `bt_insert_sparse_incompressible_block` branch and advance - // `skip_insert_until_abs` to current_abs_end. - assert!(t.skip_insert_until_abs >= t.window_size); - assert!(t.skip_insert_until_abs > before_skip_until); - } - - #[test] - fn skip_matching_bt_dense_routes_through_bt_update_tree() { - let mut t = new_table(32); - t.push_test_chunk(vec![1u8; 32]); - t.ensure_tables(); - t.uses_bt = true; - t.is_btultra2 = false; - t.search_depth = 4; - // `incompressible_hint = None` → dense bt_update_tree_until path - t.skip_matching(None); - assert_eq!(t.skip_insert_until_abs, t.history_abs_start + t.window_size); - } - - #[test] - fn replay_history_for_rebase_bt_walks_inserted_prefix() { - let mut t = new_table(64); - // Construct a contiguous mirror long enough for the BT walker - // (`bt_insert_step_no_rebase` reads 8-byte prefixes). - t.history = vec![0u8; 64]; - for (i, slot) in t.history.iter_mut().enumerate() { - *slot = (i % 17) as u8; - } - t.history_start = 0; - t.history_abs_start = 0; - t.window_size = 64; - t.position_base = 0; - t.search_depth = 4; - t.uses_bt = true; - t.ensure_tables(); - // Replay the first 32 positions; the BT walker writes entries - // into the hash table (via `hash_table[hash] = stored`) so the - // ground-truth observation is "some hash slots are no longer - // HC_EMPTY". - assert!(t.hash_table.iter().all(|&v| v == HC_EMPTY)); - t.replay_history_for_rebase_bt(0, 32); - assert!( - t.hash_table.iter().any(|&v| v != HC_EMPTY), - "BT replay must populate hash table" - ); - } - - #[test] - fn begin_rebase_clears_index_tables_and_resets_base() { - let mut t = new_table(32); - t.hash_table = vec![7; 16]; - t.chain_table = vec![9; 16]; - t.hash3_table = vec![5; 16]; - t.history_abs_start = 50; - t.position_base = 0; - t.index_shift = 4; - t.allow_zero_relative_position = false; - - t.begin_rebase(); - - assert_eq!(t.position_base, 50); - assert_eq!(t.index_shift, 0); - assert!(t.allow_zero_relative_position); - assert!(t.hash_table.iter().all(|&v| v == HC_EMPTY)); - assert!(t.chain_table.iter().all(|&v| v == HC_EMPTY)); - assert!(t.hash3_table.iter().all(|&v| v == HC_EMPTY)); - } - - /// Regression: `rebase_positions_cold` must replay the HC3 side - /// table along with the main hash / chain replay. `begin_rebase` - /// zeroes `hash3_table`, so without an explicit refill every HC3 - /// probe before `abs_pos` returns "empty" until the next encode - /// position falls due. On long-running btultra2 streams that - /// silently changes match selection (the btultra2 cascade leans - /// heavily on HC3 short matches). - #[test] - fn rebase_positions_cold_rebuilds_hash3_for_btultra2() { - let mut t = new_table(64); - t.history = b"abcdef_abcdef_abcdef_abcdef_abcdef_abcdef".to_vec(); - t.history_start = 0; - t.history_abs_start = 0; - t.window_size = t.history.len(); - // `history` is set directly above; just record it as one live chunk. - t.chunk_lens.push_back(t.history.len()); - t.hash_log = 8; - t.chain_log = 8; - // btultra2-style: HC3 side table allocated. - t.hash3_log = 6; - t.is_btultra2 = true; - t.search_depth = 4; - t.ensure_tables(); - - // Pre-fill the HC3 table the way the encoder would by walking - // positions up to the would-be rebase point. - t.update_hash3_until(20); - assert!( - t.hash3_table.iter().any(|&v| v != HC_EMPTY), - "fixture precondition: hash3 must be non-empty before rebase" - ); - - t.rebase_positions_cold(20); - - assert!( - t.hash3_table.iter().any(|&v| v != HC_EMPTY), - "rebase must repopulate the HC3 side table — \ - btultra2 short-match selection depends on it" - ); - } - - #[test] - fn insert_positions_with_step_zero_step_is_noop() { - let mut t = new_table(32); - t.history = vec![0u8; 32]; - t.push_test_chunk(vec![0u8; 32]); - t.ensure_tables(); - let next_to_update3_before = t.next_to_update3; - // step=0 must early-return without touching anything. - t.insert_positions_with_step(0, 16, 0); - assert!(t.hash_table.iter().all(|&v| v == HC_EMPTY)); - assert_eq!(t.next_to_update3, next_to_update3_before); - } - - #[test] - fn insert_positions_with_step_saturating_step_breaks_loop() { - // step = usize::MAX so first iteration overflows - // `pos.saturating_add(step)` to usize::MAX, then the `next <= pos` - // guard breaks out of the loop after one insert. - let mut t = new_table(32); - t.history = vec![1u8; 32]; - t.push_test_chunk(vec![1u8; 32]); - t.ensure_tables(); - t.insert_positions_with_step(0, 16, usize::MAX); - // Exactly one position should have been inserted before the - // loop terminated — observe that only one slot is non-empty. - let non_empty = t.hash_table.iter().filter(|&&v| v != HC_EMPTY).count(); - assert!( - non_empty <= 1, - "step=usize::MAX must break after the first insert" - ); - } - - #[test] - fn apply_limited_update_after_long_match_hc_mode_is_noop() { - // HC mode (`uses_bt = false`) — function must early-return - // without mutating `skip_insert_until_abs`. - let mut t = new_table(32); - t.uses_bt = false; - t.skip_insert_until_abs = 100; - t.apply_limited_update_after_long_match(1000); - assert_eq!( - t.skip_insert_until_abs, 100, - "HC mode must not adjust skip cursor" - ); - } - - #[test] - fn apply_limited_update_after_long_match_bt_mode_caps_gap_at_384() { - // BT mode with gap > 384 → cap the skip cursor so future - // `bt_update_tree_until` doesn't walk an unbounded prefix. - let mut t = new_table(32); - t.uses_bt = true; - t.skip_insert_until_abs = 0; - // current_abs_start = 1000 → gap = 1000 → cap subtracts - // (gap - 384).min(192) = 192, so result is 1000 - 192 = 808. - t.apply_limited_update_after_long_match(1000); - assert_eq!(t.skip_insert_until_abs, 808); - } - - #[test] - fn apply_limited_update_after_long_match_small_gap_is_noop() { - let mut t = new_table(32); - t.uses_bt = true; - t.skip_insert_until_abs = 800; - // gap = 200 < 384 → no change. - t.apply_limited_update_after_long_match(1000); - assert_eq!(t.skip_insert_until_abs, 800); - } - - #[test] - fn emit_optimal_plan_empty_plan_emits_full_literals() { - let mut t = new_table(8); - t.push_test_chunk(b"abcdefgh".to_vec()); - let mut emitted: Vec = Vec::new(); - t.emit_optimal_plan(8, &[], &mut |seq| { - if let Sequence::Literals { literals } = seq { - emitted.extend_from_slice(literals); - } - }); - assert_eq!(emitted, b"abcdefgh"); - } - - #[test] - fn emit_optimal_plan_skips_oversized_plan_item_and_emits_trailing_literals() { - let mut t = new_table(8); - t.push_test_chunk(b"abcdefgh".to_vec()); - // Plan item asks for `start + match_len > current_len` → skip. - // The function must still emit the trailing literals at the end. - let plan = [HcOptimalSequence { - offset: 1, - lit_len: 4, - match_len: 99, // overflows the 8-byte window → continue - }]; - let mut triples = 0usize; - let mut trailing: Vec = Vec::new(); - t.emit_optimal_plan(8, &plan, &mut |seq| match seq { - Sequence::Triple { .. } => triples += 1, - Sequence::Literals { literals } => trailing.extend_from_slice(literals), - }); - assert_eq!(triples, 0, "oversized plan item must be skipped"); - assert_eq!( - trailing, b"abcdefgh", - "trailing-literals path must emit the full window when plan skipped everything" - ); - } -} +mod storage_tests; diff --git a/zstd/src/encoding/match_table/storage/bt_pair_index_wrap_tests.rs b/zstd/src/encoding/match_table/storage/bt_pair_index_wrap_tests.rs new file mode 100644 index 000000000..6002fa361 --- /dev/null +++ b/zstd/src/encoding/match_table/storage/bt_pair_index_wrap_tests.rs @@ -0,0 +1,55 @@ +use super::MatchTable; + +/// `bt_pair_index_for_abs` switched from `+` to `wrapping_add` so the +/// release-mode overflow branch and the debug-mode panic stay off +/// the hot path on rare 32-bit streams where `abs_pos + index_shift` +/// overflows `usize`. This test forces that overflow and verifies +/// the returned BT slot still equals what the modulo-ring identity +/// promises: `(abs_pos + index_shift) mod 2^bt_log`, doubled +/// because the table stores pointer pairs. +#[test] +fn bt_pair_index_matches_modulo_ring_after_wraparound() { + let mut table = MatchTable::new(1 << 20); + // Small BT ring (`bt_log = chain_log - 1 = 3` → mask = 0b0111) so + // the modular identity is easy to read at a glance. + table.chain_log = 4; + // Use a wide `index_shift` so the addition wraps for many of + // the `abs_pos` values we probe below. + table.index_shift = usize::MAX - 5; + + let bt_mask = table.bt_mask(); + assert_eq!(bt_mask, 0b0111); + + for abs_pos in [0usize, 1, 4, 5, 6, 7, 8, 12, 17] { + let got = table.bt_pair_index_for_abs(abs_pos); + let expected = 2 * (abs_pos.wrapping_add(table.index_shift) & bt_mask); + assert_eq!( + got, expected, + "abs_pos={abs_pos}: wrapping_add ring slot must match the masked sum" + ); + } + + // Spot-check one value where overflow is certain (abs_pos > 5) + // and the ring index has a stable closed form. + // abs_pos=7, index_shift=usize::MAX-5: sum wraps to 1, mask -> 1, doubled -> 2. + assert_eq!(table.bt_pair_index_for_abs(7), 2); + // abs_pos=14: sum wraps to 8, masked -> 0, doubled -> 0. + assert_eq!(table.bt_pair_index_for_abs(14), 0); +} + +/// Sanity check the non-overflow path keeps the same identity so a +/// future refactor cannot regress the common case while leaving the +/// overflow case green. +#[test] +fn bt_pair_index_matches_modulo_ring_without_overflow() { + let mut table = MatchTable::new(1 << 20); + table.chain_log = 8; // bt_log = 7, mask = 0x7f + table.index_shift = 17; + + let bt_mask = table.bt_mask(); + for abs_pos in [0usize, 1, 16, 32, 64, 127, 128, 255, 1 << 20] { + let got = table.bt_pair_index_for_abs(abs_pos); + let expected = 2 * ((abs_pos + table.index_shift) & bt_mask); + assert_eq!(got, expected, "abs_pos={abs_pos}"); + } +} diff --git a/zstd/src/encoding/match_table/storage/storage_tests.rs b/zstd/src/encoding/match_table/storage/storage_tests.rs new file mode 100644 index 000000000..14434d894 --- /dev/null +++ b/zstd/src/encoding/match_table/storage/storage_tests.rs @@ -0,0 +1,302 @@ +//! Stage D coverage for `MatchTable` entry points that the +//! end-to-end compression path doesn't naturally hit on CI: +//! * `set_dictionary_limit_from_primed_bytes(0)` — the "clear" +//! branch is only entered when a dictionary frame is reset. +//! * BT-mode incompressible-block skip path on `skip_matching`. +//! * `replay_history_for_rebase_bt` — exercised only when the +//! BT cursor crosses the rolling-rebase threshold (~`u32::MAX`), +//! so we drive it directly here. +use alloc::vec; + +use super::*; + +fn new_table(window: usize) -> MatchTable { + let mut t = MatchTable::new(window); + // window_size is driven by `push_test_chunk` (= sum of live chunk + // lengths), so it is not preset here. + t.hash_log = 8; + t.chain_log = 8; + t.hash3_log = 0; + t +} + +#[test] +fn set_dictionary_limit_from_primed_bytes_zero_clears_limit() { + let mut t = new_table(64); + t.history_abs_start = 100; + t.dictionary_limit_abs = Some(123); + t.set_dictionary_limit_from_primed_bytes(0); + assert_eq!(t.dictionary_limit_abs, None); +} + +#[test] +fn set_dictionary_limit_from_primed_bytes_offsets_from_history_start() { + let mut t = new_table(64); + t.history_abs_start = 100; + t.set_dictionary_limit_from_primed_bytes(40); + assert_eq!(t.dictionary_limit_abs, Some(140)); +} + +#[test] +fn dms_cache_rebuilds_across_hc_bt_layout_switch() { + // A reused compressor that changes level across the HC↔BT boundary lands + // back in prime_dms_* with the SAME (region, mls, hash_log) but the OTHER + // builder. Without a layout discriminator in the cache key, the second + // prime would reuse the first builder's tables verbatim (HC single-link + // chain reinterpreted as a BT DUBT, or vice versa) — a silent corruption. + // The cache key must include the layout so the switch forces a rebuild. + let mut t = new_table(64); + // dms_hash_log clamps to [10, hash_log]; keep hash_log above the floor. + t.hash_log = 12; + // Match HC's fixed mls (HC_MIN_MATCH_LEN == 4) so the BT prime resolves + // the SAME (region, mls, hash_log) as the HC prime — then the LAYOUT field + // is the only differing key, which is exactly what this test guards. With + // search_mls != 4 the rebuild would happen via the mls mismatch and the + // test would pass even without the layout discriminator. + t.search_mls = 4; + t.push_test_chunk(vec![7u8; 48]); + t.ensure_tables(); + let region = 48; + + t.prime_dms_hc(region); + assert_eq!(t.dms.table().unwrap().layout, DmsDictLayout::Hc); + // HC chain has one `next` per dict position. + assert_eq!(t.dms.table().unwrap().chain_table.len(), region); + + // Same region/mls/hash_log, but the BT builder must NOT reuse the HC + // tables: it rebuilds to the BT layout (2 children per dict position). + t.prime_dms_bt(region); + assert_eq!(t.dms.table().unwrap().layout, DmsDictLayout::Bt); + assert_eq!(t.dms.table().unwrap().chain_table.len(), 2 * region); + + // And back to HC rebuilds again. + t.prime_dms_hc(region); + assert_eq!(t.dms.table().unwrap().layout, DmsDictLayout::Hc); + assert_eq!(t.dms.table().unwrap().chain_table.len(), region); +} + +#[test] +fn skip_matching_bt_incompressible_routes_through_sparse_block() { + let mut t = new_table(32); + t.push_test_chunk(vec![0u8; 32]); + t.ensure_tables(); + t.uses_bt = true; + t.is_btultra2 = false; + t.search_depth = 4; + let before_skip_until = t.skip_insert_until_abs; + t.skip_matching(Some(true)); + // BT + incompressible path must take the + // `bt_insert_sparse_incompressible_block` branch and advance + // `skip_insert_until_abs` to current_abs_end. + assert!(t.skip_insert_until_abs >= t.window_size); + assert!(t.skip_insert_until_abs > before_skip_until); +} + +#[test] +fn skip_matching_bt_dense_routes_through_bt_update_tree() { + let mut t = new_table(32); + t.push_test_chunk(vec![1u8; 32]); + t.ensure_tables(); + t.uses_bt = true; + t.is_btultra2 = false; + t.search_depth = 4; + // `incompressible_hint = None` → dense bt_update_tree_until path + t.skip_matching(None); + assert_eq!(t.skip_insert_until_abs, t.history_abs_start + t.window_size); +} + +#[test] +fn replay_history_for_rebase_bt_walks_inserted_prefix() { + let mut t = new_table(64); + // Construct a contiguous mirror long enough for the BT walker + // (`bt_insert_step_no_rebase` reads 8-byte prefixes). + t.history = vec![0u8; 64]; + for (i, slot) in t.history.iter_mut().enumerate() { + *slot = (i % 17) as u8; + } + t.history_start = 0; + t.history_abs_start = 0; + t.window_size = 64; + t.position_base = 0; + t.search_depth = 4; + t.uses_bt = true; + t.ensure_tables(); + // Replay the first 32 positions; the BT walker writes entries + // into the hash table (via `hash_table[hash] = stored`) so the + // ground-truth observation is "some hash slots are no longer + // HC_EMPTY". + assert!(t.hash_table.iter().all(|&v| v == HC_EMPTY)); + t.replay_history_for_rebase_bt(0, 32); + assert!( + t.hash_table.iter().any(|&v| v != HC_EMPTY), + "BT replay must populate hash table" + ); +} + +#[test] +fn begin_rebase_clears_index_tables_and_resets_base() { + let mut t = new_table(32); + t.hash_table = vec![7; 16]; + t.chain_table = vec![9; 16]; + t.hash3_table = vec![5; 16]; + t.history_abs_start = 50; + t.position_base = 0; + t.index_shift = 4; + t.allow_zero_relative_position = false; + + t.begin_rebase(); + + assert_eq!(t.position_base, 50); + assert_eq!(t.index_shift, 0); + assert!(t.allow_zero_relative_position); + assert!(t.hash_table.iter().all(|&v| v == HC_EMPTY)); + assert!(t.chain_table.iter().all(|&v| v == HC_EMPTY)); + assert!(t.hash3_table.iter().all(|&v| v == HC_EMPTY)); +} + +/// Regression: `rebase_positions_cold` must replay the HC3 side +/// table along with the main hash / chain replay. `begin_rebase` +/// zeroes `hash3_table`, so without an explicit refill every HC3 +/// probe before `abs_pos` returns "empty" until the next encode +/// position falls due. On long-running btultra2 streams that +/// silently changes match selection (the btultra2 cascade leans +/// heavily on HC3 short matches). +#[test] +fn rebase_positions_cold_rebuilds_hash3_for_btultra2() { + let mut t = new_table(64); + t.history = b"abcdef_abcdef_abcdef_abcdef_abcdef_abcdef".to_vec(); + t.history_start = 0; + t.history_abs_start = 0; + t.window_size = t.history.len(); + // `history` is set directly above; just record it as one live chunk. + t.chunk_lens.push_back(t.history.len()); + t.hash_log = 8; + t.chain_log = 8; + // btultra2-style: HC3 side table allocated. + t.hash3_log = 6; + t.is_btultra2 = true; + t.search_depth = 4; + t.ensure_tables(); + + // Pre-fill the HC3 table the way the encoder would by walking + // positions up to the would-be rebase point. + t.update_hash3_until(20); + assert!( + t.hash3_table.iter().any(|&v| v != HC_EMPTY), + "fixture precondition: hash3 must be non-empty before rebase" + ); + + t.rebase_positions_cold(20); + + assert!( + t.hash3_table.iter().any(|&v| v != HC_EMPTY), + "rebase must repopulate the HC3 side table — \ + btultra2 short-match selection depends on it" + ); +} + +#[test] +fn insert_positions_with_step_zero_step_is_noop() { + let mut t = new_table(32); + t.history = vec![0u8; 32]; + t.push_test_chunk(vec![0u8; 32]); + t.ensure_tables(); + let next_to_update3_before = t.next_to_update3; + // step=0 must early-return without touching anything. + t.insert_positions_with_step(0, 16, 0); + assert!(t.hash_table.iter().all(|&v| v == HC_EMPTY)); + assert_eq!(t.next_to_update3, next_to_update3_before); +} + +#[test] +fn insert_positions_with_step_saturating_step_breaks_loop() { + // step = usize::MAX so first iteration overflows + // `pos.saturating_add(step)` to usize::MAX, then the `next <= pos` + // guard breaks out of the loop after one insert. + let mut t = new_table(32); + t.history = vec![1u8; 32]; + t.push_test_chunk(vec![1u8; 32]); + t.ensure_tables(); + t.insert_positions_with_step(0, 16, usize::MAX); + // Exactly one position should have been inserted before the + // loop terminated — observe that only one slot is non-empty. + let non_empty = t.hash_table.iter().filter(|&&v| v != HC_EMPTY).count(); + assert!( + non_empty <= 1, + "step=usize::MAX must break after the first insert" + ); +} + +#[test] +fn apply_limited_update_after_long_match_hc_mode_is_noop() { + // HC mode (`uses_bt = false`) — function must early-return + // without mutating `skip_insert_until_abs`. + let mut t = new_table(32); + t.uses_bt = false; + t.skip_insert_until_abs = 100; + t.apply_limited_update_after_long_match(1000); + assert_eq!( + t.skip_insert_until_abs, 100, + "HC mode must not adjust skip cursor" + ); +} + +#[test] +fn apply_limited_update_after_long_match_bt_mode_caps_gap_at_384() { + // BT mode with gap > 384 → cap the skip cursor so future + // `bt_update_tree_until` doesn't walk an unbounded prefix. + let mut t = new_table(32); + t.uses_bt = true; + t.skip_insert_until_abs = 0; + // current_abs_start = 1000 → gap = 1000 → cap subtracts + // (gap - 384).min(192) = 192, so result is 1000 - 192 = 808. + t.apply_limited_update_after_long_match(1000); + assert_eq!(t.skip_insert_until_abs, 808); +} + +#[test] +fn apply_limited_update_after_long_match_small_gap_is_noop() { + let mut t = new_table(32); + t.uses_bt = true; + t.skip_insert_until_abs = 800; + // gap = 200 < 384 → no change. + t.apply_limited_update_after_long_match(1000); + assert_eq!(t.skip_insert_until_abs, 800); +} + +#[test] +fn emit_optimal_plan_empty_plan_emits_full_literals() { + let mut t = new_table(8); + t.push_test_chunk(b"abcdefgh".to_vec()); + let mut emitted: Vec = Vec::new(); + t.emit_optimal_plan(8, &[], &mut |seq| { + if let Sequence::Literals { literals } = seq { + emitted.extend_from_slice(literals); + } + }); + assert_eq!(emitted, b"abcdefgh"); +} + +#[test] +fn emit_optimal_plan_skips_oversized_plan_item_and_emits_trailing_literals() { + let mut t = new_table(8); + t.push_test_chunk(b"abcdefgh".to_vec()); + // Plan item asks for `start + match_len > current_len` → skip. + // The function must still emit the trailing literals at the end. + let plan = [HcOptimalSequence { + offset: 1, + lit_len: 4, + match_len: 99, // overflows the 8-byte window → continue + }]; + let mut triples = 0usize; + let mut trailing: Vec = Vec::new(); + t.emit_optimal_plan(8, &plan, &mut |seq| match seq { + Sequence::Triple { .. } => triples += 1, + Sequence::Literals { literals } => trailing.extend_from_slice(literals), + }); + assert_eq!(triples, 0, "oversized plan item must be skipped"); + assert_eq!( + trailing, b"abcdefgh", + "trailing-literals path must emit the full window when plan skipped everything" + ); +} diff --git a/zstd/src/encoding/match_table/storage/stream_abs_headroom_tests.rs b/zstd/src/encoding/match_table/storage/stream_abs_headroom_tests.rs new file mode 100644 index 000000000..1c4369673 --- /dev/null +++ b/zstd/src/encoding/match_table/storage/stream_abs_headroom_tests.rs @@ -0,0 +1,27 @@ +use super::{STREAM_ABS_HEADROOM, check_stream_abs_headroom}; + +#[test] +fn accepts_exactly_at_the_boundary() { + // `history_abs_start + window_size + data_len + STREAM_ABS_HEADROOM == usize::MAX`. + let history_abs_start = usize::MAX - STREAM_ABS_HEADROOM - 2; + check_stream_abs_headroom(history_abs_start, 1, 1); +} + +#[test] +fn accepts_well_below_the_boundary() { + check_stream_abs_headroom(0, 1 << 20, 1 << 20); +} + +#[test] +#[should_panic(expected = "STREAM_ABS_HEADROOM")] +fn rejects_one_byte_past_the_boundary() { + // One byte over: sum = usize::MAX + 1 → checked_add returns None. + let history_abs_start = usize::MAX - STREAM_ABS_HEADROOM - 1; + check_stream_abs_headroom(history_abs_start, 1, 1); +} + +#[test] +#[should_panic(expected = "STREAM_ABS_HEADROOM")] +fn rejects_history_abs_start_already_too_high() { + check_stream_abs_headroom(usize::MAX - 10, 0, 0); +} diff --git a/zstd/src/encoding/mod.rs b/zstd/src/encoding/mod.rs index e3389e333..97fea593e 100644 --- a/zstd/src/encoding/mod.rs +++ b/zstd/src/encoding/mod.rs @@ -97,9 +97,10 @@ mod streaming_encoder; pub use frame_compressor::{EncoderDictionary, FrameCompressor}; #[cfg(feature = "lsm")] pub use frame_emit_info::{BlockType, FrameBlock, FrameEmitInfo}; -pub use match_generator::{ - MatchGeneratorDriver, estimated_bt_strategy_extra_bytes, estimated_compression_workspace_bytes, +pub use levels::config::{ + estimated_bt_strategy_extra_bytes, estimated_compression_workspace_bytes, }; +pub use match_generator::MatchGeneratorDriver; pub use parameters::{ Bounds, CParameter, CompressionParameters, CompressionParametersBuilder, ParameterError, Strategy, @@ -504,35 +505,4 @@ pub enum Sequence<'data> { } #[cfg(test)] -mod compress_bound_tests { - use super::{CompressionLevel, compress_bound, compress_slice_to_vec}; - - #[test] - fn matches_upstream_formula_below_threshold() { - // src_size + (src_size >> 8) + ((128 KiB - src_size) >> 11). - assert_eq!(compress_bound(0), 64); - assert_eq!(compress_bound(4096), 4096 + 16 + 62); - } - - #[test] - fn drops_margin_at_and_above_threshold() { - let lower = 128 * 1024; - assert_eq!(compress_bound(lower), lower + (lower >> 8)); - assert_eq!(compress_bound(lower + 1), (lower + 1) + ((lower + 1) >> 8)); - } - - #[test] - fn saturates_instead_of_wrapping() { - // No allocation this large can exist; the ceiling is the right sentinel. - assert_eq!(compress_bound(usize::MAX), usize::MAX); - } - - #[test] - fn always_fits_real_compressed_output() { - for len in [0usize, 1, 100, 4096, 200_000] { - let data = alloc::vec![7u8; len]; - let out = compress_slice_to_vec(&data, CompressionLevel::Default); - assert!(out.len() <= compress_bound(len), "len={len}"); - } - } -} +mod compress_bound_tests; diff --git a/zstd/src/encoding/parameters.rs b/zstd/src/encoding/parameters.rs index ce23819e7..c050a611e 100644 --- a/zstd/src/encoding/parameters.rs +++ b/zstd/src/encoding/parameters.rs @@ -517,153 +517,4 @@ fn check(parameter: CParameter, value: Option) -> Result<(), ParameterError } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn strategy_ordinals_round_trip() { - for ordinal in 1..=9 { - let s = Strategy::from_ordinal(ordinal).expect("valid ordinal"); - assert_eq!(s.ordinal(), ordinal); - } - assert_eq!(Strategy::from_ordinal(0), None); - assert_eq!(Strategy::from_ordinal(10), None); - } - - #[test] - fn builder_default_overrides_nothing() { - let p = CompressionParameters::builder(CompressionLevel::Level(7)) - .build() - .unwrap(); - assert!(p.overrides().is_empty()); - assert_eq!(p.level(), CompressionLevel::Level(7)); - assert!(!p.long_distance_matching_enabled()); - } - - #[test] - fn builder_records_each_knob() { - let p = CompressionParameters::builder(CompressionLevel::Level(19)) - .window_log(22) - .hash_log(23) - .chain_log(24) - .search_log(7) - .min_match(4) - .target_length(256) - .strategy(Strategy::Btultra2) - .build() - .unwrap(); - let o = p.overrides(); - assert_eq!(o.window_log, Some(22)); - assert_eq!(o.hash_log, Some(23)); - assert_eq!(o.chain_log, Some(24)); - assert_eq!(o.search_log, Some(7)); - assert_eq!(o.min_match, Some(4)); - assert_eq!(o.target_length, Some(256)); - assert_eq!(o.strategy, Some(Strategy::Btultra2)); - assert!(!o.is_empty()); - } - - #[test] - fn enable_ldm_sets_override_block() { - let p = CompressionParameters::builder(CompressionLevel::Level(19)) - .enable_long_distance_matching(true) - .build() - .unwrap(); - assert!(p.long_distance_matching_enabled()); - assert_eq!(p.overrides().ldm, Some(LdmOverride::default())); - } - - #[test] - fn ldm_knob_implies_enable() { - let p = CompressionParameters::builder(CompressionLevel::Level(19)) - .ldm_hash_log(24) - .ldm_min_match(64) - .ldm_bucket_size_log(4) - .ldm_hash_rate_log(7) - .build() - .unwrap(); - assert!(p.long_distance_matching_enabled()); - let ldm = p.overrides().ldm.unwrap(); - assert_eq!(ldm.hash_log, Some(24)); - assert_eq!(ldm.min_match, Some(64)); - assert_eq!(ldm.bucket_size_log, Some(4)); - assert_eq!(ldm.hash_rate_log, Some(7)); - } - - #[test] - fn out_of_bounds_window_log_rejected() { - let err = CompressionParameters::builder(CompressionLevel::Default) - .window_log(31) - .build() - .unwrap_err(); - match err { - ParameterError::OutOfBounds { - parameter, value, .. - } => { - assert_eq!(parameter, CParameter::WindowLog); - assert_eq!(value, 31); - } - } - } - - #[test] - fn out_of_bounds_min_match_rejected() { - let err = CompressionParameters::builder(CompressionLevel::Default) - .min_match(2) - .build() - .unwrap_err(); - assert!(matches!( - err, - ParameterError::OutOfBounds { - parameter: CParameter::MinMatch, - .. - } - )); - } - - #[test] - fn ldm_bounds_only_checked_when_enabled() { - // An out-of-range LDM knob is only rejected when LDM is on. A - // builder that never enables LDM ignores the (unreachable) - // values entirely. - let err = CompressionParameters::builder(CompressionLevel::Default) - .ldm_bucket_size_log(9) - .build() - .unwrap_err(); - assert!(matches!( - err, - ParameterError::OutOfBounds { - parameter: CParameter::LdmBucketSizeLog, - .. - } - )); - } - - #[test] - fn bounds_match_c_reference() { - assert_eq!( - CParameter::WindowLog.bounds(), - Bounds { - lower_bound: 10, - upper_bound: 30 - } - ); - assert_eq!( - CParameter::Strategy.bounds(), - Bounds { - lower_bound: 1, - upper_bound: 9 - } - ); - assert_eq!( - CParameter::TargetLength.bounds(), - Bounds { - lower_bound: 0, - upper_bound: 131_072 - } - ); - assert!(CParameter::MinMatch.bounds().contains(3)); - assert!(CParameter::MinMatch.bounds().contains(7)); - assert!(!CParameter::MinMatch.bounds().contains(8)); - } -} +mod tests; diff --git a/zstd/src/encoding/parameters/tests.rs b/zstd/src/encoding/parameters/tests.rs new file mode 100644 index 000000000..99a867660 --- /dev/null +++ b/zstd/src/encoding/parameters/tests.rs @@ -0,0 +1,148 @@ +use super::*; + +#[test] +fn strategy_ordinals_round_trip() { + for ordinal in 1..=9 { + let s = Strategy::from_ordinal(ordinal).expect("valid ordinal"); + assert_eq!(s.ordinal(), ordinal); + } + assert_eq!(Strategy::from_ordinal(0), None); + assert_eq!(Strategy::from_ordinal(10), None); +} + +#[test] +fn builder_default_overrides_nothing() { + let p = CompressionParameters::builder(CompressionLevel::Level(7)) + .build() + .unwrap(); + assert!(p.overrides().is_empty()); + assert_eq!(p.level(), CompressionLevel::Level(7)); + assert!(!p.long_distance_matching_enabled()); +} + +#[test] +fn builder_records_each_knob() { + let p = CompressionParameters::builder(CompressionLevel::Level(19)) + .window_log(22) + .hash_log(23) + .chain_log(24) + .search_log(7) + .min_match(4) + .target_length(256) + .strategy(Strategy::Btultra2) + .build() + .unwrap(); + let o = p.overrides(); + assert_eq!(o.window_log, Some(22)); + assert_eq!(o.hash_log, Some(23)); + assert_eq!(o.chain_log, Some(24)); + assert_eq!(o.search_log, Some(7)); + assert_eq!(o.min_match, Some(4)); + assert_eq!(o.target_length, Some(256)); + assert_eq!(o.strategy, Some(Strategy::Btultra2)); + assert!(!o.is_empty()); +} + +#[test] +fn enable_ldm_sets_override_block() { + let p = CompressionParameters::builder(CompressionLevel::Level(19)) + .enable_long_distance_matching(true) + .build() + .unwrap(); + assert!(p.long_distance_matching_enabled()); + assert_eq!(p.overrides().ldm, Some(LdmOverride::default())); +} + +#[test] +fn ldm_knob_implies_enable() { + let p = CompressionParameters::builder(CompressionLevel::Level(19)) + .ldm_hash_log(24) + .ldm_min_match(64) + .ldm_bucket_size_log(4) + .ldm_hash_rate_log(7) + .build() + .unwrap(); + assert!(p.long_distance_matching_enabled()); + let ldm = p.overrides().ldm.unwrap(); + assert_eq!(ldm.hash_log, Some(24)); + assert_eq!(ldm.min_match, Some(64)); + assert_eq!(ldm.bucket_size_log, Some(4)); + assert_eq!(ldm.hash_rate_log, Some(7)); +} + +#[test] +fn out_of_bounds_window_log_rejected() { + let err = CompressionParameters::builder(CompressionLevel::Default) + .window_log(31) + .build() + .unwrap_err(); + match err { + ParameterError::OutOfBounds { + parameter, value, .. + } => { + assert_eq!(parameter, CParameter::WindowLog); + assert_eq!(value, 31); + } + } +} + +#[test] +fn out_of_bounds_min_match_rejected() { + let err = CompressionParameters::builder(CompressionLevel::Default) + .min_match(2) + .build() + .unwrap_err(); + assert!(matches!( + err, + ParameterError::OutOfBounds { + parameter: CParameter::MinMatch, + .. + } + )); +} + +#[test] +fn ldm_bounds_only_checked_when_enabled() { + // An out-of-range LDM knob is only rejected when LDM is on. A + // builder that never enables LDM ignores the (unreachable) + // values entirely. + let err = CompressionParameters::builder(CompressionLevel::Default) + .ldm_bucket_size_log(9) + .build() + .unwrap_err(); + assert!(matches!( + err, + ParameterError::OutOfBounds { + parameter: CParameter::LdmBucketSizeLog, + .. + } + )); +} + +#[test] +fn bounds_match_c_reference() { + assert_eq!( + CParameter::WindowLog.bounds(), + Bounds { + lower_bound: 10, + upper_bound: 30 + } + ); + assert_eq!( + CParameter::Strategy.bounds(), + Bounds { + lower_bound: 1, + upper_bound: 9 + } + ); + assert_eq!( + CParameter::TargetLength.bounds(), + Bounds { + lower_bound: 0, + upper_bound: 131_072 + } + ); + assert!(CParameter::MinMatch.bounds().contains(3)); + assert!(CParameter::MinMatch.bounds().contains(7)); + assert!(!CParameter::MinMatch.bounds().contains(8)); +} diff --git a/zstd/src/encoding/row/mod.rs b/zstd/src/encoding/row/mod.rs index 3f21a3110..e1456e98e 100644 --- a/zstd/src/encoding/row/mod.rs +++ b/zstd/src/encoding/row/mod.rs @@ -18,9 +18,10 @@ use core::convert::TryInto; use super::Sequence; use super::blocks::encode_offset_with_history; use super::dict_attach::DictAttach; +use super::levels::config::RowConfig; use super::match_generator::{ ROW_EMPTY_SLOT, ROW_HASH_BITS, ROW_HASH_KEY_LEN, ROW_LOG, ROW_MIN_MATCH_LEN, ROW_SEARCH_DEPTH, - ROW_TAG_BITS, ROW_TARGET_LEN, RowConfig, + ROW_TAG_BITS, ROW_TARGET_LEN, }; /// Immutable row-hash dictionary index (upstream zstd `ZSTD_RowFindBestMatch`'s @@ -2523,75 +2524,7 @@ impl RowMatchGenerator { feature = "std", any(target_arch = "x86", target_arch = "x86_64") ))] -mod tag_mask_tests { - use super::{row_tag_match_mask_avx2, row_tag_match_mask_scalar, row_tag_match_mask_sse2}; - - /// Deterministic LCG fill so the test exercises a realistic spread of - /// matching / non-matching tag bytes without a RNG dependency. - fn fill(buf: &mut [u8], mut state: u64) { - for b in buf.iter_mut() { - state = state - .wrapping_mul(6364136223846793005) - .wrapping_add(1442695040888963407); - *b = (state >> 56) as u8; - } - } - - /// The SIMD kernels must produce byte-identical masks to the scalar - /// reference for every supported row width (16 / 32 / 64) and tag, or - /// the match selection diverges and the compressed output changes. - #[test] - fn simd_tag_mask_matches_scalar() { - for &width in &[16usize, 32, 64] { - let mut tags = alloc::vec![0u8; width]; - for seed in 0..32u64 { - fill(&mut tags, 0x9e3779b97f4a7c15u64.wrapping_add(seed)); - // Cover both a tag that occurs in the row and arbitrary tags. - for tag in [tags[seed as usize % width], 0u8, 0xFF, (seed as u8)] { - let expected = row_tag_match_mask_scalar(&tags, tag); - if std::arch::is_x86_feature_detected!("sse2") { - let got = unsafe { row_tag_match_mask_sse2(&tags, tag) }; - assert_eq!(got, expected, "sse2 width={width} tag={tag}"); - } - if std::arch::is_x86_feature_detected!("avx2") { - let got = unsafe { row_tag_match_mask_avx2(&tags, tag) }; - assert_eq!(got, expected, "avx2 width={width} tag={tag}"); - } - } - } - } - } -} +mod tag_mask_tests; #[cfg(all(test, target_arch = "aarch64", target_endian = "little"))] -mod neon_tag_mask_tests { - use super::{row_tag_match_mask_neon, row_tag_match_mask_scalar}; - - fn fill(buf: &mut [u8], mut state: u64) { - for b in buf.iter_mut() { - state = state - .wrapping_mul(6364136223846793005) - .wrapping_add(1442695040888963407); - *b = (state >> 56) as u8; - } - } - - /// The NEON kernel must produce byte-identical masks to the scalar - /// reference for every supported row width (16 / 32 / 64) and tag, so - /// match selection (and the compressed output) is unchanged on aarch64. - #[test] - fn neon_tag_mask_matches_scalar() { - for &width in &[16usize, 32, 64] { - let mut tags = alloc::vec![0u8; width]; - for seed in 0..32u64 { - fill(&mut tags, 0x9e3779b97f4a7c15u64.wrapping_add(seed)); - for tag in [tags[seed as usize % width], 0u8, 0xFF, (seed as u8)] { - let expected = row_tag_match_mask_scalar(&tags, tag); - // SAFETY: NEON is baseline on aarch64. - let got = unsafe { row_tag_match_mask_neon(&tags, tag) }; - assert_eq!(got, expected, "neon width={width} tag={tag}"); - } - } - } - } -} +mod neon_tag_mask_tests; diff --git a/zstd/src/encoding/row/neon_tag_mask_tests.rs b/zstd/src/encoding/row/neon_tag_mask_tests.rs new file mode 100644 index 000000000..04102304b --- /dev/null +++ b/zstd/src/encoding/row/neon_tag_mask_tests.rs @@ -0,0 +1,29 @@ +use super::{row_tag_match_mask_neon, row_tag_match_mask_scalar}; + +fn fill(buf: &mut [u8], mut state: u64) { + for b in buf.iter_mut() { + state = state + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + *b = (state >> 56) as u8; + } +} + +/// The NEON kernel must produce byte-identical masks to the scalar +/// reference for every supported row width (16 / 32 / 64) and tag, so +/// match selection (and the compressed output) is unchanged on aarch64. +#[test] +fn neon_tag_mask_matches_scalar() { + for &width in &[16usize, 32, 64] { + let mut tags = alloc::vec![0u8; width]; + for seed in 0..32u64 { + fill(&mut tags, 0x9e3779b97f4a7c15u64.wrapping_add(seed)); + for tag in [tags[seed as usize % width], 0u8, 0xFF, (seed as u8)] { + let expected = row_tag_match_mask_scalar(&tags, tag); + // SAFETY: NEON is baseline on aarch64. + let got = unsafe { row_tag_match_mask_neon(&tags, tag) }; + assert_eq!(got, expected, "neon width={width} tag={tag}"); + } + } + } +} diff --git a/zstd/src/encoding/row/tag_mask_tests.rs b/zstd/src/encoding/row/tag_mask_tests.rs new file mode 100644 index 000000000..76b5b8bca --- /dev/null +++ b/zstd/src/encoding/row/tag_mask_tests.rs @@ -0,0 +1,37 @@ +use super::{row_tag_match_mask_avx2, row_tag_match_mask_scalar, row_tag_match_mask_sse2}; + +/// Deterministic LCG fill so the test exercises a realistic spread of +/// matching / non-matching tag bytes without a RNG dependency. +fn fill(buf: &mut [u8], mut state: u64) { + for b in buf.iter_mut() { + state = state + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + *b = (state >> 56) as u8; + } +} + +/// The SIMD kernels must produce byte-identical masks to the scalar +/// reference for every supported row width (16 / 32 / 64) and tag, or +/// the match selection diverges and the compressed output changes. +#[test] +fn simd_tag_mask_matches_scalar() { + for &width in &[16usize, 32, 64] { + let mut tags = alloc::vec![0u8; width]; + for seed in 0..32u64 { + fill(&mut tags, 0x9e3779b97f4a7c15u64.wrapping_add(seed)); + // Cover both a tag that occurs in the row and arbitrary tags. + for tag in [tags[seed as usize % width], 0u8, 0xFF, (seed as u8)] { + let expected = row_tag_match_mask_scalar(&tags, tag); + if std::arch::is_x86_feature_detected!("sse2") { + let got = unsafe { row_tag_match_mask_sse2(&tags, tag) }; + assert_eq!(got, expected, "sse2 width={width} tag={tag}"); + } + if std::arch::is_x86_feature_detected!("avx2") { + let got = unsafe { row_tag_match_mask_avx2(&tags, tag) }; + assert_eq!(got, expected, "avx2 width={width} tag={tag}"); + } + } + } + } +} diff --git a/zstd/src/encoding/sequence_capture.rs b/zstd/src/encoding/sequence_capture.rs index 838c8a979..c9e0778b9 100644 --- a/zstd/src/encoding/sequence_capture.rs +++ b/zstd/src/encoding/sequence_capture.rs @@ -566,303 +566,4 @@ fn detect_raw_or_rle_blocks_in_frame(frame: &[u8]) -> Result, &'stati } #[cfg(test)] -mod tests { - use super::*; - use crate::encoding::CompressionLevel; - use alloc::vec::Vec; - - /// On a 16 KiB repeating 16-byte pattern, the encoder must emit - /// at least one `Triple` sequence — every position past the first - /// 16 bytes finds a long match 16 bytes back. Constant-run input - /// (`AAAA…`) is covered separately by - /// `rejects_constant_run_with_rle_on_wire_block` because the - /// raw/RLE detector rejects that shape at the public-API - /// boundary (different contract from the per-position match - /// capture this test pins). On the rotating pattern here we - /// want at least one captured triple, which a clean - /// matcher always produces. - #[test] - fn captures_at_least_one_triple_on_repeating_pattern() { - let pattern: [u8; 16] = *b"PATTERN_1234_END"; - let data: Vec = pattern.iter().copied().cycle().take(16 * 1024).collect(); - let captured = compress_and_collect_sequences(&data, CompressionLevel::Level(3)); - let seqs = &captured.sequences; - assert!( - !seqs.is_empty(), - "expected at least one Triple sequence on 16KB repeating pattern, got 0", - ); - // Every captured sequence belongs to block 0 (single 16 KiB block fits in 128 KiB). - assert!( - seqs.iter().all(|s| s.block_idx == 0), - "16KB repeating pattern produced multi-block sequence stream: {:?}", - seqs.iter().map(|s| s.block_idx).collect::>(), - ); - // Every captured Triple must reference a sane offset/match - // (defensive: catches wrapper bugs that would leak garbage - // through the recorder). - for s in seqs { - assert!(s.of >= 1, "non-positive offset captured: {:?}", s); - assert!(s.ml >= 1, "non-positive match length captured: {:?}", s); - } - // seq_in_block must be contiguous 0..N for each block. - for (i, s) in seqs.iter().enumerate() { - assert_eq!( - s.seq_in_block, i as u32, - "seq_in_block discontinuity at idx {}: {:?}", - i, seqs, - ); - } - // Exactly one block was emitted, so exactly one tail-length entry. - assert_eq!(captured.block_tail_lengths.len(), 1); - // Reconstructed cumulative position must equal the input size: - // `Σ (ll + ml)` over triples PLUS the block's trailing-literal - // length must reach `data.len()`. This is the alignment - // invariant that motivated capturing tail lengths in the - // first place (PR #149 review). - let cumulative: u64 = seqs.iter().map(|s| s.ll as u64 + s.ml as u64).sum::() - + captured.block_tail_lengths[0] as u64; - assert_eq!( - cumulative, - data.len() as u64, - "Σ(ll+ml) + tail must reconstruct the input length exactly: \ - seqs sum + tail {} should == input {}", - cumulative, - data.len(), - ); - } - - /// Random / incompressible input is steered to a Raw_Block by the - /// encoder's raw fast-path classifier (`should_use_raw_for_block` - /// in `frame_compressor`), not by the late - /// `compressed.len() >= MAX_BLOCK_SIZE` fallback — 1 KiB of LCG - /// noise is well under one frame block, so the wire format - /// commits to `Raw_Block` before any sequence emission, and the - /// post-compress raw/RLE detector must panic. Before the - /// detector existed, this test verified the wrapper bounded - /// phantom-triple production; now the API-level guarantee is - /// stronger — the function refuses to return a misaligned - /// capture for such inputs (PR #149 review #25). - #[test] - #[should_panic(expected = "raw/RLE block")] - fn rejects_incompressible_input_with_raw_on_wire_block() { - // Deterministic non-repeating bytes via a simple LCG. - let mut state: u32 = 0x1234_5678; - let data: Vec = (0..1024) - .map(|_| { - state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); - (state >> 16) as u8 - }) - .collect(); - // Calling the function is the test: the raw-block detector - // inside `compress_and_collect_sequences` must panic before - // returning, so any code after this point is unreachable. - let _ = compress_and_collect_sequences(&data, CompressionLevel::Level(3)); - } - - /// Constant runs (`[b'A'; N]`) cause the encoder to emit an - /// RLE_Block on-wire (matcher may still produce a long - /// rep-coded triple, but the block payload is encoded as - /// `(byte, regenerated_size)`, not as sequences). The raw-block - /// detector treats RLE the same as Raw — both have no on-wire - /// sequences for the comparator to diff — and must panic to - /// prevent silently misaligned output (PR #149 review #25). - #[test] - #[should_panic(expected = "raw/RLE block")] - fn rejects_constant_run_with_rle_on_wire_block() { - let data: Vec = alloc::vec![b'A'; 16 * 1024]; - let _ = compress_and_collect_sequences(&data, CompressionLevel::Level(3)); - } - - /// Multi-block input (≥ 256 KiB, two 128 KiB frame blocks) is - /// the only shape that exercises per-block tail accounting - /// across a block boundary. Single-block tests pass even if a - /// regression records only the first/last block tail or - /// mis-increments `current_block` across the 128 KiB boundary. - /// - /// This test pins: - /// - `Σ(ll+ml) + Σ tails == input.len()` across multiple blocks, - /// - `block_tail_lengths.len() >= 2` (at least two blocks), - /// - every recorded triple's `block_idx` falls inside the - /// emitted-block range (no off-by-one on the counter), - /// - block indices observed in triples are contiguous from 0 - /// (no gaps, no missed `current_block` increments). - /// - /// 200 KiB (≈ 1.5 × 128 KiB) chosen to guarantee ≥ 2 frame blocks - /// while staying small enough that the final partial block still - /// compresses (no trailing Raw_Block from a sub-threshold tail), - /// keeping the multi-block assertion crisp (PR #149 review #19). - #[test] - fn captures_multi_block_tails_and_indices() { - // Repeating 16-byte pattern at 200 KiB (≈ 1.5 × 128 KiB - // frame block) — guaranteed to span ≥2 blocks while - // compressing densely enough to stay on the Compressed_Block - // path, avoiding the raw/RLE detector at the end of - // `compress_and_collect_sequences`. Larger / less repetitive - // fixtures risk a trailing Raw_Block when the final chunk is - // small enough that compression doesn't pay for itself. - let pattern: [u8; 16] = *b"PATTERN_1234_END"; - let data: Vec = pattern.iter().copied().cycle().take(200 * 1024).collect(); - let captured = compress_and_collect_sequences(&data, CompressionLevel::Level(3)); - - // Multi-block invariant: at least 2 blocks recorded. - assert!( - captured.block_tail_lengths.len() >= 2, - "expected ≥2 block tail entries for 200 KiB input, got {}: {:?}", - captured.block_tail_lengths.len(), - captured.block_tail_lengths, - ); - - // Cumulative reconstruction across all blocks. - let cumulative: u64 = captured - .sequences - .iter() - .map(|s| s.ll as u64 + s.ml as u64) - .sum::() - + captured - .block_tail_lengths - .iter() - .map(|t| *t as u64) - .sum::(); - assert_eq!( - cumulative, - data.len() as u64, - "multi-block Σ(ll+ml)+Σ(tails) mismatch: got {}, want {} \ - (blocks={}, triples={})", - cumulative, - data.len(), - captured.block_tail_lengths.len(), - captured.sequences.len(), - ); - - // Every triple's block_idx must be in [0, num_blocks). - let num_blocks = captured.block_tail_lengths.len() as u32; - for s in &captured.sequences { - assert!( - s.block_idx < num_blocks, - "triple block_idx={} out of range (num_blocks={})", - s.block_idx, - num_blocks, - ); - } - - // Block indices observed in triples must be contiguous - // starting at 0 (no skipped `current_block` increments). - let mut max_seen: i64 = -1; - for s in &captured.sequences { - let idx = s.block_idx as i64; - assert!( - idx <= max_seen + 1, - "non-monotonic / gapped block_idx in triple stream: \ - max_seen={max_seen}, observed={idx}", - ); - if idx > max_seen { - max_seen = idx; - } - } - } - - /// Pin the empty-input guard. Without this assert the - /// reconstruction invariant trivially passes (`0 == 0`) but - /// `block_tail_lengths` stays empty, breaking the public - /// per-emitted-block contract. - #[test] - #[should_panic(expected = "requires non-empty input")] - fn rejects_empty_input() { - let _ = compress_and_collect_sequences(&[], CompressionLevel::Level(3)); - } - - /// Pin the `CompressionLevel::Uncompressed` reject path. That - /// variant routes through the encoder's raw-block fast-path - /// without invoking the matcher, leaving the recorder empty and - /// the post-compress invariant misleading. - #[test] - #[should_panic(expected = "CompressionLevel::Uncompressed")] - fn rejects_uncompressed_level() { - let _ = compress_and_collect_sequences( - b"hello there general kenobi", - CompressionLevel::Uncompressed, - ); - } - - /// Pin the post-split-level reject path. `Level(16..=22)` (and - /// any `Level(n > 22)` that clamps to Level 22 params) goes - /// through `compress_block_with_post_split` which emits multiple - /// physical on-wire blocks per single matcher call — the - /// per-matcher-call counter cannot track that shape. - /// `Level(11..=15)` is intentionally NOT rejected: those levels - /// pre-split via `optimal_block_size` (borders), so each - /// shrunken block is processed by its own matcher call and the - /// counter stays correct. - #[test] - #[should_panic(expected = "does not support post-split levels")] - fn rejects_post_split_numeric_level() { - let _ = compress_and_collect_sequences( - b"hello there general kenobi", - CompressionLevel::Level(16), - ); - } - - /// `Best` is documented as "roughly equivalent to Level 11" but - /// `pre_split_level` matches the EXACT enum variants - /// (`Level(11..=15)` / `Level(16..=22)`) — the `Best` arm - /// falls through to `None`, so the named preset does NOT - /// trigger the block-splitter. The guard above - /// intentionally allows `Best` through; this test pins the - /// matcher path stays alignment-correct for it. Without it, a - /// future tightening of the guard to also reject `Best` would - /// silently break a valid capture path. - #[test] - fn captures_through_best_preset() { - // 32 KiB of a 32-byte rotating pattern: compressible enough - // for Best (lazy2 / btlazy2 strategy) to produce a non-empty - // sequence stream, small enough to keep the test fast. - let pattern: [u8; 32] = { - let mut p = [0u8; 32]; - for (i, b) in p.iter_mut().enumerate() { - *b = (i as u8).wrapping_mul(11).wrapping_add(3); - } - p - }; - let data: Vec = pattern.iter().copied().cycle().take(32 * 1024).collect(); - let captured = compress_and_collect_sequences(&data, CompressionLevel::Best); - let cumulative: u64 = captured - .sequences - .iter() - .map(|s| s.ll as u64 + s.ml as u64) - .sum::() - + captured - .block_tail_lengths - .iter() - .map(|t| *t as u64) - .sum::(); - assert_eq!(cumulative, data.len() as u64); - } - - /// Positive coverage for the pre-split numeric range - /// (`Level(11..=15)`). The borders splitter parks the tail in - /// `pending_input` BEFORE the matcher runs, so each shrunken - /// portion gets its own matcher call and the counter stays - /// accurate. Without this test, a future tightening of the - /// guard to reject `Level(11..=15)` would regress a valid - /// capture path. - #[test] - fn captures_through_pre_split_level_15() { - // 32 KiB of a 16-byte rotating pattern: compressible enough - // for lazy2 (Level 11..=15 strategy) to produce a non-empty - // sequence stream without tripping the raw/RLE detector. - let pattern: [u8; 16] = *b"PATTERN_1234_END"; - let data: Vec = pattern.iter().copied().cycle().take(32 * 1024).collect(); - let captured = compress_and_collect_sequences(&data, CompressionLevel::Level(15)); - let cumulative: u64 = captured - .sequences - .iter() - .map(|s| s.ll as u64 + s.ml as u64) - .sum::() - + captured - .block_tail_lengths - .iter() - .map(|t| *t as u64) - .sum::(); - assert_eq!(cumulative, data.len() as u64); - } -} +mod tests; diff --git a/zstd/src/encoding/sequence_capture/tests.rs b/zstd/src/encoding/sequence_capture/tests.rs new file mode 100644 index 000000000..8b950a322 --- /dev/null +++ b/zstd/src/encoding/sequence_capture/tests.rs @@ -0,0 +1,296 @@ +use super::*; +use crate::encoding::CompressionLevel; +use alloc::vec::Vec; + +/// On a 16 KiB repeating 16-byte pattern, the encoder must emit +/// at least one `Triple` sequence — every position past the first +/// 16 bytes finds a long match 16 bytes back. Constant-run input +/// (`AAAA…`) is covered separately by +/// `rejects_constant_run_with_rle_on_wire_block` because the +/// raw/RLE detector rejects that shape at the public-API +/// boundary (different contract from the per-position match +/// capture this test pins). On the rotating pattern here we +/// want at least one captured triple, which a clean +/// matcher always produces. +#[test] +fn captures_at_least_one_triple_on_repeating_pattern() { + let pattern: [u8; 16] = *b"PATTERN_1234_END"; + let data: Vec = pattern.iter().copied().cycle().take(16 * 1024).collect(); + let captured = compress_and_collect_sequences(&data, CompressionLevel::Level(3)); + let seqs = &captured.sequences; + assert!( + !seqs.is_empty(), + "expected at least one Triple sequence on 16KB repeating pattern, got 0", + ); + // Every captured sequence belongs to block 0 (single 16 KiB block fits in 128 KiB). + assert!( + seqs.iter().all(|s| s.block_idx == 0), + "16KB repeating pattern produced multi-block sequence stream: {:?}", + seqs.iter().map(|s| s.block_idx).collect::>(), + ); + // Every captured Triple must reference a sane offset/match + // (defensive: catches wrapper bugs that would leak garbage + // through the recorder). + for s in seqs { + assert!(s.of >= 1, "non-positive offset captured: {:?}", s); + assert!(s.ml >= 1, "non-positive match length captured: {:?}", s); + } + // seq_in_block must be contiguous 0..N for each block. + for (i, s) in seqs.iter().enumerate() { + assert_eq!( + s.seq_in_block, i as u32, + "seq_in_block discontinuity at idx {}: {:?}", + i, seqs, + ); + } + // Exactly one block was emitted, so exactly one tail-length entry. + assert_eq!(captured.block_tail_lengths.len(), 1); + // Reconstructed cumulative position must equal the input size: + // `Σ (ll + ml)` over triples PLUS the block's trailing-literal + // length must reach `data.len()`. This is the alignment + // invariant that motivated capturing tail lengths in the + // first place (PR #149 review). + let cumulative: u64 = seqs.iter().map(|s| s.ll as u64 + s.ml as u64).sum::() + + captured.block_tail_lengths[0] as u64; + assert_eq!( + cumulative, + data.len() as u64, + "Σ(ll+ml) + tail must reconstruct the input length exactly: \ + seqs sum + tail {} should == input {}", + cumulative, + data.len(), + ); +} + +/// Random / incompressible input is steered to a Raw_Block by the +/// encoder's raw fast-path classifier (`should_use_raw_for_block` +/// in `frame_compressor`), not by the late +/// `compressed.len() >= MAX_BLOCK_SIZE` fallback — 1 KiB of LCG +/// noise is well under one frame block, so the wire format +/// commits to `Raw_Block` before any sequence emission, and the +/// post-compress raw/RLE detector must panic. Before the +/// detector existed, this test verified the wrapper bounded +/// phantom-triple production; now the API-level guarantee is +/// stronger — the function refuses to return a misaligned +/// capture for such inputs (PR #149 review #25). +#[test] +#[should_panic(expected = "raw/RLE block")] +fn rejects_incompressible_input_with_raw_on_wire_block() { + // Deterministic non-repeating bytes via a simple LCG. + let mut state: u32 = 0x1234_5678; + let data: Vec = (0..1024) + .map(|_| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (state >> 16) as u8 + }) + .collect(); + // Calling the function is the test: the raw-block detector + // inside `compress_and_collect_sequences` must panic before + // returning, so any code after this point is unreachable. + let _ = compress_and_collect_sequences(&data, CompressionLevel::Level(3)); +} + +/// Constant runs (`[b'A'; N]`) cause the encoder to emit an +/// RLE_Block on-wire (matcher may still produce a long +/// rep-coded triple, but the block payload is encoded as +/// `(byte, regenerated_size)`, not as sequences). The raw-block +/// detector treats RLE the same as Raw — both have no on-wire +/// sequences for the comparator to diff — and must panic to +/// prevent silently misaligned output (PR #149 review #25). +#[test] +#[should_panic(expected = "raw/RLE block")] +fn rejects_constant_run_with_rle_on_wire_block() { + let data: Vec = alloc::vec![b'A'; 16 * 1024]; + let _ = compress_and_collect_sequences(&data, CompressionLevel::Level(3)); +} + +/// Multi-block input (≥ 256 KiB, two 128 KiB frame blocks) is +/// the only shape that exercises per-block tail accounting +/// across a block boundary. Single-block tests pass even if a +/// regression records only the first/last block tail or +/// mis-increments `current_block` across the 128 KiB boundary. +/// +/// This test pins: +/// - `Σ(ll+ml) + Σ tails == input.len()` across multiple blocks, +/// - `block_tail_lengths.len() >= 2` (at least two blocks), +/// - every recorded triple's `block_idx` falls inside the +/// emitted-block range (no off-by-one on the counter), +/// - block indices observed in triples are contiguous from 0 +/// (no gaps, no missed `current_block` increments). +/// +/// 200 KiB (≈ 1.5 × 128 KiB) chosen to guarantee ≥ 2 frame blocks +/// while staying small enough that the final partial block still +/// compresses (no trailing Raw_Block from a sub-threshold tail), +/// keeping the multi-block assertion crisp (PR #149 review #19). +#[test] +fn captures_multi_block_tails_and_indices() { + // Repeating 16-byte pattern at 200 KiB (≈ 1.5 × 128 KiB + // frame block) — guaranteed to span ≥2 blocks while + // compressing densely enough to stay on the Compressed_Block + // path, avoiding the raw/RLE detector at the end of + // `compress_and_collect_sequences`. Larger / less repetitive + // fixtures risk a trailing Raw_Block when the final chunk is + // small enough that compression doesn't pay for itself. + let pattern: [u8; 16] = *b"PATTERN_1234_END"; + let data: Vec = pattern.iter().copied().cycle().take(200 * 1024).collect(); + let captured = compress_and_collect_sequences(&data, CompressionLevel::Level(3)); + + // Multi-block invariant: at least 2 blocks recorded. + assert!( + captured.block_tail_lengths.len() >= 2, + "expected ≥2 block tail entries for 200 KiB input, got {}: {:?}", + captured.block_tail_lengths.len(), + captured.block_tail_lengths, + ); + + // Cumulative reconstruction across all blocks. + let cumulative: u64 = captured + .sequences + .iter() + .map(|s| s.ll as u64 + s.ml as u64) + .sum::() + + captured + .block_tail_lengths + .iter() + .map(|t| *t as u64) + .sum::(); + assert_eq!( + cumulative, + data.len() as u64, + "multi-block Σ(ll+ml)+Σ(tails) mismatch: got {}, want {} \ + (blocks={}, triples={})", + cumulative, + data.len(), + captured.block_tail_lengths.len(), + captured.sequences.len(), + ); + + // Every triple's block_idx must be in [0, num_blocks). + let num_blocks = captured.block_tail_lengths.len() as u32; + for s in &captured.sequences { + assert!( + s.block_idx < num_blocks, + "triple block_idx={} out of range (num_blocks={})", + s.block_idx, + num_blocks, + ); + } + + // Block indices observed in triples must be contiguous + // starting at 0 (no skipped `current_block` increments). + let mut max_seen: i64 = -1; + for s in &captured.sequences { + let idx = s.block_idx as i64; + assert!( + idx <= max_seen + 1, + "non-monotonic / gapped block_idx in triple stream: \ + max_seen={max_seen}, observed={idx}", + ); + if idx > max_seen { + max_seen = idx; + } + } +} + +/// Pin the empty-input guard. Without this assert the +/// reconstruction invariant trivially passes (`0 == 0`) but +/// `block_tail_lengths` stays empty, breaking the public +/// per-emitted-block contract. +#[test] +#[should_panic(expected = "requires non-empty input")] +fn rejects_empty_input() { + let _ = compress_and_collect_sequences(&[], CompressionLevel::Level(3)); +} + +/// Pin the `CompressionLevel::Uncompressed` reject path. That +/// variant routes through the encoder's raw-block fast-path +/// without invoking the matcher, leaving the recorder empty and +/// the post-compress invariant misleading. +#[test] +#[should_panic(expected = "CompressionLevel::Uncompressed")] +fn rejects_uncompressed_level() { + let _ = compress_and_collect_sequences( + b"hello there general kenobi", + CompressionLevel::Uncompressed, + ); +} + +/// Pin the post-split-level reject path. `Level(16..=22)` (and +/// any `Level(n > 22)` that clamps to Level 22 params) goes +/// through `compress_block_with_post_split` which emits multiple +/// physical on-wire blocks per single matcher call — the +/// per-matcher-call counter cannot track that shape. +/// `Level(11..=15)` is intentionally NOT rejected: those levels +/// pre-split via `optimal_block_size` (borders), so each +/// shrunken block is processed by its own matcher call and the +/// counter stays correct. +#[test] +#[should_panic(expected = "does not support post-split levels")] +fn rejects_post_split_numeric_level() { + let _ = + compress_and_collect_sequences(b"hello there general kenobi", CompressionLevel::Level(16)); +} + +/// `Best` is documented as "roughly equivalent to Level 11" but +/// `pre_split_level` matches the EXACT enum variants +/// (`Level(11..=15)` / `Level(16..=22)`) — the `Best` arm +/// falls through to `None`, so the named preset does NOT +/// trigger the block-splitter. The guard above +/// intentionally allows `Best` through; this test pins the +/// matcher path stays alignment-correct for it. Without it, a +/// future tightening of the guard to also reject `Best` would +/// silently break a valid capture path. +#[test] +fn captures_through_best_preset() { + // 32 KiB of a 32-byte rotating pattern: compressible enough + // for Best (lazy2 / btlazy2 strategy) to produce a non-empty + // sequence stream, small enough to keep the test fast. + let pattern: [u8; 32] = { + let mut p = [0u8; 32]; + for (i, b) in p.iter_mut().enumerate() { + *b = (i as u8).wrapping_mul(11).wrapping_add(3); + } + p + }; + let data: Vec = pattern.iter().copied().cycle().take(32 * 1024).collect(); + let captured = compress_and_collect_sequences(&data, CompressionLevel::Best); + let cumulative: u64 = captured + .sequences + .iter() + .map(|s| s.ll as u64 + s.ml as u64) + .sum::() + + captured + .block_tail_lengths + .iter() + .map(|t| *t as u64) + .sum::(); + assert_eq!(cumulative, data.len() as u64); +} + +/// Positive coverage for the pre-split numeric range +/// (`Level(11..=15)`). The borders splitter parks the tail in +/// `pending_input` BEFORE the matcher runs, so each shrunken +/// portion gets its own matcher call and the counter stays +/// accurate. Without this test, a future tightening of the +/// guard to reject `Level(11..=15)` would regress a valid +/// capture path. +#[test] +fn captures_through_pre_split_level_15() { + // 32 KiB of a 16-byte rotating pattern: compressible enough + // for lazy2 (Level 11..=15 strategy) to produce a non-empty + // sequence stream without tripping the raw/RLE detector. + let pattern: [u8; 16] = *b"PATTERN_1234_END"; + let data: Vec = pattern.iter().copied().cycle().take(32 * 1024).collect(); + let captured = compress_and_collect_sequences(&data, CompressionLevel::Level(15)); + let cumulative: u64 = captured + .sequences + .iter() + .map(|s| s.ll as u64 + s.ml as u64) + .sum::() + + captured + .block_tail_lengths + .iter() + .map(|t| *t as u64) + .sum::(); + assert_eq!(cumulative, data.len() as u64); +} diff --git a/zstd/src/encoding/simple/fast_kernel/count.rs b/zstd/src/encoding/simple/fast_kernel/count.rs index 0866d7cda..a35775a40 100644 --- a/zstd/src/encoding/simple/fast_kernel/count.rs +++ b/zstd/src/encoding/simple/fast_kernel/count.rs @@ -259,144 +259,4 @@ pub(crate) fn count_forward_dict_2segment( } #[cfg(test)] -mod tests { - use super::*; - - fn count(a: &[u8], b: &[u8]) -> usize { - let min_len = a.len().min(b.len()); - // SAFETY: both slices have at least `min_len` readable bytes, - // `iend = a.as_ptr() + min_len` stays in range. - unsafe { count_forward(a.as_ptr(), b.as_ptr(), a.as_ptr().add(min_len)) } - } - - #[test] - fn empty_inputs_return_zero() { - // Empty range → loop body never executes. - let a: [u8; 0] = []; - let b: [u8; 0] = []; - // SAFETY: iend == ip, the function never dereferences. - let n = unsafe { count_forward(a.as_ptr(), b.as_ptr(), a.as_ptr()) }; - assert_eq!(n, 0); - } - - #[test] - fn full_match_inside_8_byte_chunk() { - let a = [1, 2, 3, 4, 5, 6, 7, 8]; - let b = [1, 2, 3, 4, 5, 6, 7, 8]; - assert_eq!(count(&a, &b), 8); - } - - #[test] - fn diff_at_byte_3_in_first_chunk() { - let a = [1, 2, 3, 9, 5, 6, 7, 8]; - let b = [1, 2, 3, 4, 5, 6, 7, 8]; - assert_eq!(count(&a, &b), 3); - } - - #[test] - fn match_spanning_two_chunks() { - let mut a = [0u8; 16]; - let mut b = [0u8; 16]; - for i in 0..16 { - a[i] = i as u8; - b[i] = i as u8; - } - a[13] = 99; - assert_eq!(count(&a, &b), 13); - } - - #[test] - fn match_terminates_at_iend_within_tail() { - // 11 bytes: 1×8-chunk + 3 tail bytes (u16 + u8 fall-through). - let a = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]; - let b = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]; - assert_eq!(count(&a, &b), 11); - } - - #[test] - fn diff_in_u32_tail() { - // 12 bytes: 1×8-chunk match, then 4-byte tail diverges at - // BYTE INDEX 9 (`99` vs `10`). After the 8-chunk advances - // ip/m by 8, the upstream zstd's u32 tail check compares - // a[8..12]=[9,99,11,12] vs b[8..12]=[9,10,11,12] → unequal, - // so the u32 advance is skipped. Same for u16 - // (a[8..10]=[9,99] vs b[8..10]=[9,10] → unequal). The single - // byte cmp THEN sees a[8]=9 == b[8]=9 and advances ip by 1. - // Final match length: 8 + 1 = 9. - let a = [1, 2, 3, 4, 5, 6, 7, 8, 9, 99, 11, 12]; - let b = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]; - assert_eq!(count(&a, &b), 9); - } - - #[test] - fn diff_in_u16_tail_after_u32_match() { - // 14 bytes total, first 12 match, then u16 differs at byte 12. - let a = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 99, 14]; - let b = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]; - // 8 chunk + 4 u32 = 12 matched. u16 cmp on (99,14) vs (13,14) - // says unequal → 0 more. Single byte cmp on 99 vs 13 → 0 more. - assert_eq!(count(&a, &b), 12); - } - - #[test] - fn diff_in_single_byte_tail() { - // 13 bytes, first 12 match, then single byte differs. - let a = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 99]; - let b = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]; - // 8 chunk + 4 u32 = 12; u16 cmp not entered (only 1 byte left). - // Single byte cmp 99 != 13 → 0 more. - assert_eq!(count(&a, &b), 12); - } - - #[test] - fn long_match_thousand_bytes() { - let a = [0x5Au8; 1024]; - let b = [0x5Au8; 1024]; - assert_eq!(count(&a, &b), 1024); - } - - #[test] - fn no_match_first_byte() { - let a = [0u8, 1, 2, 3, 4, 5, 6, 7]; - let b = [9u8, 1, 2, 3, 4, 5, 6, 7]; - assert_eq!(count(&a, &b), 0); - } - - #[test] - fn dict_2segment_within_dict_only() { - // Candidate fully inside the dict; current input matches the dict tail. - let dict = [10u8, 20, 30, 40]; - let inp = [30u8, 40, 99]; - // cand=2: dict[2]=30 vs inp[0]=30 ✓; dict[3]=40 vs inp[1]=40 ✓; - // cand_idx=4 == dict.len() → inp[0]=30 vs inp[2]=99 ✗ → len 2. - assert_eq!(count_forward_dict_2segment(&dict, 2, &inp, 0), 2); - } - - #[test] - fn dict_2segment_crosses_boundary_into_input() { - // Match starts in the dict and continues past the boundary into the - // input (the logical [dict][input] window), then stops at a mismatch. - let dict = [1u8, 2, 3]; - let inp = [1u8, 2, 3, 1, 2, 3, 9]; // cur=3 → [1,2,3,9...] - // cand=0: dict[0..3] match inp[3..6]; cand_idx=3 → inp[0]=1 vs inp[6]=9 ✗ → 3. - assert_eq!(count_forward_dict_2segment(&dict, 0, &inp, 3), 3); - } - - #[test] - fn dict_2segment_continues_word_at_a_time_past_boundary() { - // Candidate exhausts the dict mid-match and keeps matching into the - // input segment — exercises the segment-2 count_forward path. - let dict = [1u8, 2, 3]; - let inp = [1u8, 2, 3, 1, 2, 3, 1, 2]; // cur=3 → [1,2,3,1,2] - // seg1: dict[0..3]=[1,2,3] vs inp[3..6]=[1,2,3] → m1=3 (dict exhausted). - // seg2: inp[0..]=[1,2,...] vs inp[6..]=[1,2] → m2=2. total 5. - assert_eq!(count_forward_dict_2segment(&dict, 0, &inp, 3), 5); - } - - #[test] - fn dict_2segment_stops_at_input_end() { - let dict = [7u8, 7]; - let inp = [7u8, 7, 7, 7]; // cur=2 → only 2 bytes left - assert_eq!(count_forward_dict_2segment(&dict, 0, &inp, 2), 2); - } -} +mod tests; diff --git a/zstd/src/encoding/simple/fast_kernel/count/tests.rs b/zstd/src/encoding/simple/fast_kernel/count/tests.rs new file mode 100644 index 000000000..d810615c7 --- /dev/null +++ b/zstd/src/encoding/simple/fast_kernel/count/tests.rs @@ -0,0 +1,139 @@ +use super::*; + +fn count(a: &[u8], b: &[u8]) -> usize { + let min_len = a.len().min(b.len()); + // SAFETY: both slices have at least `min_len` readable bytes, + // `iend = a.as_ptr() + min_len` stays in range. + unsafe { count_forward(a.as_ptr(), b.as_ptr(), a.as_ptr().add(min_len)) } +} + +#[test] +fn empty_inputs_return_zero() { + // Empty range → loop body never executes. + let a: [u8; 0] = []; + let b: [u8; 0] = []; + // SAFETY: iend == ip, the function never dereferences. + let n = unsafe { count_forward(a.as_ptr(), b.as_ptr(), a.as_ptr()) }; + assert_eq!(n, 0); +} + +#[test] +fn full_match_inside_8_byte_chunk() { + let a = [1, 2, 3, 4, 5, 6, 7, 8]; + let b = [1, 2, 3, 4, 5, 6, 7, 8]; + assert_eq!(count(&a, &b), 8); +} + +#[test] +fn diff_at_byte_3_in_first_chunk() { + let a = [1, 2, 3, 9, 5, 6, 7, 8]; + let b = [1, 2, 3, 4, 5, 6, 7, 8]; + assert_eq!(count(&a, &b), 3); +} + +#[test] +fn match_spanning_two_chunks() { + let mut a = [0u8; 16]; + let mut b = [0u8; 16]; + for i in 0..16 { + a[i] = i as u8; + b[i] = i as u8; + } + a[13] = 99; + assert_eq!(count(&a, &b), 13); +} + +#[test] +fn match_terminates_at_iend_within_tail() { + // 11 bytes: 1×8-chunk + 3 tail bytes (u16 + u8 fall-through). + let a = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]; + let b = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]; + assert_eq!(count(&a, &b), 11); +} + +#[test] +fn diff_in_u32_tail() { + // 12 bytes: 1×8-chunk match, then 4-byte tail diverges at + // BYTE INDEX 9 (`99` vs `10`). After the 8-chunk advances + // ip/m by 8, the upstream zstd's u32 tail check compares + // a[8..12]=[9,99,11,12] vs b[8..12]=[9,10,11,12] → unequal, + // so the u32 advance is skipped. Same for u16 + // (a[8..10]=[9,99] vs b[8..10]=[9,10] → unequal). The single + // byte cmp THEN sees a[8]=9 == b[8]=9 and advances ip by 1. + // Final match length: 8 + 1 = 9. + let a = [1, 2, 3, 4, 5, 6, 7, 8, 9, 99, 11, 12]; + let b = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]; + assert_eq!(count(&a, &b), 9); +} + +#[test] +fn diff_in_u16_tail_after_u32_match() { + // 14 bytes total, first 12 match, then u16 differs at byte 12. + let a = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 99, 14]; + let b = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]; + // 8 chunk + 4 u32 = 12 matched. u16 cmp on (99,14) vs (13,14) + // says unequal → 0 more. Single byte cmp on 99 vs 13 → 0 more. + assert_eq!(count(&a, &b), 12); +} + +#[test] +fn diff_in_single_byte_tail() { + // 13 bytes, first 12 match, then single byte differs. + let a = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 99]; + let b = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]; + // 8 chunk + 4 u32 = 12; u16 cmp not entered (only 1 byte left). + // Single byte cmp 99 != 13 → 0 more. + assert_eq!(count(&a, &b), 12); +} + +#[test] +fn long_match_thousand_bytes() { + let a = [0x5Au8; 1024]; + let b = [0x5Au8; 1024]; + assert_eq!(count(&a, &b), 1024); +} + +#[test] +fn no_match_first_byte() { + let a = [0u8, 1, 2, 3, 4, 5, 6, 7]; + let b = [9u8, 1, 2, 3, 4, 5, 6, 7]; + assert_eq!(count(&a, &b), 0); +} + +#[test] +fn dict_2segment_within_dict_only() { + // Candidate fully inside the dict; current input matches the dict tail. + let dict = [10u8, 20, 30, 40]; + let inp = [30u8, 40, 99]; + // cand=2: dict[2]=30 vs inp[0]=30 ✓; dict[3]=40 vs inp[1]=40 ✓; + // cand_idx=4 == dict.len() → inp[0]=30 vs inp[2]=99 ✗ → len 2. + assert_eq!(count_forward_dict_2segment(&dict, 2, &inp, 0), 2); +} + +#[test] +fn dict_2segment_crosses_boundary_into_input() { + // Match starts in the dict and continues past the boundary into the + // input (the logical [dict][input] window), then stops at a mismatch. + let dict = [1u8, 2, 3]; + let inp = [1u8, 2, 3, 1, 2, 3, 9]; // cur=3 → [1,2,3,9...] + // cand=0: dict[0..3] match inp[3..6]; cand_idx=3 → inp[0]=1 vs inp[6]=9 ✗ → 3. + assert_eq!(count_forward_dict_2segment(&dict, 0, &inp, 3), 3); +} + +#[test] +fn dict_2segment_continues_word_at_a_time_past_boundary() { + // Candidate exhausts the dict mid-match and keeps matching into the + // input segment — exercises the segment-2 count_forward path. + let dict = [1u8, 2, 3]; + let inp = [1u8, 2, 3, 1, 2, 3, 1, 2]; // cur=3 → [1,2,3,1,2] + // seg1: dict[0..3]=[1,2,3] vs inp[3..6]=[1,2,3] → m1=3 (dict exhausted). + // seg2: inp[0..]=[1,2,...] vs inp[6..]=[1,2] → m2=2. total 5. + assert_eq!(count_forward_dict_2segment(&dict, 0, &inp, 3), 5); +} + +#[test] +fn dict_2segment_stops_at_input_end() { + let dict = [7u8, 7]; + let inp = [7u8, 7, 7, 7]; // cur=2 → only 2 bytes left + assert_eq!(count_forward_dict_2segment(&dict, 0, &inp, 2), 2); +} diff --git a/zstd/src/encoding/simple/fast_kernel/hash_table.rs b/zstd/src/encoding/simple/fast_kernel/hash_table.rs index 5cf1ce79d..94a75d3bf 100644 --- a/zstd/src/encoding/simple/fast_kernel/hash_table.rs +++ b/zstd/src/encoding/simple/fast_kernel/hash_table.rs @@ -411,168 +411,4 @@ pub(crate) unsafe fn hash_ptr_raw(ptr: *const u8, hash_log: u32) } #[cfg(test)] -mod tests { - use super::*; - - /// Upstream zstd parity: `ZSTD_hash4` on `[0x01, 0x02, 0x03, 0x04]` with - /// hash_log=12 produces a specific bit pattern. Captured here as a - /// regression tripwire so any future refactor of the multiply - /// constants surfaces immediately. - #[test] - fn hash4_matches_expected_value_on_known_input() { - let table = FastHashTable::new(12, 4); - let data = [0x01u8, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]; - // SAFETY: data has 8 ≥ 4 readable bytes. - let h = unsafe { table.hash_ptr::<4>(data.as_ptr()) }; - // Manual upstream zstd calc: u32::from_le_bytes(0x04030201) * 0x9E3779B1 >> 20. - let expected = 0x04030201u32.wrapping_mul(0x9E3779B1) >> 20; - assert_eq!( - h, expected, - "hash4 must match upstream zstd multiply-shift formula" - ); - } - - #[test] - fn hash5_matches_expected_value_on_known_input() { - let table = FastHashTable::new(13, 5); - let data = [0x01u8, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]; - // SAFETY: data has 8 ≥ 5 readable bytes. - let h = unsafe { table.hash_ptr::<5>(data.as_ptr()) }; - let u = u64::from_le_bytes(data); - let expected = (((u << (64 - 40)).wrapping_mul(889_523_592_379u64)) >> (64 - 13)) as u32; - assert_eq!(h, expected); - } - - #[test] - fn get_put_round_trip_under_known_hash() { - let mut table = FastHashTable::new(8, 4); - let data = [0xAAu8, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, 0x00, 0x11]; - // SAFETY: data has 8 ≥ 4 readable bytes. - let h = unsafe { table.hash_ptr::<4>(data.as_ptr()) }; - // SAFETY: h came from hash_ptr on this table. - unsafe { - assert_eq!(table.get(h), 0, "fresh table reads sentinel"); - table.put(h, 0xCAFE_BABE); - assert_eq!(table.get(h), 0xCAFE_BABE); - } - } - - #[test] - fn clear_resets_all_entries_to_sentinel() { - let mut table = FastHashTable::new(6, 4); - let data = [1u8, 2, 3, 4, 5, 6, 7, 8]; - // SAFETY: 4 readable bytes. - let h = unsafe { table.hash_ptr::<4>(data.as_ptr()) }; - // SAFETY: hash came from hash_ptr. - unsafe { - table.put(h, 42); - } - table.clear(); - // SAFETY: hash came from hash_ptr. - let read_back = unsafe { table.get(h) }; - assert_eq!(read_back, 0, "clear must zero every entry"); - } - - /// Upstream zstd parity for mls=6: `ZSTD_hash6` uses - /// `(u << (64-48)).wrapping_mul(PRIME_6_BYTES) >> (64 - hash_log)`. - #[test] - fn hash6_matches_expected_value_on_known_input() { - let table = FastHashTable::new(14, 6); - let data = [0x01u8, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]; - // SAFETY: data has 8 ≥ 6 readable bytes (the implementation - // performs a u64 load and shifts off the unused top bits). - let h = unsafe { table.hash_ptr::<6>(data.as_ptr()) }; - let u = u64::from_le_bytes(data); - let expected = - (((u << (64 - 48)).wrapping_mul(227_718_039_650_203u64)) >> (64 - 14)) as u32; - assert_eq!( - h, expected, - "hash6 must match upstream zstd multiply-shift formula" - ); - } - - /// Upstream zstd parity for mls=7: `ZSTD_hash7` shifts by `(64-56)` and - /// multiplies by `PRIME_7_BYTES`. - #[test] - fn hash7_matches_expected_value_on_known_input() { - let table = FastHashTable::new(15, 7); - let data = [0x01u8, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]; - // SAFETY: data has 8 ≥ 7 readable bytes. - let h = unsafe { table.hash_ptr::<7>(data.as_ptr()) }; - let u = u64::from_le_bytes(data); - let expected = - (((u << (64 - 56)).wrapping_mul(58_295_818_150_454_627u64)) >> (64 - 15)) as u32; - assert_eq!( - h, expected, - "hash7 must match upstream zstd multiply-shift formula" - ); - } - - /// Upstream zstd parity for mls=8: `ZSTD_hash8` does NOT shift the input - /// (full u64), then multiplies by `PRIME_8_BYTES` (upstream zstd's - /// `prime8bytes = 0xCF1BBCDCB7A56463ULL`). - #[test] - fn hash8_matches_expected_value_on_known_input() { - let table = FastHashTable::new(16, 8); - let data = [0x01u8, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]; - // SAFETY: data has 8 readable bytes. - let h = unsafe { table.hash_ptr::<8>(data.as_ptr()) }; - let u = u64::from_le_bytes(data); - let expected = (u.wrapping_mul(0xCF1BBCDCB7A56463u64) >> (64 - 16)) as u32; - assert_eq!( - h, expected, - "hash8 must match upstream zstd multiply-shift formula" - ); - } - - /// Boundary: `hash_log = 1` is the smallest accepted value - /// (table of 2 entries). Verifies the constructor doesn't reject - /// the minimum and that the table actually allocates. - #[test] - fn hash_log_minimum_one_constructs_two_entry_table() { - let table = FastHashTable::new(1, 4); - let data = [0u8, 0, 0, 0]; - // SAFETY: 4 readable bytes; hash output occupies 1 bit so - // hash ∈ {0, 1}. - let h = unsafe { table.hash_ptr::<4>(data.as_ptr()) }; - assert!(h < 2, "hash_log=1 must produce values ∈ {{0, 1}} (got {h})"); - } - - /// Boundary: `hash_log = ZSTD_HASHLOG_MAX` (30) is the largest - /// accepted value. Calling [`FastHashTable::new`] with this value - /// would allocate ≈4 GiB (`1 << 30` × `sizeof(u32)`), well over - /// per-test memory budgets on CI runners — instead exercise the - /// same accept path via the non-allocating [`validate_params`] - /// helper. Pairs with the `should_panic` tests below that prove - /// rejection of the out-of-band cases. - #[test] - fn hash_log_maximum_thirty_is_accepted_by_validation() { - validate_params(ZSTD_HASHLOG_MAX, 4); - validate_params(ZSTD_HASHLOG_MAX, 8); - } - - #[test] - #[should_panic(expected = "hash_log must be in 1..=")] - fn panics_on_zero_hash_log() { - let _ = FastHashTable::new(0, 4); - } - - #[test] - #[should_panic(expected = "hash_log must be in 1..=")] - fn panics_on_hash_log_above_zstd_hashlog_max() { - // 31 > ZSTD_HASHLOG_MAX (30) → constructor rejects. - let _ = FastHashTable::new(31, 4); - } - - #[test] - #[should_panic(expected = "ZSTD Fast strategy only supports mls 4..=8")] - fn panics_on_mls_below_four() { - let _ = FastHashTable::new(12, 3); - } - - #[test] - #[should_panic(expected = "ZSTD Fast strategy only supports mls 4..=8")] - fn panics_on_mls_above_eight() { - let _ = FastHashTable::new(12, 9); - } -} +mod tests; diff --git a/zstd/src/encoding/simple/fast_kernel/hash_table/tests.rs b/zstd/src/encoding/simple/fast_kernel/hash_table/tests.rs new file mode 100644 index 000000000..c8d74823a --- /dev/null +++ b/zstd/src/encoding/simple/fast_kernel/hash_table/tests.rs @@ -0,0 +1,161 @@ +use super::*; + +/// Upstream zstd parity: `ZSTD_hash4` on `[0x01, 0x02, 0x03, 0x04]` with +/// hash_log=12 produces a specific bit pattern. Captured here as a +/// regression tripwire so any future refactor of the multiply +/// constants surfaces immediately. +#[test] +fn hash4_matches_expected_value_on_known_input() { + let table = FastHashTable::new(12, 4); + let data = [0x01u8, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]; + // SAFETY: data has 8 ≥ 4 readable bytes. + let h = unsafe { table.hash_ptr::<4>(data.as_ptr()) }; + // Manual upstream zstd calc: u32::from_le_bytes(0x04030201) * 0x9E3779B1 >> 20. + let expected = 0x04030201u32.wrapping_mul(0x9E3779B1) >> 20; + assert_eq!( + h, expected, + "hash4 must match upstream zstd multiply-shift formula" + ); +} + +#[test] +fn hash5_matches_expected_value_on_known_input() { + let table = FastHashTable::new(13, 5); + let data = [0x01u8, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]; + // SAFETY: data has 8 ≥ 5 readable bytes. + let h = unsafe { table.hash_ptr::<5>(data.as_ptr()) }; + let u = u64::from_le_bytes(data); + let expected = (((u << (64 - 40)).wrapping_mul(889_523_592_379u64)) >> (64 - 13)) as u32; + assert_eq!(h, expected); +} + +#[test] +fn get_put_round_trip_under_known_hash() { + let mut table = FastHashTable::new(8, 4); + let data = [0xAAu8, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, 0x00, 0x11]; + // SAFETY: data has 8 ≥ 4 readable bytes. + let h = unsafe { table.hash_ptr::<4>(data.as_ptr()) }; + // SAFETY: h came from hash_ptr on this table. + unsafe { + assert_eq!(table.get(h), 0, "fresh table reads sentinel"); + table.put(h, 0xCAFE_BABE); + assert_eq!(table.get(h), 0xCAFE_BABE); + } +} + +#[test] +fn clear_resets_all_entries_to_sentinel() { + let mut table = FastHashTable::new(6, 4); + let data = [1u8, 2, 3, 4, 5, 6, 7, 8]; + // SAFETY: 4 readable bytes. + let h = unsafe { table.hash_ptr::<4>(data.as_ptr()) }; + // SAFETY: hash came from hash_ptr. + unsafe { + table.put(h, 42); + } + table.clear(); + // SAFETY: hash came from hash_ptr. + let read_back = unsafe { table.get(h) }; + assert_eq!(read_back, 0, "clear must zero every entry"); +} + +/// Upstream zstd parity for mls=6: `ZSTD_hash6` uses +/// `(u << (64-48)).wrapping_mul(PRIME_6_BYTES) >> (64 - hash_log)`. +#[test] +fn hash6_matches_expected_value_on_known_input() { + let table = FastHashTable::new(14, 6); + let data = [0x01u8, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]; + // SAFETY: data has 8 ≥ 6 readable bytes (the implementation + // performs a u64 load and shifts off the unused top bits). + let h = unsafe { table.hash_ptr::<6>(data.as_ptr()) }; + let u = u64::from_le_bytes(data); + let expected = (((u << (64 - 48)).wrapping_mul(227_718_039_650_203u64)) >> (64 - 14)) as u32; + assert_eq!( + h, expected, + "hash6 must match upstream zstd multiply-shift formula" + ); +} + +/// Upstream zstd parity for mls=7: `ZSTD_hash7` shifts by `(64-56)` and +/// multiplies by `PRIME_7_BYTES`. +#[test] +fn hash7_matches_expected_value_on_known_input() { + let table = FastHashTable::new(15, 7); + let data = [0x01u8, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]; + // SAFETY: data has 8 ≥ 7 readable bytes. + let h = unsafe { table.hash_ptr::<7>(data.as_ptr()) }; + let u = u64::from_le_bytes(data); + let expected = (((u << (64 - 56)).wrapping_mul(58_295_818_150_454_627u64)) >> (64 - 15)) as u32; + assert_eq!( + h, expected, + "hash7 must match upstream zstd multiply-shift formula" + ); +} + +/// Upstream zstd parity for mls=8: `ZSTD_hash8` does NOT shift the input +/// (full u64), then multiplies by `PRIME_8_BYTES` (upstream zstd's +/// `prime8bytes = 0xCF1BBCDCB7A56463ULL`). +#[test] +fn hash8_matches_expected_value_on_known_input() { + let table = FastHashTable::new(16, 8); + let data = [0x01u8, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]; + // SAFETY: data has 8 readable bytes. + let h = unsafe { table.hash_ptr::<8>(data.as_ptr()) }; + let u = u64::from_le_bytes(data); + let expected = (u.wrapping_mul(0xCF1BBCDCB7A56463u64) >> (64 - 16)) as u32; + assert_eq!( + h, expected, + "hash8 must match upstream zstd multiply-shift formula" + ); +} + +/// Boundary: `hash_log = 1` is the smallest accepted value +/// (table of 2 entries). Verifies the constructor doesn't reject +/// the minimum and that the table actually allocates. +#[test] +fn hash_log_minimum_one_constructs_two_entry_table() { + let table = FastHashTable::new(1, 4); + let data = [0u8, 0, 0, 0]; + // SAFETY: 4 readable bytes; hash output occupies 1 bit so + // hash ∈ {0, 1}. + let h = unsafe { table.hash_ptr::<4>(data.as_ptr()) }; + assert!(h < 2, "hash_log=1 must produce values ∈ {{0, 1}} (got {h})"); +} + +/// Boundary: `hash_log = ZSTD_HASHLOG_MAX` (30) is the largest +/// accepted value. Calling [`FastHashTable::new`] with this value +/// would allocate ≈4 GiB (`1 << 30` × `sizeof(u32)`), well over +/// per-test memory budgets on CI runners — instead exercise the +/// same accept path via the non-allocating [`validate_params`] +/// helper. Pairs with the `should_panic` tests below that prove +/// rejection of the out-of-band cases. +#[test] +fn hash_log_maximum_thirty_is_accepted_by_validation() { + validate_params(ZSTD_HASHLOG_MAX, 4); + validate_params(ZSTD_HASHLOG_MAX, 8); +} + +#[test] +#[should_panic(expected = "hash_log must be in 1..=")] +fn panics_on_zero_hash_log() { + let _ = FastHashTable::new(0, 4); +} + +#[test] +#[should_panic(expected = "hash_log must be in 1..=")] +fn panics_on_hash_log_above_zstd_hashlog_max() { + // 31 > ZSTD_HASHLOG_MAX (30) → constructor rejects. + let _ = FastHashTable::new(31, 4); +} + +#[test] +#[should_panic(expected = "ZSTD Fast strategy only supports mls 4..=8")] +fn panics_on_mls_below_four() { + let _ = FastHashTable::new(12, 3); +} + +#[test] +#[should_panic(expected = "ZSTD Fast strategy only supports mls 4..=8")] +fn panics_on_mls_above_eight() { + let _ = FastHashTable::new(12, 9); +} diff --git a/zstd/src/encoding/simple/fast_kernel/kernel.rs b/zstd/src/encoding/simple/fast_kernel/kernel.rs index f6fb29f81..43f15fa7a 100644 --- a/zstd/src/encoding/simple/fast_kernel/kernel.rs +++ b/zstd/src/encoding/simple/fast_kernel/kernel.rs @@ -1967,722 +1967,4 @@ pub(crate) fn compress_block_fast_dict_borrowed` - /// lifetimes (a `Sequence` borrow lives only as long as the - /// closure scope; cloning the literal bytes into the tuple - /// detaches the capture from that lifetime). - fn run_block( - data: &[u8], - hash_log: u32, - mls: u32, - ) -> (Vec<(Vec, usize, usize)>, FastBlockResult) { - let mut table = FastHashTable::new(hash_log, mls); - let mut tuples: Vec<(Vec, usize, usize)> = Vec::new(); - let mut handle = |seq: Sequence<'_>| match seq { - Sequence::Triple { - literals, - offset, - match_len, - } => { - tuples.push((literals.to_vec(), offset, match_len)); - } - Sequence::Literals { literals } => { - tuples.push((literals.to_vec(), 0, 0)); - } - }; - let result = match mls { - 4 => compress_block_fast::<4, false>( - data, - 0, - PrefixBounds { - // Match production contract: - // `prefix_start_index >= 1` rejects the hash table - // empty-slot value `0` so a fresh-table probe - // cannot be mistaken for a position-0 match (the - // sentinel-1 floor documented on FastKernelMatcher). - prefix_start_index: 1, - window_low: 0, - }, - &mut table, - [0, 0], - 2, - &mut handle, - ), - 5 => compress_block_fast::<5, false>( - data, - 0, - PrefixBounds { - // Match production contract: - // `prefix_start_index >= 1` rejects the hash table - // empty-slot value `0` so a fresh-table probe - // cannot be mistaken for a position-0 match (the - // sentinel-1 floor documented on FastKernelMatcher). - prefix_start_index: 1, - window_low: 0, - }, - &mut table, - [0, 0], - 2, - &mut handle, - ), - _ => panic!("test helper only supports mls=4 and mls=5"), - }; - // Accounting invariant: literals + matches + tail == input. - let acct: usize = tuples - .iter() - .map(|(lits, _off, mlen)| lits.len() + mlen) - .sum::() - + result.tail_literals_len; - assert_eq!(acct, data.len(), "kernel must account for every input byte",); - (tuples, result) - } - - /// Tail-too-small case: input ≤ HASH_READ_SIZE produces zero - /// sequence emissions; the kernel reports the whole block as - /// `tail_literals_len` and the caller is expected to wrap it in - /// the terminal `Sequence::Literals`. - #[test] - fn short_input_reports_tail_without_emission() { - let data = [1u8, 2, 3, 4, 5]; - let (tuples, result) = run_block(&data, 8, 4); - assert!( - tuples.is_empty(), - "kernel must NOT emit sequences for short inputs (got {tuples:?})", - ); - assert_eq!(result.tail_literals_len, data.len()); - } - - /// Repeated pattern with a clear long match — the kernel should - /// detect it and emit at least one Triple. Verifies via the - /// captured tuples that an actual match was produced (`match_len - /// >= MIN_MATCH=4`, non-zero offset). - #[test] - fn finds_long_repeat_in_simple_pattern() { - let mut data = Vec::new(); - data.extend_from_slice(b"ABCDEFGHIJKLMNOP"); - data.extend_from_slice(b"ABCDEFGHIJKLMNOP"); - // Need ≥ 8 trailing bytes past the last match position so - // `ilimit = data.len() - HASH_READ_SIZE` keeps the inner - // loop active long enough to scan the repeated second half. - // Pad with distinct bytes to keep the kernel out of any - // extra repcode branches. - data.extend_from_slice(b"________"); - let (tuples, _result) = run_block(&data, 12, 4); - let triple = tuples - .iter() - .find(|(_, _, m)| *m > 0) - .expect("kernel must emit at least one Triple for the repeated half"); - assert!( - triple.2 >= 4, - "match_len must be ≥ MIN_MATCH=4 (got {})", - triple.2, - ); - assert!( - triple.1 > 0, - "explicit-offset match must have offset > 0 (got {})", - triple.1, - ); - } - - /// Helper that accepts a non-zero `rep` and pre-populated hash - /// table so individual tests can exercise specific kernel branches - /// (rep path, prefix filter, stale-entry hardening). Shares the - /// same accounting invariant as `run_block` plus returns the - /// captured tuples for behavioural assertions. - fn run_block_with_rep( - data: &[u8], - hash_log: u32, - rep: [u32; 2], - ) -> (Vec<(Vec, usize, usize)>, FastBlockResult) { - let mut table = FastHashTable::new(hash_log, 4); - let mut tuples: Vec<(Vec, usize, usize)> = Vec::new(); - let mut handle = |seq: Sequence<'_>| match seq { - Sequence::Triple { - literals, - offset, - match_len, - } => tuples.push((literals.to_vec(), offset, match_len)), - Sequence::Literals { literals } => tuples.push((literals.to_vec(), 0, 0)), - }; - let result = compress_block_fast::<4, false>( - data, - 0, - PrefixBounds { - // Match production contract: - // `prefix_start_index >= 1` rejects the hash table - // empty-slot value `0`. - prefix_start_index: 1, - window_low: 0, - }, - &mut table, - rep, - 2, - &mut handle, - ); - let acct: usize = tuples - .iter() - .map(|(lits, _off, mlen)| lits.len() + mlen) - .sum::() - + result.tail_literals_len; - assert_eq!(acct, data.len(), "kernel must account for every input byte"); - (tuples, result) - } - - /// Repcode path: uniform data + `rep[0] = 1` means every 4-byte - /// window at any `ip0 > 0` matches `data[ip0-1..ip0+3]`. The - /// kernel must emit a Triple with `offset == 1` and large - /// `match_len`. Hits the `rep_check` branch on the very first - /// loop iteration. - #[test] - fn repcode_match_emits_with_rep_offset_one() { - let data = vec![0x42u8; 64]; - let (tuples, _) = run_block_with_rep(&data, 8, [1, 4]); - let rep_triple = tuples - .iter() - .find(|(_, off, m)| *off == 1 && *m > 0) - .unwrap_or_else(|| panic!("repcode Triple at offset=1 expected, got {tuples:?}")); - assert!( - rep_triple.2 >= 4, - "match_len must be ≥ MIN_MATCH=4 (got {})", - rep_triple.2, - ); - // Uniform-buffer rep match should extend far — the first match - // covers nearly the whole tail after subtracting the initial - // literal byte and the HASH_READ_SIZE trailing cap. Assert a - // reasonable lower bound rather than an exact value (count - // logic chooses chunk boundaries deterministically but the - // chunk count depends on the LE/BE branch). - assert!( - rep_triple.2 >= 32, - "uniform-byte rep extension must consume most of the buffer, got {}", - rep_triple.2, - ); - } - - /// Explicit-match backward extension: a marker byte before the - /// repeated pattern lets the kernel walk the match back by one - /// byte once the 4-byte forward probe at the hashed position - /// fires. - /// - /// Layout: `"X"` literal at 0, then `AAAA` 4-byte block at 1..5, - /// distinct filler, then `"X"` + `AAAA` again starting at 10. The - /// kernel hashes the second `AAAA` at ip0=11 (or wherever step - /// lands close to it), reads the stored index of the first - /// `AAAA`, and the backward-extension while-loop walks back - /// because `data[ip0 - 1] == data[match_pos - 1] == 'X'`. - #[test] - fn explicit_match_backward_extension_extends_by_marker_byte() { - // Engineered so the FIRST emitted match deterministically - // backward-extends through a marker byte: - // - // [0..15] distinct prefix (no 'Z', no 'A') → table - // writebacks here can't byte-match later AAAA - // [15] 'Z' marker (first copy) - // [16..24] 'AAAAAAAA' (first AAAA copy — table[hash("AAAA")] - // gets written = 16 when ip0 reaches here) - // [24..32] distinct filler (no 'Z', no 'A') - // [32] 'Z' marker (second copy) - // [33..41] 'AAAAAAAA' (second AAAA copy — kernel matches - // this against index 16; backward extension - // walks back because data[32]='Z'==data[15]='Z') - // [41..] HASH_READ_SIZE tail - let mut data: Vec = (0..15u8).collect(); - data.push(b'Z'); - data.extend_from_slice(b"AAAAAAAA"); - for i in 0..8u8 { - data.push(0x80 + i); - } - data.push(b'Z'); - data.extend_from_slice(b"AAAAAAAA"); - for i in 0..16u8 { - data.push(0x40 + (i % 16)); - } - let (tuples, _) = run_block_with_rep(&data, 12, [0, 0]); - let triple = tuples - .iter() - .find(|(_, _, m)| *m > 0) - .unwrap_or_else(|| panic!("expected an explicit-match Triple, got {tuples:?}")); - // Backward extension must lift match_len above MIN_MATCH=4 — - // the 'Z' marker at position 32 (matching the 'Z' at 15) is - // absorbed by the backward walk. - assert!( - triple.2 >= 5, - "expected match_len ≥ 5 from backward extension (got {})", - triple.2, - ); - // Literals before the emit must NOT end with 'Z' — backward - // extension consumed the marker. - assert!( - !triple.0.ends_with(b"Z"), - "backward extension must consume the 'Z' marker (literals = {:?})", - triple.0, - ); - } - - /// `prefix_start_index` filter: a stale hash entry pointing at a - /// position BELOW `prefix_start_index` must be rejected even when - /// the byte-for-byte cmp would have succeeded. Engineered by - /// pre-populating the table with an in-range-by-bytes but - /// below-prefix index. - #[test] - fn prefix_start_index_filter_rejects_below_window() { - // Uniform data — every 4-byte window has the same hash and - // the same bytes, so a stale entry at any position would - // raw-cmp-match. Pre-set the hash slot for ip0=1 to index 0, - // then run with prefix_start_index=5. Without the filter the - // kernel would happily emit a Triple at offset=1; with it, - // the candidate is rejected. - let data = vec![0xAAu8; 64]; - let mut table = FastHashTable::new(8, 4); - // SAFETY: data has ≥ 4 readable bytes at index 1. - let h = unsafe { table.hash_ptr::<4>(data.as_ptr().add(1)) }; - // SAFETY: h came from hash_ptr on this same table. - unsafe { table.put(h, 0) }; - - let mut tuples: Vec<(Vec, usize, usize)> = Vec::new(); - let mut handle = |seq: Sequence<'_>| match seq { - Sequence::Triple { - literals, - offset, - match_len, - } => tuples.push((literals.to_vec(), offset, match_len)), - Sequence::Literals { literals } => tuples.push((literals.to_vec(), 0, 0)), - }; - // prefix_start_index=5 blocks index 0. - let _ = compress_block_fast::<4, false>( - &data, - 0, - PrefixBounds { - prefix_start_index: 5, - window_low: 5, - }, - &mut table, - [0, 0], - 2, - &mut handle, - ); - - // Walk emitted sequences in order, tracking the running - // `anchor` cursor (which equals the start of the current - // emit's literal-run). For each Triple the match begins at - // `match_start = anchor + lits.len()` and references - // `match_start - offset`; that source position MUST be at or - // above `prefix_start_index = 5`. The simpler `off <= ip0` - // form fails for the second+ Triple — `lits.len()` only - // equals `ip0` for the first emit (when anchor still sits at - // block_start=0); a single-byte tracker keeps the bound - // correct across multiple emits. - let mut anchor: usize = 0; - for (lits, off, m) in &tuples { - if *m > 0 { - // The real correctness check is `match_src >= - // prefix_start_index` below — the `offset != 1` - // form is too cadence-specific (4-cursor body's - // double writeback per iter can land an offset=1 - // emit whose SOURCE is still ≥ prefix_start_index). - let match_start = anchor + lits.len(); - let match_src = match_start - .checked_sub(*off) - .expect("offset must not exceed match_start (would wrap)"); - assert!( - match_src >= 5, - "match source {match_src} below prefix_start_index=5 \ - (match_start={match_start}, offset={off})", - ); - anchor = match_start + m; - } else { - // Pure-literals callback (currently never emitted by - // the kernel — kept defensive for future contract - // changes): advance anchor by the literal run length. - anchor += lits.len(); - } - } - } - - /// Hardening regression (round 3, finding #11): a hash entry - /// pointing AT or AFTER the current `ip0` must be rejected - /// before the 4-byte raw compare. Without this guard the kernel - /// would compute `offset = ip0 - match_pos` and wrap into a - /// gigantic offset → emit a Triple with a meaningless backward - /// reference. - /// - /// Stale hash entries below `prefix_start_index` must be rejected - /// by the upstream zstd-parity prefix filter in `match_found`. Engineered - /// scenario: pre-populate the hash slot for ip0 with a low stale - /// index (5) that points into the supposedly-out-of-window region; - /// run with `prefix_start_index = 50` so the kernel must skip - /// that candidate. The kernel's own writeback at the iteration - /// start would still leave the stale value usable if the prefix - /// filter didn't fire — uniform data ensures any survived - /// candidate would emit a non-zero match. - #[test] - fn match_found_rejects_stale_entry_below_prefix_floor() { - let data = vec![0u8; 200]; - let mut table = FastHashTable::new(8, 4); - // Force the explicit-match probe at ip0=50 (first iter once - // ip0 is bumped from prefix_start_index=50) to see the stale - // index 5. - // SAFETY: data has ≥ 4 readable bytes at index 50. - let h = unsafe { table.hash_ptr::<4>(data.as_ptr().add(50)) }; - // SAFETY: h came from hash_ptr on this same table. - unsafe { table.put(h, 5) }; - - let mut tuples: Vec<(Vec, usize, usize)> = Vec::new(); - let mut handle = |seq: Sequence<'_>| match seq { - Sequence::Triple { - literals, - offset, - match_len, - } => tuples.push((literals.to_vec(), offset, match_len)), - Sequence::Literals { literals } => tuples.push((literals.to_vec(), 0, 0)), - }; - // prefix_start_index = 50 — match_idx=5 is below the floor and - // must be rejected by the upstream zstd-parity prefix filter in - // `match_found`. - let _ = compress_block_fast::<4, false>( - &data, - 50, - PrefixBounds { - prefix_start_index: 50, - window_low: 50, - }, - &mut table, - [0, 0], - 2, - &mut handle, - ); - - // Either zero emissions (stale rejected, no other match found - // in the limited scan window) or a Triple whose offset - // references a position >= prefix_start_index = 50, never a - // 1-byte-from-stale-5 offset. - for (_, off, m) in &tuples { - if *m > 0 { - assert!( - *off > 0 && *off <= data.len(), - "every emitted offset must reference an in-buffer backward position (got {off})", - ); - } - } - } - - /// Input exactly `HASH_READ_SIZE` bytes long: the short-input - /// branch fires because `data.len() < block_start + HASH_READ_SIZE` - /// is `8 < 0 + 8` → false, so we enter the main loop, but - /// `ilimit = 8 - 8 = 0` makes `while ip0 < ilimit` zero-iteration - /// (ip0 starts at 1 ≥ 0). Result: zero emissions, entire input - /// reported as tail. - #[test] - fn block_exactly_hash_read_size_emits_no_sequences() { - let data = [1u8, 2, 3, 4, 5, 6, 7, 8]; - let (tuples, result) = run_block_with_rep(&data, 8, [0, 0]); - assert!( - tuples.is_empty(), - "exactly HASH_READ_SIZE bytes must produce no main-loop iterations", - ); - assert_eq!(result.tail_literals_len, data.len()); - } - - /// Input one byte shorter than `HASH_READ_SIZE`: the short-input - /// branch fires (`7 < 8`), the kernel returns immediately with - /// the full input as tail and no callback invocations. - #[test] - fn block_just_below_hash_read_size_emits_no_sequences() { - let data = [1u8, 2, 3, 4, 5, 6, 7]; - let (tuples, result) = run_block_with_rep(&data, 8, [0, 0]); - assert!(tuples.is_empty()); - assert_eq!(result.tail_literals_len, data.len()); - } - - /// Repcode save/restore: when the incoming `rep_offset1` is - /// larger than the addressable history (`max_rep = ip0 - - /// prefix_start_index`), the kernel stashes it into - /// `offset_saved1` and zeroes the live rep. If no explicit match - /// promotes a new rep during the block, `_cleanup` must restore - /// the saved value into the returned `rep[0]` so cross-block - /// repcode history isn't lost. The unaffected `rep[1]` is the - /// secondary witness that no mutation occurred mid-block. - #[test] - fn rep_offset_save_restore_when_out_of_range() { - // Random-looking distinct bytes — no real matches the kernel - // would discover; deterministic xorshift keeps the stream - // reproducible. - let mut data = vec![0u8; 64]; - let mut state = 0x1234_5678u32; - for byte in &mut data { - state ^= state << 13; - state ^= state >> 17; - state ^= state << 5; - *byte = state as u8; - } - // rep_offset1 huge — far exceeds any plausible ip0 in a - // 64-byte block. Must be stashed and restored unchanged. - let huge = 9999; - let mut table = FastHashTable::new(10, 4); - let mut tuples: Vec<(Vec, usize, usize)> = Vec::new(); - let mut handle = |seq: Sequence<'_>| match seq { - Sequence::Triple { - literals, - offset, - match_len, - } => tuples.push((literals.to_vec(), offset, match_len)), - Sequence::Literals { literals } => tuples.push((literals.to_vec(), 0, 0)), - }; - let result = compress_block_fast::<4, false>( - &data, - 0, - PrefixBounds { - // Match production contract: - // `prefix_start_index >= 1` rejects the hash table - // empty-slot value `0`. - prefix_start_index: 1, - window_low: 0, - }, - &mut table, - [huge, 7], - 2, - &mut handle, - ); - assert_eq!( - result.rep[0], huge, - "out-of-range rep_offset1 must be restored verbatim across the block", - ); - // rep_offset2 was also out of range (max_rep ≈ 0..63, 7 > 1). - // Upstream zstd restores it through offset_saved2; the in-range - // restoration path is the second witness. - assert_eq!(result.rep[1], 7, "rep_offset2 (also stashed) must restore"); - } - - /// cmov variant: same correctness contract as branch variant — - /// produces identical output for the same input, just lowers - /// match_idx >= prefix_start_index to a cmov instead of a - /// branch. Run a known-good fixture through both and assert - /// byte-for-byte equality of the emitted Triple stream. - #[test] - fn cmov_variant_matches_branch_variant_output() { - let mut data = alloc::vec::Vec::new(); - for i in 0..512u32 { - data.push((i & 0xFF) as u8); - } - // Repeat the first 64 bytes near the end so the kernel - // emits at least one explicit match Triple. - let tail = data[0..64].to_vec(); - data.extend_from_slice(&tail); - - let collect = |use_cmov: bool| -> alloc::vec::Vec<(alloc::vec::Vec, usize, usize)> { - let mut table = FastHashTable::new(12, 4); - let mut tuples = alloc::vec::Vec::new(); - let mut handle = |seq: Sequence<'_>| match seq { - Sequence::Triple { - literals, - offset, - match_len, - } => { - tuples.push((literals.to_vec(), offset, match_len)); - } - Sequence::Literals { literals } => { - tuples.push((literals.to_vec(), 0, 0)); - } - }; - if use_cmov { - let _ = compress_block_fast::<4, true>( - &data, - 0, - PrefixBounds { - // Match production contract: - // `prefix_start_index >= 1` rejects the hash - // table empty-slot value `0`. - prefix_start_index: 1, - window_low: 0, - }, - &mut table, - [0, 0], - 2, - &mut handle, - ); - } else { - let _ = compress_block_fast::<4, false>( - &data, - 0, - PrefixBounds { - // Match production contract: - // `prefix_start_index >= 1` rejects the hash - // table empty-slot value `0`. - prefix_start_index: 1, - window_low: 0, - }, - &mut table, - [0, 0], - 2, - &mut handle, - ); - } - tuples - }; - - let out_branch = collect(false); - let out_cmov = collect(true); - assert_eq!( - out_branch, out_cmov, - "cmov and branch variants must emit identical sequences" - ); - } - - /// Regression test for Copilot review thread on PR #219 — cmov - /// variant must NOT report a match when `match_idx < - /// prefix_start_index` even if the 4 bytes at `ip` happen to - /// equal `CMOV_DUMMY`. Without the explicit `in_range` - /// predicate the cmov path returns `true` here, producing an - /// out-of-window match the kernel would then encode with a - /// bogus offset. - #[test] - fn cmov_variant_rejects_out_of_window_when_ip_equals_dummy() { - // Layout (32 bytes total): - // data[0..4] = filler (not CMOV_DUMMY, won't accidentally match) - // data[4..] = CMOV_DUMMY bytes at position 16, so read32(ip) - // at ip_pos=16 equals read32(CMOV_DUMMY). - // - // match_idx=4 is below prefix_start=10 (out of window). - // ip_pos=16 satisfies `ip == base.add(ip_pos)`. - let mut data: alloc::vec::Vec = alloc::vec![0xAA; 32]; - data[16] = 0x12; - data[17] = 0x34; - data[18] = 0x56; - data[19] = 0x78; - // SAFETY (test fixture): ip = base + 16; both buffers cover - // ≥ 4 readable bytes (data.len()=32 ≥ 16+4 and CMOV_DUMMY is - // 4 bytes by construction). - let base = data.as_ptr(); - let ip_pos = 16usize; - let ip = unsafe { base.add(ip_pos) }; - let branch_result = unsafe { match_found::(ip, base, 4, 10) }; - assert!( - !branch_result, - "branch variant must reject out-of-window match_idx" - ); - let cmov_result = unsafe { match_found::(ip, base, 4, 10) }; - assert!( - !cmov_result, - "cmov variant must reject out-of-window match_idx even when \ - ip bytes coincide with CMOV_DUMMY", - ); - } - - /// Drive the borrowed dual-base dict kernel directly (Scalar tier — always - /// available, deterministic) over a crafted `(dict, input)` pair that - /// exercises the dict-match (2-segment + backward extension), input-match, - /// repcode and tail-literal branches. Reconstruct against the logical - /// `[dict][input]` window to prove every emitted offset is valid, and check - /// the byte-accounting invariant. - #[test] - fn borrowed_dict_kernel_reconstructs_via_dual_base() { - use crate::encoding::fastpath::FastpathKernel; - - let hash_log = 12u32; - const MLS: u32 = 4; - // Distinct dict bytes so each 4-byte key hashes uniquely; the input - // both matches the dict prefix (dict match) and repeats itself - // (input match + repcode), then ends in a literal tail. - let dict: Vec = (0u8..40).collect(); - let mut inp: Vec = Vec::new(); - inp.extend_from_slice(&dict[0..20]); // dict match against dict[0..] - inp.extend_from_slice(&dict[0..20]); // input match against the first copy - inp.extend_from_slice(&dict[4..24]); // dict match at a non-zero dict pos - inp.extend_from_slice(b"tail-literals-xyz"); // terminal literals - - let dict_end = dict.len(); - - // main table (filled during the scan) + immutable dict table primed - // over every hashable dict position. - let mut main_table = FastHashTable::new(hash_log, MLS); - let mut dict_table = FastHashTable::new(hash_log, MLS); - for pos in 0..=dict.len() - HASH_READ_SIZE { - // SAFETY: pos + 8 <= dict.len(); MLS matches the table. Tagged - // every-position last-wins fill, matching the kernel's tagged dict - // lookup (slot + packed tag). - let hat = - unsafe { hash_ptr_raw::(dict.as_ptr().add(pos), hash_log + DICT_TAG_BITS) }; - unsafe { - dict_table.put( - hat >> DICT_TAG_BITS, - ((pos as u32) << DICT_TAG_BITS) | (hat & DICT_TAG_MASK), - ) - }; - } - - let mut tuples: Vec<(Vec, usize, usize)> = Vec::new(); - let mut handle = |seq: Sequence<'_>| match seq { - Sequence::Triple { - literals, - offset, - match_len, - } => tuples.push((literals.to_vec(), offset, match_len)), - Sequence::Literals { literals } => tuples.push((literals.to_vec(), 0, 0)), - }; - - let result = compress_block_fast_dict_borrowed::( - &inp, - &dict, - 0, - inp.len(), - &mut main_table, - &dict_table, - PrefixBounds { - prefix_start_index: 1, - window_low: 0, - }, - [0, 0], - 2, - &mut handle, - FastpathKernel::Scalar, - ); - - // Reconstruct against the logical [dict][input] window: start from the - // dictionary, then replay each sequence. A Triple's offset references - // `current_len - offset` in this combined buffer, so a valid dual-base - // offset reproduces the original input exactly. - let mut window = dict.clone(); - let mut saw_dict_region_match = false; - for (literals, offset, match_len) in &tuples { - window.extend_from_slice(literals); - if *match_len > 0 { - let start = window.len() - offset; - // A match whose source lands in the dictionary prefix exercised - // the dual-base / 2-segment path (not a plain input back-ref). - if start < dict_end { - saw_dict_region_match = true; - } - for i in 0..*match_len { - let b = window[start + i]; - window.push(b); - } - } - } - // Append the terminal tail the kernel reports separately. - let tail_start = inp.len() - result.tail_literals_len; - window.extend_from_slice(&inp[tail_start..]); - - assert_eq!( - &window[dict_end..], - &inp[..], - "borrowed dict kernel must reconstruct the input from the [dict][input] window", - ); - - assert!( - tuples.iter().any(|(_, _, m)| *m >= 4), - "expected at least one match Triple", - ); - assert!( - saw_dict_region_match, - "expected at least one match reading from the dictionary region (dual-base path)", - ); - } -} +mod tests; diff --git a/zstd/src/encoding/simple/fast_kernel/kernel/tests.rs b/zstd/src/encoding/simple/fast_kernel/kernel/tests.rs new file mode 100644 index 000000000..fa0de32e8 --- /dev/null +++ b/zstd/src/encoding/simple/fast_kernel/kernel/tests.rs @@ -0,0 +1,716 @@ +use super::*; +use alloc::vec; +use alloc::vec::Vec; + +/// Capture every emitted sequence as `(literals_bytes, offset, +/// match_len)` plus the final `FastBlockResult` so each test can +/// assert byte-level accounting and the actual match decisions +/// without fighting the borrow checker over `Sequence<'_>` +/// lifetimes (a `Sequence` borrow lives only as long as the +/// closure scope; cloning the literal bytes into the tuple +/// detaches the capture from that lifetime). +fn run_block( + data: &[u8], + hash_log: u32, + mls: u32, +) -> (Vec<(Vec, usize, usize)>, FastBlockResult) { + let mut table = FastHashTable::new(hash_log, mls); + let mut tuples: Vec<(Vec, usize, usize)> = Vec::new(); + let mut handle = |seq: Sequence<'_>| match seq { + Sequence::Triple { + literals, + offset, + match_len, + } => { + tuples.push((literals.to_vec(), offset, match_len)); + } + Sequence::Literals { literals } => { + tuples.push((literals.to_vec(), 0, 0)); + } + }; + let result = match mls { + 4 => compress_block_fast::<4, false>( + data, + 0, + PrefixBounds { + // Match production contract: + // `prefix_start_index >= 1` rejects the hash table + // empty-slot value `0` so a fresh-table probe + // cannot be mistaken for a position-0 match (the + // sentinel-1 floor documented on FastKernelMatcher). + prefix_start_index: 1, + window_low: 0, + }, + &mut table, + [0, 0], + 2, + &mut handle, + ), + 5 => compress_block_fast::<5, false>( + data, + 0, + PrefixBounds { + // Match production contract: + // `prefix_start_index >= 1` rejects the hash table + // empty-slot value `0` so a fresh-table probe + // cannot be mistaken for a position-0 match (the + // sentinel-1 floor documented on FastKernelMatcher). + prefix_start_index: 1, + window_low: 0, + }, + &mut table, + [0, 0], + 2, + &mut handle, + ), + _ => panic!("test helper only supports mls=4 and mls=5"), + }; + // Accounting invariant: literals + matches + tail == input. + let acct: usize = tuples + .iter() + .map(|(lits, _off, mlen)| lits.len() + mlen) + .sum::() + + result.tail_literals_len; + assert_eq!(acct, data.len(), "kernel must account for every input byte",); + (tuples, result) +} + +/// Tail-too-small case: input ≤ HASH_READ_SIZE produces zero +/// sequence emissions; the kernel reports the whole block as +/// `tail_literals_len` and the caller is expected to wrap it in +/// the terminal `Sequence::Literals`. +#[test] +fn short_input_reports_tail_without_emission() { + let data = [1u8, 2, 3, 4, 5]; + let (tuples, result) = run_block(&data, 8, 4); + assert!( + tuples.is_empty(), + "kernel must NOT emit sequences for short inputs (got {tuples:?})", + ); + assert_eq!(result.tail_literals_len, data.len()); +} + +/// Repeated pattern with a clear long match — the kernel should +/// detect it and emit at least one Triple. Verifies via the +/// captured tuples that an actual match was produced (`match_len +/// >= MIN_MATCH=4`, non-zero offset). +#[test] +fn finds_long_repeat_in_simple_pattern() { + let mut data = Vec::new(); + data.extend_from_slice(b"ABCDEFGHIJKLMNOP"); + data.extend_from_slice(b"ABCDEFGHIJKLMNOP"); + // Need ≥ 8 trailing bytes past the last match position so + // `ilimit = data.len() - HASH_READ_SIZE` keeps the inner + // loop active long enough to scan the repeated second half. + // Pad with distinct bytes to keep the kernel out of any + // extra repcode branches. + data.extend_from_slice(b"________"); + let (tuples, _result) = run_block(&data, 12, 4); + let triple = tuples + .iter() + .find(|(_, _, m)| *m > 0) + .expect("kernel must emit at least one Triple for the repeated half"); + assert!( + triple.2 >= 4, + "match_len must be ≥ MIN_MATCH=4 (got {})", + triple.2, + ); + assert!( + triple.1 > 0, + "explicit-offset match must have offset > 0 (got {})", + triple.1, + ); +} + +/// Helper that accepts a non-zero `rep` and pre-populated hash +/// table so individual tests can exercise specific kernel branches +/// (rep path, prefix filter, stale-entry hardening). Shares the +/// same accounting invariant as `run_block` plus returns the +/// captured tuples for behavioural assertions. +fn run_block_with_rep( + data: &[u8], + hash_log: u32, + rep: [u32; 2], +) -> (Vec<(Vec, usize, usize)>, FastBlockResult) { + let mut table = FastHashTable::new(hash_log, 4); + let mut tuples: Vec<(Vec, usize, usize)> = Vec::new(); + let mut handle = |seq: Sequence<'_>| match seq { + Sequence::Triple { + literals, + offset, + match_len, + } => tuples.push((literals.to_vec(), offset, match_len)), + Sequence::Literals { literals } => tuples.push((literals.to_vec(), 0, 0)), + }; + let result = compress_block_fast::<4, false>( + data, + 0, + PrefixBounds { + // Match production contract: + // `prefix_start_index >= 1` rejects the hash table + // empty-slot value `0`. + prefix_start_index: 1, + window_low: 0, + }, + &mut table, + rep, + 2, + &mut handle, + ); + let acct: usize = tuples + .iter() + .map(|(lits, _off, mlen)| lits.len() + mlen) + .sum::() + + result.tail_literals_len; + assert_eq!(acct, data.len(), "kernel must account for every input byte"); + (tuples, result) +} + +/// Repcode path: uniform data + `rep[0] = 1` means every 4-byte +/// window at any `ip0 > 0` matches `data[ip0-1..ip0+3]`. The +/// kernel must emit a Triple with `offset == 1` and large +/// `match_len`. Hits the `rep_check` branch on the very first +/// loop iteration. +#[test] +fn repcode_match_emits_with_rep_offset_one() { + let data = vec![0x42u8; 64]; + let (tuples, _) = run_block_with_rep(&data, 8, [1, 4]); + let rep_triple = tuples + .iter() + .find(|(_, off, m)| *off == 1 && *m > 0) + .unwrap_or_else(|| panic!("repcode Triple at offset=1 expected, got {tuples:?}")); + assert!( + rep_triple.2 >= 4, + "match_len must be ≥ MIN_MATCH=4 (got {})", + rep_triple.2, + ); + // Uniform-buffer rep match should extend far — the first match + // covers nearly the whole tail after subtracting the initial + // literal byte and the HASH_READ_SIZE trailing cap. Assert a + // reasonable lower bound rather than an exact value (count + // logic chooses chunk boundaries deterministically but the + // chunk count depends on the LE/BE branch). + assert!( + rep_triple.2 >= 32, + "uniform-byte rep extension must consume most of the buffer, got {}", + rep_triple.2, + ); +} + +/// Explicit-match backward extension: a marker byte before the +/// repeated pattern lets the kernel walk the match back by one +/// byte once the 4-byte forward probe at the hashed position +/// fires. +/// +/// Layout: `"X"` literal at 0, then `AAAA` 4-byte block at 1..5, +/// distinct filler, then `"X"` + `AAAA` again starting at 10. The +/// kernel hashes the second `AAAA` at ip0=11 (or wherever step +/// lands close to it), reads the stored index of the first +/// `AAAA`, and the backward-extension while-loop walks back +/// because `data[ip0 - 1] == data[match_pos - 1] == 'X'`. +#[test] +fn explicit_match_backward_extension_extends_by_marker_byte() { + // Engineered so the FIRST emitted match deterministically + // backward-extends through a marker byte: + // + // [0..15] distinct prefix (no 'Z', no 'A') → table + // writebacks here can't byte-match later AAAA + // [15] 'Z' marker (first copy) + // [16..24] 'AAAAAAAA' (first AAAA copy — table[hash("AAAA")] + // gets written = 16 when ip0 reaches here) + // [24..32] distinct filler (no 'Z', no 'A') + // [32] 'Z' marker (second copy) + // [33..41] 'AAAAAAAA' (second AAAA copy — kernel matches + // this against index 16; backward extension + // walks back because data[32]='Z'==data[15]='Z') + // [41..] HASH_READ_SIZE tail + let mut data: Vec = (0..15u8).collect(); + data.push(b'Z'); + data.extend_from_slice(b"AAAAAAAA"); + for i in 0..8u8 { + data.push(0x80 + i); + } + data.push(b'Z'); + data.extend_from_slice(b"AAAAAAAA"); + for i in 0..16u8 { + data.push(0x40 + (i % 16)); + } + let (tuples, _) = run_block_with_rep(&data, 12, [0, 0]); + let triple = tuples + .iter() + .find(|(_, _, m)| *m > 0) + .unwrap_or_else(|| panic!("expected an explicit-match Triple, got {tuples:?}")); + // Backward extension must lift match_len above MIN_MATCH=4 — + // the 'Z' marker at position 32 (matching the 'Z' at 15) is + // absorbed by the backward walk. + assert!( + triple.2 >= 5, + "expected match_len ≥ 5 from backward extension (got {})", + triple.2, + ); + // Literals before the emit must NOT end with 'Z' — backward + // extension consumed the marker. + assert!( + !triple.0.ends_with(b"Z"), + "backward extension must consume the 'Z' marker (literals = {:?})", + triple.0, + ); +} + +/// `prefix_start_index` filter: a stale hash entry pointing at a +/// position BELOW `prefix_start_index` must be rejected even when +/// the byte-for-byte cmp would have succeeded. Engineered by +/// pre-populating the table with an in-range-by-bytes but +/// below-prefix index. +#[test] +fn prefix_start_index_filter_rejects_below_window() { + // Uniform data — every 4-byte window has the same hash and + // the same bytes, so a stale entry at any position would + // raw-cmp-match. Pre-set the hash slot for ip0=1 to index 0, + // then run with prefix_start_index=5. Without the filter the + // kernel would happily emit a Triple at offset=1; with it, + // the candidate is rejected. + let data = vec![0xAAu8; 64]; + let mut table = FastHashTable::new(8, 4); + // SAFETY: data has ≥ 4 readable bytes at index 1. + let h = unsafe { table.hash_ptr::<4>(data.as_ptr().add(1)) }; + // SAFETY: h came from hash_ptr on this same table. + unsafe { table.put(h, 0) }; + + let mut tuples: Vec<(Vec, usize, usize)> = Vec::new(); + let mut handle = |seq: Sequence<'_>| match seq { + Sequence::Triple { + literals, + offset, + match_len, + } => tuples.push((literals.to_vec(), offset, match_len)), + Sequence::Literals { literals } => tuples.push((literals.to_vec(), 0, 0)), + }; + // prefix_start_index=5 blocks index 0. + let _ = compress_block_fast::<4, false>( + &data, + 0, + PrefixBounds { + prefix_start_index: 5, + window_low: 5, + }, + &mut table, + [0, 0], + 2, + &mut handle, + ); + + // Walk emitted sequences in order, tracking the running + // `anchor` cursor (which equals the start of the current + // emit's literal-run). For each Triple the match begins at + // `match_start = anchor + lits.len()` and references + // `match_start - offset`; that source position MUST be at or + // above `prefix_start_index = 5`. The simpler `off <= ip0` + // form fails for the second+ Triple — `lits.len()` only + // equals `ip0` for the first emit (when anchor still sits at + // block_start=0); a single-byte tracker keeps the bound + // correct across multiple emits. + let mut anchor: usize = 0; + for (lits, off, m) in &tuples { + if *m > 0 { + // The real correctness check is `match_src >= + // prefix_start_index` below — the `offset != 1` + // form is too cadence-specific (4-cursor body's + // double writeback per iter can land an offset=1 + // emit whose SOURCE is still ≥ prefix_start_index). + let match_start = anchor + lits.len(); + let match_src = match_start + .checked_sub(*off) + .expect("offset must not exceed match_start (would wrap)"); + assert!( + match_src >= 5, + "match source {match_src} below prefix_start_index=5 \ + (match_start={match_start}, offset={off})", + ); + anchor = match_start + m; + } else { + // Pure-literals callback (currently never emitted by + // the kernel — kept defensive for future contract + // changes): advance anchor by the literal run length. + anchor += lits.len(); + } + } +} + +/// Hardening regression (round 3, finding #11): a hash entry +/// pointing AT or AFTER the current `ip0` must be rejected +/// before the 4-byte raw compare. Without this guard the kernel +/// would compute `offset = ip0 - match_pos` and wrap into a +/// gigantic offset → emit a Triple with a meaningless backward +/// reference. +/// +/// Stale hash entries below `prefix_start_index` must be rejected +/// by the upstream zstd-parity prefix filter in `match_found`. Engineered +/// scenario: pre-populate the hash slot for ip0 with a low stale +/// index (5) that points into the supposedly-out-of-window region; +/// run with `prefix_start_index = 50` so the kernel must skip +/// that candidate. The kernel's own writeback at the iteration +/// start would still leave the stale value usable if the prefix +/// filter didn't fire — uniform data ensures any survived +/// candidate would emit a non-zero match. +#[test] +fn match_found_rejects_stale_entry_below_prefix_floor() { + let data = vec![0u8; 200]; + let mut table = FastHashTable::new(8, 4); + // Force the explicit-match probe at ip0=50 (first iter once + // ip0 is bumped from prefix_start_index=50) to see the stale + // index 5. + // SAFETY: data has ≥ 4 readable bytes at index 50. + let h = unsafe { table.hash_ptr::<4>(data.as_ptr().add(50)) }; + // SAFETY: h came from hash_ptr on this same table. + unsafe { table.put(h, 5) }; + + let mut tuples: Vec<(Vec, usize, usize)> = Vec::new(); + let mut handle = |seq: Sequence<'_>| match seq { + Sequence::Triple { + literals, + offset, + match_len, + } => tuples.push((literals.to_vec(), offset, match_len)), + Sequence::Literals { literals } => tuples.push((literals.to_vec(), 0, 0)), + }; + // prefix_start_index = 50 — match_idx=5 is below the floor and + // must be rejected by the upstream zstd-parity prefix filter in + // `match_found`. + let _ = compress_block_fast::<4, false>( + &data, + 50, + PrefixBounds { + prefix_start_index: 50, + window_low: 50, + }, + &mut table, + [0, 0], + 2, + &mut handle, + ); + + // Either zero emissions (stale rejected, no other match found + // in the limited scan window) or a Triple whose offset + // references a position >= prefix_start_index = 50, never a + // 1-byte-from-stale-5 offset. + for (_, off, m) in &tuples { + if *m > 0 { + assert!( + *off > 0 && *off <= data.len(), + "every emitted offset must reference an in-buffer backward position (got {off})", + ); + } + } +} + +/// Input exactly `HASH_READ_SIZE` bytes long: the short-input +/// branch fires because `data.len() < block_start + HASH_READ_SIZE` +/// is `8 < 0 + 8` → false, so we enter the main loop, but +/// `ilimit = 8 - 8 = 0` makes `while ip0 < ilimit` zero-iteration +/// (ip0 starts at 1 ≥ 0). Result: zero emissions, entire input +/// reported as tail. +#[test] +fn block_exactly_hash_read_size_emits_no_sequences() { + let data = [1u8, 2, 3, 4, 5, 6, 7, 8]; + let (tuples, result) = run_block_with_rep(&data, 8, [0, 0]); + assert!( + tuples.is_empty(), + "exactly HASH_READ_SIZE bytes must produce no main-loop iterations", + ); + assert_eq!(result.tail_literals_len, data.len()); +} + +/// Input one byte shorter than `HASH_READ_SIZE`: the short-input +/// branch fires (`7 < 8`), the kernel returns immediately with +/// the full input as tail and no callback invocations. +#[test] +fn block_just_below_hash_read_size_emits_no_sequences() { + let data = [1u8, 2, 3, 4, 5, 6, 7]; + let (tuples, result) = run_block_with_rep(&data, 8, [0, 0]); + assert!(tuples.is_empty()); + assert_eq!(result.tail_literals_len, data.len()); +} + +/// Repcode save/restore: when the incoming `rep_offset1` is +/// larger than the addressable history (`max_rep = ip0 - +/// prefix_start_index`), the kernel stashes it into +/// `offset_saved1` and zeroes the live rep. If no explicit match +/// promotes a new rep during the block, `_cleanup` must restore +/// the saved value into the returned `rep[0]` so cross-block +/// repcode history isn't lost. The unaffected `rep[1]` is the +/// secondary witness that no mutation occurred mid-block. +#[test] +fn rep_offset_save_restore_when_out_of_range() { + // Random-looking distinct bytes — no real matches the kernel + // would discover; deterministic xorshift keeps the stream + // reproducible. + let mut data = vec![0u8; 64]; + let mut state = 0x1234_5678u32; + for byte in &mut data { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + *byte = state as u8; + } + // rep_offset1 huge — far exceeds any plausible ip0 in a + // 64-byte block. Must be stashed and restored unchanged. + let huge = 9999; + let mut table = FastHashTable::new(10, 4); + let mut tuples: Vec<(Vec, usize, usize)> = Vec::new(); + let mut handle = |seq: Sequence<'_>| match seq { + Sequence::Triple { + literals, + offset, + match_len, + } => tuples.push((literals.to_vec(), offset, match_len)), + Sequence::Literals { literals } => tuples.push((literals.to_vec(), 0, 0)), + }; + let result = compress_block_fast::<4, false>( + &data, + 0, + PrefixBounds { + // Match production contract: + // `prefix_start_index >= 1` rejects the hash table + // empty-slot value `0`. + prefix_start_index: 1, + window_low: 0, + }, + &mut table, + [huge, 7], + 2, + &mut handle, + ); + assert_eq!( + result.rep[0], huge, + "out-of-range rep_offset1 must be restored verbatim across the block", + ); + // rep_offset2 was also out of range (max_rep ≈ 0..63, 7 > 1). + // Upstream zstd restores it through offset_saved2; the in-range + // restoration path is the second witness. + assert_eq!(result.rep[1], 7, "rep_offset2 (also stashed) must restore"); +} + +/// cmov variant: same correctness contract as branch variant — +/// produces identical output for the same input, just lowers +/// match_idx >= prefix_start_index to a cmov instead of a +/// branch. Run a known-good fixture through both and assert +/// byte-for-byte equality of the emitted Triple stream. +#[test] +fn cmov_variant_matches_branch_variant_output() { + let mut data = alloc::vec::Vec::new(); + for i in 0..512u32 { + data.push((i & 0xFF) as u8); + } + // Repeat the first 64 bytes near the end so the kernel + // emits at least one explicit match Triple. + let tail = data[0..64].to_vec(); + data.extend_from_slice(&tail); + + let collect = |use_cmov: bool| -> alloc::vec::Vec<(alloc::vec::Vec, usize, usize)> { + let mut table = FastHashTable::new(12, 4); + let mut tuples = alloc::vec::Vec::new(); + let mut handle = |seq: Sequence<'_>| match seq { + Sequence::Triple { + literals, + offset, + match_len, + } => { + tuples.push((literals.to_vec(), offset, match_len)); + } + Sequence::Literals { literals } => { + tuples.push((literals.to_vec(), 0, 0)); + } + }; + if use_cmov { + let _ = compress_block_fast::<4, true>( + &data, + 0, + PrefixBounds { + // Match production contract: + // `prefix_start_index >= 1` rejects the hash + // table empty-slot value `0`. + prefix_start_index: 1, + window_low: 0, + }, + &mut table, + [0, 0], + 2, + &mut handle, + ); + } else { + let _ = compress_block_fast::<4, false>( + &data, + 0, + PrefixBounds { + // Match production contract: + // `prefix_start_index >= 1` rejects the hash + // table empty-slot value `0`. + prefix_start_index: 1, + window_low: 0, + }, + &mut table, + [0, 0], + 2, + &mut handle, + ); + } + tuples + }; + + let out_branch = collect(false); + let out_cmov = collect(true); + assert_eq!( + out_branch, out_cmov, + "cmov and branch variants must emit identical sequences" + ); +} + +/// Regression test for Copilot review thread on PR #219 — cmov +/// variant must NOT report a match when `match_idx < +/// prefix_start_index` even if the 4 bytes at `ip` happen to +/// equal `CMOV_DUMMY`. Without the explicit `in_range` +/// predicate the cmov path returns `true` here, producing an +/// out-of-window match the kernel would then encode with a +/// bogus offset. +#[test] +fn cmov_variant_rejects_out_of_window_when_ip_equals_dummy() { + // Layout (32 bytes total): + // data[0..4] = filler (not CMOV_DUMMY, won't accidentally match) + // data[4..] = CMOV_DUMMY bytes at position 16, so read32(ip) + // at ip_pos=16 equals read32(CMOV_DUMMY). + // + // match_idx=4 is below prefix_start=10 (out of window). + // ip_pos=16 satisfies `ip == base.add(ip_pos)`. + let mut data: alloc::vec::Vec = alloc::vec![0xAA; 32]; + data[16] = 0x12; + data[17] = 0x34; + data[18] = 0x56; + data[19] = 0x78; + // SAFETY (test fixture): ip = base + 16; both buffers cover + // ≥ 4 readable bytes (data.len()=32 ≥ 16+4 and CMOV_DUMMY is + // 4 bytes by construction). + let base = data.as_ptr(); + let ip_pos = 16usize; + let ip = unsafe { base.add(ip_pos) }; + let branch_result = unsafe { match_found::(ip, base, 4, 10) }; + assert!( + !branch_result, + "branch variant must reject out-of-window match_idx" + ); + let cmov_result = unsafe { match_found::(ip, base, 4, 10) }; + assert!( + !cmov_result, + "cmov variant must reject out-of-window match_idx even when \ + ip bytes coincide with CMOV_DUMMY", + ); +} + +/// Drive the borrowed dual-base dict kernel directly (Scalar tier — always +/// available, deterministic) over a crafted `(dict, input)` pair that +/// exercises the dict-match (2-segment + backward extension), input-match, +/// repcode and tail-literal branches. Reconstruct against the logical +/// `[dict][input]` window to prove every emitted offset is valid, and check +/// the byte-accounting invariant. +#[test] +fn borrowed_dict_kernel_reconstructs_via_dual_base() { + use crate::encoding::fastpath::FastpathKernel; + + let hash_log = 12u32; + const MLS: u32 = 4; + // Distinct dict bytes so each 4-byte key hashes uniquely; the input + // both matches the dict prefix (dict match) and repeats itself + // (input match + repcode), then ends in a literal tail. + let dict: Vec = (0u8..40).collect(); + let mut inp: Vec = Vec::new(); + inp.extend_from_slice(&dict[0..20]); // dict match against dict[0..] + inp.extend_from_slice(&dict[0..20]); // input match against the first copy + inp.extend_from_slice(&dict[4..24]); // dict match at a non-zero dict pos + inp.extend_from_slice(b"tail-literals-xyz"); // terminal literals + + let dict_end = dict.len(); + + // main table (filled during the scan) + immutable dict table primed + // over every hashable dict position. + let mut main_table = FastHashTable::new(hash_log, MLS); + let mut dict_table = FastHashTable::new(hash_log, MLS); + for pos in 0..=dict.len() - HASH_READ_SIZE { + // SAFETY: pos + 8 <= dict.len(); MLS matches the table. Tagged + // every-position last-wins fill, matching the kernel's tagged dict + // lookup (slot + packed tag). + let hat = unsafe { hash_ptr_raw::(dict.as_ptr().add(pos), hash_log + DICT_TAG_BITS) }; + unsafe { + dict_table.put( + hat >> DICT_TAG_BITS, + ((pos as u32) << DICT_TAG_BITS) | (hat & DICT_TAG_MASK), + ) + }; + } + + let mut tuples: Vec<(Vec, usize, usize)> = Vec::new(); + let mut handle = |seq: Sequence<'_>| match seq { + Sequence::Triple { + literals, + offset, + match_len, + } => tuples.push((literals.to_vec(), offset, match_len)), + Sequence::Literals { literals } => tuples.push((literals.to_vec(), 0, 0)), + }; + + let result = compress_block_fast_dict_borrowed::( + &inp, + &dict, + 0, + inp.len(), + &mut main_table, + &dict_table, + PrefixBounds { + prefix_start_index: 1, + window_low: 0, + }, + [0, 0], + 2, + &mut handle, + FastpathKernel::Scalar, + ); + + // Reconstruct against the logical [dict][input] window: start from the + // dictionary, then replay each sequence. A Triple's offset references + // `current_len - offset` in this combined buffer, so a valid dual-base + // offset reproduces the original input exactly. + let mut window = dict.clone(); + let mut saw_dict_region_match = false; + for (literals, offset, match_len) in &tuples { + window.extend_from_slice(literals); + if *match_len > 0 { + let start = window.len() - offset; + // A match whose source lands in the dictionary prefix exercised + // the dual-base / 2-segment path (not a plain input back-ref). + if start < dict_end { + saw_dict_region_match = true; + } + for i in 0..*match_len { + let b = window[start + i]; + window.push(b); + } + } + } + // Append the terminal tail the kernel reports separately. + let tail_start = inp.len() - result.tail_literals_len; + window.extend_from_slice(&inp[tail_start..]); + + assert_eq!( + &window[dict_end..], + &inp[..], + "borrowed dict kernel must reconstruct the input from the [dict][input] window", + ); + + assert!( + tuples.iter().any(|(_, _, m)| *m >= 4), + "expected at least one match Triple", + ); + assert!( + saw_dict_region_match, + "expected at least one match reading from the dictionary region (dual-base path)", + ); +} diff --git a/zstd/src/encoding/simple/fast_matcher.rs b/zstd/src/encoding/simple/fast_matcher.rs index 105603924..af2401d03 100644 --- a/zstd/src/encoding/simple/fast_matcher.rs +++ b/zstd/src/encoding/simple/fast_matcher.rs @@ -2073,1297 +2073,4 @@ fn run_fast_kernel_block_dict( } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn new_uses_level_1_defaults() { - let m = FastKernelMatcher::new(); - assert_eq!(m.window_log, FAST_LEVEL_1_WINDOW_LOG); - assert_eq!(m.hash_table.hash_log(), FAST_LEVEL_1_HASH_LOG); - assert_eq!(m.hash_table.mls(), FAST_LEVEL_1_MLS); - assert_eq!(m.rep, FAST_INITIAL_REP); - assert_eq!(m.offset_hist, FAST_INITIAL_OFFSET_HIST); - assert_eq!(m.max_window_size, 1usize << FAST_LEVEL_1_WINDOW_LOG); - // M8: history starts empty (HISTORY_DRAIN_BASE = 0). - // First input byte will live at position 0; sentinel-0 - // protection comes from the prefix filter, not from a - // dummy region. - assert_eq!(m.history.len(), HISTORY_DRAIN_BASE); - // prefix_start_index = 1 makes position 0 unmatchable so - // the hash table's empty-slot value 0 can't be confused - // with a real match. - assert_eq!(m.prefix_start_index, INITIAL_PREFIX_START_INDEX); - assert!(m.pending.is_none()); - } - - #[test] - fn borrowed_window_reads_match_owned_then_restores() { - let mut m = FastKernelMatcher::new(); - // Owned path: history_bytes mirrors the owned buffer. - m.history = b"owned-history-bytes".to_vec(); - assert_eq!(m.history_bytes(), b"owned-history-bytes"); - - // Borrowed path: history_bytes views the caller's buffer, not - // the owned one, and the owned buffer is left untouched. - let external = b"a-different-borrowed-window".to_vec(); - // SAFETY: `external` outlives every history_bytes() call below - // and is cleared before it drops at end of scope. - unsafe { m.set_borrowed_window(&external) }; - assert_eq!(m.history_bytes(), &external[..]); - assert_eq!(m.history, b"owned-history-bytes"); - - // Clearing returns to the owned path with the original bytes. - m.clear_borrowed_window(); - assert_eq!(m.history_bytes(), b"owned-history-bytes"); - } - - #[test] - fn reset_clears_borrowed_window() { - let mut m = FastKernelMatcher::new(); - let external = b"borrowed".to_vec(); - // SAFETY: `external` outlives the reset call below. - unsafe { m.set_borrowed_window(&external) }; - m.reset( - FAST_LEVEL_1_WINDOW_LOG, - FAST_LEVEL_1_HASH_LOG, - FAST_LEVEL_1_MLS, - 2, - false, - false, - ); - // After reset the borrowed window is dropped — back to the - // (now empty) owned buffer, never the dangling external range. - assert!(m.borrowed.is_none()); - assert_eq!(m.history_bytes().len(), HISTORY_DRAIN_BASE); - } - - /// The borrowed one-shot scan must emit the EXACT same sequence - /// stream (and end with the same rep / offset_hist state) as the - /// owned block-by-block path. This is the load-bearing correctness - /// check for the borrowed window: the one-shot frame path relies on - /// it producing identical output without the per-block history copy. - #[test] - fn borrowed_window_matches_owned_sequence_stream() { - #[derive(PartialEq, Debug)] - enum Seq { - Triple(alloc::vec::Vec, usize, usize), - Lits(alloc::vec::Vec), - } - - // Repeating pattern so the matcher emits real matches, split into - // two blocks. Window (1 << 15 = 32 KiB) far exceeds the input, so - // the owned path never evicts — its accumulated `history` is - // byte-identical to the borrowed buffer, the precondition for - // stream equality. - let mut whole = alloc::vec::Vec::with_capacity(320); - for i in 0..320usize { - whole.push((i % 64) as u8); - } - let split = 192usize; - - // Owned path: commit each block, then scan. - let mut owned = FastKernelMatcher::with_params(15, 12, 5, 2); - let mut owned_seqs: alloc::vec::Vec = alloc::vec::Vec::new(); - owned.accept_data(whole[..split].to_vec()); - owned.start_matching(|seq| match seq { - Sequence::Triple { - literals, - offset, - match_len, - } => owned_seqs.push(Seq::Triple(literals.to_vec(), offset, match_len)), - Sequence::Literals { literals } => owned_seqs.push(Seq::Lits(literals.to_vec())), - }); - owned.accept_data(whole[split..].to_vec()); - owned.start_matching(|seq| match seq { - Sequence::Triple { - literals, - offset, - match_len, - } => owned_seqs.push(Seq::Triple(literals.to_vec(), offset, match_len)), - Sequence::Literals { literals } => owned_seqs.push(Seq::Lits(literals.to_vec())), - }); - - // Borrowed path: same bytes, scanned in place by block range. - let mut borrowed = FastKernelMatcher::with_params(15, 12, 5, 2); - let mut borrowed_seqs: alloc::vec::Vec = alloc::vec::Vec::new(); - // SAFETY: `whole` outlives both scans below; `borrowed` is dropped - // at end of scope before `whole`, so the window never dangles. - unsafe { borrowed.set_borrowed_window(&whole) }; - borrowed.start_matching_borrowed(0, split, |seq| match seq { - Sequence::Triple { - literals, - offset, - match_len, - } => borrowed_seqs.push(Seq::Triple(literals.to_vec(), offset, match_len)), - Sequence::Literals { literals } => borrowed_seqs.push(Seq::Lits(literals.to_vec())), - }); - borrowed.start_matching_borrowed(split, whole.len(), |seq| match seq { - Sequence::Triple { - literals, - offset, - match_len, - } => borrowed_seqs.push(Seq::Triple(literals.to_vec(), offset, match_len)), - Sequence::Literals { literals } => borrowed_seqs.push(Seq::Lits(literals.to_vec())), - }); - - assert_eq!( - owned_seqs, borrowed_seqs, - "borrowed window must emit the identical sequence stream as the owned path", - ); - assert_eq!( - owned.rep, borrowed.rep, - "rep state must match after both scans" - ); - assert_eq!( - owned.offset_hist, borrowed.offset_hist, - "offset_hist must match after both scans", - ); - // The borrowed path must have produced at least one match (else - // the test would trivially pass on an all-literals stream). - assert!( - borrowed_seqs.iter().any(|s| matches!(s, Seq::Triple(..))), - "pattern must yield at least one match to make the check meaningful", - ); - } - - #[test] - fn with_params_threads_through_each_field() { - // Pick a non-default triple to prove no silent override by - // upstream zstd-default constants. - let m = FastKernelMatcher::with_params(16, 12, 5, 2); - assert_eq!(m.window_log, 16); - assert_eq!(m.hash_table.hash_log(), 12); - assert_eq!(m.hash_table.mls(), 5); - assert_eq!(m.max_window_size, 1usize << 16); - } - - #[test] - fn window_size_reports_one_shifted_window_log() { - // window_log = 16 → 64 KiB reported window. - let m = FastKernelMatcher::with_params(16, 12, 5, 2); - assert_eq!(m.window_size(), 1u64 << 16); - // Larger window_log → larger reported window. window_log = 22 - // (4 MiB) confirms the shift width (`u64` head room). - let m = FastKernelMatcher::with_params(22, 14, 7, 2); - assert_eq!(m.window_size(), 1u64 << 22); - } - - #[test] - fn last_committed_space_empty_before_commit() { - let m = FastKernelMatcher::new(); - assert!(m.last_committed_space().is_empty()); - } - - #[test] - fn reset_clears_history_and_state() { - let mut m = FastKernelMatcher::new(); - // Simulate prior-frame state — non-empty history, advanced - // prefix, non-default rep/offset stacks, a leftover pending - // block. Reset must restore the matcher to a from-scratch - // appearance regardless of which fields were dirtied. - m.history.extend_from_slice(&[1, 2, 3, 4]); - m.prefix_start_index = 7; - m.rep = [42, 99]; - m.offset_hist = [10, 20, 30]; - m.pending = Some(alloc::vec![5, 6, 7]); - - m.reset( - FAST_LEVEL_1_WINDOW_LOG, - FAST_LEVEL_1_HASH_LOG, - FAST_LEVEL_1_MLS, - 2, - false, - false, - ); - - // Post-reset: history empty (HISTORY_DRAIN_BASE=0; no - // dummy only; prefix_start_index pinned to that baseline. - assert_eq!(m.history.len(), HISTORY_DRAIN_BASE); - assert_eq!(m.prefix_start_index, INITIAL_PREFIX_START_INDEX); - assert_eq!(m.rep, FAST_INITIAL_REP); - assert_eq!(m.offset_hist, FAST_INITIAL_OFFSET_HIST); - assert!(m.pending.is_none()); - // Hash-table identity preserved (same shape) — `clear()` path, - // not a fresh `new()`. Equality test is over the params, not - // the buffer pointer, because the `Vec`-internal allocation - // identity is an internal detail the test should not lock in. - assert_eq!(m.hash_table.hash_log(), FAST_LEVEL_1_HASH_LOG); - assert_eq!(m.hash_table.mls(), FAST_LEVEL_1_MLS); - } - - #[test] - fn reset_with_changed_params_rebuilds_hash_table() { - let mut m = FastKernelMatcher::new(); - // Force a parameter change — every Vec we hand the new - // FastHashTable will be a fresh allocation. - m.reset(16, 10, 4, 2, false, false); - assert_eq!(m.hash_table.hash_log(), 10); - assert_eq!(m.hash_table.mls(), 4); - assert_eq!(m.window_log, 16); - assert_eq!(m.max_window_size, 1usize << 16); - } - - // The copy-mode restore fast path: a same-shape reset with - // `table_overwritten_by_restore` must leave the table contents - // untouched (the snapshot restore replaces them right after), while - // the plain same-shape reset must memset them. Guards against a - // refactor silently reintroducing the wasted per-frame clear — or, - // worse, dropping the clear on the plain path. - #[test] - fn reset_keeps_table_when_overwritten_by_restore() { - let mut m = FastKernelMatcher::new(); - m.reset(16, 10, 4, 2, false, false); - let probe_hash = 7u32; - // SAFETY: hash 7 < (1 << hash_log = 1024) table entries. - unsafe { m.hash_table.put(probe_hash, 0xCAFE) }; - - // Same shape + restore-pending: contents survive the reset. - m.reset(16, 10, 4, 2, false, true); - // SAFETY: same bounds as the put above. - assert_eq!( - unsafe { m.hash_table.get(probe_hash) }, - 0xCAFE, - "restore-pending reset must not clear the table" - ); - - // Plain same-shape reset: contents are memset back to empty. - m.reset(16, 10, 4, 2, false, false); - // SAFETY: same bounds as the put above. - assert_eq!( - unsafe { m.hash_table.get(probe_hash) }, - 0, - "plain reset must clear the table" - ); - - // Shape change overrides the flag: the table is rebuilt at the - // new geometry even when a restore is claimed to be pending. - unsafe { m.hash_table.put(probe_hash, 0xCAFE) }; - m.reset(16, 11, 4, 2, false, true); - assert_eq!(m.hash_table.hash_log(), 11); - // SAFETY: hash 7 < (1 << 11) table entries. - assert_eq!( - unsafe { m.hash_table.get(probe_hash) }, - 0, - "shape change must rebuild the table regardless of the flag" - ); - } - - /// Drive the matcher through a single block whose tail contains - /// a repeated 4-byte run — the kernel must emit at least one - /// `Sequence::Triple` with `match_len >= 4` and the bookkeeping - /// invariant `Σ(literals + match_len) + tail_literals_len == - /// input.len()` must hold. - #[test] - fn accept_then_start_matching_emits_match_for_repeated_run() { - // 64 bytes: 32 bytes of pseudo-random preamble + 32-byte - // verbatim copy of bytes [0..32]. The kernel scanning the - // tail should find the 32-byte repeat with offset = 32. - let mut data = alloc::vec::Vec::with_capacity(64); - for i in 0..32u8 { - // Spread the byte values so 4-byte windows are all - // distinct (avoids accidental rep hits that would skew - // the assertion). - data.push(i.wrapping_mul(7).wrapping_add(13)); - } - data.extend_from_within(0..32); - // Use a small mls=4 table so the test exercises the simpler - // hash arm; level-1 defaults (mls=7) would also work but the - // hash collisions on a 64-byte synthetic input are noisier - // for mls>=5. - let mut m = FastKernelMatcher::with_params(12, 8, 4, 2); - m.accept_data(data.clone()); - - let mut emitted_match_lens: alloc::vec::Vec = alloc::vec::Vec::new(); - let mut emitted_literal_byte_count: usize = 0; - let mut tail_byte_count: usize = 0; - m.start_matching(|seq| match seq { - Sequence::Triple { - literals, - offset: _, - match_len, - } => { - emitted_literal_byte_count += literals.len(); - emitted_match_lens.push(match_len); - } - Sequence::Literals { literals } => { - tail_byte_count += literals.len(); - } - }); - - let total_matched: usize = emitted_match_lens.iter().sum(); - assert_eq!( - emitted_literal_byte_count + total_matched + tail_byte_count, - data.len(), - "all input bytes must be accounted for as literals/matches/tail", - ); - assert!( - emitted_match_lens.iter().any(|&m| m >= 4), - "kernel must emit at least one Triple with match_len >= MIN_MATCH (got {emitted_match_lens:?})", - ); - // Pending buffer was consumed. - assert!(m.pending.is_none()); - // History grew by exactly the block size (plus the - // no dummy carried since construction — M8). - assert_eq!(m.history.len(), data.len() + HISTORY_DRAIN_BASE); - // `last_committed_space` post-processing reads from - // history[last_block_start..] (upstream zstd / legacy MatchGenerator - // parity for the frame compressor's raw-block emission - // path) — for a single-block-then-process flow it equals - // the input data verbatim. - assert_eq!(m.last_committed_space(), data.as_slice()); - } - - /// Skip path: `skip_matching` must move the pending buffer into - /// history WITHOUT emitting any sequences and WITHOUT touching - /// the rep / offset_hist state. - #[test] - fn skip_matching_extends_history_without_emissions() { - let mut m = FastKernelMatcher::with_params(12, 8, 4, 2); - let pre_rep = m.rep; - let pre_offset_hist = m.offset_hist; - - let payload: alloc::vec::Vec = (0..40u8).collect(); - m.accept_data(payload.clone()); - // Take a count of state pre-skip. - assert_eq!(m.last_committed_space().len(), payload.len()); - - m.skip_matching_with_hint(None); - - assert_eq!( - m.history.len(), - payload.len() + HISTORY_DRAIN_BASE, - "skip_matching must append the pending buffer to history", - ); - assert_eq!(m.rep, pre_rep, "skip must not touch rep state"); - assert_eq!( - m.offset_hist, pre_offset_hist, - "skip must not touch offset_hist", - ); - assert!(m.pending.is_none()); - } - - /// Two-block run with literal block then matchable block — the - /// SECOND `start_matching` must find a cross-block match against - /// the first block's bytes (cross-block matches are the headline - /// reason for keeping the hash table persistent across blocks). - /// - /// Sizing rationale: the kernel's main loop only scans up to - /// `ilimit = data.len() - HASH_READ_SIZE` (upstream zstd parity). Block - /// 2 must therefore carry enough trailing bytes past the - /// crossblock-match start for `ip0` to actually reach the copy. - /// We use a 128-byte block 1 and a 64-byte block 2 with the - /// 32-byte copy of block 1's prefix landing at block-2 offset - /// 16, leaving plenty of headroom under `ilimit`. - #[test] - fn cross_block_match_finds_first_block_payload() { - // Block 1: 128-byte pattern, distinct 4-byte windows. - let mut block1 = alloc::vec::Vec::with_capacity(128); - for i in 0..128u8 { - block1.push(i.wrapping_mul(11).wrapping_add(5)); - } - // Block 2: 16 fresh bytes followed by a 32-byte verbatim copy - // of block 1's [0..32]. The matcher must reach back into - // block 1's bytes (offset 128+16-0 = 144 ≈ length of block 1 - // plus the leading fresh bytes of block 2). Tail (16 bytes - // past the copy) gives `ip0` enough room to reach the copy - // before hitting `ilimit`. - let mut block2 = alloc::vec::Vec::with_capacity(64); - block2.extend(0..16u8); // 16 fresh bytes (different from block1) - block2.extend_from_slice(&block1[0..32]); // 32-byte cross-block copy - block2.extend(200..216u8); // 16-byte tail buffer - - let mut m = FastKernelMatcher::with_params(12, 8, 4, 2); - - // Block 1 — drain emissions, ignore. - m.accept_data(block1.clone()); - m.start_matching(|_seq| {}); - - // Block 2 — capture emissions. - m.accept_data(block2.clone()); - let mut max_match: usize = 0; - let mut saw_cross_block = false; - m.start_matching(|seq| { - if let Sequence::Triple { - offset, match_len, .. - } = seq - { - max_match = max_match.max(match_len); - // Cross-block match: offset must reach back into - // block 1, i.e. offset > position-within-block-2. - // Block 2's payload starts at history position - // `block1.len()`; the source is in block 1 when - // offset >= block2.len() (offset measured from ip0 - // backwards, so a block-1 source means offset - // exceeds any block-2-internal distance). - if offset >= block2.len() { - saw_cross_block = true; - } - } - }); - - assert!( - saw_cross_block, - "block 2's matcher must find at least one cross-block match \ - (max_len={max_match})", - ); - assert_eq!( - m.history.len(), - block1.len() + block2.len() + HISTORY_DRAIN_BASE, - "history must hold both blocks after two start_matching calls", - ); - } - - /// Dictionary-priming skip: `skip_matching(Some(false))` MUST - /// pre-populate the hash table for the just-appended range so a - /// subsequent `start_matching` can find matches against the - /// dict-primed bytes. Without that pre-population, a future - /// block that copies the dict prefix verbatim would emit only - /// literals. - #[test] - fn skip_matching_with_false_hint_populates_hashes_for_dict_priming() { - // Stage: 32 bytes "dict" via skip_matching(Some(false)), - // then a second block whose tail copies the dict prefix. - // Without the hash pre-population the kernel can't reach - // the dict bytes in block 2. - let mut dict_block = alloc::vec::Vec::with_capacity(32); - for i in 0..32u8 { - dict_block.push(i.wrapping_mul(13).wrapping_add(7)); - } - - let mut m = FastKernelMatcher::with_params(12, 8, 4, 2); - m.accept_data(dict_block.clone()); - m.skip_matching_with_hint(Some(false)); // dictionary-priming skip - - // Sanity: history grew, prefix_start_index unchanged. - assert_eq!(m.history.len(), dict_block.len() + HISTORY_DRAIN_BASE); - assert_eq!(m.prefix_start_index, INITIAL_PREFIX_START_INDEX); - - // Block 2: 16 fresh bytes + 16-byte copy of dict_block[0..16] - // + 16-byte tail buffer so the kernel can reach the copy. - let mut block2 = alloc::vec::Vec::with_capacity(48); - block2.extend(100..116u8); - block2.extend_from_slice(&dict_block[0..16]); - block2.extend(120..136u8); - m.accept_data(block2.clone()); - - let mut saw_cross_block = false; - m.start_matching(|seq| { - if let Sequence::Triple { offset, .. } = seq - && offset >= block2.len() - { - saw_cross_block = true; - } - }); - - assert!( - saw_cross_block, - "skip_matching(Some(false)) must populate hashes so block 2 \ - can match against the primed bytes", - ); - } - - /// Control case for the prime-path test: same setup but with - /// `skip_matching(None)` — the bytes are NOT hashed, so block 2 - /// must NOT find the cross-block match. - #[test] - fn skip_matching_with_none_hint_skips_hash_population() { - let mut dict_block = alloc::vec::Vec::with_capacity(32); - for i in 0..32u8 { - dict_block.push(i.wrapping_mul(13).wrapping_add(7)); - } - - let mut m = FastKernelMatcher::with_params(12, 8, 4, 2); - m.accept_data(dict_block.clone()); - m.skip_matching_with_hint(None); // plain skip — no hash pre-population - - let mut block2 = alloc::vec::Vec::with_capacity(48); - block2.extend(100..116u8); - block2.extend_from_slice(&dict_block[0..16]); - block2.extend(120..136u8); - m.accept_data(block2.clone()); - - let mut saw_cross_block = false; - m.start_matching(|seq| { - if let Sequence::Triple { offset, .. } = seq - && offset >= block2.len() - { - saw_cross_block = true; - } - }); - - assert!( - !saw_cross_block, - "skip_matching(None) must NOT populate hashes — the legacy \ - skip cost-savings only hold when future blocks are willing \ - to miss matches in the skipped region", - ); - } - - /// Window eviction: when total history would exceed `2 × - /// max_window_size`, the matcher must drain the oldest prefix - /// down to a `max_window_size` tail BEFORE appending the new - /// block, bump `prefix_start_index`, and clear the hash table. - /// - /// Post-append history can still hold up to - /// `max_window_size + block_size` bytes (the kernel needs the - /// just-arrived block for matching plus the retained prefix for - /// cross-block lookups). The hard upper bound is therefore the - /// eviction threshold itself: `2 × max_window_size`. - #[test] - fn extend_history_drains_old_prefix_past_two_window_sizes() { - // window_log = 8 → max_window_size = 256, eviction threshold - // = 512. Stage three 200-byte blocks: after the third commit, - // total would be 600 > 512 → eviction fires. - let mut m = FastKernelMatcher::with_params(8, 6, 4, 2); - for round in 0..3 { - // Distinct payload per round so a hash entry from round - // 0 referencing position 0 is identifiable as stale - // after eviction. - let block: alloc::vec::Vec = (0..200u8) - .map(|i| i.wrapping_add(round as u8 * 17)) - .collect(); - m.accept_data(block); - m.skip_matching_with_hint(None); - } - // Hard bound: post-append history can hold up to - // `max_window_size + block_size` (retained prefix + the - // just-appended block). The eviction policy keeps total - // strictly below `2 × max_window_size` for the next - // accept_data call, so the invariant we assert here is the - // post-append upper bound. - assert!( - m.history.len() <= m.max_window_size * 2 + HISTORY_DRAIN_BASE, - "after eviction, REAL history must be bounded by 2× \ - max_window_size (got history.len()={}, max_window_size={})", - m.history.len(), - m.max_window_size, - ); - assert!( - m.history.len() <= m.max_window_size + 200 + HISTORY_DRAIN_BASE, - "post-append history = retained prefix (≤ max_window_size) \ - + last block (200 bytes); got {}", - m.history.len(), - ); - // Post-fix: drain RESETS prefix_start_index back to 1 (the - // initial sentinel-0 baseline) rather than accumulating - // saturating_add — see the `drain_rebases_prefix_start_index` - // regression test for the full rationale. Eviction is - // proven by post-history shrinking, not by an - // index-advancement signal. - assert_eq!( - m.prefix_start_index, INITIAL_PREFIX_START_INDEX, - "drain must rebase prefix_start_index to the baseline (1)", - ); - } - - /// Boundary: exactly `HASH_READ_SIZE` (8) real bytes appended - /// via `skip_matching(Some(false))` — must hash one position - /// (`range_start == HISTORY_DRAIN_BASE`, `last_hashable == - /// HISTORY_DRAIN_BASE`) without overrun. - #[test] - fn skip_matching_dict_prime_handles_exactly_hash_read_size_bytes() { - let mut m = FastKernelMatcher::with_params(12, 8, 4, 2); - // 8-byte real payload appended to history → history.len() = - // 8 + HISTORY_DRAIN_BASE (= 8 under M8), last_hashable = - // HISTORY_DRAIN_BASE (= 0), hashed range = [0..=0] (one - // position). - let payload: alloc::vec::Vec = (0..8u8).collect(); - m.accept_data(payload); - m.skip_matching_with_hint(Some(false)); - assert_eq!(m.history.len(), 8 + HISTORY_DRAIN_BASE); - // No assertion on hash entries — the bug we're guarding - // against is a panic / overrun, not a behavioural one. - // Reaching this line without unwinding is the test. - } - - /// Boundary: pending block too short to hash anything (less than - /// `HASH_READ_SIZE` bytes). The dict-prime path must early-return - /// without panicking on the `last_hashable` subtract. - #[test] - fn skip_matching_dict_prime_handles_below_hash_read_size_bytes() { - let mut m = FastKernelMatcher::with_params(12, 8, 4, 2); - let payload: alloc::vec::Vec = (0..4u8).collect(); - m.accept_data(payload); - // history will be 4 bytes after append < HASH_READ_SIZE (8). - // prime_hash_table_for_range must short-circuit on the - // `history_len < HASH_READ_SIZE` guard. - m.skip_matching_with_hint(Some(false)); - assert_eq!(m.history.len(), 4 + HISTORY_DRAIN_BASE); - } - - /// Regression: priming a dictionary in two chunks must hash the - /// positions straddling the chunk seam. The second chunk's prime - /// starts at `range_start = end-of-first-chunk`, but a hash read at - /// any of the preceding `HASH_READ_SIZE - 1` positions spans the - /// seam into the second chunk — those positions belong to neither - /// chunk's `[range_start ..= last_hashable]` window unless the second - /// prime backfills them. Without the backfill a dictionary fed in - /// `slice_size` chunks silently drops every seam-spanning match. - #[test] - fn dict_prime_indexes_positions_across_chunk_seam() { - // hash_log=20 (1 Mi slots) makes a hash collision among ~32 - // primed positions negligible, so `get(hash(p)) == p` is a - // deterministic assertion on this fixed input. - let mut m = FastKernelMatcher::with_params(20, 20, 4, 2); - let chunk1: alloc::vec::Vec = (0..16u8) - .map(|i| i.wrapping_mul(37).wrapping_add(13)) - .collect(); - m.accept_data(chunk1); - m.skip_matching_for_dict_prime(); - let seam = m.history.len(); // end of first chunk - let chunk2: alloc::vec::Vec = (16..32u8) - .map(|i| i.wrapping_mul(37).wrapping_add(13)) - .collect(); - m.accept_data(chunk2); - m.skip_matching_for_dict_prime(); - - // A position in the (HASH_READ_SIZE - 1)-byte gap just below the - // seam: its 8-byte hash read straddles the chunk boundary. - let p = seam - 4; - let dt = m.dict.table().expect("dict table primed"); - // Read the dict table exactly as the kernel does: hash the position and - // return the stored dict position for that slot (0 if empty). - let found = unsafe { - crate::encoding::simple::fast_kernel::kernel::dict_lookup::<4>( - dt, - m.history.as_ptr().add(p), - dt.hash_log(), - ) - }; - assert_eq!( - found, p as u32, - "seam-spanning position {p} must be indexed in the dict table", - ); - } - - #[test] - fn block_samples_match_dict_fires_only_on_extendable_dict_match() { - // Distinct dict bytes so the 4-byte hashes are collision-free at - // hash_log=20 and a verbatim run is the only way to hit the table. - let dict: alloc::vec::Vec = (0..200u8) - .map(|i| i.wrapping_mul(37).wrapping_add(13)) - .collect(); - let mut m = FastKernelMatcher::with_params(20, 20, 4, 2); - m.accept_data(dict.clone()); - m.skip_matching_for_dict_prime(); - assert!(m.dict_is_attached(), "dict must be attached after prime"); - - // A 32-byte verbatim dict run extends well past the 16-byte useful-match - // floor → the probe fires. - let hit = dict[8..40].to_vec(); - assert!( - m.block_samples_match_dict(&hit), - "a long verbatim dict run must trip the dict probe", - ); - - // An 8-byte dict prefix followed by non-dict bytes stays BELOW the - // 16-byte floor → the probe must NOT fire (a short coincidental match - // does not signal a dict that compresses the block). - let mut short = dict[8..16].to_vec(); - short.extend((0..56u8).map(|i| i.wrapping_mul(53).wrapping_add(201))); - assert!( - !m.block_samples_match_dict(&short), - "a sub-16-byte dict match must not trip the probe", - ); - - // A high-entropy block sharing no extendable run with the dict → no hit. - let miss: alloc::vec::Vec = (0..64u8) - .map(|i| i.wrapping_mul(91).wrapping_add(7) ^ 0xA5) - .collect(); - assert!( - !m.block_samples_match_dict(&miss), - "a non-dict block must not trip the probe", - ); - - // A block too short to hash (< HASH_READ_SIZE) → false, no panic. - assert!( - !m.block_samples_match_dict(&dict[..4]), - "a sub-8-byte block cannot be probed", - ); - } - - #[test] - fn block_samples_match_dict_is_false_without_a_dictionary() { - // No dict primed → the probe short-circuits to false (the no-dict path - // never reaches it, but the guard must hold). - let m = FastKernelMatcher::with_params(20, 20, 4, 2); - let block: alloc::vec::Vec = (0..64u8) - .map(|i| i.wrapping_mul(37).wrapping_add(13)) - .collect(); - assert!(!m.dict_is_attached()); - assert!(!m.block_samples_match_dict(&block)); - } - - /// Regression: same seam-gap defect for the MAIN hash table, primed - /// per slice via `skip_matching_with_hint(Some(false))` (the dict - /// COPY path and multi-slice history priming). Cross-slice matches - /// at the seam are dropped unless the second prime backfills the - /// trailing `HASH_READ_SIZE - 1` positions. - #[test] - fn main_table_prime_indexes_positions_across_slice_seam() { - let mut m = FastKernelMatcher::with_params(20, 20, 4, 2); - let chunk1: alloc::vec::Vec = (0..16u8) - .map(|i| i.wrapping_mul(53).wrapping_add(7)) - .collect(); - m.accept_data(chunk1); - m.skip_matching_with_hint(Some(false)); - let seam = m.history.len(); - let chunk2: alloc::vec::Vec = (16..32u8) - .map(|i| i.wrapping_mul(53).wrapping_add(7)) - .collect(); - m.accept_data(chunk2); - m.skip_matching_with_hint(Some(false)); - - let p = seam - 4; - let h = unsafe { m.hash_table.hash_ptr::<4>(m.history.as_ptr().add(p)) }; - assert_eq!( - unsafe { m.hash_table.get(h) }, - p as u32, - "seam-spanning position {p} must be indexed in the main table", - ); - } - - /// After a single block emits matches, the matcher's `rep[0]` - /// (kernel's `rep_offset1` post-block) must reflect the last emitted - /// explicit offset — that is the state the next block's kernel probes - /// against. The matcher's `offset_hist` is NOT updated by matching: - /// the Fast backend drives repcode probes off `rep`, and the wire - /// offset coding is done downstream against the encode pipeline's own - /// offset history, so per-match `encode_offset_with_history` on the - /// matcher would be redundant work. `offset_hist` therefore stays at - /// whatever `reset` / `prime_offset_history` set it to. - #[test] - fn rep_tracks_last_explicit_offset_and_offset_hist_is_not_matched() { - // Engineer a single block that produces a deterministic - // explicit match. 96 bytes: 48-byte distinct-window - // preamble + 48-byte verbatim copy of bytes [0..48]. - let mut data = alloc::vec::Vec::with_capacity(96); - for i in 0..48u8 { - data.push(i.wrapping_mul(11).wrapping_add(3)); - } - data.extend_from_within(0..48); - - let mut m = FastKernelMatcher::with_params(12, 8, 4, 2); - m.accept_data(data.clone()); - - let mut emitted_offsets: alloc::vec::Vec = alloc::vec::Vec::new(); - m.start_matching(|seq| { - if let Sequence::Triple { - offset, match_len, .. - } = seq - && match_len >= 4 - { - emitted_offsets.push(offset); - } - }); - - assert!( - !emitted_offsets.is_empty(), - "test setup must produce at least one explicit match", - ); - let last_explicit = emitted_offsets[emitted_offsets.len() - 1]; - // rep[0] is the functional state: the kernel updates it from its - // own result each block and the NEXT block probes against it. - assert_eq!( - m.rep[0] as usize, last_explicit, - "kernel's rep[0] must reflect the last emitted explicit offset", - ); - // offset_hist is NOT touched by matching on the Fast backend — it - // stays at the construction default (FAST_INITIAL_OFFSET_HIST). - assert_eq!( - m.offset_hist, FAST_INITIAL_OFFSET_HIST, - "Fast matching must not mutate offset_hist (it is seeded only \ - by reset / prime_offset_history; the wire offset coding runs \ - downstream against the encode pipeline's own history)", - ); - } - - /// Eviction during a dict-priming sequence: when consecutive - /// `skip_matching(Some(false))` calls accumulate past the - /// eviction threshold, the second skip must drop the older - /// prime'd hash entries (via `hash_table.clear()`) AND bump - /// `prefix_start_index` past the dropped bytes. Otherwise the - /// matcher would carry stale absolute positions referencing - /// evicted history. - #[test] - fn eviction_during_dict_priming_drops_stale_prime_entries() { - // window_log=8 → max_window_size=256, threshold=512. - // Two 300-byte blocks both via dict-prime skip — second - // one triggers eviction. - let mut m = FastKernelMatcher::with_params(8, 6, 4, 2); - let block1: alloc::vec::Vec = (0..200u8).collect(); - m.accept_data(block1); - m.skip_matching_with_hint(Some(false)); - let block2: alloc::vec::Vec = (0..200u8).map(|i| i.wrapping_add(50)).collect(); - m.accept_data(block2); - // Second skip would push total to 400, still under 512 — no - // eviction yet. Make sure two more rounds trigger it. - m.skip_matching_with_hint(Some(false)); - let block3: alloc::vec::Vec = (0..200u8).map(|i| i.wrapping_add(100)).collect(); - m.accept_data(block3); - // Now 400+200=600 > 512 → eviction fires inside extend. - m.skip_matching_with_hint(Some(false)); - - // Post-fix: drain rebases prefix_start_index to 1 (rather - // than cumulative saturating_add); eviction is proven by - // bounded history below. - assert_eq!( - m.prefix_start_index, INITIAL_PREFIX_START_INDEX, - "drain must rebase prefix_start_index to the baseline (1)", - ); - // History within the 2× window-size hard cap. - assert!(m.history.len() <= m.max_window_size * 2); - } - - /// Regression for #216 review #1: `accept_data` MUST perform - /// window eviction immediately so the driver's `commit_space` - /// can observe the byte delta via a pre/post `history.len()` - /// comparison. Without commit-time eviction visibility, the - /// driver's `retire_dictionary_budget` never runs for this - /// backend → `max_window_size` stays inflated post-dict-prime - /// → matcher can emit offsets exceeding the frame header's - /// reported window (format-correctness risk). - #[test] - fn accept_data_evicts_eagerly_so_commit_observes_byte_delta() { - // window_log = 8 → max_window_size = 256, eviction threshold - // = 512. Stage three 200-byte blocks via accept_data + a - // start_matching cycle each so history accumulates without - // eviction (200, 400 bytes). The THIRD accept_data crosses - // the 512-byte threshold; its eviction MUST be visible at - // accept_data return-time via the history.len() drop. - let mut m = FastKernelMatcher::with_params(8, 6, 4, 2); - - m.accept_data((0..200u8).collect()); - m.skip_matching_with_hint(None); - assert_eq!(m.history.len(), 200 + HISTORY_DRAIN_BASE); - assert_eq!( - m.prefix_start_index, INITIAL_PREFIX_START_INDEX, - "no eviction yet" - ); - - m.accept_data((0..200u8).map(|i| i.wrapping_add(50)).collect()); - m.skip_matching_with_hint(None); - assert_eq!(m.history.len(), 400 + HISTORY_DRAIN_BASE); - assert_eq!( - m.prefix_start_index, INITIAL_PREFIX_START_INDEX, - "still no eviction (400 < 512)", - ); - - // Third commit: real history (400) + new space (200) = 600 > 512. - // Eviction MUST fire inside accept_data, dropping history - // back to max_window_size (256) BEFORE the kernel runs. - // `history_len_for_eviction_accounting` returns the real-data - // length (history.len() minus HISTORY_DRAIN_BASE, which is 0 - // under M8), so pre/post compare cleanly in real-byte units. - let pre = m.history_len_for_eviction_accounting(); - m.accept_data((0..200u8).map(|i| i.wrapping_add(100)).collect()); - let post = m.history_len_for_eviction_accounting(); - assert!( - pre > post, - "accept_data must shrink history at the eviction threshold \ - (pre={pre}, post={post}) — driver's commit_space relies on \ - this delta for retire_dictionary_budget accounting", - ); - assert_eq!( - post, 256, - "post-eviction retained must equal max_window_size", - ); - assert_eq!( - m.prefix_start_index, INITIAL_PREFIX_START_INDEX, - "drain rebases prefix_start_index to INITIAL_PREFIX_START_INDEX \ - — eviction is proven by the history.len() shrink above", - ); - } - - /// Regression for #216 review #2: `trim_to_window` must update - /// `last_block_start` to track the drain. Without the update, - /// the OLD position references pre-drain coordinates and - /// `last_committed_space()` would either panic with OOB or - /// return wrong bytes when `last_block_start > history.len()` - /// post-drain. - #[test] - fn trim_to_window_keeps_last_committed_space_consistent() { - // window_log = 8 → max_window_size = 256. Process a 200-byte - // block (now in history at positions [0..200], last_block_start - // = 0). Then bump the matcher's max_window_size DOWN to 128 - // (simulating a dictionary-budget retire shrinking the - // window) and call trim_to_window — drain_n = 200 - 128 = 72. - // Post-drain history is bytes [72..200] = 128 bytes. The - // last_block_start (was 0) MUST now be 0 (since 72 > 0 → - // saturating_sub gives 0) so last_committed_space() returns - // a valid in-bounds slice. - let mut m = FastKernelMatcher::with_params(8, 6, 4, 2); - let payload: alloc::vec::Vec = (0..200u8).collect(); - m.accept_data(payload); - m.skip_matching_with_hint(None); - // First block lands at history[RESERVED..RESERVED+200] after - // the seed dummy at [0..RESERVED). last_block_start tracks - // that absolute index. - assert_eq!(m.last_block_start, HISTORY_DRAIN_BASE); - assert_eq!(m.history.len(), 200 + HISTORY_DRAIN_BASE); - - // Shrink the window and trim. Without the fix, last_block_start - // stays mid-history past the post-drain end — to make the - // bug surface, use a SECOND block so last_block_start is - // somewhere AFTER the dummy + first block. - let payload2: alloc::vec::Vec = (50..150u8).collect(); - m.accept_data(payload2); - m.skip_matching_with_hint(None); - // history = [dummy] + [0..200] + [50..150] = 1 + 200 + 100 = 301. - // last_block_start = RESERVED + 200 = start of second block. - assert_eq!(m.last_block_start, HISTORY_DRAIN_BASE + 200); - assert_eq!(m.history.len(), 300 + HISTORY_DRAIN_BASE); - - // Now force trim_to_window to drain into the middle of the - // second block: shrink max_window_size below the second - // block's start. trim_to_window operates on REAL data so - // the drain target is 300 REAL bytes → 64. - m.max_window_size = 64; - let drained = m.trim_to_window(); - assert_eq!( - drained, - 300 - 64, - "trim must drain REAL history down to max_window_size = 64", - ); - // history.len() = RESERVED (dummy preserved) + 64. - assert_eq!(m.history.len(), 64 + HISTORY_DRAIN_BASE); - - // The slice MUST be in bounds — the bug would panic here OR - // return a stale slice. After the fix, last_block_start is - // saturating_sub'd by drained (236) AND clamped to >= RESERVED - // — since drained > old last_block_start - RESERVED, new - // last_block_start = RESERVED, pointing at the current first - // real byte of history (post-drain start of what remains of - // block 2). - let last = m.last_committed_space(); - assert!( - last.len() <= 64, - "last_committed_space must be in-bounds after trim \ - (got len {})", - last.len(), - ); - } - - /// Regression for #216 Copilot review #15: after - /// `prime_offset_history` the kernel's `rep[0..2]` must mirror - /// the wire-encoder's `offset_hist[0..2]` — without this the - /// kernel makes repcode decisions against stale FAST_INITIAL_REP - /// while the wire encoder uses the primed history → wrong - /// repcode wire encoding (correctness bug, not perf). - #[test] - fn prime_offset_history_keeps_rep_and_offset_hist_in_lockstep() { - let mut m = FastKernelMatcher::with_params(12, 8, 4, 2); - // Pre-prime: matcher carries the upstream zstd's initial state. - assert_eq!(m.rep, FAST_INITIAL_REP); - assert_eq!(m.offset_hist, FAST_INITIAL_OFFSET_HIST); - - // Prime with non-default history (upstream zstd's dictionary load - // restores explicit rep1/rep2/rep3 values). - let primed = [9u32, 4, 8]; - m.prime_offset_history(primed); - - // BOTH must reflect the primed values; rep[0..2] = the first - // two entries of offset_hist. - assert_eq!( - m.offset_hist, primed, - "offset_hist must be updated by prime_offset_history", - ); - assert_eq!( - m.rep, - [primed[0], primed[1]], - "rep[0..2] must mirror offset_hist[0..2] post-prime \ - (kernel's repcode decisions must match the wire \ - encoder's seeded history)", - ); - } - - /// Regression for #216 CodeRabbit #19: when a committed block - /// is larger than `max_window_size`, the pre-fix eviction math - /// always retained a full `max_window_size` of historical real - /// data and then appended the (oversized) block, blowing past - /// the documented `2 × max_window_size` post-append bound. - /// Fix retains a SMALLER prefix (or none) so the bound holds. - #[test] - fn accept_data_evicts_more_aggressively_when_block_larger_than_window() { - let mut m = FastKernelMatcher::with_params(8, 6, 4, 2); - // max_window_size = 256 (1 << 8). Threshold = 512. - - // Pre-fill history to full max_window_size of real data via - // skip_matching_with_hint (no kernel-side trimming). u8 range - // tops at 255, so the 256-byte preamble cycles via modulo. - let preamble: alloc::vec::Vec = (0..256u32).map(|i| i as u8).collect(); - m.accept_data(preamble); - m.skip_matching_with_hint(None); - assert_eq!( - m.history_len_for_eviction_accounting(), - 256, - "history pre-filled to one full window of real bytes", - ); - - // Commit a block larger than max_window_size but under 2× - // (so its append should still be allowed without rejecting - // outright). 400 > 256 but < 512. - let oversize: alloc::vec::Vec = (0..400u32) - .map(|i| (i as u8).wrapping_mul(7).wrapping_add(11)) - .collect(); - m.accept_data(oversize); - - // Post-append real_len + space.len() MUST stay within 2× - // max_window_size (the documented invariant). Pre-fix it - // jumped to 256 + 400 = 656 = 2.56× — bound violated. - let real_len_after = m.history_len_for_eviction_accounting(); - // accept_data stashes pending without appending, so the - // 400-byte block is in pending. real_len reflects post- - // drain retained real bytes. To verify the bound, run - // start_matching to commit the append. - m.start_matching(|_| {}); - let real_total = m.history_len_for_eviction_accounting(); - assert!( - real_total <= m.max_window_size * 2, - "post-append history MUST stay within 2 × max_window_size \ - (got real_total={real_total}, cap={})", - m.max_window_size * 2, - ); - // Sanity: pre-append eviction did drain something (we had - // 256 real bytes, can't accept 400 more while staying under - // 512 unless we drop at least 144). - assert!( - real_len_after < 256, - "pre-append drain must have shed historical bytes \ - (got real_len_after_drain={real_len_after}, was 256 \ - before accept)", - ); - } - - /// Regression for PR #219 round 6 (CR Critical): `start_matching` - /// computes an effective sliding prefix floor of - /// `history.len() - max_window_size` so emitted offsets never - /// exceed the advertised `max_window_size`. Without this floor, - /// after history grows past one window (commit's - /// eager-eviction keeps up to 2× max_window_size before - /// draining), the kernel could match against positions older - /// than the frame window — invalid for decoder buffer - /// reservation. - #[test] - fn start_matching_enforces_max_window_size_offset_bound() { - // Tight window: window_log=7 → max_window_size=128. - // Block lengths 200 + 200 = 400 bytes total, comfortably - // past 128 so without the sliding floor the kernel would - // emit matches at offsets > max_window. - let mut m = FastKernelMatcher::with_params(7, 8, 4, 2); - let max_window = m.max_window_size; - assert_eq!(max_window, 128, "test assumes window_log=7"); - - // Block 1: 200 bytes of a distinct ASCII pattern that - // populates the hash table at positions 0..200. - let block1: alloc::vec::Vec = (0..200u8).map(|i| 0x30 + (i % 64)).collect(); - m.accept_data(block1.clone()); - m.start_matching(|_| {}); - - // Block 2: 200 bytes that RE-USE block1's content from - // position 0 onwards. Without the sliding floor, the - // kernel would happily emit Triples with offset - // ≈ history_len (~200..400) — well past max_window=128. - let block2 = block1.clone(); - m.accept_data(block2); - - let mut max_emitted_offset = 0usize; - let mut emitted_match_count = 0usize; - m.start_matching(|seq| { - if let Sequence::Triple { - offset, match_len, .. - } = seq - && match_len > 0 - { - emitted_match_count += 1; - if offset > max_emitted_offset { - max_emitted_offset = offset; - } - } - }); - - // GUARANTEE 1: the kernel actually scanned and matched - // something — otherwise the offset bound below passes - // trivially with max_emitted_offset = 0. - assert!( - emitted_match_count > 0, - "fixture must produce at least one Triple match — \ - history.len()=~400, max_window=128, block2 is identical to block1", - ); - - // GUARANTEE 2: every emitted offset stays within the - // advertised max_window_size. Without the sliding floor, - // matches against early block1 positions (e.g. position - // 0..72) would yield offsets > 128. - assert!( - max_emitted_offset <= max_window, - "sliding floor MUST cap emitted offsets at max_window_size; \ - got max emitted offset {} vs max_window_size {}", - max_emitted_offset, - max_window, - ); - } - - /// Regression for PR #219 round 9 (Copilot Critical): the - /// sliding prefix floor in `start_matching` MUST use the - /// advertised frame window (`1 << window_log`), NOT the - /// dynamically inflated `max_window_size` (which the driver - /// adds dictionary-budget bytes to during - /// `prime_with_dictionary`). With the inflated value, - /// offsets could exceed the advertised window during - /// dictionary-primed compression — format-invalid sequences. - #[test] - fn start_matching_caps_offsets_at_window_log_not_inflated_max() { - // Advertised frame window = 1 << 7 = 128 bytes. - let mut m = FastKernelMatcher::with_params(7, 8, 4, 2); - let advertised_window: usize = 1 << m.window_log; - assert_eq!(advertised_window, 128, "test assumes window_log=7"); - - // Simulate dictionary priming: driver inflates - // max_window_size by retained_dict_budget. We add 200 - // bytes of "dictionary" content to history first, then - // bump max_window_size to reflect the dict-retention - // budget (mirrors MatchGeneratorDriver::prime_with_dictionary - // for the Simple backend). - // - // Pattern period 4 (`(i % 4)`) — dense enough that the upstream zstd- - // parity `kSearchStrength = 8` (K_STEP_INCR = 128) step - // doubling, which skips positions under step=3 in dict scan, - // still leaves matches hittable from block2: every position - // divisible by 4 inside the [in-window, hashed] subset writes - // the same hash slot, so the slot at block2's first probe - // contains a recent in-window dict position. Period 64 (the - // original fixture) only had matches at positions {0, 64, - // 128, 192} — positions 0, 64, 128 are below the sliding - // floor and 192 falls in the step-skip gap, leaving the test - // with zero emittable matches. - let dict: alloc::vec::Vec = (0..200u8).map(|i| 0x40 + (i % 4)).collect(); - m.accept_data(dict.clone()); - m.start_matching(|_| {}); // populate hash table from dict - m.max_window_size = m.max_window_size.saturating_add(200); - - // Add a block whose first 100 bytes match dict[0..100]. - // Without the fix, the kernel would emit offsets up to - // ~history_len (200..300), since the inflated max_window - // (328) keeps even the dict's earliest bytes inside the - // sliding floor. - let block: alloc::vec::Vec = (0..100u8).map(|i| 0x40 + (i % 4)).collect(); - m.accept_data(block); - - let mut max_emitted_offset = 0usize; - let mut emitted_match_count = 0usize; - m.start_matching(|seq| { - if let Sequence::Triple { - offset, match_len, .. - } = seq - && match_len > 0 - { - emitted_match_count += 1; - if offset > max_emitted_offset { - max_emitted_offset = offset; - } - } - }); - - assert!( - emitted_match_count > 0, - "fixture must produce at least one match — block content \ - repeats dict, history.len() ≈ 300, scan should find at \ - least one Triple", - ); - assert!( - max_emitted_offset <= advertised_window, - "sliding floor MUST cap emitted offsets at the ADVERTISED \ - frame window (1 << window_log = {}), NOT the inflated \ - max_window_size; got max emitted offset {}", - advertised_window, - max_emitted_offset, - ); - } - - /// Regression: at block 0 the kernel's prologue must NOT zero - /// `rep_offset1 = 1` (upstream zstd's default initial rep state). Upstream zstd - /// computes `max_rep = ip0 - windowLow` where `windowLow = 0` at - /// block 0, giving `max_rep = 1` at ip0=1 → `rep_offset1 = 1` - /// survives (`1 > 1` is false). - /// - /// Buggy prologue uses `max_rep = ip0 - prefix_start_index` with - /// `prefix_start_index = 1` (sentinel-0 floor for hash-table - /// filtering), giving `max_rep = 0` at ip0=1 → `rep_offset1 = 1 > - /// 0` → stashed → rep-at-ip2 probe disabled for the ENTIRE first - /// block. - /// - /// Symptom assertion: on a `[0x01, 0x42 × 199]` fixture, upstream zstd's - /// rep-at-ip2 fires at iter 1 (ip2=3, both `read32` reads see - /// `[42,42,42,42]`). The upstream zstd emit sequence is: - /// `new_ip = ip2 = 3`, `match0 = ip2 - rep_offset1 = 2`, then the - /// one-byte backward extension absorbs `data[2] == data[1]` - /// (both `0x42`), giving `new_ip = 2, match_len ≈ 198`. Literal - /// prefix is `data[0..new_ip] = [0x01, 0x42]` → length 2. - /// - /// The buggy path skips rep, walks the cursor via explicit-match - /// shifts until matchIdx coincides further into the run, and - /// emits a different literal prefix length. Asserting both - /// `offset == 1` AND `literals.len() == 2` pins down the - /// rep-at-ip2 path exactly — the explicit-match catch-up on - /// uniform-byte data also finds offset=1 via slot collision, so - /// an offset-only check passes both fixed and buggy paths. - #[test] - fn block_zero_prologue_preserves_default_rep_offset_one() { - let mut data = alloc::vec::Vec::with_capacity(200); - data.push(0x01); - data.resize(200, 0x42); - - let mut m = FastKernelMatcher::with_params(12, 8, 4, 2); - m.accept_data(data.clone()); - - let mut first_literals_len: Option = None; - let mut first_offset: Option = None; - m.start_matching(|seq| { - if first_literals_len.is_some() { - return; - } - if let Sequence::Triple { - literals, offset, .. - } = seq - { - first_literals_len = Some(literals.len()); - first_offset = Some(offset); - } - }); - - // Both `offset` AND `literals.len()` are asserted — together - // they pin down EXACTLY which inner-loop path emitted the - // first match. With the prologue bug (rep-at-ip2 disabled - // because `max_rep` was computed against `prefix_start_index` - // instead of `window_low`), the explicit-match path catches - // up via hash-slot collision at ip0=3 and STILL emits - // offset=1 — so an offset-only check would pass both fixed - // and buggy paths on this uniform-byte fixture. The literal - // prefix length is the actual discriminator: the rep-at-ip2 - // path consumes only the leading `0x01` literal (1 byte), - // while the explicit-match catch-up walks past two more - // `0x42` bytes before firing (3 bytes total). Asserting both - // values keeps the regression locked to the exact path the - // fix was meant to preserve. - assert_eq!( - first_offset, - Some(1), - "first emit must reference offset=1 — upstream zstd's default \ - rep_offset1=1 fires on rep-at-ip2 at iter 1, and the \ - prologue MUST NOT zero it (max_rep computed against \ - window_low=0 at block 0, NOT against the sentinel \ - prefix=1)", - ); - assert_eq!( - first_literals_len, - Some(2), - "first emit must have a 2-byte literal prefix \ - ([0x01, 0x42]) — the rep-at-ip2 probe lands at ip2=3, \ - then the one-byte backward extension drops new_ip to 2, \ - so literals = data[0..2]. A different prefix length \ - would indicate the explicit-match catch-up fired instead", - ); - } -} +mod tests; diff --git a/zstd/src/encoding/simple/fast_matcher/tests.rs b/zstd/src/encoding/simple/fast_matcher/tests.rs new file mode 100644 index 000000000..c2f6778dc --- /dev/null +++ b/zstd/src/encoding/simple/fast_matcher/tests.rs @@ -0,0 +1,1292 @@ +use super::*; + +#[test] +fn new_uses_level_1_defaults() { + let m = FastKernelMatcher::new(); + assert_eq!(m.window_log, FAST_LEVEL_1_WINDOW_LOG); + assert_eq!(m.hash_table.hash_log(), FAST_LEVEL_1_HASH_LOG); + assert_eq!(m.hash_table.mls(), FAST_LEVEL_1_MLS); + assert_eq!(m.rep, FAST_INITIAL_REP); + assert_eq!(m.offset_hist, FAST_INITIAL_OFFSET_HIST); + assert_eq!(m.max_window_size, 1usize << FAST_LEVEL_1_WINDOW_LOG); + // M8: history starts empty (HISTORY_DRAIN_BASE = 0). + // First input byte will live at position 0; sentinel-0 + // protection comes from the prefix filter, not from a + // dummy region. + assert_eq!(m.history.len(), HISTORY_DRAIN_BASE); + // prefix_start_index = 1 makes position 0 unmatchable so + // the hash table's empty-slot value 0 can't be confused + // with a real match. + assert_eq!(m.prefix_start_index, INITIAL_PREFIX_START_INDEX); + assert!(m.pending.is_none()); +} + +#[test] +fn borrowed_window_reads_match_owned_then_restores() { + let mut m = FastKernelMatcher::new(); + // Owned path: history_bytes mirrors the owned buffer. + m.history = b"owned-history-bytes".to_vec(); + assert_eq!(m.history_bytes(), b"owned-history-bytes"); + + // Borrowed path: history_bytes views the caller's buffer, not + // the owned one, and the owned buffer is left untouched. + let external = b"a-different-borrowed-window".to_vec(); + // SAFETY: `external` outlives every history_bytes() call below + // and is cleared before it drops at end of scope. + unsafe { m.set_borrowed_window(&external) }; + assert_eq!(m.history_bytes(), &external[..]); + assert_eq!(m.history, b"owned-history-bytes"); + + // Clearing returns to the owned path with the original bytes. + m.clear_borrowed_window(); + assert_eq!(m.history_bytes(), b"owned-history-bytes"); +} + +#[test] +fn reset_clears_borrowed_window() { + let mut m = FastKernelMatcher::new(); + let external = b"borrowed".to_vec(); + // SAFETY: `external` outlives the reset call below. + unsafe { m.set_borrowed_window(&external) }; + m.reset( + FAST_LEVEL_1_WINDOW_LOG, + FAST_LEVEL_1_HASH_LOG, + FAST_LEVEL_1_MLS, + 2, + false, + false, + ); + // After reset the borrowed window is dropped — back to the + // (now empty) owned buffer, never the dangling external range. + assert!(m.borrowed.is_none()); + assert_eq!(m.history_bytes().len(), HISTORY_DRAIN_BASE); +} + +/// The borrowed one-shot scan must emit the EXACT same sequence +/// stream (and end with the same rep / offset_hist state) as the +/// owned block-by-block path. This is the load-bearing correctness +/// check for the borrowed window: the one-shot frame path relies on +/// it producing identical output without the per-block history copy. +#[test] +fn borrowed_window_matches_owned_sequence_stream() { + #[derive(PartialEq, Debug)] + enum Seq { + Triple(alloc::vec::Vec, usize, usize), + Lits(alloc::vec::Vec), + } + + // Repeating pattern so the matcher emits real matches, split into + // two blocks. Window (1 << 15 = 32 KiB) far exceeds the input, so + // the owned path never evicts — its accumulated `history` is + // byte-identical to the borrowed buffer, the precondition for + // stream equality. + let mut whole = alloc::vec::Vec::with_capacity(320); + for i in 0..320usize { + whole.push((i % 64) as u8); + } + let split = 192usize; + + // Owned path: commit each block, then scan. + let mut owned = FastKernelMatcher::with_params(15, 12, 5, 2); + let mut owned_seqs: alloc::vec::Vec = alloc::vec::Vec::new(); + owned.accept_data(whole[..split].to_vec()); + owned.start_matching(|seq| match seq { + Sequence::Triple { + literals, + offset, + match_len, + } => owned_seqs.push(Seq::Triple(literals.to_vec(), offset, match_len)), + Sequence::Literals { literals } => owned_seqs.push(Seq::Lits(literals.to_vec())), + }); + owned.accept_data(whole[split..].to_vec()); + owned.start_matching(|seq| match seq { + Sequence::Triple { + literals, + offset, + match_len, + } => owned_seqs.push(Seq::Triple(literals.to_vec(), offset, match_len)), + Sequence::Literals { literals } => owned_seqs.push(Seq::Lits(literals.to_vec())), + }); + + // Borrowed path: same bytes, scanned in place by block range. + let mut borrowed = FastKernelMatcher::with_params(15, 12, 5, 2); + let mut borrowed_seqs: alloc::vec::Vec = alloc::vec::Vec::new(); + // SAFETY: `whole` outlives both scans below; `borrowed` is dropped + // at end of scope before `whole`, so the window never dangles. + unsafe { borrowed.set_borrowed_window(&whole) }; + borrowed.start_matching_borrowed(0, split, |seq| match seq { + Sequence::Triple { + literals, + offset, + match_len, + } => borrowed_seqs.push(Seq::Triple(literals.to_vec(), offset, match_len)), + Sequence::Literals { literals } => borrowed_seqs.push(Seq::Lits(literals.to_vec())), + }); + borrowed.start_matching_borrowed(split, whole.len(), |seq| match seq { + Sequence::Triple { + literals, + offset, + match_len, + } => borrowed_seqs.push(Seq::Triple(literals.to_vec(), offset, match_len)), + Sequence::Literals { literals } => borrowed_seqs.push(Seq::Lits(literals.to_vec())), + }); + + assert_eq!( + owned_seqs, borrowed_seqs, + "borrowed window must emit the identical sequence stream as the owned path", + ); + assert_eq!( + owned.rep, borrowed.rep, + "rep state must match after both scans" + ); + assert_eq!( + owned.offset_hist, borrowed.offset_hist, + "offset_hist must match after both scans", + ); + // The borrowed path must have produced at least one match (else + // the test would trivially pass on an all-literals stream). + assert!( + borrowed_seqs.iter().any(|s| matches!(s, Seq::Triple(..))), + "pattern must yield at least one match to make the check meaningful", + ); +} + +#[test] +fn with_params_threads_through_each_field() { + // Pick a non-default triple to prove no silent override by + // upstream zstd-default constants. + let m = FastKernelMatcher::with_params(16, 12, 5, 2); + assert_eq!(m.window_log, 16); + assert_eq!(m.hash_table.hash_log(), 12); + assert_eq!(m.hash_table.mls(), 5); + assert_eq!(m.max_window_size, 1usize << 16); +} + +#[test] +fn window_size_reports_one_shifted_window_log() { + // window_log = 16 → 64 KiB reported window. + let m = FastKernelMatcher::with_params(16, 12, 5, 2); + assert_eq!(m.window_size(), 1u64 << 16); + // Larger window_log → larger reported window. window_log = 22 + // (4 MiB) confirms the shift width (`u64` head room). + let m = FastKernelMatcher::with_params(22, 14, 7, 2); + assert_eq!(m.window_size(), 1u64 << 22); +} + +#[test] +fn last_committed_space_empty_before_commit() { + let m = FastKernelMatcher::new(); + assert!(m.last_committed_space().is_empty()); +} + +#[test] +fn reset_clears_history_and_state() { + let mut m = FastKernelMatcher::new(); + // Simulate prior-frame state — non-empty history, advanced + // prefix, non-default rep/offset stacks, a leftover pending + // block. Reset must restore the matcher to a from-scratch + // appearance regardless of which fields were dirtied. + m.history.extend_from_slice(&[1, 2, 3, 4]); + m.prefix_start_index = 7; + m.rep = [42, 99]; + m.offset_hist = [10, 20, 30]; + m.pending = Some(alloc::vec![5, 6, 7]); + + m.reset( + FAST_LEVEL_1_WINDOW_LOG, + FAST_LEVEL_1_HASH_LOG, + FAST_LEVEL_1_MLS, + 2, + false, + false, + ); + + // Post-reset: history empty (HISTORY_DRAIN_BASE=0; no + // dummy only; prefix_start_index pinned to that baseline. + assert_eq!(m.history.len(), HISTORY_DRAIN_BASE); + assert_eq!(m.prefix_start_index, INITIAL_PREFIX_START_INDEX); + assert_eq!(m.rep, FAST_INITIAL_REP); + assert_eq!(m.offset_hist, FAST_INITIAL_OFFSET_HIST); + assert!(m.pending.is_none()); + // Hash-table identity preserved (same shape) — `clear()` path, + // not a fresh `new()`. Equality test is over the params, not + // the buffer pointer, because the `Vec`-internal allocation + // identity is an internal detail the test should not lock in. + assert_eq!(m.hash_table.hash_log(), FAST_LEVEL_1_HASH_LOG); + assert_eq!(m.hash_table.mls(), FAST_LEVEL_1_MLS); +} + +#[test] +fn reset_with_changed_params_rebuilds_hash_table() { + let mut m = FastKernelMatcher::new(); + // Force a parameter change — every Vec we hand the new + // FastHashTable will be a fresh allocation. + m.reset(16, 10, 4, 2, false, false); + assert_eq!(m.hash_table.hash_log(), 10); + assert_eq!(m.hash_table.mls(), 4); + assert_eq!(m.window_log, 16); + assert_eq!(m.max_window_size, 1usize << 16); +} + +// The copy-mode restore fast path: a same-shape reset with +// `table_overwritten_by_restore` must leave the table contents +// untouched (the snapshot restore replaces them right after), while +// the plain same-shape reset must memset them. Guards against a +// refactor silently reintroducing the wasted per-frame clear — or, +// worse, dropping the clear on the plain path. +#[test] +fn reset_keeps_table_when_overwritten_by_restore() { + let mut m = FastKernelMatcher::new(); + m.reset(16, 10, 4, 2, false, false); + let probe_hash = 7u32; + // SAFETY: hash 7 < (1 << hash_log = 1024) table entries. + unsafe { m.hash_table.put(probe_hash, 0xCAFE) }; + + // Same shape + restore-pending: contents survive the reset. + m.reset(16, 10, 4, 2, false, true); + // SAFETY: same bounds as the put above. + assert_eq!( + unsafe { m.hash_table.get(probe_hash) }, + 0xCAFE, + "restore-pending reset must not clear the table" + ); + + // Plain same-shape reset: contents are memset back to empty. + m.reset(16, 10, 4, 2, false, false); + // SAFETY: same bounds as the put above. + assert_eq!( + unsafe { m.hash_table.get(probe_hash) }, + 0, + "plain reset must clear the table" + ); + + // Shape change overrides the flag: the table is rebuilt at the + // new geometry even when a restore is claimed to be pending. + unsafe { m.hash_table.put(probe_hash, 0xCAFE) }; + m.reset(16, 11, 4, 2, false, true); + assert_eq!(m.hash_table.hash_log(), 11); + // SAFETY: hash 7 < (1 << 11) table entries. + assert_eq!( + unsafe { m.hash_table.get(probe_hash) }, + 0, + "shape change must rebuild the table regardless of the flag" + ); +} + +/// Drive the matcher through a single block whose tail contains +/// a repeated 4-byte run — the kernel must emit at least one +/// `Sequence::Triple` with `match_len >= 4` and the bookkeeping +/// invariant `Σ(literals + match_len) + tail_literals_len == +/// input.len()` must hold. +#[test] +fn accept_then_start_matching_emits_match_for_repeated_run() { + // 64 bytes: 32 bytes of pseudo-random preamble + 32-byte + // verbatim copy of bytes [0..32]. The kernel scanning the + // tail should find the 32-byte repeat with offset = 32. + let mut data = alloc::vec::Vec::with_capacity(64); + for i in 0..32u8 { + // Spread the byte values so 4-byte windows are all + // distinct (avoids accidental rep hits that would skew + // the assertion). + data.push(i.wrapping_mul(7).wrapping_add(13)); + } + data.extend_from_within(0..32); + // Use a small mls=4 table so the test exercises the simpler + // hash arm; level-1 defaults (mls=7) would also work but the + // hash collisions on a 64-byte synthetic input are noisier + // for mls>=5. + let mut m = FastKernelMatcher::with_params(12, 8, 4, 2); + m.accept_data(data.clone()); + + let mut emitted_match_lens: alloc::vec::Vec = alloc::vec::Vec::new(); + let mut emitted_literal_byte_count: usize = 0; + let mut tail_byte_count: usize = 0; + m.start_matching(|seq| match seq { + Sequence::Triple { + literals, + offset: _, + match_len, + } => { + emitted_literal_byte_count += literals.len(); + emitted_match_lens.push(match_len); + } + Sequence::Literals { literals } => { + tail_byte_count += literals.len(); + } + }); + + let total_matched: usize = emitted_match_lens.iter().sum(); + assert_eq!( + emitted_literal_byte_count + total_matched + tail_byte_count, + data.len(), + "all input bytes must be accounted for as literals/matches/tail", + ); + assert!( + emitted_match_lens.iter().any(|&m| m >= 4), + "kernel must emit at least one Triple with match_len >= MIN_MATCH (got {emitted_match_lens:?})", + ); + // Pending buffer was consumed. + assert!(m.pending.is_none()); + // History grew by exactly the block size (plus the + // no dummy carried since construction — M8). + assert_eq!(m.history.len(), data.len() + HISTORY_DRAIN_BASE); + // `last_committed_space` post-processing reads from + // history[last_block_start..] (upstream zstd / legacy MatchGenerator + // parity for the frame compressor's raw-block emission + // path) — for a single-block-then-process flow it equals + // the input data verbatim. + assert_eq!(m.last_committed_space(), data.as_slice()); +} + +/// Skip path: `skip_matching` must move the pending buffer into +/// history WITHOUT emitting any sequences and WITHOUT touching +/// the rep / offset_hist state. +#[test] +fn skip_matching_extends_history_without_emissions() { + let mut m = FastKernelMatcher::with_params(12, 8, 4, 2); + let pre_rep = m.rep; + let pre_offset_hist = m.offset_hist; + + let payload: alloc::vec::Vec = (0..40u8).collect(); + m.accept_data(payload.clone()); + // Take a count of state pre-skip. + assert_eq!(m.last_committed_space().len(), payload.len()); + + m.skip_matching_with_hint(None); + + assert_eq!( + m.history.len(), + payload.len() + HISTORY_DRAIN_BASE, + "skip_matching must append the pending buffer to history", + ); + assert_eq!(m.rep, pre_rep, "skip must not touch rep state"); + assert_eq!( + m.offset_hist, pre_offset_hist, + "skip must not touch offset_hist", + ); + assert!(m.pending.is_none()); +} + +/// Two-block run with literal block then matchable block — the +/// SECOND `start_matching` must find a cross-block match against +/// the first block's bytes (cross-block matches are the headline +/// reason for keeping the hash table persistent across blocks). +/// +/// Sizing rationale: the kernel's main loop only scans up to +/// `ilimit = data.len() - HASH_READ_SIZE` (upstream zstd parity). Block +/// 2 must therefore carry enough trailing bytes past the +/// crossblock-match start for `ip0` to actually reach the copy. +/// We use a 128-byte block 1 and a 64-byte block 2 with the +/// 32-byte copy of block 1's prefix landing at block-2 offset +/// 16, leaving plenty of headroom under `ilimit`. +#[test] +fn cross_block_match_finds_first_block_payload() { + // Block 1: 128-byte pattern, distinct 4-byte windows. + let mut block1 = alloc::vec::Vec::with_capacity(128); + for i in 0..128u8 { + block1.push(i.wrapping_mul(11).wrapping_add(5)); + } + // Block 2: 16 fresh bytes followed by a 32-byte verbatim copy + // of block 1's [0..32]. The matcher must reach back into + // block 1's bytes (offset 128+16-0 = 144 ≈ length of block 1 + // plus the leading fresh bytes of block 2). Tail (16 bytes + // past the copy) gives `ip0` enough room to reach the copy + // before hitting `ilimit`. + let mut block2 = alloc::vec::Vec::with_capacity(64); + block2.extend(0..16u8); // 16 fresh bytes (different from block1) + block2.extend_from_slice(&block1[0..32]); // 32-byte cross-block copy + block2.extend(200..216u8); // 16-byte tail buffer + + let mut m = FastKernelMatcher::with_params(12, 8, 4, 2); + + // Block 1 — drain emissions, ignore. + m.accept_data(block1.clone()); + m.start_matching(|_seq| {}); + + // Block 2 — capture emissions. + m.accept_data(block2.clone()); + let mut max_match: usize = 0; + let mut saw_cross_block = false; + m.start_matching(|seq| { + if let Sequence::Triple { + offset, match_len, .. + } = seq + { + max_match = max_match.max(match_len); + // Cross-block match: offset must reach back into + // block 1, i.e. offset > position-within-block-2. + // Block 2's payload starts at history position + // `block1.len()`; the source is in block 1 when + // offset >= block2.len() (offset measured from ip0 + // backwards, so a block-1 source means offset + // exceeds any block-2-internal distance). + if offset >= block2.len() { + saw_cross_block = true; + } + } + }); + + assert!( + saw_cross_block, + "block 2's matcher must find at least one cross-block match \ + (max_len={max_match})", + ); + assert_eq!( + m.history.len(), + block1.len() + block2.len() + HISTORY_DRAIN_BASE, + "history must hold both blocks after two start_matching calls", + ); +} + +/// Dictionary-priming skip: `skip_matching(Some(false))` MUST +/// pre-populate the hash table for the just-appended range so a +/// subsequent `start_matching` can find matches against the +/// dict-primed bytes. Without that pre-population, a future +/// block that copies the dict prefix verbatim would emit only +/// literals. +#[test] +fn skip_matching_with_false_hint_populates_hashes_for_dict_priming() { + // Stage: 32 bytes "dict" via skip_matching(Some(false)), + // then a second block whose tail copies the dict prefix. + // Without the hash pre-population the kernel can't reach + // the dict bytes in block 2. + let mut dict_block = alloc::vec::Vec::with_capacity(32); + for i in 0..32u8 { + dict_block.push(i.wrapping_mul(13).wrapping_add(7)); + } + + let mut m = FastKernelMatcher::with_params(12, 8, 4, 2); + m.accept_data(dict_block.clone()); + m.skip_matching_with_hint(Some(false)); // dictionary-priming skip + + // Sanity: history grew, prefix_start_index unchanged. + assert_eq!(m.history.len(), dict_block.len() + HISTORY_DRAIN_BASE); + assert_eq!(m.prefix_start_index, INITIAL_PREFIX_START_INDEX); + + // Block 2: 16 fresh bytes + 16-byte copy of dict_block[0..16] + // + 16-byte tail buffer so the kernel can reach the copy. + let mut block2 = alloc::vec::Vec::with_capacity(48); + block2.extend(100..116u8); + block2.extend_from_slice(&dict_block[0..16]); + block2.extend(120..136u8); + m.accept_data(block2.clone()); + + let mut saw_cross_block = false; + m.start_matching(|seq| { + if let Sequence::Triple { offset, .. } = seq + && offset >= block2.len() + { + saw_cross_block = true; + } + }); + + assert!( + saw_cross_block, + "skip_matching(Some(false)) must populate hashes so block 2 \ + can match against the primed bytes", + ); +} + +/// Control case for the prime-path test: same setup but with +/// `skip_matching(None)` — the bytes are NOT hashed, so block 2 +/// must NOT find the cross-block match. +#[test] +fn skip_matching_with_none_hint_skips_hash_population() { + let mut dict_block = alloc::vec::Vec::with_capacity(32); + for i in 0..32u8 { + dict_block.push(i.wrapping_mul(13).wrapping_add(7)); + } + + let mut m = FastKernelMatcher::with_params(12, 8, 4, 2); + m.accept_data(dict_block.clone()); + m.skip_matching_with_hint(None); // plain skip — no hash pre-population + + let mut block2 = alloc::vec::Vec::with_capacity(48); + block2.extend(100..116u8); + block2.extend_from_slice(&dict_block[0..16]); + block2.extend(120..136u8); + m.accept_data(block2.clone()); + + let mut saw_cross_block = false; + m.start_matching(|seq| { + if let Sequence::Triple { offset, .. } = seq + && offset >= block2.len() + { + saw_cross_block = true; + } + }); + + assert!( + !saw_cross_block, + "skip_matching(None) must NOT populate hashes — the legacy \ + skip cost-savings only hold when future blocks are willing \ + to miss matches in the skipped region", + ); +} + +/// Window eviction: when total history would exceed `2 × +/// max_window_size`, the matcher must drain the oldest prefix +/// down to a `max_window_size` tail BEFORE appending the new +/// block, bump `prefix_start_index`, and clear the hash table. +/// +/// Post-append history can still hold up to +/// `max_window_size + block_size` bytes (the kernel needs the +/// just-arrived block for matching plus the retained prefix for +/// cross-block lookups). The hard upper bound is therefore the +/// eviction threshold itself: `2 × max_window_size`. +#[test] +fn extend_history_drains_old_prefix_past_two_window_sizes() { + // window_log = 8 → max_window_size = 256, eviction threshold + // = 512. Stage three 200-byte blocks: after the third commit, + // total would be 600 > 512 → eviction fires. + let mut m = FastKernelMatcher::with_params(8, 6, 4, 2); + for round in 0..3 { + // Distinct payload per round so a hash entry from round + // 0 referencing position 0 is identifiable as stale + // after eviction. + let block: alloc::vec::Vec = (0..200u8) + .map(|i| i.wrapping_add(round as u8 * 17)) + .collect(); + m.accept_data(block); + m.skip_matching_with_hint(None); + } + // Hard bound: post-append history can hold up to + // `max_window_size + block_size` (retained prefix + the + // just-appended block). The eviction policy keeps total + // strictly below `2 × max_window_size` for the next + // accept_data call, so the invariant we assert here is the + // post-append upper bound. + assert!( + m.history.len() <= m.max_window_size * 2 + HISTORY_DRAIN_BASE, + "after eviction, REAL history must be bounded by 2× \ + max_window_size (got history.len()={}, max_window_size={})", + m.history.len(), + m.max_window_size, + ); + assert!( + m.history.len() <= m.max_window_size + 200 + HISTORY_DRAIN_BASE, + "post-append history = retained prefix (≤ max_window_size) \ + + last block (200 bytes); got {}", + m.history.len(), + ); + // Post-fix: drain RESETS prefix_start_index back to 1 (the + // initial sentinel-0 baseline) rather than accumulating + // saturating_add — see the `drain_rebases_prefix_start_index` + // regression test for the full rationale. Eviction is + // proven by post-history shrinking, not by an + // index-advancement signal. + assert_eq!( + m.prefix_start_index, INITIAL_PREFIX_START_INDEX, + "drain must rebase prefix_start_index to the baseline (1)", + ); +} + +/// Boundary: exactly `HASH_READ_SIZE` (8) real bytes appended +/// via `skip_matching(Some(false))` — must hash one position +/// (`range_start == HISTORY_DRAIN_BASE`, `last_hashable == +/// HISTORY_DRAIN_BASE`) without overrun. +#[test] +fn skip_matching_dict_prime_handles_exactly_hash_read_size_bytes() { + let mut m = FastKernelMatcher::with_params(12, 8, 4, 2); + // 8-byte real payload appended to history → history.len() = + // 8 + HISTORY_DRAIN_BASE (= 8 under M8), last_hashable = + // HISTORY_DRAIN_BASE (= 0), hashed range = [0..=0] (one + // position). + let payload: alloc::vec::Vec = (0..8u8).collect(); + m.accept_data(payload); + m.skip_matching_with_hint(Some(false)); + assert_eq!(m.history.len(), 8 + HISTORY_DRAIN_BASE); + // No assertion on hash entries — the bug we're guarding + // against is a panic / overrun, not a behavioural one. + // Reaching this line without unwinding is the test. +} + +/// Boundary: pending block too short to hash anything (less than +/// `HASH_READ_SIZE` bytes). The dict-prime path must early-return +/// without panicking on the `last_hashable` subtract. +#[test] +fn skip_matching_dict_prime_handles_below_hash_read_size_bytes() { + let mut m = FastKernelMatcher::with_params(12, 8, 4, 2); + let payload: alloc::vec::Vec = (0..4u8).collect(); + m.accept_data(payload); + // history will be 4 bytes after append < HASH_READ_SIZE (8). + // prime_hash_table_for_range must short-circuit on the + // `history_len < HASH_READ_SIZE` guard. + m.skip_matching_with_hint(Some(false)); + assert_eq!(m.history.len(), 4 + HISTORY_DRAIN_BASE); +} + +/// Regression: priming a dictionary in two chunks must hash the +/// positions straddling the chunk seam. The second chunk's prime +/// starts at `range_start = end-of-first-chunk`, but a hash read at +/// any of the preceding `HASH_READ_SIZE - 1` positions spans the +/// seam into the second chunk — those positions belong to neither +/// chunk's `[range_start ..= last_hashable]` window unless the second +/// prime backfills them. Without the backfill a dictionary fed in +/// `slice_size` chunks silently drops every seam-spanning match. +#[test] +fn dict_prime_indexes_positions_across_chunk_seam() { + // hash_log=20 (1 Mi slots) makes a hash collision among ~32 + // primed positions negligible, so `get(hash(p)) == p` is a + // deterministic assertion on this fixed input. + let mut m = FastKernelMatcher::with_params(20, 20, 4, 2); + let chunk1: alloc::vec::Vec = (0..16u8) + .map(|i| i.wrapping_mul(37).wrapping_add(13)) + .collect(); + m.accept_data(chunk1); + m.skip_matching_for_dict_prime(); + let seam = m.history.len(); // end of first chunk + let chunk2: alloc::vec::Vec = (16..32u8) + .map(|i| i.wrapping_mul(37).wrapping_add(13)) + .collect(); + m.accept_data(chunk2); + m.skip_matching_for_dict_prime(); + + // A position in the (HASH_READ_SIZE - 1)-byte gap just below the + // seam: its 8-byte hash read straddles the chunk boundary. + let p = seam - 4; + let dt = m.dict.table().expect("dict table primed"); + // Read the dict table exactly as the kernel does: hash the position and + // return the stored dict position for that slot (0 if empty). + let found = unsafe { + crate::encoding::simple::fast_kernel::kernel::dict_lookup::<4>( + dt, + m.history.as_ptr().add(p), + dt.hash_log(), + ) + }; + assert_eq!( + found, p as u32, + "seam-spanning position {p} must be indexed in the dict table", + ); +} + +#[test] +fn block_samples_match_dict_fires_only_on_extendable_dict_match() { + // Distinct dict bytes so the 4-byte hashes are collision-free at + // hash_log=20 and a verbatim run is the only way to hit the table. + let dict: alloc::vec::Vec = (0..200u8) + .map(|i| i.wrapping_mul(37).wrapping_add(13)) + .collect(); + let mut m = FastKernelMatcher::with_params(20, 20, 4, 2); + m.accept_data(dict.clone()); + m.skip_matching_for_dict_prime(); + assert!(m.dict_is_attached(), "dict must be attached after prime"); + + // A 32-byte verbatim dict run extends well past the 16-byte useful-match + // floor → the probe fires. + let hit = dict[8..40].to_vec(); + assert!( + m.block_samples_match_dict(&hit), + "a long verbatim dict run must trip the dict probe", + ); + + // An 8-byte dict prefix followed by non-dict bytes stays BELOW the + // 16-byte floor → the probe must NOT fire (a short coincidental match + // does not signal a dict that compresses the block). + let mut short = dict[8..16].to_vec(); + short.extend((0..56u8).map(|i| i.wrapping_mul(53).wrapping_add(201))); + assert!( + !m.block_samples_match_dict(&short), + "a sub-16-byte dict match must not trip the probe", + ); + + // A high-entropy block sharing no extendable run with the dict → no hit. + let miss: alloc::vec::Vec = (0..64u8) + .map(|i| i.wrapping_mul(91).wrapping_add(7) ^ 0xA5) + .collect(); + assert!( + !m.block_samples_match_dict(&miss), + "a non-dict block must not trip the probe", + ); + + // A block too short to hash (< HASH_READ_SIZE) → false, no panic. + assert!( + !m.block_samples_match_dict(&dict[..4]), + "a sub-8-byte block cannot be probed", + ); +} + +#[test] +fn block_samples_match_dict_is_false_without_a_dictionary() { + // No dict primed → the probe short-circuits to false (the no-dict path + // never reaches it, but the guard must hold). + let m = FastKernelMatcher::with_params(20, 20, 4, 2); + let block: alloc::vec::Vec = (0..64u8) + .map(|i| i.wrapping_mul(37).wrapping_add(13)) + .collect(); + assert!(!m.dict_is_attached()); + assert!(!m.block_samples_match_dict(&block)); +} + +/// Regression: same seam-gap defect for the MAIN hash table, primed +/// per slice via `skip_matching_with_hint(Some(false))` (the dict +/// COPY path and multi-slice history priming). Cross-slice matches +/// at the seam are dropped unless the second prime backfills the +/// trailing `HASH_READ_SIZE - 1` positions. +#[test] +fn main_table_prime_indexes_positions_across_slice_seam() { + let mut m = FastKernelMatcher::with_params(20, 20, 4, 2); + let chunk1: alloc::vec::Vec = (0..16u8) + .map(|i| i.wrapping_mul(53).wrapping_add(7)) + .collect(); + m.accept_data(chunk1); + m.skip_matching_with_hint(Some(false)); + let seam = m.history.len(); + let chunk2: alloc::vec::Vec = (16..32u8) + .map(|i| i.wrapping_mul(53).wrapping_add(7)) + .collect(); + m.accept_data(chunk2); + m.skip_matching_with_hint(Some(false)); + + let p = seam - 4; + let h = unsafe { m.hash_table.hash_ptr::<4>(m.history.as_ptr().add(p)) }; + assert_eq!( + unsafe { m.hash_table.get(h) }, + p as u32, + "seam-spanning position {p} must be indexed in the main table", + ); +} + +/// After a single block emits matches, the matcher's `rep[0]` +/// (kernel's `rep_offset1` post-block) must reflect the last emitted +/// explicit offset — that is the state the next block's kernel probes +/// against. The matcher's `offset_hist` is NOT updated by matching: +/// the Fast backend drives repcode probes off `rep`, and the wire +/// offset coding is done downstream against the encode pipeline's own +/// offset history, so per-match `encode_offset_with_history` on the +/// matcher would be redundant work. `offset_hist` therefore stays at +/// whatever `reset` / `prime_offset_history` set it to. +#[test] +fn rep_tracks_last_explicit_offset_and_offset_hist_is_not_matched() { + // Engineer a single block that produces a deterministic + // explicit match. 96 bytes: 48-byte distinct-window + // preamble + 48-byte verbatim copy of bytes [0..48]. + let mut data = alloc::vec::Vec::with_capacity(96); + for i in 0..48u8 { + data.push(i.wrapping_mul(11).wrapping_add(3)); + } + data.extend_from_within(0..48); + + let mut m = FastKernelMatcher::with_params(12, 8, 4, 2); + m.accept_data(data.clone()); + + let mut emitted_offsets: alloc::vec::Vec = alloc::vec::Vec::new(); + m.start_matching(|seq| { + if let Sequence::Triple { + offset, match_len, .. + } = seq + && match_len >= 4 + { + emitted_offsets.push(offset); + } + }); + + assert!( + !emitted_offsets.is_empty(), + "test setup must produce at least one explicit match", + ); + let last_explicit = emitted_offsets[emitted_offsets.len() - 1]; + // rep[0] is the functional state: the kernel updates it from its + // own result each block and the NEXT block probes against it. + assert_eq!( + m.rep[0] as usize, last_explicit, + "kernel's rep[0] must reflect the last emitted explicit offset", + ); + // offset_hist is NOT touched by matching on the Fast backend — it + // stays at the construction default (FAST_INITIAL_OFFSET_HIST). + assert_eq!( + m.offset_hist, FAST_INITIAL_OFFSET_HIST, + "Fast matching must not mutate offset_hist (it is seeded only \ + by reset / prime_offset_history; the wire offset coding runs \ + downstream against the encode pipeline's own history)", + ); +} + +/// Eviction during a dict-priming sequence: when consecutive +/// `skip_matching(Some(false))` calls accumulate past the +/// eviction threshold, the second skip must drop the older +/// prime'd hash entries (via `hash_table.clear()`) AND bump +/// `prefix_start_index` past the dropped bytes. Otherwise the +/// matcher would carry stale absolute positions referencing +/// evicted history. +#[test] +fn eviction_during_dict_priming_drops_stale_prime_entries() { + // window_log=8 → max_window_size=256, threshold=512. + // Two 300-byte blocks both via dict-prime skip — second + // one triggers eviction. + let mut m = FastKernelMatcher::with_params(8, 6, 4, 2); + let block1: alloc::vec::Vec = (0..200u8).collect(); + m.accept_data(block1); + m.skip_matching_with_hint(Some(false)); + let block2: alloc::vec::Vec = (0..200u8).map(|i| i.wrapping_add(50)).collect(); + m.accept_data(block2); + // Second skip would push total to 400, still under 512 — no + // eviction yet. Make sure two more rounds trigger it. + m.skip_matching_with_hint(Some(false)); + let block3: alloc::vec::Vec = (0..200u8).map(|i| i.wrapping_add(100)).collect(); + m.accept_data(block3); + // Now 400+200=600 > 512 → eviction fires inside extend. + m.skip_matching_with_hint(Some(false)); + + // Post-fix: drain rebases prefix_start_index to 1 (rather + // than cumulative saturating_add); eviction is proven by + // bounded history below. + assert_eq!( + m.prefix_start_index, INITIAL_PREFIX_START_INDEX, + "drain must rebase prefix_start_index to the baseline (1)", + ); + // History within the 2× window-size hard cap. + assert!(m.history.len() <= m.max_window_size * 2); +} + +/// Regression for #216 review #1: `accept_data` MUST perform +/// window eviction immediately so the driver's `commit_space` +/// can observe the byte delta via a pre/post `history.len()` +/// comparison. Without commit-time eviction visibility, the +/// driver's `retire_dictionary_budget` never runs for this +/// backend → `max_window_size` stays inflated post-dict-prime +/// → matcher can emit offsets exceeding the frame header's +/// reported window (format-correctness risk). +#[test] +fn accept_data_evicts_eagerly_so_commit_observes_byte_delta() { + // window_log = 8 → max_window_size = 256, eviction threshold + // = 512. Stage three 200-byte blocks via accept_data + a + // start_matching cycle each so history accumulates without + // eviction (200, 400 bytes). The THIRD accept_data crosses + // the 512-byte threshold; its eviction MUST be visible at + // accept_data return-time via the history.len() drop. + let mut m = FastKernelMatcher::with_params(8, 6, 4, 2); + + m.accept_data((0..200u8).collect()); + m.skip_matching_with_hint(None); + assert_eq!(m.history.len(), 200 + HISTORY_DRAIN_BASE); + assert_eq!( + m.prefix_start_index, INITIAL_PREFIX_START_INDEX, + "no eviction yet" + ); + + m.accept_data((0..200u8).map(|i| i.wrapping_add(50)).collect()); + m.skip_matching_with_hint(None); + assert_eq!(m.history.len(), 400 + HISTORY_DRAIN_BASE); + assert_eq!( + m.prefix_start_index, INITIAL_PREFIX_START_INDEX, + "still no eviction (400 < 512)", + ); + + // Third commit: real history (400) + new space (200) = 600 > 512. + // Eviction MUST fire inside accept_data, dropping history + // back to max_window_size (256) BEFORE the kernel runs. + // `history_len_for_eviction_accounting` returns the real-data + // length (history.len() minus HISTORY_DRAIN_BASE, which is 0 + // under M8), so pre/post compare cleanly in real-byte units. + let pre = m.history_len_for_eviction_accounting(); + m.accept_data((0..200u8).map(|i| i.wrapping_add(100)).collect()); + let post = m.history_len_for_eviction_accounting(); + assert!( + pre > post, + "accept_data must shrink history at the eviction threshold \ + (pre={pre}, post={post}) — driver's commit_space relies on \ + this delta for retire_dictionary_budget accounting", + ); + assert_eq!( + post, 256, + "post-eviction retained must equal max_window_size", + ); + assert_eq!( + m.prefix_start_index, INITIAL_PREFIX_START_INDEX, + "drain rebases prefix_start_index to INITIAL_PREFIX_START_INDEX \ + — eviction is proven by the history.len() shrink above", + ); +} + +/// Regression for #216 review #2: `trim_to_window` must update +/// `last_block_start` to track the drain. Without the update, +/// the OLD position references pre-drain coordinates and +/// `last_committed_space()` would either panic with OOB or +/// return wrong bytes when `last_block_start > history.len()` +/// post-drain. +#[test] +fn trim_to_window_keeps_last_committed_space_consistent() { + // window_log = 8 → max_window_size = 256. Process a 200-byte + // block (now in history at positions [0..200], last_block_start + // = 0). Then bump the matcher's max_window_size DOWN to 128 + // (simulating a dictionary-budget retire shrinking the + // window) and call trim_to_window — drain_n = 200 - 128 = 72. + // Post-drain history is bytes [72..200] = 128 bytes. The + // last_block_start (was 0) MUST now be 0 (since 72 > 0 → + // saturating_sub gives 0) so last_committed_space() returns + // a valid in-bounds slice. + let mut m = FastKernelMatcher::with_params(8, 6, 4, 2); + let payload: alloc::vec::Vec = (0..200u8).collect(); + m.accept_data(payload); + m.skip_matching_with_hint(None); + // First block lands at history[RESERVED..RESERVED+200] after + // the seed dummy at [0..RESERVED). last_block_start tracks + // that absolute index. + assert_eq!(m.last_block_start, HISTORY_DRAIN_BASE); + assert_eq!(m.history.len(), 200 + HISTORY_DRAIN_BASE); + + // Shrink the window and trim. Without the fix, last_block_start + // stays mid-history past the post-drain end — to make the + // bug surface, use a SECOND block so last_block_start is + // somewhere AFTER the dummy + first block. + let payload2: alloc::vec::Vec = (50..150u8).collect(); + m.accept_data(payload2); + m.skip_matching_with_hint(None); + // history = [dummy] + [0..200] + [50..150] = 1 + 200 + 100 = 301. + // last_block_start = RESERVED + 200 = start of second block. + assert_eq!(m.last_block_start, HISTORY_DRAIN_BASE + 200); + assert_eq!(m.history.len(), 300 + HISTORY_DRAIN_BASE); + + // Now force trim_to_window to drain into the middle of the + // second block: shrink max_window_size below the second + // block's start. trim_to_window operates on REAL data so + // the drain target is 300 REAL bytes → 64. + m.max_window_size = 64; + let drained = m.trim_to_window(); + assert_eq!( + drained, + 300 - 64, + "trim must drain REAL history down to max_window_size = 64", + ); + // history.len() = RESERVED (dummy preserved) + 64. + assert_eq!(m.history.len(), 64 + HISTORY_DRAIN_BASE); + + // The slice MUST be in bounds — the bug would panic here OR + // return a stale slice. After the fix, last_block_start is + // saturating_sub'd by drained (236) AND clamped to >= RESERVED + // — since drained > old last_block_start - RESERVED, new + // last_block_start = RESERVED, pointing at the current first + // real byte of history (post-drain start of what remains of + // block 2). + let last = m.last_committed_space(); + assert!( + last.len() <= 64, + "last_committed_space must be in-bounds after trim \ + (got len {})", + last.len(), + ); +} + +/// Regression for #216 Copilot review #15: after +/// `prime_offset_history` the kernel's `rep[0..2]` must mirror +/// the wire-encoder's `offset_hist[0..2]` — without this the +/// kernel makes repcode decisions against stale FAST_INITIAL_REP +/// while the wire encoder uses the primed history → wrong +/// repcode wire encoding (correctness bug, not perf). +#[test] +fn prime_offset_history_keeps_rep_and_offset_hist_in_lockstep() { + let mut m = FastKernelMatcher::with_params(12, 8, 4, 2); + // Pre-prime: matcher carries the upstream zstd's initial state. + assert_eq!(m.rep, FAST_INITIAL_REP); + assert_eq!(m.offset_hist, FAST_INITIAL_OFFSET_HIST); + + // Prime with non-default history (upstream zstd's dictionary load + // restores explicit rep1/rep2/rep3 values). + let primed = [9u32, 4, 8]; + m.prime_offset_history(primed); + + // BOTH must reflect the primed values; rep[0..2] = the first + // two entries of offset_hist. + assert_eq!( + m.offset_hist, primed, + "offset_hist must be updated by prime_offset_history", + ); + assert_eq!( + m.rep, + [primed[0], primed[1]], + "rep[0..2] must mirror offset_hist[0..2] post-prime \ + (kernel's repcode decisions must match the wire \ + encoder's seeded history)", + ); +} + +/// Regression for #216 CodeRabbit #19: when a committed block +/// is larger than `max_window_size`, the pre-fix eviction math +/// always retained a full `max_window_size` of historical real +/// data and then appended the (oversized) block, blowing past +/// the documented `2 × max_window_size` post-append bound. +/// Fix retains a SMALLER prefix (or none) so the bound holds. +#[test] +fn accept_data_evicts_more_aggressively_when_block_larger_than_window() { + let mut m = FastKernelMatcher::with_params(8, 6, 4, 2); + // max_window_size = 256 (1 << 8). Threshold = 512. + + // Pre-fill history to full max_window_size of real data via + // skip_matching_with_hint (no kernel-side trimming). u8 range + // tops at 255, so the 256-byte preamble cycles via modulo. + let preamble: alloc::vec::Vec = (0..256u32).map(|i| i as u8).collect(); + m.accept_data(preamble); + m.skip_matching_with_hint(None); + assert_eq!( + m.history_len_for_eviction_accounting(), + 256, + "history pre-filled to one full window of real bytes", + ); + + // Commit a block larger than max_window_size but under 2× + // (so its append should still be allowed without rejecting + // outright). 400 > 256 but < 512. + let oversize: alloc::vec::Vec = (0..400u32) + .map(|i| (i as u8).wrapping_mul(7).wrapping_add(11)) + .collect(); + m.accept_data(oversize); + + // Post-append real_len + space.len() MUST stay within 2× + // max_window_size (the documented invariant). Pre-fix it + // jumped to 256 + 400 = 656 = 2.56× — bound violated. + let real_len_after = m.history_len_for_eviction_accounting(); + // accept_data stashes pending without appending, so the + // 400-byte block is in pending. real_len reflects post- + // drain retained real bytes. To verify the bound, run + // start_matching to commit the append. + m.start_matching(|_| {}); + let real_total = m.history_len_for_eviction_accounting(); + assert!( + real_total <= m.max_window_size * 2, + "post-append history MUST stay within 2 × max_window_size \ + (got real_total={real_total}, cap={})", + m.max_window_size * 2, + ); + // Sanity: pre-append eviction did drain something (we had + // 256 real bytes, can't accept 400 more while staying under + // 512 unless we drop at least 144). + assert!( + real_len_after < 256, + "pre-append drain must have shed historical bytes \ + (got real_len_after_drain={real_len_after}, was 256 \ + before accept)", + ); +} + +/// Regression for PR #219 round 6 (CR Critical): `start_matching` +/// computes an effective sliding prefix floor of +/// `history.len() - max_window_size` so emitted offsets never +/// exceed the advertised `max_window_size`. Without this floor, +/// after history grows past one window (commit's +/// eager-eviction keeps up to 2× max_window_size before +/// draining), the kernel could match against positions older +/// than the frame window — invalid for decoder buffer +/// reservation. +#[test] +fn start_matching_enforces_max_window_size_offset_bound() { + // Tight window: window_log=7 → max_window_size=128. + // Block lengths 200 + 200 = 400 bytes total, comfortably + // past 128 so without the sliding floor the kernel would + // emit matches at offsets > max_window. + let mut m = FastKernelMatcher::with_params(7, 8, 4, 2); + let max_window = m.max_window_size; + assert_eq!(max_window, 128, "test assumes window_log=7"); + + // Block 1: 200 bytes of a distinct ASCII pattern that + // populates the hash table at positions 0..200. + let block1: alloc::vec::Vec = (0..200u8).map(|i| 0x30 + (i % 64)).collect(); + m.accept_data(block1.clone()); + m.start_matching(|_| {}); + + // Block 2: 200 bytes that RE-USE block1's content from + // position 0 onwards. Without the sliding floor, the + // kernel would happily emit Triples with offset + // ≈ history_len (~200..400) — well past max_window=128. + let block2 = block1.clone(); + m.accept_data(block2); + + let mut max_emitted_offset = 0usize; + let mut emitted_match_count = 0usize; + m.start_matching(|seq| { + if let Sequence::Triple { + offset, match_len, .. + } = seq + && match_len > 0 + { + emitted_match_count += 1; + if offset > max_emitted_offset { + max_emitted_offset = offset; + } + } + }); + + // GUARANTEE 1: the kernel actually scanned and matched + // something — otherwise the offset bound below passes + // trivially with max_emitted_offset = 0. + assert!( + emitted_match_count > 0, + "fixture must produce at least one Triple match — \ + history.len()=~400, max_window=128, block2 is identical to block1", + ); + + // GUARANTEE 2: every emitted offset stays within the + // advertised max_window_size. Without the sliding floor, + // matches against early block1 positions (e.g. position + // 0..72) would yield offsets > 128. + assert!( + max_emitted_offset <= max_window, + "sliding floor MUST cap emitted offsets at max_window_size; \ + got max emitted offset {} vs max_window_size {}", + max_emitted_offset, + max_window, + ); +} + +/// Regression for PR #219 round 9 (Copilot Critical): the +/// sliding prefix floor in `start_matching` MUST use the +/// advertised frame window (`1 << window_log`), NOT the +/// dynamically inflated `max_window_size` (which the driver +/// adds dictionary-budget bytes to during +/// `prime_with_dictionary`). With the inflated value, +/// offsets could exceed the advertised window during +/// dictionary-primed compression — format-invalid sequences. +#[test] +fn start_matching_caps_offsets_at_window_log_not_inflated_max() { + // Advertised frame window = 1 << 7 = 128 bytes. + let mut m = FastKernelMatcher::with_params(7, 8, 4, 2); + let advertised_window: usize = 1 << m.window_log; + assert_eq!(advertised_window, 128, "test assumes window_log=7"); + + // Simulate dictionary priming: driver inflates + // max_window_size by retained_dict_budget. We add 200 + // bytes of "dictionary" content to history first, then + // bump max_window_size to reflect the dict-retention + // budget (mirrors MatchGeneratorDriver::prime_with_dictionary + // for the Simple backend). + // + // Pattern period 4 (`(i % 4)`) — dense enough that the upstream zstd- + // parity `kSearchStrength = 8` (K_STEP_INCR = 128) step + // doubling, which skips positions under step=3 in dict scan, + // still leaves matches hittable from block2: every position + // divisible by 4 inside the [in-window, hashed] subset writes + // the same hash slot, so the slot at block2's first probe + // contains a recent in-window dict position. Period 64 (the + // original fixture) only had matches at positions {0, 64, + // 128, 192} — positions 0, 64, 128 are below the sliding + // floor and 192 falls in the step-skip gap, leaving the test + // with zero emittable matches. + let dict: alloc::vec::Vec = (0..200u8).map(|i| 0x40 + (i % 4)).collect(); + m.accept_data(dict.clone()); + m.start_matching(|_| {}); // populate hash table from dict + m.max_window_size = m.max_window_size.saturating_add(200); + + // Add a block whose first 100 bytes match dict[0..100]. + // Without the fix, the kernel would emit offsets up to + // ~history_len (200..300), since the inflated max_window + // (328) keeps even the dict's earliest bytes inside the + // sliding floor. + let block: alloc::vec::Vec = (0..100u8).map(|i| 0x40 + (i % 4)).collect(); + m.accept_data(block); + + let mut max_emitted_offset = 0usize; + let mut emitted_match_count = 0usize; + m.start_matching(|seq| { + if let Sequence::Triple { + offset, match_len, .. + } = seq + && match_len > 0 + { + emitted_match_count += 1; + if offset > max_emitted_offset { + max_emitted_offset = offset; + } + } + }); + + assert!( + emitted_match_count > 0, + "fixture must produce at least one match — block content \ + repeats dict, history.len() ≈ 300, scan should find at \ + least one Triple", + ); + assert!( + max_emitted_offset <= advertised_window, + "sliding floor MUST cap emitted offsets at the ADVERTISED \ + frame window (1 << window_log = {}), NOT the inflated \ + max_window_size; got max emitted offset {}", + advertised_window, + max_emitted_offset, + ); +} + +/// Regression: at block 0 the kernel's prologue must NOT zero +/// `rep_offset1 = 1` (upstream zstd's default initial rep state). Upstream zstd +/// computes `max_rep = ip0 - windowLow` where `windowLow = 0` at +/// block 0, giving `max_rep = 1` at ip0=1 → `rep_offset1 = 1` +/// survives (`1 > 1` is false). +/// +/// Buggy prologue uses `max_rep = ip0 - prefix_start_index` with +/// `prefix_start_index = 1` (sentinel-0 floor for hash-table +/// filtering), giving `max_rep = 0` at ip0=1 → `rep_offset1 = 1 > +/// 0` → stashed → rep-at-ip2 probe disabled for the ENTIRE first +/// block. +/// +/// Symptom assertion: on a `[0x01, 0x42 × 199]` fixture, upstream zstd's +/// rep-at-ip2 fires at iter 1 (ip2=3, both `read32` reads see +/// `[42,42,42,42]`). The upstream zstd emit sequence is: +/// `new_ip = ip2 = 3`, `match0 = ip2 - rep_offset1 = 2`, then the +/// one-byte backward extension absorbs `data[2] == data[1]` +/// (both `0x42`), giving `new_ip = 2, match_len ≈ 198`. Literal +/// prefix is `data[0..new_ip] = [0x01, 0x42]` → length 2. +/// +/// The buggy path skips rep, walks the cursor via explicit-match +/// shifts until matchIdx coincides further into the run, and +/// emits a different literal prefix length. Asserting both +/// `offset == 1` AND `literals.len() == 2` pins down the +/// rep-at-ip2 path exactly — the explicit-match catch-up on +/// uniform-byte data also finds offset=1 via slot collision, so +/// an offset-only check passes both fixed and buggy paths. +#[test] +fn block_zero_prologue_preserves_default_rep_offset_one() { + let mut data = alloc::vec::Vec::with_capacity(200); + data.push(0x01); + data.resize(200, 0x42); + + let mut m = FastKernelMatcher::with_params(12, 8, 4, 2); + m.accept_data(data.clone()); + + let mut first_literals_len: Option = None; + let mut first_offset: Option = None; + m.start_matching(|seq| { + if first_literals_len.is_some() { + return; + } + if let Sequence::Triple { + literals, offset, .. + } = seq + { + first_literals_len = Some(literals.len()); + first_offset = Some(offset); + } + }); + + // Both `offset` AND `literals.len()` are asserted — together + // they pin down EXACTLY which inner-loop path emitted the + // first match. With the prologue bug (rep-at-ip2 disabled + // because `max_rep` was computed against `prefix_start_index` + // instead of `window_low`), the explicit-match path catches + // up via hash-slot collision at ip0=3 and STILL emits + // offset=1 — so an offset-only check would pass both fixed + // and buggy paths on this uniform-byte fixture. The literal + // prefix length is the actual discriminator: the rep-at-ip2 + // path consumes only the leading `0x01` literal (1 byte), + // while the explicit-match catch-up walks past two more + // `0x42` bytes before firing (3 bytes total). Asserting both + // values keeps the regression locked to the exact path the + // fix was meant to preserve. + assert_eq!( + first_offset, + Some(1), + "first emit must reference offset=1 — upstream zstd's default \ + rep_offset1=1 fires on rep-at-ip2 at iter 1, and the \ + prologue MUST NOT zero it (max_rep computed against \ + window_low=0 at block 0, NOT against the sentinel \ + prefix=1)", + ); + assert_eq!( + first_literals_len, + Some(2), + "first emit must have a 2-byte literal prefix \ + ([0x01, 0x42]) — the rep-at-ip2 probe lands at ip2=3, \ + then the one-byte backward extension drops new_ip to 2, \ + so literals = data[0..2]. A different prefix length \ + would indicate the explicit-match catch-up fired instead", + ); +} diff --git a/zstd/src/encoding/strategy.rs b/zstd/src/encoding/strategy.rs index 0ed60b20c..c76ee2e6d 100644 --- a/zstd/src/encoding/strategy.rs +++ b/zstd/src/encoding/strategy.rs @@ -486,149 +486,4 @@ impl StrategyTag { } #[cfg(test)] -mod tests { - use super::*; - - fn assert_strategy_matches_tag(tag: StrategyTag) { - assert_eq!(S::BACKEND, tag.backend(), "backend mismatch"); - } - - #[test] - fn strategy_consts_match_tag_bridge() { - assert_strategy_matches_tag::(StrategyTag::Fast); - assert_strategy_matches_tag::(StrategyTag::Dfast); - assert_strategy_matches_tag::(StrategyTag::Greedy); - assert_strategy_matches_tag::(StrategyTag::Lazy); - assert_strategy_matches_tag::(StrategyTag::Btlazy2); - assert_strategy_matches_tag::(StrategyTag::BtOpt); - assert_strategy_matches_tag::(StrategyTag::BtUltra); - assert_strategy_matches_tag::(StrategyTag::BtUltra2); - } - - /// Pin the `Btlazy2` tag's full bridge: it runs the BinaryTree finder on - /// the HashChain backend with a Lazy parse (upstream zstd `ZSTD_btlazy2`). - #[test] - fn btlazy2_tag_bridge_contract() { - assert_eq!(StrategyTag::Btlazy2.backend(), BackendTag::HashChain); - assert_eq!(StrategyTag::Btlazy2.search(), SearchMethod::BinaryTree); - assert_eq!(StrategyTag::Btlazy2.parse_mode(), ParseMode::Lazy2); - // The BT walk cap must let L15's search_depth = 64 govern (BtOpt's - // 32 would silently halve it); full find, no early bail. - assert_eq!(Btlazy2::MAX_CHAIN_DEPTH, 64); - assert_eq!(Btlazy2::SUFFICIENT_MATCH_LEN, usize::MAX); - } - - #[test] - fn for_compression_level_clamps_oversized_numeric_levels_to_btultra2() { - // Regression: pre-fix `Level(256)` cast `n as u8` first, - // wrapping to `0` and routing to `Dfast`. After clamp-then- - // cast every level above MAX_LEVEL (22) must land on - // BtUltra2 (the saturating top of the band). - use crate::encoding::CompressionLevel; - assert_eq!( - StrategyTag::for_compression_level(CompressionLevel::Level(23)), - StrategyTag::BtUltra2, - ); - assert_eq!( - StrategyTag::for_compression_level(CompressionLevel::Level(255)), - StrategyTag::BtUltra2, - ); - assert_eq!( - StrategyTag::for_compression_level(CompressionLevel::Level(256)), - StrategyTag::BtUltra2, - ); - assert_eq!( - StrategyTag::for_compression_level(CompressionLevel::Level(257)), - StrategyTag::BtUltra2, - ); - assert_eq!( - StrategyTag::for_compression_level(CompressionLevel::Level(i32::MAX)), - StrategyTag::BtUltra2, - ); - } - - #[test] - fn level_to_tag_matches_default_table() { - // Spot-check every band boundary and one mid-band level. - assert_eq!(StrategyTag::for_level(1), StrategyTag::Fast); - assert_eq!(StrategyTag::for_level(2), StrategyTag::Fast); - assert_eq!(StrategyTag::for_level(3), StrategyTag::Dfast); - assert_eq!(StrategyTag::for_level(4), StrategyTag::Dfast); - assert_eq!(StrategyTag::for_level(5), StrategyTag::Greedy); - assert_eq!(StrategyTag::for_level(9), StrategyTag::Lazy); - assert_eq!(StrategyTag::for_level(12), StrategyTag::Lazy); - // Upstream zstd `clevels.h` 13-15 are `ZSTD_btlazy2` — distinct from the - // Row-backed `Lazy` band. - assert_eq!(StrategyTag::for_level(13), StrategyTag::Btlazy2); - assert_eq!(StrategyTag::for_level(15), StrategyTag::Btlazy2); - assert_eq!(StrategyTag::for_level(16), StrategyTag::BtOpt); - assert_eq!(StrategyTag::for_level(17), StrategyTag::BtOpt); - assert_eq!(StrategyTag::for_level(18), StrategyTag::BtUltra); - // Upstream zstd `clevels.h` level 19 uses `ZSTD_btultra2` (searchLog 7, - // two-pass dynamic stats + hash3), not plain btultra. - assert_eq!(StrategyTag::for_level(19), StrategyTag::BtUltra2); - assert_eq!(StrategyTag::for_level(20), StrategyTag::BtUltra2); - assert_eq!(StrategyTag::for_level(22), StrategyTag::BtUltra2); - } - - // The next three blocks live at module scope so the assertions - // run at compile time and never reach the `cargo nextest` runner. - // `clippy::assertions_on_constants` requires this form for - // const-only inputs. - - // `use_bt_aligns_with_parse_mode`: Lazy2 strategies must not walk - // the BT; BtOpt / BtUltra / BtUltra2 must. Invariant that lets - // the inner optimal parser drop the `if self.parse_mode == Lazy2 - // …` branch in favour of `if !S::USE_BT`. - const _USE_BT_LAYOUT: () = { - assert!(!Fast::USE_BT); - assert!(!Dfast::USE_BT); - assert!(!Greedy::USE_BT); - assert!(!Lazy::USE_BT); - assert!(Btlazy2::USE_BT); - assert!(BtOpt::USE_BT); - assert!(BtUltra::USE_BT); - assert!(BtUltra2::USE_BT); - }; - - // hash3 short-match probe: active for btultra + btultra2 (clevels.h - // minMatch 3); btopt and below do not search it. The in-block - // two-pass dynamic-stats seed is btultra2-only. - const _USE_HASH3_LAYOUT: () = { - assert!(!Fast::USE_HASH3); - assert!(!Dfast::USE_HASH3); - assert!(!Greedy::USE_HASH3); - assert!(!Lazy::USE_HASH3); - assert!(!Btlazy2::USE_HASH3); - assert!(!Btlazy2::TWO_PASS_SEED); - assert!(!BtOpt::USE_HASH3); - assert!(BtUltra::USE_HASH3); - assert!(BtUltra2::USE_HASH3); - assert!(!BtOpt::TWO_PASS_SEED); - assert!(!BtUltra::TWO_PASS_SEED); - assert!(BtUltra2::TWO_PASS_SEED); - }; - - // Mirror the per-strategy fields the optimal-parser cost profile - // is built from, so the layout (accurate / favor_small_offsets / - // max_chain_depth / sufficient_match_len) cannot regress - // silently. - const _COST_MODEL_LAYOUT: () = { - assert!(!Lazy::ACCURATE_PRICE && Lazy::FAVOR_SMALL_OFFSETS); - assert!(!Btlazy2::ACCURATE_PRICE && Btlazy2::FAVOR_SMALL_OFFSETS); - assert!(!BtOpt::ACCURATE_PRICE && BtOpt::FAVOR_SMALL_OFFSETS); - assert!(BtUltra::ACCURATE_PRICE && !BtUltra::FAVOR_SMALL_OFFSETS); - assert!(BtUltra2::ACCURATE_PRICE && !BtUltra2::FAVOR_SMALL_OFFSETS); - // btlazy2 runs the full BT find (no early bail) and a search depth - // that lets L15's configured search_depth=64 govern; see `Btlazy2`. - assert!(Btlazy2::MAX_CHAIN_DEPTH == 64); - assert!(Btlazy2::SUFFICIENT_MATCH_LEN == usize::MAX); - assert!(BtOpt::MAX_CHAIN_DEPTH == 32); - // 1 << searchLog for clevels.h level 18 (searchLog = 6). - assert!(BtUltra::MAX_CHAIN_DEPTH == 64); - assert!(BtUltra2::MAX_CHAIN_DEPTH == 512); - assert!(BtOpt::SUFFICIENT_MATCH_LEN == usize::MAX); - assert!(BtUltra::SUFFICIENT_MATCH_LEN == usize::MAX); - assert!(BtUltra2::SUFFICIENT_MATCH_LEN == usize::MAX); - }; -} +mod tests; diff --git a/zstd/src/encoding/strategy/tests.rs b/zstd/src/encoding/strategy/tests.rs new file mode 100644 index 000000000..a5d05ac4f --- /dev/null +++ b/zstd/src/encoding/strategy/tests.rs @@ -0,0 +1,144 @@ +use super::*; + +fn assert_strategy_matches_tag(tag: StrategyTag) { + assert_eq!(S::BACKEND, tag.backend(), "backend mismatch"); +} + +#[test] +fn strategy_consts_match_tag_bridge() { + assert_strategy_matches_tag::(StrategyTag::Fast); + assert_strategy_matches_tag::(StrategyTag::Dfast); + assert_strategy_matches_tag::(StrategyTag::Greedy); + assert_strategy_matches_tag::(StrategyTag::Lazy); + assert_strategy_matches_tag::(StrategyTag::Btlazy2); + assert_strategy_matches_tag::(StrategyTag::BtOpt); + assert_strategy_matches_tag::(StrategyTag::BtUltra); + assert_strategy_matches_tag::(StrategyTag::BtUltra2); +} + +/// Pin the `Btlazy2` tag's full bridge: it runs the BinaryTree finder on +/// the HashChain backend with a Lazy parse (upstream zstd `ZSTD_btlazy2`). +#[test] +fn btlazy2_tag_bridge_contract() { + assert_eq!(StrategyTag::Btlazy2.backend(), BackendTag::HashChain); + assert_eq!(StrategyTag::Btlazy2.search(), SearchMethod::BinaryTree); + assert_eq!(StrategyTag::Btlazy2.parse_mode(), ParseMode::Lazy2); + // The BT walk cap must let L15's search_depth = 64 govern (BtOpt's + // 32 would silently halve it); full find, no early bail. + assert_eq!(Btlazy2::MAX_CHAIN_DEPTH, 64); + assert_eq!(Btlazy2::SUFFICIENT_MATCH_LEN, usize::MAX); +} + +#[test] +fn for_compression_level_clamps_oversized_numeric_levels_to_btultra2() { + // Regression: pre-fix `Level(256)` cast `n as u8` first, + // wrapping to `0` and routing to `Dfast`. After clamp-then- + // cast every level above MAX_LEVEL (22) must land on + // BtUltra2 (the saturating top of the band). + use crate::encoding::CompressionLevel; + assert_eq!( + StrategyTag::for_compression_level(CompressionLevel::Level(23)), + StrategyTag::BtUltra2, + ); + assert_eq!( + StrategyTag::for_compression_level(CompressionLevel::Level(255)), + StrategyTag::BtUltra2, + ); + assert_eq!( + StrategyTag::for_compression_level(CompressionLevel::Level(256)), + StrategyTag::BtUltra2, + ); + assert_eq!( + StrategyTag::for_compression_level(CompressionLevel::Level(257)), + StrategyTag::BtUltra2, + ); + assert_eq!( + StrategyTag::for_compression_level(CompressionLevel::Level(i32::MAX)), + StrategyTag::BtUltra2, + ); +} + +#[test] +fn level_to_tag_matches_default_table() { + // Spot-check every band boundary and one mid-band level. + assert_eq!(StrategyTag::for_level(1), StrategyTag::Fast); + assert_eq!(StrategyTag::for_level(2), StrategyTag::Fast); + assert_eq!(StrategyTag::for_level(3), StrategyTag::Dfast); + assert_eq!(StrategyTag::for_level(4), StrategyTag::Dfast); + assert_eq!(StrategyTag::for_level(5), StrategyTag::Greedy); + assert_eq!(StrategyTag::for_level(9), StrategyTag::Lazy); + assert_eq!(StrategyTag::for_level(12), StrategyTag::Lazy); + // Upstream zstd `clevels.h` 13-15 are `ZSTD_btlazy2` — distinct from the + // Row-backed `Lazy` band. + assert_eq!(StrategyTag::for_level(13), StrategyTag::Btlazy2); + assert_eq!(StrategyTag::for_level(15), StrategyTag::Btlazy2); + assert_eq!(StrategyTag::for_level(16), StrategyTag::BtOpt); + assert_eq!(StrategyTag::for_level(17), StrategyTag::BtOpt); + assert_eq!(StrategyTag::for_level(18), StrategyTag::BtUltra); + // Upstream zstd `clevels.h` level 19 uses `ZSTD_btultra2` (searchLog 7, + // two-pass dynamic stats + hash3), not plain btultra. + assert_eq!(StrategyTag::for_level(19), StrategyTag::BtUltra2); + assert_eq!(StrategyTag::for_level(20), StrategyTag::BtUltra2); + assert_eq!(StrategyTag::for_level(22), StrategyTag::BtUltra2); +} + +// The next three blocks live at module scope so the assertions +// run at compile time and never reach the `cargo nextest` runner. +// `clippy::assertions_on_constants` requires this form for +// const-only inputs. + +// `use_bt_aligns_with_parse_mode`: Lazy2 strategies must not walk +// the BT; BtOpt / BtUltra / BtUltra2 must. Invariant that lets +// the inner optimal parser drop the `if self.parse_mode == Lazy2 +// …` branch in favour of `if !S::USE_BT`. +const _USE_BT_LAYOUT: () = { + assert!(!Fast::USE_BT); + assert!(!Dfast::USE_BT); + assert!(!Greedy::USE_BT); + assert!(!Lazy::USE_BT); + assert!(Btlazy2::USE_BT); + assert!(BtOpt::USE_BT); + assert!(BtUltra::USE_BT); + assert!(BtUltra2::USE_BT); +}; + +// hash3 short-match probe: active for btultra + btultra2 (clevels.h +// minMatch 3); btopt and below do not search it. The in-block +// two-pass dynamic-stats seed is btultra2-only. +const _USE_HASH3_LAYOUT: () = { + assert!(!Fast::USE_HASH3); + assert!(!Dfast::USE_HASH3); + assert!(!Greedy::USE_HASH3); + assert!(!Lazy::USE_HASH3); + assert!(!Btlazy2::USE_HASH3); + assert!(!Btlazy2::TWO_PASS_SEED); + assert!(!BtOpt::USE_HASH3); + assert!(BtUltra::USE_HASH3); + assert!(BtUltra2::USE_HASH3); + assert!(!BtOpt::TWO_PASS_SEED); + assert!(!BtUltra::TWO_PASS_SEED); + assert!(BtUltra2::TWO_PASS_SEED); +}; + +// Mirror the per-strategy fields the optimal-parser cost profile +// is built from, so the layout (accurate / favor_small_offsets / +// max_chain_depth / sufficient_match_len) cannot regress +// silently. +const _COST_MODEL_LAYOUT: () = { + assert!(!Lazy::ACCURATE_PRICE && Lazy::FAVOR_SMALL_OFFSETS); + assert!(!Btlazy2::ACCURATE_PRICE && Btlazy2::FAVOR_SMALL_OFFSETS); + assert!(!BtOpt::ACCURATE_PRICE && BtOpt::FAVOR_SMALL_OFFSETS); + assert!(BtUltra::ACCURATE_PRICE && !BtUltra::FAVOR_SMALL_OFFSETS); + assert!(BtUltra2::ACCURATE_PRICE && !BtUltra2::FAVOR_SMALL_OFFSETS); + // btlazy2 runs the full BT find (no early bail) and a search depth + // that lets L15's configured search_depth=64 govern; see `Btlazy2`. + assert!(Btlazy2::MAX_CHAIN_DEPTH == 64); + assert!(Btlazy2::SUFFICIENT_MATCH_LEN == usize::MAX); + assert!(BtOpt::MAX_CHAIN_DEPTH == 32); + // 1 << searchLog for clevels.h level 18 (searchLog = 6). + assert!(BtUltra::MAX_CHAIN_DEPTH == 64); + assert!(BtUltra2::MAX_CHAIN_DEPTH == 512); + assert!(BtOpt::SUFFICIENT_MATCH_LEN == usize::MAX); + assert!(BtUltra::SUFFICIENT_MATCH_LEN == usize::MAX); + assert!(BtUltra2::SUFFICIENT_MATCH_LEN == usize::MAX); +}; diff --git a/zstd/src/encoding/streaming_encoder.rs b/zstd/src/encoding/streaming_encoder.rs index 4b3852a98..9225c9f3e 100644 --- a/zstd/src/encoding/streaming_encoder.rs +++ b/zstd/src/encoding/streaming_encoder.rs @@ -972,899 +972,4 @@ fn other_error(message: &str) -> Error { } #[cfg(test)] -mod tests { - use crate::decoding::StreamingDecoder; - use crate::encoding::{CompressionLevel, Matcher, Sequence, StreamingEncoder}; - use crate::io::{Error, ErrorKind, Read, Write}; - use alloc::vec; - use alloc::vec::Vec; - - struct TinyMatcher { - last_space: Vec, - window_size: u64, - } - - impl TinyMatcher { - fn new(window_size: u64) -> Self { - Self { - last_space: Vec::new(), - window_size, - } - } - } - - impl Matcher for TinyMatcher { - fn get_next_space(&mut self) -> Vec { - vec![0; self.window_size as usize] - } - - fn get_last_space(&mut self) -> &[u8] { - self.last_space.as_slice() - } - - fn commit_space(&mut self, space: Vec) { - self.last_space = space; - } - - fn skip_matching(&mut self) {} - - fn start_matching(&mut self, mut handle_sequence: impl for<'a> FnMut(Sequence<'a>)) { - handle_sequence(Sequence::Literals { - literals: self.last_space.as_slice(), - }); - } - - fn reset(&mut self, _level: CompressionLevel) { - self.last_space.clear(); - } - - fn window_size(&self) -> u64 { - self.window_size - } - } - - struct FailingWriteOnce { - writes: usize, - fail_on_write_number: usize, - sink: Vec, - } - - impl FailingWriteOnce { - fn new(fail_on_write_number: usize) -> Self { - Self { - writes: 0, - fail_on_write_number, - sink: Vec::new(), - } - } - } - - impl Write for FailingWriteOnce { - fn write(&mut self, buf: &[u8]) -> Result { - self.writes += 1; - if self.writes == self.fail_on_write_number { - return Err(super::other_error("injected write failure")); - } - self.sink.extend_from_slice(buf); - Ok(buf.len()) - } - - fn flush(&mut self) -> Result<(), Error> { - Ok(()) - } - } - - struct FailingWithKind { - writes: usize, - fail_on_write_number: usize, - kind: ErrorKind, - } - - impl FailingWithKind { - fn new(fail_on_write_number: usize, kind: ErrorKind) -> Self { - Self { - writes: 0, - fail_on_write_number, - kind, - } - } - } - - impl Write for FailingWithKind { - fn write(&mut self, buf: &[u8]) -> Result { - self.writes += 1; - if self.writes == self.fail_on_write_number { - return Err(Error::from(self.kind)); - } - Ok(buf.len()) - } - - fn flush(&mut self) -> Result<(), Error> { - Ok(()) - } - } - - struct PartialThenFailWriter { - writes: usize, - fail_on_write_number: usize, - partial_prefix_len: usize, - terminal_failure: bool, - sink: Vec, - } - - impl PartialThenFailWriter { - fn new(fail_on_write_number: usize, partial_prefix_len: usize) -> Self { - Self { - writes: 0, - fail_on_write_number, - partial_prefix_len, - terminal_failure: false, - sink: Vec::new(), - } - } - } - - impl Write for PartialThenFailWriter { - fn write(&mut self, buf: &[u8]) -> Result { - if self.terminal_failure { - return Err(super::other_error("injected terminal write failure")); - } - - self.writes += 1; - if self.writes == self.fail_on_write_number { - let written = core::cmp::min(self.partial_prefix_len, buf.len()); - if written > 0 { - self.sink.extend_from_slice(&buf[..written]); - self.terminal_failure = true; - return Ok(written); - } - return Err(super::other_error("injected terminal write failure")); - } - - self.sink.extend_from_slice(buf); - Ok(buf.len()) - } - - fn flush(&mut self) -> Result<(), Error> { - Ok(()) - } - } - - /// Pre-write `set_magicless(true)` → emitted frame omits the - /// magic prefix AND round-trips through a magicless-aware - /// decoder. - #[test] - fn streaming_encoder_set_magicless_before_write_omits_magic_and_roundtrips() { - use crate::common::MAGIC_NUM; - let payload = b"streaming-magicless-roundtrip-".repeat(64); - - let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); - encoder - .set_magicless(true) - .expect("set_magicless pre-write"); - encoder.write_all(&payload).unwrap(); - let compressed = encoder.finish().unwrap(); - - assert!( - !compressed.starts_with(&MAGIC_NUM.to_le_bytes()), - "magicless frame must omit the 4-byte magic prefix", - ); - - let mut decoder = crate::decoding::FrameDecoder::new(); - decoder.set_magicless(true); - let mut cursor: &[u8] = compressed.as_slice(); - decoder.init(&mut cursor).expect("magicless init"); - decoder - .decode_blocks(&mut cursor, crate::decoding::BlockDecodingStrategy::All) - .expect("decode_blocks"); - let mut decoded: Vec = Vec::new(); - decoder - .collect_to_writer(&mut decoded) - .expect("collect_to_writer"); - assert_eq!(decoded, payload); - } - - /// `set_magicless` after the first write MUST return an error - /// (the frame header has already been emitted, flipping the flag - /// can't affect the current frame). Mirrors - /// `set_pledged_content_size` / `set_source_size_hint` semantics. - #[test] - fn streaming_encoder_set_magicless_after_first_write_errors() { - let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); - encoder.write_all(b"first-block").unwrap(); - let err = encoder - .set_magicless(true) - .expect_err("set_magicless after first write must error"); - assert_eq!( - err.kind(), - crate::io::ErrorKind::InvalidInput, - "expected InvalidInput when setting magicless after frame_started, got {err:?}", - ); - } - - #[test] - fn streaming_encoder_roundtrip_multiple_writes() { - let payload = b"streaming-encoder-roundtrip-".repeat(1024); - let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); - for chunk in payload.chunks(313) { - encoder.write_all(chunk).unwrap(); - } - let compressed = encoder.finish().unwrap(); - - let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); - let mut decoded = Vec::new(); - decoder.read_to_end(&mut decoded).unwrap(); - assert_eq!(decoded, payload); - } - - #[test] - fn flush_emits_nonempty_partial_output() { - let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); - encoder.write_all(b"partial-block").unwrap(); - encoder.flush().unwrap(); - let flushed_len = encoder.get_ref().len(); - assert!( - flushed_len > 0, - "flush should emit header+partial block bytes" - ); - let compressed = encoder.finish().unwrap(); - let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); - let mut decoded = Vec::new(); - decoder.read_to_end(&mut decoded).unwrap(); - assert_eq!(decoded, b"partial-block"); - } - - #[test] - fn flush_without_writes_does_not_emit_frame_header() { - let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); - encoder.flush().unwrap(); - assert!(encoder.get_ref().is_empty()); - } - - #[test] - fn block_boundary_write_emits_block_in_same_call() { - let mut boundary = StreamingEncoder::new_with_matcher( - TinyMatcher::new(4), - Vec::new(), - CompressionLevel::Uncompressed, - ); - let mut below = StreamingEncoder::new_with_matcher( - TinyMatcher::new(4), - Vec::new(), - CompressionLevel::Uncompressed, - ); - - boundary.write_all(b"ABCD").unwrap(); - below.write_all(b"ABC").unwrap(); - - let boundary_len = boundary.get_ref().len(); - let below_len = below.get_ref().len(); - assert!( - boundary_len > below_len, - "full block should be emitted immediately at block boundary" - ); - } - - #[test] - fn finish_consumes_encoder_and_emits_frame() { - let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); - encoder.write_all(b"abc").unwrap(); - let compressed = encoder.finish().unwrap(); - let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); - let mut decoded = Vec::new(); - decoder.read_to_end(&mut decoded).unwrap(); - assert_eq!(decoded, b"abc"); - } - - #[test] - fn finish_without_writes_emits_empty_frame() { - let encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); - let compressed = encoder.finish().unwrap(); - let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); - let mut decoded = Vec::new(); - decoder.read_to_end(&mut decoded).unwrap(); - assert!(decoded.is_empty()); - } - - #[test] - fn write_empty_buffer_returns_zero() { - let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); - assert_eq!(encoder.write(&[]).unwrap(), 0); - let _ = encoder.finish().unwrap(); - } - - #[test] - fn uncompressed_level_roundtrip() { - let payload = b"uncompressed-streaming-roundtrip".repeat(64); - let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Uncompressed); - for chunk in payload.chunks(41) { - encoder.write_all(chunk).unwrap(); - } - let compressed = encoder.finish().unwrap(); - let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); - let mut decoded = Vec::new(); - decoder.read_to_end(&mut decoded).unwrap(); - assert_eq!(decoded, payload); - } - - #[test] - fn better_level_streaming_roundtrip() { - let payload = b"better-level-streaming-test".repeat(256); - let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Better); - for chunk in payload.chunks(53) { - encoder.write_all(chunk).unwrap(); - } - let compressed = encoder.finish().unwrap(); - let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); - let mut decoded = Vec::new(); - decoder.read_to_end(&mut decoded).unwrap(); - assert_eq!(decoded, payload); - } - - #[test] - fn zero_window_matcher_returns_invalid_input_error() { - let mut encoder = StreamingEncoder::new_with_matcher( - TinyMatcher::new(0), - Vec::new(), - CompressionLevel::Fastest, - ); - let err = encoder.write_all(b"payload").unwrap_err(); - assert_eq!(err.kind(), ErrorKind::InvalidInput); - } - - #[test] - fn best_level_streaming_roundtrip() { - // 200 KiB payload crosses the 128 KiB block boundary, exercising - // multi-block emission and matcher state carry-over for Best. - let payload = b"best-level-streaming-test".repeat(8 * 1024); - let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Best); - for chunk in payload.chunks(53) { - encoder.write_all(chunk).unwrap(); - } - let compressed = encoder.finish().unwrap(); - let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); - let mut decoded = Vec::new(); - decoder.read_to_end(&mut decoded).unwrap(); - assert_eq!(decoded, payload); - } - - #[test] - fn write_failure_poisoning_is_sticky() { - let mut encoder = StreamingEncoder::new_with_matcher( - TinyMatcher::new(4), - FailingWriteOnce::new(1), - CompressionLevel::Uncompressed, - ); - - assert!(encoder.write_all(b"ABCD").is_err()); - assert!(encoder.flush().is_err()); - assert!(encoder.write_all(b"EFGH").is_err()); - assert_eq!(encoder.get_ref().sink.len(), 0); - assert!(encoder.finish().is_err()); - } - - #[test] - fn poisoned_encoder_returns_original_error_kind() { - let mut encoder = StreamingEncoder::new_with_matcher( - TinyMatcher::new(4), - FailingWithKind::new(1, ErrorKind::BrokenPipe), - CompressionLevel::Uncompressed, - ); - - let first_error = encoder.write_all(b"ABCD").unwrap_err(); - assert_eq!(first_error.kind(), ErrorKind::BrokenPipe); - - let second_error = encoder.write_all(b"EFGH").unwrap_err(); - assert_eq!(second_error.kind(), ErrorKind::BrokenPipe); - } - - #[test] - fn write_reports_progress_but_poisoning_is_sticky_after_later_block_failure() { - let payload = b"ABCDEFGHIJKL"; - let mut encoder = StreamingEncoder::new_with_matcher( - TinyMatcher::new(4), - FailingWriteOnce::new(3), - CompressionLevel::Uncompressed, - ); - - let first_write = encoder.write(payload).unwrap(); - assert_eq!(first_write, 8); - assert!(encoder.write(&payload[first_write..]).is_err()); - assert!(encoder.flush().is_err()); - assert!(encoder.write_all(b"EFGH").is_err()); - } - - #[test] - fn partial_write_failure_after_progress_poisons_encoder() { - let payload = b"ABCDEFGHIJKL"; - let mut encoder = StreamingEncoder::new_with_matcher( - TinyMatcher::new(4), - PartialThenFailWriter::new(3, 1), - CompressionLevel::Uncompressed, - ); - - let first_write = encoder.write(payload).unwrap(); - assert_eq!(first_write, 8); - - let second_write = encoder.write(&payload[first_write..]); - assert!(second_write.is_err()); - assert!(encoder.flush().is_err()); - assert!(encoder.write_all(b"MNOP").is_err()); - } - - #[test] - fn new_with_matcher_and_get_mut_work() { - let matcher = TinyMatcher::new(128 * 1024); - let mut encoder = - StreamingEncoder::new_with_matcher(matcher, Vec::new(), CompressionLevel::Fastest); - encoder.get_mut().extend_from_slice(b""); - encoder.write_all(b"custom-matcher").unwrap(); - let compressed = encoder.finish().unwrap(); - let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); - let mut decoded = Vec::new(); - decoder.read_to_end(&mut decoded).unwrap(); - assert_eq!(decoded, b"custom-matcher"); - } - - #[test] - fn pledged_content_size_written_in_header() { - let payload = b"hello world, pledged size test"; - let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); - encoder - .set_pledged_content_size(payload.len() as u64) - .unwrap(); - encoder.write_all(payload).unwrap(); - let compressed = encoder.finish().unwrap(); - - // Verify FCS is present and correct - let header = crate::decoding::frame::read_frame_header(compressed.as_slice()) - .unwrap() - .0; - assert_eq!(header.frame_content_size(), payload.len() as u64); - - // Verify roundtrip - let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); - let mut decoded = Vec::new(); - decoder.read_to_end(&mut decoded).unwrap(); - assert_eq!(decoded, payload); - } - - #[test] - fn pledged_content_size_mismatch_returns_error() { - let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); - encoder.set_pledged_content_size(100).unwrap(); - encoder.write_all(b"short payload").unwrap(); // 13 bytes != 100 pledged - let err = encoder.finish().unwrap_err(); - assert_eq!(err.kind(), ErrorKind::InvalidInput); - } - - #[test] - fn write_exceeding_pledge_returns_error() { - let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); - encoder.set_pledged_content_size(5).unwrap(); - let err = encoder.write_all(b"exceeds five bytes").unwrap_err(); - assert_eq!(err.kind(), ErrorKind::InvalidInput); - } - - #[test] - fn write_straddling_pledge_reports_partial_progress() { - let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); - encoder.set_pledged_content_size(5).unwrap(); - // write() should accept exactly 5 bytes (partial progress) - assert_eq!(encoder.write(b"abcdef").unwrap(), 5); - // Next write should fail — pledge exhausted - let err = encoder.write(b"g").unwrap_err(); - assert_eq!(err.kind(), ErrorKind::InvalidInput); - } - - #[test] - fn encoded_scratch_capacity_is_reused_across_blocks() { - let payload = vec![0xAB; 64 * 3]; - let mut encoder = StreamingEncoder::new_with_matcher( - TinyMatcher::new(64), - Vec::new(), - CompressionLevel::Uncompressed, - ); - - encoder.write_all(&payload[..64]).unwrap(); - let first_capacity = encoder.encoded_scratch.capacity(); - assert!( - first_capacity >= 67, - "expected encoded scratch to keep block header + payload capacity", - ); - - encoder.write_all(&payload[64..128]).unwrap(); - let second_capacity = encoder.encoded_scratch.capacity(); - assert!( - second_capacity >= first_capacity, - "encoded scratch capacity should be reused across block emits", - ); - - encoder.write_all(&payload[128..]).unwrap(); - let compressed = encoder.finish().unwrap(); - let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); - let mut decoded = Vec::new(); - decoder.read_to_end(&mut decoded).unwrap(); - assert_eq!(decoded, payload); - } - - #[test] - fn pledged_content_size_after_write_returns_error() { - let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); - encoder.write_all(b"already writing").unwrap(); - let err = encoder.set_pledged_content_size(15).unwrap_err(); - assert_eq!(err.kind(), ErrorKind::InvalidInput); - } - - #[test] - fn source_size_hint_directly_reduces_window_header() { - let payload = b"streaming-source-size-hint".repeat(64); - - let mut no_hint = StreamingEncoder::new(Vec::new(), CompressionLevel::from_level(11)); - no_hint.write_all(payload.as_slice()).unwrap(); - let no_hint_frame = no_hint.finish().unwrap(); - let no_hint_header = crate::decoding::frame::read_frame_header(no_hint_frame.as_slice()) - .unwrap() - .0; - let no_hint_window = no_hint_header.window_size().unwrap(); - - let mut with_hint = StreamingEncoder::new(Vec::new(), CompressionLevel::from_level(11)); - with_hint - .set_source_size_hint(payload.len() as u64) - .unwrap(); - with_hint.write_all(payload.as_slice()).unwrap(); - let late_hint_err = with_hint - .set_source_size_hint(payload.len() as u64) - .unwrap_err(); - assert_eq!(late_hint_err.kind(), ErrorKind::InvalidInput); - let with_hint_frame = with_hint.finish().unwrap(); - let with_hint_header = - crate::decoding::frame::read_frame_header(with_hint_frame.as_slice()) - .unwrap() - .0; - let with_hint_window = with_hint_header.window_size().unwrap(); - - assert!( - with_hint_window <= no_hint_window, - "source size hint should not increase advertised window" - ); - - let mut decoder = StreamingDecoder::new(with_hint_frame.as_slice()).unwrap(); - let mut decoded = Vec::new(); - decoder.read_to_end(&mut decoded).unwrap(); - assert_eq!(decoded, payload); - } - - #[test] - fn single_segment_requires_pledged_to_fit_matcher_window() { - let payload = b"streaming-window-gate-".repeat(60); // 1320 bytes - let mut encoder = StreamingEncoder::new_with_matcher( - TinyMatcher::new(1024), - Vec::new(), - CompressionLevel::Fastest, - ); - encoder - .set_pledged_content_size(payload.len() as u64) - .unwrap(); - encoder.write_all(payload.as_slice()).unwrap(); - let compressed = encoder.finish().unwrap(); - - let header = crate::decoding::frame::read_frame_header(compressed.as_slice()) - .unwrap() - .0; - assert_eq!(header.frame_content_size(), payload.len() as u64); - assert!( - !header.descriptor.single_segment_flag(), - "single-segment must stay off when pledged content size exceeds matcher window" - ); - assert!( - header.window_size().unwrap() >= 1024, - "window descriptor should be present when single-segment is disabled" - ); - } - - #[test] - fn ensure_frame_started_refreshes_stale_strategy_tag_at_reset() { - // The literal-compression gates (`min_literals_to_compress`, - // `min_gain`) read `state.strategy_tag`. Regression: every - // reset site MUST refresh that tag from the active compression - // level — relying on construction-time initialization alone is - // not enough, because later mutations or reuse patterns can - // leave the tag stale. - // - // To exercise the RESET-time refresh (not just the - // construction-time init that `StreamingEncoder::new` does for - // free), this test deliberately corrupts `state.strategy_tag` - // to a value that does NOT match the active level, then - // triggers `ensure_frame_started` and asserts the reset path - // wrote the correct tag back. If the sync line in - // `ensure_frame_started` were deleted, the corrupted value - // would survive the write and fail the assertion. - use crate::encoding::strategy::StrategyTag; - for level in [ - CompressionLevel::Fastest, - CompressionLevel::Default, - CompressionLevel::Better, - CompressionLevel::Best, - ] { - let expected = StrategyTag::for_compression_level(level); - let mut encoder = StreamingEncoder::new(Vec::new(), level); - // Pick a sentinel that differs from the legitimate tag so - // a missing reset-time sync is observable. BtUltra2 is the - // most-aggressive variant; the four levels above resolve - // to Fast/Dfast/Lazy/Lazy respectively, none equal to it. - let sentinel = StrategyTag::BtUltra2; - assert_ne!( - expected, sentinel, - "sentinel must differ from the legitimate tag at level {level:?}", - ); - encoder.state.strategy_tag = sentinel; - encoder.write_all(b"x").unwrap(); - assert_eq!( - encoder.state.strategy_tag, expected, - "reset-time strategy_tag sync missing at level {level:?}: \ - sentinel survived `ensure_frame_started`", - ); - let _ = encoder.finish().unwrap(); - } - } - - /// Level 22 advertises the largest default window (`window_log 27` = - /// 128 MiB). Because streaming omits FCS, that window is written verbatim - /// into the frame header — so the encoder's max window MUST NOT exceed the - /// decoder's [`crate::common::MAXIMUM_ALLOWED_WINDOW_SIZE`], or our own - /// decoder rejects our own frame with `WindowSizeTooBig`. Regression for - /// the encoder↔decoder window-cap mismatch: streaming L22 must round-trip - /// through `StreamingDecoder` (and, implicitly, any stock zstd decoder, - /// which accepts up to the same 128 MiB default). - #[test] - fn level_22_streaming_window_roundtrips_in_our_decoder() { - let payload = b"level-22-streaming-window-cap-".repeat(512); - let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::from_level(22)); - for chunk in payload.chunks(101) { - encoder.write_all(chunk).unwrap(); - } - let compressed = encoder.finish().unwrap(); - - // The advertised window equals the L22 default (128 MiB) and must sit - // at or below the decoder cap — otherwise the round-trip below fails. - let header = crate::decoding::frame::read_frame_header(compressed.as_slice()) - .unwrap() - .0; - let window = header.window_size().unwrap(); - assert!( - window <= crate::common::MAXIMUM_ALLOWED_WINDOW_SIZE, - "L22 advertised window {window} exceeds decoder cap {}", - crate::common::MAXIMUM_ALLOWED_WINDOW_SIZE, - ); - - let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); - let mut decoded = Vec::new(); - decoder.read_to_end(&mut decoded).unwrap(); - assert_eq!(decoded, payload); - } - - /// `set_content_checksum(false)` before the first write must clear the - /// frame header's `Content_Checksum_flag` and the frame must still - /// round-trip through the decoder. - #[test] - fn streaming_encoder_set_content_checksum_false_clears_header_flag() { - let payload = b"streaming-checksum-toggle-".repeat(64); - let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); - encoder - .set_content_checksum(false) - .expect("set_content_checksum pre-write"); - encoder.write_all(&payload).unwrap(); - let compressed = encoder.finish().unwrap(); - - let header = crate::decoding::frame::read_frame_header(compressed.as_slice()) - .unwrap() - .0; - assert!( - !header.descriptor.content_checksum_flag(), - "content_checksum(false) must clear the frame header flag", - ); - - let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); - let mut decoded = Vec::new(); - decoder.read_to_end(&mut decoded).unwrap(); - assert_eq!(decoded, payload); - } - - /// With the `hash` feature, disabling the checksum must drop exactly the - /// 4-byte XXH64 trailer: the same payload encoded with the checksum on is - /// 4 bytes longer and its header flag is set. - #[cfg(feature = "hash")] - #[test] - fn streaming_encoder_set_content_checksum_false_omits_trailer() { - let payload = b"streaming-checksum-trailer-".repeat(64); - - let mut with = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); - // Explicit: the encoder default is off (upstream library parity). - with.set_content_checksum(true) - .expect("set_content_checksum pre-write"); - with.write_all(&payload).unwrap(); - let with_checksum = with.finish().unwrap(); - - let mut without = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); - without - .set_content_checksum(false) - .expect("set_content_checksum pre-write"); - without.write_all(&payload).unwrap(); - let without_checksum = without.finish().unwrap(); - - assert!( - crate::decoding::frame::read_frame_header(with_checksum.as_slice()) - .unwrap() - .0 - .descriptor - .content_checksum_flag(), - "default checksum-on frame must set the header flag", - ); - assert_eq!( - with_checksum.len(), - without_checksum.len() + 4, - "checksum-on frame must carry exactly the 4-byte XXH64 trailer", - ); - } - - /// `set_content_checksum` after the first write must error: the frame - /// header (and its checksum flag) is already emitted, so a late flip would - /// desync the header flag from the emitted trailer. Mirrors - /// `set_magicless` / `set_pledged_content_size` semantics. - #[test] - fn streaming_encoder_set_content_checksum_after_first_write_errors() { - let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); - encoder.write_all(b"first-block").unwrap(); - let err = encoder - .set_content_checksum(false) - .expect_err("set_content_checksum after first write must error"); - assert_eq!( - err.kind(), - ErrorKind::InvalidInput, - "expected InvalidInput when setting content checksum after frame_started, got {err:?}", - ); - } - - #[test] - fn no_pledged_size_omits_fcs_from_header() { - let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); - encoder.write_all(b"no pledged size").unwrap(); - let compressed = encoder.finish().unwrap(); - - // FCS should be omitted from the header; the decoder reports absent FCS as 0. - let header = crate::decoding::frame::read_frame_header(compressed.as_slice()) - .unwrap() - .0; - assert_eq!(header.frame_content_size(), 0); - // Verify the descriptor confirms FCS field is truly absent (0 bytes), - // not just FCS present with value 0. - assert_eq!(header.descriptor.frame_content_size_bytes().unwrap(), 0); - } - - #[test] - fn streaming_encoder_with_dictionary_roundtrips_and_carries_dict_id() { - use alloc::format; - let dict_raw = include_bytes!("../../dict_tests/dictionary"); - let dict_id = crate::decoding::Dictionary::decode_dict(dict_raw) - .unwrap() - .id; - - // Dictionary-resembling payload (the dict was trained on similar lines), - // fed in many small writes so the dict + cross-block matching are both - // exercised by the streaming path. - let mut payload = Vec::new(); - for i in 0..400u32 { - payload.extend_from_slice( - format!("tenant=demo table=orders key={i} region=eu payload=aaaaabbbbbccccc\n") - .as_bytes(), - ); - } - - let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Level(19)); - encoder - .set_dictionary_from_bytes(dict_raw) - .expect("attach dictionary"); - for chunk in payload.chunks(777) { - encoder.write_all(chunk).unwrap(); - } - let compressed = encoder.finish().unwrap(); - - // The frame header advertises the dictionary ID (single-segment is - // disabled for dictionary frames, so an explicit window is present). - let header = crate::decoding::frame::read_frame_header(compressed.as_slice()) - .unwrap() - .0; - assert_eq!(header.dictionary_id(), Some(dict_id)); - - // Round-trip through a decoder primed with the SAME dictionary. - let mut decoder = - StreamingDecoder::new_with_dictionary_bytes(compressed.as_slice(), dict_raw).unwrap(); - let mut decoded = Vec::new(); - decoder.read_to_end(&mut decoded).unwrap(); - assert_eq!(decoded, payload); - - // The dictionary is actually used: the dict frame is no larger than the - // no-dictionary frame on this dict-resembling payload (a dict that was - // ignored could only ever make the frame the same size or bigger). - let mut nodict = StreamingEncoder::new(Vec::new(), CompressionLevel::Level(19)); - for chunk in payload.chunks(777) { - nodict.write_all(chunk).unwrap(); - } - let nodict_frame = nodict.finish().unwrap(); - assert!( - compressed.len() <= nodict_frame.len(), - "dict frame {} should not exceed no-dict frame {}", - compressed.len(), - nodict_frame.len() - ); - } - - #[test] - fn streaming_encoder_strategy_override_survives_frame_start() { - // A `.strategy(...)` override must drive BOTH the matcher and the - // literal-compression gates (`state.strategy_tag`) once the frame - // starts. `ensure_frame_started` re-syncs the tag, so without persisting - // the override it would silently fall back to the level's strategy and - // diverge from `FrameCompressor` for the same parameters. - use crate::encoding::{CompressionParameters, Strategy}; - let level = CompressionLevel::Fastest; - let level_tag = crate::encoding::strategy::StrategyTag::for_compression_level(level); - let override_tag = Strategy::Greedy.tag(); - assert_ne!( - level_tag, override_tag, - "test needs an override that changes the derived tag" - ); - - let params = CompressionParameters::builder(level) - .strategy(Strategy::Greedy) - .build() - .unwrap(); - let payload = b"override must outlive the frame header"; - let mut encoder = StreamingEncoder::new(Vec::new(), level); - encoder.set_parameters(¶ms).unwrap(); - encoder.write_all(payload).unwrap(); - assert_eq!( - encoder.state.strategy_tag, override_tag, - "strategy override was discarded when the frame started" - ); - - let compressed = encoder.finish().unwrap(); - let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); - let mut decoded = Vec::new(); - decoder.read_to_end(&mut decoded).unwrap(); - assert_eq!(decoded, payload); - } - - #[test] - fn streaming_encoder_uncompressed_with_dictionary_omits_dict_id() { - // At `Uncompressed` the matcher cannot prime a dictionary, so an - // attached dictionary must NOT be reflected in the frame: advertising a - // `Dictionary_ID` would force a dictionary at decode time for a frame - // that does not actually depend on one. Mirrors `FrameCompressor`'s - // `use_dictionary_state` gate. - let dict_raw = include_bytes!("../../dict_tests/dictionary"); - let payload = b"tenant=demo table=orders region=eu payload=aaaaabbbbbccccc"; - let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Uncompressed); - encoder - .set_dictionary_from_bytes(dict_raw) - .expect("attach dictionary"); - encoder.write_all(payload).unwrap(); - let compressed = encoder.finish().unwrap(); - - let header = crate::decoding::frame::read_frame_header(compressed.as_slice()) - .unwrap() - .0; - assert_eq!( - header.dictionary_id(), - None, - "uncompressed frame must not require a dictionary at decode time" - ); - - // Decodes WITHOUT any dictionary. - let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); - let mut decoded = Vec::new(); - decoder.read_to_end(&mut decoded).unwrap(); - assert_eq!(decoded, payload); - } -} +mod tests; diff --git a/zstd/src/encoding/streaming_encoder/tests.rs b/zstd/src/encoding/streaming_encoder/tests.rs new file mode 100644 index 000000000..d12109add --- /dev/null +++ b/zstd/src/encoding/streaming_encoder/tests.rs @@ -0,0 +1,893 @@ +use crate::decoding::StreamingDecoder; +use crate::encoding::{CompressionLevel, Matcher, Sequence, StreamingEncoder}; +use crate::io::{Error, ErrorKind, Read, Write}; +use alloc::vec; +use alloc::vec::Vec; + +struct TinyMatcher { + last_space: Vec, + window_size: u64, +} + +impl TinyMatcher { + fn new(window_size: u64) -> Self { + Self { + last_space: Vec::new(), + window_size, + } + } +} + +impl Matcher for TinyMatcher { + fn get_next_space(&mut self) -> Vec { + vec![0; self.window_size as usize] + } + + fn get_last_space(&mut self) -> &[u8] { + self.last_space.as_slice() + } + + fn commit_space(&mut self, space: Vec) { + self.last_space = space; + } + + fn skip_matching(&mut self) {} + + fn start_matching(&mut self, mut handle_sequence: impl for<'a> FnMut(Sequence<'a>)) { + handle_sequence(Sequence::Literals { + literals: self.last_space.as_slice(), + }); + } + + fn reset(&mut self, _level: CompressionLevel) { + self.last_space.clear(); + } + + fn window_size(&self) -> u64 { + self.window_size + } +} + +struct FailingWriteOnce { + writes: usize, + fail_on_write_number: usize, + sink: Vec, +} + +impl FailingWriteOnce { + fn new(fail_on_write_number: usize) -> Self { + Self { + writes: 0, + fail_on_write_number, + sink: Vec::new(), + } + } +} + +impl Write for FailingWriteOnce { + fn write(&mut self, buf: &[u8]) -> Result { + self.writes += 1; + if self.writes == self.fail_on_write_number { + return Err(super::other_error("injected write failure")); + } + self.sink.extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> Result<(), Error> { + Ok(()) + } +} + +struct FailingWithKind { + writes: usize, + fail_on_write_number: usize, + kind: ErrorKind, +} + +impl FailingWithKind { + fn new(fail_on_write_number: usize, kind: ErrorKind) -> Self { + Self { + writes: 0, + fail_on_write_number, + kind, + } + } +} + +impl Write for FailingWithKind { + fn write(&mut self, buf: &[u8]) -> Result { + self.writes += 1; + if self.writes == self.fail_on_write_number { + return Err(Error::from(self.kind)); + } + Ok(buf.len()) + } + + fn flush(&mut self) -> Result<(), Error> { + Ok(()) + } +} + +struct PartialThenFailWriter { + writes: usize, + fail_on_write_number: usize, + partial_prefix_len: usize, + terminal_failure: bool, + sink: Vec, +} + +impl PartialThenFailWriter { + fn new(fail_on_write_number: usize, partial_prefix_len: usize) -> Self { + Self { + writes: 0, + fail_on_write_number, + partial_prefix_len, + terminal_failure: false, + sink: Vec::new(), + } + } +} + +impl Write for PartialThenFailWriter { + fn write(&mut self, buf: &[u8]) -> Result { + if self.terminal_failure { + return Err(super::other_error("injected terminal write failure")); + } + + self.writes += 1; + if self.writes == self.fail_on_write_number { + let written = core::cmp::min(self.partial_prefix_len, buf.len()); + if written > 0 { + self.sink.extend_from_slice(&buf[..written]); + self.terminal_failure = true; + return Ok(written); + } + return Err(super::other_error("injected terminal write failure")); + } + + self.sink.extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> Result<(), Error> { + Ok(()) + } +} + +/// Pre-write `set_magicless(true)` → emitted frame omits the +/// magic prefix AND round-trips through a magicless-aware +/// decoder. +#[test] +fn streaming_encoder_set_magicless_before_write_omits_magic_and_roundtrips() { + use crate::common::MAGIC_NUM; + let payload = b"streaming-magicless-roundtrip-".repeat(64); + + let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); + encoder + .set_magicless(true) + .expect("set_magicless pre-write"); + encoder.write_all(&payload).unwrap(); + let compressed = encoder.finish().unwrap(); + + assert!( + !compressed.starts_with(&MAGIC_NUM.to_le_bytes()), + "magicless frame must omit the 4-byte magic prefix", + ); + + let mut decoder = crate::decoding::FrameDecoder::new(); + decoder.set_magicless(true); + let mut cursor: &[u8] = compressed.as_slice(); + decoder.init(&mut cursor).expect("magicless init"); + decoder + .decode_blocks(&mut cursor, crate::decoding::BlockDecodingStrategy::All) + .expect("decode_blocks"); + let mut decoded: Vec = Vec::new(); + decoder + .collect_to_writer(&mut decoded) + .expect("collect_to_writer"); + assert_eq!(decoded, payload); +} + +/// `set_magicless` after the first write MUST return an error +/// (the frame header has already been emitted, flipping the flag +/// can't affect the current frame). Mirrors +/// `set_pledged_content_size` / `set_source_size_hint` semantics. +#[test] +fn streaming_encoder_set_magicless_after_first_write_errors() { + let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); + encoder.write_all(b"first-block").unwrap(); + let err = encoder + .set_magicless(true) + .expect_err("set_magicless after first write must error"); + assert_eq!( + err.kind(), + crate::io::ErrorKind::InvalidInput, + "expected InvalidInput when setting magicless after frame_started, got {err:?}", + ); +} + +#[test] +fn streaming_encoder_roundtrip_multiple_writes() { + let payload = b"streaming-encoder-roundtrip-".repeat(1024); + let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); + for chunk in payload.chunks(313) { + encoder.write_all(chunk).unwrap(); + } + let compressed = encoder.finish().unwrap(); + + let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); + let mut decoded = Vec::new(); + decoder.read_to_end(&mut decoded).unwrap(); + assert_eq!(decoded, payload); +} + +#[test] +fn flush_emits_nonempty_partial_output() { + let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); + encoder.write_all(b"partial-block").unwrap(); + encoder.flush().unwrap(); + let flushed_len = encoder.get_ref().len(); + assert!( + flushed_len > 0, + "flush should emit header+partial block bytes" + ); + let compressed = encoder.finish().unwrap(); + let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); + let mut decoded = Vec::new(); + decoder.read_to_end(&mut decoded).unwrap(); + assert_eq!(decoded, b"partial-block"); +} + +#[test] +fn flush_without_writes_does_not_emit_frame_header() { + let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); + encoder.flush().unwrap(); + assert!(encoder.get_ref().is_empty()); +} + +#[test] +fn block_boundary_write_emits_block_in_same_call() { + let mut boundary = StreamingEncoder::new_with_matcher( + TinyMatcher::new(4), + Vec::new(), + CompressionLevel::Uncompressed, + ); + let mut below = StreamingEncoder::new_with_matcher( + TinyMatcher::new(4), + Vec::new(), + CompressionLevel::Uncompressed, + ); + + boundary.write_all(b"ABCD").unwrap(); + below.write_all(b"ABC").unwrap(); + + let boundary_len = boundary.get_ref().len(); + let below_len = below.get_ref().len(); + assert!( + boundary_len > below_len, + "full block should be emitted immediately at block boundary" + ); +} + +#[test] +fn finish_consumes_encoder_and_emits_frame() { + let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); + encoder.write_all(b"abc").unwrap(); + let compressed = encoder.finish().unwrap(); + let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); + let mut decoded = Vec::new(); + decoder.read_to_end(&mut decoded).unwrap(); + assert_eq!(decoded, b"abc"); +} + +#[test] +fn finish_without_writes_emits_empty_frame() { + let encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); + let compressed = encoder.finish().unwrap(); + let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); + let mut decoded = Vec::new(); + decoder.read_to_end(&mut decoded).unwrap(); + assert!(decoded.is_empty()); +} + +#[test] +fn write_empty_buffer_returns_zero() { + let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); + assert_eq!(encoder.write(&[]).unwrap(), 0); + let _ = encoder.finish().unwrap(); +} + +#[test] +fn uncompressed_level_roundtrip() { + let payload = b"uncompressed-streaming-roundtrip".repeat(64); + let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Uncompressed); + for chunk in payload.chunks(41) { + encoder.write_all(chunk).unwrap(); + } + let compressed = encoder.finish().unwrap(); + let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); + let mut decoded = Vec::new(); + decoder.read_to_end(&mut decoded).unwrap(); + assert_eq!(decoded, payload); +} + +#[test] +fn better_level_streaming_roundtrip() { + let payload = b"better-level-streaming-test".repeat(256); + let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Better); + for chunk in payload.chunks(53) { + encoder.write_all(chunk).unwrap(); + } + let compressed = encoder.finish().unwrap(); + let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); + let mut decoded = Vec::new(); + decoder.read_to_end(&mut decoded).unwrap(); + assert_eq!(decoded, payload); +} + +#[test] +fn zero_window_matcher_returns_invalid_input_error() { + let mut encoder = StreamingEncoder::new_with_matcher( + TinyMatcher::new(0), + Vec::new(), + CompressionLevel::Fastest, + ); + let err = encoder.write_all(b"payload").unwrap_err(); + assert_eq!(err.kind(), ErrorKind::InvalidInput); +} + +#[test] +fn best_level_streaming_roundtrip() { + // 200 KiB payload crosses the 128 KiB block boundary, exercising + // multi-block emission and matcher state carry-over for Best. + let payload = b"best-level-streaming-test".repeat(8 * 1024); + let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Best); + for chunk in payload.chunks(53) { + encoder.write_all(chunk).unwrap(); + } + let compressed = encoder.finish().unwrap(); + let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); + let mut decoded = Vec::new(); + decoder.read_to_end(&mut decoded).unwrap(); + assert_eq!(decoded, payload); +} + +#[test] +fn write_failure_poisoning_is_sticky() { + let mut encoder = StreamingEncoder::new_with_matcher( + TinyMatcher::new(4), + FailingWriteOnce::new(1), + CompressionLevel::Uncompressed, + ); + + assert!(encoder.write_all(b"ABCD").is_err()); + assert!(encoder.flush().is_err()); + assert!(encoder.write_all(b"EFGH").is_err()); + assert_eq!(encoder.get_ref().sink.len(), 0); + assert!(encoder.finish().is_err()); +} + +#[test] +fn poisoned_encoder_returns_original_error_kind() { + let mut encoder = StreamingEncoder::new_with_matcher( + TinyMatcher::new(4), + FailingWithKind::new(1, ErrorKind::BrokenPipe), + CompressionLevel::Uncompressed, + ); + + let first_error = encoder.write_all(b"ABCD").unwrap_err(); + assert_eq!(first_error.kind(), ErrorKind::BrokenPipe); + + let second_error = encoder.write_all(b"EFGH").unwrap_err(); + assert_eq!(second_error.kind(), ErrorKind::BrokenPipe); +} + +#[test] +fn write_reports_progress_but_poisoning_is_sticky_after_later_block_failure() { + let payload = b"ABCDEFGHIJKL"; + let mut encoder = StreamingEncoder::new_with_matcher( + TinyMatcher::new(4), + FailingWriteOnce::new(3), + CompressionLevel::Uncompressed, + ); + + let first_write = encoder.write(payload).unwrap(); + assert_eq!(first_write, 8); + assert!(encoder.write(&payload[first_write..]).is_err()); + assert!(encoder.flush().is_err()); + assert!(encoder.write_all(b"EFGH").is_err()); +} + +#[test] +fn partial_write_failure_after_progress_poisons_encoder() { + let payload = b"ABCDEFGHIJKL"; + let mut encoder = StreamingEncoder::new_with_matcher( + TinyMatcher::new(4), + PartialThenFailWriter::new(3, 1), + CompressionLevel::Uncompressed, + ); + + let first_write = encoder.write(payload).unwrap(); + assert_eq!(first_write, 8); + + let second_write = encoder.write(&payload[first_write..]); + assert!(second_write.is_err()); + assert!(encoder.flush().is_err()); + assert!(encoder.write_all(b"MNOP").is_err()); +} + +#[test] +fn new_with_matcher_and_get_mut_work() { + let matcher = TinyMatcher::new(128 * 1024); + let mut encoder = + StreamingEncoder::new_with_matcher(matcher, Vec::new(), CompressionLevel::Fastest); + encoder.get_mut().extend_from_slice(b""); + encoder.write_all(b"custom-matcher").unwrap(); + let compressed = encoder.finish().unwrap(); + let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); + let mut decoded = Vec::new(); + decoder.read_to_end(&mut decoded).unwrap(); + assert_eq!(decoded, b"custom-matcher"); +} + +#[test] +fn pledged_content_size_written_in_header() { + let payload = b"hello world, pledged size test"; + let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); + encoder + .set_pledged_content_size(payload.len() as u64) + .unwrap(); + encoder.write_all(payload).unwrap(); + let compressed = encoder.finish().unwrap(); + + // Verify FCS is present and correct + let header = crate::decoding::frame::read_frame_header(compressed.as_slice()) + .unwrap() + .0; + assert_eq!(header.frame_content_size(), payload.len() as u64); + + // Verify roundtrip + let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); + let mut decoded = Vec::new(); + decoder.read_to_end(&mut decoded).unwrap(); + assert_eq!(decoded, payload); +} + +#[test] +fn pledged_content_size_mismatch_returns_error() { + let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); + encoder.set_pledged_content_size(100).unwrap(); + encoder.write_all(b"short payload").unwrap(); // 13 bytes != 100 pledged + let err = encoder.finish().unwrap_err(); + assert_eq!(err.kind(), ErrorKind::InvalidInput); +} + +#[test] +fn write_exceeding_pledge_returns_error() { + let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); + encoder.set_pledged_content_size(5).unwrap(); + let err = encoder.write_all(b"exceeds five bytes").unwrap_err(); + assert_eq!(err.kind(), ErrorKind::InvalidInput); +} + +#[test] +fn write_straddling_pledge_reports_partial_progress() { + let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); + encoder.set_pledged_content_size(5).unwrap(); + // write() should accept exactly 5 bytes (partial progress) + assert_eq!(encoder.write(b"abcdef").unwrap(), 5); + // Next write should fail — pledge exhausted + let err = encoder.write(b"g").unwrap_err(); + assert_eq!(err.kind(), ErrorKind::InvalidInput); +} + +#[test] +fn encoded_scratch_capacity_is_reused_across_blocks() { + let payload = vec![0xAB; 64 * 3]; + let mut encoder = StreamingEncoder::new_with_matcher( + TinyMatcher::new(64), + Vec::new(), + CompressionLevel::Uncompressed, + ); + + encoder.write_all(&payload[..64]).unwrap(); + let first_capacity = encoder.encoded_scratch.capacity(); + assert!( + first_capacity >= 67, + "expected encoded scratch to keep block header + payload capacity", + ); + + encoder.write_all(&payload[64..128]).unwrap(); + let second_capacity = encoder.encoded_scratch.capacity(); + assert!( + second_capacity >= first_capacity, + "encoded scratch capacity should be reused across block emits", + ); + + encoder.write_all(&payload[128..]).unwrap(); + let compressed = encoder.finish().unwrap(); + let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); + let mut decoded = Vec::new(); + decoder.read_to_end(&mut decoded).unwrap(); + assert_eq!(decoded, payload); +} + +#[test] +fn pledged_content_size_after_write_returns_error() { + let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); + encoder.write_all(b"already writing").unwrap(); + let err = encoder.set_pledged_content_size(15).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::InvalidInput); +} + +#[test] +fn source_size_hint_directly_reduces_window_header() { + let payload = b"streaming-source-size-hint".repeat(64); + + let mut no_hint = StreamingEncoder::new(Vec::new(), CompressionLevel::from_level(11)); + no_hint.write_all(payload.as_slice()).unwrap(); + let no_hint_frame = no_hint.finish().unwrap(); + let no_hint_header = crate::decoding::frame::read_frame_header(no_hint_frame.as_slice()) + .unwrap() + .0; + let no_hint_window = no_hint_header.window_size().unwrap(); + + let mut with_hint = StreamingEncoder::new(Vec::new(), CompressionLevel::from_level(11)); + with_hint + .set_source_size_hint(payload.len() as u64) + .unwrap(); + with_hint.write_all(payload.as_slice()).unwrap(); + let late_hint_err = with_hint + .set_source_size_hint(payload.len() as u64) + .unwrap_err(); + assert_eq!(late_hint_err.kind(), ErrorKind::InvalidInput); + let with_hint_frame = with_hint.finish().unwrap(); + let with_hint_header = crate::decoding::frame::read_frame_header(with_hint_frame.as_slice()) + .unwrap() + .0; + let with_hint_window = with_hint_header.window_size().unwrap(); + + assert!( + with_hint_window <= no_hint_window, + "source size hint should not increase advertised window" + ); + + let mut decoder = StreamingDecoder::new(with_hint_frame.as_slice()).unwrap(); + let mut decoded = Vec::new(); + decoder.read_to_end(&mut decoded).unwrap(); + assert_eq!(decoded, payload); +} + +#[test] +fn single_segment_requires_pledged_to_fit_matcher_window() { + let payload = b"streaming-window-gate-".repeat(60); // 1320 bytes + let mut encoder = StreamingEncoder::new_with_matcher( + TinyMatcher::new(1024), + Vec::new(), + CompressionLevel::Fastest, + ); + encoder + .set_pledged_content_size(payload.len() as u64) + .unwrap(); + encoder.write_all(payload.as_slice()).unwrap(); + let compressed = encoder.finish().unwrap(); + + let header = crate::decoding::frame::read_frame_header(compressed.as_slice()) + .unwrap() + .0; + assert_eq!(header.frame_content_size(), payload.len() as u64); + assert!( + !header.descriptor.single_segment_flag(), + "single-segment must stay off when pledged content size exceeds matcher window" + ); + assert!( + header.window_size().unwrap() >= 1024, + "window descriptor should be present when single-segment is disabled" + ); +} + +#[test] +fn ensure_frame_started_refreshes_stale_strategy_tag_at_reset() { + // The literal-compression gates (`min_literals_to_compress`, + // `min_gain`) read `state.strategy_tag`. Regression: every + // reset site MUST refresh that tag from the active compression + // level — relying on construction-time initialization alone is + // not enough, because later mutations or reuse patterns can + // leave the tag stale. + // + // To exercise the RESET-time refresh (not just the + // construction-time init that `StreamingEncoder::new` does for + // free), this test deliberately corrupts `state.strategy_tag` + // to a value that does NOT match the active level, then + // triggers `ensure_frame_started` and asserts the reset path + // wrote the correct tag back. If the sync line in + // `ensure_frame_started` were deleted, the corrupted value + // would survive the write and fail the assertion. + use crate::encoding::strategy::StrategyTag; + for level in [ + CompressionLevel::Fastest, + CompressionLevel::Default, + CompressionLevel::Better, + CompressionLevel::Best, + ] { + let expected = StrategyTag::for_compression_level(level); + let mut encoder = StreamingEncoder::new(Vec::new(), level); + // Pick a sentinel that differs from the legitimate tag so + // a missing reset-time sync is observable. BtUltra2 is the + // most-aggressive variant; the four levels above resolve + // to Fast/Dfast/Lazy/Lazy respectively, none equal to it. + let sentinel = StrategyTag::BtUltra2; + assert_ne!( + expected, sentinel, + "sentinel must differ from the legitimate tag at level {level:?}", + ); + encoder.state.strategy_tag = sentinel; + encoder.write_all(b"x").unwrap(); + assert_eq!( + encoder.state.strategy_tag, expected, + "reset-time strategy_tag sync missing at level {level:?}: \ + sentinel survived `ensure_frame_started`", + ); + let _ = encoder.finish().unwrap(); + } +} + +/// Level 22 advertises the largest default window (`window_log 27` = +/// 128 MiB). Because streaming omits FCS, that window is written verbatim +/// into the frame header — so the encoder's max window MUST NOT exceed the +/// decoder's [`crate::common::MAXIMUM_ALLOWED_WINDOW_SIZE`], or our own +/// decoder rejects our own frame with `WindowSizeTooBig`. Regression for +/// the encoder↔decoder window-cap mismatch: streaming L22 must round-trip +/// through `StreamingDecoder` (and, implicitly, any stock zstd decoder, +/// which accepts up to the same 128 MiB default). +#[test] +fn level_22_streaming_window_roundtrips_in_our_decoder() { + let payload = b"level-22-streaming-window-cap-".repeat(512); + let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::from_level(22)); + for chunk in payload.chunks(101) { + encoder.write_all(chunk).unwrap(); + } + let compressed = encoder.finish().unwrap(); + + // The advertised window equals the L22 default (128 MiB) and must sit + // at or below the decoder cap — otherwise the round-trip below fails. + let header = crate::decoding::frame::read_frame_header(compressed.as_slice()) + .unwrap() + .0; + let window = header.window_size().unwrap(); + assert!( + window <= crate::common::MAXIMUM_ALLOWED_WINDOW_SIZE, + "L22 advertised window {window} exceeds decoder cap {}", + crate::common::MAXIMUM_ALLOWED_WINDOW_SIZE, + ); + + let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); + let mut decoded = Vec::new(); + decoder.read_to_end(&mut decoded).unwrap(); + assert_eq!(decoded, payload); +} + +/// `set_content_checksum(false)` before the first write must clear the +/// frame header's `Content_Checksum_flag` and the frame must still +/// round-trip through the decoder. +#[test] +fn streaming_encoder_set_content_checksum_false_clears_header_flag() { + let payload = b"streaming-checksum-toggle-".repeat(64); + let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); + encoder + .set_content_checksum(false) + .expect("set_content_checksum pre-write"); + encoder.write_all(&payload).unwrap(); + let compressed = encoder.finish().unwrap(); + + let header = crate::decoding::frame::read_frame_header(compressed.as_slice()) + .unwrap() + .0; + assert!( + !header.descriptor.content_checksum_flag(), + "content_checksum(false) must clear the frame header flag", + ); + + let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); + let mut decoded = Vec::new(); + decoder.read_to_end(&mut decoded).unwrap(); + assert_eq!(decoded, payload); +} + +/// With the `hash` feature, disabling the checksum must drop exactly the +/// 4-byte XXH64 trailer: the same payload encoded with the checksum on is +/// 4 bytes longer and its header flag is set. +#[cfg(feature = "hash")] +#[test] +fn streaming_encoder_set_content_checksum_false_omits_trailer() { + let payload = b"streaming-checksum-trailer-".repeat(64); + + let mut with = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); + // Explicit: the encoder default is off (upstream library parity). + with.set_content_checksum(true) + .expect("set_content_checksum pre-write"); + with.write_all(&payload).unwrap(); + let with_checksum = with.finish().unwrap(); + + let mut without = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); + without + .set_content_checksum(false) + .expect("set_content_checksum pre-write"); + without.write_all(&payload).unwrap(); + let without_checksum = without.finish().unwrap(); + + assert!( + crate::decoding::frame::read_frame_header(with_checksum.as_slice()) + .unwrap() + .0 + .descriptor + .content_checksum_flag(), + "default checksum-on frame must set the header flag", + ); + assert_eq!( + with_checksum.len(), + without_checksum.len() + 4, + "checksum-on frame must carry exactly the 4-byte XXH64 trailer", + ); +} + +/// `set_content_checksum` after the first write must error: the frame +/// header (and its checksum flag) is already emitted, so a late flip would +/// desync the header flag from the emitted trailer. Mirrors +/// `set_magicless` / `set_pledged_content_size` semantics. +#[test] +fn streaming_encoder_set_content_checksum_after_first_write_errors() { + let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); + encoder.write_all(b"first-block").unwrap(); + let err = encoder + .set_content_checksum(false) + .expect_err("set_content_checksum after first write must error"); + assert_eq!( + err.kind(), + ErrorKind::InvalidInput, + "expected InvalidInput when setting content checksum after frame_started, got {err:?}", + ); +} + +#[test] +fn no_pledged_size_omits_fcs_from_header() { + let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Fastest); + encoder.write_all(b"no pledged size").unwrap(); + let compressed = encoder.finish().unwrap(); + + // FCS should be omitted from the header; the decoder reports absent FCS as 0. + let header = crate::decoding::frame::read_frame_header(compressed.as_slice()) + .unwrap() + .0; + assert_eq!(header.frame_content_size(), 0); + // Verify the descriptor confirms FCS field is truly absent (0 bytes), + // not just FCS present with value 0. + assert_eq!(header.descriptor.frame_content_size_bytes().unwrap(), 0); +} + +#[test] +fn streaming_encoder_with_dictionary_roundtrips_and_carries_dict_id() { + use alloc::format; + let dict_raw = include_bytes!("../../../dict_tests/dictionary"); + let dict_id = crate::decoding::Dictionary::decode_dict(dict_raw) + .unwrap() + .id; + + // Dictionary-resembling payload (the dict was trained on similar lines), + // fed in many small writes so the dict + cross-block matching are both + // exercised by the streaming path. + let mut payload = Vec::new(); + for i in 0..400u32 { + payload.extend_from_slice( + format!("tenant=demo table=orders key={i} region=eu payload=aaaaabbbbbccccc\n") + .as_bytes(), + ); + } + + let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Level(19)); + encoder + .set_dictionary_from_bytes(dict_raw) + .expect("attach dictionary"); + for chunk in payload.chunks(777) { + encoder.write_all(chunk).unwrap(); + } + let compressed = encoder.finish().unwrap(); + + // The frame header advertises the dictionary ID (single-segment is + // disabled for dictionary frames, so an explicit window is present). + let header = crate::decoding::frame::read_frame_header(compressed.as_slice()) + .unwrap() + .0; + assert_eq!(header.dictionary_id(), Some(dict_id)); + + // Round-trip through a decoder primed with the SAME dictionary. + let mut decoder = + StreamingDecoder::new_with_dictionary_bytes(compressed.as_slice(), dict_raw).unwrap(); + let mut decoded = Vec::new(); + decoder.read_to_end(&mut decoded).unwrap(); + assert_eq!(decoded, payload); + + // The dictionary is actually used: the dict frame is no larger than the + // no-dictionary frame on this dict-resembling payload (a dict that was + // ignored could only ever make the frame the same size or bigger). + let mut nodict = StreamingEncoder::new(Vec::new(), CompressionLevel::Level(19)); + for chunk in payload.chunks(777) { + nodict.write_all(chunk).unwrap(); + } + let nodict_frame = nodict.finish().unwrap(); + assert!( + compressed.len() <= nodict_frame.len(), + "dict frame {} should not exceed no-dict frame {}", + compressed.len(), + nodict_frame.len() + ); +} + +#[test] +fn streaming_encoder_strategy_override_survives_frame_start() { + // A `.strategy(...)` override must drive BOTH the matcher and the + // literal-compression gates (`state.strategy_tag`) once the frame + // starts. `ensure_frame_started` re-syncs the tag, so without persisting + // the override it would silently fall back to the level's strategy and + // diverge from `FrameCompressor` for the same parameters. + use crate::encoding::{CompressionParameters, Strategy}; + let level = CompressionLevel::Fastest; + let level_tag = crate::encoding::strategy::StrategyTag::for_compression_level(level); + let override_tag = Strategy::Greedy.tag(); + assert_ne!( + level_tag, override_tag, + "test needs an override that changes the derived tag" + ); + + let params = CompressionParameters::builder(level) + .strategy(Strategy::Greedy) + .build() + .unwrap(); + let payload = b"override must outlive the frame header"; + let mut encoder = StreamingEncoder::new(Vec::new(), level); + encoder.set_parameters(¶ms).unwrap(); + encoder.write_all(payload).unwrap(); + assert_eq!( + encoder.state.strategy_tag, override_tag, + "strategy override was discarded when the frame started" + ); + + let compressed = encoder.finish().unwrap(); + let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); + let mut decoded = Vec::new(); + decoder.read_to_end(&mut decoded).unwrap(); + assert_eq!(decoded, payload); +} + +#[test] +fn streaming_encoder_uncompressed_with_dictionary_omits_dict_id() { + // At `Uncompressed` the matcher cannot prime a dictionary, so an + // attached dictionary must NOT be reflected in the frame: advertising a + // `Dictionary_ID` would force a dictionary at decode time for a frame + // that does not actually depend on one. Mirrors `FrameCompressor`'s + // `use_dictionary_state` gate. + let dict_raw = include_bytes!("../../../dict_tests/dictionary"); + let payload = b"tenant=demo table=orders region=eu payload=aaaaabbbbbccccc"; + let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Uncompressed); + encoder + .set_dictionary_from_bytes(dict_raw) + .expect("attach dictionary"); + encoder.write_all(payload).unwrap(); + let compressed = encoder.finish().unwrap(); + + let header = crate::decoding::frame::read_frame_header(compressed.as_slice()) + .unwrap() + .0; + assert_eq!( + header.dictionary_id(), + None, + "uncompressed frame must not require a dictionary at decode time" + ); + + // Decodes WITHOUT any dictionary. + let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); + let mut decoded = Vec::new(); + decoder.read_to_end(&mut decoded).unwrap(); + assert_eq!(decoded, payload); +} diff --git a/zstd/src/encoding/util.rs b/zstd/src/encoding/util.rs index 62b8a3b89..2ca30f963 100644 --- a/zstd/src/encoding/util.rs +++ b/zstd/src/encoding/util.rs @@ -53,66 +53,4 @@ pub fn find_fcs_field_size(val: u64, single_segment: bool) -> usize { } #[cfg(test)] -mod tests { - use super::find_fcs_field_size; - use super::find_min_size; - use super::write_minified_val; - use alloc::vec; - use alloc::vec::Vec; - - fn minify_val(val: u64) -> Vec { - let mut out = Vec::new(); - write_minified_val(val, &mut out); - out - } - - #[test] - fn min_size_detection() { - assert_eq!(find_min_size(0), 1); - assert_eq!(find_min_size(0xff), 1); - assert_eq!(find_min_size(0xff_ff), 2); - assert_eq!(find_min_size(0x00_ff_ff_ff), 4); - assert_eq!(find_min_size(0xff_ff_ff_ff), 4); - assert_eq!(find_min_size(0x00ff_ffff_ffff_ffff), 8); - assert_eq!(find_min_size(0xffff_ffff_ffff_ffff), 8); - } - - #[test] - fn fcs_field_size_single_segment() { - // 1-byte range: 0–255 when single_segment is true - assert_eq!(find_fcs_field_size(0, true), 1); - assert_eq!(find_fcs_field_size(255, true), 1); - // 2-byte range: 256–65791 - assert_eq!(find_fcs_field_size(256, true), 2); - assert_eq!(find_fcs_field_size(65791, true), 2); - // 4-byte range - assert_eq!(find_fcs_field_size(65792, true), 4); - assert_eq!(find_fcs_field_size(u32::MAX as u64, true), 4); - // 8-byte range - assert_eq!(find_fcs_field_size(u32::MAX as u64 + 1, true), 8); - } - - #[test] - fn fcs_field_size_no_single_segment() { - // Without single_segment, 0–255 cannot use 1-byte → falls to 4 bytes - assert_eq!(find_fcs_field_size(0, false), 4); - assert_eq!(find_fcs_field_size(255, false), 4); - // 256–65791 still fits in 2 bytes - assert_eq!(find_fcs_field_size(256, false), 2); - assert_eq!(find_fcs_field_size(65791, false), 2); - // Values that find_min_size would map to 4 but FCS can still fit in 2 - assert_eq!(find_fcs_field_size(65536, false), 2); - } - - #[test] - fn bytes_minified() { - assert_eq!(minify_val(0), vec![0]); - assert_eq!(minify_val(0xff), vec![0xff]); - assert_eq!(minify_val(0xff_ff), vec![0xff, 0xff]); - assert_eq!(minify_val(0xff_ff_ff_ff), vec![0xff, 0xff, 0xff, 0xff]); - assert_eq!( - minify_val(0xffff_ffff_ffff_ffff), - vec![0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff] - ); - } -} +mod tests; diff --git a/zstd/src/encoding/util/tests.rs b/zstd/src/encoding/util/tests.rs new file mode 100644 index 000000000..805a173e1 --- /dev/null +++ b/zstd/src/encoding/util/tests.rs @@ -0,0 +1,61 @@ +use super::find_fcs_field_size; +use super::find_min_size; +use super::write_minified_val; +use alloc::vec; +use alloc::vec::Vec; + +fn minify_val(val: u64) -> Vec { + let mut out = Vec::new(); + write_minified_val(val, &mut out); + out +} + +#[test] +fn min_size_detection() { + assert_eq!(find_min_size(0), 1); + assert_eq!(find_min_size(0xff), 1); + assert_eq!(find_min_size(0xff_ff), 2); + assert_eq!(find_min_size(0x00_ff_ff_ff), 4); + assert_eq!(find_min_size(0xff_ff_ff_ff), 4); + assert_eq!(find_min_size(0x00ff_ffff_ffff_ffff), 8); + assert_eq!(find_min_size(0xffff_ffff_ffff_ffff), 8); +} + +#[test] +fn fcs_field_size_single_segment() { + // 1-byte range: 0–255 when single_segment is true + assert_eq!(find_fcs_field_size(0, true), 1); + assert_eq!(find_fcs_field_size(255, true), 1); + // 2-byte range: 256–65791 + assert_eq!(find_fcs_field_size(256, true), 2); + assert_eq!(find_fcs_field_size(65791, true), 2); + // 4-byte range + assert_eq!(find_fcs_field_size(65792, true), 4); + assert_eq!(find_fcs_field_size(u32::MAX as u64, true), 4); + // 8-byte range + assert_eq!(find_fcs_field_size(u32::MAX as u64 + 1, true), 8); +} + +#[test] +fn fcs_field_size_no_single_segment() { + // Without single_segment, 0–255 cannot use 1-byte → falls to 4 bytes + assert_eq!(find_fcs_field_size(0, false), 4); + assert_eq!(find_fcs_field_size(255, false), 4); + // 256–65791 still fits in 2 bytes + assert_eq!(find_fcs_field_size(256, false), 2); + assert_eq!(find_fcs_field_size(65791, false), 2); + // Values that find_min_size would map to 4 but FCS can still fit in 2 + assert_eq!(find_fcs_field_size(65536, false), 2); +} + +#[test] +fn bytes_minified() { + assert_eq!(minify_val(0), vec![0]); + assert_eq!(minify_val(0xff), vec![0xff]); + assert_eq!(minify_val(0xff_ff), vec![0xff, 0xff]); + assert_eq!(minify_val(0xff_ff_ff_ff), vec![0xff, 0xff, 0xff, 0xff]); + assert_eq!( + minify_val(0xffff_ffff_ffff_ffff), + vec![0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff] + ); +} diff --git a/zstd/src/fse/fse_decoder.rs b/zstd/src/fse/fse_decoder.rs index 5d7d3cac8..7f3443c1f 100644 --- a/zstd/src/fse/fse_decoder.rs +++ b/zstd/src/fse/fse_decoder.rs @@ -1174,113 +1174,4 @@ fn next_position(mut p: usize, table_size: usize) -> usize { } #[cfg(test)] -mod tests { - use super::*; - use crate::decoding::errors::FSETableError; - - #[test] - fn build_from_probabilities_rejects_sum_too_small() { - // acc_log=4 → table_size=16. Valid sum (treating -1 as 1) - // is 16. Use a distribution that sums to 8 (probs sum 8, - // no -1s). - let mut t = FSETable::new(8); - let probs: [i32; 4] = [4, 2, 1, 1]; - let result = t.build_from_probabilities(4, &probs); - assert!( - matches!( - result, - Err(FSETableError::ProbabilityCounterMismatch { .. }) - ), - "expected ProbabilityCounterMismatch for sum=8 vs expected=16, got {result:?}", - ); - } - - #[test] - fn build_from_probabilities_rejects_sum_too_large() { - // acc_log=4 → table_size=16. Probs summing to 20 (>16) - // would over-fill the spread. - let mut t = FSETable::new(8); - let probs: [i32; 4] = [8, 6, 4, 2]; - let result = t.build_from_probabilities(4, &probs); - assert!( - matches!( - result, - Err(FSETableError::ProbabilityCounterMismatch { .. }) - ), - "expected ProbabilityCounterMismatch for sum=20 vs expected=16, got {result:?}", - ); - } - - #[test] - fn build_from_probabilities_accepts_negative_one_in_sum() { - // acc_log=4 → table_size=16. Probs: 12 positive + 4 × -1 - // (counted as 1 each) sum to 16. Should succeed. - let mut t = FSETable::new(8); - let probs: [i32; 6] = [8, 4, -1, -1, -1, -1]; - let result = t.build_from_probabilities(4, &probs); - assert!( - result.is_ok(), - "expected Ok for sum=16 with -1s, got {result:?}" - ); - } - - #[test] - fn build_from_probabilities_rejects_overflow_in_sum() { - // DoS-shaped exploit: two i32::MAX values + 0x42 cast to u32 - // and summed wrapping wraps to 0x40 = 64 = 1 << acc_log. The - // pre-fix u32 sum check would accept the input, then - // build_decoding_table would run `for _ in 0..prob` against - // prob = i32::MAX, looping 2^31-1 times per such symbol. - let mut t = FSETable::new(8); - let probs: [i32; 3] = [i32::MAX, i32::MAX, 0x42]; - let result = t.build_from_probabilities(6, &probs); - assert!( - matches!(result, Err(FSETableError::InvalidProbability { .. })), - "expected InvalidProbability for overflow-shaped exploit, got {result:?}", - ); - } - - #[test] - fn build_from_probabilities_rejects_negative_below_minus_one() { - // RFC 8878 §4.1.1: probability values are in {-1, 0, positive}. - // p = -2 is outside the spec; clamping to 0 silently is wrong. - let mut t = FSETable::new(8); - let probs: [i32; 4] = [-2, 8, 4, 4]; - let result = t.build_from_probabilities(4, &probs); - assert!( - matches!( - result, - Err(FSETableError::InvalidProbability { value: -2, .. }) - ), - "expected InvalidProbability{{value: -2, ..}}, got {result:?}", - ); - } - - #[test] - fn build_from_probabilities_accepts_exact_sum() { - // Sum 4+4+4+4 = 16 = 1 << 4. No -1s. - let mut t = FSETable::new(8); - let probs: [i32; 4] = [4, 4, 4, 4]; - let result = t.build_from_probabilities(4, &probs); - assert!(result.is_ok(), "expected Ok for exact sum, got {result:?}"); - } - - /// The decoder accepts tables with `accuracy_log` up to - /// `ENTRY_MAX_ACCURACY_LOG = 16`, but the encoder table builder indexes a - /// fixed 512-entry (`1 << 9`) scratch. A decoder table with `accuracy_log > - /// 9` reached through `to_encoder_table` (e.g. a non-sequence table carried - /// by a dictionary) must return None rather than slice past the scratch and - /// panic. - #[test] - fn to_encoder_table_rejects_accuracy_log_above_nine() { - let mut t = FSETable::new(8); - // A table with accuracy_log 10 (1 << 10 = 1024 entries) and a plausible - // probability distribution summing to 1024. - t.accuracy_log = 10; - t.symbol_probabilities = alloc::vec![512, 512]; - assert!( - t.to_encoder_table().is_none(), - "accuracy_log > 9 must not be reusable as an encoder table" - ); - } -} +mod tests; diff --git a/zstd/src/fse/fse_decoder/tests.rs b/zstd/src/fse/fse_decoder/tests.rs new file mode 100644 index 000000000..77cc8b7d0 --- /dev/null +++ b/zstd/src/fse/fse_decoder/tests.rs @@ -0,0 +1,108 @@ +use super::*; +use crate::decoding::errors::FSETableError; + +#[test] +fn build_from_probabilities_rejects_sum_too_small() { + // acc_log=4 → table_size=16. Valid sum (treating -1 as 1) + // is 16. Use a distribution that sums to 8 (probs sum 8, + // no -1s). + let mut t = FSETable::new(8); + let probs: [i32; 4] = [4, 2, 1, 1]; + let result = t.build_from_probabilities(4, &probs); + assert!( + matches!( + result, + Err(FSETableError::ProbabilityCounterMismatch { .. }) + ), + "expected ProbabilityCounterMismatch for sum=8 vs expected=16, got {result:?}", + ); +} + +#[test] +fn build_from_probabilities_rejects_sum_too_large() { + // acc_log=4 → table_size=16. Probs summing to 20 (>16) + // would over-fill the spread. + let mut t = FSETable::new(8); + let probs: [i32; 4] = [8, 6, 4, 2]; + let result = t.build_from_probabilities(4, &probs); + assert!( + matches!( + result, + Err(FSETableError::ProbabilityCounterMismatch { .. }) + ), + "expected ProbabilityCounterMismatch for sum=20 vs expected=16, got {result:?}", + ); +} + +#[test] +fn build_from_probabilities_accepts_negative_one_in_sum() { + // acc_log=4 → table_size=16. Probs: 12 positive + 4 × -1 + // (counted as 1 each) sum to 16. Should succeed. + let mut t = FSETable::new(8); + let probs: [i32; 6] = [8, 4, -1, -1, -1, -1]; + let result = t.build_from_probabilities(4, &probs); + assert!( + result.is_ok(), + "expected Ok for sum=16 with -1s, got {result:?}" + ); +} + +#[test] +fn build_from_probabilities_rejects_overflow_in_sum() { + // DoS-shaped exploit: two i32::MAX values + 0x42 cast to u32 + // and summed wrapping wraps to 0x40 = 64 = 1 << acc_log. The + // pre-fix u32 sum check would accept the input, then + // build_decoding_table would run `for _ in 0..prob` against + // prob = i32::MAX, looping 2^31-1 times per such symbol. + let mut t = FSETable::new(8); + let probs: [i32; 3] = [i32::MAX, i32::MAX, 0x42]; + let result = t.build_from_probabilities(6, &probs); + assert!( + matches!(result, Err(FSETableError::InvalidProbability { .. })), + "expected InvalidProbability for overflow-shaped exploit, got {result:?}", + ); +} + +#[test] +fn build_from_probabilities_rejects_negative_below_minus_one() { + // RFC 8878 §4.1.1: probability values are in {-1, 0, positive}. + // p = -2 is outside the spec; clamping to 0 silently is wrong. + let mut t = FSETable::new(8); + let probs: [i32; 4] = [-2, 8, 4, 4]; + let result = t.build_from_probabilities(4, &probs); + assert!( + matches!( + result, + Err(FSETableError::InvalidProbability { value: -2, .. }) + ), + "expected InvalidProbability{{value: -2, ..}}, got {result:?}", + ); +} + +#[test] +fn build_from_probabilities_accepts_exact_sum() { + // Sum 4+4+4+4 = 16 = 1 << 4. No -1s. + let mut t = FSETable::new(8); + let probs: [i32; 4] = [4, 4, 4, 4]; + let result = t.build_from_probabilities(4, &probs); + assert!(result.is_ok(), "expected Ok for exact sum, got {result:?}"); +} + +/// The decoder accepts tables with `accuracy_log` up to +/// `ENTRY_MAX_ACCURACY_LOG = 16`, but the encoder table builder indexes a +/// fixed 512-entry (`1 << 9`) scratch. A decoder table with `accuracy_log > +/// 9` reached through `to_encoder_table` (e.g. a non-sequence table carried +/// by a dictionary) must return None rather than slice past the scratch and +/// panic. +#[test] +fn to_encoder_table_rejects_accuracy_log_above_nine() { + let mut t = FSETable::new(8); + // A table with accuracy_log 10 (1 << 10 = 1024 entries) and a plausible + // probability distribution summing to 1024. + t.accuracy_log = 10; + t.symbol_probabilities = alloc::vec![512, 512]; + assert!( + t.to_encoder_table().is_none(), + "accuracy_log > 9 must not be reusable as an encoder table" + ); +} diff --git a/zstd/src/fse/mod.rs b/zstd/src/fse/mod.rs index c2162fd8a..35d9a9459 100644 --- a/zstd/src/fse/mod.rs +++ b/zstd/src/fse/mod.rs @@ -18,194 +18,6 @@ pub use fse_decoder::*; pub mod fse_encoder; -#[test] -fn decoder_entry_layout_12_bytes_seqsymbol_shape() { - // Upstream zstd `ZSTD_seqSymbol` shape: 4-byte header (new_state / - // symbol / num_bits), 4-byte `base_value`, 1-byte - // `num_additional_bits` + 3-byte tail padding for natural u32 - // alignment. Total 12 bytes. Grown from the classical 4-byte - // layout to remove the per-sequence `lookup_ll_code` / - // `lookup_ml_code` indirection: LL and ML enrich `base_value` - // / `num_additional_bits` from the packed code-meta tables - // (`enrich_with_packed_seq_meta`), OF enriches closed-form - // `base_value = 1 << code`, `num_additional_bits = code` - // (`enrich_for_offsets`, no meta table). HUF-weight FSE tables - // never enrich and leave both fields zero. - assert_eq!(core::mem::size_of::(), 12); - assert_eq!(core::mem::offset_of!(fse_decoder::Entry, new_state), 0); - assert_eq!(core::mem::offset_of!(fse_decoder::Entry, symbol), 2); - assert_eq!(core::mem::offset_of!(fse_decoder::Entry, num_bits), 3); - assert_eq!(core::mem::offset_of!(fse_decoder::Entry, base_value), 4); - assert_eq!( - core::mem::offset_of!(fse_decoder::Entry, num_additional_bits), - 8 - ); -} - -#[test] -fn build_from_probabilities_rejects_acc_log_over_entry_limit() { - let mut dec_table = FSETable::new(255); - let err = dec_table - .build_from_probabilities(17, &[1, 1, 1, 1]) - .unwrap_err(); - assert!(matches!( - err, - crate::decoding::errors::FSETableError::AccLogTooBig { got: 17, max: 16 } - )); -} - -#[test] -fn build_decoder_empty_input_reports_bits_error_with_large_max_log() { - let mut dec_table = FSETable::new(255); - let err = dec_table.build_decoder(&[], 17).unwrap_err(); - assert!(matches!( - err, - crate::decoding::errors::FSETableError::GetBitsError(_) - )); -} - -#[test] -fn tables_equal() { - let probs = &[0, 0, -1, 3, 2, 2, (1 << 6) - 8]; - let mut dec_table = FSETable::new(255); - dec_table.build_from_probabilities(6, probs).unwrap(); - let enc_table = fse_encoder::build_table_from_probabilities(probs, 6); - - check_tables(&dec_table, &enc_table); -} - -#[cfg(any(test, feature = "fuzz_exports"))] -fn check_tables(dec_table: &fse_decoder::FSETable, enc_table: &fse_encoder::FSETable) { - // Per-symbol `Vec` storage was dropped in #110 — the encoder now - // holds only `nextStateTable` (upstream zstd parity) and per-symbol - // `symbolTT`. Verify decoder/encoder parity by enumerating every - // valid `(symbol, input_state)` transition and asserting that: - // (a) `next.index` lands on a decoder slot owned by `symbol` - // (the transition reaches a state that decodes back to the - // same symbol — without this, a broken encoder routing - // symbol A into symbol B's slot would slip through), and - // (b) `baseline` / `num_bits` from the encoder match what the - // decoder records for that slot. - // After enumerating, every decoder slot must have been hit at - // least once (full coverage). - let table_size = enc_table.table_size; - let mut hit = alloc::vec![false; table_size]; - for symbol_u in 0..=255u16 { - let symbol = symbol_u as u8; - if enc_table.symbol_probability(symbol) == 0 { - continue; - } - for input_state in 0..table_size { - let next = enc_table.next_state(symbol, input_state); - let dec_state = &dec_table.decode()[next.index]; - assert_eq!( - dec_state.symbol, symbol, - "encoder transition for symbol {symbol} from state {input_state} \ - lands on decoder slot {} which decodes to symbol {} \ - (encoder/decoder routing mismatch)", - next.index, dec_state.symbol - ); - assert_eq!( - next.baseline, dec_state.new_state as usize, - "decode/encode baseline mismatch at slot {} (symbol {symbol})", - next.index - ); - assert_eq!( - next.num_bits, dec_state.num_bits, - "decode/encode num_bits mismatch at slot {} (symbol {symbol})", - next.index - ); - hit[next.index] = true; - } - } - for (idx, was_hit) in hit.iter().enumerate() { - assert!( - *was_hit, - "decoder slot {idx} not reached by any (symbol, prev_state) encoder transition" - ); - } -} - -/// Verify `table_header_bits()` matches the actual byte count written by `write_table()`. -#[test] -#[allow(clippy::borrow_deref_ref)] -fn table_header_bits_exact() { - use crate::bit_io::BitWriter; - use fse_encoder::{ - build_table_from_data, build_table_from_probabilities, default_ll_table, default_ml_table, - default_of_table, - }; - - let check = |table: &fse_encoder::FSETable| { - let mut buf = alloc::vec::Vec::new(); - let mut writer = BitWriter::from(&mut buf); - table.write_table(&mut writer); - writer.flush(); - let written_bits = buf.len() * 8; // flush pads to byte boundary - let computed_bits = table.table_header_bits(); - assert_eq!( - computed_bits, written_bits, - "table_header_bits() mismatch: computed={computed_bits}, written={written_bits}" - ); - }; - - // Predefined tables. Borrow via `&*` so the call compiles on - // both `FseDefaultTable` shapes — `&'static FSETable` (atomic / - // `critical-section`) and `Box` (no-atomic-no-CS). - check(&*default_ll_table()); - check(&*default_ml_table()); - check(&*default_of_table()); - - // Tables built from synthetic data - let data: alloc::vec::Vec = (0u8..32).cycle().take(1000).collect(); - check(&build_table_from_data(data.iter().copied(), 9, true)); - - let data2: alloc::vec::Vec = alloc::vec![0, 1, 2, 3] - .into_iter() - .cycle() - .take(500) - .collect(); - check(&build_table_from_data(data2.iter().copied(), 8, true)); - - // Uniform distribution: 32 symbols × prob=2 = 64 = 1<<6 (acc_log=6 requires sum=64) - check(&build_table_from_probabilities( - &[ - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, - ], - 6, - )); -} - -#[test] -fn roundtrip() { - round_trip(&(0..64).collect::>()); - let mut data = alloc::vec![]; - data.extend(0..32); - data.extend(0..32); - data.extend(0..32); - data.extend(0..32); - data.extend(0..32); - data.extend(20..32); - data.extend(20..32); - data.extend(0..32); - data.extend(20..32); - data.extend(100..255); - data.extend(20..32); - data.extend(20..32); - round_trip(&data); - - #[cfg(feature = "std")] - if std::fs::exists("fuzz/artifacts/fse").unwrap_or(false) { - for file in std::fs::read_dir("fuzz/artifacts/fse").unwrap() { - if file.as_ref().unwrap().file_type().unwrap().is_file() { - let data = std::fs::read(file.unwrap().path()).unwrap(); - round_trip(&data); - } - } - } -} - /// Only needed for testing. /// /// Encodes the data with a table built from that data @@ -283,38 +95,57 @@ pub fn round_trip(data: &[u8]) { assert_eq!(br.bits_remaining(), 0); } -#[test] -#[should_panic(expected = "FSE table requires at least 2 samples")] -fn fse_header_pricing_rejects_single_sample_histogram() { - // A histogram with fewer than two samples cannot form a valid FSE - // table. Header pricing must trip the same `total > 1` guard as the - // builder it mirrors, rather than fall through into table-log / - // normalization arithmetic that underflows on `total <= 1`. - let _ = fse_encoder::fse_header_bits_for_counts(&[1], 9, false); -} - -#[test] -fn fse_header_pricing_matches_built_table_header_bits() { - // The custom-table mode selector trusts that pricing a header from - // counts returns exactly what the eventually built table's - // `table_header_bits()` reports. Lock that invariant across a few - // representative histograms and both `avoid_0_numbit` modes. - let histograms: [&[usize]; 4] = [ - &[50, 50], - &[4, 4, 4, 4], - &[100, 5, 1, 1], - &[200, 100, 50, 25, 12, 6, 3, 1], - ]; - for counts in histograms { - for avoid_0_numbit in [false, true] { - let priced = fse_encoder::fse_header_bits_for_counts(counts, 9, avoid_0_numbit); - let built = fse_encoder::build_table_from_symbol_counts(counts, 9, avoid_0_numbit) - .table_header_bits(); +#[cfg(any(test, feature = "fuzz_exports"))] +fn check_tables(dec_table: &fse_decoder::FSETable, enc_table: &fse_encoder::FSETable) { + // Per-symbol `Vec` storage was dropped in #110 — the encoder now + // holds only `nextStateTable` (upstream zstd parity) and per-symbol + // `symbolTT`. Verify decoder/encoder parity by enumerating every + // valid `(symbol, input_state)` transition and asserting that: + // (a) `next.index` lands on a decoder slot owned by `symbol` + // (the transition reaches a state that decodes back to the + // same symbol — without this, a broken encoder routing + // symbol A into symbol B's slot would slip through), and + // (b) `baseline` / `num_bits` from the encoder match what the + // decoder records for that slot. + // After enumerating, every decoder slot must have been hit at + // least once (full coverage). + let table_size = enc_table.table_size; + let mut hit = alloc::vec![false; table_size]; + for symbol_u in 0..=255u16 { + let symbol = symbol_u as u8; + if enc_table.symbol_probability(symbol) == 0 { + continue; + } + for input_state in 0..table_size { + let next = enc_table.next_state(symbol, input_state); + let dec_state = &dec_table.decode()[next.index]; + assert_eq!( + dec_state.symbol, symbol, + "encoder transition for symbol {symbol} from state {input_state} \ + lands on decoder slot {} which decodes to symbol {} \ + (encoder/decoder routing mismatch)", + next.index, dec_state.symbol + ); assert_eq!( - priced, built, - "header pricing diverged from built table for counts={counts:?} \ - avoid_0_numbit={avoid_0_numbit}" + next.baseline, dec_state.new_state as usize, + "decode/encode baseline mismatch at slot {} (symbol {symbol})", + next.index + ); + assert_eq!( + next.num_bits, dec_state.num_bits, + "decode/encode num_bits mismatch at slot {} (symbol {symbol})", + next.index ); + hit[next.index] = true; } } + for (idx, was_hit) in hit.iter().enumerate() { + assert!( + *was_hit, + "decoder slot {idx} not reached by any (symbol, prev_state) encoder transition" + ); + } } + +#[cfg(test)] +mod tests; diff --git a/zstd/src/fse/tests.rs b/zstd/src/fse/tests.rs new file mode 100644 index 000000000..a173f204c --- /dev/null +++ b/zstd/src/fse/tests.rs @@ -0,0 +1,173 @@ +use super::*; + +#[test] +fn decoder_entry_layout_12_bytes_seqsymbol_shape() { + // Upstream zstd `ZSTD_seqSymbol` shape: 4-byte header (new_state / + // symbol / num_bits), 4-byte `base_value`, 1-byte + // `num_additional_bits` + 3-byte tail padding for natural u32 + // alignment. Total 12 bytes. Grown from the classical 4-byte + // layout to remove the per-sequence `lookup_ll_code` / + // `lookup_ml_code` indirection: LL and ML enrich `base_value` + // / `num_additional_bits` from the packed code-meta tables + // (`enrich_with_packed_seq_meta`), OF enriches closed-form + // `base_value = 1 << code`, `num_additional_bits = code` + // (`enrich_for_offsets`, no meta table). HUF-weight FSE tables + // never enrich and leave both fields zero. + assert_eq!(core::mem::size_of::(), 12); + assert_eq!(core::mem::offset_of!(fse_decoder::Entry, new_state), 0); + assert_eq!(core::mem::offset_of!(fse_decoder::Entry, symbol), 2); + assert_eq!(core::mem::offset_of!(fse_decoder::Entry, num_bits), 3); + assert_eq!(core::mem::offset_of!(fse_decoder::Entry, base_value), 4); + assert_eq!( + core::mem::offset_of!(fse_decoder::Entry, num_additional_bits), + 8 + ); +} + +#[test] +fn build_from_probabilities_rejects_acc_log_over_entry_limit() { + let mut dec_table = FSETable::new(255); + let err = dec_table + .build_from_probabilities(17, &[1, 1, 1, 1]) + .unwrap_err(); + assert!(matches!( + err, + crate::decoding::errors::FSETableError::AccLogTooBig { got: 17, max: 16 } + )); +} + +#[test] +fn build_decoder_empty_input_reports_bits_error_with_large_max_log() { + let mut dec_table = FSETable::new(255); + let err = dec_table.build_decoder(&[], 17).unwrap_err(); + assert!(matches!( + err, + crate::decoding::errors::FSETableError::GetBitsError(_) + )); +} + +#[test] +fn tables_equal() { + let probs = &[0, 0, -1, 3, 2, 2, (1 << 6) - 8]; + let mut dec_table = FSETable::new(255); + dec_table.build_from_probabilities(6, probs).unwrap(); + let enc_table = fse_encoder::build_table_from_probabilities(probs, 6); + + check_tables(&dec_table, &enc_table); +} + +/// Verify `table_header_bits()` matches the actual byte count written by `write_table()`. +#[test] +#[allow(clippy::borrow_deref_ref)] +fn table_header_bits_exact() { + use crate::bit_io::BitWriter; + use fse_encoder::{ + build_table_from_data, build_table_from_probabilities, default_ll_table, default_ml_table, + default_of_table, + }; + + let check = |table: &fse_encoder::FSETable| { + let mut buf = alloc::vec::Vec::new(); + let mut writer = BitWriter::from(&mut buf); + table.write_table(&mut writer); + writer.flush(); + let written_bits = buf.len() * 8; // flush pads to byte boundary + let computed_bits = table.table_header_bits(); + assert_eq!( + computed_bits, written_bits, + "table_header_bits() mismatch: computed={computed_bits}, written={written_bits}" + ); + }; + + // Predefined tables. Borrow via `&*` so the call compiles on + // both `FseDefaultTable` shapes — `&'static FSETable` (atomic / + // `critical-section`) and `Box` (no-atomic-no-CS). + check(&*default_ll_table()); + check(&*default_ml_table()); + check(&*default_of_table()); + + // Tables built from synthetic data + let data: alloc::vec::Vec = (0u8..32).cycle().take(1000).collect(); + check(&build_table_from_data(data.iter().copied(), 9, true)); + + let data2: alloc::vec::Vec = alloc::vec![0, 1, 2, 3] + .into_iter() + .cycle() + .take(500) + .collect(); + check(&build_table_from_data(data2.iter().copied(), 8, true)); + + // Uniform distribution: 32 symbols × prob=2 = 64 = 1<<6 (acc_log=6 requires sum=64) + check(&build_table_from_probabilities( + &[ + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, + ], + 6, + )); +} + +#[test] +fn roundtrip() { + round_trip(&(0..64).collect::>()); + let mut data = alloc::vec![]; + data.extend(0..32); + data.extend(0..32); + data.extend(0..32); + data.extend(0..32); + data.extend(0..32); + data.extend(20..32); + data.extend(20..32); + data.extend(0..32); + data.extend(20..32); + data.extend(100..255); + data.extend(20..32); + data.extend(20..32); + round_trip(&data); + + #[cfg(feature = "std")] + if std::fs::exists("fuzz/artifacts/fse").unwrap_or(false) { + for file in std::fs::read_dir("fuzz/artifacts/fse").unwrap() { + if file.as_ref().unwrap().file_type().unwrap().is_file() { + let data = std::fs::read(file.unwrap().path()).unwrap(); + round_trip(&data); + } + } + } +} + +#[test] +#[should_panic(expected = "FSE table requires at least 2 samples")] +fn fse_header_pricing_rejects_single_sample_histogram() { + // A histogram with fewer than two samples cannot form a valid FSE + // table. Header pricing must trip the same `total > 1` guard as the + // builder it mirrors, rather than fall through into table-log / + // normalization arithmetic that underflows on `total <= 1`. + let _ = fse_encoder::fse_header_bits_for_counts(&[1], 9, false); +} + +#[test] +fn fse_header_pricing_matches_built_table_header_bits() { + // The custom-table mode selector trusts that pricing a header from + // counts returns exactly what the eventually built table's + // `table_header_bits()` reports. Lock that invariant across a few + // representative histograms and both `avoid_0_numbit` modes. + let histograms: [&[usize]; 4] = [ + &[50, 50], + &[4, 4, 4, 4], + &[100, 5, 1, 1], + &[200, 100, 50, 25, 12, 6, 3, 1], + ]; + for counts in histograms { + for avoid_0_numbit in [false, true] { + let priced = fse_encoder::fse_header_bits_for_counts(counts, 9, avoid_0_numbit); + let built = fse_encoder::build_table_from_symbol_counts(counts, 9, avoid_0_numbit) + .table_header_bits(); + assert_eq!( + priced, built, + "header pricing diverged from built table for counts={counts:?} \ + avoid_0_numbit={avoid_0_numbit}" + ); + } + } +} diff --git a/zstd/src/histogram.rs b/zstd/src/histogram.rs index 15bbfcc7e..fbc7484bb 100644 --- a/zstd/src/histogram.rs +++ b/zstd/src/histogram.rs @@ -169,59 +169,4 @@ unsafe fn count_bytes_sve2(data: &[u8], counts: &mut [usize; 256]) -> (usize, us } #[cfg(test)] -mod tests { - use super::{PARALLEL_COUNT_THRESHOLD, count_bytes, count_bytes_scalar, merge_lane_counts}; - - fn make_data(len: usize, seed: u64) -> alloc::vec::Vec { - let mut state = seed; - let mut out = alloc::vec![0u8; len]; - for byte in out.iter_mut() { - state = state.wrapping_mul(6364136223846793005).wrapping_add(1); - *byte = (state >> 32) as u8; - } - out - } - - #[test] - fn count_bytes_matches_scalar_for_large_input() { - let data = make_data(8192, 0xDEADBEEF); - let mut fast = [0usize; 256]; - let mut scalar = [0usize; 256]; - - let fast_meta = count_bytes(&data, &mut fast); - let scalar_meta = count_bytes_scalar(&data, &mut scalar); - - assert_eq!(fast, scalar); - assert_eq!(fast_meta, scalar_meta); - } - - #[test] - fn count_bytes_handles_empty_input() { - let mut counts = [123usize; 256]; - let meta = count_bytes(&[], &mut counts); - - assert_eq!(meta, (0, 0)); - assert!(counts.iter().all(|value| *value == 0)); - } - - #[test] - fn count_bytes_parallel_handles_tail() { - let data = make_data(PARALLEL_COUNT_THRESHOLD + 7, 42); - let mut fast = [0usize; 256]; - let mut scalar = [0usize; 256]; - - let fast_meta = count_bytes(&data, &mut fast); - let scalar_meta = count_bytes_scalar(&data, &mut scalar); - - assert_eq!(fast, scalar); - assert_eq!(fast_meta, scalar_meta); - } - - #[test] - fn merge_lane_counts_widens_before_sum() { - let lane = u32::MAX / 4; - let sum = merge_lane_counts(lane, lane, lane, lane); - let expected = 4u64 * (lane as u64); - assert_eq!(sum as u64, expected); - } -} +mod tests; diff --git a/zstd/src/histogram/tests.rs b/zstd/src/histogram/tests.rs new file mode 100644 index 000000000..2b46ba2c4 --- /dev/null +++ b/zstd/src/histogram/tests.rs @@ -0,0 +1,54 @@ +use super::{PARALLEL_COUNT_THRESHOLD, count_bytes, count_bytes_scalar, merge_lane_counts}; + +fn make_data(len: usize, seed: u64) -> alloc::vec::Vec { + let mut state = seed; + let mut out = alloc::vec![0u8; len]; + for byte in out.iter_mut() { + state = state.wrapping_mul(6364136223846793005).wrapping_add(1); + *byte = (state >> 32) as u8; + } + out +} + +#[test] +fn count_bytes_matches_scalar_for_large_input() { + let data = make_data(8192, 0xDEADBEEF); + let mut fast = [0usize; 256]; + let mut scalar = [0usize; 256]; + + let fast_meta = count_bytes(&data, &mut fast); + let scalar_meta = count_bytes_scalar(&data, &mut scalar); + + assert_eq!(fast, scalar); + assert_eq!(fast_meta, scalar_meta); +} + +#[test] +fn count_bytes_handles_empty_input() { + let mut counts = [123usize; 256]; + let meta = count_bytes(&[], &mut counts); + + assert_eq!(meta, (0, 0)); + assert!(counts.iter().all(|value| *value == 0)); +} + +#[test] +fn count_bytes_parallel_handles_tail() { + let data = make_data(PARALLEL_COUNT_THRESHOLD + 7, 42); + let mut fast = [0usize; 256]; + let mut scalar = [0usize; 256]; + + let fast_meta = count_bytes(&data, &mut fast); + let scalar_meta = count_bytes_scalar(&data, &mut scalar); + + assert_eq!(fast, scalar); + assert_eq!(fast_meta, scalar_meta); +} + +#[test] +fn merge_lane_counts_widens_before_sum() { + let lane = u32::MAX / 4; + let sum = merge_lane_counts(lane, lane, lane, lane); + let expected = 4u64 * (lane as u64); + assert_eq!(sum as u64, expected); +} diff --git a/zstd/src/huff0/huf_cstream.rs b/zstd/src/huff0/huf_cstream.rs index 6bef6e4c7..4b141a105 100644 --- a/zstd/src/huff0/huf_cstream.rs +++ b/zstd/src/huff0/huf_cstream.rs @@ -470,80 +470,4 @@ impl<'a> HufCStream<'a> { } #[cfg(test)] -mod tests { - use super::*; - - /// Roundtrip a single short symbol through HufCStream and verify - /// the byte output decodes back to the same bit pattern. - #[test] - fn add_bits_single_symbol_emits_correct_byte() { - let mut out: Vec = Vec::new(); - let mut s = HufCStream::new(&mut out, 64).expect("init ok"); - // Symbol: nb_bits=4, value=0b1011 (11). Packed: low=4, top=11<<60. - let elt = pack_huf_celt(0b1011, 4); - s.add_bits::(elt, 0); - let n = s.close(); - assert!(n > 0); - assert_eq!(out.len(), 1); - // Upstream zstd `HUF_addBits` + `HUF_flushBits` layout (top-down - // packing in the 64-bit container, then `flushBits` shifts - // the buffered bits down to the bottom of a 0-padded word - // and `MEM_writeLE` stores 8 bytes little-endian — emitted - // byte 0 is the LOW byte of that word): - // - // After `add_bits(pack_huf_celt(0b1011, 4), 0)`: - // container top 4 bits = 0b1011, bit_pos = 4 - // After `close()` prepends end-mark `(value=1, nb_bits=1)`: - // container top 5 bits = [1, 1, 0, 1, 1] (high → low), - // bit_pos = 5 - // `flush_bits` then `container >> (64 - 5)` produces 0b11011 - // = 27 = 0x1B, which lands in `out[0]`. - assert_eq!( - out[0], 0x1B, - "first emitted byte must mirror upstream zstd's HUF_addBits + \ - HUF_endMark packing collapsed to a 5-bit prefix 0b11011", - ); - } - - /// Encode multiple symbols summing to > 64 bits; expect the - /// container to flush partway and write whole bytes to output. - #[test] - fn add_bits_overflowing_container_flushes_correctly() { - let mut out: Vec = Vec::new(); - let mut s = HufCStream::new(&mut out, 256).expect("init ok"); - // 8 symbols of 8 bits each = 64 bits — exactly fills container. - for i in 0..8 { - let elt = pack_huf_celt(i as u32, 8); - s.add_bits::(elt, 0); - } - s.flush_bits::(); - // After flushing 64 bits = 8 bytes; cursor advanced 8. - assert_eq!(s.cursor - s.start_idx, 8); - // pending bits should be 0 (cleanly flushed). - assert_eq!(s.pending_bits(), 0); - let n = s.close(); - // close adds 1-bit end mark + flush → 1 trailing byte for end mark. - assert!(n >= 8); - } - - /// Dual-container parallel encode through `encode_unrolled` (which - /// inlines the zero/merge of container 1 into container 0). With a - /// uniform 4-bit code over 16 symbols, the total emitted size is - /// order-independent: 16 * 4 = 64 payload bits + a 1-bit end mark = - /// 65 bits → 9 bytes. K_UNROLL=4 with 16 symbols runs phase 3 (the - /// dual-container loop) twice, so the merge path is exercised. - #[test] - fn encode_unrolled_dual_container_size_is_deterministic() { - let mut out: Vec = Vec::new(); - let mut s = HufCStream::new(&mut out, 64).expect("init ok"); - // Every symbol maps to the same 4-bit code (value 0b1010). - let table = [pack_huf_celt(0b1010, 4); 256]; - let data = [0u8; 16]; - s.encode_unrolled::<4, false, false>(&table, &data); - let n = s.close(); - assert_eq!( - n, 9, - "16 symbols * 4 bits + 1 end-mark bit = 65 bits = 9 bytes" - ); - } -} +mod tests; diff --git a/zstd/src/huff0/huf_cstream/tests.rs b/zstd/src/huff0/huf_cstream/tests.rs new file mode 100644 index 000000000..e180a72da --- /dev/null +++ b/zstd/src/huff0/huf_cstream/tests.rs @@ -0,0 +1,75 @@ +use super::*; + +/// Roundtrip a single short symbol through HufCStream and verify +/// the byte output decodes back to the same bit pattern. +#[test] +fn add_bits_single_symbol_emits_correct_byte() { + let mut out: Vec = Vec::new(); + let mut s = HufCStream::new(&mut out, 64).expect("init ok"); + // Symbol: nb_bits=4, value=0b1011 (11). Packed: low=4, top=11<<60. + let elt = pack_huf_celt(0b1011, 4); + s.add_bits::(elt, 0); + let n = s.close(); + assert!(n > 0); + assert_eq!(out.len(), 1); + // Upstream zstd `HUF_addBits` + `HUF_flushBits` layout (top-down + // packing in the 64-bit container, then `flushBits` shifts + // the buffered bits down to the bottom of a 0-padded word + // and `MEM_writeLE` stores 8 bytes little-endian — emitted + // byte 0 is the LOW byte of that word): + // + // After `add_bits(pack_huf_celt(0b1011, 4), 0)`: + // container top 4 bits = 0b1011, bit_pos = 4 + // After `close()` prepends end-mark `(value=1, nb_bits=1)`: + // container top 5 bits = [1, 1, 0, 1, 1] (high → low), + // bit_pos = 5 + // `flush_bits` then `container >> (64 - 5)` produces 0b11011 + // = 27 = 0x1B, which lands in `out[0]`. + assert_eq!( + out[0], 0x1B, + "first emitted byte must mirror upstream zstd's HUF_addBits + \ + HUF_endMark packing collapsed to a 5-bit prefix 0b11011", + ); +} + +/// Encode multiple symbols summing to > 64 bits; expect the +/// container to flush partway and write whole bytes to output. +#[test] +fn add_bits_overflowing_container_flushes_correctly() { + let mut out: Vec = Vec::new(); + let mut s = HufCStream::new(&mut out, 256).expect("init ok"); + // 8 symbols of 8 bits each = 64 bits — exactly fills container. + for i in 0..8 { + let elt = pack_huf_celt(i as u32, 8); + s.add_bits::(elt, 0); + } + s.flush_bits::(); + // After flushing 64 bits = 8 bytes; cursor advanced 8. + assert_eq!(s.cursor - s.start_idx, 8); + // pending bits should be 0 (cleanly flushed). + assert_eq!(s.pending_bits(), 0); + let n = s.close(); + // close adds 1-bit end mark + flush → 1 trailing byte for end mark. + assert!(n >= 8); +} + +/// Dual-container parallel encode through `encode_unrolled` (which +/// inlines the zero/merge of container 1 into container 0). With a +/// uniform 4-bit code over 16 symbols, the total emitted size is +/// order-independent: 16 * 4 = 64 payload bits + a 1-bit end mark = +/// 65 bits → 9 bytes. K_UNROLL=4 with 16 symbols runs phase 3 (the +/// dual-container loop) twice, so the merge path is exercised. +#[test] +fn encode_unrolled_dual_container_size_is_deterministic() { + let mut out: Vec = Vec::new(); + let mut s = HufCStream::new(&mut out, 64).expect("init ok"); + // Every symbol maps to the same 4-bit code (value 0b1010). + let table = [pack_huf_celt(0b1010, 4); 256]; + let data = [0u8; 16]; + s.encode_unrolled::<4, false, false>(&table, &data); + let n = s.close(); + assert_eq!( + n, 9, + "16 symbols * 4 bits + 1 end-mark bit = 65 bits = 9 bytes" + ); +} diff --git a/zstd/src/huff0/huff0_decoder.rs b/zstd/src/huff0/huff0_decoder.rs index 1dce67dd9..0523cd4b6 100644 --- a/zstd/src/huff0/huff0_decoder.rs +++ b/zstd/src/huff0/huff0_decoder.rs @@ -944,123 +944,4 @@ fn highest_bit_set(x: u32) -> u32 { } #[cfg(test)] -mod tests { - use super::*; - use alloc::vec; - - fn test_table() -> HuffmanTable { - // Packed `symbol | (num_bits << 8)` per state index (upstream zstd `HUF_DEltX1`). - let packed_decode = vec![ - u16::from(b'A') | (1u16 << 8), - u16::from(b'B') | (2u16 << 8), - u16::from(b'C') | (1u16 << 8), - u16::from(b'D') | (2u16 << 8), - ]; - - HuffmanTable { - packed_decode, - weights: Vec::new(), - max_num_bits: 2, - state_mask: 0b11, - bits: Vec::new(), - bit_ranks: Vec::new(), - weight_sum: 0, - weight_rank_count: [0; (MAX_MAX_NUM_BITS as usize) + 1], - last_weight: 0, - fse_table: FSETable::new(255), - } - } - - #[test] - fn build_decoder_rejects_fse_streams_with_256_explicit_weights() { - // The format caps explicit weights at 255: symbols are u8 and one - // more weight is inferred, so 256 explicit weights would create a - // 257-symbol table whose last index wraps through `symbol as u8`. - // FSE-encode exactly 256 weights (alternating 1/2 keeps the table - // otherwise valid: weight_sum 384, leftover 128 = 2^7) the same way - // the encoder's weight-description path does, and require a loud - // `TooManyWeights` instead of acceptance. - use crate::bit_io::BitWriter; - use crate::fse::fse_encoder::{FSEEncoder, build_table_from_symbol_counts}; - - let weights: Vec = (0..256).map(|i| if i % 2 == 0 { 1 } else { 2 }).collect(); - - let mut encoded = Vec::new(); - { - let mut writer = BitWriter::from(&mut encoded); - let mut counts = [0usize; 13]; - for &w in &weights { - counts[w as usize] += 1; - } - let mut encoder = FSEEncoder::new( - build_table_from_symbol_counts(&counts, 6, false), - &mut writer, - ); - encoder.encode_interleaved(&weights); - writer.flush(); - } - assert!( - encoded.len() < 128, - "fixture must fit the FSE-described header byte, got {}", - encoded.len() - ); - - let mut description = Vec::with_capacity(encoded.len() + 1); - description.push(encoded.len() as u8); - description.extend_from_slice(&encoded); - - let mut table = HuffmanTable::new(); - let result = table.build_decoder(description.as_slice()); - assert!( - matches!(result, Err(HuffmanTableError::TooManyWeights { .. })), - "256 explicit weights must be rejected, got {result:?}" - ); - } - - #[test] - fn decode_symbol_and_advance_scalar_matches_manual_transition() { - let table = test_table(); - let initial_state = 1_u64; - let packed = table.packed_decode[initial_state as usize]; - let entry_num_bits = (packed >> 8) as u8; - let entry_symbol = packed as u8; - let mut manual_br = - BitReaderReversed::::new(&[0b10101010, 0b01010101]); - let expected_new_bits = manual_br.get_bits(entry_num_bits); - let expected_state = - ((initial_state << entry_num_bits) & table.state_mask) | expected_new_bits; - - let mut decoder = HuffmanDecoder { - table: &table, - kernel: HuffmanDecodeKernel::Scalar, - state: initial_state, - }; - let mut br = - BitReaderReversed::::new(&[0b10101010, 0b01010101]); - let symbol = decoder.decode_symbol_and_advance(&mut br); - - assert_eq!(symbol, entry_symbol); - assert_eq!(decoder.state, expected_state); - } - - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - #[test] - fn select_x86_kernel_ordering_is_stable() { - assert_eq!( - select_x86_huffman_decode_kernel(true, true, true, true, true, true), - HuffmanDecodeKernel::X86Vbmi2 - ); - assert_eq!( - select_x86_huffman_decode_kernel(false, false, false, false, true, true), - HuffmanDecodeKernel::X86Avx2 - ); - assert_eq!( - select_x86_huffman_decode_kernel(false, false, false, false, true, false), - HuffmanDecodeKernel::X86Bmi2 - ); - assert_eq!( - select_x86_huffman_decode_kernel(false, false, false, false, false, true), - HuffmanDecodeKernel::Scalar - ); - } -} +mod tests; diff --git a/zstd/src/huff0/huff0_decoder/tests.rs b/zstd/src/huff0/huff0_decoder/tests.rs new file mode 100644 index 000000000..2dbf563f7 --- /dev/null +++ b/zstd/src/huff0/huff0_decoder/tests.rs @@ -0,0 +1,117 @@ +use super::*; +use alloc::vec; + +fn test_table() -> HuffmanTable { + // Packed `symbol | (num_bits << 8)` per state index (upstream zstd `HUF_DEltX1`). + let packed_decode = vec![ + u16::from(b'A') | (1u16 << 8), + u16::from(b'B') | (2u16 << 8), + u16::from(b'C') | (1u16 << 8), + u16::from(b'D') | (2u16 << 8), + ]; + + HuffmanTable { + packed_decode, + weights: Vec::new(), + max_num_bits: 2, + state_mask: 0b11, + bits: Vec::new(), + bit_ranks: Vec::new(), + weight_sum: 0, + weight_rank_count: [0; (MAX_MAX_NUM_BITS as usize) + 1], + last_weight: 0, + fse_table: FSETable::new(255), + } +} + +#[test] +fn build_decoder_rejects_fse_streams_with_256_explicit_weights() { + // The format caps explicit weights at 255: symbols are u8 and one + // more weight is inferred, so 256 explicit weights would create a + // 257-symbol table whose last index wraps through `symbol as u8`. + // FSE-encode exactly 256 weights (alternating 1/2 keeps the table + // otherwise valid: weight_sum 384, leftover 128 = 2^7) the same way + // the encoder's weight-description path does, and require a loud + // `TooManyWeights` instead of acceptance. + use crate::bit_io::BitWriter; + use crate::fse::fse_encoder::{FSEEncoder, build_table_from_symbol_counts}; + + let weights: Vec = (0..256).map(|i| if i % 2 == 0 { 1 } else { 2 }).collect(); + + let mut encoded = Vec::new(); + { + let mut writer = BitWriter::from(&mut encoded); + let mut counts = [0usize; 13]; + for &w in &weights { + counts[w as usize] += 1; + } + let mut encoder = FSEEncoder::new( + build_table_from_symbol_counts(&counts, 6, false), + &mut writer, + ); + encoder.encode_interleaved(&weights); + writer.flush(); + } + assert!( + encoded.len() < 128, + "fixture must fit the FSE-described header byte, got {}", + encoded.len() + ); + + let mut description = Vec::with_capacity(encoded.len() + 1); + description.push(encoded.len() as u8); + description.extend_from_slice(&encoded); + + let mut table = HuffmanTable::new(); + let result = table.build_decoder(description.as_slice()); + assert!( + matches!(result, Err(HuffmanTableError::TooManyWeights { .. })), + "256 explicit weights must be rejected, got {result:?}" + ); +} + +#[test] +fn decode_symbol_and_advance_scalar_matches_manual_transition() { + let table = test_table(); + let initial_state = 1_u64; + let packed = table.packed_decode[initial_state as usize]; + let entry_num_bits = (packed >> 8) as u8; + let entry_symbol = packed as u8; + let mut manual_br = + BitReaderReversed::::new(&[0b10101010, 0b01010101]); + let expected_new_bits = manual_br.get_bits(entry_num_bits); + let expected_state = ((initial_state << entry_num_bits) & table.state_mask) | expected_new_bits; + + let mut decoder = HuffmanDecoder { + table: &table, + kernel: HuffmanDecodeKernel::Scalar, + state: initial_state, + }; + let mut br = + BitReaderReversed::::new(&[0b10101010, 0b01010101]); + let symbol = decoder.decode_symbol_and_advance(&mut br); + + assert_eq!(symbol, entry_symbol); + assert_eq!(decoder.state, expected_state); +} + +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +#[test] +fn select_x86_kernel_ordering_is_stable() { + assert_eq!( + select_x86_huffman_decode_kernel(true, true, true, true, true, true), + HuffmanDecodeKernel::X86Vbmi2 + ); + assert_eq!( + select_x86_huffman_decode_kernel(false, false, false, false, true, true), + HuffmanDecodeKernel::X86Avx2 + ); + assert_eq!( + select_x86_huffman_decode_kernel(false, false, false, false, true, false), + HuffmanDecodeKernel::X86Bmi2 + ); + assert_eq!( + select_x86_huffman_decode_kernel(false, false, false, false, false, true), + HuffmanDecodeKernel::Scalar + ); +} diff --git a/zstd/src/huff0/huff0_encoder.rs b/zstd/src/huff0/huff0_encoder.rs index 64136384b..f9c02fb25 100644 --- a/zstd/src/huff0/huff0_encoder.rs +++ b/zstd/src/huff0/huff0_encoder.rs @@ -1348,24 +1348,6 @@ fn highest_bit_set(x: usize) -> usize { usize::BITS as usize - x.leading_zeros() as usize } -#[test] -fn huffman() { - let table = HuffmanTable::build_from_weights(&[2, 2, 2, 1, 1]); - assert_eq!(table.codes[0], (1, 2)); - assert_eq!(table.codes[1], (2, 2)); - assert_eq!(table.codes[2], (3, 2)); - assert_eq!(table.codes[3], (0, 3)); - assert_eq!(table.codes[4], (1, 3)); - - let table = HuffmanTable::build_from_weights(&[4, 3, 2, 0, 1, 1]); - assert_eq!(table.codes[0], (1, 1)); - assert_eq!(table.codes[1], (1, 2)); - assert_eq!(table.codes[2], (1, 3)); - assert_eq!(table.codes[3], (0, 0)); - assert_eq!(table.codes[4], (0, 4)); - assert_eq!(table.codes[5], (1, 4)); -} - /// Distributes weights that add up to a clean power of two fn distribute_weights(amount: usize) -> Vec { assert!(amount >= 2); @@ -1473,436 +1455,6 @@ fn redistribute_weights(weights: &mut [usize], max_num_bits: usize) { } } -#[test] -fn weights() { - // assert_eq!(distribute_weights(5).as_slice(), &[1, 1, 2, 3, 4]); - for amount in 2..=256 { - let mut weights = distribute_weights(amount); - assert_eq!(weights.len(), amount); - let sum = weights - .iter() - .copied() - .map(|weight| 1 << weight) - .sum::(); - assert!(sum.is_power_of_two()); - - for num_bit_limit in (amount.ilog2() as usize + 1)..=11 { - redistribute_weights(&mut weights, num_bit_limit); - let sum = weights - .iter() - .copied() - .map(|weight| 1 << weight) - .sum::(); - assert!(sum.is_power_of_two()); - assert!( - sum.ilog2() <= 11, - "Max bits too big: sum: {} {weights:?}", - sum - ); - - let codes = HuffmanTable::build_from_weights(&weights).codes; - for (code, num_bits) in codes.iter().copied() { - for (code2, num_bits2) in codes.iter().copied() { - if num_bits == 0 || num_bits2 == 0 || (code, num_bits) == (code2, num_bits2) { - continue; - } - if num_bits <= num_bits2 { - let code2_shifted = code2 >> (num_bits2 - num_bits); - assert_ne!( - code, code2_shifted, - "{code:b},{num_bits:} is prefix of {code2:b},{num_bits2:}" - ); - } - } - } - } - } -} - -#[test] -fn counts() { - let counts = &[3, 0, 4, 1, 5]; - let table = HuffmanTable::build_from_counts(counts).codes; - - assert_eq!(table[1].1, 0); - assert!(table[3].1 >= table[0].1); - assert!(table[0].1 >= table[2].1); - assert!(table[2].1 >= table[4].1); - - let counts = &[3, 0, 4, 0, 7, 2, 2, 2, 0, 2, 2, 1, 5]; - let table = HuffmanTable::build_from_counts(counts).codes; - - assert_eq!(table[1].1, 0); - assert_eq!(table[3].1, 0); - assert_eq!(table[8].1, 0); - assert!(table[11].1 >= table[5].1); - assert!(table[5].1 >= table[6].1); - assert!(table[6].1 >= table[7].1); - assert!(table[7].1 >= table[9].1); - assert!(table[9].1 >= table[10].1); - assert!(table[10].1 >= table[0].1); - assert!(table[0].1 >= table[2].1); - assert!(table[2].1 >= table[12].1); - assert!(table[12].1 >= table[4].1); -} - -#[test] -fn from_data() { - let counts = &[3, 0, 4, 1, 2]; - let table = HuffmanTable::build_from_counts(counts).codes; - - let data = &[0, 2, 4, 4, 0, 3, 2, 2, 0, 2]; - let table2 = HuffmanTable::build_from_data(data).codes; - - assert_eq!(table, table2); -} - -/// `cheap_desc_size_proxy` is the cheap analytic estimate used inside -/// `HuffmanTable::build_from_counts` to score `table_log` candidates -/// without paying a full FSE encode per iteration. Issue #167. -/// -/// Sanity invariants checked here on synthetic weight distributions: -/// -/// - The proxy is **conservative** vs the exact serialized size — it -/// may overestimate by a few bytes (entropy upper bound + 8 B FSE -/// header constant), but **never undershoots so far that the proxy -/// estimate falls below the raw nibble representation** for the same -/// weight stream. This is the guardrail that prevents the loop from -/// picking a `table_log` whose real description is larger than the -/// proxy claims. -/// - The proxy returns `Some` exactly when the real -/// `encode_weight_description` / raw fallback would also produce a -/// serializable description. -#[test] -fn cheap_desc_size_proxy_is_conservative_vs_exact() { - // Fixtures are synthesized via `HuffmanTable::build_from_counts` so - // every weight vector is Kraft-valid by construction (the encoder's - // own output passes its own `huffman_weight_sum_is_power_of_two` - // gate). Hand-curated weight arrays were prone to silently being - // rejected by the Kraft check, leaving the test body unreached - // (caught by CodeRabbit on PR #168). - // - // Each case is `(counts_input, label)` — fed through - // `build_from_counts`, then `table.weights()` is the full weight - // vector and `[..len-1]` is what `try_table_description_size` - // trims internally before calling the encoder. The proxy is - // exercised on the same trimmed slice for a fair comparison. - let cases: &[(Vec, &str)] = &[ - (alloc::vec![5, 3, 2, 1], "4-symbol skewed"), - (alloc::vec![1, 1, 1, 1, 1, 1, 1, 1], "8-symbol uniform"), - (alloc::vec![100, 50, 25, 12, 6, 3, 2, 1], "geometric decay"), - // Wider alphabet: cycle counts over 32 symbols. Build will - // produce a valid Huffman code regardless of exact frequencies. - ((1..=32usize).collect(), "32-symbol increasing"), - // Very wide alphabet that pushes weight count near the raw limit. - ((1..=120usize).collect(), "120-symbol near raw limit"), - ]; - let mut exercised = 0usize; - for (counts, label) in cases { - let table = HuffmanTable::build_from_counts(counts); - let weights = table.weights(); - if weights.is_empty() { - // Single-cardinality fallback path can produce empty - // weights; nothing for the proxy to score. - continue; - } - // `try_table_description_size` trims internally; mirror that - // on the proxy call so both score the same slice. - let trimmed = &weights[..weights.len() - 1]; - let exact = table.try_table_description_size(); - let proxy = cheap_desc_size_proxy(trimmed); - match (proxy, exact) { - (Some(p), Some(e)) => { - exercised += 1; - // Raw representation floor on the trimmed slice — what - // `write_raw_weight_description` would actually emit - // for `trimmed`: ceil(n/2) packed nibbles + 1 length - // byte. The proxy must either be within +2 B of the - // exact size or at least cover this floor (overestimate - // is fine; under-shooting raw is the bug we're - // guarding against). - let raw_floor = trimmed.len().div_ceil(2) + 1; - assert!( - p + 2 >= e || p >= raw_floor, - "[{label}] proxy {p} under-shot exact {e} (raw_floor {raw_floor})" - ); - } - (None, None) => {} // both reject — fine (empty trimmed slice case) - (proxy_res, exact_res) => panic!( - "[{label}] proxy/exact disagreement on representability: proxy={proxy_res:?} exact={exact_res:?}" - ), - } - } - assert!( - exercised > 0, - "no fixture exercised the proxy/exact assertion — synthetic counts must produce Kraft-valid Huffman tables" - ); -} - -/// Edge-case coverage for [`cheap_desc_size_proxy`] — every return arm of -/// the `(fse_ok, raw_ok)` match exercised + the `n == 0` early-out + the -/// `ratio <= 1` clamp. Plugs uncovered branches that the -/// `is_conservative_vs_exact` table didn't reach. Issue #167. -#[test] -fn cheap_desc_size_proxy_edge_cases() { - // `n == 0` → `None` (early-out before the histogram loop). - assert_eq!(cheap_desc_size_proxy(&[]), None); - - // `n == 1`: single symbol, ratio = 1 / 1 = 1 → `<= 1` clamp branch - // fires (1 bit / symbol minimum). FSE estimate = 1 byte payload + 8 - // header = 9 B; raw = 1.div_ceil(2) + 1 = 2 B. Proxy picks min = 2. - assert_eq!(cheap_desc_size_proxy(&[3]), Some(2)); - - // Highly-skewed (one dominant weight): exercises the `ratio > 1` - // branch with `bits_per_symbol == 1` for the dominant bin. - let skew = alloc::vec![1u8; 64]; - let s = cheap_desc_size_proxy(&skew).expect("skewed-small case must be representable"); - assert!(s <= 64usize.div_ceil(2) + 1, "skewed proxy {s} ≤ raw 33"); - - // Exactly at the raw boundary (`weights.len() == 128`): raw is - // representable, both arms reachable depending on which is smaller. - let at_limit: Vec = (0u8..13).cycle().take(128).collect(); - let s = cheap_desc_size_proxy(&at_limit).expect("len=128 stays in (_, raw_ok=true)"); - assert!(s > 0); - - // Past raw boundary (`weights.len() == 129`): `raw_ok = false`. - // The 13-bin uniform-ish histogram still fits FSE → `(true, false)` arm. - let over_raw: Vec = (0u8..13).cycle().take(129).collect(); - let s = cheap_desc_size_proxy(&over_raw) - .expect("uniform 129-symbol stream still fits FSE: (true, false) arm"); - assert!(s > 0); - - // High-entropy + huge length: both representations fail → - // `(false, false)` arm returns `None`. With 256 weights cycling - // over 13 bins, `bits/sym ≈ ceil_log2(ceil(256/20)) = 4`. Total - // payload bits ≈ 1024 b = 128 B, +8 header = 136 > 128 → fse_ok=false. - // raw is also off the table (256 > 128) → None. - let way_over: Vec = (0u8..13).cycle().take(256).collect(); - assert_eq!( - cheap_desc_size_proxy(&way_over), - None, - "huge high-entropy stream hits (false, false) → None" - ); -} - -#[test] -fn encoded_weight_description_roundtrips() { - let data = &include_bytes!("../../decodecorpus_files/z000033")[..16 * 1024]; - let table = HuffmanTable::build_from_data(data); - let mut encoded = Vec::new(); - { - let mut writer = BitWriter::from(&mut encoded); - let mut encoder = HuffmanEncoder::new(&table, &mut writer); - encoder.write_table(); - writer.flush(); - } - - let mut decoded = crate::huff0::huff0_decoder::HuffmanTable::new(); - decoded.build_decoder(&encoded).unwrap(); - let decoded = decoded.to_encoder_table().unwrap(); - - let table_weights = { - let mut out = Vec::new(); - let mut writer = BitWriter::from(&mut out); - let encoder = HuffmanEncoder::new(&table, &mut writer); - encoder.weights() - }; - let decoded_weights = { - let mut out = Vec::new(); - let mut writer = BitWriter::from(&mut out); - let encoder = HuffmanEncoder::new(&decoded, &mut writer); - encoder.weights() - }; - assert_eq!(table_weights, decoded_weights); -} - -#[test] -fn fse_weight_descriptions_roundtrip() { - // Regression for the FSE weight-description encode/decode bug: every weight - // stream that `encode_weight_description` actually FSE-encodes (i.e. passes - // the upstream-zstd early-outs) MUST decode back to the same weights, so the - // encoder can trust its output without a runtime round-trip. Sweep many - // (cardinality, distribution) alphabets; for each, FSE-encode the weight - // description exactly as `encode_weight_description` does and confirm it - // round-trips. Before the early-outs, a single-distinct-weight (uniform) - // alphabet such as 4 symbols → weights [1,1,1] produced a description the - // decoder rejected. - let mut fails: Vec<(usize, u32, alloc::vec::Vec)> = alloc::vec::Vec::new(); - for card in 2usize..=255 { - for skew in 0u32..4 { - let mut data: Vec = Vec::new(); - for s in 0..card { - let n = match skew { - 0 => 1usize, - 1 => s + 1, - 2 => card - s, - _ => ((s * 7 + 1) % 17) + 1, - }; - data.extend(core::iter::repeat_n(s as u8, n)); - } - let table = HuffmanTable::build_from_data(&data); - let mut weights = { - let mut out = Vec::new(); - let mut writer = BitWriter::from(&mut out); - let encoder = HuffmanEncoder::new(&table, &mut writer); - encoder.weights() - }; - weights.pop(); // serialized description omits the final weight - if weights.len() <= 2 { - continue; - } - // Call the PRODUCTION encoder directly so the test can never drift - // from its early-out / FSE-encode logic (re-implementing the counts - // + early-outs inline would silently diverge if the encoder - // changed). `encode_weight_description` returns Some(fse_bytes) only - // for streams it actually FSE-encodes; None means it chose the raw - // description (nothing to round-trip). Every Some MUST decode back. - let Some(encoded) = HuffmanEncoder::>::encode_weight_description(&weights) - else { - continue; - }; - let mut description = Vec::with_capacity(encoded.len() + 1); - description.push(encoded.len() as u8); - description.extend_from_slice(&encoded); - - let mut decoded = crate::huff0::huff0_decoder::HuffmanTable::new(); - let build = decoded.build_decoder(&description); - let decoded_weights = build - .ok() - .and_then(|_| decoded.to_encoder_table()) - .map(|t| { - let mut out = Vec::new(); - let mut writer = BitWriter::from(&mut out); - let encoder = HuffmanEncoder::new(&t, &mut writer); - encoder.weights() - }); - let ok = decoded_weights.as_ref().is_some_and(|dw| { - dw.len() == weights.len() + 1 && dw[..weights.len()] == weights[..] - }); - if !ok { - fails.push((card, skew, weights.clone())); - } - } - } - assert!( - fails.is_empty(), - "{} FSE weight cases still fail to round-trip after upstream-zstd early-outs; first 5: {:?}", - fails.len(), - &fails[..fails.len().min(5)] - ); -} - -#[test] -fn large_alphabet_weight_description_uses_fse_when_raw_is_unrepresentable() { - let mut data = Vec::new(); - for symbol in 0u8..=255 { - data.extend(core::iter::repeat_n(symbol, usize::from(symbol) + 1)); - } - let table = HuffmanTable::build_from_data(&data); - let mut weights = { - let mut out = Vec::new(); - let mut writer = BitWriter::from(&mut out); - let encoder = HuffmanEncoder::new(&table, &mut writer); - encoder.weights() - }; - weights.pop(); - assert!( - weights.len() > 128, - "fixture must require an FSE table description" - ); - - let encoded = HuffmanEncoder::>::encode_weight_description(&weights) - .expect("FSE weight description must be available when raw weights cannot be represented"); - let mut description = Vec::with_capacity(encoded.len() + 1); - description.push(encoded.len() as u8); - description.extend_from_slice(&encoded); - - // The encoder no longer round-trip-verifies at runtime (it trusts the FSE - // encoding after the upstream-zstd early-outs, matching upstream zstd); assert the - // decodes-back property here instead. - let mut decoded = crate::huff0::huff0_decoder::HuffmanTable::new(); - decoded - .build_decoder(&description) - .expect("FSE weight description must decode"); - let decoded = decoded - .to_encoder_table() - .expect("decoded weight table must convert to an encoder table"); - let decoded_weights = { - let mut out = Vec::new(); - let mut writer = BitWriter::from(&mut out); - let encoder = HuffmanEncoder::new(&decoded, &mut writer); - encoder.weights() - }; - assert_eq!(decoded_weights.len(), weights.len() + 1); - assert_eq!(&decoded_weights[..weights.len()], &weights[..]); -} - -#[cfg(feature = "std")] -#[test] -fn cached_encoded_weight_description_is_reused_for_write_table() { - let mut data = Vec::new(); - for symbol in 0u8..=255 { - data.extend(core::iter::repeat_n(symbol, usize::from(symbol) + 1)); - } - let table = HuffmanTable::build_from_data(&data); - let desc_size = table - .writeable_table_description_size() - .expect("table description must be writable"); - let cached = table - .cached_encoded_weight_description - .get() - .and_then(Option::as_ref) - .expect("large alphabet fixture must cache FSE description") - .clone(); - assert_eq!(desc_size, cached.len() + 1); - - let mut encoded = Vec::new(); - { - let mut writer = BitWriter::from(&mut encoded); - let mut encoder = HuffmanEncoder::new(&table, &mut writer); - encoder.write_table(); - writer.flush(); - } - assert_eq!(encoded[0] as usize, cached.len()); - assert_eq!(&encoded[1..], cached.as_slice()); -} - -#[cfg(feature = "std")] -#[test] -fn write_table_raw_path_initializes_none_cache() { - let table = HuffmanTable::build_from_weights(&[1, 1]); - assert!(table.cached_encoded_weight_description.get().is_none()); - - let mut expected = Vec::new(); - let weights = { - let mut out = Vec::new(); - let mut writer = BitWriter::from(&mut out); - let encoder = HuffmanEncoder::new(&table, &mut writer); - encoder.weights() - }; - { - let mut writer = BitWriter::from(&mut expected); - HuffmanEncoder::>::write_raw_weight_description( - &mut writer, - &weights[..weights.len() - 1], - ); - writer.flush(); - } - - let mut encoded = Vec::new(); - { - let mut writer = BitWriter::from(&mut encoded); - let mut encoder = HuffmanEncoder::new(&table, &mut writer); - encoder.write_table(); - writer.flush(); - } - assert_eq!(encoded, expected); - assert!(matches!( - table.cached_encoded_weight_description.get(), - Some(None) - )); -} - /// White-box capture of the FSE-coded Huffman weight description our encoder /// emits for `data` (a length byte followed by the FSE payload), plus the raw /// per-symbol weights. Returns `(description, weights)`. The C-conformance @@ -1942,3 +1494,6 @@ pub(crate) fn huf_encode4x_for_test(data: &[u8]) -> Vec { } encoded } + +#[cfg(test)] +mod tests; diff --git a/zstd/src/huff0/huff0_encoder/tests.rs b/zstd/src/huff0/huff0_encoder/tests.rs new file mode 100644 index 000000000..57193c53d --- /dev/null +++ b/zstd/src/huff0/huff0_encoder/tests.rs @@ -0,0 +1,449 @@ +use super::*; + +#[test] +fn huffman() { + let table = HuffmanTable::build_from_weights(&[2, 2, 2, 1, 1]); + assert_eq!(table.codes[0], (1, 2)); + assert_eq!(table.codes[1], (2, 2)); + assert_eq!(table.codes[2], (3, 2)); + assert_eq!(table.codes[3], (0, 3)); + assert_eq!(table.codes[4], (1, 3)); + + let table = HuffmanTable::build_from_weights(&[4, 3, 2, 0, 1, 1]); + assert_eq!(table.codes[0], (1, 1)); + assert_eq!(table.codes[1], (1, 2)); + assert_eq!(table.codes[2], (1, 3)); + assert_eq!(table.codes[3], (0, 0)); + assert_eq!(table.codes[4], (0, 4)); + assert_eq!(table.codes[5], (1, 4)); +} + +#[test] +fn weights() { + // assert_eq!(distribute_weights(5).as_slice(), &[1, 1, 2, 3, 4]); + for amount in 2..=256 { + let mut weights = distribute_weights(amount); + assert_eq!(weights.len(), amount); + let sum = weights + .iter() + .copied() + .map(|weight| 1 << weight) + .sum::(); + assert!(sum.is_power_of_two()); + + for num_bit_limit in (amount.ilog2() as usize + 1)..=11 { + redistribute_weights(&mut weights, num_bit_limit); + let sum = weights + .iter() + .copied() + .map(|weight| 1 << weight) + .sum::(); + assert!(sum.is_power_of_two()); + assert!( + sum.ilog2() <= 11, + "Max bits too big: sum: {} {weights:?}", + sum + ); + + let codes = HuffmanTable::build_from_weights(&weights).codes; + for (code, num_bits) in codes.iter().copied() { + for (code2, num_bits2) in codes.iter().copied() { + if num_bits == 0 || num_bits2 == 0 || (code, num_bits) == (code2, num_bits2) { + continue; + } + if num_bits <= num_bits2 { + let code2_shifted = code2 >> (num_bits2 - num_bits); + assert_ne!( + code, code2_shifted, + "{code:b},{num_bits:} is prefix of {code2:b},{num_bits2:}" + ); + } + } + } + } + } +} + +#[test] +fn counts() { + let counts = &[3, 0, 4, 1, 5]; + let table = HuffmanTable::build_from_counts(counts).codes; + + assert_eq!(table[1].1, 0); + assert!(table[3].1 >= table[0].1); + assert!(table[0].1 >= table[2].1); + assert!(table[2].1 >= table[4].1); + + let counts = &[3, 0, 4, 0, 7, 2, 2, 2, 0, 2, 2, 1, 5]; + let table = HuffmanTable::build_from_counts(counts).codes; + + assert_eq!(table[1].1, 0); + assert_eq!(table[3].1, 0); + assert_eq!(table[8].1, 0); + assert!(table[11].1 >= table[5].1); + assert!(table[5].1 >= table[6].1); + assert!(table[6].1 >= table[7].1); + assert!(table[7].1 >= table[9].1); + assert!(table[9].1 >= table[10].1); + assert!(table[10].1 >= table[0].1); + assert!(table[0].1 >= table[2].1); + assert!(table[2].1 >= table[12].1); + assert!(table[12].1 >= table[4].1); +} + +#[test] +fn from_data() { + let counts = &[3, 0, 4, 1, 2]; + let table = HuffmanTable::build_from_counts(counts).codes; + + let data = &[0, 2, 4, 4, 0, 3, 2, 2, 0, 2]; + let table2 = HuffmanTable::build_from_data(data).codes; + + assert_eq!(table, table2); +} + +/// `cheap_desc_size_proxy` is the cheap analytic estimate used inside +/// `HuffmanTable::build_from_counts` to score `table_log` candidates +/// without paying a full FSE encode per iteration. Issue #167. +/// +/// Sanity invariants checked here on synthetic weight distributions: +/// +/// - The proxy is **conservative** vs the exact serialized size — it +/// may overestimate by a few bytes (entropy upper bound + 8 B FSE +/// header constant), but **never undershoots so far that the proxy +/// estimate falls below the raw nibble representation** for the same +/// weight stream. This is the guardrail that prevents the loop from +/// picking a `table_log` whose real description is larger than the +/// proxy claims. +/// - The proxy returns `Some` exactly when the real +/// `encode_weight_description` / raw fallback would also produce a +/// serializable description. +#[test] +fn cheap_desc_size_proxy_is_conservative_vs_exact() { + // Fixtures are synthesized via `HuffmanTable::build_from_counts` so + // every weight vector is Kraft-valid by construction (the encoder's + // own output passes its own `huffman_weight_sum_is_power_of_two` + // gate). Hand-curated weight arrays were prone to silently being + // rejected by the Kraft check, leaving the test body unreached + // (caught by CodeRabbit on PR #168). + // + // Each case is `(counts_input, label)` — fed through + // `build_from_counts`, then `table.weights()` is the full weight + // vector and `[..len-1]` is what `try_table_description_size` + // trims internally before calling the encoder. The proxy is + // exercised on the same trimmed slice for a fair comparison. + let cases: &[(Vec, &str)] = &[ + (alloc::vec![5, 3, 2, 1], "4-symbol skewed"), + (alloc::vec![1, 1, 1, 1, 1, 1, 1, 1], "8-symbol uniform"), + (alloc::vec![100, 50, 25, 12, 6, 3, 2, 1], "geometric decay"), + // Wider alphabet: cycle counts over 32 symbols. Build will + // produce a valid Huffman code regardless of exact frequencies. + ((1..=32usize).collect(), "32-symbol increasing"), + // Very wide alphabet that pushes weight count near the raw limit. + ((1..=120usize).collect(), "120-symbol near raw limit"), + ]; + let mut exercised = 0usize; + for (counts, label) in cases { + let table = HuffmanTable::build_from_counts(counts); + let weights = table.weights(); + if weights.is_empty() { + // Single-cardinality fallback path can produce empty + // weights; nothing for the proxy to score. + continue; + } + // `try_table_description_size` trims internally; mirror that + // on the proxy call so both score the same slice. + let trimmed = &weights[..weights.len() - 1]; + let exact = table.try_table_description_size(); + let proxy = cheap_desc_size_proxy(trimmed); + match (proxy, exact) { + (Some(p), Some(e)) => { + exercised += 1; + // Raw representation floor on the trimmed slice — what + // `write_raw_weight_description` would actually emit + // for `trimmed`: ceil(n/2) packed nibbles + 1 length + // byte. The proxy must either be within +2 B of the + // exact size or at least cover this floor (overestimate + // is fine; under-shooting raw is the bug we're + // guarding against). + let raw_floor = trimmed.len().div_ceil(2) + 1; + assert!( + p + 2 >= e || p >= raw_floor, + "[{label}] proxy {p} under-shot exact {e} (raw_floor {raw_floor})" + ); + } + (None, None) => {} // both reject — fine (empty trimmed slice case) + (proxy_res, exact_res) => panic!( + "[{label}] proxy/exact disagreement on representability: proxy={proxy_res:?} exact={exact_res:?}" + ), + } + } + assert!( + exercised > 0, + "no fixture exercised the proxy/exact assertion — synthetic counts must produce Kraft-valid Huffman tables" + ); +} + +/// Edge-case coverage for [`cheap_desc_size_proxy`] — every return arm of +/// the `(fse_ok, raw_ok)` match exercised + the `n == 0` early-out + the +/// `ratio <= 1` clamp. Plugs uncovered branches that the +/// `is_conservative_vs_exact` table didn't reach. Issue #167. +#[test] +fn cheap_desc_size_proxy_edge_cases() { + // `n == 0` → `None` (early-out before the histogram loop). + assert_eq!(cheap_desc_size_proxy(&[]), None); + + // `n == 1`: single symbol, ratio = 1 / 1 = 1 → `<= 1` clamp branch + // fires (1 bit / symbol minimum). FSE estimate = 1 byte payload + 8 + // header = 9 B; raw = 1.div_ceil(2) + 1 = 2 B. Proxy picks min = 2. + assert_eq!(cheap_desc_size_proxy(&[3]), Some(2)); + + // Highly-skewed (one dominant weight): exercises the `ratio > 1` + // branch with `bits_per_symbol == 1` for the dominant bin. + let skew = alloc::vec![1u8; 64]; + let s = cheap_desc_size_proxy(&skew).expect("skewed-small case must be representable"); + assert!(s <= 64usize.div_ceil(2) + 1, "skewed proxy {s} ≤ raw 33"); + + // Exactly at the raw boundary (`weights.len() == 128`): raw is + // representable, both arms reachable depending on which is smaller. + let at_limit: Vec = (0u8..13).cycle().take(128).collect(); + let s = cheap_desc_size_proxy(&at_limit).expect("len=128 stays in (_, raw_ok=true)"); + assert!(s > 0); + + // Past raw boundary (`weights.len() == 129`): `raw_ok = false`. + // The 13-bin uniform-ish histogram still fits FSE → `(true, false)` arm. + let over_raw: Vec = (0u8..13).cycle().take(129).collect(); + let s = cheap_desc_size_proxy(&over_raw) + .expect("uniform 129-symbol stream still fits FSE: (true, false) arm"); + assert!(s > 0); + + // High-entropy + huge length: both representations fail → + // `(false, false)` arm returns `None`. With 256 weights cycling + // over 13 bins, `bits/sym ≈ ceil_log2(ceil(256/20)) = 4`. Total + // payload bits ≈ 1024 b = 128 B, +8 header = 136 > 128 → fse_ok=false. + // raw is also off the table (256 > 128) → None. + let way_over: Vec = (0u8..13).cycle().take(256).collect(); + assert_eq!( + cheap_desc_size_proxy(&way_over), + None, + "huge high-entropy stream hits (false, false) → None" + ); +} + +#[test] +fn encoded_weight_description_roundtrips() { + let data = &include_bytes!("../../../decodecorpus_files/z000033")[..16 * 1024]; + let table = HuffmanTable::build_from_data(data); + let mut encoded = Vec::new(); + { + let mut writer = BitWriter::from(&mut encoded); + let mut encoder = HuffmanEncoder::new(&table, &mut writer); + encoder.write_table(); + writer.flush(); + } + + let mut decoded = crate::huff0::huff0_decoder::HuffmanTable::new(); + decoded.build_decoder(&encoded).unwrap(); + let decoded = decoded.to_encoder_table().unwrap(); + + let table_weights = { + let mut out = Vec::new(); + let mut writer = BitWriter::from(&mut out); + let encoder = HuffmanEncoder::new(&table, &mut writer); + encoder.weights() + }; + let decoded_weights = { + let mut out = Vec::new(); + let mut writer = BitWriter::from(&mut out); + let encoder = HuffmanEncoder::new(&decoded, &mut writer); + encoder.weights() + }; + assert_eq!(table_weights, decoded_weights); +} + +#[test] +fn fse_weight_descriptions_roundtrip() { + // Regression for the FSE weight-description encode/decode bug: every weight + // stream that `encode_weight_description` actually FSE-encodes (i.e. passes + // the upstream-zstd early-outs) MUST decode back to the same weights, so the + // encoder can trust its output without a runtime round-trip. Sweep many + // (cardinality, distribution) alphabets; for each, FSE-encode the weight + // description exactly as `encode_weight_description` does and confirm it + // round-trips. Before the early-outs, a single-distinct-weight (uniform) + // alphabet such as 4 symbols → weights [1,1,1] produced a description the + // decoder rejected. + let mut fails: Vec<(usize, u32, alloc::vec::Vec)> = alloc::vec::Vec::new(); + for card in 2usize..=255 { + for skew in 0u32..4 { + let mut data: Vec = Vec::new(); + for s in 0..card { + let n = match skew { + 0 => 1usize, + 1 => s + 1, + 2 => card - s, + _ => ((s * 7 + 1) % 17) + 1, + }; + data.extend(core::iter::repeat_n(s as u8, n)); + } + let table = HuffmanTable::build_from_data(&data); + let mut weights = { + let mut out = Vec::new(); + let mut writer = BitWriter::from(&mut out); + let encoder = HuffmanEncoder::new(&table, &mut writer); + encoder.weights() + }; + weights.pop(); // serialized description omits the final weight + if weights.len() <= 2 { + continue; + } + // Call the PRODUCTION encoder directly so the test can never drift + // from its early-out / FSE-encode logic (re-implementing the counts + // + early-outs inline would silently diverge if the encoder + // changed). `encode_weight_description` returns Some(fse_bytes) only + // for streams it actually FSE-encodes; None means it chose the raw + // description (nothing to round-trip). Every Some MUST decode back. + let Some(encoded) = HuffmanEncoder::>::encode_weight_description(&weights) + else { + continue; + }; + let mut description = Vec::with_capacity(encoded.len() + 1); + description.push(encoded.len() as u8); + description.extend_from_slice(&encoded); + + let mut decoded = crate::huff0::huff0_decoder::HuffmanTable::new(); + let build = decoded.build_decoder(&description); + let decoded_weights = build + .ok() + .and_then(|_| decoded.to_encoder_table()) + .map(|t| { + let mut out = Vec::new(); + let mut writer = BitWriter::from(&mut out); + let encoder = HuffmanEncoder::new(&t, &mut writer); + encoder.weights() + }); + let ok = decoded_weights.as_ref().is_some_and(|dw| { + dw.len() == weights.len() + 1 && dw[..weights.len()] == weights[..] + }); + if !ok { + fails.push((card, skew, weights.clone())); + } + } + } + assert!( + fails.is_empty(), + "{} FSE weight cases still fail to round-trip after upstream-zstd early-outs; first 5: {:?}", + fails.len(), + &fails[..fails.len().min(5)] + ); +} + +#[test] +fn large_alphabet_weight_description_uses_fse_when_raw_is_unrepresentable() { + let mut data = Vec::new(); + for symbol in 0u8..=255 { + data.extend(core::iter::repeat_n(symbol, usize::from(symbol) + 1)); + } + let table = HuffmanTable::build_from_data(&data); + let mut weights = { + let mut out = Vec::new(); + let mut writer = BitWriter::from(&mut out); + let encoder = HuffmanEncoder::new(&table, &mut writer); + encoder.weights() + }; + weights.pop(); + assert!( + weights.len() > 128, + "fixture must require an FSE table description" + ); + + let encoded = HuffmanEncoder::>::encode_weight_description(&weights) + .expect("FSE weight description must be available when raw weights cannot be represented"); + let mut description = Vec::with_capacity(encoded.len() + 1); + description.push(encoded.len() as u8); + description.extend_from_slice(&encoded); + + // The encoder no longer round-trip-verifies at runtime (it trusts the FSE + // encoding after the upstream-zstd early-outs, matching upstream zstd); assert the + // decodes-back property here instead. + let mut decoded = crate::huff0::huff0_decoder::HuffmanTable::new(); + decoded + .build_decoder(&description) + .expect("FSE weight description must decode"); + let decoded = decoded + .to_encoder_table() + .expect("decoded weight table must convert to an encoder table"); + let decoded_weights = { + let mut out = Vec::new(); + let mut writer = BitWriter::from(&mut out); + let encoder = HuffmanEncoder::new(&decoded, &mut writer); + encoder.weights() + }; + assert_eq!(decoded_weights.len(), weights.len() + 1); + assert_eq!(&decoded_weights[..weights.len()], &weights[..]); +} + +#[cfg(feature = "std")] +#[test] +fn cached_encoded_weight_description_is_reused_for_write_table() { + let mut data = Vec::new(); + for symbol in 0u8..=255 { + data.extend(core::iter::repeat_n(symbol, usize::from(symbol) + 1)); + } + let table = HuffmanTable::build_from_data(&data); + let desc_size = table + .writeable_table_description_size() + .expect("table description must be writable"); + let cached = table + .cached_encoded_weight_description + .get() + .and_then(Option::as_ref) + .expect("large alphabet fixture must cache FSE description") + .clone(); + assert_eq!(desc_size, cached.len() + 1); + + let mut encoded = Vec::new(); + { + let mut writer = BitWriter::from(&mut encoded); + let mut encoder = HuffmanEncoder::new(&table, &mut writer); + encoder.write_table(); + writer.flush(); + } + assert_eq!(encoded[0] as usize, cached.len()); + assert_eq!(&encoded[1..], cached.as_slice()); +} + +#[cfg(feature = "std")] +#[test] +fn write_table_raw_path_initializes_none_cache() { + let table = HuffmanTable::build_from_weights(&[1, 1]); + assert!(table.cached_encoded_weight_description.get().is_none()); + + let mut expected = Vec::new(); + let weights = { + let mut out = Vec::new(); + let mut writer = BitWriter::from(&mut out); + let encoder = HuffmanEncoder::new(&table, &mut writer); + encoder.weights() + }; + { + let mut writer = BitWriter::from(&mut expected); + HuffmanEncoder::>::write_raw_weight_description( + &mut writer, + &weights[..weights.len() - 1], + ); + writer.flush(); + } + + let mut encoded = Vec::new(); + { + let mut writer = BitWriter::from(&mut encoded); + let mut encoder = HuffmanEncoder::new(&table, &mut writer); + encoder.write_table(); + writer.flush(); + } + assert_eq!(encoded, expected); + assert!(matches!( + table.cached_encoded_weight_description.get(), + Some(None) + )); +} diff --git a/zstd/src/huff0/mod.rs b/zstd/src/huff0/mod.rs index fd4c38197..42be18c2a 100644 --- a/zstd/src/huff0/mod.rs +++ b/zstd/src/huff0/mod.rs @@ -59,30 +59,5 @@ pub fn round_trip(data: &[u8]) { assert_eq!(&decoded, data); } -#[test] -fn roundtrip() { - use alloc::vec::Vec; - round_trip(&[1, 1, 1, 1, 2, 3]); - round_trip(&[1, 1, 1, 1, 2, 3, 5, 45, 12, 90]); - - for size in 2..255 { - use alloc::vec; - let data = vec![123; size]; - round_trip(&data); - let mut data = Vec::new(); - for x in 0..size { - data.push(x as u8); - } - round_trip(&data); - } - - #[cfg(feature = "std")] - if std::fs::exists("fuzz/artifacts/huff0").unwrap_or(false) { - for file in std::fs::read_dir("fuzz/artifacts/huff0").unwrap() { - if file.as_ref().unwrap().file_type().unwrap().is_file() { - let data = std::fs::read(file.unwrap().path()).unwrap(); - round_trip(&data); - } - } - } -} +#[cfg(test)] +mod tests; diff --git a/zstd/src/huff0/tests.rs b/zstd/src/huff0/tests.rs new file mode 100644 index 000000000..5485935a0 --- /dev/null +++ b/zstd/src/huff0/tests.rs @@ -0,0 +1,29 @@ +use super::*; + +#[test] +fn roundtrip() { + use alloc::vec::Vec; + round_trip(&[1, 1, 1, 1, 2, 3]); + round_trip(&[1, 1, 1, 1, 2, 3, 5, 45, 12, 90]); + + for size in 2..255 { + use alloc::vec; + let data = vec![123; size]; + round_trip(&data); + let mut data = Vec::new(); + for x in 0..size { + data.push(x as u8); + } + round_trip(&data); + } + + #[cfg(feature = "std")] + if std::fs::exists("fuzz/artifacts/huff0").unwrap_or(false) { + for file in std::fs::read_dir("fuzz/artifacts/huff0").unwrap() { + if file.as_ref().unwrap().file_type().unwrap().is_file() { + let data = std::fs::read(file.unwrap().path()).unwrap(); + round_trip(&data); + } + } + } +} diff --git a/zstd/src/skippable.rs b/zstd/src/skippable.rs index 22f915531..faaced4d4 100644 --- a/zstd/src/skippable.rs +++ b/zstd/src/skippable.rs @@ -427,317 +427,4 @@ impl std::error::Error for DecodeSkippableFrameError { } #[cfg(all(test, feature = "std"))] -mod tests { - use super::*; - - fn build_expected_skippable(magic_variant: u8, payload: &[u8]) -> Vec { - // Upstream zstd `ZSTD_writeSkippableFrame` (zstd v1.5.7 - // `lib/compress/zstd_compress.c:4751-4763`) emits exactly - // `4-byte LE magic || 4-byte LE size || payload`. Mirror that - // here as the byte-parity oracle. Re-implementing the upstream zstd - // layout in the test (rather than calling out to zstd-sys) - // keeps this test independent of the dev-dep wiring and - // makes the parity expectation visible inline. - let magic = (SKIPPABLE_MAGIC_START + u32::from(magic_variant)).to_le_bytes(); - let size = (payload.len() as u32).to_le_bytes(); - let mut out = Vec::with_capacity(payload.len() + SKIPPABLE_HEADER_SIZE); - out.extend_from_slice(&magic); - out.extend_from_slice(&size); - out.extend_from_slice(payload); - out - } - - #[test] - fn round_trip_all_sixteen_variants() { - for variant in 0u8..=15 { - let payload = alloc::vec![variant; 32 + variant as usize]; - let frame = SkippableFrame::new(variant, payload.clone()).expect("variant in range"); - let mut wire = Vec::new(); - frame - .encode_into(&mut wire) - .expect("encode into Vec succeeds"); - - let mut cursor: &[u8] = wire.as_slice(); - let decoded = SkippableFrame::decode_from(&mut cursor).expect("round-trip decode"); - assert_eq!(decoded.magic_variant(), variant); - assert_eq!( - decoded.magic_number(), - SKIPPABLE_MAGIC_START + u32::from(variant) - ); - assert_eq!(decoded.payload(), payload.as_slice()); - assert!( - cursor.is_empty(), - "decode_from must consume exactly the frame bytes, no overshoot or undershoot" - ); - } - } - - #[test] - fn empty_payload_round_trips() { - let frame = SkippableFrame::new(7, Vec::new()).expect("empty payload OK"); - assert_eq!(frame.serialized_size(), SKIPPABLE_HEADER_SIZE); - - let mut wire = Vec::new(); - frame.encode_into(&mut wire).unwrap(); - assert_eq!(wire.len(), SKIPPABLE_HEADER_SIZE); - - let mut cursor: &[u8] = wire.as_slice(); - let decoded = SkippableFrame::decode_from(&mut cursor).unwrap(); - assert!(decoded.payload().is_empty()); - assert_eq!(decoded.magic_variant(), 7); - } - - #[test] - fn large_payload_round_trips() { - // 1 MiB so the 4-byte length field carries a non-trivial - // value (0x100000) — the byte-parity test below verifies the - // LE serialisation explicitly. - let payload = alloc::vec![0xABu8; 1024 * 1024]; - let frame = SkippableFrame::new(0, payload.clone()).unwrap(); - let mut wire = Vec::new(); - frame.encode_into(&mut wire).unwrap(); - assert_eq!(wire.len(), payload.len() + SKIPPABLE_HEADER_SIZE); - - let mut cursor: &[u8] = wire.as_slice(); - let decoded = SkippableFrame::decode_from(&mut cursor).unwrap(); - assert_eq!(decoded.payload().len(), payload.len()); - assert!(decoded.payload() == payload.as_slice()); - } - - #[test] - fn new_rejects_variant_sixteen() { - let err = SkippableFrame::new(16, Vec::new()).expect_err("variant 16 out of range"); - match err { - SkippableFrameError::InvalidMagicVariant(v) => assert_eq!(v, 16), - other => panic!("expected InvalidMagicVariant(16), got {other:?}"), - } - } - - #[test] - fn new_rejects_variant_max() { - // u8::MAX = 255 — clearly outside the spec's 0..=15 range. - let err = SkippableFrame::new(255, Vec::new()).unwrap_err(); - match err { - SkippableFrameError::InvalidMagicVariant(v) => assert_eq!(v, 255), - other => panic!("expected InvalidMagicVariant(255), got {other:?}"), - } - } - - #[test] - fn write_function_rejects_invalid_variant() { - let mut sink: Vec = Vec::new(); - let err = write_skippable_frame(16, b"x", &mut sink).unwrap_err(); - assert!(matches!(err, SkippableFrameError::InvalidMagicVariant(16))); - assert!( - sink.is_empty(), - "no bytes must be written on rejected input" - ); - } - - #[test] - fn byte_parity_with_spec_layout() { - // For every variant + a handful of payload sizes, our output - // bytes must equal the upstream zstd's `ZSTD_writeSkippableFrame` - // layout byte-for-byte. This locks the wire-format contract - // against future drift. - for &payload_len in &[0usize, 1, 8, 256, 4096] { - let payload: Vec = (0..payload_len).map(|i| (i % 251) as u8).collect(); - for variant in 0u8..=15 { - let expected = build_expected_skippable(variant, &payload); - - let mut via_struct = Vec::new(); - SkippableFrame::new(variant, payload.clone()) - .unwrap() - .encode_into(&mut via_struct) - .unwrap(); - assert_eq!( - via_struct, expected, - "struct encode mismatch: variant={variant} len={payload_len}" - ); - - let mut via_free = Vec::new(); - let written = write_skippable_frame(variant, &payload, &mut via_free).unwrap(); - assert_eq!(written, expected.len()); - assert_eq!( - via_free, expected, - "free-fn encode mismatch: variant={variant} len={payload_len}" - ); - } - } - } - - #[test] - fn decode_rejects_non_skippable_magic() { - // Zstd-1 magic 0xFD2FB528 is NOT in the skippable range. - let mut wire = Vec::new(); - wire.extend_from_slice(&0xFD2F_B528u32.to_le_bytes()); - wire.extend_from_slice(&0u32.to_le_bytes()); - let mut cursor: &[u8] = wire.as_slice(); - let err = SkippableFrame::decode_from(&mut cursor).unwrap_err(); - match err { - DecodeSkippableFrameError::BadMagicNumber(m) => assert_eq!(m, 0xFD2F_B528), - other => panic!("expected BadMagicNumber, got {other:?}"), - } - } - - #[test] - fn decode_rejects_magic_above_band() { - // 0x184D2A60 is one past the skippable band — must be - // rejected via BadMagicNumber, not silently accepted as - // variant 16. - let mut wire = Vec::new(); - wire.extend_from_slice(&0x184D_2A60u32.to_le_bytes()); - wire.extend_from_slice(&0u32.to_le_bytes()); - let mut cursor: &[u8] = wire.as_slice(); - let err = SkippableFrame::decode_from(&mut cursor).unwrap_err(); - assert!(matches!( - err, - DecodeSkippableFrameError::BadMagicNumber(0x184D_2A60) - )); - } - - #[test] - fn decode_truncated_magic_surfaces_typed_error() { - // Three bytes (one less than a magic) — must fail on the - // magic read step, not panic. - let wire = [0x50u8, 0x2A, 0x4D]; - let mut cursor: &[u8] = &wire; - let err = SkippableFrame::decode_from(&mut cursor).unwrap_err(); - assert!( - matches!(err, DecodeSkippableFrameError::Magic(_)), - "expected Magic, got {err:?}" - ); - } - - #[test] - fn decode_truncated_length_surfaces_typed_error() { - // Magic OK, but length field is short (3 bytes instead of 4). - let mut wire = Vec::new(); - wire.extend_from_slice(&SKIPPABLE_MAGIC_START.to_le_bytes()); - wire.extend_from_slice(&[0u8, 0, 0]); - let mut cursor: &[u8] = wire.as_slice(); - let err = SkippableFrame::decode_from(&mut cursor).unwrap_err(); - assert!( - matches!(err, DecodeSkippableFrameError::Length(_)), - "expected Length, got {err:?}" - ); - } - - #[test] - fn decode_truncated_payload_surfaces_typed_error() { - // Header claims 16-byte payload but only 4 bytes follow. - // The error must point at the PAYLOAD read step, not get - // misreported as a header / descriptor read. - let mut wire = Vec::new(); - wire.extend_from_slice(&SKIPPABLE_MAGIC_START.to_le_bytes()); - wire.extend_from_slice(&16u32.to_le_bytes()); - wire.extend_from_slice(&[0u8; 4]); - let mut cursor: &[u8] = wire.as_slice(); - let err = SkippableFrame::decode_from(&mut cursor).unwrap_err(); - assert!( - matches!(err, DecodeSkippableFrameError::Payload(_)), - "expected Payload, got {err:?}" - ); - } - - #[test] - fn serialized_size_matches_encoded_length() { - for payload_len in [0usize, 1, 7, 8, 9, 255, 256, 1023, 1024] { - let payload = alloc::vec![0u8; payload_len]; - let frame = SkippableFrame::new(3, payload).unwrap(); - let mut wire = Vec::new(); - frame.encode_into(&mut wire).unwrap(); - assert_eq!( - wire.len(), - frame.serialized_size(), - "serialized_size() must match actual encode length for payload_len={payload_len}" - ); - } - } - - #[test] - fn decode_huge_length_returns_typed_error_not_oom_abort() { - // Crafted wire declares a u32::MAX payload but provides - // zero payload bytes. The decoder must surface a typed - // error rather than aborting the process or panicking. - // Three paths are acceptable, each gated by the host's - // ABI / allocator behaviour: - // - // - `PayloadTooLarge { length }` — 32-bit host, where - // `length + 8` overflows `usize`. The decoder rejects - // the length before allocating. - // - `AllocationFailed { requested }` — 64-bit host, no - // memory overcommit (Windows / configured Linux): - // `try_reserve_exact` reports failure. - // - `Payload(io_err)` — 64-bit host, memory overcommit - // (Linux default / macOS): allocation succeeds for the - // address range, chunked read on truncated stream - // surfaces the I/O error after committing one page - // for the scratch buffer. - // - // What it must NOT do: abort the process on OOM or panic - // via Vec::with_capacity / Vec::resize. - let huge: u32 = u32::MAX; - let mut wire = Vec::new(); - wire.extend_from_slice(&SKIPPABLE_MAGIC_START.to_le_bytes()); - wire.extend_from_slice(&huge.to_le_bytes()); - let mut cursor: &[u8] = wire.as_slice(); - let err = SkippableFrame::decode_from(&mut cursor).unwrap_err(); - match err { - DecodeSkippableFrameError::PayloadTooLarge { length } => { - assert_eq!(length, huge); - } - DecodeSkippableFrameError::AllocationFailed { requested } => { - assert_eq!(requested, huge as usize); - } - DecodeSkippableFrameError::Payload(_) => { - // Chunked read on the truncated payload surfaced - // the I/O error after the OS overcommitted the - // address range. Also acceptable. - } - other => panic!("expected PayloadTooLarge / AllocationFailed / Payload, got {other:?}"), - } - } - - #[test] - fn payload_too_large_check_branches_on_pointer_width() { - // The `validate_payload_size` invariant is twofold: - // - // 1. `len > u32::MAX` is rejected on every target (the - // on-wire length field is u32). - // 2. `len + SKIPPABLE_HEADER_SIZE` overflowing `usize` is - // rejected on every target. On 64-bit this is - // unreachable because `u32::MAX + 8 < usize::MAX`. On - // 32-bit `len == u32::MAX` itself trips condition 2: - // `u32::MAX + 8` wraps `usize`. - // - // Branch the boundary expectation on pointer width so the - // test passes on both i686 (CI cross-i686 shard) and - // x86_64 hosts. - #[cfg(target_pointer_width = "64")] - { - let result = validate_payload_size(u32::MAX as usize + 1); - assert!(matches!( - result, - Err(SkippableFrameError::PayloadTooLarge(_)) - )); - let ok = validate_payload_size(u32::MAX as usize); - assert!(ok.is_ok(), "u32::MAX representable on 64-bit"); - } - - #[cfg(target_pointer_width = "32")] - { - // `u32::MAX + 1` literally cannot be expressed as - // `usize` on 32-bit — `u32::MAX as usize + 1` wraps - // to 0. So construct the test only through values - // that are validly representable. - let result = validate_payload_size(u32::MAX as usize); - assert!( - matches!(result, Err(SkippableFrameError::PayloadTooLarge(_))), - "u32::MAX overflows when combined with the 8-byte header on 32-bit" - ); - let ok = validate_payload_size((u32::MAX as usize) - SKIPPABLE_HEADER_SIZE); - assert!(ok.is_ok(), "below the header-overflow boundary on 32-bit"); - } - } -} +mod tests; diff --git a/zstd/src/skippable/tests.rs b/zstd/src/skippable/tests.rs new file mode 100644 index 000000000..05ee80a25 --- /dev/null +++ b/zstd/src/skippable/tests.rs @@ -0,0 +1,312 @@ +use super::*; + +fn build_expected_skippable(magic_variant: u8, payload: &[u8]) -> Vec { + // Upstream zstd `ZSTD_writeSkippableFrame` (zstd v1.5.7 + // `lib/compress/zstd_compress.c:4751-4763`) emits exactly + // `4-byte LE magic || 4-byte LE size || payload`. Mirror that + // here as the byte-parity oracle. Re-implementing the upstream zstd + // layout in the test (rather than calling out to zstd-sys) + // keeps this test independent of the dev-dep wiring and + // makes the parity expectation visible inline. + let magic = (SKIPPABLE_MAGIC_START + u32::from(magic_variant)).to_le_bytes(); + let size = (payload.len() as u32).to_le_bytes(); + let mut out = Vec::with_capacity(payload.len() + SKIPPABLE_HEADER_SIZE); + out.extend_from_slice(&magic); + out.extend_from_slice(&size); + out.extend_from_slice(payload); + out +} + +#[test] +fn round_trip_all_sixteen_variants() { + for variant in 0u8..=15 { + let payload = alloc::vec![variant; 32 + variant as usize]; + let frame = SkippableFrame::new(variant, payload.clone()).expect("variant in range"); + let mut wire = Vec::new(); + frame + .encode_into(&mut wire) + .expect("encode into Vec succeeds"); + + let mut cursor: &[u8] = wire.as_slice(); + let decoded = SkippableFrame::decode_from(&mut cursor).expect("round-trip decode"); + assert_eq!(decoded.magic_variant(), variant); + assert_eq!( + decoded.magic_number(), + SKIPPABLE_MAGIC_START + u32::from(variant) + ); + assert_eq!(decoded.payload(), payload.as_slice()); + assert!( + cursor.is_empty(), + "decode_from must consume exactly the frame bytes, no overshoot or undershoot" + ); + } +} + +#[test] +fn empty_payload_round_trips() { + let frame = SkippableFrame::new(7, Vec::new()).expect("empty payload OK"); + assert_eq!(frame.serialized_size(), SKIPPABLE_HEADER_SIZE); + + let mut wire = Vec::new(); + frame.encode_into(&mut wire).unwrap(); + assert_eq!(wire.len(), SKIPPABLE_HEADER_SIZE); + + let mut cursor: &[u8] = wire.as_slice(); + let decoded = SkippableFrame::decode_from(&mut cursor).unwrap(); + assert!(decoded.payload().is_empty()); + assert_eq!(decoded.magic_variant(), 7); +} + +#[test] +fn large_payload_round_trips() { + // 1 MiB so the 4-byte length field carries a non-trivial + // value (0x100000) — the byte-parity test below verifies the + // LE serialisation explicitly. + let payload = alloc::vec![0xABu8; 1024 * 1024]; + let frame = SkippableFrame::new(0, payload.clone()).unwrap(); + let mut wire = Vec::new(); + frame.encode_into(&mut wire).unwrap(); + assert_eq!(wire.len(), payload.len() + SKIPPABLE_HEADER_SIZE); + + let mut cursor: &[u8] = wire.as_slice(); + let decoded = SkippableFrame::decode_from(&mut cursor).unwrap(); + assert_eq!(decoded.payload().len(), payload.len()); + assert!(decoded.payload() == payload.as_slice()); +} + +#[test] +fn new_rejects_variant_sixteen() { + let err = SkippableFrame::new(16, Vec::new()).expect_err("variant 16 out of range"); + match err { + SkippableFrameError::InvalidMagicVariant(v) => assert_eq!(v, 16), + other => panic!("expected InvalidMagicVariant(16), got {other:?}"), + } +} + +#[test] +fn new_rejects_variant_max() { + // u8::MAX = 255 — clearly outside the spec's 0..=15 range. + let err = SkippableFrame::new(255, Vec::new()).unwrap_err(); + match err { + SkippableFrameError::InvalidMagicVariant(v) => assert_eq!(v, 255), + other => panic!("expected InvalidMagicVariant(255), got {other:?}"), + } +} + +#[test] +fn write_function_rejects_invalid_variant() { + let mut sink: Vec = Vec::new(); + let err = write_skippable_frame(16, b"x", &mut sink).unwrap_err(); + assert!(matches!(err, SkippableFrameError::InvalidMagicVariant(16))); + assert!( + sink.is_empty(), + "no bytes must be written on rejected input" + ); +} + +#[test] +fn byte_parity_with_spec_layout() { + // For every variant + a handful of payload sizes, our output + // bytes must equal the upstream zstd's `ZSTD_writeSkippableFrame` + // layout byte-for-byte. This locks the wire-format contract + // against future drift. + for &payload_len in &[0usize, 1, 8, 256, 4096] { + let payload: Vec = (0..payload_len).map(|i| (i % 251) as u8).collect(); + for variant in 0u8..=15 { + let expected = build_expected_skippable(variant, &payload); + + let mut via_struct = Vec::new(); + SkippableFrame::new(variant, payload.clone()) + .unwrap() + .encode_into(&mut via_struct) + .unwrap(); + assert_eq!( + via_struct, expected, + "struct encode mismatch: variant={variant} len={payload_len}" + ); + + let mut via_free = Vec::new(); + let written = write_skippable_frame(variant, &payload, &mut via_free).unwrap(); + assert_eq!(written, expected.len()); + assert_eq!( + via_free, expected, + "free-fn encode mismatch: variant={variant} len={payload_len}" + ); + } + } +} + +#[test] +fn decode_rejects_non_skippable_magic() { + // Zstd-1 magic 0xFD2FB528 is NOT in the skippable range. + let mut wire = Vec::new(); + wire.extend_from_slice(&0xFD2F_B528u32.to_le_bytes()); + wire.extend_from_slice(&0u32.to_le_bytes()); + let mut cursor: &[u8] = wire.as_slice(); + let err = SkippableFrame::decode_from(&mut cursor).unwrap_err(); + match err { + DecodeSkippableFrameError::BadMagicNumber(m) => assert_eq!(m, 0xFD2F_B528), + other => panic!("expected BadMagicNumber, got {other:?}"), + } +} + +#[test] +fn decode_rejects_magic_above_band() { + // 0x184D2A60 is one past the skippable band — must be + // rejected via BadMagicNumber, not silently accepted as + // variant 16. + let mut wire = Vec::new(); + wire.extend_from_slice(&0x184D_2A60u32.to_le_bytes()); + wire.extend_from_slice(&0u32.to_le_bytes()); + let mut cursor: &[u8] = wire.as_slice(); + let err = SkippableFrame::decode_from(&mut cursor).unwrap_err(); + assert!(matches!( + err, + DecodeSkippableFrameError::BadMagicNumber(0x184D_2A60) + )); +} + +#[test] +fn decode_truncated_magic_surfaces_typed_error() { + // Three bytes (one less than a magic) — must fail on the + // magic read step, not panic. + let wire = [0x50u8, 0x2A, 0x4D]; + let mut cursor: &[u8] = &wire; + let err = SkippableFrame::decode_from(&mut cursor).unwrap_err(); + assert!( + matches!(err, DecodeSkippableFrameError::Magic(_)), + "expected Magic, got {err:?}" + ); +} + +#[test] +fn decode_truncated_length_surfaces_typed_error() { + // Magic OK, but length field is short (3 bytes instead of 4). + let mut wire = Vec::new(); + wire.extend_from_slice(&SKIPPABLE_MAGIC_START.to_le_bytes()); + wire.extend_from_slice(&[0u8, 0, 0]); + let mut cursor: &[u8] = wire.as_slice(); + let err = SkippableFrame::decode_from(&mut cursor).unwrap_err(); + assert!( + matches!(err, DecodeSkippableFrameError::Length(_)), + "expected Length, got {err:?}" + ); +} + +#[test] +fn decode_truncated_payload_surfaces_typed_error() { + // Header claims 16-byte payload but only 4 bytes follow. + // The error must point at the PAYLOAD read step, not get + // misreported as a header / descriptor read. + let mut wire = Vec::new(); + wire.extend_from_slice(&SKIPPABLE_MAGIC_START.to_le_bytes()); + wire.extend_from_slice(&16u32.to_le_bytes()); + wire.extend_from_slice(&[0u8; 4]); + let mut cursor: &[u8] = wire.as_slice(); + let err = SkippableFrame::decode_from(&mut cursor).unwrap_err(); + assert!( + matches!(err, DecodeSkippableFrameError::Payload(_)), + "expected Payload, got {err:?}" + ); +} + +#[test] +fn serialized_size_matches_encoded_length() { + for payload_len in [0usize, 1, 7, 8, 9, 255, 256, 1023, 1024] { + let payload = alloc::vec![0u8; payload_len]; + let frame = SkippableFrame::new(3, payload).unwrap(); + let mut wire = Vec::new(); + frame.encode_into(&mut wire).unwrap(); + assert_eq!( + wire.len(), + frame.serialized_size(), + "serialized_size() must match actual encode length for payload_len={payload_len}" + ); + } +} + +#[test] +fn decode_huge_length_returns_typed_error_not_oom_abort() { + // Crafted wire declares a u32::MAX payload but provides + // zero payload bytes. The decoder must surface a typed + // error rather than aborting the process or panicking. + // Three paths are acceptable, each gated by the host's + // ABI / allocator behaviour: + // + // - `PayloadTooLarge { length }` — 32-bit host, where + // `length + 8` overflows `usize`. The decoder rejects + // the length before allocating. + // - `AllocationFailed { requested }` — 64-bit host, no + // memory overcommit (Windows / configured Linux): + // `try_reserve_exact` reports failure. + // - `Payload(io_err)` — 64-bit host, memory overcommit + // (Linux default / macOS): allocation succeeds for the + // address range, chunked read on truncated stream + // surfaces the I/O error after committing one page + // for the scratch buffer. + // + // What it must NOT do: abort the process on OOM or panic + // via Vec::with_capacity / Vec::resize. + let huge: u32 = u32::MAX; + let mut wire = Vec::new(); + wire.extend_from_slice(&SKIPPABLE_MAGIC_START.to_le_bytes()); + wire.extend_from_slice(&huge.to_le_bytes()); + let mut cursor: &[u8] = wire.as_slice(); + let err = SkippableFrame::decode_from(&mut cursor).unwrap_err(); + match err { + DecodeSkippableFrameError::PayloadTooLarge { length } => { + assert_eq!(length, huge); + } + DecodeSkippableFrameError::AllocationFailed { requested } => { + assert_eq!(requested, huge as usize); + } + DecodeSkippableFrameError::Payload(_) => { + // Chunked read on the truncated payload surfaced + // the I/O error after the OS overcommitted the + // address range. Also acceptable. + } + other => panic!("expected PayloadTooLarge / AllocationFailed / Payload, got {other:?}"), + } +} + +#[test] +fn payload_too_large_check_branches_on_pointer_width() { + // The `validate_payload_size` invariant is twofold: + // + // 1. `len > u32::MAX` is rejected on every target (the + // on-wire length field is u32). + // 2. `len + SKIPPABLE_HEADER_SIZE` overflowing `usize` is + // rejected on every target. On 64-bit this is + // unreachable because `u32::MAX + 8 < usize::MAX`. On + // 32-bit `len == u32::MAX` itself trips condition 2: + // `u32::MAX + 8` wraps `usize`. + // + // Branch the boundary expectation on pointer width so the + // test passes on both i686 (CI cross-i686 shard) and + // x86_64 hosts. + #[cfg(target_pointer_width = "64")] + { + let result = validate_payload_size(u32::MAX as usize + 1); + assert!(matches!( + result, + Err(SkippableFrameError::PayloadTooLarge(_)) + )); + let ok = validate_payload_size(u32::MAX as usize); + assert!(ok.is_ok(), "u32::MAX representable on 64-bit"); + } + + #[cfg(target_pointer_width = "32")] + { + // `u32::MAX + 1` literally cannot be expressed as + // `usize` on 32-bit — `u32::MAX as usize + 1` wraps + // to 0. So construct the test only through values + // that are validly representable. + let result = validate_payload_size(u32::MAX as usize); + assert!( + matches!(result, Err(SkippableFrameError::PayloadTooLarge(_))), + "u32::MAX overflows when combined with the 8-byte header on 32-bit" + ); + let ok = validate_payload_size((u32::MAX as usize) - SKIPPABLE_HEADER_SIZE); + assert!(ok.is_ok(), "below the header-overflow boundary on 32-bit"); + } +}