diff --git a/zstd/src/encoding/frame_compressor/tests.rs b/zstd/src/encoding/frame_compressor/tests.rs index b4d4c2298..faa6d0f9b 100644 --- a/zstd/src/encoding/frame_compressor/tests.rs +++ b/zstd/src/encoding/frame_compressor/tests.rs @@ -2687,3 +2687,45 @@ fn exact_block_multiple_marks_last_real_block() { } } } + +#[test] +fn dict_compress_bt_level_tiny_source_round_trips_through_prime_dms_bt() { + // End-to-end cover for the dictionary match binary-tree (`prime_dms_bt`, a + // ZSTD_dictMatchState analog built only on the BT strategies, level >= 13): + // compress a tiny source with a small raw-content dictionary at level 19 + // (BtUltra2) and round-trip it. This drives the BT dict-prime + dict-match + // search path that non-BT levels (Fast/Dfast/Lazy) never reach. + // + // The `prime_dms_bt` dms-table sizing previously used + // `ceil_log2(region).clamp(10, hash_log)`, which panicked ("min > max") when + // the matcher's `hash_log` adjusted below the 10 floor — the exact bound + // arithmetic is pinned directly by `storage::dms_hash_log_tests`. On this + // build the window-log floor keeps `hash_log >= 10` so this end-to-end path + // stays above the boundary; the unit test exercises `hash_log < 10`. + let raw_dict: Vec = (0..100u32) + .map(|i| (i.wrapping_mul(2_654_435_761) >> 24) as u8) + .collect(); + let dict_id = 1u32; + let dict_for_encoder = + crate::decoding::Dictionary::from_raw_content(dict_id, raw_dict.clone()).unwrap(); + let dict_for_decoder = + crate::decoding::Dictionary::from_raw_content(dict_id, raw_dict).unwrap(); + + let data = b"hello world".to_vec(); + + let mut compressor: FrameCompressor = + FrameCompressor::new(super::CompressionLevel::from_level(19)); + compressor + .set_dictionary(dict_for_encoder) + .expect("raw-content dictionary should attach"); + // Runs the BT dict-prime + dict-match path end to end. + let out = compressor.compress_independent_frame(data.as_slice()); + + let mut decoder = FrameDecoder::new(); + decoder.add_dict(dict_for_decoder).unwrap(); + let mut decoded = Vec::with_capacity(data.len()); + decoder + .decode_all_to_vec(&out, &mut decoded) + .expect("dict BT-level frame should round-trip"); + assert_eq!(decoded, data); +} diff --git a/zstd/src/encoding/match_table/storage.rs b/zstd/src/encoding/match_table/storage.rs index 208345aed..9b0ebfb59 100644 --- a/zstd/src/encoding/match_table/storage.rs +++ b/zstd/src/encoding/match_table/storage.rs @@ -97,6 +97,9 @@ mod bt_pair_index_wrap_tests; #[cfg(test)] mod stream_abs_headroom_tests; +#[cfg(test)] +mod dms_hash_log_tests; + /// Knuth-style 3-byte hash multiplier. Upstream zstd parity: /// `ZSTD_HASH3PRIME` in `lib/compress/zstd_compress_internal.h`. Used /// by the HC3 short-match side table and by the 3-byte branch of the @@ -350,6 +353,23 @@ impl Clone for MatchTable { } } +/// Hash-log for a dictionary-match-state table: `ceil_log2(region)` (a table +/// sized to the dict region), with a 10-bit minimum but never wider than the +/// matcher's own `hash_log` (the dms table need not exceed the live table). +/// +/// A tiny source makes `ZSTD_adjustCParams` legitimately shrink the matcher's +/// `hash_log` below 10 (a BT strategy, level >= 13, with a dictionary). The +/// minimum is therefore `min(10, hash_log)`, not a fixed 10: when `hash_log` +/// is already below 10 the 10-bit floor would invert the clamp bounds +/// (`min > max`) and panic. Collapsing the floor to `hash_log` keeps the clamp +/// valid and sizes the dms table to the (small) live-table width — matching +/// what upstream's dict cParams `hashLog` yields for the same small dictionary. +pub(crate) fn dms_hash_log(region: usize, hash_log: usize) -> usize { + debug_assert!(region >= 1, "dms_hash_log called with empty region"); + let hash_log = hash_log as u32; + (usize::BITS - (region - 1).leading_zeros()).clamp(10.min(hash_log), hash_log) as usize +} + /// Shared scaffold for the two `prime_dms_*` dictionary-match-state builders. /// /// Both resolve the dict `region` (capped to the live history), bail on an @@ -372,9 +392,10 @@ macro_rules! build_dms { $self.dms.invalidate(); return; } - // Dict-sized hash log: ceil-log2(region) clamped to [10, hash_log]. + // Dict-sized hash log: ceil-log2(region), bounded to the matcher's + // `hash_log`. See [`dms_hash_log`] for the bound rationale. let dms_hash_log = - (usize::BITS - (region - 1).leading_zeros()).clamp(10, $self.hash_log as u32) as usize; + $crate::encoding::match_table::storage::dms_hash_log(region, $self.hash_log); // CDict cache: the tables depend only on the dict bytes (re-committed // identically to the front of history every frame) and the // (region, mls, hash_log) shape, so a primed same-shape table is valid diff --git a/zstd/src/encoding/match_table/storage/dms_hash_log_tests.rs b/zstd/src/encoding/match_table/storage/dms_hash_log_tests.rs new file mode 100644 index 000000000..e0e922f8d --- /dev/null +++ b/zstd/src/encoding/match_table/storage/dms_hash_log_tests.rs @@ -0,0 +1,37 @@ +use super::dms_hash_log; + +/// Regression: a tiny source makes `ZSTD_adjustCParams` shrink the BT matcher's +/// `hash_log` below the dms table's nominal 10-bit floor. The sizing must bound +/// that floor by `hash_log` rather than `clamp(10, hash_log)` with `min > max`, +/// which panicked (`min = 10, max = 7`) when priming the dict match binary-tree +/// on a BT strategy (level >= 13) with a dictionary attached. +#[test] +fn dms_hash_log_does_not_panic_when_matcher_hash_log_below_ten() { + // region 128 -> ceil_log2 = 7; matcher hash_log = 7 (the reported case). + let h = dms_hash_log(128, 7); + assert!( + h <= 7, + "dms hash_log must never exceed the matcher hash_log, got {h}" + ); + assert!( + h >= 1, + "dms hash_log must size at least one bucket, got {h}" + ); + + // Smallest sane matcher hash_log: still no panic, table sized to it. + assert_eq!(dms_hash_log(64, 6), 6); + assert_eq!(dms_hash_log(8, 8), 8); +} + +/// With `hash_log >= 10` the established behaviour is unchanged: the dms table +/// gets at least 10 bits, grows with `ceil_log2(region)`, and is capped at the +/// matcher's `hash_log`. +#[test] +fn dms_hash_log_keeps_ten_floor_and_hash_log_cap_when_room_allows() { + // Tiny region -> floored at 10. + assert_eq!(dms_hash_log(64, 22), 10); + // ceil_log2(region) wins between the 10 floor and the hash_log cap. + assert_eq!(dms_hash_log(1 << 14, 22), 14); + // Capped at hash_log. + assert_eq!(dms_hash_log(1 << 20, 16), 16); +}