Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 42 additions & 0 deletions zstd/src/encoding/frame_compressor/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2687,3 +2687,45 @@ fn exact_block_multiple_marks_last_real_block() {
}
}
}

#[test]
fn dict_compress_bt_level_tiny_source_round_trips_through_prime_dms_bt() {
// End-to-end cover for the dictionary match binary-tree (`prime_dms_bt`, a
// ZSTD_dictMatchState analog built only on the BT strategies, level >= 13):
// compress a tiny source with a small raw-content dictionary at level 19
// (BtUltra2) and round-trip it. This drives the BT dict-prime + dict-match
// search path that non-BT levels (Fast/Dfast/Lazy) never reach.
//
// The `prime_dms_bt` dms-table sizing previously used
// `ceil_log2(region).clamp(10, hash_log)`, which panicked ("min > max") when
// the matcher's `hash_log` adjusted below the 10 floor — the exact bound
// arithmetic is pinned directly by `storage::dms_hash_log_tests`. On this
// build the window-log floor keeps `hash_log >= 10` so this end-to-end path
// stays above the boundary; the unit test exercises `hash_log < 10`.
let raw_dict: Vec<u8> = (0..100u32)
.map(|i| (i.wrapping_mul(2_654_435_761) >> 24) as u8)
.collect();
let dict_id = 1u32;
let dict_for_encoder =
crate::decoding::Dictionary::from_raw_content(dict_id, raw_dict.clone()).unwrap();
let dict_for_decoder =
crate::decoding::Dictionary::from_raw_content(dict_id, raw_dict).unwrap();

let data = b"hello world".to_vec();

let mut compressor: FrameCompressor =
FrameCompressor::new(super::CompressionLevel::from_level(19));
compressor
.set_dictionary(dict_for_encoder)
.expect("raw-content dictionary should attach");
// Runs the BT dict-prime + dict-match path end to end.
let out = compressor.compress_independent_frame(data.as_slice());

let mut decoder = FrameDecoder::new();
decoder.add_dict(dict_for_decoder).unwrap();
let mut decoded = Vec::with_capacity(data.len());
decoder
.decode_all_to_vec(&out, &mut decoded)
.expect("dict BT-level frame should round-trip");
assert_eq!(decoded, data);
}
25 changes: 23 additions & 2 deletions zstd/src/encoding/match_table/storage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,9 @@ mod bt_pair_index_wrap_tests;
#[cfg(test)]
mod stream_abs_headroom_tests;

#[cfg(test)]
mod dms_hash_log_tests;

/// Knuth-style 3-byte hash multiplier. Upstream zstd parity:
/// `ZSTD_HASH3PRIME` in `lib/compress/zstd_compress_internal.h`. Used
/// by the HC3 short-match side table and by the 3-byte branch of the
Expand Down Expand Up @@ -350,6 +353,23 @@ impl Clone for MatchTable {
}
}

/// Hash-log for a dictionary-match-state table: `ceil_log2(region)` (a table
/// sized to the dict region), with a 10-bit minimum but never wider than the
/// matcher's own `hash_log` (the dms table need not exceed the live table).
///
/// A tiny source makes `ZSTD_adjustCParams` legitimately shrink the matcher's
/// `hash_log` below 10 (a BT strategy, level >= 13, with a dictionary). The
/// minimum is therefore `min(10, hash_log)`, not a fixed 10: when `hash_log`
/// is already below 10 the 10-bit floor would invert the clamp bounds
/// (`min > max`) and panic. Collapsing the floor to `hash_log` keeps the clamp
/// valid and sizes the dms table to the (small) live-table width — matching
/// what upstream's dict cParams `hashLog` yields for the same small dictionary.
pub(crate) fn dms_hash_log(region: usize, hash_log: usize) -> usize {
debug_assert!(region >= 1, "dms_hash_log called with empty region");
let hash_log = hash_log as u32;
(usize::BITS - (region - 1).leading_zeros()).clamp(10.min(hash_log), hash_log) as usize
}

/// Shared scaffold for the two `prime_dms_*` dictionary-match-state builders.
///
/// Both resolve the dict `region` (capped to the live history), bail on an
Expand All @@ -372,9 +392,10 @@ macro_rules! build_dms {
$self.dms.invalidate();
return;
}
// Dict-sized hash log: ceil-log2(region) clamped to [10, hash_log].
// Dict-sized hash log: ceil-log2(region), bounded to the matcher's
// `hash_log`. See [`dms_hash_log`] for the bound rationale.
let dms_hash_log =
(usize::BITS - (region - 1).leading_zeros()).clamp(10, $self.hash_log as u32) as usize;
$crate::encoding::match_table::storage::dms_hash_log(region, $self.hash_log);
// CDict cache: the tables depend only on the dict bytes (re-committed
// identically to the front of history every frame) and the
// (region, mls, hash_log) shape, so a primed same-shape table is valid
Expand Down
37 changes: 37 additions & 0 deletions zstd/src/encoding/match_table/storage/dms_hash_log_tests.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
use super::dms_hash_log;

/// Regression: a tiny source makes `ZSTD_adjustCParams` shrink the BT matcher's
/// `hash_log` below the dms table's nominal 10-bit floor. The sizing must bound
/// that floor by `hash_log` rather than `clamp(10, hash_log)` with `min > max`,
/// which panicked (`min = 10, max = 7`) when priming the dict match binary-tree
/// on a BT strategy (level >= 13) with a dictionary attached.
#[test]
fn dms_hash_log_does_not_panic_when_matcher_hash_log_below_ten() {
// region 128 -> ceil_log2 = 7; matcher hash_log = 7 (the reported case).
let h = dms_hash_log(128, 7);
assert!(
h <= 7,
"dms hash_log must never exceed the matcher hash_log, got {h}"
);
assert!(
h >= 1,
"dms hash_log must size at least one bucket, got {h}"
);

// Smallest sane matcher hash_log: still no panic, table sized to it.
assert_eq!(dms_hash_log(64, 6), 6);
assert_eq!(dms_hash_log(8, 8), 8);
}

/// With `hash_log >= 10` the established behaviour is unchanged: the dms table
/// gets at least 10 bits, grows with `ceil_log2(region)`, and is capped at the
/// matcher's `hash_log`.
#[test]
fn dms_hash_log_keeps_ten_floor_and_hash_log_cap_when_room_allows() {
// Tiny region -> floored at 10.
assert_eq!(dms_hash_log(64, 22), 10);
// ceil_log2(region) wins between the 10 floor and the hash_log cap.
assert_eq!(dms_hash_log(1 << 14, 22), 14);
// Capped at hash_log.
assert_eq!(dms_hash_log(1 << 20, 16), 16);
}
Loading