diff --git a/.github/scripts/run-benchmarks.sh b/.github/scripts/run-benchmarks.sh index 0763c1e9c..854e897a1 100755 --- a/.github/scripts/run-benchmarks.sh +++ b/.github/scripts/run-benchmarks.sh @@ -458,7 +458,7 @@ if not benchmark_results: # and `level_22_btultra2` (the ALERT_LEVELS) ever fire # github-action-benchmark regressions. Falling back to # unfiltered `timings` here would reintroduce non-canonical - # shard siblings (e.g. level_2_dfast on the `dfast` shard) and + # shard siblings (e.g. level_4_dfast on the `fast-dfast` shard) and # they'd trip alerts they were explicitly excluded from. benchmark_results = [ { diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 55f0c8fdc..5b2f4d4ae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -413,36 +413,24 @@ jobs: "levels": "level_-7_fast,level_-6_fast,level_-5_fast,level_-4_fast,level_-3_fast" }, { - "id": "fast-pos", - "levels": "level_-2_fast,level_-1_fast,level_1_fast" - }, - { - "id": "dfast", - "levels": "level_2_dfast,level_3_dfast" - }, - { - "id": "greedy", - "levels": "level_4_greedy" + "id": "fast-dfast", + "levels": "level_-2_fast,level_-1_fast,level_1_fast,level_2_fast,level_3_dfast,level_4_dfast,level_5_greedy" }, { "id": "lazy-lower", - "levels": "level_5_lazy,level_6_lazy,level_7_lazy,level_8_lazy,level_9_lazy" + "levels": "level_6_lazy,level_7_lazy,level_8_lazy,level_9_lazy,level_10_lazy" }, { "id": "lazy-upper", - "levels": "level_10_lazy,level_11_lazy,level_12_lazy,level_13_lazy,level_14_lazy,level_15_lazy" + "levels": "level_11_lazy,level_12_lazy,level_13_lazy,level_14_lazy,level_15_lazy" }, { "id": "btopt", "levels": "level_16_btopt,level_17_btopt" }, - { - "id": "btultra", - "levels": "level_18_btultra,level_19_btultra" - }, { "id": "btultra2", - "levels": "level_20_btultra2,level_21_btultra2,level_22_btultra2" + "levels": "level_18_btultra,level_19_btultra2,level_20_btultra2,level_21_btultra2,level_22_btultra2" } ] EOF diff --git a/zstd/benches/support/mod.rs b/zstd/benches/support/mod.rs index 481fb032d..fdfcd6067 100644 --- a/zstd/benches/support/mod.rs +++ b/zstd/benches/support/mod.rs @@ -147,12 +147,16 @@ fn strategy_suffix(level: i32) -> &'static str { reaches this helper. Resolve to the canonical numeric level \ before calling." ), - 1 => "fast", - 2 | 3 => "dfast", - 4 => "greedy", - 5..=15 => "lazy", + // Names mirror the encoder's actual per-level strategy + // (`StrategyTag::for_level`, itself a port of donor `clevels.h` + // for the srcSize > 256 KiB tier): 1-2 fast, 3-4 dfast, 5 greedy, + // 6-15 lazy, 16-17 btopt, 18 btultra, 19-22 btultra2. + 1 | 2 => "fast", + 3 | 4 => "dfast", + 5 => "greedy", + 6..=15 => "lazy", 16 | 17 => "btopt", - 18 | 19 => "btultra", + 18 => "btultra", _ => "btultra2", } } diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs index 3d962f866..796818753 100644 --- a/zstd/src/encoding/match_generator.rs +++ b/zstd/src/encoding/match_generator.rs @@ -174,20 +174,6 @@ const HC_CONFIG: HcConfig = HcConfig { target_len: HC_TARGET_LEN, }; -const BTOPT_HC_CONFIG: HcConfig = HcConfig { - hash_log: 23, - chain_log: 22, - search_depth: 32, - target_len: 256, -}; - -const BTULTRA_HC_CONFIG: HcConfig = HcConfig { - hash_log: 23, - chain_log: 23, - search_depth: 32, - target_len: 256, -}; - const BTULTRA2_HC_CONFIG: HcConfig = HcConfig { hash_log: 24, chain_log: 24, @@ -280,7 +266,8 @@ fn row_hash_bits_for_window(max_window_size: usize) -> usize { /// /// Each entry maps a zstd compression level to the best-available matcher /// backend and tuning knobs. High levels map to dedicated parse modes: -/// btopt (16-17), btultra (18-19), btultra2 (20-22). +/// btopt (16-17), btultra (18), btultra2 (19-22) — matching donor +/// `clevels.h` (level 19 is `ZSTD_btultra2`, not plain btultra). /// /// Index 0 = level 1, index 21 = level 22. #[rustfmt::skip] @@ -317,11 +304,11 @@ const LEVEL_TABLE: [LevelParams; 22] = [ /*13 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 22, chain_log: 22, search_depth: 16, target_len: 32 }, row: ROW_CONFIG }, /*14 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 23, chain_log: 22, search_depth: 32, target_len: 32 }, row: ROW_CONFIG }, /*15 */ LevelParams { strategy_tag: super::strategy::StrategyTag::Lazy, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 23, chain_log: 23, search_depth: 64, target_len: 32 }, row: ROW_CONFIG }, - /*16 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtOpt, window_log: 26, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: BTOPT_HC_CONFIG, row: ROW_CONFIG }, - /*17 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtOpt, window_log: 26, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: BTOPT_HC_CONFIG, row: ROW_CONFIG }, - /*18 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra, window_log: 26, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: BTULTRA_HC_CONFIG, row: ROW_CONFIG }, - /*19 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra, window_log: 26, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: BTULTRA_HC_CONFIG, row: ROW_CONFIG }, - /*20 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, window_log: 26, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: BTULTRA2_HC_CONFIG, row: ROW_CONFIG }, + /*16 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtOpt, window_log: 22, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 22, chain_log: 22, search_depth: 32, target_len: 48 }, row: ROW_CONFIG }, + /*17 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtOpt, window_log: 23, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 22, chain_log: 23, search_depth: 32, target_len: 64 }, row: ROW_CONFIG }, + /*18 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra, window_log: 23, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 22, chain_log: 23, search_depth: 64, target_len: 64 }, row: ROW_CONFIG }, + /*19 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, window_log: 23, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 22, chain_log: 24, search_depth: 128, target_len: 256 }, row: ROW_CONFIG }, + /*20 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, window_log: 25, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: HcConfig { hash_log: 23, chain_log: 25, search_depth: 128, target_len: 256 }, row: ROW_CONFIG }, /*21 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, window_log: 26, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: BTULTRA2_HC_CONFIG, row: ROW_CONFIG }, /*22 */ LevelParams { strategy_tag: super::strategy::StrategyTag::BtUltra2, window_log: 27, fast_hash_log: 14, fast_mls: 7, fast_step_size: 2, lazy_depth: 2, hc: BTULTRA2_HC_CONFIG_L22, row: ROW_CONFIG }, ]; @@ -2998,13 +2985,12 @@ impl HcMatchGenerator { &self, current_len: usize, ) -> bool { - // Donor `initStats_ultra` (the seed pass) is the BtUltra2-only - // first-pass dynamic stats refinement. With `S` plumbed in, - // every non-BtUltra2 monomorphisation drops both this call - // and the seed-pass body at codegen time — the const below - // resolves to `false` for Fast / Dfast / Greedy / Lazy / - // BtOpt / BtUltra and short-circuits the entire predicate. - if !(S::OPT_LEVEL == 2 && S::USE_HASH3) { + // The in-block two-pass dynamic-stats seed (`initStats_ultra`) + // is btultra2-only. `TWO_PASS_SEED` is `false` for every other + // strategy — including btultra, which now shares the hash3 + // short-match probe but stays single-pass — so the seed call and + // its body drop at codegen time for all non-btultra2 kernels. + if !S::TWO_PASS_SEED { return false; } let HcBackend::Bt(bt) = &self.backend else { @@ -3048,7 +3034,12 @@ impl HcMatchGenerator { tag, StrategyTag::BtOpt | StrategyTag::BtUltra | StrategyTag::BtUltra2 ); - let next_hash3_log = if is_btultra2 { + // btultra and btultra2 both run the mls=3 hash3 short-match probe + // (clevels.h minMatch 3). The `is_btultra2` flag below stays + // exclusive to btultra2 because it tweaks the BT rebase boundary, + // not match finding. + let wants_hash3 = matches!(tag, StrategyTag::BtUltra | StrategyTag::BtUltra2); + let next_hash3_log = if wants_hash3 { HC3_HASH_LOG.min(window_log as usize) } else { 0 @@ -4622,14 +4613,18 @@ fn level_16_17_map_to_btopt_strategy() { } #[test] -fn level_18_19_map_to_btultra_strategy() { +fn level_18_maps_to_btultra_level_19_to_btultra2_strategy() { use super::strategy::{BackendTag, StrategyTag}; + // Donor `clevels.h` (srcSize > 256 KiB tier): level 18 = `ZSTD_btultra`, + // level 19 = `ZSTD_btultra2`. Level 19 was previously mapped to plain + // btultra, which under-searched (searchLog 6 vs 7) and lost ~3.7% ratio + // on the repo corpus. let p18 = resolve_level_params(CompressionLevel::Level(18), None); let p19 = resolve_level_params(CompressionLevel::Level(19), None); assert_eq!(p18.backend(), BackendTag::HashChain); assert_eq!(p19.backend(), BackendTag::HashChain); assert_eq!(StrategyTag::for_level(18), StrategyTag::BtUltra); - assert_eq!(StrategyTag::for_level(19), StrategyTag::BtUltra); + assert_eq!(StrategyTag::for_level(19), StrategyTag::BtUltra2); } #[test] @@ -4652,6 +4647,33 @@ fn level22_uses_donor_target_length_and_large_input_tables() { assert_eq!(params.hc.target_len, 999); } +#[test] +fn bt_levels_16_to_21_pin_clevels_params() { + // Pins the BT-level (window_log, hash_log, chain_log, search_depth, + // target_len) tuples so the clevels.h alignment cannot silently drift. + // Levels 16-20 mirror upstream `clevels.h` (srcSize > 256 KiB tier, + // search_depth = 1 << searchLog); level 21 intentionally keeps a deeper + // search_depth (512 vs upstream's 128) — it beats C on ratio there and + // the deeper walk is a deliberate ratio-positive divergence. + let expected = [ + // (level, window_log, hash_log, chain_log, search_depth, target_len) + (16u8, 22u8, 22usize, 22usize, 32usize, 48usize), + (17, 23, 22, 23, 32, 64), + (18, 23, 22, 23, 64, 64), + (19, 23, 22, 24, 128, 256), + (20, 25, 23, 25, 128, 256), + (21, 26, 24, 24, 512, 256), + ]; + for (level, wlog, hlog, clog, sd, tl) in expected { + let p = resolve_level_params(CompressionLevel::Level(level as i32), None); + assert_eq!(p.window_log, wlog, "level {level} window_log"); + assert_eq!(p.hc.hash_log, hlog, "level {level} hash_log"); + assert_eq!(p.hc.chain_log, clog, "level {level} chain_log"); + assert_eq!(p.hc.search_depth, sd, "level {level} search_depth"); + assert_eq!(p.hc.target_len, tl, "level {level} target_len"); + } +} + #[test] fn level22_source_size_hint_uses_donor_btultra2_tiers() { let p16k = resolve_level_params(CompressionLevel::Level(22), Some(16 * 1024)); @@ -4753,8 +4775,8 @@ fn btultra2_profile_disables_small_offset_handicap() { fn btultra_profile_keeps_donor_search_depth_budget() { let p = HcOptimalCostProfile::const_for_strategy::(); assert_eq!( - p.max_chain_depth, 32, - "btultra should not cap chain depth below donor opt2 search budget" + p.max_chain_depth, 64, + "btultra chain-depth budget must match clevels.h level 18 searchLog 6 (1 << 6 = 64)" ); } diff --git a/zstd/src/encoding/strategy.rs b/zstd/src/encoding/strategy.rs index b2dd97fc3..61f611dd9 100644 --- a/zstd/src/encoding/strategy.rs +++ b/zstd/src/encoding/strategy.rs @@ -61,7 +61,13 @@ pub(crate) trait Strategy: Copy + 'static { /// Match-finder backend this strategy runs on. const BACKEND: BackendTag; - /// Minimum match length the parser will produce. + /// Nominal minimum match length for this strategy, mirroring the + /// upstream `clevels.h` `searchLength` column (3 for btultra/btultra2, + /// where the mls=3 hash3 probe is active). Descriptive: the optimal + /// parser's actual acceptance floor is the crate-wide + /// `HC_OPT_MIN_MATCH_LEN` (= 3) and the row matcher uses + /// `ROW_MIN_MATCH_LEN`; this const documents the strategy's intent and + /// is not read on the hot path. const MIN_MATCH: usize; /// `accurate` flag for [`crate::encoding::cost_model::HcOptimalCostProfile`] @@ -73,11 +79,18 @@ pub(crate) trait Strategy: Copy + 'static { /// favour decompression speed; disabled for BtUltra / BtUltra2. const FAVOR_SMALL_OFFSETS: bool; - /// Compile-time gate for the donor `static (mls==3)` short-match - /// probe inside `ZSTD_insertBtAndGetAllMatches`. Only BtUltra2 - /// drives the hash3 table today. + /// Compile-time gate for the `static (mls==3)` short-match probe. + /// Both btultra and btultra2 search the hash3 table (their + /// `clevels.h` minMatch is 3); btopt does not (minMatch >= 4). const USE_HASH3: bool; + /// Whether the optimal parser runs the in-block two-pass dynamic + /// statistics seed (upstream `ZSTD_initStats_ultra`). Only btultra2 + /// does this; btultra is single-pass even though it now shares the + /// hash3 short-match probe. Defaults to `false` so every other + /// strategy drops the seed-pass body at codegen time. + const TWO_PASS_SEED: bool = false; + /// Whether the optimal parser walks the BT — `false` for Lazy2, /// `true` for BtOpt / BtUltra / BtUltra2. const USE_BT: bool; @@ -202,34 +215,42 @@ impl Strategy for BtOpt { const SUFFICIENT_MATCH_LEN: usize = usize::MAX; } -/// Levels 18-19 — donor `ZSTD_btultra`. BT + opt with refined price -/// tables and no small-offset bias. +/// Level 18 — upstream `ZSTD_btultra`. BT + opt with refined price +/// tables and no small-offset bias; shares the mls=3 hash3 short-match +/// probe but stays single-pass (no two-pass dynamic-stats seed). #[derive(Copy, Clone, Debug, Default)] pub(crate) struct BtUltra; impl Strategy for BtUltra { const BACKEND: BackendTag = BackendTag::HashChain; - const MIN_MATCH: usize = 4; + const MIN_MATCH: usize = 3; const ACCURATE_PRICE: bool = true; const FAVOR_SMALL_OFFSETS: bool = false; - const USE_HASH3: bool = false; + // clevels.h level 18 minMatch = 3 → the hash3 short-match probe is + // active, same as btultra2. The two-pass seed stays btultra2-only + // (TWO_PASS_SEED defaults to false here). + const USE_HASH3: bool = true; const USE_BT: bool = true; const OPT_LEVEL: u8 = 2; - const MAX_CHAIN_DEPTH: usize = 32; + // 1 << searchLog for level 18 (clevels.h searchLog = 6). The BT walk + // caps compares at min(MAX_CHAIN_DEPTH, hc.search_depth); both must be + // >= 64 for level 18 to reach upstream search depth. + const MAX_CHAIN_DEPTH: usize = 64; const SUFFICIENT_MATCH_LEN: usize = usize::MAX; } -/// Levels 20-22 — donor `ZSTD_btultra2`. BT + opt with the two-pass +/// Levels 19-22 — upstream `ZSTD_btultra2`. BT + opt with the two-pass /// dynamic-statistics seed and the hash3 short-match table. #[derive(Copy, Clone, Debug, Default)] pub(crate) struct BtUltra2; impl Strategy for BtUltra2 { const BACKEND: BackendTag = BackendTag::HashChain; - const MIN_MATCH: usize = 4; + const MIN_MATCH: usize = 3; const ACCURATE_PRICE: bool = true; const FAVOR_SMALL_OFFSETS: bool = false; const USE_HASH3: bool = true; + const TWO_PASS_SEED: bool = true; const USE_BT: bool = true; const OPT_LEVEL: u8 = 2; const MAX_CHAIN_DEPTH: usize = 512; @@ -267,8 +288,8 @@ impl StrategyTag { /// 13-15=btlazy2; we collapse all three onto our `Lazy` tag and /// carry the lazy_depth variance via `LevelParams.lazy_depth`) /// * 16-17 → `BtOpt` - /// * 18-19 → `BtUltra` - /// * 20-22 → `BtUltra2` + /// * 18 → `BtUltra` + /// * 19-22 → `BtUltra2` pub(crate) const fn for_level(level: u8) -> Self { match level { 1 | 2 => Self::Fast, @@ -276,7 +297,8 @@ impl StrategyTag { 5 => Self::Greedy, 6..=15 => Self::Lazy, 16 | 17 => Self::BtOpt, - 18 | 19 => Self::BtUltra, + 18 => Self::BtUltra, + 19 => Self::BtUltra2, _ => Self::BtUltra2, } } @@ -381,7 +403,9 @@ mod tests { assert_eq!(StrategyTag::for_level(16), StrategyTag::BtOpt); assert_eq!(StrategyTag::for_level(17), StrategyTag::BtOpt); assert_eq!(StrategyTag::for_level(18), StrategyTag::BtUltra); - assert_eq!(StrategyTag::for_level(19), StrategyTag::BtUltra); + // Donor `clevels.h` level 19 uses `ZSTD_btultra2` (searchLog 7, + // two-pass dynamic stats + hash3), not plain btultra. + assert_eq!(StrategyTag::for_level(19), StrategyTag::BtUltra2); assert_eq!(StrategyTag::for_level(20), StrategyTag::BtUltra2); assert_eq!(StrategyTag::for_level(22), StrategyTag::BtUltra2); } @@ -405,16 +429,20 @@ mod tests { assert!(BtUltra2::USE_BT); }; - // `use_hash3_only_set_for_btultra2`: hash3 is exclusively a - // BtUltra2 feature (donor parity). + // hash3 short-match probe: active for btultra + btultra2 (clevels.h + // minMatch 3); btopt and below do not search it. The in-block + // two-pass dynamic-stats seed is btultra2-only. const _USE_HASH3_LAYOUT: () = { assert!(!Fast::USE_HASH3); assert!(!Dfast::USE_HASH3); assert!(!Greedy::USE_HASH3); assert!(!Lazy::USE_HASH3); assert!(!BtOpt::USE_HASH3); - assert!(!BtUltra::USE_HASH3); + assert!(BtUltra::USE_HASH3); assert!(BtUltra2::USE_HASH3); + assert!(!BtOpt::TWO_PASS_SEED); + assert!(!BtUltra::TWO_PASS_SEED); + assert!(BtUltra2::TWO_PASS_SEED); }; // Mirror the per-strategy fields the optimal-parser cost profile @@ -427,7 +455,8 @@ mod tests { assert!(BtUltra::ACCURATE_PRICE && !BtUltra::FAVOR_SMALL_OFFSETS); assert!(BtUltra2::ACCURATE_PRICE && !BtUltra2::FAVOR_SMALL_OFFSETS); assert!(BtOpt::MAX_CHAIN_DEPTH == 32); - assert!(BtUltra::MAX_CHAIN_DEPTH == 32); + // 1 << searchLog for clevels.h level 18 (searchLog = 6). + assert!(BtUltra::MAX_CHAIN_DEPTH == 64); assert!(BtUltra2::MAX_CHAIN_DEPTH == 512); assert!(BtOpt::SUFFICIENT_MATCH_LEN == usize::MAX); assert!(BtUltra::SUFFICIENT_MATCH_LEN == usize::MAX);