perf(huf+lazy): gate HUF table-log on the size-adaptive strategy, unify the lazy parser

ffi-bench/tests/frame_compressor_ffi.rs

@@ -342,3 +342,49 @@ fn hinted_small_compressible_frames_use_single_segment_across_levels() {
         assert_eq!(c_decode(&compressed), data);
     }
 }
+
+/// The bench `small-4k-log-lines` fixture: four rotating log lines tiled to
+/// `len` bytes (byte-identical to the `compare_ffi` scenario).
+fn repeated_log_lines(len: usize) -> Vec<u8> {
+    const LINES: &[&str] = &[
+        "ts=2026-03-26T21:39:28Z level=INFO msg=\"flush memtable\" tenant=demo table=orders region=eu-west\n",
+        "ts=2026-03-26T21:39:29Z level=INFO msg=\"rotate segment\" tenant=demo table=orders region=eu-west\n",
+        "ts=2026-03-26T21:39:30Z level=INFO msg=\"compact level\" tenant=demo table=orders region=eu-west\n",
+        "ts=2026-03-26T21:39:31Z level=INFO msg=\"write block\" tenant=demo table=orders region=eu-west\n",
+    ];
+    let mut bytes = Vec::with_capacity(len);
+    while bytes.len() < len {
+        for line in LINES {
+            if bytes.len() == len {
+                break;
+            }
+            let remaining = len - bytes.len();
+            bytes.extend_from_slice(&line.as_bytes()[..line.len().min(remaining)]);
+        }
+    }
+    bytes
+}
+
+/// Regression: a one-shot frame whose source-size hint places it in a small
+/// cParams tier must run the same parse strategy C resolves, so its literal
+/// section is never larger than C's. For a <=16 KiB frame upstream
+/// `ZSTD_getCParams` promotes levels 13-17 to btultra/btultra2, which enables
+/// the HUF table-log search. The matcher already resolved that strategy, but
+/// the literal-compression / HUF-search gate used to re-derive it from the bare
+/// level (btlazy2/btopt) and skip the search, overshooting C by 4 bytes on the
+/// 4 KiB log fixture. The gate now reads the matcher's resolved strategy.
+#[test]
+fn small_hinted_frames_match_c_literal_section_levels_13_to_17() {
+    let data = repeated_log_lines(4 * 1024);
+    for level in 13..=17 {
+        let ours = compress_hinted(&data, CompressionLevel::Level(level));
+        let c = zstd::bulk::compress(data.as_slice(), level).expect("C compress");
+        assert!(
+            ours.len() <= c.len(),
+            "level {level}: ours {} > C {} (small-tier strategy must match C)",
+            ours.len(),
+            c.len()
+        );
+        assert_eq!(c_decode(&ours), data, "roundtrip level={level}");
+    }
+}

zstd/src/encoding/frame_compressor.rs

@@ -724,25 +724,18 @@ pub(crate) struct CompressState<M: Matcher> {
 
 /// Whether the HUF literal build should run the #167 table-log search for a
 /// frame of `source_size` bytes (see [`CompressState::huf_optimal_search`]).
-/// The Fast and DoubleFast strategies do little matching work, so per-frame
-/// HUF setup dominates a small frame and the search's ~1.5 us is a large
-/// relative cost for a ratio gain the cheap single-build forgoes while still
-/// tying upstream zstd. Gate the search to frames above 128 KiB (one full
-/// block) for those two. Higher strategies do enough matching that the search
-/// is a negligible fraction and always run it for the ratio edge. Unknown size
-/// (streaming) keeps the search for conservative ratio.
+/// Upstream gates the optimal-depth tableLog probe to
+/// `HUF_OPTIMAL_DEPTH_THRESHOLD = ZSTD_btultra` (huf.h:117): only btultra /
+/// btultra2 search the tableLog, every lower strategy (fast .. btopt) takes the
+/// single-shot fast path (`HUF_optimalTableLog`, huf_compress.c:1284-1287).
+/// Mirror that so our literal tableLog choice tracks upstream's instead of
+/// spending the search to beat it on ratio at a speed cost.
 pub(crate) fn huf_search_enabled(
     strategy: crate::encoding::strategy::StrategyTag,
-    source_size: Option<u64>,
+    _source_size: Option<u64>,
 ) -> bool {
     use crate::encoding::strategy::StrategyTag;
-    if !matches!(strategy, StrategyTag::Fast | StrategyTag::Dfast) {
-        return true;
-    }
-    match source_size {
-        Some(size) => size > 128 * 1024,
-        None => true,
-    }
+    matches!(strategy, StrategyTag::BtUltra | StrategyTag::BtUltra2)
 }
 
 impl<M: Matcher> CompressState<M> {
@@ -1042,9 +1035,16 @@ impl<R: Read, W: Write> FrameCompressor<R, W, MatchGeneratorDriver> {
         self.strategy_override = overrides.strategy.map(|s| s.tag());
         // Keep `state.strategy_tag` consistent immediately so the borrowed
         // one-shot eligibility gate (`borrowed_eligible`) and literal gates
-        // are correct even before the next `compress()` re-sync.
+        // are correct even before the next `compress()` re-sync. Resolve it
+        // size-adaptively (same `resolve_level_params` path `prepare_frame`
+        // uses) so a hint already set here yields the same strategy the matcher
+        // will run, not the bare level-only mapping.
         self.state.strategy_tag = self.strategy_override.unwrap_or_else(|| {
-            crate::encoding::strategy::StrategyTag::for_compression_level(self.compression_level)
+            crate::encoding::levels::config::resolve_level_params(
+                self.compression_level,
+                self.source_size_hint,
+            )
+            .strategy_tag
         });
         self.state.huf_optimal_search =
             huf_search_enabled(self.state.strategy_tag, self.source_size_hint);
@@ -1658,10 +1658,22 @@ impl<R: Read, W: Write, M: Matcher> FrameCompressor<R, W, M> {
         // through this same `compress()` entry point, so re-syncing here
         // covers level switches without touching the matcher dispatch.
         // A public-parameter strategy override (#27) wins over the level's
-        // derived tag so the literal-compression gates and dict-attach
-        // cutoff below see the strategy the matcher actually runs.
+        // derived tag so the literal-compression gates and dict-attach cutoff
+        // below see the strategy the matcher actually runs. Otherwise resolve
+        // the strategy SIZE-ADAPTIVELY through the same path the matcher's reset
+        // used (`resolve_level_params` -> `get_cparams`, the port of upstream
+        // `ZSTD_getCParams`): a small frame promotes a level to a higher
+        // strategy (e.g. L13 over a <=16 KiB frame becomes btultra). Re-deriving
+        // from the bare level would make the literal-compression / HUF-search
+        // gates disagree with the matcher's actual parse on small frames (the
+        // gate would think btlazy2 and skip the HUF table-log search the btultra
+        // frame runs, costing a few bytes on small literal sections).
         self.state.strategy_tag = self.strategy_override.unwrap_or_else(|| {
-            crate::encoding::strategy::StrategyTag::for_compression_level(self.compression_level)
+            crate::encoding::levels::config::resolve_level_params(
+                self.compression_level,
+                initial_size_hint,
+            )
+            .strategy_tag
         });
         // `initial_size_hint` (captured before the `.take()` above) — by here
         // `self.source_size_hint` is None.

zstd/src/encoding/hc/generator.rs

@@ -12,7 +12,7 @@ use alloc::vec::Vec;
 use crate::encoding::Sequence;
 use crate::encoding::blocks::encode_offset_with_history;
 use crate::encoding::cost_model::HC_PREDEF_THRESHOLD;
-use crate::encoding::hc::{HC_MIN_MATCH_LEN, MAX_HC_SEARCH_DEPTH};
+use crate::encoding::hc::{HC_MIN_MATCH_LEN, HcMatch, MAX_HC_SEARCH_DEPTH};
 use crate::encoding::levels::config::HcConfig;
 use crate::encoding::match_generator::{HC_SEARCH_DEPTH, HC_TARGET_LEN, HcBackend};
 use crate::encoding::match_table::storage::HC3_HASH_LOG;
@@ -1270,70 +1270,93 @@ impl HcMatchGenerator {
 
         let mut pos = 0usize;
         let mut literals_start = 0usize;
+        // Lookahead carry (upstream's lazy depth loop searches each position
+        // ONCE): when the lookahead defers to `abs_pos + 1`, its already-computed
+        // match is carried here and reused next iteration instead of re-searched.
+        // `None` means search this position fresh.
+        let mut carried: Option<HcMatch> = None;
+        // Set when the previous position deferred WITHOUT a carry (depth-2:
+        // `abs_pos + 2` won but `abs_pos + 1` was cold). If this position then
+        // misses, the no-match skip below must advance by exactly ONE so it
+        // cannot hop over the already-proven `abs_pos + 2` winner.
+        let mut deferred_without_carry = false;
         while pos + HC_MIN_MATCH_LEN <= current_len {
             let abs_pos = current_abs_start + pos;
             let lit_len = pos - literals_start;
+            let forced_single_step = core::mem::take(&mut deferred_without_carry);
 
-            // `find_best_match` returns the forward `(offset, length)` in
-            // registers (`HcMatch`, 16 bytes) — no 24-byte `MatchCandidate` /
-            // 32-byte `Option` spilled-and-copied per position. The backward
-            // extension that yields `start` runs ONCE here, after the lazy
-            // decision settles, exactly like upstream's lazy loop.
-            let best =
-                self.hc
-                    .find_best_match::<DICT>(concat, dms_primed, &self.table, abs_pos, lit_len);
+            // Reuse the carried lookahead match (searched WITH the previous
+            // position already inserted, so its offset-1 candidate is visible)
+            // or search this position fresh. `find_best_match` returns the
+            // forward `(offset, length)` in registers (`HcMatch`, 16 bytes).
+            let best = match carried.take() {
+                Some(m) => m,
+                None => self.hc.find_best_match::<DICT>(
+                    concat,
+                    dms_primed,
+                    &self.table,
+                    abs_pos,
+                    lit_len,
+                ),
+            };
             if best.is_match() {
-                if self.hc.pick_lazy_match::<DICT>(
+                // Insert `abs_pos` BEFORE the lookahead so the `abs_pos + 1`
+                // probe sees it at offset 1 — upstream inserts the searched
+                // position during its own search, before the depth loop probes
+                // the next one.
+                self.table.insert_position(abs_pos);
+                if let Some(carry) = self.hc.lazy_decide_carry::<DICT>(
                     concat,
                     dms_primed,
                     &self.table,
                     abs_pos,
                     lit_len,
                     best,
                 ) {
-                    // Backward-extend over the literal run (upstream `zstd_lazy.c`
-                    // after rep-vs-chain selection). The offset is preserved;
-                    // `start` and `match_len` grow by the same amount, bounded by
-                    // `literals_start` (the `min_abs` floor) so it never crosses
-                    // an already-emitted sequence.
-                    let history_abs_start = self.table.history_abs_start;
-                    let min_abs = abs_pos - lit_len;
-                    let mut start_abs = abs_pos;
-                    let mut cand_abs = abs_pos - best.offset;
-                    let mut match_len = best.match_len;
-                    while start_abs > min_abs
-                        && cand_abs > history_abs_start
-                        && concat[cand_abs - history_abs_start - 1]
-                            == concat[start_abs - history_abs_start - 1]
-                    {
-                        start_abs -= 1;
-                        cand_abs -= 1;
-                        match_len += 1;
-                    }
-                    self.table.insert_match_span(abs_pos, start_abs + match_len);
-                    let start = start_abs - current_abs_start;
-                    let literals = &current[literals_start..start];
-                    handle_sequence(Sequence::Triple {
-                        literals,
-                        offset: best.offset,
-                        match_len,
-                    });
-                    let _ = encode_offset_with_history(
-                        best.offset as u32,
-                        literals.len() as u32,
-                        &mut self.table.offset_hist,
-                    );
-                    pos = start + match_len;
-                    literals_start = pos;
+                    // DEFER: the lazy lookahead found a better match one (or two)
+                    // bytes ahead. Advance exactly ONE byte. Reuse the lookahead
+                    // match when it is real (so it is not re-searched — upstream
+                    // searches each position once); the `NONE` sentinel is the
+                    // depth-2 defer-without-carry case (`abs_pos + 2` won but
+                    // `abs_pos + 1` had no match), so search the next fresh and
+                    // pin the following advance to one byte (below) so the skip
+                    // heuristic cannot hop over that `abs_pos + 2` winner.
+                    carried = carry.is_match().then_some(carry);
+                    deferred_without_carry = carried.is_none();
+                    pos += 1;
                     continue;
                 }
-                // Lazy lookahead found a better match at `abs_pos + 1` / `+ 2`
-                // (defer): advance exactly ONE byte (upstream
-                // `ZSTD_compressBlock_lazy_generic`) so the deferred candidate is
-                // re-evaluated at its own position; the no-match skip below could
-                // jump past it once the literal run reaches 256+ bytes.
-                self.table.insert_position(abs_pos);
-                pos += 1;
+                // COMMIT `best`. Backward-extend over the literal run (upstream
+                // `zstd_lazy.c` after rep-vs-chain selection) through the shared
+                // raw-pointer helper the Row / Dfast probes use — one
+                // back-extension source, no per-step slice bounds checks.
+                // `abs_pos` is already inserted above, so the forward span fill
+                // starts at `abs_pos + 1`.
+                let ext = crate::encoding::match_table::helpers::extend_backwards_shared(
+                    concat,
+                    self.table.history_abs_start,
+                    abs_pos - best.offset,
+                    abs_pos,
+                    best.match_len,
+                    lit_len,
+                );
+                let match_len = ext.match_len;
+                self.table
+                    .insert_match_span(abs_pos + 1, ext.start + match_len);
+                let start = ext.start - current_abs_start;
+                let literals = &current[literals_start..start];
+                handle_sequence(Sequence::Triple {
+                    literals,
+                    offset: best.offset,
+                    match_len,
+                });
+                let _ = encode_offset_with_history(
+                    best.offset as u32,
+                    literals.len() as u32,
+                    &mut self.table.offset_hist,
+                );
+                pos = start + match_len;
+                literals_start = pos;
                 continue;
             }
             // No match found.
@@ -1349,7 +1372,13 @@ impl HcMatchGenerator {
             // every byte. Skipped positions are not inserted, mirroring
             // upstream (it inserts only searched positions during a no-match
             // run). Ratio follows upstream (not byte-identical).
-            let step = ((pos - literals_start) >> 8) + 1;
+            // A miss one byte after a depth-2 defer-without-carry must advance by
+            // exactly one so the skip cannot pass the proven `abs_pos + 2` match.
+            let step = if forced_single_step {
+                1
+            } else {
+                ((pos - literals_start) >> 8) + 1
+            };
             pos += step;
             // No clamp needed before the tail loop: the search bound and the
             // hashable bound are both `pos + HC_MIN_MATCH_LEN <= current_len`