From f9c0649044580bbcaedae4bd8f91310aac184e7e Mon Sep 17 00:00:00 2001
From: Dmitry Prudnikov <mail@polaz.com>
Date: Sat, 13 Jun 2026 04:33:15 +0300
Subject: [PATCH 01/12] feat(c-api): complete the stable ZSTDLIB_API symbol
 surface

The last three stable symbols land: the deprecated
ZSTD_getDecompressedSize alias (getFrameContentSize with the
unknown/error sentinels collapsed to 0, the pre-1.3 contract) and the
ZSTD_sizeof_CStream / ZSTD_sizeof_DStream context aliases. The CI
symbol gate covers them; with this the cdylib exports every stable
zstd.h / zdict.h entry point except multi-threaded compression and
legacy-format decoding.

Part of #127
---
 .github/workflows/ci.yml |  1 +
 c-api/src/simple.rs      | 18 ++++++++++++++++++
 c-api/src/streaming.rs   | 20 ++++++++++++++++++++
 c-api/src/tests.rs       | 33 +++++++++++++++++++++++++++++++--
 4 files changed, 70 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c76691a74..222f2b9b6 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -238,6 +238,7 @@ jobs:
             ZSTD_estimateCCtxSize ZSTD_estimateCCtxSize_usingCParams \
             ZSTD_estimateCStreamSize_usingCParams \
             ZSTD_estimateDCtxSize ZSTD_estimateCStreamSize ZSTD_estimateDStreamSize \
+            ZSTD_getDecompressedSize ZSTD_sizeof_CStream ZSTD_sizeof_DStream \
             ZDICT_trainFromBuffer_fastCover ZDICT_optimizeTrainFromBuffer_fastCover; do
             if ! grep -qx "$sym" <<<"$exported"; then
               echo "::error::symbol $sym not exported from $so"
diff --git a/c-api/src/simple.rs b/c-api/src/simple.rs
index 15852ea1c..b0858c306 100644
--- a/c-api/src/simple.rs
+++ b/c-api/src/simple.rs
@@ -168,6 +168,24 @@ pub unsafe extern "C" fn ZSTD_getFrameContentSize(src: *const u8, src_size: usiz
     }
 }
 
+/// `unsigned long long ZSTD_getDecompressedSize(const void* src, size_t
+/// srcSize)` — DEPRECATED upstream alias of [`ZSTD_getFrameContentSize`]
+/// that collapses both `CONTENTSIZE_UNKNOWN` and `CONTENTSIZE_ERROR` to
+/// `0` (the pre-1.3 contract; kept because the symbol is still part of the
+/// stable `ZSTDLIB_API` surface).
+///
+/// # Safety
+/// `src` must be valid for `src_size` bytes (or `NULL` with `src_size == 0`).
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn ZSTD_getDecompressedSize(src: *const u8, src_size: usize) -> u64 {
+    let size = unsafe { ZSTD_getFrameContentSize(src, src_size) };
+    if size == CONTENTSIZE_UNKNOWN || size == CONTENTSIZE_ERROR {
+        0
+    } else {
+        size
+    }
+}
+
 /// `size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)`.
 ///
 /// Returns the on-disk size of the first frame in `src` (so a caller can step
diff --git a/c-api/src/streaming.rs b/c-api/src/streaming.rs
index 66deb238a..e97834298 100644
--- a/c-api/src/streaming.rs
+++ b/c-api/src/streaming.rs
@@ -182,6 +182,16 @@ pub unsafe extern "C" fn ZSTD_freeCStream(zcs: *mut ZSTD_CCtx) -> usize {
     unsafe { crate::context::ZSTD_freeCCtx(zcs) }
 }
 
+/// `size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)` — `ZSTD_CStream`
+/// is the same object as a `ZSTD_CCtx`, so this is `ZSTD_sizeof_CCtx`.
+///
+/// # Safety
+/// Same as [`ZSTD_freeCStream`].
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn ZSTD_sizeof_CStream(zcs: *const ZSTD_CCtx) -> usize {
+    unsafe { crate::context::ZSTD_sizeof_CCtx(zcs) }
+}
+
 /// `size_t ZSTD_CStreamInSize(void)` — recommended input granule
 /// (`ZSTD_BLOCKSIZE_MAX`).
 #[unsafe(no_mangle)]
@@ -450,6 +460,16 @@ pub unsafe extern "C" fn ZSTD_freeDStream(zds: *mut ZSTD_DCtx) -> usize {
     unsafe { crate::context::ZSTD_freeDCtx(zds) }
 }
 
+/// `size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds)` — `ZSTD_DStream`
+/// is the same object as a `ZSTD_DCtx`, so this is `ZSTD_sizeof_DCtx`.
+///
+/// # Safety
+/// Same as [`ZSTD_freeDStream`].
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn ZSTD_sizeof_DStream(zds: *const ZSTD_DCtx) -> usize {
+    unsafe { crate::context::ZSTD_sizeof_DCtx(zds) }
+}
+
 /// `size_t ZSTD_DStreamInSize(void)` — recommended input granule
 /// (`ZSTD_BLOCKSIZE_MAX + blockHeader(3)`).
 #[unsafe(no_mangle)]
diff --git a/c-api/src/tests.rs b/c-api/src/tests.rs
index cf8fa1416..9e425b887 100644
--- a/c-api/src/tests.rs
+++ b/c-api/src/tests.rs
@@ -20,8 +20,8 @@ use crate::frame::{
 };
 use crate::simple::{
     ZSTD_compress, ZSTD_compressBound, ZSTD_decompress, ZSTD_defaultCLevel,
-    ZSTD_findFrameCompressedSize, ZSTD_getFrameContentSize, ZSTD_maxCLevel, ZSTD_minCLevel,
-    ZSTD_versionNumber,
+    ZSTD_findFrameCompressedSize, ZSTD_getDecompressedSize, ZSTD_getFrameContentSize,
+    ZSTD_maxCLevel, ZSTD_minCLevel, ZSTD_versionNumber,
 };
 
 fn sample(len: usize) -> Vec<u8> {
@@ -50,6 +50,15 @@ fn simple_roundtrips_one_mib() {
 
     let declared = unsafe { ZSTD_getFrameContentSize(compressed.as_ptr(), csize) };
     assert_eq!(declared, input.len() as u64);
+    // The deprecated alias agrees on known sizes and collapses the
+    // unknown/error sentinels to 0 (garbage header → 0, not a sentinel).
+    let legacy = unsafe { ZSTD_getDecompressedSize(compressed.as_ptr(), csize) };
+    assert_eq!(legacy, input.len() as u64);
+    let garbage = [0u8; 3];
+    assert_eq!(
+        unsafe { ZSTD_getDecompressedSize(garbage.as_ptr(), garbage.len()) },
+        0
+    );
 
     let mut restored = vec![0u8; input.len()];
     let dsize = unsafe {
@@ -589,6 +598,26 @@ use crate::streaming::{
 };
 use core::ffi::c_int;
 
+#[test]
+fn stream_sizeof_aliases_match_context_sizeof() {
+    // ZSTD_CStream / ZSTD_DStream are the same objects as the contexts, so
+    // the stream sizeof entry points must agree with the context ones.
+    let cctx = crate::context::ZSTD_createCCtx();
+    let dctx = crate::context::ZSTD_createDCtx();
+    unsafe {
+        assert_eq!(
+            crate::streaming::ZSTD_sizeof_CStream(cctx),
+            crate::context::ZSTD_sizeof_CCtx(cctx)
+        );
+        assert_eq!(
+            crate::streaming::ZSTD_sizeof_DStream(dctx),
+            crate::context::ZSTD_sizeof_DCtx(dctx)
+        );
+        crate::context::ZSTD_freeCCtx(cctx);
+        crate::context::ZSTD_freeDCtx(dctx);
+    }
+}
+
 // ABI invariants: struct sizes match upstream `sizeof` per pointer width
 // (`size_t` / pointers halve the layouts on 32-bit).
 #[test]

From 6a1a05fa50be782fb4000206130840f37db385c5 Mon Sep 17 00:00:00 2001
From: Dmitry Prudnikov <mail@polaz.com>
Date: Sat, 13 Jun 2026 05:32:52 +0300
Subject: [PATCH 02/12] perf(encode): seed the lazy row probe with the rep
 candidate

row_best_match computed the repcode candidate, ran the row probe from
an empty best, then merged the two at the end. Seeding the probe's
best with the rep candidate instead lets the speculative tail-gate
prune row candidates against the rep length from the first hit (fewer
full common_prefix_len calls) and drops the rep value's separate
liveness across the probe. The merge is byte-identical: the seed stays
the permanent lhs, exactly as the trailing best_len_offset_candidate
made it.

i9 z000033 L8 encode_loop: -0.39% instructions (31.97G -> 31.85G,
deterministic), output byte-identical (803 tests).
---
 zstd/src/encoding/row/mod.rs | 27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/zstd/src/encoding/row/mod.rs b/zstd/src/encoding/row/mod.rs
index e0e1dd8d3..240d85016 100644
--- a/zstd/src/encoding/row/mod.rs
+++ b/zstd/src/encoding/row/mod.rs
@@ -327,7 +327,11 @@ macro_rules! greedy_parse_body {
                     // Probe body expanded inline (tier SIMD pairing via the
                     // enclosing monolith macro), not a function call.
                     None => {
-                        row_probe_body!($m, abs_pos, lit_len, hash, $rl, $use_mask, $maskmac, $cpl)
+                        // Greedy already short-circuited on a rep hit above,
+                        // so the probe starts from no candidate here.
+                        row_probe_body!(
+                            $m, abs_pos, lit_len, hash, None, $rl, $use_mask, $maskmac, $cpl
+                        )
                     }
                 };
 
@@ -444,10 +448,9 @@ macro_rules! greedy_parse_body {
 macro_rules! row_best_match {
     ($m:expr, $abs_pos:expr, $lit_len:expr, $hash:expr, $rl:expr, $use_mask:literal, $maskmac:ident, $cpl:path) => {{
         let rep = $m.repcode_candidate($abs_pos, $lit_len);
-        let row = row_probe_body!(
-            $m, $abs_pos, $lit_len, $hash, $rl, $use_mask, $maskmac, $cpl
-        );
-        best_len_offset_candidate(rep, row)
+        row_probe_body!(
+            $m, $abs_pos, $lit_len, $hash, rep, $rl, $use_mask, $maskmac, $cpl
+        )
     }};
 }
 
@@ -866,7 +869,7 @@ macro_rules! row_tag_mask_simd128 {
 /// because a `return` inside a macro body would return from the EXPANSION
 /// SITE's function.
 macro_rules! row_probe_body {
-    ($m:expr, $abs_pos:expr, $lit_len:expr, $hash:expr, $rl:expr, $use_mask:literal, $maskmac:ident, $cpl:path) => {{
+    ($m:expr, $abs_pos:expr, $lit_len:expr, $hash:expr, $seed:expr, $rl:expr, $use_mask:literal, $maskmac:ident, $cpl:path) => {{
         #[allow(unused_labels)]
         'probe: {
             debug_assert_eq!($rl, $m.row_log);
@@ -919,7 +922,13 @@ macro_rules! row_probe_body {
                 0
             };
 
-            let mut best: Option<MatchCandidate> = None;
+            // Seeded with the rep candidate (when present) so the tail-gate
+            // below prunes row candidates against the rep length from the
+            // first hit, and the rep value need not stay live separately
+            // across the probe. Merge stays byte-identical: the seed is the
+            // permanent lhs, exactly as the former trailing
+            // `best_len_offset_candidate(rep, row)` merge made it.
+            let mut best: Option<MatchCandidate> = $seed;
             // Donor `ZSTD_RowFindBestMatch` mask iteration: rotate the tag
             // mask into head (newest-first) order once, then visit ONLY the
             // set bits via tzcnt + clear-lowest. The former per-slot loop
@@ -1162,7 +1171,9 @@ macro_rules! gen_row_probe {
             lit_len: usize,
             hash: Option<(usize, u8)>,
         ) -> Option<MatchCandidate> {
-            row_probe_body!(self, abs_pos, lit_len, hash, ROW_LOG, $use_mask, $maskmac, $cpl)
+            // Standalone probe: the caller merges the rep candidate, so the
+            // probe itself starts unseeded.
+            row_probe_body!(self, abs_pos, lit_len, hash, None, ROW_LOG, $use_mask, $maskmac, $cpl)
         }
     };
 }

From 340b0046539baa8f3a876ef47bf588fcaa4e3bc5 Mon Sep 17 00:00:00 2001
From: Dmitry Prudnikov <mail@polaz.com>
Date: Sat, 13 Jun 2026 05:56:30 +0300
Subject: [PATCH 03/12] perf(encode): table-lookup the literal-length code in
 the cost model
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

lit_code_and_bits derived the LL code + extra-bit count via a 20-arm
range match that lowered to a jae comparison ladder — ~8% of self
time in the L16 btopt+dict profile because the optimal parser calls
lit_length_price per candidate. Replaced with the donor's direct
LL_Code[64] / LL_bits[36] table lookups for ll < 64 and
highbit32(ll)+19 above (ZSTD_LLcode, zstd_compress_internal.h). Same
(code, nbBits) values — byte-identical output (803 tests).

i9 small-10k-random compress-dict L16 btopt: -5.3% instructions
(23.23G -> 22.00G deterministic), -8.2% wall (1.162ms -> 1.067ms,
flat c_ffi control). Helps the whole btopt/btultra band since the
cost model is shared.
---
 zstd/src/encoding/cost_model/mod.rs | 48 ++++++++++++++---------------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/zstd/src/encoding/cost_model/mod.rs b/zstd/src/encoding/cost_model/mod.rs
index 8a2b86a35..7784bb9b9 100644
--- a/zstd/src/encoding/cost_model/mod.rs
+++ b/zstd/src/encoding/cost_model/mod.rs
@@ -229,31 +229,31 @@ impl HcOptState {
 
     #[inline(always)]
     pub(crate) fn lit_code_and_bits(lit_len: usize) -> (usize, u32) {
+        // Donor parity (`ZSTD_LLcode` + `LL_bits`, zstd_compress_internal.h):
+        // a direct table lookup for ll < 64 and `highbit32(ll) + delta` above,
+        // replacing the former 20-arm range cascade (a `jae` comparison ladder
+        // that was ~8% of L16-btopt-dict). Same (code, nbBits) values — only
+        // the derivation changes.
+        const LL_CODE: [u8; 64] = [
+            0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16, 17, 17, 18, 18, 19, 19,
+            20, 20, 20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23, 23, 23,
+            23, 23, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+        ];
+        // Extra bits per LL code (index = code, 0..=35).
+        const LL_BITS: [u8; 36] = [
+            0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,
+            10, 11, 12, 13, 14, 15, 16,
+        ];
         let ll = lit_len.min(131_071) as u32;
-        let (code, _, extra_bits) = match ll {
-            0..=15 => (ll as u8, 0, 0),
-            16..=17 => (16, ll - 16, 1),
-            18..=19 => (17, ll - 18, 1),
-            20..=21 => (18, ll - 20, 1),
-            22..=23 => (19, ll - 22, 1),
-            24..=27 => (20, ll - 24, 2),
-            28..=31 => (21, ll - 28, 2),
-            32..=39 => (22, ll - 32, 3),
-            40..=47 => (23, ll - 40, 3),
-            48..=63 => (24, ll - 48, 4),
-            64..=127 => (25, ll - 64, 6),
-            128..=255 => (26, ll - 128, 7),
-            256..=511 => (27, ll - 256, 8),
-            512..=1023 => (28, ll - 512, 9),
-            1024..=2047 => (29, ll - 1024, 10),
-            2048..=4095 => (30, ll - 2048, 11),
-            4096..=8191 => (31, ll - 4096, 12),
-            8192..=16383 => (32, ll - 8192, 13),
-            16384..=32767 => (33, ll - 16384, 14),
-            32768..=65535 => (34, ll - 32768, 15),
-            _ => (35, ll - 65536, 16),
-        };
-        (code as usize, extra_bits as u32)
+        if ll < 64 {
+            let code = LL_CODE[ll as usize] as usize;
+            (code, LL_BITS[code] as u32)
+        } else {
+            // ll in 64..=131_071 ⇒ highbit32 in 6..=16; code = hb + 19
+            // (25..=35), nbBits = hb. Matches the cascade's upper arms.
+            let hb = 31 - ll.leading_zeros();
+            ((hb + 19) as usize, hb)
+        }
     }
 
     #[inline(always)]

From 888b07034dabd2251b5a00145389f7c5e12fa31b Mon Sep 17 00:00:00 2001
From: Dmitry Prudnikov <mail@polaz.com>
Date: Sat, 13 Jun 2026 06:03:31 +0300
Subject: [PATCH 04/12] perf(encode): table-lookup the match-length code in the
 cost model

Same transform as the literal-length LUT, applied to ml_code_and_bits:
the 22-arm range cascade becomes the donor's ML_Code[128] / ML_bits[53]
lookup on mlBase for mlBase < 128 and highbit32(mlBase)+36 above
(ZSTD_MLcode). Cold on random data (few matches) but the optimal
parser calls match_length_price per candidate on compressible input.

i9 z000033 L16 btopt: -2.05% instructions (72.54G -> 71.05G
deterministic), byte-identical (803 tests). Stacks with the LL LUT
across the whole btopt/btultra band.
---
 zstd/src/encoding/cost_model/mod.rs | 55 +++++++++++++++--------------
 1 file changed, 29 insertions(+), 26 deletions(-)

diff --git a/zstd/src/encoding/cost_model/mod.rs b/zstd/src/encoding/cost_model/mod.rs
index 7784bb9b9..aa4d6ceba 100644
--- a/zstd/src/encoding/cost_model/mod.rs
+++ b/zstd/src/encoding/cost_model/mod.rs
@@ -258,32 +258,35 @@ impl HcOptState {
 
     #[inline(always)]
     pub(crate) fn ml_code_and_bits(match_len: usize) -> (usize, u32) {
-        let ml = match_len.clamp(3, 131_074) as u32;
-        let (code, _, extra_bits) = match ml {
-            3..=34 => (ml as u8 - 3, 0, 0),
-            35..=36 => (32, ml - 35, 1),
-            37..=38 => (33, ml - 37, 1),
-            39..=40 => (34, ml - 39, 1),
-            41..=42 => (35, ml - 41, 1),
-            43..=46 => (36, ml - 43, 2),
-            47..=50 => (37, ml - 47, 2),
-            51..=58 => (38, ml - 51, 3),
-            59..=66 => (39, ml - 59, 3),
-            67..=82 => (40, ml - 67, 4),
-            83..=98 => (41, ml - 83, 4),
-            99..=130 => (42, ml - 99, 5),
-            131..=258 => (43, ml - 131, 7),
-            259..=514 => (44, ml - 259, 8),
-            515..=1026 => (45, ml - 515, 9),
-            1027..=2050 => (46, ml - 1027, 10),
-            2051..=4098 => (47, ml - 2051, 11),
-            4099..=8194 => (48, ml - 4099, 12),
-            8195..=16386 => (49, ml - 8195, 13),
-            16387..=32770 => (50, ml - 16387, 14),
-            32771..=65538 => (51, ml - 32771, 15),
-            _ => (52, ml - 65539, 16),
-        };
-        (code as usize, extra_bits as u32)
+        // Donor parity (`ZSTD_MLcode` + `ML_bits`, zstd_compress_internal.h):
+        // direct table lookup on `mlBase = ml - MINMATCH` for mlBase < 128 and
+        // `highbit32(mlBase) + 36` above — same (code, nbBits) values as the
+        // former 22-arm range cascade, replacing its jae ladder. Cold on
+        // random data but hot on the compressible btopt/btultra band, where
+        // the optimal parser calls match_length_price per candidate.
+        const ML_CODE: [u8; 128] = [
+            0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
+            24, 25, 26, 27, 28, 29, 30, 31, 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37,
+            37, 37, 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, 40, 40, 40, 40,
+            40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
+            41, 41, 41, 41, 41, 41, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
+            42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
+        ];
+        // Extra bits per ML code (index = code, 0..=52).
+        const ML_BITS: [u8; 53] = [
+            0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+            0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
+        ];
+        let ml = match_len.clamp(3, 131_074);
+        let ml_base = (ml - 3) as u32;
+        if ml_base < 128 {
+            let code = ML_CODE[ml_base as usize] as usize;
+            (code, ML_BITS[code] as u32)
+        } else {
+            // mlBase in 128..=131_071 ⇒ highbit32 in 7..=16; code = hb + 36.
+            let hb = 31 - ml_base.leading_zeros();
+            ((hb + 36) as usize, hb)
+        }
     }
 
     pub(crate) fn rescale_freqs(&mut self, src: &[u8], profile: HcOptimalCostProfile) {

From dcc99ce66d402e6cf13b03cec6e97b907cb7a79f Mon Sep 17 00:00:00 2001
From: Dmitry Prudnikov <mail@polaz.com>
Date: Sat, 13 Jun 2026 06:18:12 +0300
Subject: [PATCH 05/12] perf(encode): hoist the BT pointer-pair base out of
 self in the tree walk

bt_insert_and_collect / bt_insert_step walked the binary tree via
chain_table[idx] through &mut self, forcing a Vec (ptr,len) reload from
the struct plus a bounds check on every tree step (the donor uses a raw
U32* btable). Hoisting the base pointer once and indexing unchecked --
indices are in bounds by the BT pair-index invariant -- removes both.
i9 z000033 L16: -2.0% instructions (71.0G->69.6G), byte-identical.
---
 zstd/src/encoding/match_generator.rs | 68 +++++++++++++++++++++++-----
 1 file changed, 56 insertions(+), 12 deletions(-)

diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs
index ea13aef14..5900d2a6a 100644
--- a/zstd/src/encoding/match_generator.rs
+++ b/zstd/src/encoding/match_generator.rs
@@ -3087,6 +3087,11 @@ macro_rules! bt_insert_step_no_rebase_body {
         // triggers early; raw subtraction would underflow into a huge
         // sentinel that ALWAYS triggers.
         let bt_low = $abs_pos.saturating_sub(bt_mask);
+        // Hoist the BT pointer-pair base out of `self` once — see the
+        // collect-matches body for the full rationale (per-step Vec reload +
+        // bounds check through `&mut self` vs the donor's raw `U32*` walk).
+        let chain_ptr = $table.chain_table.as_mut_ptr();
+        debug_assert_eq!($table.chain_table.len(), 2 << $table.bt_log());
         let window_low = $table.window_low_abs_for_target($target_abs);
         // `abs_pos + 9` is safe in raw form: `MatchTable::add_data` caps
         // total input at `usize::MAX - STREAM_ABS_HEADROOM` (where
@@ -3123,8 +3128,11 @@ macro_rules! bt_insert_step_no_rebase_body {
             compares_left -= 1;
 
             let next_pair_idx = $table.bt_pair_index_for_abs(candidate_abs);
-            let next_smaller = $table.chain_table[next_pair_idx];
-            let next_larger = $table.chain_table[next_pair_idx + 1];
+            // SAFETY: `next_pair_idx (+1)` = `2*(candidate_abs & bt_mask) (+1)`
+            // ≤ `chain_table.len()-1`; `chain_ptr` is the hoisted live base,
+            // table not realloc'd during the walk.
+            let next_smaller = unsafe { *chain_ptr.add(next_pair_idx) };
+            let next_larger = unsafe { *chain_ptr.add(next_pair_idx + 1) };
             let seed_len = common_length_smaller.min(common_length_larger);
             let candidate_idx = candidate_abs - $table.history_abs_start;
             // SAFETY: BT walk invariant — `candidate_idx + tail_limit ≤
@@ -3151,7 +3159,10 @@ macro_rules! bt_insert_step_no_rebase_body {
             let candidate_next = candidate_idx + match_len;
             let current_next = idx + match_len;
             if concat[candidate_next] < concat[current_next] {
-                $table.chain_table[smaller_slot] = match_stored;
+                // SAFETY: `smaller_slot` holds a valid pair index (init
+                // `pair_idx`, updated to `next_pair_idx + 1`); the `usize::MAX`
+                // sentinel is set only just before `break`, never written here.
+                unsafe { *chain_ptr.add(smaller_slot) = match_stored };
                 common_length_smaller = match_len;
                 if candidate_abs <= bt_low {
                     smaller_slot = usize::MAX;
@@ -3160,7 +3171,8 @@ macro_rules! bt_insert_step_no_rebase_body {
                 smaller_slot = next_pair_idx + 1;
                 match_stored = next_larger;
             } else {
-                $table.chain_table[larger_slot] = match_stored;
+                // SAFETY: as above for `larger_slot`.
+                unsafe { *chain_ptr.add(larger_slot) = match_stored };
                 common_length_larger = match_len;
                 if candidate_abs <= bt_low {
                     larger_slot = usize::MAX;
@@ -3171,11 +3183,17 @@ macro_rules! bt_insert_step_no_rebase_body {
             }
         }
 
+        // SAFETY: both slots, when not the `usize::MAX` sentinel, hold valid
+        // pair indices into the hoisted `chain_table` base.
         if smaller_slot != usize::MAX {
-            $table.chain_table[smaller_slot] = $crate::encoding::match_table::storage::HC_EMPTY;
+            unsafe {
+                *chain_ptr.add(smaller_slot) = $crate::encoding::match_table::storage::HC_EMPTY
+            };
         }
         if larger_slot != usize::MAX {
-            $table.chain_table[larger_slot] = $crate::encoding::match_table::storage::HC_EMPTY;
+            unsafe {
+                *chain_ptr.add(larger_slot) = $crate::encoding::match_table::storage::HC_EMPTY
+            };
         }
 
         let speed_positions = if best_len > 384 {
@@ -4660,6 +4678,19 @@ macro_rules! bt_insert_and_collect_matches_body {
         };
         let stored = relative_pos + 1;
         let bt_mask = $table.bt_mask();
+        // Hoist the BT pointer-pair table's base out of `self` once: every
+        // access below is `chain_table[computed_index]` through `&mut self`,
+        // which the optimizer cannot prove loop-invariant, so it reloads the
+        // Vec's (ptr,len) from the struct AND bounds-checks on every tree
+        // step (the donor walks a raw `U32* btable`, zstd_opt.c). The raw
+        // base carries no borrow, so the `&self` helper calls in the loop
+        // (`bt_pair_index_for_abs`, `window_low_abs_for_target`,
+        // `relative_position`) coexist — they read other fields, never
+        // `chain_table`. Indices are in bounds by the BT invariants:
+        // `bt_pair_index_for_abs` returns `2*(abs & bt_mask) (+1)` ≤
+        // `chain_table.len()-1`, and the slots only ever hold those values.
+        let chain_ptr = $table.chain_table.as_mut_ptr();
+        debug_assert_eq!($table.chain_table.len(), 2 << $table.bt_log());
         // See `bt_insert_step_no_rebase_body!`: saturating is needed for the
         // first BT walk of a fresh frame where `abs_pos < bt_mask`.
         let bt_low = $abs_pos.saturating_sub(bt_mask);
@@ -4702,8 +4733,11 @@ macro_rules! bt_insert_and_collect_matches_body {
             compares_left -= 1;
 
             let next_pair_idx = $table.bt_pair_index_for_abs(candidate_abs);
-            let next_smaller = $table.chain_table[next_pair_idx];
-            let next_larger = $table.chain_table[next_pair_idx + 1];
+            // SAFETY: `next_pair_idx (+1)` = `2*(candidate_abs & bt_mask) (+1)`
+            // ≤ `chain_table.len()-1`; `chain_ptr` is the hoisted live base,
+            // table not realloc'd during the walk.
+            let next_smaller = unsafe { *chain_ptr.add(next_pair_idx) };
+            let next_larger = unsafe { *chain_ptr.add(next_pair_idx + 1) };
             let seed_len = common_length_smaller.min(common_length_larger);
             let candidate_idx = candidate_abs - $table.history_abs_start;
             // SAFETY: BT walk invariant — `candidate_idx + tail_limit ≤
@@ -4750,7 +4784,10 @@ macro_rules! bt_insert_and_collect_matches_body {
             let candidate_next = candidate_idx + match_len;
             let current_next = idx + match_len;
             if concat[candidate_next] < concat[current_next] {
-                $table.chain_table[smaller_slot] = match_stored;
+                // SAFETY: `smaller_slot` holds a valid pair index (init
+                // `pair_idx`, updated to `next_pair_idx + 1`); the `usize::MAX`
+                // sentinel is set only just before `break`, never written here.
+                unsafe { *chain_ptr.add(smaller_slot) = match_stored };
                 common_length_smaller = match_len;
                 if candidate_abs <= bt_low {
                     smaller_slot = usize::MAX;
@@ -4759,7 +4796,8 @@ macro_rules! bt_insert_and_collect_matches_body {
                 smaller_slot = next_pair_idx + 1;
                 match_stored = next_larger;
             } else {
-                $table.chain_table[larger_slot] = match_stored;
+                // SAFETY: as above for `larger_slot`.
+                unsafe { *chain_ptr.add(larger_slot) = match_stored };
                 common_length_larger = match_len;
                 if candidate_abs <= bt_low {
                     larger_slot = usize::MAX;
@@ -4770,11 +4808,17 @@ macro_rules! bt_insert_and_collect_matches_body {
             }
         }
 
+        // SAFETY: both slots, when not the `usize::MAX` sentinel, hold valid
+        // pair indices into the hoisted `chain_table` base.
         if smaller_slot != usize::MAX {
-            $table.chain_table[smaller_slot] = $crate::encoding::match_table::storage::HC_EMPTY;
+            unsafe {
+                *chain_ptr.add(smaller_slot) = $crate::encoding::match_table::storage::HC_EMPTY
+            };
         }
         if larger_slot != usize::MAX {
-            $table.chain_table[larger_slot] = $crate::encoding::match_table::storage::HC_EMPTY;
+            unsafe {
+                *chain_ptr.add(larger_slot) = $crate::encoding::match_table::storage::HC_EMPTY
+            };
         }
 
         // Dict dual-probe (donor `ZSTD_dictMatchState`, zstd_opt.c:777-813):

From 12ffbf9ce5def8ad3146896d9a9e11308ed5a000 Mon Sep 17 00:00:00 2001
From: Dmitry Prudnikov <mail@polaz.com>
Date: Sat, 13 Jun 2026 07:07:26 +0300
Subject: [PATCH 06/12] perf(encode): fold the BT decode underflow branch into
 the window check

The BT walk decoded each stored entry via stored_abs_position_fast,
which carries two None branches (HC_EMPTY sentinel + index_shift
underflow). The underflow case only ever arises for stale post-rebase
slots, whose decoded position is below the window and already rejected
by the candidate-abs window test. Inlining the decode with a wrapping
subtraction lets an underflow wrap to a near-usize::MAX value that the
existing >= abs_pos test catches, collapsing two per-candidate branches
into one -- the donor's single matchIndex < lowLimit shape.

i9 z000033 L16: -0.46% instructions over the chain-hoist baseline,
byte-identical (803 tests incl. level22 parity).
---
 zstd/src/encoding/match_generator.rs | 42 +++++++++++++++++-----------
 1 file changed, 26 insertions(+), 16 deletions(-)

diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs
index 5900d2a6a..15038461c 100644
--- a/zstd/src/encoding/match_generator.rs
+++ b/zstd/src/encoding/match_generator.rs
@@ -3113,15 +3113,20 @@ macro_rules! bt_insert_step_no_rebase_body {
         $table.hash_table[hash] = stored;
 
         while compares_left > 0 {
-            let Some(candidate_abs) =
-                $crate::encoding::match_table::storage::MatchTable::stored_abs_position_fast(
-                    match_stored,
-                    $table.position_base,
-                    $table.index_shift,
-                )
-            else {
+            if match_stored == $crate::encoding::match_table::storage::HC_EMPTY {
                 break;
-            };
+            }
+            // Inline decode without the separate underflow branch that
+            // `stored_abs_position_fast` carries: an entry that underflows
+            // `index_shift` (a stale, post-rebase slot) wraps to a
+            // near-`usize::MAX` value, which the `>= abs_pos` window test
+            // below rejects anyway. Folding the underflow check into the
+            // window check mirrors the donor's single `matchIndex < lowLimit`
+            // test — one branch per candidate instead of two on the hottest
+            // BT-walk path. `match_stored != HC_EMPTY` here, so `- 1` cannot
+            // underflow.
+            let candidate_abs = ($table.position_base + (match_stored as usize - 1))
+                .wrapping_sub($table.index_shift);
             if candidate_abs < window_low || candidate_abs >= $abs_pos {
                 break;
             }
@@ -4718,15 +4723,20 @@ macro_rules! bt_insert_and_collect_matches_body {
         let mut best_len = (*$best_len_for_skip).max($min_match_len - 1);
 
         while compares_left > 0 {
-            let Some(candidate_abs) =
-                $crate::encoding::match_table::storage::MatchTable::stored_abs_position_fast(
-                    match_stored,
-                    $table.position_base,
-                    $table.index_shift,
-                )
-            else {
+            if match_stored == $crate::encoding::match_table::storage::HC_EMPTY {
                 break;
-            };
+            }
+            // Inline decode without the separate underflow branch that
+            // `stored_abs_position_fast` carries: an entry that underflows
+            // `index_shift` (a stale, post-rebase slot) wraps to a
+            // near-`usize::MAX` value, which the `>= abs_pos` window test
+            // below rejects anyway. Folding the underflow check into the
+            // window check mirrors the donor's single `matchIndex < lowLimit`
+            // test — one branch per candidate instead of two on the hottest
+            // BT-walk path. `match_stored != HC_EMPTY` here, so `- 1` cannot
+            // underflow.
+            let candidate_abs = ($table.position_base + (match_stored as usize - 1))
+                .wrapping_sub($table.index_shift);
             if candidate_abs < window_low || candidate_abs >= $abs_pos {
                 break;
             }

From d0b3d8ab7b49c3f61d6c1ca4ef1a50cb32b655a1 Mon Sep 17 00:00:00 2001
From: Dmitry Prudnikov <mail@polaz.com>
Date: Sat, 13 Jun 2026 14:51:36 +0300
Subject: [PATCH 07/12] perf(encode): read opt[cur] on demand in DP, drop held
 node copy

The per-position optimal-parser loop copied the 28-byte opt[cur] node
into a local held live across the (non-inlinable) candidate-collection
call, forcing LLVM to spill reps[3]+litlen around it (asm: 112 SIMD
moves vs C's 39, the excess being vmovaps stack traffic). Read the
fields on demand instead: base_cost as a scalar, the rep recompute in a
tight inner scope, and reps/litlen re-read fresh after the call (nodes
is stable across collect). Mirrors upstream's memory-resident opt[cur]
access. Byte-identical (803 tests incl. level22 parity).
---
 zstd/src/encoding/match_generator.rs | 50 ++++++++++++++++++----------
 1 file changed, 32 insertions(+), 18 deletions(-)

diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs
index 15038461c..c08a20bfa 100644
--- a/zstd/src/encoding/match_generator.rs
+++ b/zstd/src/encoding/match_generator.rs
@@ -3859,27 +3859,33 @@ macro_rules! build_optimal_plan_impl_body {
                 }
             }
 
-            let mut base_node = unsafe { *nodes.get_unchecked(pos) };
-            if base_node.price == u32::MAX {
+            // Memory-resident DP (donor parity): read opt[cur] fields on
+            // demand instead of holding a 28-byte node copy live across the
+            // per-position `$collect` call below. The held copy forced LLVM
+            // to spill reps[3] + litlen around the (non-inlinable) call;
+            // reading the fields fresh on each side keeps them out of the
+            // cross-call live set. `nodes[pos]` is stable across `$collect`
+            // (it only fills `candidates`), so post-call reads are identical.
+            let base_cost = unsafe { nodes.get_unchecked(pos).price };
+            if base_cost == u32::MAX {
                 pos += 1;
                 continue;
             }
-            if base_node.mlen > 0 && base_node.litlen == 0 {
-                // Donor parity (zstd_opt.c:1255): `cur >= opt[cur].mlen`.
-                debug_assert!(pos >= base_node.mlen as usize);
-                let prev_pos = pos - base_node.mlen as usize;
-                {
+            {
+                let base_node = unsafe { *nodes.get_unchecked(pos) };
+                if base_node.mlen > 0 && base_node.litlen == 0 {
+                    // Donor parity (zstd_opt.c:1255): `cur >= opt[cur].mlen`.
+                    debug_assert!(pos >= base_node.mlen as usize);
+                    let prev_pos = pos - base_node.mlen as usize;
                     let prev_state = unsafe { *nodes.get_unchecked(prev_pos) };
                     let (_, reps_after_match) = BtMatcher::encode_offset_with_reps(
                         base_node.off,
                         prev_state.litlen as usize,
                         prev_state.reps,
                     );
-                    base_node.reps = reps_after_match;
                     unsafe { nodes.get_unchecked_mut(pos).reps = reps_after_match };
                 }
             }
-            let base_cost = base_node.price;
 
             if pos + 8 > $current_len {
                 pos += 1;
@@ -3910,31 +3916,39 @@ macro_rules! build_optimal_plan_impl_body {
                 None
             };
             candidates.clear();
-            // SAFETY: same umbrella as `$collect`.
+            // SAFETY: same umbrella as `$collect`. Query fields are read
+            // fresh here (consumed into the call's argument) so they do not
+            // stay live across the call; the post-call reads below are a
+            // separate, fresh load of the same stable `nodes[pos]`.
             unsafe {
                 $self.$collect::<$strategy_ty, true>(
                     abs_pos,
                     current_abs_end,
                     profile,
                     HcCandidateQuery {
-                        reps: base_node.reps,
-                        lit_len: base_node.litlen as usize,
+                        reps: nodes.get_unchecked(pos).reps,
+                        lit_len: nodes.get_unchecked(pos).litlen as usize,
                         ldm_candidate,
                     },
                     &mut candidates,
                 )
             };
+            // Post-call reads of opt[cur]: fresh, born after `$collect`, so
+            // never part of the cross-call live set (see memory-resident note
+            // above). `nodes[pos]` is untouched by `$collect`.
+            let base_reps = unsafe { nodes.get_unchecked(pos).reps };
+            let base_litlen = unsafe { nodes.get_unchecked(pos).litlen as usize };
             if let Some(candidate) = candidates.last() {
                 let longest_len = candidate.match_len.min($current_len - pos);
                 if longest_len > sufficient_len
                     || pos + longest_len >= HC_OPT_NUM
                     || pos + longest_len >= $current_len
                 {
-                    let lit_len = base_node.litlen as usize;
+                    let lit_len = base_litlen;
                     let off_base = BtMatcher::encode_offset_base_with_reps(
                         candidate.offset as u32,
                         lit_len,
-                        base_node.reps,
+                        base_reps,
                     );
                     let off_price = profile
                         .offset_price_for::<ACCURATE_PRICE, FAVOR_SMALL_OFFSETS>($stats, off_base);
@@ -3958,7 +3972,7 @@ macro_rules! build_optimal_plan_impl_body {
                         off: candidate.offset as u32,
                         mlen: longest_len as u32,
                         litlen: 0,
-                        reps: base_node.reps,
+                        reps: base_reps,
                     });
                     break;
                 }
@@ -3986,11 +4000,11 @@ macro_rules! build_optimal_plan_impl_body {
                 if max_next > last_pos {
                     BtMatcher::reset_opt_nodes(&mut nodes, last_pos + 1, max_next);
                 }
-                let lit_len = base_node.litlen as usize;
+                let lit_len = base_litlen;
                 let off_base = BtMatcher::encode_offset_base_with_reps(
                     candidate.offset as u32,
                     lit_len,
-                    base_node.reps,
+                    base_reps,
                 );
                 let off_price = profile
                     .offset_price_for::<ACCURATE_PRICE, FAVOR_SMALL_OFFSETS>($stats, off_base);
@@ -4019,7 +4033,7 @@ macro_rules! build_optimal_plan_impl_body {
                             off: candidate.offset as u32,
                             mlen: match_len as u32,
                             litlen: 0,
-                            reps: base_node.reps,
+                            reps: base_reps,
                         };
                         if next > last_pos {
                             last_pos = next;

From ed13b42557098b8a6a8275b5707afe60f9666d4e Mon Sep 17 00:00:00 2001
From: Dmitry Prudnikov <mail@polaz.com>
Date: Sat, 13 Jun 2026 15:57:23 +0300
Subject: [PATCH 08/12] perf(encode): prefetch BT hash bucket to hide cold-miss
 stall

The read+write of hash_table[hash] in the BT match-collect stalled ~35%
of that function's cycles: for the large L16+ hash table over
high-entropy input the bucket is L3/DRAM-cold, and our split BT-collect
(rep/hash3 live in the seed, not inlined here) has nothing to overlap the
miss, unlike upstream's monolithic ZSTD_btGetAllMatches. Issue an
_mm_prefetch hint right after the hash is computed so the miss overlaps
the address setup before the read. x86-only (cfg-gated), byte-identical
(803 tests incl. level22 parity).
---
 zstd/src/encoding/match_generator.rs | 38 ++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs
index c08a20bfa..2cf377ba6 100644
--- a/zstd/src/encoding/match_generator.rs
+++ b/zstd/src/encoding/match_generator.rs
@@ -3076,6 +3076,25 @@ macro_rules! bt_insert_step_no_rebase_body {
             $table.hash_log,
             $table.search_mls,
         );
+        // Prefetch the hash bucket now. For the large L16+ hash table over
+        // high-entropy input the bucket is L3/DRAM-cold, and unlike upstream's
+        // monolithic ZSTD_btGetAllMatches (which overlaps this miss with its
+        // inline rep/hash3 prologue) the read+write of `hash_table[hash]`
+        // below is reached with nothing to hide it behind — it stalled a large
+        // share of this function's cycles. Issuing the hint here lets the miss
+        // overlap the address setup that follows.
+        #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+        {
+            #[cfg(target_arch = "x86")]
+            use core::arch::x86::{_MM_HINT_T0, _mm_prefetch};
+            #[cfg(target_arch = "x86_64")]
+            use core::arch::x86_64::{_MM_HINT_T0, _mm_prefetch};
+            // SAFETY: prefetch is a hint that never faults; `hash` indexes
+            // `hash_table` directly below, so it is in bounds.
+            unsafe {
+                _mm_prefetch($table.hash_table.as_ptr().add(hash).cast(), _MM_HINT_T0);
+            }
+        }
         let Some(relative_pos) = $table.relative_position($abs_pos) else {
             return 1;
         };
@@ -4692,6 +4711,25 @@ macro_rules! bt_insert_and_collect_matches_body {
             $table.hash_log,
             $table.search_mls,
         );
+        // Prefetch the hash bucket now. For the large L16+ hash table over
+        // high-entropy input the bucket is L3/DRAM-cold, and unlike upstream's
+        // monolithic ZSTD_btGetAllMatches (which overlaps this miss with its
+        // inline rep/hash3 prologue) the read+write of `hash_table[hash]`
+        // below is reached with nothing to hide it behind — it stalled a large
+        // share of this function's cycles. Issuing the hint here lets the miss
+        // overlap the address setup that follows.
+        #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+        {
+            #[cfg(target_arch = "x86")]
+            use core::arch::x86::{_MM_HINT_T0, _mm_prefetch};
+            #[cfg(target_arch = "x86_64")]
+            use core::arch::x86_64::{_MM_HINT_T0, _mm_prefetch};
+            // SAFETY: prefetch is a hint that never faults; `hash` indexes
+            // `hash_table` directly below, so it is in bounds.
+            unsafe {
+                _mm_prefetch($table.hash_table.as_ptr().add(hash).cast(), _MM_HINT_T0);
+            }
+        }
         let Some(relative_pos) = $table.relative_position($abs_pos) else {
             return;
         };

From d7598e536e4170104915f3f13fa1da9ff74e2afe Mon Sep 17 00:00:00 2001
From: Dmitry Prudnikov <mail@polaz.com>
Date: Sat, 13 Jun 2026 16:24:35 +0300
Subject: [PATCH 09/12] perf(encode): prefetch next position's BT hash bucket a
 full iteration ahead

The same-call hash prefetch only led the read by ~30 instructions, too
few to hide the L3/DRAM miss on the cold bucket. Also prefetch the next
position's bucket from the current collect: the optimal-parser DP
advances one position per iteration, so the hint is issued a full BT walk
plus the next iteration's pre-collect work ahead of the read that
consumes it. Byte-identical (803 tests incl. level22 parity).
---
 zstd/src/encoding/match_generator.rs | 44 ++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs
index 2cf377ba6..a040aca7e 100644
--- a/zstd/src/encoding/match_generator.rs
+++ b/zstd/src/encoding/match_generator.rs
@@ -3094,6 +3094,28 @@ macro_rules! bt_insert_step_no_rebase_body {
             unsafe {
                 _mm_prefetch($table.hash_table.as_ptr().add(hash).cast(), _MM_HINT_T0);
             }
+            // Prefetch the NEXT position's bucket too. The optimal-parser DP
+            // advances one position per iteration, so this miss is issued a
+            // full BT walk plus the next iteration's pre-collect work ahead of
+            // the collect that will read it — far more lead than the same-call
+            // hint above, enough to hide the full DRAM latency.
+            if idx + 1 + 8 <= concat.len() {
+                let hash_next =
+                    $crate::encoding::match_table::storage::MatchTable::hash_position_at(
+                        concat,
+                        idx + 1,
+                        $table.hash_log,
+                        $table.search_mls,
+                    );
+                // SAFETY: prefetch never faults; an out-of-range index is a
+                // harmless no-op hint.
+                unsafe {
+                    _mm_prefetch(
+                        $table.hash_table.as_ptr().add(hash_next).cast(),
+                        _MM_HINT_T0,
+                    );
+                }
+            }
         }
         let Some(relative_pos) = $table.relative_position($abs_pos) else {
             return 1;
@@ -4729,6 +4751,28 @@ macro_rules! bt_insert_and_collect_matches_body {
             unsafe {
                 _mm_prefetch($table.hash_table.as_ptr().add(hash).cast(), _MM_HINT_T0);
             }
+            // Prefetch the NEXT position's bucket too. The optimal-parser DP
+            // advances one position per iteration, so this miss is issued a
+            // full BT walk plus the next iteration's pre-collect work ahead of
+            // the collect that will read it — far more lead than the same-call
+            // hint above, enough to hide the full DRAM latency.
+            if idx + 1 + 8 <= concat.len() {
+                let hash_next =
+                    $crate::encoding::match_table::storage::MatchTable::hash_position_at(
+                        concat,
+                        idx + 1,
+                        $table.hash_log,
+                        $table.search_mls,
+                    );
+                // SAFETY: prefetch never faults; an out-of-range index is a
+                // harmless no-op hint.
+                unsafe {
+                    _mm_prefetch(
+                        $table.hash_table.as_ptr().add(hash_next).cast(),
+                        _MM_HINT_T0,
+                    );
+                }
+            }
         }
         let Some(relative_pos) = $table.relative_position($abs_pos) else {
             return;

From 43a9e39772fed59625c37ddfbf03e2e048e51baa Mon Sep 17 00:00:00 2001
From: Dmitry Prudnikov <mail@polaz.com>
Date: Sat, 13 Jun 2026 20:48:44 +0300
Subject: [PATCH 10/12] perf(encode): get_unchecked BT child-descent compare
 (helps L18/L22)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The BT walk child-descent compare used checked indexing; both indices are
first-differing positions after a match_len-long prefix bounded by
concat.len() (match_len < tail_limit + walk invariant), so get_unchecked
is safe (donor compares raw match[ml]<ip[ml]). Neutral at L16 but −0.5%
(L18) / −0.6% (L22) where deeper search walks pay more per-step checks.
---
 zstd/src/encoding/match_generator.rs | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs
index a040aca7e..e1bc2aba7 100644
--- a/zstd/src/encoding/match_generator.rs
+++ b/zstd/src/encoding/match_generator.rs
@@ -3204,7 +3204,12 @@ macro_rules! bt_insert_step_no_rebase_body {
 
             let candidate_next = candidate_idx + match_len;
             let current_next = idx + match_len;
-            if concat[candidate_next] < concat[current_next] {
+            // SAFETY: first-differing positions after a match_len-long prefix;
+            // match_len < tail_limit (break above) + BT-walk bound
+            // idx/candidate_idx + tail_limit <= concat.len() keep both in range.
+            if unsafe {
+                *concat.get_unchecked(candidate_next) < *concat.get_unchecked(current_next)
+            } {
                 // SAFETY: `smaller_slot` holds a valid pair index (init
                 // `pair_idx`, updated to `next_pair_idx + 1`); the `usize::MAX`
                 // sentinel is set only just before `break`, never written here.
@@ -4889,7 +4894,12 @@ macro_rules! bt_insert_and_collect_matches_body {
 
             let candidate_next = candidate_idx + match_len;
             let current_next = idx + match_len;
-            if concat[candidate_next] < concat[current_next] {
+            // SAFETY: first-differing positions after a match_len-long prefix;
+            // match_len < tail_limit (break above) + BT-walk bound
+            // idx/candidate_idx + tail_limit <= concat.len() keep both in range.
+            if unsafe {
+                *concat.get_unchecked(candidate_next) < *concat.get_unchecked(current_next)
+            } {
                 // SAFETY: `smaller_slot` holds a valid pair index (init
                 // `pair_idx`, updated to `next_pair_idx + 1`); the `usize::MAX`
                 // sentinel is set only just before `break`, never written here.

From 1583226ee95571f9c80cf0e6b6483ce938e7babc Mon Sep 17 00:00:00 2001
From: Dmitry Prudnikov <mail@polaz.com>
Date: Sat, 13 Jun 2026 21:53:14 +0300
Subject: [PATCH 11/12] perf(encode): donor-form BT-walk window check in the
 loop condition

The BT-walk decoded candidate_abs and then broke on the window test, so
every walk's terminating step entered the body and paid a wasted decode
that upstream avoids by testing matchIndex >= matchLow in the while
condition. Precompute the window bound in stored space and fold it into
the loop condition (s.wrapping_add(win_off) < win_range), decoding
candidate_abs only for productive steps. Byte-identical (803 tests incl.
level22 parity).
---
 zstd/src/encoding/match_generator.rs | 43 +++++++++++++++++-----------
 1 file changed, 26 insertions(+), 17 deletions(-)

diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs
index e1bc2aba7..6cafe2881 100644
--- a/zstd/src/encoding/match_generator.rs
+++ b/zstd/src/encoding/match_generator.rs
@@ -4801,6 +4801,22 @@ macro_rules! bt_insert_and_collect_matches_body {
         // first BT walk of a fresh frame where `abs_pos < bt_mask`.
         let bt_low = $abs_pos.saturating_sub(bt_mask);
         let window_low = $table.window_low_abs_for_target($abs_pos);
+        // Donor-style window bound in stored space so the BT-walk loop
+        // condition rejects out-of-window / HC_EMPTY candidates WITHOUT
+        // decoding them (mirrors upstream `while ... matchIndex >= matchLow`):
+        // one range check on `match_stored` instead of decode-then-break,
+        // dropping the wasted candidate_abs decode on every walk's terminating
+        // step. candidate_abs(s) = (position_base + s - 1) - index_shift =
+        // base + s (wrapping); in-window ⟺ candidate_abs - window_low <
+        // abs_pos - window_low ⟺ s.wrapping_add(win_off) < win_range.
+        // HC_EMPTY (s = 0) maps to base = (lowest representable abs) - 1 <
+        // window_low, so it falls out of range and ends the walk.
+        let win_off = $table
+            .position_base
+            .wrapping_sub(1)
+            .wrapping_sub($table.index_shift)
+            .wrapping_sub(window_low);
+        let win_range = $abs_pos - window_low;
         // Raw `+ 9` is safe here — see `bt_insert_step_no_rebase_body!`
         // for the full discussion of the upstream `STREAM_ABS_HEADROOM`
         // cap in `MatchTable::add_data`.
@@ -4823,25 +4839,18 @@ macro_rules! bt_insert_and_collect_matches_body {
         );
         let mut best_len = (*$best_len_for_skip).max($min_match_len - 1);
 
-        while compares_left > 0 {
-            if match_stored == $crate::encoding::match_table::storage::HC_EMPTY {
-                break;
-            }
-            // Inline decode without the separate underflow branch that
-            // `stored_abs_position_fast` carries: an entry that underflows
-            // `index_shift` (a stale, post-rebase slot) wraps to a
-            // near-`usize::MAX` value, which the `>= abs_pos` window test
-            // below rejects anyway. Folding the underflow check into the
-            // window check mirrors the donor's single `matchIndex < lowLimit`
-            // test — one branch per candidate instead of two on the hottest
-            // BT-walk path. `match_stored != HC_EMPTY` here, so `- 1` cannot
-            // underflow.
+        // Donor-form loop condition: the stored-space window range check
+        // (`s.wrapping_add(win_off) < win_range`) rejects out-of-window and
+        // HC_EMPTY candidates here, so the terminating step never enters the
+        // body — no wasted candidate_abs decode, matching upstream's
+        // `while ... matchIndex >= matchLow`.
+        while compares_left > 0 && (match_stored as usize).wrapping_add(win_off) < win_range {
+            compares_left -= 1;
+            // The condition proved this candidate is in `[window_low,
+            // abs_pos)`, so `match_stored >= 1` (HC_EMPTY is out of range) and
+            // the `- 1` cannot underflow; candidate_abs == base + match_stored.
             let candidate_abs = ($table.position_base + (match_stored as usize - 1))
                 .wrapping_sub($table.index_shift);
-            if candidate_abs < window_low || candidate_abs >= $abs_pos {
-                break;
-            }
-            compares_left -= 1;
 
             let next_pair_idx = $table.bt_pair_index_for_abs(candidate_abs);
             // SAFETY: `next_pair_idx (+1)` = `2*(candidate_abs & bt_mask) (+1)`

From a39aad5243117a05dcf7d0e673976f5b013016fc Mon Sep 17 00:00:00 2001
From: Dmitry Prudnikov <mail@polaz.com>
Date: Sat, 13 Jun 2026 22:06:10 +0300
Subject: [PATCH 12/12] fix(encode): reject stale BT candidate underflow on
 rebased streams

- BT insert-walk: use checked_sub instead of wrapping_sub when decoding a
  candidate absolute position, so a stale post-rebase slot below
  index_shift ends the walk rather than wrapping into [window_low,
  abs_pos) when abs_pos nears the integer ceiling (reachable on 32-bit
  long streams). The collect-walk already rejects this via its
  stored-space range check anchored at the true lower bound.
- Gate the BT hash-bucket prefetch blocks on target_feature sse too,
  since _mm_prefetch is an SSE intrinsic (compiled out on no-SSE x86).
- c-api: assert created contexts are non-NULL before the sizeof alias
  equality checks so they cannot pass vacuously.
---
 c-api/src/tests.rs                   |  4 ++++
 zstd/src/encoding/match_generator.rs | 35 +++++++++++++++++-----------
 2 files changed, 26 insertions(+), 13 deletions(-)

diff --git a/c-api/src/tests.rs b/c-api/src/tests.rs
index 9e425b887..0f329e386 100644
--- a/c-api/src/tests.rs
+++ b/c-api/src/tests.rs
@@ -604,6 +604,10 @@ fn stream_sizeof_aliases_match_context_sizeof() {
     // the stream sizeof entry points must agree with the context ones.
     let cctx = crate::context::ZSTD_createCCtx();
     let dctx = crate::context::ZSTD_createDCtx();
+    // Both `sizeof` calls return 0 on a NULL handle, so the equality checks
+    // would pass vacuously if creation failed — assert non-NULL first.
+    assert!(!cctx.is_null(), "ZSTD_createCCtx returned NULL");
+    assert!(!dctx.is_null(), "ZSTD_createDCtx returned NULL");
     unsafe {
         assert_eq!(
             crate::streaming::ZSTD_sizeof_CStream(cctx),
diff --git a/zstd/src/encoding/match_generator.rs b/zstd/src/encoding/match_generator.rs
index 6cafe2881..22e5ccd52 100644
--- a/zstd/src/encoding/match_generator.rs
+++ b/zstd/src/encoding/match_generator.rs
@@ -3083,7 +3083,10 @@ macro_rules! bt_insert_step_no_rebase_body {
         // below is reached with nothing to hide it behind — it stalled a large
         // share of this function's cycles. Issuing the hint here lets the miss
         // overlap the address setup that follows.
-        #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+        #[cfg(all(
+            target_feature = "sse",
+            any(target_arch = "x86", target_arch = "x86_64")
+        ))]
         {
             #[cfg(target_arch = "x86")]
             use core::arch::x86::{_MM_HINT_T0, _mm_prefetch};
@@ -3157,17 +3160,20 @@ macro_rules! bt_insert_step_no_rebase_body {
             if match_stored == $crate::encoding::match_table::storage::HC_EMPTY {
                 break;
             }
-            // Inline decode without the separate underflow branch that
-            // `stored_abs_position_fast` carries: an entry that underflows
-            // `index_shift` (a stale, post-rebase slot) wraps to a
-            // near-`usize::MAX` value, which the `>= abs_pos` window test
-            // below rejects anyway. Folding the underflow check into the
-            // window check mirrors the donor's single `matchIndex < lowLimit`
-            // test — one branch per candidate instead of two on the hottest
-            // BT-walk path. `match_stored != HC_EMPTY` here, so `- 1` cannot
-            // underflow.
-            let candidate_abs = ($table.position_base + (match_stored as usize - 1))
-                .wrapping_sub($table.index_shift);
+            // Reject stale post-rebase slots whose pre-shift position is below
+            // `index_shift` explicitly. A `wrapping_sub` maps such a slot to a
+            // near-`usize::MAX` value that the `>= abs_pos` test only rejects
+            // while `abs_pos` is far from the integer ceiling; on a
+            // long-running rebased stream (reachable on 32-bit) `abs_pos` can
+            // approach the ceiling and the wrapped value can land back inside
+            // `[window_low, abs_pos)`. `checked_sub` ends the walk on the
+            // underflow instead. `match_stored != HC_EMPTY` here, so the `- 1`
+            // cannot underflow.
+            let Some(candidate_abs) = ($table.position_base + (match_stored as usize - 1))
+                .checked_sub($table.index_shift)
+            else {
+                break;
+            };
             if candidate_abs < window_low || candidate_abs >= $abs_pos {
                 break;
             }
@@ -4745,7 +4751,10 @@ macro_rules! bt_insert_and_collect_matches_body {
         // below is reached with nothing to hide it behind — it stalled a large
         // share of this function's cycles. Issuing the hint here lets the miss
         // overlap the address setup that follows.
-        #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+        #[cfg(all(
+            target_feature = "sse",
+            any(target_arch = "x86", target_arch = "x86_64")
+        ))]
         {
             #[cfg(target_arch = "x86")]
             use core::arch::x86::{_MM_HINT_T0, _mm_prefetch};