perf(levels): consolidate cparams + negative-band byte-parity with C

ffi-bench/Cargo.toml

@@ -32,6 +32,9 @@ zstd = { version = "0.13.3", features = ["zdict_builder", "experimental"] }
 criterion = "0.8"
 rand = "0.10"
 dhat = "0.3"
+# Third-party pure-Rust zstd codec, for the cross-implementation comparison
+# example (`zrip_compare`). Encode-focused (levels -7..4, Fast/DFast).
+zrip = "0.7"
 
 [features]
 # Forward the structured-zstd features the bench/test/example sources gate on
@@ -52,6 +55,11 @@ name = "decode_all"
 path = "../zstd/benches/decode_all.rs"
 harness = false
 
+[[bench]]
+name = "decode_loop"
+path = "../zstd/benches/decode_loop.rs"
+harness = false
+
 [[bench]]
 name = "huf_decode_kernels"
 path = "../zstd/benches/huf_decode_kernels.rs"
@@ -188,6 +196,10 @@ path = "../zstd/examples/decode_loop_dict.rs"
 name = "decode_loop_z000033"
 path = "../zstd/examples/decode_loop_z000033.rs"
 
+[[example]]
+name = "zrip_compare"
+path = "examples/zrip_compare.rs"
+
 [[example]]
 name = "encode_l4"
 path = "../zstd/examples/encode_l4.rs"
@@ -237,3 +249,15 @@ required-features = ["bench_internals"]
 name = "dict_matrix"
 path = "../zstd/examples/dict_matrix.rs"
 required-features = ["dict_builder"]
+
+[[example]]
+name = "interop_small"
+path = "examples/interop_small.rs"
+
+[[example]]
+name = "encode_small"
+path = "examples/encode_small.rs"
+
+[[example]]
+name = "ratio_parity"
+path = "examples/ratio_parity.rs"

ffi-bench/examples/encode_small.rs

@@ -0,0 +1,34 @@
+//! Tight small-input compress loop for flamegraph profiling. Isolates the
+//! per-frame encoder cost that dominates small frames (where setup / entropy /
+//! table work is not amortised over many bytes). Pure encoder, no FFI.
+//!
+//! Usage: encode_small [level] [iters]   (defaults: level 3, 2_000_000 iters)
+//! Profile on the i9: cargo flamegraph --example encode_small --features dict_builder -- 3 2000000
+
+use structured_zstd::encoding::{CompressionLevel, compress_slice_to_vec};
+
+fn main() {
+    let data = std::fs::read(
+        std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+            .join("../zstd/decodecorpus_files/z000002"),
+    )
+    .expect("z000002 fixture");
+    let level: i32 = std::env::args()
+        .nth(1)
+        .and_then(|s| s.parse().ok())
+        .unwrap_or(3);
+    let iters: usize = std::env::args()
+        .nth(2)
+        .and_then(|s| s.parse().ok())
+        .unwrap_or(2_000_000);
+
+    let mut sink = 0u64;
+    for _ in 0..iters {
+        let out = compress_slice_to_vec(&data, CompressionLevel::Level(level));
+        sink = sink.wrapping_add(out.len() as u64);
+    }
+    println!(
+        "done level={level} iters={iters} input={} sink={sink}",
+        data.len()
+    );
+}

ffi-bench/examples/interop_small.rs

@@ -0,0 +1,52 @@
+//! Cross-implementation interop check for small frames: our encoder now sizes
+//! the window the C-faithful way (`get_cparams` clamps it to the source, e.g.
+//! window_log 10 for a 1 KiB input), dropping the old MIN_HINTED_WINDOW_LOG
+//! 16 KiB floor. This verifies a frame WE produce with such a small window
+//! still decodes in the C reference decoder (the interop the old floor guarded).
+
+use structured_zstd::encoding::{CompressionLevel, compress_slice_to_vec};
+
+fn load(name: &str) -> Option<Vec<u8>> {
+    let p = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("../zstd/decodecorpus_files")
+        .join(name);
+    std::fs::read(p).ok()
+}
+
+fn check(name: &str, data: &[u8]) {
+    for lvl in [-5i32, -1, 1, 2, 3, 4] {
+        let ours = compress_slice_to_vec(data, CompressionLevel::Level(lvl));
+        match zstd::bulk::decompress(&ours, data.len().max(1)) {
+            Ok(dec) if dec == data => {
+                println!(
+                    "OK   {name:<16} L{lvl:<3} {} -> {} B, C-decoded",
+                    data.len(),
+                    ours.len()
+                );
+            }
+            Ok(dec) => panic!(
+                "{name} L{lvl}: C decoded {} bytes but != input ({} bytes)",
+                dec.len(),
+                data.len()
+            ),
+            Err(e) => panic!("{name} L{lvl}: C FAILED to decode our frame: {e}"),
+        }
+    }
+}
+
+fn main() {
+    // Synthetic small inputs that drive the window below the old 16 KiB floor.
+    let pat512: Vec<u8> = (0..512).map(|i| (i % 37) as u8).collect();
+    let pat1k: Vec<u8> = (0..1024).map(|i| ((i * 7) % 251) as u8).collect();
+    let pat4k: Vec<u8> = (0..4096).map(|i| ((i * 3 + i / 11) % 97) as u8).collect();
+    check("synthetic-512", &pat512);
+    check("synthetic-1k", &pat1k);
+    check("synthetic-4k", &pat4k);
+    if let Some(d) = load("z000001") {
+        check("z000001", &d);
+    }
+    if let Some(d) = load("z000002") {
+        check("z000002", &d);
+    }
+    println!("\nALL small-window frames round-tripped through the C decoder.");
+}

ffi-bench/examples/ratio_parity.rs

@@ -0,0 +1,53 @@
+//! Ground-truth ratio comparison: OUR real compressed frame vs the C reference's
+//! real compressed frame (zstd crate), byte sizes per level. Used to confirm the
+//! negative-level over-compression (ours < C bytes) that ZSTD_generateSequences
+//! does NOT reflect, and to drive ratio parity with C.
+
+use structured_zstd::encoding::{CompressionLevel, compress_slice_to_vec};
+
+fn main() {
+    let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("../zstd/decodecorpus_files")
+        .join(std::env::args().nth(1).unwrap_or_else(|| "z000002".into()));
+    let data = std::fs::read(&path).unwrap_or_else(|e| panic!("read {path:?}: {e}"));
+    println!("fixture {path:?} ({} bytes)", data.len());
+    for lvl in [-7i32, -5, -3, -1, 1, 3] {
+        let ours = compress_slice_to_vec(&data, CompressionLevel::Level(lvl));
+        let c = zstd::bulk::compress(&data, lvl).expect("c compress");
+        // Sanity: ours MUST round-trip through the C decoder (drop-in contract).
+        // Fail loud on the exact regression this example exists to catch instead
+        // of printing `false` and exiting 0.
+        let c_decoded =
+            zstd::bulk::decompress(&ours, data.len().max(1)).expect("C decoder rejected ours");
+        assert_eq!(c_decoded, data, "C decoder mismatch for L{lvl}");
+        println!(
+            "L{lvl:<3} ours={:>5}B  C={:>5}B  ours/C={:.3}  (C decodes ours: ok)",
+            ours.len(),
+            c.len(),
+            ours.len() as f64 / c.len() as f64,
+        );
+        // Byte-level diff: first differing index + a hex window around it from
+        // each frame, so a 1-byte header divergence is pinpointed exactly.
+        if ours != c {
+            let first = (0..ours.len().min(c.len()))
+                .find(|&i| ours[i] != c[i])
+                .unwrap_or(ours.len().min(c.len()));
+            let lo = first.saturating_sub(4);
+            let hi_o = (first + 6).min(ours.len());
+            let hi_c = (first + 6).min(c.len());
+            let hx = |s: &[u8]| {
+                s.iter()
+                    .map(|b| format!("{b:02x}"))
+                    .collect::<Vec<_>>()
+                    .join(" ")
+            };
+            println!(
+                "   DIFF L{lvl}: first differ at byte {first} (ours {}B vs C {}B)\n     ours[{lo}..]: {}\n     C   [{lo}..]: {}",
+                ours.len(),
+                c.len(),
+                hx(&ours[lo..hi_o]),
+                hx(&c[lo..hi_c]),
+            );
+        }
+    }
+}