fix(dfast): upstream-identical sequence generation + faster HUF decode

zstd/benches/compare_ffi.rs

@@ -293,6 +293,8 @@ fn bench_compress(c: &mut Criterion) {
                 emit_report_line(scenario, level, &rust_compressed, &ffi_compressed);
                 emit_frame_header_report(scenario, level, "rust", &rust_compressed);
                 emit_frame_header_report(scenario, level, "ffi", &ffi_compressed);
+                emit_block_structure_report(scenario, level, "rust", &rust_compressed);
+                emit_block_structure_report(scenario, level, "ffi", &ffi_compressed);
             }
 
             let benchmark_name = format!("compress/{}/{}/{}", level.name, scenario.id, "matrix");
@@ -1085,6 +1087,114 @@ fn emit_frame_header_report(
     );
 }
 
+fn emit_block_structure_report(
+    scenario: &Scenario,
+    level: LevelConfig,
+    encoder: &'static str,
+    compressed: &[u8],
+) {
+    if compressed.len() < 5 {
+        return;
+    }
+    // Raw hex of the frame head: lets us read the literals-section header and
+    // Huffman tree description (FSE weights vs direct nibbles) by hand when
+    // comparing rust vs ffi generation byte-for-byte on a fixture.
+    let head_len = compressed.len().min(48);
+    let hex: String = compressed[..head_len]
+        .iter()
+        .map(|b| format!("{b:02x}"))
+        .collect::<Vec<_>>()
+        .join(" ");
+    println!(
+        "REPORT_HEX scenario={} level={} encoder={} head=[{}]",
+        scenario.id, level.name, encoder, hex
+    );
+    let desc = compressed[4];
+    let fcs_flag = desc >> 6;
+    let single_segment = ((desc >> 5) & 0x1) == 1;
+    let checksum = ((desc >> 2) & 0x1) == 1;
+    let dict_id_bytes: usize = match desc & 0x3 {
+        0 => 0,
+        1 => 1,
+        2 => 2,
+        _ => 4,
+    };
+    let fcs_bytes: usize = match fcs_flag {
+        0 => usize::from(single_segment),
+        1 => 2,
+        2 => 4,
+        _ => 8,
+    };
+    let mut pos = 4 + 1 + usize::from(!single_segment) + dict_id_bytes + fcs_bytes;
+    let payload_end = compressed.len() - if checksum { 4 } else { 0 };
+    // A truncated/invalid frame must report `parse=error`, not fall through to a
+    // normal REPORT_BLK that would make it look like a valid empty/partial frame.
+    if pos > payload_end {
+        println!(
+            "REPORT_BLK scenario={} level={} encoder={} parse=error",
+            scenario.id, level.name, encoder
+        );
+        return;
+    }
+    let mut blocks: Vec<(char, usize, bool)> = Vec::new();
+    let mut saw_last = false;
+    while pos + 3 <= payload_end {
+        let h = compressed[pos] as usize
+            | (compressed[pos + 1] as usize) << 8
+            | (compressed[pos + 2] as usize) << 16;
+        let last = (h & 1) == 1;
+        let btype = (h >> 1) & 0x3;
+        let bsize = h >> 3;
+        let (kind, advance) = match btype {
+            0 => ('R', bsize), // Raw
+            1 => ('L', 1),     // RLE (1 byte payload)
+            2 => ('C', bsize), // Compressed
+            _ => {
+                println!(
+                    "REPORT_BLK scenario={} level={} encoder={} parse=error",
+                    scenario.id, level.name, encoder
+                );
+                return;
+            }
+        };
+        if pos + 3 + advance > payload_end {
+            println!(
+                "REPORT_BLK scenario={} level={} encoder={} parse=error",
+                scenario.id, level.name, encoder
+            );
+            return;
+        }
+        blocks.push((kind, bsize, last));
+        pos += 3 + advance;
+        if last {
+            saw_last = true;
+            break;
+        }
+    }
+    // A clean frame ends exactly on a terminal (`last`) block with no leftover
+    // bytes; otherwise it was truncated (e.g. 1-2 trailing bytes that cannot
+    // form a block header) and must report a parse error, not a block list.
+    if !saw_last || pos != payload_end {
+        println!(
+            "REPORT_BLK scenario={} level={} encoder={} parse=error",
+            scenario.id, level.name, encoder
+        );
+        return;
+    }
+    let list: Vec<String> = blocks
+        .iter()
+        .map(|(k, s, l)| format!("{k}{s}{}", if *l { "!" } else { "" }))
+        .collect();
+    println!(
+        "REPORT_BLK scenario={} level={} encoder={} n_blocks={} blocks=[{}]",
+        scenario.id,
+        level.name,
+        encoder,
+        blocks.len(),
+        list.join(","),
+    );
+}
+
 fn emit_report_line(
     scenario: &Scenario,
     level: LevelConfig,

zstd/src/decoding/frame.rs

@@ -18,6 +18,7 @@ pub(crate) fn read_frame_header(r: impl Read) -> Result<(FrameHeader, u8), ReadF
 /// frame detection is bypassed — the caller MUST know out-of-band
 /// that the stream is magicless. Upstream zstd parity:
 /// `ZSTD_f_zstd1_magicless` via `ZSTD_d_format`.
+#[inline]
 pub fn read_frame_header_with_format(
     mut r: impl Read,
     magicless: bool,
@@ -60,6 +61,10 @@ pub fn read_frame_header_with_format(
         window_descriptor: 0,
     };
 
+    // Each variable header field is read with its own field-specific error so
+    // a truncated frame reports which field is missing. The slice `read_exact`
+    // override (`io_nostd`) makes each of these a single bounds-checked copy,
+    // so the small per-field reads are not the cost they once were.
     if !desc.single_segment_flag() {
         r.read_exact(&mut buf[0..1])
             .map_err(err::WindowDescriptorReadError)?;
@@ -73,10 +78,8 @@ pub fn read_frame_header_with_format(
         r.read_exact(buf).map_err(err::DictionaryIdReadError)?;
         bytes_read += dict_id_len;
         let mut dict_id = 0u32;
-
-        #[allow(clippy::needless_range_loop)]
-        for i in 0..dict_id_len {
-            dict_id += (buf[i] as u32) << (8 * i);
+        for (i, &b) in buf.iter().enumerate() {
+            dict_id += (b as u32) << (8 * i);
         }
         if dict_id != 0 {
             frame_header.dict_id = Some(dict_id);
@@ -91,10 +94,8 @@ pub fn read_frame_header_with_format(
             .map_err(err::FrameContentSizeReadError)?;
         bytes_read += fcs_len;
         let mut fcs = 0u64;
-
-        #[allow(clippy::needless_range_loop)]
-        for i in 0..fcs_len {
-            fcs += (fcs_buf[i] as u64) << (8 * i);
+        for (i, &b) in fcs_buf.iter().enumerate() {
+            fcs += (b as u64) << (8 * i);
         }
         if fcs_len == 2 {
             fcs += 256;

zstd/src/decoding/frame_decoder.rs

@@ -854,6 +854,7 @@ impl FrameDecoderState {
     /// non-direct fallback reserves `window_size` once in
     /// `decode_all_impl` / `decode_blocks` via `reserve_buffer` before
     /// any block write.
+    #[inline]
     pub(crate) fn new_with_format(
         source: impl Read,
         magicless: bool,
@@ -898,6 +899,7 @@ impl FrameDecoderState {
     /// `DecoderScratchKind::reserve_buffer(window_size)` before any block
     /// write. A reused scratch whose new frame fits within prior capacity
     /// reuses it; a larger one grows on that same `reserve_buffer` call.
+    #[inline]
     pub(crate) fn reset_with_format(
         &mut self,
         source: impl Read,
@@ -1186,6 +1188,7 @@ impl FrameDecoder {
     /// Note that all bytes currently in the decodebuffer from any previous frame will be lost. Collect them with collect()/collect_to_writer()
     ///
     /// equivalent to reset()
+    #[inline]
     pub fn init(&mut self, source: impl Read) -> Result<(), FrameDecoderError> {
         self.reset(source)
     }
@@ -1222,6 +1225,7 @@ impl FrameDecoder {
     /// Note that all bytes currently in the decodebuffer from any previous frame will be lost. Collect them with collect()/collect_to_writer()
     ///
     /// equivalent to init()
+    #[inline]
     pub fn reset(&mut self, source: impl Read) -> Result<(), FrameDecoderError> {
         use FrameDecoderError as err;
         // Fresh frame → start with an empty per-block checksum vec so

zstd/src/decoding/literals_section_decoder.rs

@@ -446,13 +446,46 @@ fn decompress_literals_impl<K: CpuKernel>(
         // a no-op; eliding it also lets us hold `target_ptr` without an
         // intervening `&mut Vec<u8>` borrow that invalidates the pointer
         // under stacked-borrows.
+        // Per-stream tail decode, mirroring upstream zstd `HUF_decodeStreamX1`
+        // (huf_decompress.c:546): decode in groups of 4 with ONE reload per
+        // group, then the final < 4 symbols per-symbol. The previous form
+        // reloaded the bit reader on EVERY symbol, so each trailing symbol near
+        // a stream end paid `refill_slow` — the dominant drain cost on a
+        // literal-heavy frame.
+        let group_bits = 4 * max_num_bits;
         for i in 0..4 {
-            while brs[i].bits_remaining() > -max_bits && cursors[i] < ends[i] {
-                let byte = decoders[i].decode_symbol_and_advance(&mut brs[i]);
-                unsafe {
-                    target_ptr.add(cursors[i]).write(byte);
+            // Phase 1 (upstream `HUF_decodeStreamX1` lines 551-557): decode in
+            // groups of four with a SINGLE reload per group. Output-based bound
+            // (`cursors[i] + 4 <= ends[i]`), like upstream's `p < pEnd-3`, so a
+            // group only runs while four whole symbols remain — it never reads
+            // past the last symbol into the zero padding.
+            while cursors[i] + 4 <= ends[i] {
+                brs[i].ensure_bits(group_bits);
+                for _ in 0..4 {
+                    let byte = decoders[i].decode_symbol_and_advance_no_refill(&mut brs[i]);
+                    unsafe {
+                        target_ptr.add(cursors[i]).write(byte);
+                    }
+                    cursors[i] += 1;
+                }
+            }
+            // Phase 2 (upstream `HUF_decodeStreamX1` line 568 `while (p < pEnd)`):
+            // the final < 4 symbols. ONE reload covers them (<= 3 * max_num_bits
+            // bits), then NO per-symbol reload — so the trailing symbols at the
+            // stream end never pay the cold `refill_slow` each. `bits_remaining`
+            // is reload-timing-independent (the padding lands in either
+            // `extra_bits` or `bits_consumed`, and `(64 - bits_consumed) -
+            // extra_bits` is identical), so the end-of-stream check below still
+            // holds exactly.
+            if cursors[i] < ends[i] {
+                brs[i].ensure_bits(group_bits);
+                while cursors[i] < ends[i] {
+                    let byte = decoders[i].decode_symbol_and_advance_no_refill(&mut brs[i]);
+                    unsafe {
+                        target_ptr.add(cursors[i]).write(byte);
+                    }
+                    cursors[i] += 1;
                 }
-                cursors[i] += 1;
             }
             if brs[i].bits_remaining() != -max_bits {
                 return Err(DecompressLiteralsError::BitstreamReadMismatch {