perf(decode): route RLE-mode sequence tables through the fused path

zstd/src/decoding/block_decoder.rs

@@ -237,7 +237,6 @@ impl BlockDecoder {
             parts.buffer,
             parts.offset_hist,
             parts.literals_buffer,
-            parts.sequences,
             raw,
         )
     }
@@ -276,7 +275,6 @@ impl BlockDecoder {
             parts.buffer,
             parts.offset_hist,
             parts.literals_buffer,
-            parts.sequences,
             raw,
         )
     }
@@ -298,7 +296,6 @@ impl BlockDecoder {
         buffer: &mut crate::decoding::decode_buffer::DecodeBuffer<B>,
         offset_hist: &mut [u32; 3],
         literals_buffer: &mut alloc::vec::Vec<u8>,
-        sequences: &mut alloc::vec::Vec<crate::blocks::sequence_section::Sequence>,
         raw: &[u8],
     ) -> Result<(), DecompressBlockError> {
         let mut section = LiteralsSection::new();
@@ -374,9 +371,10 @@ impl BlockDecoder {
         if seq_section.num_sequences != 0 {
             // Fused decode + execute: avoids the Vec<Sequence> round-trip
             // and inlines the per-iter execute_one_sequence work next to
-            // the FSE state advance. Falls back to the legacy two-pass
-            // pipeline internally when any of LL/ML/OF is in RLE mode.
-            // Pass field-level borrows from the WorkspaceRef so `raw`
+            // the FSE state advance. RLE-mode axes are handled in the same
+            // fused loop via a degenerate single-state FSE table built by
+            // `maybe_update_fse_tables`, so there is no separate two-pass
+            // path. Pass field-level borrows from the WorkspaceRef so `raw`
             // (immutable view into block_content_buffer) can coexist
             // with the mutable borrows on the FSE / decode-buffer /
             // offset-hist fields.
@@ -387,7 +385,6 @@ impl BlockDecoder {
                 buffer,
                 offset_hist,
                 literals_view,
-                sequences,
             )?;
         } else {
             if !raw.is_empty() {
@@ -398,7 +395,6 @@ impl BlockDecoder {
                 ));
             }
             buffer.push(literals_view);
-            sequences.clear();
         }
 
         Ok(())
@@ -638,14 +634,12 @@ mod tests {
         let mut fse = FSEScratch::new();
         let mut offset_hist = [1u32, 4, 8];
         let mut literals_buffer = alloc::vec::Vec::new();
-        let mut sequences = alloc::vec::Vec::new();
         let mut block_content_buffer = alloc::vec::Vec::new();
         let mut direct = DirectScratch {
             huf: &mut huf,
             fse: &mut fse,
             offset_hist: &mut offset_hist,
             literals_buffer: &mut literals_buffer,
-            sequences: &mut sequences,
             block_content_buffer: &mut block_content_buffer,
             buffer,
         };
@@ -686,14 +680,12 @@ mod tests {
         let mut fse = FSEScratch::new();
         let mut offset_hist = [1u32, 4, 8];
         let mut literals_buffer = alloc::vec::Vec::new();
-        let mut sequences = alloc::vec::Vec::new();
         let mut block_content_buffer = alloc::vec::Vec::new();
         let mut direct = DirectScratch {
             huf: &mut huf,
             fse: &mut fse,
             offset_hist: &mut offset_hist,
             literals_buffer: &mut literals_buffer,
-            sequences: &mut sequences,
             block_content_buffer: &mut block_content_buffer,
             buffer,
         };

zstd/src/decoding/errors.rs

@@ -951,15 +951,28 @@ pub enum DecodeSequenceError {
     GetBitsError(GetBitsError),
     FSEDecoderError(FSEDecoderError),
     FSETableError(FSETableError),
-    ExtraPadding { skipped_bits: i32 },
-    UnsupportedOffset { offset_code: u8 },
+    ExtraPadding {
+        skipped_bits: i32,
+    },
+    UnsupportedOffset {
+        offset_code: u8,
+    },
     ZeroOffset,
     NotEnoughBytesForNumSequences,
-    ExtraBits { bits_remaining: isize },
+    ExtraBits {
+        bits_remaining: isize,
+    },
     MissingCompressionMode,
     MissingByteForRleLlTable,
     MissingByteForRleOfTable,
     MissingByteForRleMlTable,
+    /// An RLE-mode sequence table's single symbol code is out of range
+    /// for its axis (LL/ML/OF). `axis` names the axis; `code` is the
+    /// offending value read from the stream.
+    InvalidRleCode {
+        axis: &'static str,
+        code: u8,
+    },
 }
 
 #[cfg(feature = "std")]
@@ -1014,6 +1027,9 @@ impl core::fmt::Display for DecodeSequenceError {
             DecodeSequenceError::MissingByteForRleMlTable => {
                 write!(f, "Need a byte to read for RLE ml table")
             }
+            DecodeSequenceError::InvalidRleCode { axis, code } => {
+                write!(f, "RLE {axis} table code {code} is out of range")
+            }
         }
     }
 }

zstd/src/decoding/frame_decoder.rs

@@ -1627,14 +1627,13 @@ impl FrameDecoder {
         // fields; only `buffer` differs and we don't use that here.
         // Macro-style binding avoids the closure / generic
         // gymnastics of returning multiple `&mut` from a match arm.
-        let (huf, fse, offset_hist, literals_buffer, sequences, block_content_buffer, window_size) =
+        let (huf, fse, offset_hist, literals_buffer, block_content_buffer, window_size) =
             match &mut state.decoder_scratch {
                 DecoderScratchKind::Flat(s) => (
                     &mut s.huf,
                     &mut s.fse,
                     &mut s.offset_hist,
                     &mut s.literals_buffer,
-                    &mut s.sequences,
                     &mut s.block_content_buffer,
                     s.buffer.window_size,
                 ),
@@ -1643,7 +1642,6 @@ impl FrameDecoder {
                     &mut s.fse,
                     &mut s.offset_hist,
                     &mut s.literals_buffer,
-                    &mut s.sequences,
                     &mut s.block_content_buffer,
                     s.buffer.window_size,
                 ),
@@ -1655,7 +1653,6 @@ impl FrameDecoder {
             fse,
             offset_hist,
             literals_buffer,
-            sequences,
             block_content_buffer,
             buffer,
         };

zstd/src/decoding/scratch.rs

@@ -1,6 +1,5 @@
 //! Structures that wrap around various decoders to make decoding easier.
 
-use super::super::blocks::sequence_section::Sequence;
 use super::buffer_backend::BufferBackend;
 use super::decode_buffer::DecodeBuffer;
 use super::ringbuffer::RingBuffer;
@@ -30,7 +29,6 @@ pub struct DecoderScratch<B: BufferBackend = RingBuffer> {
     pub offset_hist: [u32; 3],
 
     pub literals_buffer: Vec<u8>,
-    pub sequences: Vec<Sequence>,
     pub block_content_buffer: Vec<u8>,
 }
 
@@ -52,7 +50,6 @@ pub struct WorkspaceRef<'a, B: BufferBackend> {
     pub buffer: &'a mut DecodeBuffer<B>,
     pub offset_hist: &'a mut [u32; 3],
     pub literals_buffer: &'a mut Vec<u8>,
-    pub sequences: &'a mut Vec<Sequence>,
     pub block_content_buffer: &'a mut Vec<u8>,
 }
 
@@ -83,7 +80,6 @@ impl<B: BufferBackend> Workspace for DecoderScratch<B> {
             buffer: &mut self.buffer,
             offset_hist: &mut self.offset_hist,
             literals_buffer: &mut self.literals_buffer,
-            sequences: &mut self.sequences,
             block_content_buffer: &mut self.block_content_buffer,
         }
     }
@@ -115,7 +111,6 @@ pub struct DirectScratch<'o, 'p> {
     pub buffer: DecodeBuffer<super::user_slice_buf::UserSliceBackend<'o>>,
     pub offset_hist: &'p mut [u32; 3],
     pub literals_buffer: &'p mut Vec<u8>,
-    pub sequences: &'p mut Vec<Sequence>,
     pub block_content_buffer: &'p mut Vec<u8>,
 }
 
@@ -133,7 +128,6 @@ impl<'o, 'p> Workspace for DirectScratch<'o, 'p> {
             buffer: &mut self.buffer,
             offset_hist: &mut *self.offset_hist,
             literals_buffer: &mut *self.literals_buffer,
-            sequences: &mut *self.sequences,
             block_content_buffer: &mut *self.block_content_buffer,
         }
     }
@@ -147,11 +141,8 @@ impl<B: BufferBackend> DecoderScratch<B> {
             },
             fse: FSEScratch {
                 offsets: AlignedFSETable::new(MAX_OFFSET_CODE),
-                of_rle: None,
                 literal_lengths: AlignedFSETable::new(MAX_LITERAL_LENGTH_CODE),
-                ll_rle: None,
                 match_lengths: AlignedFSETable::new(MAX_MATCH_LENGTH_CODE),
-                ml_rle: None,
                 offsets_long_share: 0,
                 ddict_is_cold: false,
             },
@@ -160,14 +151,12 @@ impl<B: BufferBackend> DecoderScratch<B> {
 
             block_content_buffer: Vec::new(),
             literals_buffer: Vec::new(),
-            sequences: Vec::new(),
         }
     }
 
     pub fn reset(&mut self, window_size: usize) {
         self.offset_hist = [1, 4, 8];
         self.literals_buffer.clear();
-        self.sequences.clear();
         self.block_content_buffer.clear();
 
         // Pre-allocate the per-block scratch Vecs to `min(window_size,
@@ -214,9 +203,6 @@ impl<B: BufferBackend> DecoderScratch<B> {
         self.fse.literal_lengths.reset();
         self.fse.match_lengths.reset();
         self.fse.offsets.reset();
-        self.fse.ll_rle = None;
-        self.fse.ml_rle = None;
-        self.fse.of_rle = None;
         // Reset the cached pipeline-gate signal alongside the FSE
         // table reset — otherwise scratch reuse across frames could
         // engage the long pipeline on a new frame's Repeat-mode
@@ -274,11 +260,8 @@ impl Default for HuffmanScratch {
 
 pub struct FSEScratch {
     pub offsets: AlignedFSETable,
-    pub of_rle: Option<u8>,
     pub literal_lengths: AlignedFSETable,
-    pub ll_rle: Option<u8>,
     pub match_lengths: AlignedFSETable,
-    pub ml_rle: Option<u8>,
     /// Cached "share of offset codes strictly > LONG_OFFSET_CODE_THRESHOLD
     /// (i.e. codes ≥ 23 when the threshold is 22)" scaled to donor's
     /// `OffFSELog = 8` (256-entry reference).
@@ -308,11 +291,8 @@ impl FSEScratch {
     pub fn new() -> FSEScratch {
         FSEScratch {
             offsets: AlignedFSETable::new(MAX_OFFSET_CODE),
-            of_rle: None,
             literal_lengths: AlignedFSETable::new(MAX_LITERAL_LENGTH_CODE),
-            ll_rle: None,
             match_lengths: AlignedFSETable::new(MAX_MATCH_LENGTH_CODE),
-            ml_rle: None,
             offsets_long_share: 0,
             ddict_is_cold: false,
         }
@@ -322,9 +302,6 @@ impl FSEScratch {
         self.offsets.reinit_from(&other.offsets);
         self.literal_lengths.reinit_from(&other.literal_lengths);
         self.match_lengths.reinit_from(&other.match_lengths);
-        self.of_rle = other.of_rle;
-        self.ll_rle = other.ll_rle;
-        self.ml_rle = other.ml_rle;
         // Recompute the share from the just-copied offsets table
         // rather than trusting `other.offsets_long_share`. Two source
         // shapes produce a populated `offsets` table but a still-zero

zstd/src/decoding/seq_decoder_avx2.rs

@@ -20,17 +20,15 @@ use super::buffer_backend::BufferBackend;
 use super::decode_buffer::DecodeBuffer;
 use super::scratch::FSEScratch;
 use super::sequence_section_decoder::{
-    ADVANCE, ADVANCE_MASK, ExecSeq, compute_use_long_pipeline, decode_sequences_with_rle,
-    maybe_update_fse_tables,
+    ADVANCE, ADVANCE_MASK, ExecSeq, compute_use_long_pipeline, maybe_update_fse_tables,
 };
 use crate::bit_io::BitReaderReversed;
 use crate::blocks::sequence_section::{MAX_OFFSET_CODE, Sequence, SequencesHeader};
 use crate::common::MAX_BLOCK_SIZE;
 use crate::cpu_kernel::Avx2Kernel;
 use crate::decoding::errors::{DecodeSequenceError, DecompressBlockError, ExecuteSequencesError};
-use crate::decoding::sequence_execution::{do_offset_history, execute_sequences_fields};
+use crate::decoding::sequence_execution::do_offset_history;
 use crate::fse::SeqFSEDecoder;
-use alloc::vec::Vec;
 
 /// Textual expansion of per-sequence decode. Reads LL/ML/OF state,
 /// performs triple-bit extract via `peek_bits_triple_bmi2` (`_pext_u64`
@@ -194,10 +192,7 @@ pub(crate) unsafe fn decode_and_execute_sequences_avx2<B: BufferBackend>(
     buffer: &mut DecodeBuffer<B>,
     offset_hist: &mut [u32; 3],
     literals_buffer: &[u8],
-    rle_fallback_sequences: &mut Vec<Sequence>,
 ) -> Result<(), DecompressBlockError> {
-    rle_fallback_sequences.clear();
-
     let ddict_is_cold = fse.ddict_is_cold;
     fse.ddict_is_cold = false;
 
@@ -217,12 +212,6 @@ pub(crate) unsafe fn decode_and_execute_sequences_avx2<B: BufferBackend>(
         return Err(DecodeSequenceError::ExtraPadding { skipped_bits }.into());
     }
 
-    if fse.ll_rle.is_some() || fse.ml_rle.is_some() || fse.of_rle.is_some() {
-        decode_sequences_with_rle(section, &mut br, fse, rle_fallback_sequences)?;
-        execute_sequences_fields(buffer, literals_buffer, offset_hist, rle_fallback_sequences)?;
-        return Ok(());
-    }
-
     let mut ll_dec = SeqFSEDecoder::new(&fse.literal_lengths);
     let mut ml_dec = SeqFSEDecoder::new(&fse.match_lengths);
     let mut of_dec = SeqFSEDecoder::new(&fse.offsets);

zstd/src/decoding/seq_decoder_bmi2.rs

@@ -11,17 +11,15 @@ use super::buffer_backend::BufferBackend;
 use super::decode_buffer::DecodeBuffer;
 use super::scratch::FSEScratch;
 use super::sequence_section_decoder::{
-    ADVANCE, ADVANCE_MASK, ExecSeq, compute_use_long_pipeline, decode_sequences_with_rle,
-    maybe_update_fse_tables,
+    ADVANCE, ADVANCE_MASK, ExecSeq, compute_use_long_pipeline, maybe_update_fse_tables,
 };
 use crate::bit_io::BitReaderReversed;
 use crate::blocks::sequence_section::{MAX_OFFSET_CODE, Sequence, SequencesHeader};
 use crate::common::MAX_BLOCK_SIZE;
 use crate::cpu_kernel::Bmi2Kernel;
 use crate::decoding::errors::{DecodeSequenceError, DecompressBlockError, ExecuteSequencesError};
-use crate::decoding::sequence_execution::{do_offset_history, execute_sequences_fields};
+use crate::decoding::sequence_execution::do_offset_history;
 use crate::fse::SeqFSEDecoder;
-use alloc::vec::Vec;
 
 /// Textual decode-one body. PEXT-direct via `peek_bits_triple_bmi2`
 /// when vendor cache enables it.
@@ -166,10 +164,7 @@ pub(crate) unsafe fn decode_and_execute_sequences_bmi2<B: BufferBackend>(
     buffer: &mut DecodeBuffer<B>,
     offset_hist: &mut [u32; 3],
     literals_buffer: &[u8],
-    rle_fallback_sequences: &mut Vec<Sequence>,
 ) -> Result<(), DecompressBlockError> {
-    rle_fallback_sequences.clear();
-
     let ddict_is_cold = fse.ddict_is_cold;
     fse.ddict_is_cold = false;
 
@@ -189,12 +184,6 @@ pub(crate) unsafe fn decode_and_execute_sequences_bmi2<B: BufferBackend>(
         return Err(DecodeSequenceError::ExtraPadding { skipped_bits }.into());
     }
 
-    if fse.ll_rle.is_some() || fse.ml_rle.is_some() || fse.of_rle.is_some() {
-        decode_sequences_with_rle(section, &mut br, fse, rle_fallback_sequences)?;
-        execute_sequences_fields(buffer, literals_buffer, offset_hist, rle_fallback_sequences)?;
-        return Ok(());
-    }
-
     let mut ll_dec = SeqFSEDecoder::new(&fse.literal_lengths);
     let mut ml_dec = SeqFSEDecoder::new(&fse.match_lengths);
     let mut of_dec = SeqFSEDecoder::new(&fse.offsets);