diff --git a/zstd/src/decoding/frame_decoder.rs b/zstd/src/decoding/frame_decoder.rs index 864beff07..57c19afab 100644 --- a/zstd/src/decoding/frame_decoder.rs +++ b/zstd/src/decoding/frame_decoder.rs @@ -1453,6 +1453,14 @@ impl FrameDecoder { state.bytes_read_counter } + /// Test-only: number of frames decoded through the single-copy direct + /// path (`run_direct_decode`). Lets cross-module tests assert that a + /// given decode took the decode-in-place path rather than the ring drain. + #[cfg(test)] + pub(crate) fn direct_frames(&self) -> u64 { + self.direct_frames + } + /// Whether the current frames last block has been decoded yet /// If this returns true you can call the drain* functions to get all content /// (the read() function will drain automatically if this returns true) @@ -2279,6 +2287,202 @@ impl FrameDecoder { } } + /// Whether the decoder sits at the very start of an initialised frame: + /// the header has been read (state populated) but no block has been + /// decoded and the frame is not finished. In this state the wrapped + /// source is positioned exactly after the frame header, so + /// [`Self::decode_current_frame_to_vec`] can decode the rest of the frame + /// straight from the remaining source bytes. + pub(crate) fn is_at_frame_start(&self) -> bool { + self.state + .as_ref() + .is_some_and(|s| s.block_counter == 0 && !s.frame_finished) + } + + /// Decode the CURRENT (already-initialised) frame, APPENDING the + /// decompressed bytes to `output`, and return the number appended. + /// + /// `input` must be the frame's post-header bytes (the wrapped source after + /// `init` consumed the header). Unlike [`Self::decode_all_to_vec`] this + /// neither re-reads a header nor requires the caller to pre-reserve + /// capacity: a frame that declares its content size decodes DIRECTLY into + /// freshly-grown `output` capacity via the single-copy direct path + /// ([`Self::run_direct_decode`]) — bypassing the `Ring`/`FlatBuf` → + /// `read()` drain copy the streaming loop pays — while an unsized frame + /// falls back to the window-bounded ring drain (still one copy, into + /// `output`). Backs [`StreamingDecoder`](crate::decoding::StreamingDecoder)'s + /// `read_to_end` fast path; the caller must ensure + /// [`Self::is_at_frame_start`]. + /// + /// # Errors + /// + /// Propagates any [`FrameDecoderError`] from block decode, content-size + /// mismatch, or (in `Verify` mode) checksum validation. + pub(crate) fn decode_current_frame_to_vec( + &mut self, + mut input: &[u8], + output: &mut Vec, + dict: Option<&DictionaryHandle>, + ) -> Result { + let start_len = output.len(); + // The current frame is already initialised (its header consumed by the + // caller, WITH `dict` applied if the decoder was constructed with one). + // Decode it, then decode any FOLLOWING concatenated / skippable frames + // in `input` so the whole source is consumed to EOF and nothing is + // dropped (matching `read_to_end` semantics). + self.decode_one_frame_to_vec(&mut input, output)?; + self.decode_concatenated_frames_to_vec(&mut input, output, dict)?; + Ok(output.len() - start_len) + } + + /// Initialise and decode every frame remaining in `input` (concatenated / + /// skippable), APPENDING to `output`. `input` is advanced as frames are + /// consumed; on return it is empty. Re-initialisation honours `dict`: when + /// `Some`, each following frame is initialised via + /// [`Self::init_with_dict_handle`] so a forced dictionary is preserved even + /// for frames that omit the dictionary id (plain [`Self::init`] would + /// resolve dictionaries by id only). Backs the `read_to_end` fast path (the + /// frames after the current one) and its mid-frame fallback (the frames + /// after the partially-read one). + pub(crate) fn decode_concatenated_frames_to_vec( + &mut self, + input: &mut &[u8], + output: &mut Vec, + dict: Option<&DictionaryHandle>, + ) -> Result { + let start_len = output.len(); + while !input.is_empty() { + let init_result = match dict { + Some(d) => self.init_with_dict_handle(&mut *input, d), + None => self.init(&mut *input), + }; + match init_result { + Ok(_) => {} + Err(FrameDecoderError::ReadFrameHeaderError( + crate::decoding::errors::ReadFrameHeaderError::SkipFrame { length, .. }, + )) => { + *input = input + .get(length as usize..) + .ok_or(FrameDecoderError::FailedToSkipFrame)?; + continue; + } + Err(e) => return Err(e), + } + self.decode_one_frame_to_vec(&mut *input, output)?; + } + Ok(output.len() - start_len) + } + + /// Decode the single CURRENT (already-initialised) frame, APPENDING to + /// `output`. Helper for [`Self::decode_current_frame_to_vec`]. + fn decode_one_frame_to_vec( + &mut self, + input: &mut &[u8], + output: &mut Vec, + ) -> Result { + let frame_start = output.len(); + let (content_size, fcs_declared) = { + let s = self.state.as_ref().expect("frame is initialised"); + ( + s.frame_header.frame_content_size(), + s.frame_header.fcs_declared(), + ) + }; + // Direct path: a declared, non-empty content size that FITS in `usize` + // (and whose end offset does not overflow). `usize::try_from` guards the + // 32-bit / oversized-FCS truncation; an unrepresentable size falls + // through to the window-bounded ring drain rather than allocating a + // truncated buffer that would violate `run_direct_decode`'s precondition. + // + // Plausibility gate: the direct path `resize`s `output` to the declared + // size up front, so a tiny/truncated frame declaring a huge (but + // representable) FCS would allocate + zero that whole size before the + // body is validated. zstd's per-block ceiling is MAX_BLOCK_SIZE from as + // little as ~4 input bytes, so the declared size cannot legitimately + // exceed `input.len() * (MAX_BLOCK_SIZE / 4)`. Anything larger falls + // through to the ring drain, which grows only as real bytes are produced + // and errors out cheaply on truncated input. `input` spans the remaining + // source (this frame plus any following ones), so the bound only ever + // over-permits — a legitimate frame is never forced off the direct path. + // saturating_mul is intentional: an overflow means the available input + // is so large that any representable FCS is plausible (cap = "no limit"). + const MAX_DECOMPRESSION_RATIO: usize = (crate::common::MAX_BLOCK_SIZE / 4) as usize; + if content_size > 0 + && let Ok(cs) = usize::try_from(content_size) + && cs <= input.len().saturating_mul(MAX_DECOMPRESSION_RATIO) + && let Some(frame_end) = frame_start.checked_add(cs) + { + // Reserve exactly the frame's content and decode straight into it + // (single copy, no ring). The direct path writes precisely + // `content_size` bytes (erroring otherwise), so the grown region is + // fully written. + output.resize(frame_end, 0); + // On error, drop the just-grown (zeroed) tail before propagating so + // callers never observe bytes that were never decoded. + let written = + match self.run_direct_decode(&mut *input, &mut output[frame_start..], content_size) + { + Ok(n) => n, + Err(e) => { + output.truncate(frame_start); + return Err(e); + } + }; + output.truncate(frame_start + written); + #[cfg(feature = "hash")] + self.verify_content_checksum()?; + return Ok(written); + } + // The ring-drain fallback below pre-reserves `useful_window_size()` + // (= `window.min(FCS)`), which for a single-segment frame is the + // declared FCS itself — so a truncated single-segment frame lying about + // its size would still allocate the pledged window before the body + // errors, sidestepping the direct-path gate above. Reject such a frame + // up front when its declared (FCS-bearing) window exceeds what the + // available input could plausibly produce. Frames without a declared + // size keep their window-descriptor reservation (already capped at + // `MAXIMUM_ALLOWED_WINDOW_SIZE` at init); a small-window multi-segment + // frame still falls through to the ring drain, which errors cheaply on + // the truncated body. + if fcs_declared + && let Some(state) = self.state.as_ref() + && state.useful_window_size() > input.len().saturating_mul(MAX_DECOMPRESSION_RATIO) + { + return Err(FrameDecoderError::FrameContentSizeMismatch { + declared: content_size, + produced: 0, + }); + } + // No declared size, explicit FCS=0, or an unrepresentable FCS: window- + // bounded ring drain, appended directly to `output` via + // `collect_to_writer` (no staging buffer). + loop { + self.decode_blocks(&mut *input, BlockDecodingStrategy::UptoBytes(1024 * 1024))?; + self.collect_to_writer(&mut *output) + .map_err(FrameDecoderError::FailedToDrainDecodebuffer)?; + if self.is_finished() { + // Final flush of the retained window tail. + self.collect_to_writer(&mut *output) + .map_err(FrameDecoderError::FailedToDrainDecodebuffer)?; + break; + } + } + let produced = (output.len() - frame_start) as u64; + // A declared content size MUST match what the body produced — otherwise + // accept the same corrupt frames `decode_all_impl` rejects (e.g. an + // explicit FCS=0 whose body emits bytes). Use `fcs_declared()` so an + // on-wire FCS=0 is validated, while an unknown size is not. + if fcs_declared && produced != content_size { + return Err(FrameDecoderError::FrameContentSizeMismatch { + declared: content_size, + produced, + }); + } + #[cfg(feature = "hash")] + self.verify_content_checksum()?; + Ok(produced as usize) + } + /// Default-feature decode_all_impl: no visitor parameter so the /// no-lsm build's call surface and codegen are byte-identical to /// the pre-#172 implementation. Compiles only when `lsm` is OFF. @@ -3817,6 +4021,79 @@ mod tests { ); } + #[test] + fn implausible_content_size_skips_eager_alloc_direct_path() { + // Adversarial frame: a 1 KiB window (small ring) but a declared + // content size of 4 MiB, followed by a truncated raw block. The + // direct path would `resize` the caller's Vec to the pledged 4 MiB + // (allocating + zeroing it) BEFORE the truncated body is validated. + // The gate must reject the implausible size (4 MiB cannot come from + // 3 compressed bytes) and fall through to the window-bounded ring + // drain, which errors without ever allocating the pledged size. + // + // Hand-built so the declared size is fully decoupled from the real + // (tiny) input — the encoder always writes a truthful FCS. + let frame: &[u8] = &[ + 0x28, 0xB5, 0x2F, 0xFD, // magic + 0x80, // FHD: multi-segment, 4-byte FCS field, no dict + 0x00, // window descriptor -> 1 KiB window + 0x00, 0x00, 0x40, 0x00, // FCS = 4 MiB + 0x21, 0x03, 0x00, // raw block header: last, size 100, no body + ]; + + let mut dec = FrameDecoder::new(); + let mut src = frame; + dec.init(&mut src).expect("header must parse"); + // `src` now points past the header at the truncated 3-byte block. + let mut out = Vec::new(); + let err = dec.decode_current_frame_to_vec(src, &mut out, None); + assert!( + err.is_err(), + "truncated body must fail regardless of decode path" + ); + assert_eq!( + dec.direct_frames, 0, + "implausible FCS must NOT take the eager-alloc direct path" + ); + } + + #[test] + fn implausible_single_segment_fcs_rejected_before_window_reservation() { + // Single-segment adversarial frame: the window equals the declared + // content size (4 MiB) by definition, so the fallback ring drain would + // pre-reserve that whole window via `useful_window_size()` before the + // truncated body errors — the multi-segment gate test does not cover + // this. The implausible size (4 MiB cannot come from 3 compressed + // bytes) must be rejected up front with a content-size error, NOT a + // block-body error after the reservation. + let frame: &[u8] = &[ + 0x28, 0xB5, 0x2F, 0xFD, // magic + 0xA0, // FHD: single-segment, 4-byte FCS field + 0x00, 0x00, 0x40, 0x00, // FCS = 4 MiB (== window for single-segment) + 0x21, 0x03, 0x00, // raw block header: last, size 100, no body + ]; + + let mut dec = FrameDecoder::new(); + let mut src = frame; + dec.init(&mut src).expect("header must parse"); + let mut out = Vec::new(); + let err = dec + .decode_current_frame_to_vec(src, &mut out, None) + .expect_err("implausible single-segment FCS must be rejected"); + match err { + super::FrameDecoderError::FrameContentSizeMismatch { declared, .. } => { + assert_eq!(declared, 4 * 1024 * 1024); + } + other => panic!( + "expected early FrameContentSizeMismatch (no window reservation), got {other:?}" + ), + } + assert_eq!( + dec.direct_frames, 0, + "implausible FCS must not take the eager-alloc direct path" + ); + } + #[cfg(feature = "lsm")] mod expect_validation { use super::*; diff --git a/zstd/src/decoding/streaming_decoder.rs b/zstd/src/decoding/streaming_decoder.rs index d608838cc..837a26fd7 100644 --- a/zstd/src/decoding/streaming_decoder.rs +++ b/zstd/src/decoding/streaming_decoder.rs @@ -20,16 +20,18 @@ use crate::io::{Error, Read}; /// [FrameDecoder::decode_blocks] repeatedly to decode the entire frame. /// /// ## Caveat -/// [StreamingDecoder] expects the underlying stream to only contain a single frame, -/// yet the specification states that a single archive may contain multiple frames. +/// Plain `read` / `read_exact` operate on the single frame this decoder was +/// initialised with: they do not advance into following frames. `read_to_end`, +/// by contrast, is specialised to consume a finite source to EOF, decoding +/// concatenated frames and skipping skippable frames along the way. /// -/// To decode all the frames in a finite stream, the calling code needs to recreate -/// the instance of the decoder and handle +/// To recover the bytes that follow one frame WITHOUT consuming the rest of the +/// source, recreate the decoder manually and handle /// [crate::decoding::errors::ReadFrameHeaderError::SkipFrame] /// errors by skipping forward the `length` amount of bytes, see /// /// ```no_run -/// // `read_to_end` is not implemented by the no_std implementation. +/// // `File` is std-only; `read_to_end` itself is available under no_std too. /// #[cfg(feature = "std")] /// { /// use std::fs::File; @@ -46,6 +48,12 @@ use crate::io::{Error, Read}; pub struct StreamingDecoder> { pub decoder: DEC, source: READ, + /// Dictionary the decoder was constructed with, if any. Retained so the + /// `read_to_end` paths can re-initialise FOLLOWING concatenated frames with + /// the same forced dictionary (a plain re-init resolves dictionaries by + /// frame id only and would lose a forced dict for frames omitting the id). + /// Cheap to hold: `DictionaryHandle` is an `Arc`/`Rc` handle. + dict: Option, } impl> StreamingDecoder { @@ -54,7 +62,11 @@ impl> StreamingDecoder { mut decoder: DEC, ) -> Result, FrameDecoderError> { decoder.borrow_mut().init(&mut source)?; - Ok(StreamingDecoder { decoder, source }) + Ok(StreamingDecoder { + decoder, + source, + dict: None, + }) } } @@ -64,7 +76,11 @@ impl StreamingDecoder { ) -> Result, FrameDecoderError> { let mut decoder = FrameDecoder::new(); decoder.init(&mut source)?; - Ok(StreamingDecoder { decoder, source }) + Ok(StreamingDecoder { + decoder, + source, + dict: None, + }) } /// Create a streaming decoder using a pre-parsed dictionary handle. @@ -82,7 +98,11 @@ impl StreamingDecoder { ) -> Result, FrameDecoderError> { let mut decoder = FrameDecoder::new(); decoder.init_with_dict_handle(&mut source, dict)?; - Ok(StreamingDecoder { decoder, source }) + Ok(StreamingDecoder { + decoder, + source, + dict: Some(dict.clone()), + }) } /// Create a streaming decoder using a serialized dictionary blob. @@ -211,6 +231,126 @@ impl> Read for StreamingDecoder `RingBuffer` -> copy into the + /// caller buffer), buffer the (compressed, hence small) source and decode + /// STRAIGHT into `output`'s spare capacity via the single-copy direct path, + /// pre-sized from the frame's declared content size. Only taken when the + /// decoder is at a frame boundary (nothing partially decoded / undrained); + /// otherwise it falls back to the generic grow-and-`read` loop so a caller + /// that mixed `read` with `read_to_end` still gets correct output. + /// + /// Per the `Read::read_to_end` contract this consumes the source to EOF: if + /// the stream holds several concatenated frames they are ALL decoded (and + /// skippable frames skipped). To recover bytes that follow a single frame, + /// use `read` plus the + /// [`SkipFrame`](crate::decoding::errors::ReadFrameHeaderError::SkipFrame) + /// recreate-the-decoder pattern instead. + #[cfg(feature = "std")] + fn read_to_end(&mut self, output: &mut alloc::vec::Vec) -> Result { + let start_total = output.len(); + // `new()` already read the frame header, so the fast path applies when + // the decoder sits at the start of that frame with nothing decoded yet. + let at_start = { + let d = self.decoder.borrow_mut(); + d.is_at_frame_start() && d.can_collect() == 0 + }; + // Clone the (cheap Arc/Rc) dict handle out so the `decoder` borrow below + // does not conflict with borrowing `self.dict`. + let dict = self.dict.clone(); + if at_start { + let mut compressed = alloc::vec::Vec::new(); + self.source.read_to_end(&mut compressed)?; + self.decoder + .borrow_mut() + .decode_current_frame_to_vec(&compressed, output, dict.as_ref()) + .map_err(Error::other)?; + return Ok(output.len() - start_total); + } + // Mid-frame fallback: drain the partially-read CURRENT frame through the + // generic path, then decode any FOLLOWING concatenated frames so + // read_to_end still consumes the source to true EOF. + loop { + let start = output.len(); + output.resize(start + MAX_BLOCK_SIZE as usize, 0); + // On error, drop the just-grown (zeroed) tail before propagating so + // the caller never observes bytes that were never decoded. + let n = match self.read(&mut output[start..]) { + Ok(n) => n, + Err(e) => { + output.truncate(start); + return Err(e); + } + }; + output.truncate(start + n); + if n == 0 { + break; + } + } + // Current frame fully drained; `source` is positioned at the next frame. + let mut rest = alloc::vec::Vec::new(); + self.source.read_to_end(&mut rest)?; + if !rest.is_empty() { + let mut input = rest.as_slice(); + self.decoder + .borrow_mut() + .decode_concatenated_frames_to_vec(&mut input, output, dict.as_ref()) + .map_err(Error::other)?; + } + Ok(output.len() - start_total) + } + + /// no_std counterpart of the decode-in-place `read_to_end` fast path above + /// (the no_std `Read::read_to_end` returns `()` instead of the byte count). + #[cfg(not(feature = "std"))] + fn read_to_end(&mut self, output: &mut alloc::vec::Vec) -> Result<(), Error> { + let at_start = { + let d = self.decoder.borrow_mut(); + d.is_at_frame_start() && d.can_collect() == 0 + }; + // Cheap Arc/Rc clone so the `decoder` borrow does not conflict with + // borrowing `self.dict`. + let dict = self.dict.clone(); + if at_start { + let mut compressed = alloc::vec::Vec::new(); + self.source.read_to_end(&mut compressed)?; + self.decoder + .borrow_mut() + .decode_current_frame_to_vec(&compressed, output, dict.as_ref()) + .map_err(|e| Error::new(ErrorKind::Other, alloc::boxed::Box::new(e)))?; + return Ok(()); + } + // Mid-frame fallback: drain the partial CURRENT frame, then decode the + // FOLLOWING concatenated frames so the source is consumed to true EOF. + loop { + let start = output.len(); + output.resize(start + MAX_BLOCK_SIZE as usize, 0); + // On error, drop the just-grown (zeroed) tail before propagating so + // the caller never observes bytes that were never decoded. + let n = match self.read(&mut output[start..]) { + Ok(n) => n, + Err(e) => { + output.truncate(start); + return Err(e); + } + }; + output.truncate(start + n); + if n == 0 { + break; + } + } + let mut rest = alloc::vec::Vec::new(); + self.source.read_to_end(&mut rest)?; + if !rest.is_empty() { + let mut input = rest.as_slice(); + self.decoder + .borrow_mut() + .decode_concatenated_frames_to_vec(&mut input, output, dict.as_ref()) + .map_err(|e| Error::new(ErrorKind::Other, alloc::boxed::Box::new(e)))?; + } + Ok(()) + } } #[cfg(test)] @@ -269,4 +409,275 @@ mod tests { .expect_err("deferred checksum mismatch must surface on the terminating read"); assert_eq!(err.kind(), ErrorKind::Other); } + + /// A fresh `read_to_end` must take the single-copy decode-in-place path + /// (FCS-declared frame decoded straight into the output `Vec`, no ring + /// drain) AND reproduce the payload byte-for-byte. + #[cfg(feature = "std")] + #[test] + fn read_to_end_decode_in_place_matches_and_takes_direct_path() { + use crate::encoding::{CompressionLevel, FrameCompressor}; + use alloc::vec::Vec; + + let payload: Vec = (0..20_000u32) + .map(|i| (i.wrapping_mul(2654435761) >> 24) as u8) + .collect(); + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_source(payload.as_slice()); + let mut compressed = Vec::new(); + compressor.set_drain(&mut compressed); + compressor.compress(); + + let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); + let mut out = Vec::new(); + let n = decoder.read_to_end(&mut out).unwrap(); + assert_eq!(n, payload.len()); + assert_eq!(out, payload); + // FrameCompressor declares FCS, so the fresh fast path used the direct + // (decode-in-place) route, not the ring drain. + assert_eq!(decoder.decoder.direct_frames(), 1); + } + + /// `read_to_end` after a partial `read` must still produce the full + /// payload. The decoder is mid-frame, so the fast path is skipped and the + /// generic grow-and-drain fallback runs (no direct frame). + #[cfg(feature = "std")] + #[test] + fn read_to_end_after_partial_read_is_complete() { + use crate::encoding::{CompressionLevel, FrameCompressor}; + use alloc::vec; + use alloc::vec::Vec; + + let payload: Vec = (0..20_000u32).map(|i| (i & 0xFF) as u8).collect(); + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_source(payload.as_slice()); + let mut compressed = Vec::new(); + compressor.set_drain(&mut compressed); + compressor.compress(); + + let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); + let mut head = vec![0u8; 4096]; + let got = decoder.read(&mut head).unwrap(); + assert!(got > 0 && got <= head.len()); + + let mut out = Vec::new(); + out.extend_from_slice(&head[..got]); + decoder.read_to_end(&mut out).unwrap(); + assert_eq!(out, payload); + // Mid-frame entry → fallback path, never the direct route. + assert_eq!(decoder.decoder.direct_frames(), 0); + } + + /// `read_to_end` reads the WHOLE source to EOF: a stream of concatenated + /// frames must decode every frame, not just the first. (The fast path + /// buffers the whole source, so dropping the trailing frame would lose + /// data.) + #[cfg(feature = "std")] + #[test] + fn read_to_end_decodes_all_concatenated_frames() { + use crate::encoding::{CompressionLevel, compress_slice_to_vec}; + use alloc::vec::Vec; + + let a: Vec = (0..5000u32).map(|i| (i & 0xFF) as u8).collect(); + let b: Vec = (0..3000u32) + .map(|i| ((i.wrapping_mul(7)) & 0xFF) as u8) + .collect(); + let mut stream = compress_slice_to_vec(&a, CompressionLevel::Level(3)); + stream.extend_from_slice(&compress_slice_to_vec(&b, CompressionLevel::Level(3))); + + let mut decoder = StreamingDecoder::new(stream.as_slice()).unwrap(); + let mut out = Vec::new(); + decoder.read_to_end(&mut out).unwrap(); + + let mut expected = a.clone(); + expected.extend_from_slice(&b); + assert_eq!(out, expected); + // Both FCS-declared frames took the direct path. + assert_eq!(decoder.decoder.direct_frames(), 2); + } + + /// `read_to_end` after a partial `read` must STILL consume the source to + /// EOF across concatenated frames, not stop at the current frame's end. The + /// partial read forces the mid-frame fallback path; with two concatenated + /// frames the fallback must finish frame 1, then advance through frame 2. + #[cfg(feature = "std")] + #[test] + fn read_to_end_after_partial_read_decodes_all_concatenated_frames() { + use crate::encoding::{CompressionLevel, compress_slice_to_vec}; + use alloc::vec; + use alloc::vec::Vec; + + let a: Vec = (0..6000u32).map(|i| (i & 0xFF) as u8).collect(); + let b: Vec = (0..4000u32) + .map(|i| ((i.wrapping_mul(11)) & 0xFF) as u8) + .collect(); + let mut stream = compress_slice_to_vec(&a, CompressionLevel::Level(3)); + stream.extend_from_slice(&compress_slice_to_vec(&b, CompressionLevel::Level(3))); + + let mut decoder = StreamingDecoder::new(stream.as_slice()).unwrap(); + // Partial read of frame 1 → mid-frame, so read_to_end takes the fallback. + let mut head = vec![0u8; 2048]; + let got = decoder.read(&mut head).unwrap(); + assert!(got > 0 && got <= head.len()); + + let mut out = Vec::new(); + out.extend_from_slice(&head[..got]); + decoder.read_to_end(&mut out).unwrap(); + + let mut expected = a.clone(); + expected.extend_from_slice(&b); + assert_eq!( + out, expected, + "fallback path must decode frame 2 too, not stop at frame 1 EOF" + ); + } + + /// `read_to_end` on a stream of concatenated DICTIONARY frames must decode + /// every frame WITH the dictionary the decoder was constructed with. The + /// fast-path concatenated loop re-initialises following frames, and a plain + /// re-init resolves dictionaries by frame id only — losing the forced + /// dictionary for frames that omit (or can't resolve) the id. + #[cfg(feature = "std")] + #[test] + fn read_to_end_concatenated_dict_frames_decode_with_dictionary() { + use crate::encoding::{CompressionLevel, FrameCompressor}; + use alloc::vec::Vec; + + let dict_raw = include_bytes!("../../dict_tests/dictionary"); + let compress_with_dict = |payload: &[u8]| -> Vec { + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor + .set_dictionary_from_bytes(dict_raw) + .expect("dict load"); + compressor.set_source(payload); + let mut compressed = Vec::new(); + compressor.set_drain(&mut compressed); + compressor.compress(); + compressed + }; + + let a = b"first dictionary-compressed frame payload".to_vec(); + let b = b"second dictionary-compressed frame payload".to_vec(); + let mut stream = compress_with_dict(&a); + stream.extend_from_slice(&compress_with_dict(&b)); + + let mut decoder = + StreamingDecoder::new_with_dictionary_bytes(stream.as_slice(), dict_raw).unwrap(); + let mut out = Vec::new(); + decoder + .read_to_end(&mut out) + .expect("both dict frames must decode with the forced dictionary"); + + let mut expected = a.clone(); + expected.extend_from_slice(&b); + assert_eq!(out, expected); + } + + /// A direct-path decode error must NOT leave non-decoded bytes in `output`. + /// The fast path resizes `output` to the declared content size before + /// decoding; if decode fails, the enlarged (zeroed) tail must be truncated + /// away so callers never observe bytes that were never decoded. + #[cfg(feature = "std")] + #[test] + fn read_to_end_truncates_output_on_direct_decode_error() { + use crate::encoding::{CompressionLevel, FrameCompressor}; + use alloc::vec::Vec; + + let payload: Vec = (0..5000u32).map(|i| (i & 0xFF) as u8).collect(); + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_source(payload.as_slice()); + let mut compressed = Vec::new(); + compressor.set_drain(&mut compressed); + compressor.compress(); + // Truncate the block bytes (the FCS-bearing header at the front stays + // intact) so the header parses but the direct-path block decode hits a + // premature end → error after `output` was already resized. + compressed.truncate(compressed.len() - 40); + + let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); + let mut out = b"SENTINEL".to_vec(); + let result = decoder.read_to_end(&mut out); + assert!(result.is_err(), "truncated block must fail the decode"); + assert_eq!( + out, b"SENTINEL", + "failed direct decode must not append non-decoded bytes to output" + ); + } + + /// The mid-frame fallback grows `output` by `MAX_BLOCK_SIZE` before each + /// `self.read`. When that read errors (truncated current frame), the grown + /// zero-filled tail must be truncated away before the error propagates, so + /// the caller never observes `MAX_BLOCK_SIZE` worth of bytes that were never + /// decoded. + #[cfg(feature = "std")] + #[test] + fn read_to_end_truncates_output_on_midframe_fallback_error() { + use crate::encoding::{CompressionLevel, CompressionParameters, FrameCompressor}; + use alloc::vec; + use alloc::vec::Vec; + + // Incompressible payload with a window (128 KiB) SMALLER than the input, + // so the frame holds several blocks and bytes become collectable while + // the frame is still mid-decode. Without a sub-input window the decoder + // retains the whole input until the frame finishes, and a partial read + // could only ever finish or error, never leave a truncated remainder for + // the fallback to trip on. + let payload: Vec = (0..320_000u32) + .map(|i| (i.wrapping_mul(2654435761) >> 24) as u8) + .collect(); + let params = CompressionParameters::builder(CompressionLevel::Default) + .window_log(17) + .build() + .expect("window_log within bounds"); + let mut compressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_parameters(¶ms); + compressor.set_source(payload.as_slice()); + let mut compressed = Vec::new(); + compressor.set_drain(&mut compressed); + compressor.compress(); + // Truncate the tail so the final block decode fails partway through. + compressed.truncate(compressed.len() - 40); + + let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); + // A partial `read` first: leaves the decoder mid-frame so `read_to_end` + // takes the grow-and-drain fallback (not the decode-in-place fast path). + let mut head = vec![0u8; 4096]; + let got = decoder.read(&mut head).unwrap(); + assert!(got > 0); + + let mut out = Vec::new(); + out.extend_from_slice(&head[..got]); + let result = decoder.read_to_end(&mut out); + assert!( + result.is_err(), + "truncated current frame must fail the decode" + ); + assert!( + out.len() <= payload.len(), + "failed fallback read must not leave a zero-filled tail (len {} > payload {})", + out.len(), + payload.len() + ); + assert_eq!( + out.as_slice(), + &payload[..out.len()], + "decoded prefix must match the payload, with no appended non-decoded bytes" + ); + } + + /// An empty (`Frame_Content_Size = 0`) frame decodes to nothing through the + /// `read_to_end` fast path — the declared-size validation accepts the valid + /// case (produced == 0) instead of erroring. + #[cfg(feature = "std")] + #[test] + fn read_to_end_empty_frame_decodes_to_empty() { + use crate::encoding::{CompressionLevel, compress_slice_to_vec}; + use alloc::vec::Vec; + + let compressed = compress_slice_to_vec(&[], CompressionLevel::Level(3)); + let mut decoder = StreamingDecoder::new(compressed.as_slice()).unwrap(); + let mut out = Vec::new(); + decoder.read_to_end(&mut out).unwrap(); + assert!(out.is_empty()); + } }