feat(checksum): content-checksum modes, encoder toggle, off-thread one-shot hashing, wasm bindings

zstd-wasm/Cargo.toml

@@ -34,7 +34,10 @@ wasm-bindgen = "0.2"
 # go stale on the next core bump and break packaging again. The requirement
 # is purely ceremonial: `path` always wins for local/workspace builds and
 # this crate is never published to crates.io.
-structured-zstd = { version = "0", path = "../zstd", default-features = false, features = ["kernel_simd128"] }
+# `hash` enables XXH64 content-checksum support (the `ContentChecksum` modes +
+# encoder toggle exposed below). twox-hash is `no_std`, so this adds the
+# checksum code without pulling `std`.
+structured-zstd = { version = "0", path = "../zstd", default-features = false, features = ["kernel_simd128", "hash"] }
 
 # Size-tuned wasm-pack release profile: optimise for size + a wasm-opt -Oz
 # post-pass, since payload bytes are the budget for a published wasm module.

zstd-wasm/npm/index.ts

@@ -17,19 +17,41 @@
 
 import { simd } from "wasm-feature-detect";
 
+/**
+ * How the decoder treats a frame's optional XXH64 content checksum. Numeric
+ * values match the wasm-bindgen `ContentChecksum` enum.
+ */
+export enum ContentChecksum {
+  /** Skip the XXH64 pass entirely (fastest; no verification). */
+  None = 0,
+  /** Compute the checksum but do not error on a mismatch (default). */
+  EmitOnly = 1,
+  /** Compute and verify; a mismatch rejects the decode. */
+  Verify = 2,
+}
+
 /** Shape of a wasm-pack `web`-target payload glue module (simd or scalar). */
 interface Payload {
   /** Async wasm initialiser. Accepts wasm bytes / a URL, or nothing (browser). */
   // Typed `unknown`: wasm-bindgen's `web` init accepts the broad `InitInput`
   // (bytes / URL / Response / Module), and Node's `fs.readFile` returns
   // `Buffer<ArrayBufferLike>`, wider than the DOM `BufferSource` in the .d.ts.
   default: (moduleOrPath?: unknown) => Promise<unknown>;
-  compress: (data: Uint8Array, level: number) => Uint8Array;
-  decompress: (data: Uint8Array) => Uint8Array;
-  compressUsingDict: (data: Uint8Array, dict: Uint8Array, level: number) => Uint8Array;
-  decompressUsingDict: (data: Uint8Array, dict: Uint8Array) => Uint8Array;
-  ZstdDecompressStream: new () => DecompressStream;
-  ZstdCompressStream: new (level: number) => CompressStream;
+  compress: (data: Uint8Array, level: number, checksum?: boolean) => Uint8Array;
+  decompress: (data: Uint8Array, checksum?: ContentChecksum) => Uint8Array;
+  compressUsingDict: (
+    data: Uint8Array,
+    dict: Uint8Array,
+    level: number,
+    checksum?: boolean,
+  ) => Uint8Array;
+  decompressUsingDict: (
+    data: Uint8Array,
+    dict: Uint8Array,
+    checksum?: ContentChecksum,
+  ) => Uint8Array;
+  ZstdDecompressStream: new (checksum?: ContentChecksum) => DecompressStream;
+  ZstdCompressStream: new (level: number, checksum?: boolean) => CompressStream;
 }
 
 /**
@@ -42,8 +64,22 @@ interface Payload {
 export interface DecompressStream {
   /** Feed compressed bytes; returns decompressed output available so far. */
   push(chunk: Uint8Array): Uint8Array;
-  /** Signal end of input; returns the final bytes. Throws if incomplete. */
+  /**
+   * Signal end of input; returns the final bytes. Throws if incomplete, or
+   * (in {@link ContentChecksum.Verify} mode) if the content checksum is wrong.
+   */
   finish(): Uint8Array;
+  /**
+   * Content checksum stored in the frame trailer, or `undefined` if none.
+   * Meaningful after {@link DecompressStream.finish}.
+   */
+  storedChecksum(): number | undefined;
+  /**
+   * XXH64 digest computed over the output (low 32 bits), or `undefined` under
+   * {@link ContentChecksum.None} or when the frame carried no checksum. Lets
+   * callers verify manually under {@link ContentChecksum.EmitOnly}.
+   */
+  calculatedChecksum(): number | undefined;
   /** Release the underlying wasm handle. */
   free(): void;
 }
@@ -124,18 +160,25 @@ export async function init(): Promise<void> {
 export async function compress(
   data: Uint8Array,
   level: number = DEFAULT_LEVEL,
+  checksum?: boolean,
 ): Promise<Uint8Array> {
   loading ??= load();
-  return (await loading).compress(data, level);
+  return (await loading).compress(data, level, checksum);
 }
 
 /**
  * Decompress a complete Zstandard frame. Rejects if the input is not a valid,
- * complete frame.
+ * complete frame, or — when `checksum` is {@link ContentChecksum.Verify} — if
+ * the content checksum does not match. Defaults to
+ * {@link ContentChecksum.EmitOnly}; pass {@link ContentChecksum.None} to skip
+ * the XXH64 pass for speed.
  */
-export async function decompress(data: Uint8Array): Promise<Uint8Array> {
+export async function decompress(
+  data: Uint8Array,
+  checksum?: ContentChecksum,
+): Promise<Uint8Array> {
   loading ??= load();
-  return (await loading).decompress(data);
+  return (await loading).decompress(data, checksum);
 }
 
 /**
@@ -148,9 +191,10 @@ export async function compressUsingDict(
   data: Uint8Array,
   dict: Uint8Array,
   level: number = DEFAULT_LEVEL,
+  checksum?: boolean,
 ): Promise<Uint8Array> {
   loading ??= load();
-  return (await loading).compressUsingDict(data, dict, level);
+  return (await loading).compressUsingDict(data, dict, level, checksum);
 }
 
 /**
@@ -161,9 +205,10 @@ export async function compressUsingDict(
 export async function decompressUsingDict(
   data: Uint8Array,
   dict: Uint8Array,
+  checksum?: ContentChecksum,
 ): Promise<Uint8Array> {
   loading ??= load();
-  return (await loading).decompressUsingDict(data, dict);
+  return (await loading).decompressUsingDict(data, dict, checksum);
 }
 
 /**
@@ -172,9 +217,11 @@ export async function decompressUsingDict(
  * Unlike the common npm wasm zstd packages, the frame need not be fully
  * buffered — the decoder window lives on the wasm side across chunks.
  */
-export async function createDecompressStream(): Promise<DecompressStream> {
+export async function createDecompressStream(
+  checksum?: ContentChecksum,
+): Promise<DecompressStream> {
   loading ??= load();
-  return new (await loading).ZstdDecompressStream();
+  return new (await loading).ZstdDecompressStream(checksum);
 }
 
 /**
@@ -188,7 +235,8 @@ export async function createDecompressStream(): Promise<DecompressStream> {
  */
 export async function createCompressStream(
   level: number = DEFAULT_LEVEL,
+  checksum?: boolean,
 ): Promise<CompressStream> {
   loading ??= load();
-  return new (await loading).ZstdCompressStream(level);
+  return new (await loading).ZstdCompressStream(level, checksum);
 }

zstd-wasm/src/lib.rs

@@ -21,26 +21,70 @@ use structured_zstd::encoding::{
 use structured_zstd::io::{Read, Write};
 use wasm_bindgen::prelude::*;
 
+/// How the decoder treats a frame's optional content checksum. Mirrors the
+/// core `structured_zstd::decoding::ContentChecksum`.
+#[wasm_bindgen]
+#[derive(Copy, Clone, PartialEq, Eq, Debug)]
+pub enum ContentChecksum {
+    /// Skip the XXH64 pass entirely (fastest; no verification).
+    None = 0,
+    /// Compute the checksum but do not error on a mismatch (default).
+    EmitOnly = 1,
+    /// Compute and verify; a mismatch throws on decode.
+    Verify = 2,
+}
+
+/// Resolve an optional JS-supplied mode to the core enum. Defaults to `None`
+/// (skip the XXH64 pass) when omitted: this matches the package's prior
+/// behaviour (it shipped without the checksum code at all) and libzstd's
+/// `checksumFlag = 0` default, and avoids paying the XXH64 cost on a digest no
+/// wasm accessor even exposes. Callers opt into `Verify` explicitly.
+fn core_checksum(mode: Option<ContentChecksum>) -> structured_zstd::decoding::ContentChecksum {
+    use structured_zstd::decoding::ContentChecksum as Core;
+    match mode.unwrap_or(ContentChecksum::None) {
+        ContentChecksum::None => Core::None,
+        ContentChecksum::EmitOnly => Core::EmitOnly,
+        ContentChecksum::Verify => Core::Verify,
+    }
+}
+
 /// Compress `data` into a standard Zstandard frame at compression `level`.
 ///
 /// `level` follows the zstd scale: `1..=22` (higher = smaller/slower) and
 /// negative levels (`-7..=-1`) for the ultra-fast tier. The returned frame
 /// decodes in any compliant zstd decoder, including the native C library.
+///
+/// `checksum` is optional (default `true`): pass `false` to emit a frame
+/// without the trailing XXH64 content checksum (semantics of
+/// `ZSTD_c_checksumFlag`).
 #[wasm_bindgen]
-pub fn compress(data: &[u8], level: i32) -> Vec<u8> {
-    compress_slice_to_vec(data, CompressionLevel::Level(level))
+pub fn compress(data: &[u8], level: i32, checksum: Option<bool>) -> Vec<u8> {
+    if checksum.unwrap_or(true) {
+        // Default: keep the historical fast path verbatim.
+        compress_slice_to_vec(data, CompressionLevel::Level(level))
+    } else {
+        let mut enc: FrameCompressor = FrameCompressor::new(CompressionLevel::Level(level));
+        enc.set_content_checksum(false);
+        enc.compress_independent_frame(data)
+    }
 }
 
 /// Decompress a complete Zstandard frame back into its original bytes.
 ///
-/// Throws a JavaScript `Error` if the input is not a valid, complete frame.
+/// Throws a JavaScript `Error` if the input is not a valid, complete frame,
+/// or (when `checksum` is `Verify`) if the content checksum does not match.
+/// `checksum` is optional (default `EmitOnly`): pass `ContentChecksum.None`
+/// to skip the XXH64 pass for speed, or `ContentChecksum.Verify` to validate.
 #[wasm_bindgen]
-pub fn decompress(data: &[u8]) -> Result<Vec<u8>, JsError> {
+pub fn decompress(data: &[u8], checksum: Option<ContentChecksum>) -> Result<Vec<u8>, JsError> {
     // Stream the frame so the output Vec grows to fit — works for frames with
     // or without a content-size header (the fixed-size `decode_all_to_vec`
     // requires the caller to know the decoded length up front).
     let mut decoder = StreamingDecoder::new(data)
         .map_err(|err| JsError::new(&format!("structured-zstd: invalid frame: {err:?}")))?;
+    decoder
+        .decoder
+        .set_content_checksum(core_checksum(checksum));
     let mut out = Vec::new();
     decoder
         .read_to_end(&mut out)
@@ -54,8 +98,14 @@ pub fn decompress(data: &[u8]) -> Result<Vec<u8>, JsError> {
 /// small, similar payloads compress far better. The dictionary is the raw
 /// zstd dictionary blob (e.g. from `zstd --train`). Throws if it is invalid.
 #[wasm_bindgen(js_name = compressUsingDict)]
-pub fn compress_using_dict(data: &[u8], dict: &[u8], level: i32) -> Result<Vec<u8>, JsError> {
+pub fn compress_using_dict(
+    data: &[u8],
+    dict: &[u8],
+    level: i32,
+    checksum: Option<bool>,
+) -> Result<Vec<u8>, JsError> {
     let mut enc: FrameCompressor = FrameCompressor::new(CompressionLevel::Level(level));
+    enc.set_content_checksum(checksum.unwrap_or(true));
     enc.set_dictionary_from_bytes(dict)
         .map_err(|err| JsError::new(&format!("structured-zstd: invalid dictionary: {err:?}")))?;
     Ok(enc.compress_independent_frame(data))
@@ -67,12 +117,19 @@ pub fn compress_using_dict(data: &[u8], dict: &[u8], level: i32) -> Result<Vec<u
 /// dictionary the frame was compressed with. Throws on a malformed frame or a
 /// dictionary mismatch.
 #[wasm_bindgen(js_name = decompressUsingDict)]
-pub fn decompress_using_dict(data: &[u8], dict: &[u8]) -> Result<Vec<u8>, JsError> {
+pub fn decompress_using_dict(
+    data: &[u8],
+    dict: &[u8],
+    checksum: Option<ContentChecksum>,
+) -> Result<Vec<u8>, JsError> {
     let mut decoder = StreamingDecoder::new_with_dictionary_bytes(data, dict).map_err(|err| {
         JsError::new(&format!(
             "structured-zstd: dict decode init failed: {err:?}"
         ))
     })?;
+    decoder
+        .decoder
+        .set_content_checksum(core_checksum(checksum));
     let mut out = Vec::new();
     decoder.read_to_end(&mut out).map_err(|err| {
         JsError::new(&format!("structured-zstd: dict decompress failed: {err:?}"))
@@ -145,10 +202,15 @@ pub struct ZstdDecompressStream {
 
 #[wasm_bindgen]
 impl ZstdDecompressStream {
+    /// `checksum` is optional (default `EmitOnly`) and applies to the whole
+    /// stream, so set it here rather than mid-stream: `None` skips the XXH64
+    /// pass, `Verify` validates the content checksum at [`Self::finish`].
     #[wasm_bindgen(constructor)]
-    pub fn new() -> ZstdDecompressStream {
+    pub fn new(checksum: Option<ContentChecksum>) -> ZstdDecompressStream {
+        let mut decoder = FrameDecoder::new();
+        decoder.set_content_checksum(core_checksum(checksum));
         ZstdDecompressStream {
-            decoder: FrameDecoder::new(),
+            decoder,
             pending: Vec::new(),
             header_done: false,
             checksum: false,
@@ -164,21 +226,45 @@ impl ZstdDecompressStream {
     }
 
     /// Signal end of input; returns the final decompressed bytes. Throws if the
-    /// stream ended before the frame completed.
+    /// stream ended before the frame completed, or (in `Verify` mode) if the
+    /// content checksum does not match.
     pub fn finish(&mut self) -> Result<Vec<u8>, JsError> {
         let out = self.pump()?;
         if !self.finished {
             return Err(JsError::new(
                 "structured-zstd: stream ended before the frame completed",
             ));
         }
+        // The frame is fully decoded and drained (pump collects every block),
+        // so the running digest is final: validate it in Verify mode (no-op
+        // otherwise). The Display of `ChecksumMismatch` names it a corrupt
+        // frame, so the thrown JS error reads clearly on the TS side.
+        self.decoder
+            .verify_content_checksum()
+            .map_err(|err| JsError::new(&format!("structured-zstd: {err}")))?;
         Ok(out)
     }
+
+    /// The content checksum stored in the frame's 4-byte trailer, or
+    /// `undefined` if the frame carried none. Meaningful after [`Self::finish`].
+    #[wasm_bindgen(js_name = storedChecksum)]
+    pub fn stored_checksum(&self) -> Option<u32> {
+        self.decoder.get_checksum_from_data()
+    }
+
+    /// The XXH64 digest the decoder computed over the output (low 32 bits), or
+    /// `undefined` when the mode is `None` or the frame carried no checksum.
+    /// Meaningful after [`Self::finish`]; lets callers verify manually under
+    /// `EmitOnly` without enabling the throwing `Verify` mode.
+    #[wasm_bindgen(js_name = calculatedChecksum)]
+    pub fn calculated_checksum(&self) -> Option<u32> {
+        self.decoder.get_calculated_checksum()
+    }
 }
 
 impl Default for ZstdDecompressStream {
     fn default() -> Self {
-        Self::new()
+        Self::new(None)
     }
 }
 
@@ -243,14 +329,18 @@ pub struct ZstdCompressStream {
 #[wasm_bindgen]
 impl ZstdCompressStream {
     /// Open a streaming compressor at `level` (zstd scale: `1..=22`, negatives
-    /// for the ultra-fast tier).
+    /// for the ultra-fast tier). `checksum` is optional (default `true`): pass
+    /// `false` to seal the frame without a trailing content checksum.
     #[wasm_bindgen(constructor)]
-    pub fn new(level: i32) -> ZstdCompressStream {
+    pub fn new(level: i32, checksum: Option<bool>) -> ZstdCompressStream {
+        let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Level(level));
+        // Provably Ok: the encoder is fresh, so no frame header has been
+        // emitted yet (the only failure mode of this setter).
+        encoder
+            .set_content_checksum(checksum.unwrap_or(true))
+            .expect("fresh streaming encoder accepts the content-checksum toggle");
         ZstdCompressStream {
-            encoder: Some(StreamingEncoder::new(
-                Vec::new(),
-                CompressionLevel::Level(level),
-            )),
+            encoder: Some(encoder),
         }
     }