From d7c376b51f4ffb6fe40472668317c501e3c075ad Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Tue, 9 Jun 2026 02:52:58 +0300 Subject: [PATCH 1/3] feat(c-api): C ABI core, a libzstd-compatible drop-in (simple/context/error/frame/dict) New c-api/ workspace member building a cdylib (SONAME libzstd.so.1) + staticlib over the pure-Rust codec, with verbatim-vendored upstream v1.5.7 headers and hand-written extern "C" wrappers. - Simple: compress/decompress/compressBound/getFrameContentSize/ findFrameCompressedSize/version/level bounds - Context: create/free/compress/decompress CCtx+DCtx, sizeof (decoder workspace included), panic-safe decoder reset - Error: ZSTD_ErrorCode, size_t encoding, decoder-error mapping, isError/getErrorCode/getErrorName/getErrorString (primitive-typed across FFI) - Frame inspection: frameHeaderSize, getFrameHeader(_advanced) with layout-locked ZSTD_FrameHeader, findDecompressedSize, decompressBound - Dictionary: ZDICT_trainFromBuffer/finalizeDictionary/getDictID/ getDictHeaderSize/isError/getErrorName over the FastCOVER builder - Content-checksum: no emission by default (upstream ZSTD_c_checksumFlag = 0) across the C ABI and the wasm/npm bindings; decode verifies when present - Public codec helpers the wrappers need (compress_bound, frame-size/header inspection, FrameDecoder::workspace_size); oversized-block rejection in the frame-size walk; ContentChecksum modes wired through wasm bindings Closes #126 --- .github/workflows/ci.yml | 114 + Cargo.toml | 2 +- c-api/Cargo.toml | 40 + c-api/build.rs | 61 + c-api/include/zdict.h | 481 ++++ c-api/include/zstd.h | 3198 +++++++++++++++++++++++++ c-api/include/zstd_errors.h | 107 + c-api/src/context.rs | 188 ++ c-api/src/dict.rs | 197 ++ c-api/src/error.rs | 246 ++ c-api/src/ffi.rs | 31 + c-api/src/frame.rs | 252 ++ c-api/src/lib.rs | 40 + c-api/src/simple.rs | 190 ++ c-api/src/tests.rs | 400 ++++ c-api/tests/c_consumer.c | 90 + zstd-wasm/npm/index.ts | 15 +- zstd-wasm/src/lib.rs | 38 +- zstd/src/decoding/decode_buffer.rs | 8 + zstd/src/decoding/frame_decoder.rs | 33 + zstd/src/decoding/mod.rs | 515 ++++ zstd/src/decoding/scratch.rs | 30 + zstd/src/encoding/frame_compressor.rs | 5 +- zstd/src/encoding/mod.rs | 69 + zstd/src/fse/fse_decoder.rs | 8 + zstd/src/huff0/huff0_decoder.rs | 11 + 26 files changed, 6342 insertions(+), 27 deletions(-) create mode 100644 c-api/Cargo.toml create mode 100644 c-api/build.rs create mode 100644 c-api/include/zdict.h create mode 100644 c-api/include/zstd.h create mode 100644 c-api/include/zstd_errors.h create mode 100644 c-api/src/context.rs create mode 100644 c-api/src/dict.rs create mode 100644 c-api/src/error.rs create mode 100644 c-api/src/ffi.rs create mode 100644 c-api/src/frame.rs create mode 100644 c-api/src/lib.rs create mode 100644 c-api/src/simple.rs create mode 100644 c-api/src/tests.rs create mode 100644 c-api/tests/c_consumer.c diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c17a022b2..59e7e5e30 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,6 +140,120 @@ jobs: exit 1 fi + c-abi: + # Builds the libzstd-compatible C ABI front end and verifies it is a real + # drop-in: vendored headers match upstream verbatim, the cdylib advertises + # SONAME libzstd.so.1, every declared symbol is exported, and a genuine C + # consumer links + round-trips through the vendored header. + needs: lint + timeout-minutes: 12 + runs-on: ubuntu-latest + env: + UPSTREAM_TAG: v1.5.7 + steps: + - uses: actions/checkout@v6 + with: + # Build + test + header-diff job (no push); don't leave the token + # in the checkout's git config. + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: c-abi + - name: Clippy (c-api) + run: cargo clippy -p structured-zstd-c --all-targets -- -D warnings + - name: Unit + ABI tests + run: cargo test -p structured-zstd-c + - name: Vendored headers match upstream verbatim + # The headers are copied byte-for-byte from the pinned upstream tag; a + # diff means someone edited a vendored header (forbidden) or the pin + # moved without re-vendoring. + run: | + set -euo pipefail + base="https://raw.githubusercontent.com/facebook/zstd/${UPSTREAM_TAG}/lib" + for h in zstd.h zdict.h zstd_errors.h; do + curl -fsSL "${base}/${h}" -o "/tmp/${h}.upstream" + if ! diff -u "/tmp/${h}.upstream" "c-api/include/${h}"; then + echo "::error::c-api/include/${h} diverges from upstream ${UPSTREAM_TAG}" + exit 1 + fi + done + echo "vendored headers identical to upstream ${UPSTREAM_TAG}" + - name: Build cdylib + staticlib + run: cargo build -p structured-zstd-c + - name: SONAME is libzstd.so.1 + run: | + set -euo pipefail + so=target/debug/libstructured_zstd.so + soname=$(readelf -d "$so" | sed -n 's/.*SONAME.*\[\(.*\)\]/\1/p') + echo "SONAME=$soname" + test "$soname" = "libzstd.so.1" + - name: All declared symbols are exported + run: | + set -euo pipefail + so=target/debug/libstructured_zstd.so + exported=$(nm -D --defined-only "$so" | awk '{print $NF}') + missing=0 + for sym in \ + ZSTD_compress ZSTD_decompress ZSTD_compressBound \ + ZSTD_getFrameContentSize ZSTD_findFrameCompressedSize \ + ZSTD_isError ZSTD_getErrorCode ZSTD_getErrorName ZSTD_getErrorString \ + ZSTD_minCLevel ZSTD_maxCLevel ZSTD_defaultCLevel \ + ZSTD_versionNumber ZSTD_versionString \ + ZSTD_createCCtx ZSTD_freeCCtx ZSTD_createDCtx ZSTD_freeDCtx \ + ZSTD_compressCCtx ZSTD_decompressDCtx ZSTD_sizeof_CCtx ZSTD_sizeof_DCtx \ + ZSTD_frameHeaderSize ZSTD_getFrameHeader ZSTD_getFrameHeader_advanced \ + ZSTD_findDecompressedSize ZSTD_decompressBound \ + ZDICT_trainFromBuffer ZDICT_finalizeDictionary ZDICT_getDictID \ + ZDICT_getDictHeaderSize ZDICT_isError ZDICT_getErrorName; do + if ! grep -qx "$sym" <<<"$exported"; then + echo "::error::symbol $sym not exported from $so" + missing=1 + fi + done + test "$missing" -eq 0 + - name: pkg-config reports upstream version + run: | + set -euo pipefail + # Validate every discovered libzstd.pc, not just the first hit: a + # restored cache can leave a stale file ahead of the fresh build. + mapfile -t pcs < <(find target -type f -name libzstd.pc) + test "${#pcs[@]}" -gt 0 + for pc in "${pcs[@]}"; do + echo "checking $pc"; cat "$pc" + grep -qx "Version: 1.5.7" "$pc" + done + - name: Real C consumer links + round-trips + run: | + set -euo pipefail + # The cdylib advertises SONAME libzstd.so.1, so a consumer records a + # NEEDED dependency on that name; provide it via a symlink. Also + # expose the canonical `libzstd.so` link name and link with `-lzstd`, + # exactly the path a real drop-in C consumer uses. + ln -sf libstructured_zstd.so target/debug/libzstd.so.1 + ln -sf libstructured_zstd.so target/debug/libzstd.so + cc -std=c11 -Wall -Wextra -Ic-api/include c-api/tests/c_consumer.c \ + -Ltarget/debug -lzstd -o /tmp/c_consumer + LD_LIBRARY_PATH=target/debug /tmp/c_consumer + - name: musl static drop-in builds + exports symbols + # musl is a std target (not no-std); its default `+crt-static` profile + # makes the static archive `libstructured_zstd.a` the canonical drop-in + # for Alpine / fully-static binaries (the cdylib is dropped under + # crt-static, which is expected). Verify the archive builds and carries + # the exported wrappers. + run: | + set -euo pipefail + rustup target add x86_64-unknown-linux-musl + cargo build -p structured-zstd-c --target x86_64-unknown-linux-musl + a=target/x86_64-unknown-linux-musl/debug/libstructured_zstd.a + test -f "$a" + for sym in ZSTD_compress ZSTD_decompress ZSTD_versionNumber ZSTD_getFrameHeader; do + nm "$a" | grep -qE " T ${sym}$" || { echo "::error::$sym missing from musl staticlib"; exit 1; } + done + echo "musl staticlib OK ($(du -h "$a" | cut -f1))" + test: needs: lint timeout-minutes: 15 diff --git a/Cargo.toml b/Cargo.toml index ba1013b9c..59446b82f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "3" -members = ["zstd", "cli", "zstd-wasm"] +members = ["zstd", "cli", "zstd-wasm", "c-api"] # Bench profile inherits from release but overrides LTO/codegen-units so # the perf measurement reflects full cross-crate inlining. We deliberately diff --git a/c-api/Cargo.toml b/c-api/Cargo.toml new file mode 100644 index 000000000..a632d6f30 --- /dev/null +++ b/c-api/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "structured-zstd-c" +version = "0.0.33" +rust-version = "1.92" +authors = ["Structured World Foundation "] +edition = "2024" +license = "Apache-2.0" +homepage = "https://github.com/structured-world/structured-zstd" +repository = "https://github.com/structured-world/structured-zstd" +description = "C ABI (libzstd-compatible) front end for the structured-zstd pure-Rust codec." +# The cdylib is consumed as a system library (drop-in for libzstd.so.1), not +# from crates.io, so it stays out of the release-plz publish set for now. +publish = false + +[lib] +# The output artifacts are `libstructured_zstd.so` / `.a`; the SONAME is +# rewritten to `libzstd.so.1` by build.rs so a consumer linked against +# upstream libzstd resolves against this build with no rebuild. +name = "structured_zstd" +# `lib` (rlib) is added alongside the C artifacts so the crate's own test +# harness can link the wrappers; the shipped artifacts are the cdylib + staticlib. +crate-type = ["lib", "cdylib", "staticlib"] + +# No `[features]` / `std` contract by design: this is a host-only cdylib/ +# staticlib whose `extern "C"` wrappers guard the boundary with +# `std::panic::catch_unwind` (a Rust panic must not unwind into C), so the +# crate cannot build without std. A default-on `std` feature would advertise a +# `--no-default-features` build that fails to compile. The no_std + alloc +# surface lives in the `codec` crate below. + +[dependencies] +# The codec is imported under the alias `codec` rather than its own name: this +# lib is itself named `structured_zstd` (so the artifact is `libstructured_zstd.so`), +# and a same-named extern would make rustdoc's doctest harness see two +# `structured_zstd` rlibs (E0464). The alias keeps the artifact name while +# giving the dependency a distinct crate name in this crate. +# Default features pull in std + hash (checksums/LDM) + the per-CPU kernel +# tiers — the C ABI is a host library, so std is always available here. +# `dict_builder` adds the dictionary-training surface backing the ZDICT_* C API. +codec = { path = "../zstd", package = "structured-zstd", features = ["dict_builder"] } diff --git a/c-api/build.rs b/c-api/build.rs new file mode 100644 index 000000000..6b88bbafe --- /dev/null +++ b/c-api/build.rs @@ -0,0 +1,61 @@ +//! Build glue for the C ABI front end: +//! - emits the SONAME / install-name a consumer of upstream `libzstd` +//! expects, so the cdylib is a true drop-in; +//! - installs the vendored headers under `OUT_DIR/include` for packaging; +//! - generates `libzstd.pc` reporting the vendored upstream version. + +use std::env; +use std::fs; +use std::path::PathBuf; + +/// Upstream zstd release the vendored headers + reported pkg-config version +/// track. Keep in sync with the tracking comment at the top of each header. +const UPSTREAM_VERSION: &str = "1.5.7"; + +fn main() { + let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap_or_default(); + + // A binary linked against upstream `libzstd.so.1` resolves by SONAME, so + // the cdylib must advertise that exact name to be substitutable. + match target_os.as_str() { + "linux" | "android" => { + println!("cargo:rustc-cdylib-link-arg=-Wl,-soname,libzstd.so.1"); + } + "macos" | "ios" => { + println!("cargo:rustc-cdylib-link-arg=-Wl,-install_name,libzstd.1.dylib"); + } + _ => {} + } + + let out_dir = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR set by cargo")); + let include_src = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("include"); + + // Stage the vendored headers next to the built library so downstream + // packaging can install them under /usr/include alongside the .so. + let include_dst = out_dir.join("include"); + fs::create_dir_all(&include_dst).expect("create OUT_DIR/include"); + for header in ["zstd.h", "zdict.h", "zstd_errors.h"] { + fs::copy(include_src.join(header), include_dst.join(header)) + .unwrap_or_else(|e| panic!("copy vendored header {header}: {e}")); + println!("cargo:rerun-if-changed=include/{header}"); + } + + // `pkg-config --modversion libzstd` must report the upstream version a + // consumer's build system pins against. + let pc = format!( + "prefix=/usr\n\ + exec_prefix=${{prefix}}\n\ + libdir=${{exec_prefix}}/lib\n\ + includedir=${{prefix}}/include\n\ + \n\ + Name: zstd\n\ + Description: structured-zstd, a libzstd-compatible pure-Rust zstd codec\n\ + URL: https://github.com/structured-world/structured-zstd\n\ + Version: {UPSTREAM_VERSION}\n\ + Libs: -L${{libdir}} -lzstd\n\ + Cflags: -I${{includedir}}\n" + ); + fs::write(out_dir.join("libzstd.pc"), pc).expect("write libzstd.pc"); + + println!("cargo:rerun-if-changed=build.rs"); +} diff --git a/c-api/include/zdict.h b/c-api/include/zdict.h new file mode 100644 index 000000000..599b79301 --- /dev/null +++ b/c-api/include/zdict.h @@ -0,0 +1,481 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_ZDICT_H +#define ZSTD_ZDICT_H + + +/*====== Dependencies ======*/ +#include /* size_t */ + +#if defined (__cplusplus) +extern "C" { +#endif + +/* ===== ZDICTLIB_API : control library symbols visibility ===== */ +#ifndef ZDICTLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZDICTLIB_VISIBILITY +# define ZDICTLIB_VISIBLE ZDICTLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZDICTLIB_VISIBLE __attribute__ ((visibility ("default"))) +# else +# define ZDICTLIB_VISIBLE +# endif +#endif + +#ifndef ZDICTLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZDICTLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else +# define ZDICTLIB_HIDDEN +# endif +#endif + +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBLE +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZDICTLIB_API ZDICTLIB_VISIBLE +#endif + +/******************************************************************************* + * Zstd dictionary builder + * + * FAQ + * === + * Why should I use a dictionary? + * ------------------------------ + * + * Zstd can use dictionaries to improve compression ratio of small data. + * Traditionally small files don't compress well because there is very little + * repetition in a single sample, since it is small. But, if you are compressing + * many similar files, like a bunch of JSON records that share the same + * structure, you can train a dictionary on ahead of time on some samples of + * these files. Then, zstd can use the dictionary to find repetitions that are + * present across samples. This can vastly improve compression ratio. + * + * When is a dictionary useful? + * ---------------------------- + * + * Dictionaries are useful when compressing many small files that are similar. + * The larger a file is, the less benefit a dictionary will have. Generally, + * we don't expect dictionary compression to be effective past 100KB. And the + * smaller a file is, the more we would expect the dictionary to help. + * + * How do I use a dictionary? + * -------------------------- + * + * Simply pass the dictionary to the zstd compressor with + * `ZSTD_CCtx_loadDictionary()`. The same dictionary must then be passed to + * the decompressor, using `ZSTD_DCtx_loadDictionary()`. There are other + * more advanced functions that allow selecting some options, see zstd.h for + * complete documentation. + * + * What is a zstd dictionary? + * -------------------------- + * + * A zstd dictionary has two pieces: Its header, and its content. The header + * contains a magic number, the dictionary ID, and entropy tables. These + * entropy tables allow zstd to save on header costs in the compressed file, + * which really matters for small data. The content is just bytes, which are + * repeated content that is common across many samples. + * + * What is a raw content dictionary? + * --------------------------------- + * + * A raw content dictionary is just bytes. It doesn't have a zstd dictionary + * header, a dictionary ID, or entropy tables. Any buffer is a valid raw + * content dictionary. + * + * How do I train a dictionary? + * ---------------------------- + * + * Gather samples from your use case. These samples should be similar to each + * other. If you have several use cases, you could try to train one dictionary + * per use case. + * + * Pass those samples to `ZDICT_trainFromBuffer()` and that will train your + * dictionary. There are a few advanced versions of this function, but this + * is a great starting point. If you want to further tune your dictionary + * you could try `ZDICT_optimizeTrainFromBuffer_cover()`. If that is too slow + * you can try `ZDICT_optimizeTrainFromBuffer_fastCover()`. + * + * If the dictionary training function fails, that is likely because you + * either passed too few samples, or a dictionary would not be effective + * for your data. Look at the messages that the dictionary trainer printed, + * if it doesn't say too few samples, then a dictionary would not be effective. + * + * How large should my dictionary be? + * ---------------------------------- + * + * A reasonable dictionary size, the `dictBufferCapacity`, is about 100KB. + * The zstd CLI defaults to a 110KB dictionary. You likely don't need a + * dictionary larger than that. But, most use cases can get away with a + * smaller dictionary. The advanced dictionary builders can automatically + * shrink the dictionary for you, and select the smallest size that doesn't + * hurt compression ratio too much. See the `shrinkDict` parameter. + * A smaller dictionary can save memory, and potentially speed up + * compression. + * + * How many samples should I provide to the dictionary builder? + * ------------------------------------------------------------ + * + * We generally recommend passing ~100x the size of the dictionary + * in samples. A few thousand should suffice. Having too few samples + * can hurt the dictionaries effectiveness. Having more samples will + * only improve the dictionaries effectiveness. But having too many + * samples can slow down the dictionary builder. + * + * How do I determine if a dictionary will be effective? + * ----------------------------------------------------- + * + * Simply train a dictionary and try it out. You can use zstd's built in + * benchmarking tool to test the dictionary effectiveness. + * + * # Benchmark levels 1-3 without a dictionary + * zstd -b1e3 -r /path/to/my/files + * # Benchmark levels 1-3 with a dictionary + * zstd -b1e3 -r /path/to/my/files -D /path/to/my/dictionary + * + * When should I retrain a dictionary? + * ----------------------------------- + * + * You should retrain a dictionary when its effectiveness drops. Dictionary + * effectiveness drops as the data you are compressing changes. Generally, we do + * expect dictionaries to "decay" over time, as your data changes, but the rate + * at which they decay depends on your use case. Internally, we regularly + * retrain dictionaries, and if the new dictionary performs significantly + * better than the old dictionary, we will ship the new dictionary. + * + * I have a raw content dictionary, how do I turn it into a zstd dictionary? + * ------------------------------------------------------------------------- + * + * If you have a raw content dictionary, e.g. by manually constructing it, or + * using a third-party dictionary builder, you can turn it into a zstd + * dictionary by using `ZDICT_finalizeDictionary()`. You'll also have to + * provide some samples of the data. It will add the zstd header to the + * raw content, which contains a dictionary ID and entropy tables, which + * will improve compression ratio, and allow zstd to write the dictionary ID + * into the frame, if you so choose. + * + * Do I have to use zstd's dictionary builder? + * ------------------------------------------- + * + * No! You can construct dictionary content however you please, it is just + * bytes. It will always be valid as a raw content dictionary. If you want + * a zstd dictionary, which can improve compression ratio, use + * `ZDICT_finalizeDictionary()`. + * + * What is the attack surface of a zstd dictionary? + * ------------------------------------------------ + * + * Zstd is heavily fuzz tested, including loading fuzzed dictionaries, so + * zstd should never crash, or access out-of-bounds memory no matter what + * the dictionary is. However, if an attacker can control the dictionary + * during decompression, they can cause zstd to generate arbitrary bytes, + * just like if they controlled the compressed data. + * + ******************************************************************************/ + + +/*! ZDICT_trainFromBuffer(): + * Train a dictionary from an array of samples. + * Redirect towards ZDICT_optimizeTrainFromBuffer_fastCover() single-threaded, with d=8, steps=4, + * f=20, and accel=1. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * The resulting dictionary will be saved into `dictBuffer`. + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * Note: Dictionary training will fail if there are not enough samples to construct a + * dictionary, or if most of the samples are too small (< 8 bytes being the lower limit). + * If dictionary training fails, you should use zstd without a dictionary, as the dictionary + * would've been ineffective anyways. If you believe your samples would benefit from a dictionary + * please open an issue with details, and we can look into it. + * Note: ZDICT_trainFromBuffer()'s memory usage is about 6 MB. + * Tips: In general, a reasonable dictionary has a size of ~ 100 KB. + * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`. + * In general, it's recommended to provide a few thousands samples, though this can vary a lot. + * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. + */ +ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, + const size_t* samplesSizes, unsigned nbSamples); + +typedef struct { + int compressionLevel; /**< optimize for a specific zstd compression level; 0 means default */ + unsigned notificationLevel; /**< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */ + unsigned dictID; /**< force dictID value; 0 means auto mode (32-bits random value) + * NOTE: The zstd format reserves some dictionary IDs for future use. + * You may use them in private settings, but be warned that they + * may be used by zstd in a public dictionary registry in the future. + * These dictionary IDs are: + * - low range : <= 32767 + * - high range : >= (2^31) + */ +} ZDICT_params_t; + +/*! ZDICT_finalizeDictionary(): + * Given a custom content as a basis for dictionary, and a set of samples, + * finalize dictionary by adding headers and statistics according to the zstd + * dictionary format. + * + * Samples must be stored concatenated in a flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each + * sample in order. The samples are used to construct the statistics, so they + * should be representative of what you will compress with this dictionary. + * + * The compression level can be set in `parameters`. You should pass the + * compression level you expect to use in production. The statistics for each + * compression level differ, so tuning the dictionary for the compression level + * can help quite a bit. + * + * You can set an explicit dictionary ID in `parameters`, or allow us to pick + * a random dictionary ID for you, but we can't guarantee no collisions. + * + * The dstDictBuffer and the dictContent may overlap, and the content will be + * appended to the end of the header. If the header + the content doesn't fit in + * maxDictSize the beginning of the content is truncated to make room, since it + * is presumed that the most profitable content is at the end of the dictionary, + * since that is the cheapest to reference. + * + * `maxDictSize` must be >= max(dictContentSize, ZDICT_DICTSIZE_MIN). + * + * @return: size of dictionary stored into `dstDictBuffer` (<= `maxDictSize`), + * or an error code, which can be tested by ZDICT_isError(). + * Note: ZDICT_finalizeDictionary() will push notifications into stderr if + * instructed to, using notificationLevel>0. + * NOTE: This function currently may fail in several edge cases including: + * * Not enough samples + * * Samples are uncompressible + * * Samples are all exactly the same + */ +ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dstDictBuffer, size_t maxDictSize, + const void* dictContent, size_t dictContentSize, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_params_t parameters); + + +/*====== Helper functions ======*/ +ZDICTLIB_API unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize); /**< extracts dictID; @return zero if error (not a valid dictionary) */ +ZDICTLIB_API size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize); /* returns dict header size; returns a ZSTD error code on failure */ +ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode); +ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode); + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_ZDICT_H */ + +#if defined(ZDICT_STATIC_LINKING_ONLY) && !defined(ZSTD_ZDICT_H_STATIC) +#define ZSTD_ZDICT_H_STATIC + +#if defined (__cplusplus) +extern "C" { +#endif + +/* This can be overridden externally to hide static symbols. */ +#ifndef ZDICTLIB_STATIC_API +# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZDICTLIB_STATIC_API __declspec(dllexport) ZDICTLIB_VISIBLE +# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZDICTLIB_STATIC_API __declspec(dllimport) ZDICTLIB_VISIBLE +# else +# define ZDICTLIB_STATIC_API ZDICTLIB_VISIBLE +# endif +#endif + +/* ==================================================================================== + * The definitions in this section are considered experimental. + * They should never be used with a dynamic library, as they may change in the future. + * They are provided for advanced usages. + * Use them only in association with static linking. + * ==================================================================================== */ + +#define ZDICT_DICTSIZE_MIN 256 +/* Deprecated: Remove in v1.6.0 */ +#define ZDICT_CONTENTSIZE_MIN 128 + +/*! ZDICT_cover_params_t: + * k and d are the only required parameters. + * For others, value 0 means default. + */ +typedef struct { + unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */ + unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */ + unsigned steps; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */ + unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */ + double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (1.0), 1.0 when all samples are used for both training and testing */ + unsigned shrinkDict; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */ + unsigned shrinkDictMaxRegression; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */ + ZDICT_params_t zParams; +} ZDICT_cover_params_t; + +typedef struct { + unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */ + unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */ + unsigned f; /* log of size of frequency array : constraint: 0 < f <= 31 : 1 means default(20)*/ + unsigned steps; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */ + unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */ + double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (0.75), 1.0 when all samples are used for both training and testing */ + unsigned accel; /* Acceleration level: constraint: 0 < accel <= 10, higher means faster and less accurate, 0 means default(1) */ + unsigned shrinkDict; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */ + unsigned shrinkDictMaxRegression; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */ + + ZDICT_params_t zParams; +} ZDICT_fastCover_params_t; + +/*! ZDICT_trainFromBuffer_cover(): + * Train a dictionary from an array of samples using the COVER algorithm. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * The resulting dictionary will be saved into `dictBuffer`. + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * See ZDICT_trainFromBuffer() for details on failure modes. + * Note: ZDICT_trainFromBuffer_cover() requires about 9 bytes of memory for each input byte. + * Tips: In general, a reasonable dictionary has a size of ~ 100 KB. + * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`. + * In general, it's recommended to provide a few thousands samples, though this can vary a lot. + * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. + */ +ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_cover( + void *dictBuffer, size_t dictBufferCapacity, + const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, + ZDICT_cover_params_t parameters); + +/*! ZDICT_optimizeTrainFromBuffer_cover(): + * The same requirements as above hold for all the parameters except `parameters`. + * This function tries many parameter combinations and picks the best parameters. + * `*parameters` is filled with the best parameters found, + * dictionary constructed with those parameters is stored in `dictBuffer`. + * + * All of the parameters d, k, steps are optional. + * If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}. + * if steps is zero it defaults to its default value. + * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000]. + * + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * On success `*parameters` contains the parameters selected. + * See ZDICT_trainFromBuffer() for details on failure modes. + * Note: ZDICT_optimizeTrainFromBuffer_cover() requires about 8 bytes of memory for each input byte and additionally another 5 bytes of memory for each byte of memory for each thread. + */ +ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover( + void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_cover_params_t* parameters); + +/*! ZDICT_trainFromBuffer_fastCover(): + * Train a dictionary from an array of samples using a modified version of COVER algorithm. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * d and k are required. + * All other parameters are optional, will use default values if not provided + * The resulting dictionary will be saved into `dictBuffer`. + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * See ZDICT_trainFromBuffer() for details on failure modes. + * Note: ZDICT_trainFromBuffer_fastCover() requires 6 * 2^f bytes of memory. + * Tips: In general, a reasonable dictionary has a size of ~ 100 KB. + * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`. + * In general, it's recommended to provide a few thousands samples, though this can vary a lot. + * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. + */ +ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer, + size_t dictBufferCapacity, const void *samplesBuffer, + const size_t *samplesSizes, unsigned nbSamples, + ZDICT_fastCover_params_t parameters); + +/*! ZDICT_optimizeTrainFromBuffer_fastCover(): + * The same requirements as above hold for all the parameters except `parameters`. + * This function tries many parameter combinations (specifically, k and d combinations) + * and picks the best parameters. `*parameters` is filled with the best parameters found, + * dictionary constructed with those parameters is stored in `dictBuffer`. + * All of the parameters d, k, steps, f, and accel are optional. + * If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}. + * if steps is zero it defaults to its default value. + * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000]. + * If f is zero, default value of 20 is used. + * If accel is zero, default value of 1 is used. + * + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * On success `*parameters` contains the parameters selected. + * See ZDICT_trainFromBuffer() for details on failure modes. + * Note: ZDICT_optimizeTrainFromBuffer_fastCover() requires about 6 * 2^f bytes of memory for each thread. + */ +ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer, + size_t dictBufferCapacity, const void* samplesBuffer, + const size_t* samplesSizes, unsigned nbSamples, + ZDICT_fastCover_params_t* parameters); + +typedef struct { + unsigned selectivityLevel; /* 0 means default; larger => select more => larger dictionary */ + ZDICT_params_t zParams; +} ZDICT_legacy_params_t; + +/*! ZDICT_trainFromBuffer_legacy(): + * Train a dictionary from an array of samples. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * The resulting dictionary will be saved into `dictBuffer`. + * `parameters` is optional and can be provided with values set to 0 to mean "default". + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * See ZDICT_trainFromBuffer() for details on failure modes. + * Tips: In general, a reasonable dictionary has a size of ~ 100 KB. + * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`. + * In general, it's recommended to provide a few thousands samples, though this can vary a lot. + * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. + * Note: ZDICT_trainFromBuffer_legacy() will send notifications into stderr if instructed to, using notificationLevel>0. + */ +ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_legacy( + void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_legacy_params_t parameters); + + +/* Deprecation warnings */ +/* It is generally possible to disable deprecation warnings from compiler, + for example with -Wno-deprecated-declarations for gcc + or _CRT_SECURE_NO_WARNINGS in Visual. + Otherwise, it's also possible to manually define ZDICT_DISABLE_DEPRECATE_WARNINGS */ +#ifdef ZDICT_DISABLE_DEPRECATE_WARNINGS +# define ZDICT_DEPRECATED(message) /* disable deprecation warnings */ +#else +# define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define ZDICT_DEPRECATED(message) [[deprecated(message)]] +# elif defined(__clang__) || (ZDICT_GCC_VERSION >= 405) +# define ZDICT_DEPRECATED(message) __attribute__((deprecated(message))) +# elif (ZDICT_GCC_VERSION >= 301) +# define ZDICT_DEPRECATED(message) __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define ZDICT_DEPRECATED(message) __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement ZDICT_DEPRECATED for this compiler") +# define ZDICT_DEPRECATED(message) +# endif +#endif /* ZDICT_DISABLE_DEPRECATE_WARNINGS */ + +ZDICT_DEPRECATED("use ZDICT_finalizeDictionary() instead") +ZDICTLIB_STATIC_API +size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples); + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_ZDICT_H_STATIC */ diff --git a/c-api/include/zstd.h b/c-api/include/zstd.h new file mode 100644 index 000000000..b8c0644a7 --- /dev/null +++ b/c-api/include/zstd.h @@ -0,0 +1,3198 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_H_235446 +#define ZSTD_H_235446 + + +/* ====== Dependencies ======*/ +#include /* size_t */ + +#include "zstd_errors.h" /* list of errors */ +#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) +#include /* INT_MAX */ +#endif /* ZSTD_STATIC_LINKING_ONLY */ + +#if defined (__cplusplus) +extern "C" { +#endif + +/* ===== ZSTDLIB_API : control library symbols visibility ===== */ +#ifndef ZSTDLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZSTDLIB_VISIBILITY +# define ZSTDLIB_VISIBLE ZSTDLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDLIB_VISIBLE __attribute__ ((visibility ("default"))) +# else +# define ZSTDLIB_VISIBLE +# endif +#endif + +#ifndef ZSTDLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else +# define ZSTDLIB_HIDDEN +# endif +#endif + +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBLE +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZSTDLIB_API ZSTDLIB_VISIBLE +#endif + +/* Deprecation warnings : + * Should these warnings be a problem, it is generally possible to disable them, + * typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual. + * Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS. + */ +#ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS +# define ZSTD_DEPRECATED(message) /* disable deprecation warnings */ +#else +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define ZSTD_DEPRECATED(message) [[deprecated(message)]] +# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__) || defined(__IAR_SYSTEMS_ICC__) +# define ZSTD_DEPRECATED(message) __attribute__((deprecated(message))) +# elif defined(__GNUC__) && (__GNUC__ >= 3) +# define ZSTD_DEPRECATED(message) __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define ZSTD_DEPRECATED(message) __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler") +# define ZSTD_DEPRECATED(message) +# endif +#endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */ + + +/******************************************************************************* + Introduction + + zstd, short for Zstandard, is a fast lossless compression algorithm, targeting + real-time compression scenarios at zlib-level and better compression ratios. + The zstd compression library provides in-memory compression and decompression + functions. + + The library supports regular compression levels from 1 up to ZSTD_maxCLevel(), + which is currently 22. Levels >= 20, labeled `--ultra`, should be used with + caution, as they require more memory. The library also offers negative + compression levels, which extend the range of speed vs. ratio preferences. + The lower the level, the faster the speed (at the cost of compression). + + Compression can be done in: + - a single step (described as Simple API) + - a single step, reusing a context (described as Explicit context) + - unbounded multiple steps (described as Streaming compression) + + The compression ratio achievable on small data can be highly improved using + a dictionary. Dictionary compression can be performed in: + - a single step (described as Simple dictionary API) + - a single step, reusing a dictionary (described as Bulk-processing + dictionary API) + + Advanced experimental functions can be accessed using + `#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h. + + Advanced experimental APIs should never be used with a dynamically-linked + library. They are not "stable"; their definitions or signatures may change in + the future. Only static linking is allowed. +*******************************************************************************/ + +/*------ Version ------*/ +#define ZSTD_VERSION_MAJOR 1 +#define ZSTD_VERSION_MINOR 5 +#define ZSTD_VERSION_RELEASE 7 +#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) + +/*! ZSTD_versionNumber() : + * Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). */ +ZSTDLIB_API unsigned ZSTD_versionNumber(void); + +#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE +#define ZSTD_QUOTE(str) #str +#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str) +#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) + +/*! ZSTD_versionString() : + * Return runtime library version, like "1.4.5". Requires v1.3.0+. */ +ZSTDLIB_API const char* ZSTD_versionString(void); + +/* ************************************* + * Default constant + ***************************************/ +#ifndef ZSTD_CLEVEL_DEFAULT +# define ZSTD_CLEVEL_DEFAULT 3 +#endif + +/* ************************************* + * Constants + ***************************************/ + +/* All magic numbers are supposed read/written to/from files/memory using little-endian convention */ +#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */ +#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */ +#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */ +#define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0 + +#define ZSTD_BLOCKSIZELOG_MAX 17 +#define ZSTD_BLOCKSIZE_MAX (1<= ZSTD_compressBound(srcSize)` guarantees that zstd will have + * enough space to successfully compress the data. + * @return : compressed size written into `dst` (<= `dstCapacity), + * or an error code if it fails (which can be tested using ZSTD_isError()). */ +ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel); + +/*! ZSTD_decompress() : + * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames. + * Multiple compressed frames can be decompressed at once with this method. + * The result will be the concatenation of all decompressed frames, back to back. + * `dstCapacity` is an upper bound of originalSize to regenerate. + * First frame's decompressed size can be extracted using ZSTD_getFrameContentSize(). + * If maximum upper bound isn't known, prefer using streaming mode to decompress data. + * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), + * or an errorCode if it fails (which can be tested using ZSTD_isError()). */ +ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, + const void* src, size_t compressedSize); + + +/*====== Decompression helper functions ======*/ + +/*! ZSTD_getFrameContentSize() : requires v1.3.0+ + * `src` should point to the start of a ZSTD encoded frame. + * `srcSize` must be at least as large as the frame header. + * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough. + * @return : - decompressed size of `src` frame content, if known + * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined + * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) + * note 1 : a 0 return value means the frame is valid but "empty". + * When invoking this method on a skippable frame, it will return 0. + * note 2 : decompressed size is an optional field, it may not be present (typically in streaming mode). + * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. + * In which case, it's necessary to use streaming mode to decompress data. + * Optionally, application can rely on some implicit limit, + * as ZSTD_decompress() only needs an upper bound of decompressed size. + * (For example, data could be necessarily cut into blocks <= 16 KB). + * note 3 : decompressed size is always present when compression is completed using single-pass functions, + * such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict(). + * note 4 : decompressed size can be very large (64-bits value), + * potentially larger than what local system can handle as a single memory segment. + * In which case, it's necessary to use streaming mode to decompress data. + * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified. + * Always ensure return value fits within application's authorized limits. + * Each application can set its own limits. + * note 6 : This function replaces ZSTD_getDecompressedSize() */ +#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) +#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) +ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); + +/*! ZSTD_getDecompressedSize() (obsolete): + * This function is now obsolete, in favor of ZSTD_getFrameContentSize(). + * Both functions work the same way, but ZSTD_getDecompressedSize() blends + * "empty", "unknown" and "error" results to the same return value (0), + * while ZSTD_getFrameContentSize() gives them separate return values. + * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */ +ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize") +ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); + +/*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+ + * `src` should point to the start of a ZSTD frame or skippable frame. + * `srcSize` must be >= first frame size + * @return : the compressed size of the first frame starting at `src`, + * suitable to pass as `srcSize` to `ZSTD_decompress` or similar, + * or an error code if input is invalid + * Note 1: this method is called _find*() because it's not enough to read the header, + * it may have to scan through the frame's content, to reach its end. + * Note 2: this method also works with Skippable Frames. In which case, + * it returns the size of the complete skippable frame, + * which is always equal to its content size + 8 bytes for headers. */ +ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); + + +/*====== Compression helper functions ======*/ + +/*! ZSTD_compressBound() : + * maximum compressed size in worst case single-pass scenario. + * When invoking `ZSTD_compress()`, or any other one-pass compression function, + * it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize) + * as it eliminates one potential failure scenario, + * aka not enough room in dst buffer to write the compressed frame. + * Note : ZSTD_compressBound() itself can fail, if @srcSize >= ZSTD_MAX_INPUT_SIZE . + * In which case, ZSTD_compressBound() will return an error code + * which can be tested using ZSTD_isError(). + * + * ZSTD_COMPRESSBOUND() : + * same as ZSTD_compressBound(), but as a macro. + * It can be used to produce constants, which can be useful for static allocation, + * for example to size a static array on stack. + * Will produce constant value 0 if srcSize is too large. + */ +#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00ULL : 0xFF00FF00U) +#define ZSTD_COMPRESSBOUND(srcSize) (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ +ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ + + +/*====== Error helper functions ======*/ +/* ZSTD_isError() : + * Most ZSTD_* functions returning a size_t value can be tested for error, + * using ZSTD_isError(). + * @return 1 if error, 0 otherwise + */ +ZSTDLIB_API unsigned ZSTD_isError(size_t result); /*!< tells if a `size_t` function result is an error code */ +ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); /* convert a result into an error code, which can be compared to error enum list */ +ZSTDLIB_API const char* ZSTD_getErrorName(size_t result); /*!< provides readable string from a function result */ +ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed, requires v1.4.0+ */ +ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ +ZSTDLIB_API int ZSTD_defaultCLevel(void); /*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ */ + + +/*************************************** +* Explicit context +***************************************/ +/*= Compression context + * When compressing many times, + * it is recommended to allocate a compression context just once, + * and reuse it for each successive compression operation. + * This will make the workload easier for system's memory. + * Note : re-using context is just a speed / resource optimization. + * It doesn't change the compression ratio, which remains identical. + * Note 2: For parallel execution in multi-threaded environments, + * use one different context per thread . + */ +typedef struct ZSTD_CCtx_s ZSTD_CCtx; +ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); +ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /* compatible with NULL pointer */ + +/*! ZSTD_compressCCtx() : + * Same as ZSTD_compress(), using an explicit ZSTD_CCtx. + * Important : in order to mirror `ZSTD_compress()` behavior, + * this function compresses at the requested compression level, + * __ignoring any other advanced parameter__ . + * If any advanced parameter was set using the advanced API, + * they will all be reset. Only @compressionLevel remains. + */ +ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel); + +/*= Decompression context + * When decompressing many times, + * it is recommended to allocate a context only once, + * and reuse it for each successive compression operation. + * This will make workload friendlier for system's memory. + * Use one context per thread for parallel execution. */ +typedef struct ZSTD_DCtx_s ZSTD_DCtx; +ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); +ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /* accept NULL pointer */ + +/*! ZSTD_decompressDCtx() : + * Same as ZSTD_decompress(), + * requires an allocated ZSTD_DCtx. + * Compatible with sticky parameters (see below). + */ +ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + + +/********************************************* +* Advanced compression API (Requires v1.4.0+) +**********************************************/ + +/* API design : + * Parameters are pushed one by one into an existing context, + * using ZSTD_CCtx_set*() functions. + * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame. + * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` ! + * __They do not apply to one-shot variants such as ZSTD_compressCCtx()__ . + * + * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset(). + * + * This API supersedes all other "advanced" API entry points in the experimental section. + * In the future, we expect to remove API entry points from experimental which are redundant with this API. + */ + + +/* Compression strategies, listed from fastest to strongest */ +typedef enum { ZSTD_fast=1, + ZSTD_dfast=2, + ZSTD_greedy=3, + ZSTD_lazy=4, + ZSTD_lazy2=5, + ZSTD_btlazy2=6, + ZSTD_btopt=7, + ZSTD_btultra=8, + ZSTD_btultra2=9 + /* note : new strategies _might_ be added in the future. + Only the order (from fast to strong) is guaranteed */ +} ZSTD_strategy; + +typedef enum { + + /* compression parameters + * Note: When compressing with a ZSTD_CDict these parameters are superseded + * by the parameters used to construct the ZSTD_CDict. + * See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */ + ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table. + * Note that exact compression parameters are dynamically determined, + * depending on both compression level and srcSize (when known). + * Default level is ZSTD_CLEVEL_DEFAULT==3. + * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT. + * Note 1 : it's possible to pass a negative compression level. + * Note 2 : setting a level does not automatically set all other compression parameters + * to default. Setting this will however eventually dynamically impact the compression + * parameters which have not been manually set. The manually set + * ones will 'stick'. */ + /* Advanced compression parameters : + * It's possible to pin down compression parameters to some specific values. + * In which case, these values are no longer dynamically selected by the compressor */ + ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2. + * This will set a memory budget for streaming decompression, + * with larger values requiring more memory + * and typically compressing more. + * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX. + * Special: value 0 means "use default windowLog". + * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT + * requires explicitly allowing such size at streaming decompression stage. */ + ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2. + * Resulting memory usage is (1 << (hashLog+2)). + * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX. + * Larger tables improve compression ratio of strategies <= dFast, + * and improve speed of strategies > dFast. + * Special: value 0 means "use default hashLog". */ + ZSTD_c_chainLog=103, /* Size of the multi-probe search table, as a power of 2. + * Resulting memory usage is (1 << (chainLog+2)). + * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX. + * Larger tables result in better and slower compression. + * This parameter is useless for "fast" strategy. + * It's still useful when using "dfast" strategy, + * in which case it defines a secondary probe table. + * Special: value 0 means "use default chainLog". */ + ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2. + * More attempts result in better and slower compression. + * This parameter is useless for "fast" and "dFast" strategies. + * Special: value 0 means "use default searchLog". */ + ZSTD_c_minMatch=105, /* Minimum size of searched matches. + * Note that Zstandard can still find matches of smaller size, + * it just tweaks its search algorithm to look for this size and larger. + * Larger values increase compression and decompression speed, but decrease ratio. + * Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX. + * Note that currently, for all strategies < btopt, effective minimum is 4. + * , for all strategies > fast, effective maximum is 6. + * Special: value 0 means "use default minMatchLength". */ + ZSTD_c_targetLength=106, /* Impact of this field depends on strategy. + * For strategies btopt, btultra & btultra2: + * Length of Match considered "good enough" to stop search. + * Larger values make compression stronger, and slower. + * For strategy fast: + * Distance between match sampling. + * Larger values make compression faster, and weaker. + * Special: value 0 means "use default targetLength". */ + ZSTD_c_strategy=107, /* See ZSTD_strategy enum definition. + * The higher the value of selected strategy, the more complex it is, + * resulting in stronger and slower compression. + * Special: value 0 means "use default strategy". */ + + ZSTD_c_targetCBlockSize=130, /* v1.5.6+ + * Attempts to fit compressed block size into approximately targetCBlockSize. + * Bound by ZSTD_TARGETCBLOCKSIZE_MIN and ZSTD_TARGETCBLOCKSIZE_MAX. + * Note that it's not a guarantee, just a convergence target (default:0). + * No target when targetCBlockSize == 0. + * This is helpful in low bandwidth streaming environments to improve end-to-end latency, + * when a client can make use of partial documents (a prominent example being Chrome). + * Note: this parameter is stable since v1.5.6. + * It was present as an experimental parameter in earlier versions, + * but it's not recommended using it with earlier library versions + * due to massive performance regressions. + */ + /* LDM mode parameters */ + ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching. + * This parameter is designed to improve compression ratio + * for large inputs, by finding large matches at long distance. + * It increases memory usage and window size. + * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB + * except when expressly set to a different value. + * Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and + * compression strategy >= ZSTD_btopt (== compression level 16+) */ + ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2. + * Larger values increase memory usage and compression ratio, + * but decrease compression speed. + * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX + * default: windowlog - 7. + * Special: value 0 means "automatically determine hashlog". */ + ZSTD_c_ldmMinMatch=162, /* Minimum match size for long distance matcher. + * Larger/too small values usually decrease compression ratio. + * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX. + * Special: value 0 means "use default value" (default: 64). */ + ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution. + * Larger values improve collision resolution but decrease compression speed. + * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX. + * Special: value 0 means "use default value" (default: 3). */ + ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table. + * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN). + * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage. + * Larger values improve compression speed. + * Deviating far from default value will likely result in a compression ratio decrease. + * Special: value 0 means "automatically determine hashRateLog". */ + + /* frame parameters */ + ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1) + * Content size must be known at the beginning of compression. + * This is automatically the case when using ZSTD_compress2(), + * For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */ + ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */ + ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */ + + /* multi-threading parameters */ + /* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD). + * Otherwise, trying to set any other value than default (0) will be a no-op and return an error. + * In a situation where it's unknown if the linked library supports multi-threading or not, + * setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property. + */ + ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel. + * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() : + * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller, + * while compression is performed in parallel, within worker thread(s). + * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end : + * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call). + * More workers improve speed, but also increase memory usage. + * Default value is `0`, aka "single-threaded mode" : no worker is spawned, + * compression is performed inside Caller's thread, and all invocations are blocking */ + ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1. + * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads. + * 0 means default, which is dynamically determined based on compression parameters. + * Job size must be a minimum of overlap size, or ZSTDMT_JOBSIZE_MIN (= 512 KB), whichever is largest. + * The minimum size is automatically and transparently enforced. */ + ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size. + * The overlap size is an amount of data reloaded from previous job at the beginning of a new job. + * It helps preserve compression ratio, while each job is compressed in parallel. + * This value is enforced only when nbWorkers >= 1. + * Larger values increase compression ratio, but decrease speed. + * Possible values range from 0 to 9 : + * - 0 means "default" : value will be determined by the library, depending on strategy + * - 1 means "no overlap" + * - 9 means "full overlap", using a full window size. + * Each intermediate rank increases/decreases load size by a factor 2 : + * 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no overlap; 0:default + * default value varies between 6 and 9, depending on strategy */ + + /* note : additional experimental parameters are also available + * within the experimental section of the API. + * At the time of this writing, they include : + * ZSTD_c_rsyncable + * ZSTD_c_format + * ZSTD_c_forceMaxWindow + * ZSTD_c_forceAttachDict + * ZSTD_c_literalCompressionMode + * ZSTD_c_srcSizeHint + * ZSTD_c_enableDedicatedDictSearch + * ZSTD_c_stableInBuffer + * ZSTD_c_stableOutBuffer + * ZSTD_c_blockDelimiters + * ZSTD_c_validateSequences + * ZSTD_c_blockSplitterLevel + * ZSTD_c_splitAfterSequences + * ZSTD_c_useRowMatchFinder + * ZSTD_c_prefetchCDictTables + * ZSTD_c_enableSeqProducerFallback + * ZSTD_c_maxBlockSize + * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. + * note : never ever use experimentalParam? names directly; + * also, the enums values themselves are unstable and can still change. + */ + ZSTD_c_experimentalParam1=500, + ZSTD_c_experimentalParam2=10, + ZSTD_c_experimentalParam3=1000, + ZSTD_c_experimentalParam4=1001, + ZSTD_c_experimentalParam5=1002, + /* was ZSTD_c_experimentalParam6=1003; is now ZSTD_c_targetCBlockSize */ + ZSTD_c_experimentalParam7=1004, + ZSTD_c_experimentalParam8=1005, + ZSTD_c_experimentalParam9=1006, + ZSTD_c_experimentalParam10=1007, + ZSTD_c_experimentalParam11=1008, + ZSTD_c_experimentalParam12=1009, + ZSTD_c_experimentalParam13=1010, + ZSTD_c_experimentalParam14=1011, + ZSTD_c_experimentalParam15=1012, + ZSTD_c_experimentalParam16=1013, + ZSTD_c_experimentalParam17=1014, + ZSTD_c_experimentalParam18=1015, + ZSTD_c_experimentalParam19=1016, + ZSTD_c_experimentalParam20=1017 +} ZSTD_cParameter; + +typedef struct { + size_t error; + int lowerBound; + int upperBound; +} ZSTD_bounds; + +/*! ZSTD_cParam_getBounds() : + * All parameters must belong to an interval with lower and upper bounds, + * otherwise they will either trigger an error or be automatically clamped. + * @return : a structure, ZSTD_bounds, which contains + * - an error status field, which must be tested using ZSTD_isError() + * - lower and upper bounds, both inclusive + */ +ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam); + +/*! ZSTD_CCtx_setParameter() : + * Set one compression parameter, selected by enum ZSTD_cParameter. + * All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds(). + * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). + * Setting a parameter is generally only possible during frame initialization (before starting compression). + * Exception : when using multi-threading mode (nbWorkers >= 1), + * the following parameters can be updated _during_ compression (within same frame): + * => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy. + * new parameters will be active for next job only (after a flush()). + * @return : an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value); + +/*! ZSTD_CCtx_setPledgedSrcSize() : + * Total input data size to be compressed as a single frame. + * Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag. + * This value will also be controlled at end of frame, and trigger an error if not respected. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame. + * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN. + * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame. + * Note 2 : pledgedSrcSize is only valid once, for the next frame. + * It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN. + * Note 3 : Whenever all input data is provided and consumed in a single round, + * for example with ZSTD_compress2(), + * or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end), + * this value is automatically overridden by srcSize instead. + */ +ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize); + +typedef enum { + ZSTD_reset_session_only = 1, + ZSTD_reset_parameters = 2, + ZSTD_reset_session_and_parameters = 3 +} ZSTD_ResetDirective; + +/*! ZSTD_CCtx_reset() : + * There are 2 different things that can be reset, independently or jointly : + * - The session : will stop compressing current frame, and make CCtx ready to start a new one. + * Useful after an error, or to interrupt any ongoing compression. + * Any internal data not yet flushed is cancelled. + * Compression parameters and dictionary remain unchanged. + * They will be used to compress next frame. + * Resetting session never fails. + * - The parameters : changes all parameters back to "default". + * This also removes any reference to any dictionary or external sequence producer. + * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing) + * otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError()) + * - Both : similar to resetting the session, followed by resetting parameters. + */ +ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset); + +/*! ZSTD_compress2() : + * Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API. + * (note that this entry point doesn't even expose a compression level parameter). + * ZSTD_compress2() always starts a new frame. + * Should cctx hold data from a previously unfinished frame, everything about it is forgotten. + * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() + * - The function is always blocking, returns when compression is completed. + * NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have + * enough space to successfully compress the data, though it is possible it fails for other reasons. + * @return : compressed size written into `dst` (<= `dstCapacity), + * or an error code if it fails (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + + +/*********************************************** +* Advanced decompression API (Requires v1.4.0+) +************************************************/ + +/* The advanced API pushes parameters one by one into an existing DCtx context. + * Parameters are sticky, and remain valid for all following frames + * using the same DCtx context. + * It's possible to reset parameters to default values using ZSTD_DCtx_reset(). + * Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream(). + * Therefore, no new decompression function is necessary. + */ + +typedef enum { + + ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which + * the streaming API will refuse to allocate memory buffer + * in order to protect the host from unreasonable memory requirements. + * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. + * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT). + * Special: value 0 means "use default maximum windowLog". */ + + /* note : additional experimental parameters are also available + * within the experimental section of the API. + * At the time of this writing, they include : + * ZSTD_d_format + * ZSTD_d_stableOutBuffer + * ZSTD_d_forceIgnoreChecksum + * ZSTD_d_refMultipleDDicts + * ZSTD_d_disableHuffmanAssembly + * ZSTD_d_maxBlockSize + * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. + * note : never ever use experimentalParam? names directly + */ + ZSTD_d_experimentalParam1=1000, + ZSTD_d_experimentalParam2=1001, + ZSTD_d_experimentalParam3=1002, + ZSTD_d_experimentalParam4=1003, + ZSTD_d_experimentalParam5=1004, + ZSTD_d_experimentalParam6=1005 + +} ZSTD_dParameter; + +/*! ZSTD_dParam_getBounds() : + * All parameters must belong to an interval with lower and upper bounds, + * otherwise they will either trigger an error or be automatically clamped. + * @return : a structure, ZSTD_bounds, which contains + * - an error status field, which must be tested using ZSTD_isError() + * - both lower and upper bounds, inclusive + */ +ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam); + +/*! ZSTD_DCtx_setParameter() : + * Set one compression parameter, selected by enum ZSTD_dParameter. + * All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds(). + * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). + * Setting a parameter is only possible during frame initialization (before starting decompression). + * @return : 0, or an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value); + +/*! ZSTD_DCtx_reset() : + * Return a DCtx to clean state. + * Session and parameters can be reset jointly or separately. + * Parameters can only be reset when no active frame is being decompressed. + * @return : 0, or an error code, which can be tested with ZSTD_isError() + */ +ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset); + + +/**************************** +* Streaming +****************************/ + +typedef struct ZSTD_inBuffer_s { + const void* src; /**< start of input buffer */ + size_t size; /**< size of input buffer */ + size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_inBuffer; + +typedef struct ZSTD_outBuffer_s { + void* dst; /**< start of output buffer */ + size_t size; /**< size of output buffer */ + size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_outBuffer; + + + +/*-*********************************************************************** +* Streaming compression - HowTo +* +* A ZSTD_CStream object is required to track streaming operation. +* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. +* ZSTD_CStream objects can be reused multiple times on consecutive compression operations. +* It is recommended to reuse ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory. +* +* For parallel execution, use one separate ZSTD_CStream per thread. +* +* note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing. +* +* Parameters are sticky : when starting a new compression on the same context, +* it will reuse the same sticky parameters as previous compression session. +* When in doubt, it's recommended to fully initialize the context before usage. +* Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(), +* ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to +* set more specific parameters, the pledged source size, or load a dictionary. +* +* Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to +* consume input stream. The function will automatically update both `pos` +* fields within `input` and `output`. +* Note that the function may not consume the entire input, for example, because +* the output buffer is already full, in which case `input.pos < input.size`. +* The caller must check if input has been entirely consumed. +* If not, the caller must make some room to receive more compressed data, +* and then present again remaining input data. +* note: ZSTD_e_continue is guaranteed to make some forward progress when called, +* but doesn't guarantee maximal forward progress. This is especially relevant +* when compressing with multiple threads. The call won't block if it can +* consume some input, but if it can't it will wait for some, but not all, +* output to be flushed. +* @return : provides a minimum amount of data remaining to be flushed from internal buffers +* or an error code, which can be tested using ZSTD_isError(). +* +* At any moment, it's possible to flush whatever data might remain stuck within internal buffer, +* using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated. +* Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0). +* In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush. +* You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the +* operation. +* note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will +* block until the flush is complete or the output buffer is full. +* @return : 0 if internal buffers are entirely flushed, +* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), +* or an error code, which can be tested using ZSTD_isError(). +* +* Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame. +* It will perform a flush and write frame epilogue. +* The epilogue is required for decoders to consider a frame completed. +* flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush. +* You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to +* start a new frame. +* note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will +* block until the flush is complete or the output buffer is full. +* @return : 0 if frame fully completed and fully flushed, +* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), +* or an error code, which can be tested using ZSTD_isError(). +* +* *******************************************************************/ + +typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */ + /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */ +/*===== ZSTD_CStream management functions =====*/ +ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); +ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); /* accept NULL pointer */ + +/*===== Streaming compression functions =====*/ +typedef enum { + ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */ + ZSTD_e_flush=1, /* flush any data provided so far, + * it creates (at least) one new block, that can be decoded immediately on reception; + * frame will continue: any future data can still reference previously compressed data, improving compression. + * note : multithreaded compression will block to flush as much output as possible. */ + ZSTD_e_end=2 /* flush any remaining data _and_ close current frame. + * note that frame is only closed after compressed data is fully flushed (return value == 0). + * After that point, any additional data starts a new frame. + * note : each frame is independent (does not reference any content from previous frame). + : note : multithreaded compression will block to flush as much output as possible. */ +} ZSTD_EndDirective; + +/*! ZSTD_compressStream2() : Requires v1.4.0+ + * Behaves about the same as ZSTD_compressStream, with additional control on end directive. + * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() + * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode) + * - output->pos must be <= dstCapacity, input->pos must be <= srcSize + * - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit. + * - endOp must be a valid directive + * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller. + * - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available, + * and then immediately returns, just indicating that there is some data remaining to be flushed. + * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte. + * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking. + * - @return provides a minimum amount of data remaining to be flushed from internal buffers + * or an error code, which can be tested using ZSTD_isError(). + * if @return != 0, flush is not fully completed, there is still some data left within internal buffers. + * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers. + * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed. + * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0), + * only ZSTD_e_end or ZSTD_e_flush operations are allowed. + * Before starting a new compression job, or changing compression parameters, + * it is required to fully flush internal buffers. + * - note: if an operation ends with an error, it may leave @cctx in an undefined state. + * Therefore, it's UB to invoke ZSTD_compressStream2() of ZSTD_compressStream() on such a state. + * In order to be re-employed after an error, a state must be reset, + * which can be done explicitly (ZSTD_CCtx_reset()), + * or is sometimes implied by methods starting a new compression job (ZSTD_initCStream(), ZSTD_compressCCtx()) + */ +ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp); + + +/* These buffer sizes are softly recommended. + * They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output. + * Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(), + * reducing the amount of memory shuffling and buffering, resulting in minor performance savings. + * + * However, note that these recommendations are from the perspective of a C caller program. + * If the streaming interface is invoked from some other language, + * especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo, + * a major performance rule is to reduce crossing such interface to an absolute minimum. + * It's not rare that performance ends being spent more into the interface, rather than compression itself. + * In which cases, prefer using large buffers, as large as practical, + * for both input and output, to reduce the nb of roundtrips. + */ +ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */ + + +/* ***************************************************************************** + * This following is a legacy streaming API, available since v1.0+ . + * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2(). + * It is redundant, but remains fully supported. + ******************************************************************************/ + +/*! + * Equivalent to: + * + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * + * Note that ZSTD_initCStream() clears any previously set dictionary. Use the new API + * to compress with a dictionary. + */ +ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); +/*! + * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue). + * NOTE: The return value is different. ZSTD_compressStream() returns a hint for + * the next read size (if non-zero and not an error). ZSTD_compressStream2() + * returns the minimum nb of bytes left to flush (if non-zero and not an error). + */ +ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); +/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */ +ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); +/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */ +ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); + + +/*-*************************************************************************** +* Streaming decompression - HowTo +* +* A ZSTD_DStream object is required to track streaming operations. +* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. +* ZSTD_DStream objects can be re-employed multiple times. +* +* Use ZSTD_initDStream() to start a new decompression operation. +* @return : recommended first input size +* Alternatively, use advanced API to set specific properties. +* +* Use ZSTD_decompressStream() repetitively to consume your input. +* The function will update both `pos` fields. +* If `input.pos < input.size`, some input has not been consumed. +* It's up to the caller to present again remaining data. +* +* The function tries to flush all data decoded immediately, respecting output buffer size. +* If `output.pos < output.size`, decoder has flushed everything it could. +* +* However, when `output.pos == output.size`, it's more difficult to know. +* If @return > 0, the frame is not complete, meaning +* either there is still some data left to flush within internal buffers, +* or there is more input to read to complete the frame (or both). +* In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer. +* Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX. +* @return : 0 when a frame is completely decoded and fully flushed, +* or an error code, which can be tested using ZSTD_isError(), +* or any other value > 0, which means there is still some decoding or flushing to do to complete current frame : +* the return value is a suggested next input size (just a hint for better latency) +* that will never request more than the remaining content of the compressed frame. +* *******************************************************************************/ + +typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */ + /* For compatibility with versions <= v1.2.0, prefer differentiating them. */ +/*===== ZSTD_DStream management functions =====*/ +ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); +ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); /* accept NULL pointer */ + +/*===== Streaming decompression functions =====*/ + +/*! ZSTD_initDStream() : + * Initialize/reset DStream state for new decompression operation. + * Call before new decompression operation using same DStream. + * + * Note : This function is redundant with the advanced API and equivalent to: + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_refDDict(zds, NULL); + */ +ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); + +/*! ZSTD_decompressStream() : + * Streaming decompression function. + * Call repetitively to consume full input updating it as necessary. + * Function will update both input and output `pos` fields exposing current state via these fields: + * - `input.pos < input.size`, some input remaining and caller should provide remaining input + * on the next call. + * - `output.pos < output.size`, decoder flushed internal output buffer. + * - `output.pos == output.size`, unflushed data potentially present in the internal buffers, + * check ZSTD_decompressStream() @return value, + * if > 0, invoke it again to flush remaining data to output. + * Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX. + * + * @return : 0 when a frame is completely decoded and fully flushed, + * or an error code, which can be tested using ZSTD_isError(), + * or any other value > 0, which means there is some decoding or flushing to do to complete current frame. + * + * Note: when an operation returns with an error code, the @zds state may be left in undefined state. + * It's UB to invoke `ZSTD_decompressStream()` on such a state. + * In order to re-use such a state, it must be first reset, + * which can be done explicitly (`ZSTD_DCtx_reset()`), + * or is implied for operations starting some new decompression job (`ZSTD_initDStream`, `ZSTD_decompressDCtx()`, `ZSTD_decompress_usingDict()`) + */ +ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); + +ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */ + + +/************************** +* Simple dictionary API +***************************/ +/*! ZSTD_compress_usingDict() : + * Compression at an explicit compression level using a Dictionary. + * A dictionary can be any arbitrary data segment (also called a prefix), + * or a buffer with specified information (see zdict.h). + * Note : This function loads the dictionary, resulting in significant startup delay. + * It's intended for a dictionary used only once. + * Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */ +ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + int compressionLevel); + +/*! ZSTD_decompress_usingDict() : + * Decompression using a known Dictionary. + * Dictionary must be identical to the one used during compression. + * Note : This function loads the dictionary, resulting in significant startup delay. + * It's intended for a dictionary used only once. + * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize); + + +/*********************************** + * Bulk processing dictionary API + **********************************/ +typedef struct ZSTD_CDict_s ZSTD_CDict; + +/*! ZSTD_createCDict() : + * When compressing multiple messages or blocks using the same dictionary, + * it's recommended to digest the dictionary only once, since it's a costly operation. + * ZSTD_createCDict() will create a state from digesting a dictionary. + * The resulting state can be used for future compression operations with very limited startup cost. + * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. + * @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict. + * Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content. + * Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer, + * in which case the only thing that it transports is the @compressionLevel. + * This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively, + * expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */ +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, + int compressionLevel); + +/*! ZSTD_freeCDict() : + * Function frees memory allocated by ZSTD_createCDict(). + * If a NULL pointer is passed, no operation is performed. */ +ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); + +/*! ZSTD_compress_usingCDict() : + * Compression using a digested Dictionary. + * Recommended when same dictionary is used multiple times. + * Note : compression level is _decided at dictionary creation time_, + * and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */ +ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict); + + +typedef struct ZSTD_DDict_s ZSTD_DDict; + +/*! ZSTD_createDDict() : + * Create a digested dictionary, ready to start decompression operation without startup delay. + * dictBuffer can be released after DDict creation, as its content is copied inside DDict. */ +ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize); + +/*! ZSTD_freeDDict() : + * Function frees memory allocated with ZSTD_createDDict() + * If a NULL pointer is passed, no operation is performed. */ +ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); + +/*! ZSTD_decompress_usingDDict() : + * Decompression using a digested Dictionary. + * Recommended when same dictionary is used multiple times. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict); + + +/******************************** + * Dictionary helper functions + *******************************/ + +/*! ZSTD_getDictID_fromDict() : Requires v1.4.0+ + * Provides the dictID stored within dictionary. + * if @return == 0, the dictionary is not conformant with Zstandard specification. + * It can still be loaded, but as a content-only dictionary. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); + +/*! ZSTD_getDictID_fromCDict() : Requires v1.5.0+ + * Provides the dictID of the dictionary loaded into `cdict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict); + +/*! ZSTD_getDictID_fromDDict() : Requires v1.4.0+ + * Provides the dictID of the dictionary loaded into `ddict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); + +/*! ZSTD_getDictID_fromFrame() : Requires v1.4.0+ + * Provides the dictID required to decompressed the frame stored within `src`. + * If @return == 0, the dictID could not be decoded. + * This could for one of the following reasons : + * - The frame does not require a dictionary to be decoded (most common case). + * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden piece of information. + * Note : this use case also happens when using a non-conformant dictionary. + * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). + * - This is not a Zstandard frame. + * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); + + +/******************************************************************************* + * Advanced dictionary and prefix API (Requires v1.4.0+) + * + * This API allows dictionaries to be used with ZSTD_compress2(), + * ZSTD_compressStream2(), and ZSTD_decompressDCtx(). + * Dictionaries are sticky, they remain valid when same context is reused, + * they only reset when the context is reset + * with ZSTD_reset_parameters or ZSTD_reset_session_and_parameters. + * In contrast, Prefixes are single-use. + ******************************************************************************/ + + +/*! ZSTD_CCtx_loadDictionary() : Requires v1.4.0+ + * Create an internal CDict from `dict` buffer. + * Decompression will have to use same dictionary. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary, + * meaning "return to no-dictionary mode". + * Note 1 : Dictionary is sticky, it will be used for all future compressed frames, + * until parameters are reset, a new dictionary is loaded, or the dictionary + * is explicitly invalidated by loading a NULL dictionary. + * Note 2 : Loading a dictionary involves building tables. + * It's also a CPU consuming operation, with non-negligible impact on latency. + * Tables are dependent on compression parameters, and for this reason, + * compression parameters can no longer be changed after loading a dictionary. + * Note 3 :`dict` content will be copied internally. + * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead. + * In such a case, dictionary buffer must outlive its users. + * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced() + * to precisely select how dictionary content must be interpreted. + * Note 5 : This method does not benefit from LDM (long distance mode). + * If you want to employ LDM on some large dictionary content, + * prefer employing ZSTD_CCtx_refPrefix() described below. + */ +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); + +/*! ZSTD_CCtx_refCDict() : Requires v1.4.0+ + * Reference a prepared dictionary, to be used for all future compressed frames. + * Note that compression parameters are enforced from within CDict, + * and supersede any compression parameter previously set within CCtx. + * The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs. + * The ignored parameters will be used again if the CCtx is returned to no-dictionary mode. + * The dictionary will remain valid for future compressed frames using same CCtx. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special : Referencing a NULL CDict means "return to no-dictionary mode". + * Note 1 : Currently, only one dictionary can be managed. + * Referencing a new dictionary effectively "discards" any previous one. + * Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */ +ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); + +/*! ZSTD_CCtx_refPrefix() : Requires v1.4.0+ + * Reference a prefix (single-usage dictionary) for next compressed frame. + * A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end). + * Decompression will need same prefix to properly regenerate data. + * Compressing with a prefix is similar in outcome as performing a diff and compressing it, + * but performs much faster, especially during decompression (compression speed is tunable with compression level). + * This method is compatible with LDM (long distance mode). + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary + * Note 1 : Prefix buffer is referenced. It **must** outlive compression. + * Its content must remain unmodified during compression. + * Note 2 : If the intention is to diff some large src data blob with some prior version of itself, + * ensure that the window size is large enough to contain the entire source. + * See ZSTD_c_windowLog. + * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. + * It's a CPU consuming operation, with non-negligible impact on latency. + * If there is a need to use the same prefix multiple times, consider loadDictionary instead. + * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent). + * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */ +ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, + const void* prefix, size_t prefixSize); + +/*! ZSTD_DCtx_loadDictionary() : Requires v1.4.0+ + * Create an internal DDict from dict buffer, to be used to decompress all future frames. + * The dictionary remains valid for all future frames, until explicitly invalidated, or + * a new dictionary is loaded. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary, + * meaning "return to no-dictionary mode". + * Note 1 : Loading a dictionary involves building tables, + * which has a non-negligible impact on CPU usage and latency. + * It's recommended to "load once, use many times", to amortize the cost + * Note 2 :`dict` content will be copied internally, so `dict` can be released after loading. + * Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead. + * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of + * how dictionary content is loaded and interpreted. + */ +ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); + +/*! ZSTD_DCtx_refDDict() : Requires v1.4.0+ + * Reference a prepared dictionary, to be used to decompress next frames. + * The dictionary remains active for decompression of future frames using same DCtx. + * + * If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function + * will store the DDict references in a table, and the DDict used for decompression + * will be determined at decompression time, as per the dict ID in the frame. + * The memory for the table is allocated on the first call to refDDict, and can be + * freed with ZSTD_freeDCtx(). + * + * If called with ZSTD_d_refMultipleDDicts disabled (the default), only one dictionary + * will be managed, and referencing a dictionary effectively "discards" any previous one. + * + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: referencing a NULL DDict means "return to no-dictionary mode". + * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx. + */ +ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + +/*! ZSTD_DCtx_refPrefix() : Requires v1.4.0+ + * Reference a prefix (single-usage dictionary) to decompress next frame. + * This is the reverse operation of ZSTD_CCtx_refPrefix(), + * and must use the same prefix as the one used during compression. + * Prefix is **only used once**. Reference is discarded at end of frame. + * End of frame is reached when ZSTD_decompressStream() returns 0. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary + * Note 2 : Prefix buffer is referenced. It **must** outlive decompression. + * Prefix buffer must remain unmodified up to the end of frame, + * reached when ZSTD_decompressStream() returns 0. + * Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent). + * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section) + * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. + * A full dictionary is more costly, as it requires building tables. + */ +ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, + const void* prefix, size_t prefixSize); + +/* === Memory management === */ + +/*! ZSTD_sizeof_*() : Requires v1.4.0+ + * These functions give the _current_ memory usage of selected object. + * Note that object memory usage can evolve (increase or decrease) over time. */ +ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); +ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); +ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); +ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); +ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_H_235446 */ + + +/* ************************************************************************************** + * ADVANCED AND EXPERIMENTAL FUNCTIONS + **************************************************************************************** + * The definitions in the following section are considered experimental. + * They are provided for advanced scenarios. + * They should never be used with a dynamic library, as prototypes may change in the future. + * Use them only in association with static linking. + * ***************************************************************************************/ + +#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) +#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY + +#if defined (__cplusplus) +extern "C" { +#endif + +/* This can be overridden externally to hide static symbols. */ +#ifndef ZSTDLIB_STATIC_API +# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_STATIC_API __declspec(dllexport) ZSTDLIB_VISIBLE +# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDLIB_STATIC_API __declspec(dllimport) ZSTDLIB_VISIBLE +# else +# define ZSTDLIB_STATIC_API ZSTDLIB_VISIBLE +# endif +#endif + +/**************************************************************************************** + * experimental API (static linking only) + **************************************************************************************** + * The following symbols and constants + * are not planned to join "stable API" status in the near future. + * They can still change in future versions. + * Some of them are planned to remain in the static_only section indefinitely. + * Some of them might be removed in the future (especially when redundant with existing stable functions) + * ***************************************************************************************/ + +#define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */ +#define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2) +#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */ +#define ZSTD_SKIPPABLEHEADERSIZE 8 + +/* compression parameter bounds */ +#define ZSTD_WINDOWLOG_MAX_32 30 +#define ZSTD_WINDOWLOG_MAX_64 31 +#define ZSTD_WINDOWLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) +#define ZSTD_WINDOWLOG_MIN 10 +#define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30) +#define ZSTD_HASHLOG_MIN 6 +#define ZSTD_CHAINLOG_MAX_32 29 +#define ZSTD_CHAINLOG_MAX_64 30 +#define ZSTD_CHAINLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64)) +#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN +#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) +#define ZSTD_SEARCHLOG_MIN 1 +#define ZSTD_MINMATCH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */ +#define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */ +#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX +#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */ +#define ZSTD_STRATEGY_MIN ZSTD_fast +#define ZSTD_STRATEGY_MAX ZSTD_btultra2 +#define ZSTD_BLOCKSIZE_MAX_MIN (1 << 10) /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */ + + +#define ZSTD_OVERLAPLOG_MIN 0 +#define ZSTD_OVERLAPLOG_MAX 9 + +#define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame + * requiring larger than (1< 0: + * If litLength != 0: + * rep == 1 --> offset == repeat_offset_1 + * rep == 2 --> offset == repeat_offset_2 + * rep == 3 --> offset == repeat_offset_3 + * If litLength == 0: + * rep == 1 --> offset == repeat_offset_2 + * rep == 2 --> offset == repeat_offset_3 + * rep == 3 --> offset == repeat_offset_1 - 1 + * + * Note: This field is optional. ZSTD_generateSequences() will calculate the value of + * 'rep', but repeat offsets do not necessarily need to be calculated from an external + * sequence provider perspective. For example, ZSTD_compressSequences() does not + * use this 'rep' field at all (as of now). + */ +} ZSTD_Sequence; + +typedef struct { + unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ + unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ + unsigned hashLog; /**< dispatch table : larger == faster, more memory */ + unsigned searchLog; /**< nb of searches : larger == more compression, slower */ + unsigned minMatch; /**< match length searched : larger == faster decompression, sometimes less compression */ + unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */ + ZSTD_strategy strategy; /**< see ZSTD_strategy definition above */ +} ZSTD_compressionParameters; + +typedef struct { + int contentSizeFlag; /**< 1: content size will be in frame header (when known) */ + int checksumFlag; /**< 1: generate a 32-bits checksum using XXH64 algorithm at end of frame, for error detection */ + int noDictIDFlag; /**< 1: no dictID will be saved into frame header (dictID is only useful for dictionary compression) */ +} ZSTD_frameParameters; + +typedef struct { + ZSTD_compressionParameters cParams; + ZSTD_frameParameters fParams; +} ZSTD_parameters; + +typedef enum { + ZSTD_dct_auto = 0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */ + ZSTD_dct_rawContent = 1, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */ + ZSTD_dct_fullDict = 2 /* refuses to load a dictionary if it does not respect Zstandard's specification, starting with ZSTD_MAGIC_DICTIONARY */ +} ZSTD_dictContentType_e; + +typedef enum { + ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */ + ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */ +} ZSTD_dictLoadMethod_e; + +typedef enum { + ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */ + ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number. + * Useful to save 4 bytes per generated frame. + * Decoder cannot recognise automatically this format, requiring this instruction. */ +} ZSTD_format_e; + +typedef enum { + /* Note: this enum controls ZSTD_d_forceIgnoreChecksum */ + ZSTD_d_validateChecksum = 0, + ZSTD_d_ignoreChecksum = 1 +} ZSTD_forceIgnoreChecksum_e; + +typedef enum { + /* Note: this enum controls ZSTD_d_refMultipleDDicts */ + ZSTD_rmd_refSingleDDict = 0, + ZSTD_rmd_refMultipleDDicts = 1 +} ZSTD_refMultipleDDicts_e; + +typedef enum { + /* Note: this enum and the behavior it controls are effectively internal + * implementation details of the compressor. They are expected to continue + * to evolve and should be considered only in the context of extremely + * advanced performance tuning. + * + * Zstd currently supports the use of a CDict in three ways: + * + * - The contents of the CDict can be copied into the working context. This + * means that the compression can search both the dictionary and input + * while operating on a single set of internal tables. This makes + * the compression faster per-byte of input. However, the initial copy of + * the CDict's tables incurs a fixed cost at the beginning of the + * compression. For small compressions (< 8 KB), that copy can dominate + * the cost of the compression. + * + * - The CDict's tables can be used in-place. In this model, compression is + * slower per input byte, because the compressor has to search two sets of + * tables. However, this model incurs no start-up cost (as long as the + * working context's tables can be reused). For small inputs, this can be + * faster than copying the CDict's tables. + * + * - The CDict's tables are not used at all, and instead we use the working + * context alone to reload the dictionary and use params based on the source + * size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict(). + * This method is effective when the dictionary sizes are very small relative + * to the input size, and the input size is fairly large to begin with. + * + * Zstd has a simple internal heuristic that selects which strategy to use + * at the beginning of a compression. However, if experimentation shows that + * Zstd is making poor choices, it is possible to override that choice with + * this enum. + */ + ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ + ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ + ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ + ZSTD_dictForceLoad = 3 /* Always reload the dictionary */ +} ZSTD_dictAttachPref_e; + +typedef enum { + ZSTD_lcm_auto = 0, /**< Automatically determine the compression mode based on the compression level. + * Negative compression levels will be uncompressed, and positive compression + * levels will be compressed. */ + ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be + * emitted if Huffman compression is not profitable. */ + ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */ +} ZSTD_literalCompressionMode_e; + +typedef enum { + /* Note: This enum controls features which are conditionally beneficial. + * Zstd can take a decision on whether or not to enable the feature (ZSTD_ps_auto), + * but setting the switch to ZSTD_ps_enable or ZSTD_ps_disable force enable/disable the feature. + */ + ZSTD_ps_auto = 0, /* Let the library automatically determine whether the feature shall be enabled */ + ZSTD_ps_enable = 1, /* Force-enable the feature */ + ZSTD_ps_disable = 2 /* Do not use the feature */ +} ZSTD_ParamSwitch_e; +#define ZSTD_paramSwitch_e ZSTD_ParamSwitch_e /* old name */ + +/*************************************** +* Frame header and size functions +***************************************/ + +/*! ZSTD_findDecompressedSize() : + * `src` should point to the start of a series of ZSTD encoded and/or skippable frames + * `srcSize` must be the _exact_ size of this series + * (i.e. there should be a frame boundary at `src + srcSize`) + * @return : - decompressed size of all data in all successive frames + * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN + * - if an error occurred: ZSTD_CONTENTSIZE_ERROR + * + * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. + * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. + * In which case, it's necessary to use streaming mode to decompress data. + * note 2 : decompressed size is always present when compression is done with ZSTD_compress() + * note 3 : decompressed size can be very large (64-bits value), + * potentially larger than what local system can handle as a single memory segment. + * In which case, it's necessary to use streaming mode to decompress data. + * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. + * Always ensure result fits within application's authorized limits. + * Each application can set its own limits. + * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to + * read each contained frame header. This is fast as most of the data is skipped, + * however it does mean that all frame data must be present and valid. */ +ZSTDLIB_STATIC_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); + +/*! ZSTD_decompressBound() : + * `src` should point to the start of a series of ZSTD encoded and/or skippable frames + * `srcSize` must be the _exact_ size of this series + * (i.e. there should be a frame boundary at `src + srcSize`) + * @return : - upper-bound for the decompressed size of all data in all successive frames + * - if an error occurred: ZSTD_CONTENTSIZE_ERROR + * + * note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame. + * note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`. + * in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value. + * note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by: + * upper-bound = # blocks * min(128 KB, Window_Size) + */ +ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize); + +/*! ZSTD_frameHeaderSize() : + * srcSize must be large enough, aka >= ZSTD_FRAMEHEADERSIZE_PREFIX. + * @return : size of the Frame Header, + * or an error code (if srcSize is too small) */ +ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); + +typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_FrameType_e; +#define ZSTD_frameType_e ZSTD_FrameType_e /* old name */ +typedef struct { + unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */ + unsigned long long windowSize; /* can be very large, up to <= frameContentSize */ + unsigned blockSizeMax; + ZSTD_FrameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */ + unsigned headerSize; + unsigned dictID; /* for ZSTD_skippableFrame, contains the skippable magic variant [0-15] */ + unsigned checksumFlag; + unsigned _reserved1; + unsigned _reserved2; +} ZSTD_FrameHeader; +#define ZSTD_frameHeader ZSTD_FrameHeader /* old name */ + +/*! ZSTD_getFrameHeader() : + * decode Frame Header into `zfhPtr`, or requires larger `srcSize`. + * @return : 0 => header is complete, `zfhPtr` is correctly filled, + * >0 => `srcSize` is too small, @return value is the wanted `srcSize` amount, `zfhPtr` is not filled, + * or an error code, which can be tested using ZSTD_isError() */ +ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize); +/*! ZSTD_getFrameHeader_advanced() : + * same as ZSTD_getFrameHeader(), + * with added capability to select a format (like ZSTD_f_zstd1_magicless) */ +ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); + +/*! ZSTD_decompressionMargin() : + * Zstd supports in-place decompression, where the input and output buffers overlap. + * In this case, the output buffer must be at least (Margin + Output_Size) bytes large, + * and the input buffer must be at the end of the output buffer. + * + * _______________________ Output Buffer ________________________ + * | | + * | ____ Input Buffer ____| + * | | | + * v v v + * |---------------------------------------|-----------|----------| + * ^ ^ ^ + * |___________________ Output_Size ___________________|_ Margin _| + * + * NOTE: See also ZSTD_DECOMPRESSION_MARGIN(). + * NOTE: This applies only to single-pass decompression through ZSTD_decompress() or + * ZSTD_decompressDCtx(). + * NOTE: This function supports multi-frame input. + * + * @param src The compressed frame(s) + * @param srcSize The size of the compressed frame(s) + * @returns The decompression margin or an error that can be checked with ZSTD_isError(). + */ +ZSTDLIB_STATIC_API size_t ZSTD_decompressionMargin(const void* src, size_t srcSize); + +/*! ZSTD_DECOMPRESS_MARGIN() : + * Similar to ZSTD_decompressionMargin(), but instead of computing the margin from + * the compressed frame, compute it from the original size and the blockSizeLog. + * See ZSTD_decompressionMargin() for details. + * + * WARNING: This macro does not support multi-frame input, the input must be a single + * zstd frame. If you need that support use the function, or implement it yourself. + * + * @param originalSize The original uncompressed size of the data. + * @param blockSize The block size == MIN(windowSize, ZSTD_BLOCKSIZE_MAX). + * Unless you explicitly set the windowLog smaller than + * ZSTD_BLOCKSIZELOG_MAX you can just use ZSTD_BLOCKSIZE_MAX. + */ +#define ZSTD_DECOMPRESSION_MARGIN(originalSize, blockSize) ((size_t)( \ + ZSTD_FRAMEHEADERSIZE_MAX /* Frame header */ + \ + 4 /* checksum */ + \ + ((originalSize) == 0 ? 0 : 3 * (((originalSize) + (blockSize) - 1) / blockSize)) /* 3 bytes per block */ + \ + (blockSize) /* One block of margin */ \ + )) + +typedef enum { + ZSTD_sf_noBlockDelimiters = 0, /* ZSTD_Sequence[] has no block delimiters, just sequences */ + ZSTD_sf_explicitBlockDelimiters = 1 /* ZSTD_Sequence[] contains explicit block delimiters */ +} ZSTD_SequenceFormat_e; +#define ZSTD_sequenceFormat_e ZSTD_SequenceFormat_e /* old name */ + +/*! ZSTD_sequenceBound() : + * `srcSize` : size of the input buffer + * @return : upper-bound for the number of sequences that can be generated + * from a buffer of srcSize bytes + * + * note : returns number of sequences - to get bytes, multiply by sizeof(ZSTD_Sequence). + */ +ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize); + +/*! ZSTD_generateSequences() : + * WARNING: This function is meant for debugging and informational purposes ONLY! + * Its implementation is flawed, and it will be deleted in a future version. + * It is not guaranteed to succeed, as there are several cases where it will give + * up and fail. You should NOT use this function in production code. + * + * This function is deprecated, and will be removed in a future version. + * + * Generate sequences using ZSTD_compress2(), given a source buffer. + * + * @param zc The compression context to be used for ZSTD_compress2(). Set any + * compression parameters you need on this context. + * @param outSeqs The output sequences buffer of size @p outSeqsSize + * @param outSeqsCapacity The size of the output sequences buffer. + * ZSTD_sequenceBound(srcSize) is an upper bound on the number + * of sequences that can be generated. + * @param src The source buffer to generate sequences from of size @p srcSize. + * @param srcSize The size of the source buffer. + * + * Each block will end with a dummy sequence + * with offset == 0, matchLength == 0, and litLength == length of last literals. + * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0) + * simply acts as a block delimiter. + * + * @returns The number of sequences generated, necessarily less than + * ZSTD_sequenceBound(srcSize), or an error code that can be checked + * with ZSTD_isError(). + */ +ZSTD_DEPRECATED("For debugging only, will be replaced by ZSTD_extractSequences()") +ZSTDLIB_STATIC_API size_t +ZSTD_generateSequences(ZSTD_CCtx* zc, + ZSTD_Sequence* outSeqs, size_t outSeqsCapacity, + const void* src, size_t srcSize); + +/*! ZSTD_mergeBlockDelimiters() : + * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals + * by merging them into the literals of the next sequence. + * + * As such, the final generated result has no explicit representation of block boundaries, + * and the final last literals segment is not represented in the sequences. + * + * The output of this function can be fed into ZSTD_compressSequences() with CCtx + * setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters + * @return : number of sequences left after merging + */ +ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize); + +/*! ZSTD_compressSequences() : + * Compress an array of ZSTD_Sequence, associated with @src buffer, into dst. + * @src contains the entire input (not just the literals). + * If @srcSize > sum(sequence.length), the remaining bytes are considered all literals + * If a dictionary is included, then the cctx should reference the dict (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.). + * The entire source is compressed into a single frame. + * + * The compression behavior changes based on cctx params. In particular: + * If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain + * no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on + * the block size derived from the cctx, and sequences may be split. This is the default setting. + * + * If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain + * valid block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided. + * + * When ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, it's possible to decide generating repcodes + * using the advanced parameter ZSTD_c_repcodeResolution. Repcodes will improve compression ratio, though the benefit + * can vary greatly depending on Sequences. On the other hand, repcode resolution is an expensive operation. + * By default, it's disabled at low (<10) compression levels, and enabled above the threshold (>=10). + * ZSTD_c_repcodeResolution makes it possible to directly manage this processing in either direction. + * + * If ZSTD_c_validateSequences == 0, this function blindly accepts the Sequences provided. Invalid Sequences cause undefined + * behavior. If ZSTD_c_validateSequences == 1, then the function will detect invalid Sequences (see doc/zstd_compression_format.md for + * specifics regarding offset/matchlength requirements) and then bail out and return an error. + * + * In addition to the two adjustable experimental params, there are other important cctx params. + * - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN. + * - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression. + * - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset + * is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md + * + * Note: Repcodes are, as of now, always re-calculated within this function, ZSTD_Sequence.rep is effectively unused. + * Dev Note: Once ability to ingest repcodes become available, the explicit block delims mode must respect those repcodes exactly, + * and cannot emit an RLE block that disagrees with the repcode history. + * @return : final compressed size, or a ZSTD error code. + */ +ZSTDLIB_STATIC_API size_t +ZSTD_compressSequences(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize); + + +/*! ZSTD_compressSequencesAndLiterals() : + * This is a variant of ZSTD_compressSequences() which, + * instead of receiving (src,srcSize) as input parameter, receives (literals,litSize), + * aka all the literals, already extracted and laid out into a single continuous buffer. + * This can be useful if the process generating the sequences also happens to generate the buffer of literals, + * thus skipping an extraction + caching stage. + * It's a speed optimization, useful when the right conditions are met, + * but it also features the following limitations: + * - Only supports explicit delimiter mode + * - Currently does not support Sequences validation (so input Sequences are trusted) + * - Not compatible with frame checksum, which must be disabled + * - If any block is incompressible, will fail and return an error + * - @litSize must be == sum of all @.litLength fields in @inSeqs. Any discrepancy will generate an error. + * - @litBufCapacity is the size of the underlying buffer into which literals are written, starting at address @literals. + * @litBufCapacity must be at least 8 bytes larger than @litSize. + * - @decompressedSize must be correct, and correspond to the sum of all Sequences. Any discrepancy will generate an error. + * @return : final compressed size, or a ZSTD error code. + */ +ZSTDLIB_STATIC_API size_t +ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const ZSTD_Sequence* inSeqs, size_t nbSequences, + const void* literals, size_t litSize, size_t litBufCapacity, + size_t decompressedSize); + + +/*! ZSTD_writeSkippableFrame() : + * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer. + * + * Skippable frames begin with a 4-byte magic number. There are 16 possible choices of magic number, + * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15. + * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, + * so the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant. + * + * Returns an error if destination buffer is not large enough, if the source size is not representable + * with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid). + * + * @return : number of bytes written or a ZSTD error. + */ +ZSTDLIB_STATIC_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + unsigned magicVariant); + +/*! ZSTD_readSkippableFrame() : + * Retrieves the content of a zstd skippable frame starting at @src, and writes it to @dst buffer. + * + * The parameter @magicVariant will receive the magicVariant that was supplied when the frame was written, + * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. + * This can be NULL if the caller is not interested in the magicVariant. + * + * Returns an error if destination buffer is not large enough, or if the frame is not skippable. + * + * @return : number of bytes written or a ZSTD error. + */ +ZSTDLIB_STATIC_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, + unsigned* magicVariant, + const void* src, size_t srcSize); + +/*! ZSTD_isSkippableFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame. + */ +ZSTDLIB_STATIC_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size); + + + +/*************************************** +* Memory management +***************************************/ + +/*! ZSTD_estimate*() : + * These functions make it possible to estimate memory usage + * of a future {D,C}Ctx, before its creation. + * This is useful in combination with ZSTD_initStatic(), + * which makes it possible to employ a static buffer for ZSTD_CCtx* state. + * + * ZSTD_estimateCCtxSize() will provide a memory budget large enough + * to compress data of any size using one-shot compression ZSTD_compressCCtx() or ZSTD_compress2() + * associated with any compression level up to max specified one. + * The estimate will assume the input may be arbitrarily large, + * which is the worst case. + * + * Note that the size estimation is specific for one-shot compression, + * it is not valid for streaming (see ZSTD_estimateCStreamSize*()) + * nor other potential ways of using a ZSTD_CCtx* state. + * + * When srcSize can be bound by a known and rather "small" value, + * this knowledge can be used to provide a tighter budget estimation + * because the ZSTD_CCtx* state will need less memory for small inputs. + * This tighter estimation can be provided by employing more advanced functions + * ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(), + * and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter(). + * Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits. + * + * Note : only single-threaded compression is supported. + * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int maxCompressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void); + +/*! ZSTD_estimateCStreamSize() : + * ZSTD_estimateCStreamSize() will provide a memory budget large enough for streaming compression + * using any compression level up to the max specified one. + * It will also consider src size to be arbitrarily "large", which is a worst case scenario. + * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation. + * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. + * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. + * Note : CStream size estimation is only correct for single-threaded compression. + * ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. + * Note 2 : ZSTD_estimateCStreamSize* functions are not compatible with the Block-Level Sequence Producer API at this time. + * Size estimates assume that no external sequence producer is registered. + * + * ZSTD_DStream memory budget depends on frame's window Size. + * This information can be passed manually, using ZSTD_estimateDStreamSize, + * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame(); + * Any frame requesting a window size larger than max specified one will be rejected. + * Note : if streaming is init with function ZSTD_init?Stream_usingDict(), + * an internal ?Dict will be created, which additional size is not estimated here. + * In this case, get total size by adding ZSTD_estimate?DictSize + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int maxCompressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize(size_t maxWindowSize); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize); + +/*! ZSTD_estimate?DictSize() : + * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict(). + * ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced(). + * Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller. + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod); + +/*! ZSTD_initStatic*() : + * Initialize an object using a pre-allocated fixed-size buffer. + * workspace: The memory area to emplace the object into. + * Provided pointer *must be 8-bytes aligned*. + * Buffer must outlive object. + * workspaceSize: Use ZSTD_estimate*Size() to determine + * how large workspace must be to support target scenario. + * @return : pointer to object (same address as workspace, just different type), + * or NULL if error (size too small, incorrect alignment, etc.) + * Note : zstd will never resize nor malloc() when using a static buffer. + * If the object requires more memory than available, + * zstd will just error out (typically ZSTD_error_memory_allocation). + * Note 2 : there is no corresponding "free" function. + * Since workspace is allocated externally, it must be freed externally too. + * Note 3 : cParams : use ZSTD_getCParams() to convert a compression level + * into its associated cParams. + * Limitation 1 : currently not compatible with internal dictionary creation, triggered by + * ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict(). + * Limitation 2 : static cctx currently not compatible with multi-threading. + * Limitation 3 : static dctx is incompatible with legacy support. + */ +ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize); +ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */ + +ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize); +ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */ + +ZSTDLIB_STATIC_API const ZSTD_CDict* ZSTD_initStaticCDict( + void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams); + +ZSTDLIB_STATIC_API const ZSTD_DDict* ZSTD_initStaticDDict( + void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType); + + +/*! Custom memory allocation : + * These prototypes make it possible to pass your own allocation/free functions. + * ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below. + * All allocation/free operations will be completed using these custom variants instead of regular ones. + */ +typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); +typedef void (*ZSTD_freeFunction) (void* opaque, void* address); +typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; +static +#ifdef __GNUC__ +__attribute__((__unused__)) +#endif + +#if defined(__clang__) && __clang_major__ >= 5 +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" +#endif +ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ +#if defined(__clang__) && __clang_major__ >= 5 +#pragma clang diagnostic pop +#endif + +ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); + +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams, + ZSTD_customMem customMem); + +/*! Thread pool : + * These prototypes make it possible to share a thread pool among multiple compression contexts. + * This can limit resources for applications with multiple threads where each one uses + * a threaded compression mode (via ZSTD_c_nbWorkers parameter). + * ZSTD_createThreadPool creates a new thread pool with a given number of threads. + * Note that the lifetime of such pool must exist while being used. + * ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value + * to use an internal thread pool). + * ZSTD_freeThreadPool frees a thread pool, accepts NULL pointer. + */ +typedef struct POOL_ctx_s ZSTD_threadPool; +ZSTDLIB_STATIC_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads); +ZSTDLIB_STATIC_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); /* accept NULL pointer */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool); + + +/* + * This API is temporary and is expected to change or disappear in the future! + */ +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced2( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + const ZSTD_CCtx_params* cctxParams, + ZSTD_customMem customMem); + +ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_advanced( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_customMem customMem); + + +/*************************************** +* Advanced compression functions +***************************************/ + +/*! ZSTD_createCDict_byReference() : + * Create a digested dictionary for compression + * Dictionary content is just referenced, not duplicated. + * As a consequence, `dictBuffer` **must** outlive CDict, + * and its content must remain unmodified throughout the lifetime of CDict. + * note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */ +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); + +/*! ZSTD_getCParams() : + * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. + * `estimatedSrcSize` value is optional, select 0 if not known */ +ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); + +/*! ZSTD_getParams() : + * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`. + * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */ +ZSTDLIB_STATIC_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); + +/*! ZSTD_checkCParams() : + * Ensure param values remain within authorized range. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */ +ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); + +/*! ZSTD_adjustCParams() : + * optimize params for a given `srcSize` and `dictSize`. + * `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN. + * `dictSize` must be `0` when there is no dictionary. + * cPar can be invalid : all parameters will be clamped within valid range in the @return struct. + * This function never fails (wide contract) */ +ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); + +/*! ZSTD_CCtx_setCParams() : + * Set all parameters provided within @p cparams into the working @p cctx. + * Note : if modifying parameters during compression (MT mode only), + * note that changes to the .windowLog parameter will be ignored. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + * On failure, no parameters are updated. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams); + +/*! ZSTD_CCtx_setFParams() : + * Set all parameters provided within @p fparams into the working @p cctx. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams); + +/*! ZSTD_CCtx_setParams() : + * Set all parameters provided within @p params into the working @p cctx. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params); + +/*! ZSTD_compress_advanced() : + * Note : this function is now DEPRECATED. + * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters. + * This prototype will generate compilation warnings. */ +ZSTD_DEPRECATED("use ZSTD_compress2") +ZSTDLIB_STATIC_API +size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params); + +/*! ZSTD_compress_usingCDict_advanced() : + * Note : this function is now DEPRECATED. + * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters. + * This prototype will generate compilation warnings. */ +ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary") +ZSTDLIB_STATIC_API +size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams); + + +/*! ZSTD_CCtx_loadDictionary_byReference() : + * Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx. + * It saves some memory, but also requires that `dict` outlives its usage within `cctx` */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); + +/*! ZSTD_CCtx_loadDictionary_advanced() : + * Same as ZSTD_CCtx_loadDictionary(), but gives finer control over + * how to load the dictionary (by copy ? by reference ?) + * and how to interpret it (automatic ? force raw mode ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_CCtx_refPrefix_advanced() : + * Same as ZSTD_CCtx_refPrefix(), but gives finer control over + * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); + +/* === experimental parameters === */ +/* these parameters can be used with ZSTD_setParameter() + * they are not guaranteed to remain supported in the future */ + + /* Enables rsyncable mode, + * which makes compressed files more rsync friendly + * by adding periodic synchronization points to the compressed data. + * The target average block size is ZSTD_c_jobSize / 2. + * It's possible to modify the job size to increase or decrease + * the granularity of the synchronization point. + * Once the jobSize is smaller than the window size, + * it will result in compression ratio degradation. + * NOTE 1: rsyncable mode only works when multithreading is enabled. + * NOTE 2: rsyncable performs poorly in combination with long range mode, + * since it will decrease the effectiveness of synchronization points, + * though mileage may vary. + * NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s. + * If the selected compression level is already running significantly slower, + * the overall speed won't be significantly impacted. + */ + #define ZSTD_c_rsyncable ZSTD_c_experimentalParam1 + +/* Select a compression format. + * The value must be of type ZSTD_format_e. + * See ZSTD_format_e enum definition for details */ +#define ZSTD_c_format ZSTD_c_experimentalParam2 + +/* Force back-reference distances to remain < windowSize, + * even when referencing into Dictionary content (default:0) */ +#define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam3 + +/* Controls whether the contents of a CDict + * are used in place, or copied into the working context. + * Accepts values from the ZSTD_dictAttachPref_e enum. + * See the comments on that enum for an explanation of the feature. */ +#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4 + +/* Controlled with ZSTD_ParamSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never compress literals. + * Set to ZSTD_ps_enable to always compress literals. (Note: uncompressed literals + * may still be emitted if huffman is not beneficial to use.) + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * literals compression based on the compression parameters - specifically, + * negative compression levels do not use literal compression. + */ +#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5 + +/* User's best guess of source size. + * Hint is not valid when srcSizeHint == 0. + * There is no guarantee that hint is close to actual source size, + * but compression ratio may regress significantly if guess considerably underestimates */ +#define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7 + +/* Controls whether the new and experimental "dedicated dictionary search + * structure" can be used. This feature is still rough around the edges, be + * prepared for surprising behavior! + * + * How to use it: + * + * When using a CDict, whether to use this feature or not is controlled at + * CDict creation, and it must be set in a CCtxParams set passed into that + * construction (via ZSTD_createCDict_advanced2()). A compression will then + * use the feature or not based on how the CDict was constructed; the value of + * this param, set in the CCtx, will have no effect. + * + * However, when a dictionary buffer is passed into a CCtx, such as via + * ZSTD_CCtx_loadDictionary(), this param can be set on the CCtx to control + * whether the CDict that is created internally can use the feature or not. + * + * What it does: + * + * Normally, the internal data structures of the CDict are analogous to what + * would be stored in a CCtx after compressing the contents of a dictionary. + * To an approximation, a compression using a dictionary can then use those + * data structures to simply continue what is effectively a streaming + * compression where the simulated compression of the dictionary left off. + * Which is to say, the search structures in the CDict are normally the same + * format as in the CCtx. + * + * It is possible to do better, since the CDict is not like a CCtx: the search + * structures are written once during CDict creation, and then are only read + * after that, while the search structures in the CCtx are both read and + * written as the compression goes along. This means we can choose a search + * structure for the dictionary that is read-optimized. + * + * This feature enables the use of that different structure. + * + * Note that some of the members of the ZSTD_compressionParameters struct have + * different semantics and constraints in the dedicated search structure. It is + * highly recommended that you simply set a compression level in the CCtxParams + * you pass into the CDict creation call, and avoid messing with the cParams + * directly. + * + * Effects: + * + * This will only have any effect when the selected ZSTD_strategy + * implementation supports this feature. Currently, that's limited to + * ZSTD_greedy, ZSTD_lazy, and ZSTD_lazy2. + * + * Note that this means that the CDict tables can no longer be copied into the + * CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be + * usable. The dictionary can only be attached or reloaded. + * + * In general, you should expect compression to be faster--sometimes very much + * so--and CDict creation to be slightly slower. Eventually, we will probably + * make this mode the default. + */ +#define ZSTD_c_enableDedicatedDictSearch ZSTD_c_experimentalParam8 + +/* ZSTD_c_stableInBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells the compressor that input data presented with ZSTD_inBuffer + * will ALWAYS be the same between calls. + * Technically, the @src pointer must never be changed, + * and the @pos field can only be updated by zstd. + * However, it's possible to increase the @size field, + * allowing scenarios where more data can be appended after compressions starts. + * These conditions are checked by the compressor, + * and compression will fail if they are not respected. + * Also, data in the ZSTD_inBuffer within the range [src, src + pos) + * MUST not be modified during compression or it will result in data corruption. + * + * When this flag is enabled zstd won't allocate an input window buffer, + * because the user guarantees it can reference the ZSTD_inBuffer until + * the frame is complete. But, it will still allocate an output buffer + * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also + * avoid the memcpy() from the input buffer to the input window buffer. + * + * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using + * this flag is ALWAYS memory safe, and will never access out-of-bounds + * memory. However, compression WILL fail if conditions are not respected. + * + * WARNING: The data in the ZSTD_inBuffer in the range [src, src + pos) MUST + * not be modified during compression or it will result in data corruption. + * This is because zstd needs to reference data in the ZSTD_inBuffer to find + * matches. Normally zstd maintains its own window buffer for this purpose, + * but passing this flag tells zstd to rely on user provided buffer instead. + */ +#define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9 + +/* ZSTD_c_stableOutBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells he compressor that the ZSTD_outBuffer will not be resized between + * calls. Specifically: (out.size - out.pos) will never grow. This gives the + * compressor the freedom to say: If the compressed data doesn't fit in the + * output buffer then return ZSTD_error_dstSizeTooSmall. This allows us to + * always decompress directly into the output buffer, instead of decompressing + * into an internal buffer and copying to the output buffer. + * + * When this flag is enabled zstd won't allocate an output buffer, because + * it can write directly to the ZSTD_outBuffer. It will still allocate the + * input window buffer (see ZSTD_c_stableInBuffer). + * + * Zstd will check that (out.size - out.pos) never grows and return an error + * if it does. While not strictly necessary, this should prevent surprises. + */ +#define ZSTD_c_stableOutBuffer ZSTD_c_experimentalParam10 + +/* ZSTD_c_blockDelimiters + * Default is 0 == ZSTD_sf_noBlockDelimiters. + * + * For use with sequence compression API: ZSTD_compressSequences(). + * + * Designates whether or not the given array of ZSTD_Sequence contains block delimiters + * and last literals, which are defined as sequences with offset == 0 and matchLength == 0. + * See the definition of ZSTD_Sequence for more specifics. + */ +#define ZSTD_c_blockDelimiters ZSTD_c_experimentalParam11 + +/* ZSTD_c_validateSequences + * Default is 0 == disabled. Set to 1 to enable sequence validation. + * + * For use with sequence compression API: ZSTD_compressSequences*(). + * Designates whether or not provided sequences are validated within ZSTD_compressSequences*() + * during function execution. + * + * When Sequence validation is disabled (default), Sequences are compressed as-is, + * so they must correct, otherwise it would result in a corruption error. + * + * Sequence validation adds some protection, by ensuring that all values respect boundary conditions. + * If a Sequence is detected invalid (see doc/zstd_compression_format.md for + * specifics regarding offset/matchlength requirements) then the function will bail out and + * return an error. + */ +#define ZSTD_c_validateSequences ZSTD_c_experimentalParam12 + +/* ZSTD_c_blockSplitterLevel + * note: this parameter only influences the first splitter stage, + * which is active before producing the sequences. + * ZSTD_c_splitAfterSequences controls the next splitter stage, + * which is active after sequence production. + * Note that both can be combined. + * Allowed values are between 0 and ZSTD_BLOCKSPLITTER_LEVEL_MAX included. + * 0 means "auto", which will select a value depending on current ZSTD_c_strategy. + * 1 means no splitting. + * Then, values from 2 to 6 are sorted in increasing cpu load order. + * + * Note that currently the first block is never split, + * to ensure expansion guarantees in presence of incompressible data. + */ +#define ZSTD_BLOCKSPLITTER_LEVEL_MAX 6 +#define ZSTD_c_blockSplitterLevel ZSTD_c_experimentalParam20 + +/* ZSTD_c_splitAfterSequences + * This is a stronger splitter algorithm, + * based on actual sequences previously produced by the selected parser. + * It's also slower, and as a consequence, mostly used for high compression levels. + * While the post-splitter does overlap with the pre-splitter, + * both can nonetheless be combined, + * notably with ZSTD_c_blockSplitterLevel at ZSTD_BLOCKSPLITTER_LEVEL_MAX, + * resulting in higher compression ratio than just one of them. + * + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never use block splitter. + * Set to ZSTD_ps_enable to always use block splitter. + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * block splitting based on the compression parameters. + */ +#define ZSTD_c_splitAfterSequences ZSTD_c_experimentalParam13 + +/* ZSTD_c_useRowMatchFinder + * Controlled with ZSTD_ParamSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never use row-based matchfinder. + * Set to ZSTD_ps_enable to force usage of row-based matchfinder. + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * the row-based matchfinder based on support for SIMD instructions and the window log. + * Note that this only pertains to compression strategies: greedy, lazy, and lazy2 + */ +#define ZSTD_c_useRowMatchFinder ZSTD_c_experimentalParam14 + +/* ZSTD_c_deterministicRefPrefix + * Default is 0 == disabled. Set to 1 to enable. + * + * Zstd produces different results for prefix compression when the prefix is + * directly adjacent to the data about to be compressed vs. when it isn't. + * This is because zstd detects that the two buffers are contiguous and it can + * use a more efficient match finding algorithm. However, this produces different + * results than when the two buffers are non-contiguous. This flag forces zstd + * to always load the prefix in non-contiguous mode, even if it happens to be + * adjacent to the data, to guarantee determinism. + * + * If you really care about determinism when using a dictionary or prefix, + * like when doing delta compression, you should select this option. It comes + * at a speed penalty of about ~2.5% if the dictionary and data happened to be + * contiguous, and is free if they weren't contiguous. We don't expect that + * intentionally making the dictionary and data contiguous will be worth the + * cost to memcpy() the data. + */ +#define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15 + +/* ZSTD_c_prefetchCDictTables + * Controlled with ZSTD_ParamSwitch_e enum. Default is ZSTD_ps_auto. + * + * In some situations, zstd uses CDict tables in-place rather than copying them + * into the working context. (See docs on ZSTD_dictAttachPref_e above for details). + * In such situations, compression speed is seriously impacted when CDict tables are + * "cold" (outside CPU cache). This parameter instructs zstd to prefetch CDict tables + * when they are used in-place. + * + * For sufficiently small inputs, the cost of the prefetch will outweigh the benefit. + * For sufficiently large inputs, zstd will by default memcpy() CDict tables + * into the working context, so there is no need to prefetch. This parameter is + * targeted at a middle range of input sizes, where a prefetch is cheap enough to be + * useful but memcpy() is too expensive. The exact range of input sizes where this + * makes sense is best determined by careful experimentation. + * + * Note: for this parameter, ZSTD_ps_auto is currently equivalent to ZSTD_ps_disable, + * but in the future zstd may conditionally enable this feature via an auto-detection + * heuristic for cold CDicts. + * Use ZSTD_ps_disable to opt out of prefetching under any circumstances. + */ +#define ZSTD_c_prefetchCDictTables ZSTD_c_experimentalParam16 + +/* ZSTD_c_enableSeqProducerFallback + * Allowed values are 0 (disable) and 1 (enable). The default setting is 0. + * + * Controls whether zstd will fall back to an internal sequence producer if an + * external sequence producer is registered and returns an error code. This fallback + * is block-by-block: the internal sequence producer will only be called for blocks + * where the external sequence producer returns an error code. Fallback parsing will + * follow any other cParam settings, such as compression level, the same as in a + * normal (fully-internal) compression operation. + * + * The user is strongly encouraged to read the full Block-Level Sequence Producer API + * documentation (below) before setting this parameter. */ +#define ZSTD_c_enableSeqProducerFallback ZSTD_c_experimentalParam17 + +/* ZSTD_c_maxBlockSize + * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB). + * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default. + * + * This parameter can be used to set an upper bound on the blocksize + * that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper + * bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make + * compressBound() inaccurate). Only currently meant to be used for testing. + */ +#define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18 + +/* ZSTD_c_repcodeResolution + * This parameter only has an effect if ZSTD_c_blockDelimiters is + * set to ZSTD_sf_explicitBlockDelimiters (may change in the future). + * + * This parameter affects how zstd parses external sequences, + * provided via the ZSTD_compressSequences*() API + * or from an external block-level sequence producer. + * + * If set to ZSTD_ps_enable, the library will check for repeated offsets within + * external sequences, even if those repcodes are not explicitly indicated in + * the "rep" field. Note that this is the only way to exploit repcode matches + * while using compressSequences*() or an external sequence producer, since zstd + * currently ignores the "rep" field of external sequences. + * + * If set to ZSTD_ps_disable, the library will not exploit repeated offsets in + * external sequences, regardless of whether the "rep" field has been set. This + * reduces sequence compression overhead by about 25% while sacrificing some + * compression ratio. + * + * The default value is ZSTD_ps_auto, for which the library will enable/disable + * based on compression level (currently: level<10 disables, level>=10 enables). + */ +#define ZSTD_c_repcodeResolution ZSTD_c_experimentalParam19 +#define ZSTD_c_searchForExternalRepcodes ZSTD_c_experimentalParam19 /* older name */ + + +/*! ZSTD_CCtx_getParameter() : + * Get the requested compression parameter value, selected by enum ZSTD_cParameter, + * and store it into int* value. + * @return : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value); + + +/*! ZSTD_CCtx_params : + * Quick howto : + * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure + * - ZSTD_CCtxParams_setParameter() : Push parameters one by one into + * an existing ZSTD_CCtx_params structure. + * This is similar to + * ZSTD_CCtx_setParameter(). + * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to + * an existing CCtx. + * These parameters will be applied to + * all subsequent frames. + * - ZSTD_compressStream2() : Do compression using the CCtx. + * - ZSTD_freeCCtxParams() : Free the memory, accept NULL pointer. + * + * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams() + * for static allocation of CCtx for single-threaded compression. + */ +ZSTDLIB_STATIC_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); +ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); /* accept NULL pointer */ + +/*! ZSTD_CCtxParams_reset() : + * Reset params to default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params); + +/*! ZSTD_CCtxParams_init() : + * Initializes the compression parameters of cctxParams according to + * compression level. All other parameters are reset to their default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel); + +/*! ZSTD_CCtxParams_init_advanced() : + * Initializes the compression and frame parameters of cctxParams according to + * params. All other parameters are reset to their default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); + +/*! ZSTD_CCtxParams_setParameter() : Requires v1.4.0+ + * Similar to ZSTD_CCtx_setParameter. + * Set one compression parameter, selected by enum ZSTD_cParameter. + * Parameters must be applied to a ZSTD_CCtx using + * ZSTD_CCtx_setParametersUsingCCtxParams(). + * @result : a code representing success or failure (which can be tested with + * ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); + +/*! ZSTD_CCtxParams_getParameter() : + * Similar to ZSTD_CCtx_getParameter. + * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value); + +/*! ZSTD_CCtx_setParametersUsingCCtxParams() : + * Apply a set of ZSTD_CCtx_params to the compression context. + * This can be done even after compression is started, + * if nbWorkers==0, this will have no impact until a new compression is started. + * if nbWorkers>=1, new parameters will be picked up at next job, + * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParametersUsingCCtxParams( + ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params); + +/*! ZSTD_compressStream2_simpleArgs() : + * Same as ZSTD_compressStream2(), + * but using only integral types as arguments. + * This variant might be helpful for binders from dynamic languages + * which have troubles handling structures containing memory pointers. + */ +ZSTDLIB_STATIC_API size_t ZSTD_compressStream2_simpleArgs ( + ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos, + ZSTD_EndDirective endOp); + + +/*************************************** +* Advanced decompression functions +***************************************/ + +/*! ZSTD_isFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. + * Note 3 : Skippable Frame Identifiers are considered valid. */ +ZSTDLIB_STATIC_API unsigned ZSTD_isFrame(const void* buffer, size_t size); + +/*! ZSTD_createDDict_byReference() : + * Create a digested dictionary, ready to start decompression operation without startup delay. + * Dictionary content is referenced, and therefore stays in dictBuffer. + * It is important that dictBuffer outlives DDict, + * it must remain read accessible throughout the lifetime of DDict */ +ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize); + +/*! ZSTD_DCtx_loadDictionary_byReference() : + * Same as ZSTD_DCtx_loadDictionary(), + * but references `dict` content instead of copying it into `dctx`. + * This saves memory if `dict` remains around., + * However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); + +/*! ZSTD_DCtx_loadDictionary_advanced() : + * Same as ZSTD_DCtx_loadDictionary(), + * but gives direct control over + * how to load the dictionary (by copy ? by reference ?) + * and how to interpret it (automatic ? force raw mode ? full mode only ?). */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_DCtx_refPrefix_advanced() : + * Same as ZSTD_DCtx_refPrefix(), but gives finer control over + * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_DCtx_setMaxWindowSize() : + * Refuses allocating internal buffers for frames requiring a window size larger than provided limit. + * This protects a decoder context from reserving too much memory for itself (potential attack scenario). + * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. + * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + * @return : 0, or an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize); + +/*! ZSTD_DCtx_getParameter() : + * Get the requested decompression parameter value, selected by enum ZSTD_dParameter, + * and store it into int* value. + * @return : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value); + +/* ZSTD_d_format + * experimental parameter, + * allowing selection between ZSTD_format_e input compression formats + */ +#define ZSTD_d_format ZSTD_d_experimentalParam1 +/* ZSTD_d_stableOutBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same + * between calls, except for the modifications that zstd makes to pos (the + * caller must not modify pos). This is checked by the decompressor, and + * decompression will fail if it ever changes. Therefore the ZSTD_outBuffer + * MUST be large enough to fit the entire decompressed frame. This will be + * checked when the frame content size is known. The data in the ZSTD_outBuffer + * in the range [dst, dst + pos) MUST not be modified during decompression + * or you will get data corruption. + * + * When this flag is enabled zstd won't allocate an output buffer, because + * it can write directly to the ZSTD_outBuffer, but it will still allocate + * an input buffer large enough to fit any compressed block. This will also + * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer. + * If you need to avoid the input buffer allocation use the buffer-less + * streaming API. + * + * NOTE: So long as the ZSTD_outBuffer always points to valid memory, using + * this flag is ALWAYS memory safe, and will never access out-of-bounds + * memory. However, decompression WILL fail if you violate the preconditions. + * + * WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST + * not be modified during decompression or you will get data corruption. This + * is because zstd needs to reference data in the ZSTD_outBuffer to regenerate + * matches. Normally zstd maintains its own buffer for this purpose, but passing + * this flag tells zstd to use the user provided buffer. + */ +#define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2 + +/* ZSTD_d_forceIgnoreChecksum + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable + * + * Tells the decompressor to skip checksum validation during decompression, regardless + * of whether checksumming was specified during compression. This offers some + * slight performance benefits, and may be useful for debugging. + * Param has values of type ZSTD_forceIgnoreChecksum_e + */ +#define ZSTD_d_forceIgnoreChecksum ZSTD_d_experimentalParam3 + +/* ZSTD_d_refMultipleDDicts + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable + * + * If enabled and dctx is allocated on the heap, then additional memory will be allocated + * to store references to multiple ZSTD_DDict. That is, multiple calls of ZSTD_refDDict() + * using a given ZSTD_DCtx, rather than overwriting the previous DDict reference, will instead + * store all references. At decompression time, the appropriate dictID is selected + * from the set of DDicts based on the dictID in the frame. + * + * Usage is simply calling ZSTD_refDDict() on multiple dict buffers. + * + * Param has values of byte ZSTD_refMultipleDDicts_e + * + * WARNING: Enabling this parameter and calling ZSTD_DCtx_refDDict(), will trigger memory + * allocation for the hash table. ZSTD_freeDCtx() also frees this memory. + * Memory is allocated as per ZSTD_DCtx::customMem. + * + * Although this function allocates memory for the table, the user is still responsible for + * memory management of the underlying ZSTD_DDict* themselves. + */ +#define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4 + +/* ZSTD_d_disableHuffmanAssembly + * Set to 1 to disable the Huffman assembly implementation. + * The default value is 0, which allows zstd to use the Huffman assembly + * implementation if available. + * + * This parameter can be used to disable Huffman assembly at runtime. + * If you want to disable it at compile time you can define the macro + * ZSTD_DISABLE_ASM. + */ +#define ZSTD_d_disableHuffmanAssembly ZSTD_d_experimentalParam5 + +/* ZSTD_d_maxBlockSize + * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB). + * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default. + * + * Forces the decompressor to reject blocks whose content size is + * larger than the configured maxBlockSize. When maxBlockSize is + * larger than the windowSize, the windowSize is used instead. + * This saves memory on the decoder when you know all blocks are small. + * + * This option is typically used in conjunction with ZSTD_c_maxBlockSize. + * + * WARNING: This causes the decoder to reject otherwise valid frames + * that have block sizes larger than the configured maxBlockSize. + */ +#define ZSTD_d_maxBlockSize ZSTD_d_experimentalParam6 + + +/*! ZSTD_DCtx_setFormat() : + * This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter(). + * Instruct the decoder context about what kind of data to decode next. + * This instruction is mandatory to decode data without a fully-formed header, + * such ZSTD_f_zstd1_magicless for example. + * @return : 0, or an error code (which can be tested using ZSTD_isError()). */ +ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead") +ZSTDLIB_STATIC_API +size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); + +/*! ZSTD_decompressStream_simpleArgs() : + * Same as ZSTD_decompressStream(), + * but using only integral types as arguments. + * This can be helpful for binders from dynamic languages + * which have troubles handling structures containing memory pointers. + */ +ZSTDLIB_STATIC_API size_t ZSTD_decompressStream_simpleArgs ( + ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos); + + +/******************************************************************** +* Advanced streaming functions +* Warning : most of these functions are now redundant with the Advanced API. +* Once Advanced API reaches "stable" status, +* redundant functions will be deprecated, and then at some point removed. +********************************************************************/ + +/*===== Advanced Streaming compression functions =====*/ + +/*! ZSTD_initCStream_srcSize() : + * This function is DEPRECATED, and equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * + * pledgedSrcSize must be correct. If it is not known at init time, use + * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, + * "0" also disables frame content size field. It may be enabled in the future. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, + int compressionLevel, + unsigned long long pledgedSrcSize); + +/*! ZSTD_initCStream_usingDict() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); + * + * Creates of an internal CDict (incompatible with static CCtx), except if + * dict == NULL or dictSize < 8, in which case no dict is used. + * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if + * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + int compressionLevel); + +/*! ZSTD_initCStream_advanced() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setParams(zcs, params); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); + * + * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy. + * pledgedSrcSize must be correct. + * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + ZSTD_parameters params, + unsigned long long pledgedSrcSize); + +/*! ZSTD_initCStream_usingCDict() : + * This function is DEPRECATED, and equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, cdict); + * + * note : cdict will just be referenced, and must outlive compression session + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); + +/*! ZSTD_initCStream_usingCDict_advanced() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setFParams(zcs, fParams); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * ZSTD_CCtx_refCDict(zcs, cdict); + * + * same as ZSTD_initCStream_usingCDict(), with control over frame parameters. + * pledgedSrcSize must be correct. If srcSize is not known at init time, use + * value ZSTD_CONTENTSIZE_UNKNOWN. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams, + unsigned long long pledgedSrcSize); + +/*! ZSTD_resetCStream() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * Note: ZSTD_resetCStream() interprets pledgedSrcSize == 0 as ZSTD_CONTENTSIZE_UNKNOWN, but + * ZSTD_CCtx_setPledgedSrcSize() does not do the same, so ZSTD_CONTENTSIZE_UNKNOWN must be + * explicitly specified. + * + * start a new frame, using same parameters from previous frame. + * This is typically useful to skip dictionary loading stage, since it will reuse it in-place. + * Note that zcs must be init at least once before using ZSTD_resetCStream(). + * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN. + * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. + * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, + * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. + * @return : 0, or an error code (which can be tested using ZSTD_isError()) + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); + + +typedef struct { + unsigned long long ingested; /* nb input bytes read and buffered */ + unsigned long long consumed; /* nb input bytes actually compressed */ + unsigned long long produced; /* nb of compressed bytes generated and buffered */ + unsigned long long flushed; /* nb of compressed bytes flushed : not provided; can be tracked from caller side */ + unsigned currentJobID; /* MT only : latest started job nb */ + unsigned nbActiveWorkers; /* MT only : nb of workers actively compressing at probe time */ +} ZSTD_frameProgression; + +/* ZSTD_getFrameProgression() : + * tells how much data has been ingested (read from input) + * consumed (input actually compressed) and produced (output) for current frame. + * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed. + * Aggregates progression inside active worker threads. + */ +ZSTDLIB_STATIC_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx); + +/*! ZSTD_toFlushNow() : + * Tell how many bytes are ready to be flushed immediately. + * Useful for multithreading scenarios (nbWorkers >= 1). + * Probe the oldest active job, defined as oldest job not yet entirely flushed, + * and check its output buffer. + * @return : amount of data stored in oldest job and ready to be flushed immediately. + * if @return == 0, it means either : + * + there is no active job (could be checked with ZSTD_frameProgression()), or + * + oldest job is still actively compressing data, + * but everything it has produced has also been flushed so far, + * therefore flush speed is limited by production speed of oldest job + * irrespective of the speed of concurrent (and newer) jobs. + */ +ZSTDLIB_STATIC_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); + + +/*===== Advanced Streaming decompression functions =====*/ + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_loadDictionary(zds, dict, dictSize); + * + * note: no dictionary will be used if dict == NULL or dictSize < 8 + */ +ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_loadDictionary, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_refDDict(zds, ddict); + * + * note : ddict is referenced, it must outlive decompression session + */ +ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_refDDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * + * reuse decompression parameters from previous init; saves dictionary loading + */ +ZSTD_DEPRECATED("use ZSTD_DCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); + + +/* ********************* BLOCK-LEVEL SEQUENCE PRODUCER API ********************* + * + * *** OVERVIEW *** + * The Block-Level Sequence Producer API allows users to provide their own custom + * sequence producer which libzstd invokes to process each block. The produced list + * of sequences (literals and matches) is then post-processed by libzstd to produce + * valid compressed blocks. + * + * This block-level offload API is a more granular complement of the existing + * frame-level offload API compressSequences() (introduced in v1.5.1). It offers + * an easier migration story for applications already integrated with libzstd: the + * user application continues to invoke the same compression functions + * ZSTD_compress2() or ZSTD_compressStream2() as usual, and transparently benefits + * from the specific advantages of the external sequence producer. For example, + * the sequence producer could be tuned to take advantage of known characteristics + * of the input, to offer better speed / ratio, or could leverage hardware + * acceleration not available within libzstd itself. + * + * See contrib/externalSequenceProducer for an example program employing the + * Block-Level Sequence Producer API. + * + * *** USAGE *** + * The user is responsible for implementing a function of type + * ZSTD_sequenceProducer_F. For each block, zstd will pass the following + * arguments to the user-provided function: + * + * - sequenceProducerState: a pointer to a user-managed state for the sequence + * producer. + * + * - outSeqs, outSeqsCapacity: an output buffer for the sequence producer. + * outSeqsCapacity is guaranteed >= ZSTD_sequenceBound(srcSize). The memory + * backing outSeqs is managed by the CCtx. + * + * - src, srcSize: an input buffer for the sequence producer to parse. + * srcSize is guaranteed to be <= ZSTD_BLOCKSIZE_MAX. + * + * - dict, dictSize: a history buffer, which may be empty, which the sequence + * producer may reference as it parses the src buffer. Currently, zstd will + * always pass dictSize == 0 into external sequence producers, but this will + * change in the future. + * + * - compressionLevel: a signed integer representing the zstd compression level + * set by the user for the current operation. The sequence producer may choose + * to use this information to change its compression strategy and speed/ratio + * tradeoff. Note: the compression level does not reflect zstd parameters set + * through the advanced API. + * + * - windowSize: a size_t representing the maximum allowed offset for external + * sequences. Note that sequence offsets are sometimes allowed to exceed the + * windowSize if a dictionary is present, see doc/zstd_compression_format.md + * for details. + * + * The user-provided function shall return a size_t representing the number of + * sequences written to outSeqs. This return value will be treated as an error + * code if it is greater than outSeqsCapacity. The return value must be non-zero + * if srcSize is non-zero. The ZSTD_SEQUENCE_PRODUCER_ERROR macro is provided + * for convenience, but any value greater than outSeqsCapacity will be treated as + * an error code. + * + * If the user-provided function does not return an error code, the sequences + * written to outSeqs must be a valid parse of the src buffer. Data corruption may + * occur if the parse is not valid. A parse is defined to be valid if the + * following conditions hold: + * - The sum of matchLengths and literalLengths must equal srcSize. + * - All sequences in the parse, except for the final sequence, must have + * matchLength >= ZSTD_MINMATCH_MIN. The final sequence must have + * matchLength >= ZSTD_MINMATCH_MIN or matchLength == 0. + * - All offsets must respect the windowSize parameter as specified in + * doc/zstd_compression_format.md. + * - If the final sequence has matchLength == 0, it must also have offset == 0. + * + * zstd will only validate these conditions (and fail compression if they do not + * hold) if the ZSTD_c_validateSequences cParam is enabled. Note that sequence + * validation has a performance cost. + * + * If the user-provided function returns an error, zstd will either fall back + * to an internal sequence producer or fail the compression operation. The user can + * choose between the two behaviors by setting the ZSTD_c_enableSeqProducerFallback + * cParam. Fallback compression will follow any other cParam settings, such as + * compression level, the same as in a normal compression operation. + * + * The user shall instruct zstd to use a particular ZSTD_sequenceProducer_F + * function by calling + * ZSTD_registerSequenceProducer(cctx, + * sequenceProducerState, + * sequenceProducer) + * This setting will persist until the next parameter reset of the CCtx. + * + * The sequenceProducerState must be initialized by the user before calling + * ZSTD_registerSequenceProducer(). The user is responsible for destroying the + * sequenceProducerState. + * + * *** LIMITATIONS *** + * This API is compatible with all zstd compression APIs which respect advanced parameters. + * However, there are three limitations: + * + * First, the ZSTD_c_enableLongDistanceMatching cParam is not currently supported. + * COMPRESSION WILL FAIL if it is enabled and the user tries to compress with a block-level + * external sequence producer. + * - Note that ZSTD_c_enableLongDistanceMatching is auto-enabled by default in some + * cases (see its documentation for details). Users must explicitly set + * ZSTD_c_enableLongDistanceMatching to ZSTD_ps_disable in such cases if an external + * sequence producer is registered. + * - As of this writing, ZSTD_c_enableLongDistanceMatching is disabled by default + * whenever ZSTD_c_windowLog < 128MB, but that's subject to change. Users should + * check the docs on ZSTD_c_enableLongDistanceMatching whenever the Block-Level Sequence + * Producer API is used in conjunction with advanced settings (like ZSTD_c_windowLog). + * + * Second, history buffers are not currently supported. Concretely, zstd will always pass + * dictSize == 0 to the external sequence producer (for now). This has two implications: + * - Dictionaries are not currently supported. Compression will *not* fail if the user + * references a dictionary, but the dictionary won't have any effect. + * - Stream history is not currently supported. All advanced compression APIs, including + * streaming APIs, work with external sequence producers, but each block is treated as + * an independent chunk without history from previous blocks. + * + * Third, multi-threading within a single compression is not currently supported. In other words, + * COMPRESSION WILL FAIL if ZSTD_c_nbWorkers > 0 and an external sequence producer is registered. + * Multi-threading across compressions is fine: simply create one CCtx per thread. + * + * Long-term, we plan to overcome all three limitations. There is no technical blocker to + * overcoming them. It is purely a question of engineering effort. + */ + +#define ZSTD_SEQUENCE_PRODUCER_ERROR ((size_t)(-1)) + +typedef size_t (*ZSTD_sequenceProducer_F) ( + void* sequenceProducerState, + ZSTD_Sequence* outSeqs, size_t outSeqsCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + int compressionLevel, + size_t windowSize +); + +/*! ZSTD_registerSequenceProducer() : + * Instruct zstd to use a block-level external sequence producer function. + * + * The sequenceProducerState must be initialized by the caller, and the caller is + * responsible for managing its lifetime. This parameter is sticky across + * compressions. It will remain set until the user explicitly resets compression + * parameters. + * + * Sequence producer registration is considered to be an "advanced parameter", + * part of the "advanced API". This means it will only have an effect on compression + * APIs which respect advanced parameters, such as compress2() and compressStream2(). + * Older compression APIs such as compressCCtx(), which predate the introduction of + * "advanced parameters", will ignore any external sequence producer setting. + * + * The sequence producer can be "cleared" by registering a NULL function pointer. This + * removes all limitations described above in the "LIMITATIONS" section of the API docs. + * + * The user is strongly encouraged to read the full API documentation (above) before + * calling this function. */ +ZSTDLIB_STATIC_API void +ZSTD_registerSequenceProducer( + ZSTD_CCtx* cctx, + void* sequenceProducerState, + ZSTD_sequenceProducer_F sequenceProducer +); + +/*! ZSTD_CCtxParams_registerSequenceProducer() : + * Same as ZSTD_registerSequenceProducer(), but operates on ZSTD_CCtx_params. + * This is used for accurate size estimation with ZSTD_estimateCCtxSize_usingCCtxParams(), + * which is needed when creating a ZSTD_CCtx with ZSTD_initStaticCCtx(). + * + * If you are using the external sequence producer API in a scenario where ZSTD_initStaticCCtx() + * is required, then this function is for you. Otherwise, you probably don't need it. + * + * See tests/zstreamtest.c for example usage. */ +ZSTDLIB_STATIC_API void +ZSTD_CCtxParams_registerSequenceProducer( + ZSTD_CCtx_params* params, + void* sequenceProducerState, + ZSTD_sequenceProducer_F sequenceProducer +); + + +/********************************************************************* +* Buffer-less and synchronous inner streaming functions (DEPRECATED) +* +* This API is deprecated, and will be removed in a future version. +* It allows streaming (de)compression with user allocated buffers. +* However, it is hard to use, and not as well tested as the rest of +* our API. +* +* Please use the normal streaming API instead: ZSTD_compressStream2, +* and ZSTD_decompressStream. +* If there is functionality that you need, but it doesn't provide, +* please open an issue on our GitHub. +********************************************************************* */ + +/** + Buffer-less streaming compression (synchronous mode) + + A ZSTD_CCtx object is required to track streaming operations. + Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. + ZSTD_CCtx object can be reused multiple times within successive compression operations. + + Start by initializing a context. + Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression. + + Then, consume your input using ZSTD_compressContinue(). + There are some important considerations to keep in mind when using this advanced function : + - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only. + - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks. + - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario. + Worst case evaluation is provided by ZSTD_compressBound(). + ZSTD_compressContinue() doesn't guarantee recover after a failed compression. + - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog). + It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks) + - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps. + In which case, it will "discard" the relevant memory section from its history. + + Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum. + It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame. + Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders. + + `ZSTD_CCtx` object can be reused (ZSTD_compressBegin()) to compress again. +*/ + +/*===== Buffer-less streaming compression functions =====*/ +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */ + +ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.") +ZSTDLIB_STATIC_API +size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */ + +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */ +ZSTD_DEPRECATED("use advanced API to access custom parameters") +ZSTDLIB_STATIC_API +size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */ +ZSTD_DEPRECATED("use advanced API to access custom parameters") +ZSTDLIB_STATIC_API +size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */ +/** + Buffer-less streaming decompression (synchronous mode) + + A ZSTD_DCtx object is required to track streaming operations. + Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. + A ZSTD_DCtx object can be reused multiple times. + + First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader(). + Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough. + Data fragment must be large enough to ensure successful decoding. + `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough. + result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled. + >0 : `srcSize` is too small, please provide at least result bytes on next attempt. + errorCode, which can be tested using ZSTD_isError(). + + It fills a ZSTD_FrameHeader structure with important information to correctly decode the frame, + such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`). + Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information. + As a consequence, check that values remain within valid application range. + For example, do not allocate memory blindly, check that `windowSize` is within expectation. + Each application can set its own limits, depending on local restrictions. + For extended interoperability, it is recommended to support `windowSize` of at least 8 MB. + + ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes. + ZSTD_decompressContinue() is very sensitive to contiguity, + if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place, + or that previous contiguous segment is large enough to properly handle maximum back-reference distance. + There are multiple ways to guarantee this condition. + + The most memory efficient way is to use a round buffer of sufficient size. + Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(), + which can return an error code if required value is too large for current system (in 32-bits mode). + In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one, + up to the moment there is not enough room left in the buffer to guarantee decoding another full block, + which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`. + At which point, decoding can resume from the beginning of the buffer. + Note that already decoded data stored in the buffer should be flushed before being overwritten. + + There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory. + + Finally, if you control the compression process, you can also ignore all buffer size rules, + as long as the encoder and decoder progress in "lock-step", + aka use exactly the same buffer sizes, break contiguity at the same place, etc. + + Once buffers are setup, start decompression, with ZSTD_decompressBegin(). + If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict(). + + Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. + ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue(). + ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail. + + result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). + It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item. + It can also be an error code, which can be tested with ZSTD_isError(). + + A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. + Context can then be reset to start a new decompression. + + Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType(). + This information is not required to properly decode a frame. + + == Special case : skippable frames == + + Skippable frames allow integration of user-defined data into a flow of concatenated frames. + Skippable frames will be ignored (skipped) by decompressor. + The format of skippable frames is as follows : + a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F + b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits + c) Frame Content - any content (User Data) of length equal to Frame Size + For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame. + For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content. +*/ + +/*===== Buffer-less streaming decompression functions =====*/ + +ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */ + +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + +ZSTDLIB_STATIC_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); +ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* misc */ +ZSTD_DEPRECATED("This function will likely be removed in the next minor release. It is misleading and has very limited utility.") +ZSTDLIB_STATIC_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); +typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; +ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); + + + + +/* ========================================= */ +/** Block level API (DEPRECATED) */ +/* ========================================= */ + +/*! + + This API is deprecated in favor of the regular compression API. + You can get the frame header down to 2 bytes by setting: + - ZSTD_c_format = ZSTD_f_zstd1_magicless + - ZSTD_c_contentSizeFlag = 0 + - ZSTD_c_checksumFlag = 0 + - ZSTD_c_dictIDFlag = 0 + + This API is not as well tested as our normal API, so we recommend not using it. + We will be removing it in a future version. If the normal API doesn't provide + the functionality you need, please open a GitHub issue. + + Block functions produce and decode raw zstd blocks, without frame metadata. + Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes). + But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes. + + A few rules to respect : + - Compressing and decompressing require a context structure + + Use ZSTD_createCCtx() and ZSTD_createDCtx() + - It is necessary to init context before starting + + compression : any ZSTD_compressBegin*() variant, including with dictionary + + decompression : any ZSTD_decompressBegin*() variant, including with dictionary + - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB + + If input is larger than a block size, it's necessary to split input data into multiple blocks + + For inputs larger than a single block, consider using regular ZSTD_compress() instead. + Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block. + - When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) ! + ===> In which case, nothing is produced into `dst` ! + + User __must__ test for such outcome and deal directly with uncompressed data + + A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0. + Doing so would mess up with statistics history, leading to potential data corruption. + + ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !! + + In case of multiple successive blocks, should some of them be uncompressed, + decoder must be informed of their existence in order to follow proper history. + Use ZSTD_insertBlock() for such a case. +*/ + +/*===== Raw zstd block functions =====*/ +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */ + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */ diff --git a/c-api/include/zstd_errors.h b/c-api/include/zstd_errors.h new file mode 100644 index 000000000..8ebc95cbb --- /dev/null +++ b/c-api/include/zstd_errors.h @@ -0,0 +1,107 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_ERRORS_H_398273423 +#define ZSTD_ERRORS_H_398273423 + +#if defined (__cplusplus) +extern "C" { +#endif + +/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */ +#ifndef ZSTDERRORLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZSTDERRORLIB_VISIBILITY +# define ZSTDERRORLIB_VISIBLE ZSTDERRORLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDERRORLIB_VISIBLE __attribute__ ((visibility ("default"))) +# else +# define ZSTDERRORLIB_VISIBLE +# endif +#endif + +#ifndef ZSTDERRORLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDERRORLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else +# define ZSTDERRORLIB_HIDDEN +# endif +#endif + +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBLE +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBLE +#endif + +/*-********************************************* + * Error codes list + *-********************************************* + * Error codes _values_ are pinned down since v1.3.1 only. + * Therefore, don't rely on values if you may link to any version < v1.3.1. + * + * Only values < 100 are considered stable. + * + * note 1 : this API shall be used with static linking only. + * dynamic linking is not yet officially supported. + * note 2 : Prefer relying on the enum than on its value whenever possible + * This is the only supported way to use the error list < v1.3.1 + * note 3 : ZSTD_isError() is always correct, whatever the library version. + **********************************************/ +typedef enum { + ZSTD_error_no_error = 0, + ZSTD_error_GENERIC = 1, + ZSTD_error_prefix_unknown = 10, + ZSTD_error_version_unsupported = 12, + ZSTD_error_frameParameter_unsupported = 14, + ZSTD_error_frameParameter_windowTooLarge = 16, + ZSTD_error_corruption_detected = 20, + ZSTD_error_checksum_wrong = 22, + ZSTD_error_literals_headerWrong = 24, + ZSTD_error_dictionary_corrupted = 30, + ZSTD_error_dictionary_wrong = 32, + ZSTD_error_dictionaryCreation_failed = 34, + ZSTD_error_parameter_unsupported = 40, + ZSTD_error_parameter_combination_unsupported = 41, + ZSTD_error_parameter_outOfBound = 42, + ZSTD_error_tableLog_tooLarge = 44, + ZSTD_error_maxSymbolValue_tooLarge = 46, + ZSTD_error_maxSymbolValue_tooSmall = 48, + ZSTD_error_cannotProduce_uncompressedBlock = 49, + ZSTD_error_stabilityCondition_notRespected = 50, + ZSTD_error_stage_wrong = 60, + ZSTD_error_init_missing = 62, + ZSTD_error_memory_allocation = 64, + ZSTD_error_workSpace_tooSmall= 66, + ZSTD_error_dstSize_tooSmall = 70, + ZSTD_error_srcSize_wrong = 72, + ZSTD_error_dstBuffer_null = 74, + ZSTD_error_noForwardProgress_destFull = 80, + ZSTD_error_noForwardProgress_inputEmpty = 82, + /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */ + ZSTD_error_frameIndex_tooLarge = 100, + ZSTD_error_seekableIO = 102, + ZSTD_error_dstBuffer_wrong = 104, + ZSTD_error_srcBuffer_wrong = 105, + ZSTD_error_sequenceProducer_failed = 106, + ZSTD_error_externalSequences_invalid = 107, + ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */ +} ZSTD_ErrorCode; + +ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */ + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_ERRORS_H_398273423 */ diff --git a/c-api/src/context.rs b/c-api/src/context.rs new file mode 100644 index 000000000..bb2f92074 --- /dev/null +++ b/c-api/src/context.rs @@ -0,0 +1,188 @@ +//! Synchronous context API: `ZSTD_CCtx` / `ZSTD_DCtx` create / free / one-shot +//! compress / decompress / sizeof. +//! +//! The contexts are opaque heap allocations holding reusable scratch (the +//! compressor's output buffer; the decoder instance) so repeated one-shot +//! calls reuse allocations — the reason a caller holds a context rather than +//! calling the simple API. Advanced parameter setters land in Phase 6.2. + +use core::ffi::c_int; +use std::panic::{AssertUnwindSafe, catch_unwind}; + +use codec::decoding::{ContentChecksum, FrameDecoder}; +use codec::encoding::{CompressionLevel, FrameCompressor}; + +use crate::error::{ZSTD_ErrorCode, code_for_decoder_error, encode}; +use crate::ffi::{in_slice, out_slice}; + +/// Opaque compression context. Carries a reusable output buffer so repeated +/// `ZSTD_compressCCtx` calls amortise the destination allocation. +#[allow(non_camel_case_types)] +pub struct ZSTD_CCtx { + scratch: Vec, +} + +/// Opaque decompression context. Wraps a reusable [`FrameDecoder`] so its +/// internal buffers persist across `ZSTD_decompressDCtx` calls. +#[allow(non_camel_case_types)] +pub struct ZSTD_DCtx { + decoder: FrameDecoder, +} + +/// `ZSTD_CCtx* ZSTD_createCCtx(void)`. +#[unsafe(no_mangle)] +pub extern "C" fn ZSTD_createCCtx() -> *mut ZSTD_CCtx { + Box::into_raw(Box::new(ZSTD_CCtx { + scratch: Vec::new(), + })) +} + +/// `size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)` — frees the context; `NULL` is a +/// no-op (returns 0), matching upstream. +/// +/// # Safety +/// `cctx` must be a pointer returned by [`ZSTD_createCCtx`] and not already +/// freed, or `NULL`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ZSTD_freeCCtx(cctx: *mut ZSTD_CCtx) -> usize { + if !cctx.is_null() { + drop(unsafe { Box::from_raw(cctx) }); + } + 0 +} + +/// `size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx)` — current heap footprint, +/// or 0 for `NULL`. +/// +/// # Safety +/// `cctx` must be a live pointer from [`ZSTD_createCCtx`], or `NULL`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ZSTD_sizeof_CCtx(cctx: *const ZSTD_CCtx) -> usize { + if cctx.is_null() { + return 0; + } + let cctx = unsafe { &*cctx }; + core::mem::size_of::() + cctx.scratch.capacity() +} + +/// `size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, +/// const void* src, size_t srcSize, int compressionLevel)` — same result as +/// [`ZSTD_compress`](crate::simple::ZSTD_compress), reusing the context's +/// output buffer. +/// +/// # Safety +/// `cctx` must be live; `dst`/`src` valid for their lengths (or `NULL`+0). +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ZSTD_compressCCtx( + cctx: *mut ZSTD_CCtx, + dst: *mut u8, + dst_capacity: usize, + src: *const u8, + src_size: usize, + compression_level: c_int, +) -> usize { + if cctx.is_null() { + return encode(ZSTD_ErrorCode::ZSTD_error_GENERIC); + } + let cctx = unsafe { &mut *cctx }; + let src = unsafe { in_slice(src, src_size) }; + let level = CompressionLevel::from_level(compression_level); + let outcome = catch_unwind(AssertUnwindSafe(|| { + cctx.scratch.clear(); + let mut enc: FrameCompressor = FrameCompressor::new(level); + // Upstream ZSTD_compressCCtx defaults ZSTD_c_checksumFlag = 0; match it. + enc.set_content_checksum(false); + enc.compress_independent_frame_into(src, &mut cctx.scratch); + })); + if outcome.is_err() { + return encode(ZSTD_ErrorCode::ZSTD_error_GENERIC); + } + let len = cctx.scratch.len(); + if len > dst_capacity { + return encode(ZSTD_ErrorCode::ZSTD_error_dstSize_tooSmall); + } + let dst = unsafe { out_slice(dst, dst_capacity) }; + dst[..len].copy_from_slice(&cctx.scratch); + len +} + +/// `ZSTD_DCtx* ZSTD_createDCtx(void)`. +#[unsafe(no_mangle)] +pub extern "C" fn ZSTD_createDCtx() -> *mut ZSTD_DCtx { + Box::into_raw(Box::new(ZSTD_DCtx { + decoder: FrameDecoder::new(), + })) +} + +/// `size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)` — frees the context; `NULL` is a +/// no-op (returns 0). +/// +/// # Safety +/// `dctx` must be a pointer from [`ZSTD_createDCtx`] and not already freed, or +/// `NULL`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ZSTD_freeDCtx(dctx: *mut ZSTD_DCtx) -> usize { + if !dctx.is_null() { + drop(unsafe { Box::from_raw(dctx) }); + } + 0 +} + +/// `size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx)` — total footprint of the +/// context, or 0 for `NULL`. +/// +/// Sums the inline struct size and the `FrameDecoder`'s lazily-grown workspace +/// (decode-window buffer, per-block literal/content buffers, entropy tables), +/// matching the workspace term of upstream's `ZSTD_sizeof_DCtx`. The workspace +/// is 0 until the first frame allocates it. Shared dictionaries (ref-counted +/// handles) are not counted, as upstream excludes `refDDict` memory. +/// +/// # Safety +/// `dctx` must be a live pointer from [`ZSTD_createDCtx`], or `NULL`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ZSTD_sizeof_DCtx(dctx: *const ZSTD_DCtx) -> usize { + if dctx.is_null() { + return 0; + } + let dctx = unsafe { &*dctx }; + core::mem::size_of::() + dctx.decoder.workspace_size() +} + +/// `size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, +/// const void* src, size_t srcSize)` — same result as +/// [`ZSTD_decompress`](crate::simple::ZSTD_decompress), reusing the context's +/// decoder. +/// +/// # Safety +/// `dctx` must be live; `dst`/`src` valid for their lengths (or `NULL`+0). +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ZSTD_decompressDCtx( + dctx: *mut ZSTD_DCtx, + dst: *mut u8, + dst_capacity: usize, + src: *const u8, + src_size: usize, +) -> usize { + if dctx.is_null() { + return encode(ZSTD_ErrorCode::ZSTD_error_GENERIC); + } + let dctx = unsafe { &mut *dctx }; + let src = unsafe { in_slice(src, src_size) }; + let dst = unsafe { out_slice(dst, dst_capacity) }; + // Verify the trailing content checksum like upstream ZSTD_decompress; the + // setter is idempotent, so reapplying it on a reused DCtx is fine. + dctx.decoder.set_content_checksum(ContentChecksum::Verify); + let outcome = catch_unwind(AssertUnwindSafe(|| dctx.decoder.decode_all(src, dst))); + match outcome { + Ok(Ok(written)) => written, + Ok(Err(err)) => encode(code_for_decoder_error(&err)), + Err(_) => { + // A panic mid-decode can leave the decoder's internal state + // partially consumed; replace it with a fresh one (same as + // ZSTD_createDCtx) so a later call on this reused DCtx starts clean + // instead of observing a broken invariant. + dctx.decoder = FrameDecoder::new(); + encode(ZSTD_ErrorCode::ZSTD_error_GENERIC) + } + } +} diff --git a/c-api/src/dict.rs b/c-api/src/dict.rs new file mode 100644 index 000000000..cb423f2ad --- /dev/null +++ b/c-api/src/dict.rs @@ -0,0 +1,197 @@ +//! Dictionary-builder API: the stable `ZDICTLIB_API` slice of `zdict.h` +//! (train / finalize / inspect). Wraps the codec crate's `dictionary` module +//! (FastCOVER trainer + raw-dictionary finalizer). The experimental +//! `ZDICT_STATIC_LINKING_ONLY` cover/fastcover/legacy entry points are not part +//! of the stable shared-library ABI and are intentionally not exported here. + +use core::ffi::{c_char, c_int, c_uint}; +use std::panic::{AssertUnwindSafe, catch_unwind}; + +use codec::decoding::Dictionary; +use codec::dictionary::{FastCoverOptions, FinalizeOptions, create_fastcover_dict_from_source}; + +use crate::error::{ZSTD_ErrorCode, encode}; +use crate::ffi::in_slice; + +/// Little-endian `ZSTD_MAGIC_DICTIONARY` (0xEC30A437) that prefixes a valid +/// zstd dictionary; the 4 bytes after it are the dictionary ID. +const DICT_MAGIC: u32 = 0xEC30_A437; + +/// `ZDICT_params_t` — finalize parameters, ABI-identical to `zdict.h`. +#[repr(C)] +#[derive(Copy, Clone)] +#[allow(non_snake_case)] +pub struct ZDICT_params_t { + /// Compression level used while analysing the samples (0 = codec default). + pub compressionLevel: c_int, + /// Verbosity of the (no-op here) builder logging. + pub notificationLevel: c_uint, + /// Forced dictionary ID; 0 lets the builder derive a compliant one. + pub dictID: c_uint, +} + +/// Sum the per-sample sizes, returning `None` on overflow or a NULL array. +/// +/// # Safety +/// `samples_sizes` must be valid for `nb_samples` `usize` reads, or NULL. +unsafe fn total_sample_len(samples_sizes: *const usize, nb_samples: c_uint) -> Option { + if nb_samples == 0 { + return Some(0); + } + if samples_sizes.is_null() { + return None; + } + let sizes = unsafe { core::slice::from_raw_parts(samples_sizes, nb_samples as usize) }; + let mut total: usize = 0; + for &s in sizes { + total = total.checked_add(s)?; + } + Some(total) +} + +/// `size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity, +/// const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples)`. +/// +/// Trains a dictionary (FastCOVER) from the concatenated `samplesBuffer` and +/// writes up to `dictBufferCapacity` bytes into `dictBuffer`, returning the +/// dictionary size or an error code (test with `ZDICT_isError`). +/// +/// # Safety +/// `dictBuffer` valid for `dictBufferCapacity` bytes; `samplesBuffer` valid for +/// the summed `samplesSizes`; `samplesSizes` valid for `nbSamples` entries. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ZDICT_trainFromBuffer( + dict_buffer: *mut u8, + dict_capacity: usize, + samples_buffer: *const u8, + samples_sizes: *const usize, + nb_samples: c_uint, +) -> usize { + let Some(total) = (unsafe { total_sample_len(samples_sizes, nb_samples) }) else { + return encode(ZSTD_ErrorCode::ZSTD_error_dictionaryCreation_failed); + }; + let samples = unsafe { in_slice(samples_buffer, total) }; + + let outcome = catch_unwind(AssertUnwindSafe(|| { + let mut out = Vec::new(); + create_fastcover_dict_from_source( + samples, + &mut out, + dict_capacity, + &FastCoverOptions::default(), + FinalizeOptions::default(), + ) + .map(|_| out) + })); + let dict = match outcome { + Ok(Ok(dict)) => dict, + _ => return encode(ZSTD_ErrorCode::ZSTD_error_dictionaryCreation_failed), + }; + if dict.len() > dict_capacity { + return encode(ZSTD_ErrorCode::ZSTD_error_dstSize_tooSmall); + } + let out = unsafe { crate::ffi::out_slice(dict_buffer, dict_capacity) }; + out[..dict.len()].copy_from_slice(&dict); + dict.len() +} + +/// `size_t ZDICT_finalizeDictionary(void* dstDictBuffer, size_t maxDictSize, +/// const void* dictContent, size_t dictContentSize, const void* samplesBuffer, +/// const size_t* samplesSizes, unsigned nbSamples, ZDICT_params_t parameters)`. +/// +/// Wraps raw `dictContent` (plus entropy tables analysed from the samples) into +/// a full zstd dictionary, writing up to `maxDictSize` bytes into +/// `dstDictBuffer`. Returns the dictionary size or an error code. +/// +/// # Safety +/// All buffers valid for their stated lengths; `samplesSizes` valid for +/// `nbSamples` entries. +#[unsafe(no_mangle)] +#[allow(clippy::too_many_arguments)] +pub unsafe extern "C" fn ZDICT_finalizeDictionary( + dst_dict_buffer: *mut u8, + max_dict_size: usize, + dict_content: *const u8, + dict_content_size: usize, + samples_buffer: *const u8, + samples_sizes: *const usize, + nb_samples: c_uint, + parameters: ZDICT_params_t, +) -> usize { + let Some(total) = (unsafe { total_sample_len(samples_sizes, nb_samples) }) else { + return encode(ZSTD_ErrorCode::ZSTD_error_dictionaryCreation_failed); + }; + let content = unsafe { in_slice(dict_content, dict_content_size) }; + let samples = unsafe { in_slice(samples_buffer, total) }; + // dictID 0 means "derive a compliant id"; any non-zero value is forced. + let finalize = FinalizeOptions { + dict_id: (parameters.dictID != 0).then_some(parameters.dictID), + }; + + let outcome = catch_unwind(AssertUnwindSafe(|| { + codec::dictionary::finalize_raw_dict(content, samples, max_dict_size, finalize) + })); + let dict = match outcome { + Ok(Ok(dict)) => dict, + _ => return encode(ZSTD_ErrorCode::ZSTD_error_dictionaryCreation_failed), + }; + if dict.len() > max_dict_size { + return encode(ZSTD_ErrorCode::ZSTD_error_dstSize_tooSmall); + } + let out = unsafe { crate::ffi::out_slice(dst_dict_buffer, max_dict_size) }; + out[..dict.len()].copy_from_slice(&dict); + dict.len() +} + +/// `unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize)` — the +/// dictionary ID from the header, or 0 if `dictBuffer` is not a valid +/// dictionary (bad magic or too short). +/// +/// # Safety +/// `dictBuffer` must be valid for `dictSize` bytes (or NULL with `dictSize == 0`). +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ZDICT_getDictID(dict_buffer: *const u8, dict_size: usize) -> c_uint { + let dict = unsafe { in_slice(dict_buffer, dict_size) }; + if dict.len() < 8 { + return 0; + } + if u32::from_le_bytes(dict[..4].try_into().expect("4 bytes")) != DICT_MAGIC { + return 0; + } + u32::from_le_bytes(dict[4..8].try_into().expect("4 bytes")) +} + +/// `size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize)` — +/// the dictionary header length (everything before the raw content), or a ZSTD +/// error code on a malformed dictionary. +/// +/// # Safety +/// `dictBuffer` must be valid for `dictSize` bytes (or NULL with `dictSize == 0`). +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ZDICT_getDictHeaderSize( + dict_buffer: *const u8, + dict_size: usize, +) -> usize { + let dict = unsafe { in_slice(dict_buffer, dict_size) }; + let outcome = catch_unwind(AssertUnwindSafe(|| Dictionary::decode_dict(dict))); + match outcome { + // Header size is the prefix before the raw content (magic + id + + // entropy tables + offset history): total minus the content length. + Ok(Ok(parsed)) => dict_size - parsed.dict_content.len(), + _ => encode(ZSTD_ErrorCode::ZSTD_error_dictionary_corrupted), + } +} + +/// `unsigned ZDICT_isError(size_t errorCode)` — non-zero iff `errorCode` is an +/// error. ZDICT shares ZSTD's `size_t` error encoding. +#[unsafe(no_mangle)] +pub extern "C" fn ZDICT_isError(error_code: usize) -> c_uint { + crate::error::ZSTD_isError(error_code) +} + +/// `const char* ZDICT_getErrorName(size_t errorCode)` — readable string for a +/// ZDICT error code (same table as ZSTD). +#[unsafe(no_mangle)] +pub extern "C" fn ZDICT_getErrorName(error_code: usize) -> *const c_char { + crate::error::ZSTD_getErrorName(error_code) +} diff --git a/c-api/src/error.rs b/c-api/src/error.rs new file mode 100644 index 000000000..8bcbb36cd --- /dev/null +++ b/c-api/src/error.rs @@ -0,0 +1,246 @@ +//! Error-code surface mirroring `zstd_errors.h`. +//! +//! Upstream encodes errors in the `size_t` return of most API functions: a +//! result `r` is an error when `r > (size_t)-ZSTD_error_maxCode`, and the +//! error code is recovered as `(ZSTD_ErrorCode)(0 - r)`. We reproduce that +//! encoding exactly so a consumer's existing `ZSTD_isError` / `ZSTD_getErrorCode` +//! calls behave identically against this library. + +use core::ffi::{CStr, c_char, c_uint}; + +use codec::decoding::errors::FrameDecoderError; + +/// Error codes from `zstd_errors.h` (upstream v1.5.7), numeric values pinned +/// since zstd v1.3.1. Exposed `#[repr(u32)]` so the discriminants are the +/// exact integers a C consumer compares against the `ZSTD_ErrorCode` enum. +#[repr(u32)] +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +#[allow(non_camel_case_types)] +pub enum ZSTD_ErrorCode { + ZSTD_error_no_error = 0, + ZSTD_error_GENERIC = 1, + ZSTD_error_prefix_unknown = 10, + ZSTD_error_version_unsupported = 12, + ZSTD_error_frameParameter_unsupported = 14, + ZSTD_error_frameParameter_windowTooLarge = 16, + ZSTD_error_corruption_detected = 20, + ZSTD_error_checksum_wrong = 22, + ZSTD_error_literals_headerWrong = 24, + ZSTD_error_dictionary_corrupted = 30, + ZSTD_error_dictionary_wrong = 32, + ZSTD_error_dictionaryCreation_failed = 34, + ZSTD_error_parameter_unsupported = 40, + ZSTD_error_parameter_combination_unsupported = 41, + ZSTD_error_parameter_outOfBound = 42, + ZSTD_error_tableLog_tooLarge = 44, + ZSTD_error_maxSymbolValue_tooLarge = 46, + ZSTD_error_maxSymbolValue_tooSmall = 48, + ZSTD_error_cannotProduce_uncompressedBlock = 49, + ZSTD_error_stabilityCondition_notRespected = 50, + ZSTD_error_stage_wrong = 60, + ZSTD_error_init_missing = 62, + ZSTD_error_memory_allocation = 64, + ZSTD_error_workSpace_tooSmall = 66, + ZSTD_error_dstSize_tooSmall = 70, + ZSTD_error_srcSize_wrong = 72, + ZSTD_error_dstBuffer_null = 74, + ZSTD_error_noForwardProgress_destFull = 80, + ZSTD_error_noForwardProgress_inputEmpty = 82, + ZSTD_error_frameIndex_tooLarge = 100, + ZSTD_error_seekableIO = 102, + ZSTD_error_dstBuffer_wrong = 104, + ZSTD_error_srcBuffer_wrong = 105, + ZSTD_error_sequenceProducer_failed = 106, + ZSTD_error_externalSequences_invalid = 107, + ZSTD_error_maxCode = 120, +} + +/// `(size_t)-ZSTD_error_maxCode`. A `size_t` result strictly greater than +/// this sentinel is an error (matches upstream `ERR_isError`). +const MAXCODE_SENTINEL: usize = 0usize.wrapping_sub(ZSTD_ErrorCode::ZSTD_error_maxCode as usize); + +/// Encode a `ZSTD_ErrorCode` as the `size_t` error value upstream returns: +/// `(size_t)(0 - code)`. Used as the return of the `size_t`-typed wrappers. +#[inline] +pub fn encode(code: ZSTD_ErrorCode) -> usize { + 0usize.wrapping_sub(code as usize) +} + +/// Whether a `size_t` function result is an error code. Mirrors +/// `ERR_isError`: `result > (size_t)-ZSTD_error_maxCode`. +#[inline] +pub fn result_is_error(result: usize) -> bool { + result > MAXCODE_SENTINEL +} + +/// Recover the `ZSTD_ErrorCode` from a `size_t` result. Returns +/// `ZSTD_error_no_error` for non-error results and `ZSTD_error_GENERIC` for an +/// error value that does not decode to a known code (cannot happen for values +/// this library produces, but keeps the mapping total for foreign inputs). +pub fn code_from_result(result: usize) -> ZSTD_ErrorCode { + if !result_is_error(result) { + return ZSTD_ErrorCode::ZSTD_error_no_error; + } + code_from_u32((0usize.wrapping_sub(result)) as u32) +} + +fn code_from_u32(value: u32) -> ZSTD_ErrorCode { + use ZSTD_ErrorCode::*; + match value { + 0 => ZSTD_error_no_error, + 1 => ZSTD_error_GENERIC, + 10 => ZSTD_error_prefix_unknown, + 12 => ZSTD_error_version_unsupported, + 14 => ZSTD_error_frameParameter_unsupported, + 16 => ZSTD_error_frameParameter_windowTooLarge, + 20 => ZSTD_error_corruption_detected, + 22 => ZSTD_error_checksum_wrong, + 24 => ZSTD_error_literals_headerWrong, + 30 => ZSTD_error_dictionary_corrupted, + 32 => ZSTD_error_dictionary_wrong, + 34 => ZSTD_error_dictionaryCreation_failed, + 40 => ZSTD_error_parameter_unsupported, + 41 => ZSTD_error_parameter_combination_unsupported, + 42 => ZSTD_error_parameter_outOfBound, + 44 => ZSTD_error_tableLog_tooLarge, + 46 => ZSTD_error_maxSymbolValue_tooLarge, + 48 => ZSTD_error_maxSymbolValue_tooSmall, + 49 => ZSTD_error_cannotProduce_uncompressedBlock, + 50 => ZSTD_error_stabilityCondition_notRespected, + 60 => ZSTD_error_stage_wrong, + 62 => ZSTD_error_init_missing, + 64 => ZSTD_error_memory_allocation, + 66 => ZSTD_error_workSpace_tooSmall, + 70 => ZSTD_error_dstSize_tooSmall, + 72 => ZSTD_error_srcSize_wrong, + 74 => ZSTD_error_dstBuffer_null, + 80 => ZSTD_error_noForwardProgress_destFull, + 82 => ZSTD_error_noForwardProgress_inputEmpty, + 100 => ZSTD_error_frameIndex_tooLarge, + 102 => ZSTD_error_seekableIO, + 104 => ZSTD_error_dstBuffer_wrong, + 105 => ZSTD_error_srcBuffer_wrong, + 106 => ZSTD_error_sequenceProducer_failed, + 107 => ZSTD_error_externalSequences_invalid, + 120 => ZSTD_error_maxCode, + _ => ZSTD_error_GENERIC, + } +} + +/// Map a decoder error to the closest stable `ZSTD_ErrorCode`. Conservative: +/// any variant without an exact upstream analogue (including the +/// feature-gated and future ones caught by the wildcard) collapses to +/// `corruption_detected`, the code upstream uses for a malformed frame. +pub fn code_for_decoder_error(err: &FrameDecoderError) -> ZSTD_ErrorCode { + use ZSTD_ErrorCode::*; + match err { + FrameDecoderError::ReadFrameHeaderError(_) => ZSTD_error_prefix_unknown, + FrameDecoderError::FrameHeaderError(_) | FrameDecoderError::FailedToInitialize(_) => { + ZSTD_error_frameParameter_unsupported + } + FrameDecoderError::WindowSizeTooBig { .. } => ZSTD_error_frameParameter_windowTooLarge, + FrameDecoderError::DictionaryDecodeError(_) => ZSTD_error_dictionary_corrupted, + FrameDecoderError::DictNotProvided { .. } + | FrameDecoderError::DictIdMismatch { .. } + | FrameDecoderError::DictAlreadyRegistered { .. } => ZSTD_error_dictionary_wrong, + FrameDecoderError::TargetTooSmall => ZSTD_error_dstSize_tooSmall, + FrameDecoderError::FrameContentSizeMismatch { .. } => ZSTD_error_corruption_detected, + FrameDecoderError::NotYetInitialized => ZSTD_error_init_missing, + // Trailing content checksum disagreed with the decoded output — upstream + // reports this distinctly from generic corruption. + FrameDecoderError::ChecksumMismatch { .. } => ZSTD_error_checksum_wrong, + // Block-body / drain / skip failures and any feature-gated or future + // variant: a malformed or unparseable frame. + _ => ZSTD_error_corruption_detected, + } +} + +/// Static, NUL-terminated message for a code (the strings upstream's +/// `ZSTD_getErrorString` returns for the stable codes). +fn message(code: ZSTD_ErrorCode) -> &'static CStr { + use ZSTD_ErrorCode::*; + match code { + ZSTD_error_no_error => c"No error detected", + ZSTD_error_GENERIC => c"Error (generic)", + ZSTD_error_prefix_unknown => c"Unknown frame descriptor", + ZSTD_error_version_unsupported => c"Version not supported", + ZSTD_error_frameParameter_unsupported => c"Unsupported frame parameter", + ZSTD_error_frameParameter_windowTooLarge => c"Frame requires too much memory for decoding", + ZSTD_error_corruption_detected => c"Data corruption detected", + ZSTD_error_checksum_wrong => c"Restored data doesn't match checksum", + ZSTD_error_literals_headerWrong => { + c"Header of Literals' block doesn't respect format specification" + } + ZSTD_error_dictionary_corrupted => c"Dictionary is corrupted", + ZSTD_error_dictionary_wrong => c"Dictionary mismatch", + ZSTD_error_dictionaryCreation_failed => c"Cannot create Dictionary from provided samples", + ZSTD_error_parameter_unsupported => c"Unsupported parameter", + ZSTD_error_parameter_combination_unsupported => c"Unsupported combination of parameters", + ZSTD_error_parameter_outOfBound => c"Parameter is out of bound", + ZSTD_error_tableLog_tooLarge => c"tableLog requires too much memory : unsupported", + ZSTD_error_maxSymbolValue_tooLarge => c"Unsupported max Symbol Value : too large", + ZSTD_error_maxSymbolValue_tooSmall => c"Specified maxSymbolValue is too small", + ZSTD_error_cannotProduce_uncompressedBlock => { + c"This mode cannot generate an uncompressed block" + } + ZSTD_error_stabilityCondition_notRespected => { + c"pinned buffer stability condition is not respected" + } + ZSTD_error_stage_wrong => c"Operation not authorized at current processing stage", + ZSTD_error_init_missing => c"Context should be init first", + ZSTD_error_memory_allocation => c"Allocation error : not enough memory", + ZSTD_error_workSpace_tooSmall => c"workSpace buffer is not large enough", + ZSTD_error_dstSize_tooSmall => c"Destination buffer is too small", + ZSTD_error_srcSize_wrong => c"Src size is incorrect", + ZSTD_error_dstBuffer_null => c"Operation on NULL destination buffer", + ZSTD_error_noForwardProgress_destFull => { + c"Operation made no progress over multiple calls, due to output buffer being full" + } + ZSTD_error_noForwardProgress_inputEmpty => { + c"Operation made no progress over multiple calls, due to input being empty" + } + ZSTD_error_frameIndex_tooLarge => c"Frame index is too large", + ZSTD_error_seekableIO => c"An I/O error occurred when reading/seeking", + ZSTD_error_dstBuffer_wrong => c"Destination buffer is wrong", + ZSTD_error_srcBuffer_wrong => c"Source buffer is wrong", + ZSTD_error_sequenceProducer_failed => { + c"Block-level external sequence producer returned an error code" + } + ZSTD_error_externalSequences_invalid => c"External sequences are not valid", + ZSTD_error_maxCode => c"Unspecified error code", + } +} + +// ===== extern "C" surface ===== + +/// `unsigned ZSTD_isError(size_t result)` — non-zero iff `result` is an error. +/// +/// # Safety +/// FFI boundary; no pointers dereferenced. +#[unsafe(no_mangle)] +pub extern "C" fn ZSTD_isError(result: usize) -> c_uint { + result_is_error(result) as c_uint +} + +/// `ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult)`. +#[unsafe(no_mangle)] +pub extern "C" fn ZSTD_getErrorCode(function_result: usize) -> ZSTD_ErrorCode { + code_from_result(function_result) +} + +/// `const char* ZSTD_getErrorName(size_t result)` — readable string for a result. +#[unsafe(no_mangle)] +pub extern "C" fn ZSTD_getErrorName(result: usize) -> *const c_char { + message(code_from_result(result)).as_ptr() +} + +/// `const char* ZSTD_getErrorString(ZSTD_ErrorCode code)` — readable string +/// for a code (the `zstd_errors.h` entry point). Takes the discriminant as a +/// primitive `c_uint`, not the `ZSTD_ErrorCode` enum: a C caller may pass a +/// value outside the known set, and materializing an out-of-range enum +/// discriminant across the FFI boundary is undefined behavior. `code_from_u32` +/// maps any unknown value to the generic error. +#[unsafe(no_mangle)] +pub extern "C" fn ZSTD_getErrorString(code: c_uint) -> *const c_char { + message(code_from_u32(code)).as_ptr() +} diff --git a/c-api/src/ffi.rs b/c-api/src/ffi.rs new file mode 100644 index 000000000..03ebc0a1e --- /dev/null +++ b/c-api/src/ffi.rs @@ -0,0 +1,31 @@ +//! Raw-pointer ↔ slice helpers shared by the wrappers. +//! +//! All upstream entry points tolerate a `(NULL, 0)` buffer pair, so a zero +//! length never dereferences the pointer. + +/// Build a shared slice from a C `(ptr, len)` pair. +/// +/// # Safety +/// When `len > 0`, `ptr` must be non-null and valid for reads of `len` bytes +/// for the slice's lifetime. +pub(crate) unsafe fn in_slice<'a>(ptr: *const u8, len: usize) -> &'a [u8] { + if len == 0 { + &[] + } else { + unsafe { core::slice::from_raw_parts(ptr, len) } + } +} + +/// Build a mutable slice from a C `(ptr, len)` pair, with the same `len == 0` +/// NULL tolerance as [`in_slice`]. +/// +/// # Safety +/// When `len > 0`, `ptr` must be non-null, valid for reads and writes of +/// `len` bytes, and unaliased for the slice's lifetime. +pub(crate) unsafe fn out_slice<'a>(ptr: *mut u8, len: usize) -> &'a mut [u8] { + if len == 0 { + &mut [] + } else { + unsafe { core::slice::from_raw_parts_mut(ptr, len) } + } +} diff --git a/c-api/src/frame.rs b/c-api/src/frame.rs new file mode 100644 index 000000000..3cc8f8e4e --- /dev/null +++ b/c-api/src/frame.rs @@ -0,0 +1,252 @@ +//! Frame-inspection entry points (the experimental `ZSTDLIB_STATIC_API` slice +//! of `zstd.h`): frame-header size, header decode, and decompressed-size +//! queries across one or more concatenated frames. + +use core::ffi::{c_uint, c_ulonglong}; + +use codec::decoding::errors::ReadFrameHeaderError; +use codec::decoding::{ + FrameContentSize, find_frame_compressed_size, frame_decompressed_bound, frame_header_size, + read_frame_content_size, read_frame_header_info, +}; + +use crate::error::{ZSTD_ErrorCode, encode}; +use crate::ffi::in_slice; + +/// `(0ULL - 1)` — content size could not be determined. +const CONTENTSIZE_UNKNOWN: c_ulonglong = u64::MAX; +/// `(0ULL - 2)` — an error occurred. +const CONTENTSIZE_ERROR: c_ulonglong = u64::MAX - 1; +/// `ZSTD_FRAMEHEADERSIZE_MAX` — the "need at most this many bytes" hint +/// `ZSTD_getFrameHeader` returns when `src` is too short. +const FRAMEHEADERSIZE_MAX: usize = 18; +/// `ZSTD_BLOCKSIZE_MAX` = `1 << 17` (128 KiB). +const BLOCKSIZE_MAX: u64 = 1 << 17; +/// Base magic of the skippable-frame range; the low nibble is the variant. +const SKIPPABLE_MAGIC_BASE: u32 = 0x184D_2A50; + +/// `ZSTD_FrameType_e` — `ZSTD_frame` (0) or `ZSTD_skippableFrame` (1). +#[repr(C)] +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +#[allow(non_camel_case_types)] +pub enum ZSTD_FrameType_e { + ZSTD_frame = 0, + ZSTD_skippableFrame = 1, +} + +/// `ZSTD_format_e` — selects the frame format for the `_advanced` query. +#[repr(C)] +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +#[allow(non_camel_case_types)] +pub enum ZSTD_format_e { + ZSTD_f_zstd1 = 0, + ZSTD_f_zstd1_magicless = 1, +} + +/// `ZSTD_FrameHeader` — layout-compatible with `zstd.h` (field order, sizes, +/// and types are byte-for-byte identical so a C consumer reads it correctly). +#[repr(C)] +#[derive(Copy, Clone)] +#[allow(non_camel_case_types, non_snake_case)] +pub struct ZSTD_FrameHeader { + pub frameContentSize: c_ulonglong, + pub windowSize: c_ulonglong, + pub blockSizeMax: c_uint, + pub frameType: ZSTD_FrameType_e, + pub headerSize: c_uint, + pub dictID: c_uint, + pub checksumFlag: c_uint, + pub _reserved1: c_uint, + pub _reserved2: c_uint, +} + +/// `size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)` — length of +/// the frame header (including the magic number), or an error code. +/// +/// # Safety +/// `src` must be valid for `src_size` bytes (or `NULL` with `src_size == 0`). +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ZSTD_frameHeaderSize(src: *const u8, src_size: usize) -> usize { + let src = unsafe { in_slice(src, src_size) }; + match frame_header_size(src) { + Ok(size) => size, + Err(ReadFrameHeaderError::BadMagicNumber(_) | ReadFrameHeaderError::SkipFrame { .. }) => { + encode(ZSTD_ErrorCode::ZSTD_error_prefix_unknown) + } + // A bad frame descriptor is a corrupt frame, not a too-short read: + // report it as such (mirrors `fill_frame_header`) instead of the + // retryable `srcSize_wrong`. + Err(ReadFrameHeaderError::InvalidFrameDescriptor(_)) => { + encode(ZSTD_ErrorCode::ZSTD_error_corruption_detected) + } + // The remaining variants are genuine short reads. + Err(_) => encode(ZSTD_ErrorCode::ZSTD_error_srcSize_wrong), + } +} + +/// Shared body for [`ZSTD_getFrameHeader`] and [`ZSTD_getFrameHeader_advanced`]. +/// +/// # Safety +/// `zfh` must be a valid, writable `ZSTD_FrameHeader` pointer; `src` valid for +/// its length. +unsafe fn fill_frame_header(zfh: *mut ZSTD_FrameHeader, src: &[u8], magicless: bool) -> usize { + if zfh.is_null() { + return encode(ZSTD_ErrorCode::ZSTD_error_GENERIC); + } + match read_frame_header_info(src, magicless) { + Ok(info) => { + let header = ZSTD_FrameHeader { + frameContentSize: match info.content_size { + FrameContentSize::Known(size) => size, + FrameContentSize::Unknown => CONTENTSIZE_UNKNOWN, + }, + windowSize: info.window_size, + blockSizeMax: info.window_size.min(BLOCKSIZE_MAX) as c_uint, + frameType: ZSTD_FrameType_e::ZSTD_frame, + headerSize: info.header_size as c_uint, + dictID: info.dictionary_id.unwrap_or(0), + checksumFlag: u32::from(info.content_checksum), + _reserved1: 0, + _reserved2: 0, + }; + unsafe { zfh.write(header) }; + 0 + } + // Skippable frames are only recognised when the magic is present. + Err(ReadFrameHeaderError::SkipFrame { + magic_number, + length, + }) if !magicless => { + let header = ZSTD_FrameHeader { + frameContentSize: c_ulonglong::from(length), + windowSize: 0, + blockSizeMax: 0, + frameType: ZSTD_FrameType_e::ZSTD_skippableFrame, + headerSize: 8, + // For skippable frames upstream stores the magic variant (0-15). + dictID: magic_number.wrapping_sub(SKIPPABLE_MAGIC_BASE), + checksumFlag: 0, + _reserved1: 0, + _reserved2: 0, + }; + unsafe { zfh.write(header) }; + 0 + } + Err(ReadFrameHeaderError::BadMagicNumber(_) | ReadFrameHeaderError::SkipFrame { .. }) => { + encode(ZSTD_ErrorCode::ZSTD_error_prefix_unknown) + } + Err(ReadFrameHeaderError::InvalidFrameDescriptor(_)) => { + encode(ZSTD_ErrorCode::ZSTD_error_corruption_detected) + } + // The remaining variants are short-read failures: ask for more input. + Err(_) => FRAMEHEADERSIZE_MAX, + } +} + +/// `size_t ZSTD_getFrameHeader(ZSTD_FrameHeader* zfhPtr, const void* src, +/// size_t srcSize)` — fills `*zfhPtr`; returns 0 on success, a positive +/// "wanted srcSize" hint when `src` is too short, or an error code. +/// +/// # Safety +/// `zfhPtr` must be writable; `src` valid for `src_size` (or `NULL`+0). +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ZSTD_getFrameHeader( + zfh_ptr: *mut ZSTD_FrameHeader, + src: *const u8, + src_size: usize, +) -> usize { + let src = unsafe { in_slice(src, src_size) }; + unsafe { fill_frame_header(zfh_ptr, src, false) } +} + +/// `size_t ZSTD_getFrameHeader_advanced(ZSTD_FrameHeader* zfhPtr, const void* +/// src, size_t srcSize, ZSTD_format_e format)`. +/// +/// `format` is taken as a primitive `c_uint`, not the `ZSTD_format_e` enum: a C +/// caller may pass any integer, and materializing an out-of-range enum +/// discriminant across the FFI boundary is undefined behavior. Only the two +/// defined values are accepted (`0` = zstd1, `1` = magicless); anything else +/// returns a generic error. +/// +/// # Safety +/// As [`ZSTD_getFrameHeader`]. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ZSTD_getFrameHeader_advanced( + zfh_ptr: *mut ZSTD_FrameHeader, + src: *const u8, + src_size: usize, + format: c_uint, +) -> usize { + let magicless = if format == ZSTD_format_e::ZSTD_f_zstd1 as c_uint { + false + } else if format == ZSTD_format_e::ZSTD_f_zstd1_magicless as c_uint { + true + } else { + return encode(ZSTD_ErrorCode::ZSTD_error_GENERIC); + }; + let src = unsafe { in_slice(src, src_size) }; + unsafe { fill_frame_header(zfh_ptr, src, magicless) } +} + +/// `unsigned long long ZSTD_findDecompressedSize(const void* src, size_t +/// srcSize)` — sum of declared content sizes across every frame in `src`. +/// +/// Returns `ZSTD_CONTENTSIZE_UNKNOWN` if any frame omits its size, or +/// `ZSTD_CONTENTSIZE_ERROR` on a malformed frame. +/// +/// # Safety +/// `src` must be valid for `src_size` bytes (or `NULL` with `src_size == 0`). +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ZSTD_findDecompressedSize(src: *const u8, src_size: usize) -> c_ulonglong { + let mut rest = unsafe { in_slice(src, src_size) }; + let mut total: u64 = 0; + while !rest.is_empty() { + match read_frame_content_size(rest) { + // checked_add, not saturating: a saturated u64::MAX would alias the + // CONTENTSIZE_UNKNOWN sentinel and mask the overflow. + Ok(FrameContentSize::Known(size)) => match total.checked_add(size) { + Some(sum) => total = sum, + None => return CONTENTSIZE_ERROR, + }, + Ok(FrameContentSize::Unknown) => return CONTENTSIZE_UNKNOWN, + // Skippable frames add nothing to the decompressed size. + Err(ReadFrameHeaderError::SkipFrame { .. }) => {} + Err(_) => return CONTENTSIZE_ERROR, + } + match find_frame_compressed_size(rest) { + Ok(consumed) if consumed > 0 && consumed <= rest.len() => rest = &rest[consumed..], + _ => return CONTENTSIZE_ERROR, + } + } + total +} + +/// `unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize)` — +/// an upper bound on the total decompressed size of every frame in `src`. +/// +/// Always returns a bound (exact when sizes are declared, otherwise a loose +/// per-block bound) or `ZSTD_CONTENTSIZE_ERROR` on a malformed frame. +/// +/// # Safety +/// `src` must be valid for `src_size` bytes (or `NULL` with `src_size == 0`). +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ZSTD_decompressBound(src: *const u8, src_size: usize) -> c_ulonglong { + let mut rest = unsafe { in_slice(src, src_size) }; + let mut total: u64 = 0; + while !rest.is_empty() { + match frame_decompressed_bound(rest) { + // checked_add, not saturating: a saturated u64::MAX would alias the + // CONTENTSIZE_UNKNOWN sentinel and mask the overflow. + Ok(bound) => match total.checked_add(bound) { + Some(sum) => total = sum, + None => return CONTENTSIZE_ERROR, + }, + Err(_) => return CONTENTSIZE_ERROR, + } + match find_frame_compressed_size(rest) { + Ok(consumed) if consumed > 0 && consumed <= rest.len() => rest = &rest[consumed..], + _ => return CONTENTSIZE_ERROR, + } + } + total +} diff --git a/c-api/src/lib.rs b/c-api/src/lib.rs new file mode 100644 index 000000000..635d8137b --- /dev/null +++ b/c-api/src/lib.rs @@ -0,0 +1,40 @@ +//! C ABI front end for `structured-zstd` — a drop-in `libzstd` replacement. +//! +//! This crate exposes hand-written `extern "C"` wrappers whose signatures and +//! error semantics match upstream zstd v1.5.7 (the vendored headers under +//! `include/`), bottomed on the pure-Rust [`codec`] public API. It +//! builds as both a `cdylib` (SONAME `libzstd.so.1`) and a `staticlib`. +//! +//! Phase 6.1 scope: the synchronous slice of `zstd.h` — the simple one-shot +//! API, the synchronous context API, error-code mapping, and frame content +//! inspection. Streaming, advanced parameters, dictionaries, and the CLI land +//! in later phases. +//! +//! Every wrapper here is `unsafe extern "C"`; the safety contracts mirror the +//! upstream documentation (valid `(ptr, len)` buffers, live context handles). +//! +//! This crate is intentionally std-only — it has no `no_std` prologue and no +//! `std` feature gate. It exists solely to emit a host shared object / static +//! archive (`libzstd.so.1` / `libzstd.a`) and every `extern "C"` wrapper guards +//! the boundary with [`std::panic::catch_unwind`], which is mandatory for +//! soundness: a Rust panic must not unwind into C. `catch_unwind` requires the +//! standard library, so the wrappers cannot build under `no_std`. The +//! `no_std + alloc` surface lives in the pure-Rust [`codec`] crate this depends +//! on; consumers wanting an embedded build link `codec` directly. + +mod context; +mod dict; +mod error; +mod ffi; +mod frame; +mod simple; + +#[cfg(test)] +mod tests; + +// The `extern "C"` entry points are exported by their `#[no_mangle]` symbols; +// re-export the public types so rustdoc and in-crate tests can name them. +pub use context::{ZSTD_CCtx, ZSTD_DCtx}; +pub use dict::ZDICT_params_t; +pub use error::ZSTD_ErrorCode; +pub use frame::{ZSTD_FrameHeader, ZSTD_FrameType_e, ZSTD_format_e}; diff --git a/c-api/src/simple.rs b/c-api/src/simple.rs new file mode 100644 index 000000000..71e6db8e2 --- /dev/null +++ b/c-api/src/simple.rs @@ -0,0 +1,190 @@ +//! Simple one-shot API: the synchronous `ZSTDLIB_API` slice of `zstd.h` +//! (compress / decompress / sizing / version / level bounds). + +use core::ffi::{c_char, c_int, c_uint}; +use std::panic::{AssertUnwindSafe, catch_unwind}; + +use codec::decoding::errors::ReadFrameHeaderError; +use codec::decoding::{ + ContentChecksum, FrameContentSize, FrameDecoder, FrameSizeError, find_frame_compressed_size, + read_frame_content_size, +}; +use codec::encoding::{CompressionLevel, FrameCompressor, compress_bound}; + +use crate::error::{ZSTD_ErrorCode, code_for_decoder_error, encode}; +use crate::ffi::{in_slice, out_slice}; + +/// `ZSTD_VERSION_NUMBER` for the vendored upstream: MAJOR*10000 + MINOR*100 + +/// RELEASE = 1*10000 + 5*100 + 7. +const VERSION_NUMBER: c_uint = 10_507; + +/// `(0ULL - 1)` — frame content size could not be determined from the header. +const CONTENTSIZE_UNKNOWN: u64 = u64::MAX; +/// `(0ULL - 2)` — an error occurred reading the frame header. +const CONTENTSIZE_ERROR: u64 = u64::MAX - 1; + +/// `ZSTD_MAX_INPUT_SIZE`: above this `ZSTD_compressBound` reports an error. +const MAX_INPUT_SIZE: usize = if usize::BITS >= 64 { + 0xFF00_FF00_FF00_FF00 +} else { + 0xFF00_FF00 +}; + +/// `unsigned ZSTD_versionNumber(void)`. +#[unsafe(no_mangle)] +pub extern "C" fn ZSTD_versionNumber() -> c_uint { + VERSION_NUMBER +} + +/// `const char* ZSTD_versionString(void)`. +#[unsafe(no_mangle)] +pub extern "C" fn ZSTD_versionString() -> *const c_char { + c"1.5.7".as_ptr() +} + +/// `int ZSTD_minCLevel(void)` — lowest (most negative) level accepted. +#[unsafe(no_mangle)] +pub extern "C" fn ZSTD_minCLevel() -> c_int { + CompressionLevel::MIN_LEVEL +} + +/// `int ZSTD_maxCLevel(void)`. +#[unsafe(no_mangle)] +pub extern "C" fn ZSTD_maxCLevel() -> c_int { + CompressionLevel::MAX_LEVEL +} + +/// `int ZSTD_defaultCLevel(void)`. +#[unsafe(no_mangle)] +pub extern "C" fn ZSTD_defaultCLevel() -> c_int { + CompressionLevel::DEFAULT_LEVEL +} + +/// `size_t ZSTD_compressBound(size_t srcSize)` — worst-case compressed size, +/// or an error code when `srcSize >= ZSTD_MAX_INPUT_SIZE`. +#[unsafe(no_mangle)] +pub extern "C" fn ZSTD_compressBound(src_size: usize) -> usize { + if src_size >= MAX_INPUT_SIZE { + encode(ZSTD_ErrorCode::ZSTD_error_srcSize_wrong) + } else { + compress_bound(src_size) + } +} + +/// `size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, +/// size_t srcSize, int compressionLevel)`. +/// +/// Returns the compressed byte count, or an error code (test with +/// `ZSTD_isError`) when the destination buffer is too small. +/// +/// # Safety +/// `dst`/`src` must each be valid for the given capacity/size (or `NULL` with +/// a zero length), per the upstream contract. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ZSTD_compress( + dst: *mut u8, + dst_capacity: usize, + src: *const u8, + src_size: usize, + compression_level: c_int, +) -> usize { + let src = unsafe { in_slice(src, src_size) }; + let level = CompressionLevel::from_level(compression_level); + // The bulk encoder aborts via the global allocator on OOM and otherwise + // does not return errors, but it can panic on an internal invariant + // break; catch it so a panic never unwinds across the FFI boundary. + let compressed = match catch_unwind(AssertUnwindSafe(|| { + let mut enc: FrameCompressor = FrameCompressor::new(level); + // Upstream ZSTD_compress defaults ZSTD_c_checksumFlag = 0; match it. + enc.set_content_checksum(false); + enc.compress_independent_frame(src) + })) { + Ok(buf) => buf, + Err(_) => return encode(ZSTD_ErrorCode::ZSTD_error_GENERIC), + }; + if compressed.len() > dst_capacity { + return encode(ZSTD_ErrorCode::ZSTD_error_dstSize_tooSmall); + } + let dst = unsafe { out_slice(dst, dst_capacity) }; + dst[..compressed.len()].copy_from_slice(&compressed); + compressed.len() +} + +/// `size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, +/// size_t compressedSize)`. +/// +/// Returns the decompressed byte count, or an error code (test with +/// `ZSTD_isError`). +/// +/// # Safety +/// `dst`/`src` must each be valid for the given capacity/size (or `NULL` with +/// a zero length). +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ZSTD_decompress( + dst: *mut u8, + dst_capacity: usize, + src: *const u8, + compressed_size: usize, +) -> usize { + let src = unsafe { in_slice(src, compressed_size) }; + let dst = unsafe { out_slice(dst, dst_capacity) }; + let outcome = catch_unwind(AssertUnwindSafe(|| { + let mut decoder = FrameDecoder::new(); + // Verify the trailing content checksum (when the frame carries one), as + // upstream ZSTD_decompress does: a mismatch surfaces as ChecksumMismatch. + decoder.set_content_checksum(ContentChecksum::Verify); + decoder.decode_all(src, dst) + })); + match outcome { + Ok(Ok(written)) => written, + Ok(Err(err)) => encode(code_for_decoder_error(&err)), + Err(_) => encode(ZSTD_ErrorCode::ZSTD_error_GENERIC), + } +} + +/// `unsigned long long ZSTD_getFrameContentSize(const void* src, size_t srcSize)`. +/// +/// Returns the declared content size, `ZSTD_CONTENTSIZE_UNKNOWN` when the +/// header omits it, or `ZSTD_CONTENTSIZE_ERROR` when the header is unreadable. +/// +/// # Safety +/// `src` must be valid for `src_size` bytes (or `NULL` with `src_size == 0`). +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ZSTD_getFrameContentSize(src: *const u8, src_size: usize) -> u64 { + let src = unsafe { in_slice(src, src_size) }; + match read_frame_content_size(src) { + Ok(FrameContentSize::Known(size)) => size, + Ok(FrameContentSize::Unknown) => CONTENTSIZE_UNKNOWN, + Err(_) => CONTENTSIZE_ERROR, + } +} + +/// `size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)`. +/// +/// Returns the on-disk size of the first frame in `src` (so a caller can step +/// to the next concatenated frame), or an error code (test with `ZSTD_isError`). +/// +/// # Safety +/// `src` must be valid for `src_size` bytes (or `NULL` with `src_size == 0`). +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ZSTD_findFrameCompressedSize(src: *const u8, src_size: usize) -> usize { + let src = unsafe { in_slice(src, src_size) }; + match find_frame_compressed_size(src) { + Ok(size) => size, + // A non-zstd prefix (bad magic / skippable) is "unknown prefix", but a + // corrupt frame descriptor is a corrupt frame, not an unknown prefix: + // map it accordingly instead of hiding it as prefix_unknown. Mirrors + // `fill_frame_header`. + Err(FrameSizeError::Header( + ReadFrameHeaderError::BadMagicNumber(_) | ReadFrameHeaderError::SkipFrame { .. }, + )) => encode(ZSTD_ErrorCode::ZSTD_error_prefix_unknown), + Err(FrameSizeError::Header(ReadFrameHeaderError::InvalidFrameDescriptor(_))) => { + encode(ZSTD_ErrorCode::ZSTD_error_corruption_detected) + } + Err(FrameSizeError::Header(_)) => encode(ZSTD_ErrorCode::ZSTD_error_srcSize_wrong), + Err(FrameSizeError::Truncated) => encode(ZSTD_ErrorCode::ZSTD_error_srcSize_wrong), + Err(FrameSizeError::ReservedBlock) | Err(FrameSizeError::OversizedBlock) => { + encode(ZSTD_ErrorCode::ZSTD_error_corruption_detected) + } + } +} diff --git a/c-api/src/tests.rs b/c-api/src/tests.rs new file mode 100644 index 000000000..dd2cfc29a --- /dev/null +++ b/c-api/src/tests.rs @@ -0,0 +1,400 @@ +//! In-crate correctness tests for the C ABI wrappers. These call the +//! `extern "C"` entry points directly (reachable in-crate) to exercise the +//! real symbol behaviour; the C-consumer link test in `tests/` is added +//! separately and verifies a real `#include ` consumer. + +use crate::context::{ + ZSTD_compressCCtx, ZSTD_createCCtx, ZSTD_createDCtx, ZSTD_decompressDCtx, ZSTD_freeCCtx, + ZSTD_freeDCtx, ZSTD_sizeof_CCtx, +}; +use crate::dict::{ZDICT_getDictHeaderSize, ZDICT_getDictID, ZDICT_isError, ZDICT_trainFromBuffer}; +use crate::error::{ZSTD_ErrorCode, ZSTD_getErrorCode, ZSTD_isError}; +use crate::frame::{ + ZSTD_FrameHeader, ZSTD_FrameType_e, ZSTD_decompressBound, ZSTD_findDecompressedSize, + ZSTD_frameHeaderSize, ZSTD_getFrameHeader, +}; +use crate::simple::{ + ZSTD_compress, ZSTD_compressBound, ZSTD_decompress, ZSTD_defaultCLevel, + ZSTD_findFrameCompressedSize, ZSTD_getFrameContentSize, ZSTD_maxCLevel, ZSTD_minCLevel, + ZSTD_versionNumber, +}; + +fn sample(len: usize) -> Vec { + // Deterministic, mildly compressible bytes (no rng dependency). + (0..len) + .map(|i| (i.wrapping_mul(2654435761) >> 13) as u8) + .collect() +} + +#[test] +fn simple_roundtrips_one_mib() { + let input = sample(1 << 20); + let bound = ZSTD_compressBound(input.len()); + let mut compressed = vec![0u8; bound]; + let csize = unsafe { + ZSTD_compress( + compressed.as_mut_ptr(), + compressed.len(), + input.as_ptr(), + input.len(), + 3, + ) + }; + assert_eq!(ZSTD_isError(csize), 0, "compress reported an error"); + assert!(csize <= bound); + + let declared = unsafe { ZSTD_getFrameContentSize(compressed.as_ptr(), csize) }; + assert_eq!(declared, input.len() as u64); + + let mut restored = vec![0u8; input.len()]; + let dsize = unsafe { + ZSTD_decompress( + restored.as_mut_ptr(), + restored.len(), + compressed.as_ptr(), + csize, + ) + }; + assert_eq!(ZSTD_isError(dsize), 0, "decompress reported an error"); + assert_eq!(dsize, input.len()); + assert_eq!(restored, input); +} + +#[test] +fn compress_into_too_small_dst_is_error() { + let input = sample(4096); + let mut tiny = [0u8; 4]; + let r = unsafe { + ZSTD_compress( + tiny.as_mut_ptr(), + tiny.len(), + input.as_ptr(), + input.len(), + 3, + ) + }; + assert_ne!(ZSTD_isError(r), 0); + assert_eq!( + ZSTD_getErrorCode(r), + ZSTD_ErrorCode::ZSTD_error_dstSize_tooSmall + ); +} + +#[test] +fn decompress_garbage_maps_to_error_code() { + let garbage = [0xABu8; 64]; + let mut out = [0u8; 64]; + let r = + unsafe { ZSTD_decompress(out.as_mut_ptr(), out.len(), garbage.as_ptr(), garbage.len()) }; + assert_ne!(ZSTD_isError(r), 0); + // Bad magic -> prefix_unknown (the code upstream returns for a non-frame). + assert_eq!( + ZSTD_getErrorCode(r), + ZSTD_ErrorCode::ZSTD_error_prefix_unknown + ); +} + +#[test] +fn context_api_roundtrips_and_reuses() { + let cctx = ZSTD_createCCtx(); + let dctx = ZSTD_createDCtx(); + assert!(!cctx.is_null() && !dctx.is_null()); + + for len in [0usize, 1, 4096, 200_000] { + let input = sample(len); + let bound = ZSTD_compressBound(len); + let mut compressed = vec![0u8; bound]; + let csize = unsafe { + ZSTD_compressCCtx( + cctx, + compressed.as_mut_ptr(), + compressed.len(), + input.as_ptr(), + input.len(), + 5, + ) + }; + assert_eq!(ZSTD_isError(csize), 0); + + let mut restored = vec![0u8; len]; + let dsize = unsafe { + ZSTD_decompressDCtx( + dctx, + restored.as_mut_ptr(), + restored.len(), + compressed.as_ptr(), + csize, + ) + }; + assert_eq!(ZSTD_isError(dsize), 0); + assert_eq!(dsize, len); + assert_eq!(restored, input); + } + + // Context tracks a non-zero footprint after use; free is a clean no-op on NULL. + assert!(unsafe { ZSTD_sizeof_CCtx(cctx) } >= core::mem::size_of::()); + assert_eq!(unsafe { ZSTD_freeCCtx(cctx) }, 0); + assert_eq!(unsafe { ZSTD_freeDCtx(dctx) }, 0); + assert_eq!(unsafe { ZSTD_freeCCtx(core::ptr::null_mut()) }, 0); +} + +#[test] +fn decompress_rejects_corrupted_content_checksum() { + let input = sample(4096); + // ZSTD_compress mirrors upstream and emits no content checksum by default, + // so build a checksum-bearing frame explicitly through the core encoder. + // Flipping the trailing 4-byte checksum makes the stored value disagree + // with the decoded output while leaving the block data (and the decode + // itself) intact. A faithful drop-in must report this as + // ZSTD_error_checksum_wrong, not silently accept the frame. + let mut frame = { + let mut enc: codec::encoding::FrameCompressor = + codec::encoding::FrameCompressor::new(codec::encoding::CompressionLevel::from_level(3)); + enc.set_content_checksum(true); + enc.compress_independent_frame(&input) + }; + let last = frame.len() - 1; + frame[last] ^= 0xFF; + + let mut out = vec![0u8; input.len()]; + let r = unsafe { ZSTD_decompress(out.as_mut_ptr(), out.len(), frame.as_ptr(), frame.len()) }; + assert_ne!( + ZSTD_isError(r), + 0, + "corrupted content checksum must be rejected" + ); + assert_eq!( + ZSTD_getErrorCode(r), + ZSTD_ErrorCode::ZSTD_error_checksum_wrong + ); +} + +fn compress_frame(input: &[u8]) -> Vec { + let bound = ZSTD_compressBound(input.len()); + let mut out = vec![0u8; bound]; + let n = unsafe { ZSTD_compress(out.as_mut_ptr(), out.len(), input.as_ptr(), input.len(), 3) }; + assert_eq!(ZSTD_isError(n), 0); + out.truncate(n); + out +} + +#[test] +fn find_frame_compressed_size_locates_frame_boundary() { + let frame = compress_frame(&sample(4096)); + // A lone frame's compressed size is the whole buffer. + let size = unsafe { ZSTD_findFrameCompressedSize(frame.as_ptr(), frame.len()) }; + assert_eq!(ZSTD_isError(size), 0); + assert_eq!(size, frame.len()); + + // With a second frame appended, it still reports only the first frame, so + // a caller can step to the next one. + let mut two = frame.clone(); + two.extend_from_slice(&compress_frame(&sample(100))); + let first = unsafe { ZSTD_findFrameCompressedSize(two.as_ptr(), two.len()) }; + assert_eq!(first, frame.len()); +} + +#[test] +fn find_frame_compressed_size_rejects_garbage() { + let garbage = [0u8; 16]; + let r = unsafe { ZSTD_findFrameCompressedSize(garbage.as_ptr(), garbage.len()) }; + assert_ne!(ZSTD_isError(r), 0); +} + +#[test] +fn get_frame_header_fills_fields() { + let frame = compress_frame(&sample(2048)); + let hdr_size = unsafe { ZSTD_frameHeaderSize(frame.as_ptr(), frame.len()) }; + assert_eq!(ZSTD_isError(hdr_size), 0); + assert!((5..=18).contains(&hdr_size)); + + let mut zfh = ZSTD_FrameHeader { + frameContentSize: 0, + windowSize: 0, + blockSizeMax: 0, + frameType: ZSTD_FrameType_e::ZSTD_skippableFrame, + headerSize: 0, + dictID: 0, + checksumFlag: 7, + _reserved1: 0, + _reserved2: 0, + }; + let r = unsafe { ZSTD_getFrameHeader(&mut zfh, frame.as_ptr(), frame.len()) }; + assert_eq!(r, 0, "header complete"); + assert_eq!(zfh.frameType, ZSTD_FrameType_e::ZSTD_frame); + assert_eq!(zfh.frameContentSize, 2048); + assert!(zfh.windowSize >= 2048); + assert_eq!(zfh.headerSize as usize, hdr_size); + assert_eq!(zfh.dictID, 0); +} + +#[test] +fn get_frame_header_short_input_asks_for_more() { + let frame = compress_frame(&sample(2048)); + let mut zfh = ZSTD_FrameHeader { + frameContentSize: 0, + windowSize: 0, + blockSizeMax: 0, + frameType: ZSTD_FrameType_e::ZSTD_frame, + headerSize: 0, + dictID: 0, + checksumFlag: 0, + _reserved1: 0, + _reserved2: 0, + }; + // Only 2 bytes: too short for even the magic; expect a positive size hint. + let r = unsafe { ZSTD_getFrameHeader(&mut zfh, frame.as_ptr(), 2) }; + assert_eq!(ZSTD_isError(r), 0); + assert!(r > 0 && r <= 18); +} + +#[test] +fn decompressed_size_queries_span_multiple_frames() { + let mut two = compress_frame(&sample(4096)); + two.extend_from_slice(&compress_frame(&sample(1000))); + + let total = unsafe { ZSTD_findDecompressedSize(two.as_ptr(), two.len()) }; + assert_eq!(total, 4096 + 1000); + + let bound = unsafe { ZSTD_decompressBound(two.as_ptr(), two.len()) }; + assert!(bound >= 4096 + 1000, "bound must not undercount"); +} + +#[test] +fn level_bounds_match_crate() { + assert_eq!(ZSTD_minCLevel(), -131072); + assert_eq!(ZSTD_maxCLevel(), 22); + assert_eq!(ZSTD_defaultCLevel(), 3); + assert_eq!(ZSTD_versionNumber(), 10_507); +} + +/// ABI layout lock for `ZSTD_FrameHeader`: it is passed by value across the C +/// boundary, so its field offsets MUST match the `zstd.h` struct. A field +/// reorder / type change that breaks a C consumer fails here rather than +/// silently corrupting reads. Offsets are pointer-width independent (the two +/// leading u64s sit at 0/8 on every ABI); the total size is 8-aligned on +/// 64-bit targets. +#[test] +fn frame_header_abi_layout_is_stable() { + use core::mem::{align_of, offset_of, size_of}; + assert_eq!(offset_of!(ZSTD_FrameHeader, frameContentSize), 0); + assert_eq!(offset_of!(ZSTD_FrameHeader, windowSize), 8); + assert_eq!(offset_of!(ZSTD_FrameHeader, blockSizeMax), 16); + assert_eq!(offset_of!(ZSTD_FrameHeader, frameType), 20); + assert_eq!(offset_of!(ZSTD_FrameHeader, headerSize), 24); + assert_eq!(offset_of!(ZSTD_FrameHeader, dictID), 28); + assert_eq!(offset_of!(ZSTD_FrameHeader, checksumFlag), 32); + assert_eq!(offset_of!(ZSTD_FrameHeader, _reserved1), 36); + assert_eq!(offset_of!(ZSTD_FrameHeader, _reserved2), 40); + // C enums and the `unsigned` fields are 4 bytes. + assert_eq!(size_of::(), 4); + #[cfg(target_pointer_width = "64")] + { + assert_eq!(size_of::(), 48); + assert_eq!(align_of::(), 8); + } +} + +#[test] +fn compress_emits_no_content_checksum_by_default() { + // Upstream ZSTD_compress defaults ZSTD_c_checksumFlag = 0, so the simple + // wrapper must emit no trailing content checksum (cleared flag in header). + let input = sample(4096); + let bound = ZSTD_compressBound(input.len()); + let mut frame = vec![0u8; bound]; + let n = unsafe { + ZSTD_compress( + frame.as_mut_ptr(), + frame.len(), + input.as_ptr(), + input.len(), + 3, + ) + }; + assert_eq!(ZSTD_isError(n), 0); + frame.truncate(n); + let descriptor = frame[4]; + assert_eq!( + (descriptor >> 2) & 1, + 0, + "default ZSTD_compress frame must not set the content-checksum flag" + ); +} + +#[test] +fn compress_cctx_emits_no_content_checksum_by_default() { + // Same guarantee for the context path: ZSTD_compressCCtx must also default + // to no content checksum, matching upstream's ZSTD_c_checksumFlag = 0. + let input = sample(4096); + let cctx = ZSTD_createCCtx(); + assert!(!cctx.is_null()); + + let bound = ZSTD_compressBound(input.len()); + let mut frame = vec![0u8; bound]; + let n = unsafe { + ZSTD_compressCCtx( + cctx, + frame.as_mut_ptr(), + frame.len(), + input.as_ptr(), + input.len(), + 3, + ) + }; + assert_eq!(ZSTD_isError(n), 0); + frame.truncate(n); + + let descriptor = frame[4]; + assert_eq!( + (descriptor >> 2) & 1, + 0, + "default ZSTD_compressCCtx frame must not set the content-checksum flag" + ); + + assert_eq!(unsafe { ZSTD_freeCCtx(cctx) }, 0); +} + +#[test] +fn zdict_train_produces_a_valid_dictionary() { + // Many small, similar samples: a concatenated buffer plus per-sample sizes, + // the exact layout ZDICT_trainFromBuffer expects. + let mut samples: Vec = Vec::new(); + let mut sizes: Vec = Vec::new(); + for i in 0..512u32 { + let s = format!("tenant=demo table=orders key={i} region=eu payload=aaaaabbbbbccccc\n"); + sizes.push(s.len()); + samples.extend_from_slice(s.as_bytes()); + } + + let mut dict = vec![0u8; 64 * 1024]; + let n = unsafe { + ZDICT_trainFromBuffer( + dict.as_mut_ptr(), + dict.len(), + samples.as_ptr(), + sizes.as_ptr(), + sizes.len() as u32, + ) + }; + assert_eq!(ZDICT_isError(n), 0, "training reported an error"); + assert!(n > 0 && n <= dict.len()); + dict.truncate(n); + + // A valid dictionary carries a non-zero ID and a header smaller than itself. + let id = unsafe { ZDICT_getDictID(dict.as_ptr(), dict.len()) }; + assert_ne!(id, 0, "trained dictionary must carry a non-zero ID"); + let header = unsafe { ZDICT_getDictHeaderSize(dict.as_ptr(), dict.len()) }; + assert_eq!( + ZDICT_isError(header), + 0, + "header size query reported an error" + ); + assert!(header > 0 && header < dict.len()); + + // A buffer that is not a dictionary reports ID 0. + let garbage = [0xABu8; 16]; + assert_eq!( + unsafe { ZDICT_getDictID(garbage.as_ptr(), garbage.len()) }, + 0 + ); +} diff --git a/c-api/tests/c_consumer.c b/c-api/tests/c_consumer.c new file mode 100644 index 000000000..d71df44ef --- /dev/null +++ b/c-api/tests/c_consumer.c @@ -0,0 +1,90 @@ +/* + * Real C consumer of the vendored against the structured-zstd C ABI. + * + * Catches "compiles in Rust but a genuine C consumer can't link / call it" + * regressions: it includes ONLY the vendored header and links the built + * library, exercising the simple, error, context, and frame-inspection + * surface with a 1 MiB round trip. Exits 0 on success; a non-zero code marks + * which check failed. + * + * Compiled + run by the `c-abi` CI job: + * cc -Ic-api/include c-api/tests/c_consumer.c -Ltarget/<...> -lstructured_zstd -o consumer + */ +#define ZSTD_STATIC_LINKING_ONLY /* expose the experimental frame-inspection API */ +#include + +#include +#include +#include + +int main(void) { + const size_t n = (size_t)1 << 20; + unsigned char *input = (unsigned char *)malloc(n); + unsigned char *out = (unsigned char *)malloc(n); + if (!input || !out) return 2; + for (size_t i = 0; i < n; i++) { + input[i] = (unsigned char)((i * 2654435761u) >> 13); + } + + size_t bound = ZSTD_compressBound(n); + if (ZSTD_isError(bound)) { + fprintf(stderr, "compressBound: %s\n", ZSTD_getErrorName(bound)); + return 3; + } + unsigned char *comp = (unsigned char *)malloc(bound); + if (!comp) return 2; + + size_t csize = ZSTD_compress(comp, bound, input, n, 3); + if (ZSTD_isError(csize)) { + fprintf(stderr, "compress: %s\n", ZSTD_getErrorName(csize)); + return 4; + } + + if (ZSTD_getFrameContentSize(comp, csize) != (unsigned long long)n) return 5; + if (ZSTD_findFrameCompressedSize(comp, csize) != csize) return 6; + + size_t dsize = ZSTD_decompress(out, n, comp, csize); + if (ZSTD_isError(dsize) || dsize != n || memcmp(out, input, n) != 0) { + fprintf(stderr, "roundtrip failed\n"); + return 7; + } + + if (ZSTD_versionNumber() != 10507) return 8; + if (strcmp(ZSTD_versionString(), "1.5.7") != 0) return 9; + + /* Context API round trip. */ + ZSTD_CCtx *cctx = ZSTD_createCCtx(); + ZSTD_DCtx *dctx = ZSTD_createDCtx(); + if (!cctx || !dctx) return 10; + size_t c2 = ZSTD_compressCCtx(cctx, comp, bound, input, n, 5); + /* Validate c2 before it is reused as srcSize: on error it is an + error-encoded size_t, and passing that to decompress would feed a bogus + (huge) length and read out of bounds. */ + if (ZSTD_isError(c2)) { + fprintf(stderr, "compressCCtx: %s\n", ZSTD_getErrorName(c2)); + return 11; + } + size_t d2 = ZSTD_decompressDCtx(dctx, out, n, comp, c2); + if (ZSTD_isError(d2) || d2 != n || memcmp(out, input, n) != 0) { + return 11; + } + ZSTD_freeCCtx(cctx); + ZSTD_freeDCtx(dctx); + + /* Experimental frame-header inspection. */ + ZSTD_FrameHeader zfh; + if (ZSTD_getFrameHeader(&zfh, comp, csize) != 0) return 12; + if (zfh.frameContentSize != (unsigned long long)n) return 13; + if (zfh.frameType != ZSTD_frame) return 14; + + /* Error mapping: a bad buffer must report an error code. */ + unsigned char garbage[16] = {0}; + size_t bad = ZSTD_decompress(out, n, garbage, sizeof garbage); + if (!ZSTD_isError(bad)) return 15; + + free(input); + free(out); + free(comp); + printf("c_consumer: OK (csize=%zu)\n", csize); + return 0; +} diff --git a/zstd-wasm/npm/index.ts b/zstd-wasm/npm/index.ts index 7011297af..a3939c93d 100644 --- a/zstd-wasm/npm/index.ts +++ b/zstd-wasm/npm/index.ts @@ -162,8 +162,9 @@ export async function init(): Promise { * negatives for the ultra-fast tier; defaults to {@link DEFAULT_LEVEL}). The * frame decodes in any compliant zstd decoder, including the native C library. * - * @param checksum Defaults to `true` (emit the trailing XXH64 content - * checksum); pass `false` to omit it from the frame. + * @param checksum Defaults to `false` (no trailing checksum, matching + * libzstd's `ZSTD_c_checksumFlag = 0`); pass `true` to append the XXH64 + * content checksum. */ export async function compress( data: Uint8Array, @@ -195,8 +196,9 @@ export async function decompress( * `ZSTD_compress_usingDict` — small, similar payloads compress far better. * Rejects if the dictionary is invalid. * - * @param checksum Defaults to `true` (emit the trailing XXH64 content - * checksum); pass `false` to omit it from the frame. + * @param checksum Defaults to `false` (no trailing checksum, matching + * libzstd's `ZSTD_c_checksumFlag = 0`); pass `true` to append the XXH64 + * content checksum. */ export async function compressUsingDict( data: Uint8Array, @@ -251,8 +253,9 @@ export async function createDecompressStream( * decodes in any compliant zstd decoder. Symmetric with * {@link createDecompressStream}. * - * @param checksum Defaults to `true` (seal the frame with a trailing XXH64 - * content checksum); pass `false` to omit it. + * @param checksum Defaults to `false` (no trailing checksum, matching + * libzstd's `ZSTD_c_checksumFlag = 0`); pass `true` to seal the frame with a + * trailing XXH64 content checksum. */ export async function createCompressStream( level: number = DEFAULT_LEVEL, diff --git a/zstd-wasm/src/lib.rs b/zstd-wasm/src/lib.rs index fd9d43a7b..ed3ec3196 100644 --- a/zstd-wasm/src/lib.rs +++ b/zstd-wasm/src/lib.rs @@ -15,9 +15,7 @@ #![cfg(target_arch = "wasm32")] use structured_zstd::decoding::{BlockDecodingStrategy, FrameDecoder, StreamingDecoder}; -use structured_zstd::encoding::{ - CompressionLevel, FrameCompressor, StreamingEncoder, compress_slice_to_vec, -}; +use structured_zstd::encoding::{CompressionLevel, FrameCompressor, StreamingEncoder}; use structured_zstd::io::{Read, Write}; use wasm_bindgen::prelude::*; @@ -54,19 +52,16 @@ fn core_checksum(mode: Option) -> structured_zstd::decoding::Co /// negative levels (`-7..=-1`) for the ultra-fast tier. The returned frame /// decodes in any compliant zstd decoder, including the native C library. /// -/// `checksum` is optional (default `true`): pass `false` to emit a frame -/// without the trailing XXH64 content checksum (semantics of -/// `ZSTD_c_checksumFlag`). +/// `checksum` is optional (default `false`, matching libzstd's +/// `ZSTD_c_checksumFlag = 0`): pass `true` to append the trailing XXH64 content +/// checksum. #[wasm_bindgen] pub fn compress(data: &[u8], level: i32, checksum: Option) -> Vec { - if checksum.unwrap_or(true) { - // Default: keep the historical fast path verbatim. - compress_slice_to_vec(data, CompressionLevel::Level(level)) - } else { - let mut enc: FrameCompressor = FrameCompressor::new(CompressionLevel::Level(level)); - enc.set_content_checksum(false); - enc.compress_independent_frame(data) - } + // Set the checksum flag explicitly in both cases rather than leaning on the + // encoder's own default, so the wasm default stays decoupled from it. + let mut enc: FrameCompressor = FrameCompressor::new(CompressionLevel::Level(level)); + enc.set_content_checksum(checksum.unwrap_or(false)); + enc.compress_independent_frame(data) } /// Decompress a complete Zstandard frame back into its original bytes. @@ -98,6 +93,9 @@ pub fn decompress(data: &[u8], checksum: Option) -> Result, ) -> Result, JsError> { let mut enc: FrameCompressor = FrameCompressor::new(CompressionLevel::Level(level)); - enc.set_content_checksum(checksum.unwrap_or(true)); + enc.set_content_checksum(checksum.unwrap_or(false)); enc.set_dictionary_from_bytes(dict) .map_err(|err| JsError::new(&format!("structured-zstd: invalid dictionary: {err:?}")))?; Ok(enc.compress_independent_frame(data)) @@ -315,7 +313,8 @@ impl ZstdDecompressStream { /// Incremental streaming compressor: feed plaintext chunks via /// [`ZstdCompressStream::push`] and receive complete compressed blocks as the /// matcher window fills, then [`ZstdCompressStream::finish`] to seal the frame -/// (final block + content checksum). Peak working set is O(window), not +/// (final block, plus the XXH64 trailer only when `checksum` was enabled). +/// Peak working set is O(window), not /// O(input) — emitted blocks are flushed to the caller while only the matcher /// window is retained — so a large payload never has to be buffered whole. The /// produced frame omits `Frame_Content_Size` (the total is unknown while @@ -331,15 +330,16 @@ pub struct ZstdCompressStream { #[wasm_bindgen] impl ZstdCompressStream { /// Open a streaming compressor at `level` (zstd scale: `1..=22`, negatives - /// for the ultra-fast tier). `checksum` is optional (default `true`): pass - /// `false` to seal the frame without a trailing content checksum. + /// for the ultra-fast tier). `checksum` is optional (default `false`, + /// matching libzstd's `ZSTD_c_checksumFlag = 0`): pass `true` to seal the + /// frame with a trailing content checksum. #[wasm_bindgen(constructor)] pub fn new(level: i32, checksum: Option) -> ZstdCompressStream { let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Level(level)); // Provably Ok: the encoder is fresh, so no frame header has been // emitted yet (the only failure mode of this setter). encoder - .set_content_checksum(checksum.unwrap_or(true)) + .set_content_checksum(checksum.unwrap_or(false)) .expect("fresh streaming encoder accepts the content-checksum toggle"); ZstdCompressStream { encoder: Some(encoder), diff --git a/zstd/src/decoding/decode_buffer.rs b/zstd/src/decoding/decode_buffer.rs index 058eb9363..ddd900413 100644 --- a/zstd/src/decoding/decode_buffer.rs +++ b/zstd/src/decoding/decode_buffer.rs @@ -182,6 +182,14 @@ impl DecodeBuffer { self.buffer.len() } + /// Allocated byte capacity of the backing buffer (the decode window). + /// Backs the workspace-footprint reporting; the value is the backend's + /// `cap()` (RingBuffer's ring-indexing capacity / FlatBuf's `Vec` + /// capacity), so it tracks the real heap reservation. + pub fn capacity(&self) -> usize { + self.buffer.cap() + } + /// Active dictionary content bytes, borrowed through the shared handle /// (no copy). Empty slice when no dictionary is attached. #[inline] diff --git a/zstd/src/decoding/frame_decoder.rs b/zstd/src/decoding/frame_decoder.rs index f48f26469..7ee702222 100644 --- a/zstd/src/decoding/frame_decoder.rs +++ b/zstd/src/decoding/frame_decoder.rs @@ -488,6 +488,13 @@ impl DecoderScratchKind { } } + fn workspace_bytes(&self) -> usize { + match self { + Self::Ring(s) => s.workspace_bytes(), + Self::Flat(s) => s.workspace_bytes(), + } + } + /// Pre-reserve the backing buffer to `window_size` in a single /// allocation. Called once on the non-direct (`decode_blocks`) path /// after direct-eligibility is ruled out, so multi-segment fallback @@ -890,6 +897,20 @@ impl FrameDecoder { } } + /// Heap bytes currently held by the decoder's lazily-grown workspace: + /// the decode-window buffer plus the per-block literal/content buffers + /// and the entropy tables. Returns 0 before the first frame is initialised + /// (no workspace allocated yet). The window allocation dominates and grows + /// with the frame's window size; this is the value to track for decode-time + /// memory pressure, mirroring the workspace term of upstream + /// `ZSTD_sizeof_DCtx`. Shared dictionaries (ref-counted handles) are not + /// counted, matching upstream excluding `refDDict` memory. + pub fn workspace_size(&self) -> usize { + self.state + .as_ref() + .map_or(0, |s| s.decoder_scratch.workspace_bytes()) + } + /// Select how the frame's optional content checksum is handled /// (compute, expose, verify, or skip). See [`ContentChecksum`]. /// Default [`ContentChecksum::EmitOnly`]. Takes effect on the next @@ -1642,9 +1663,21 @@ impl FrameDecoder { emit_resume: bool, ) -> Result { use FrameDecoderError as err; + #[cfg(feature = "hash")] + let checksum_mode = self.content_checksum; let magicless = self.magicless; let state = self.state.as_mut().ok_or(err::NotYetInitialized)?; + // Honor the checksum mode before any drain/read can hash: `None` must + // compute no XXH64. `decode_blocks` sets this; the partial path must too, + // or a reused scratch keeps hashing with the default-enabled state. + #[cfg(feature = "hash")] + { + let compute_hash = checksum_mode != ContentChecksum::None + && state.frame_header.descriptor.content_checksum_flag(); + state.decoder_scratch.set_compute_hash(compute_hash); + } + // Mirror `decode_blocks`: pre-reserve the backing buffer to // `window_size` so multi-block frames don't pay repeated grow steps. let window_size = state.frame_header.window_size().unwrap_or(0) as usize; diff --git a/zstd/src/decoding/mod.rs b/zstd/src/decoding/mod.rs index 392da05d3..42b97e8e4 100644 --- a/zstd/src/decoding/mod.rs +++ b/zstd/src/decoding/mod.rs @@ -39,6 +39,317 @@ pub use frame_decoder::{BlockDecodingStrategy, ContentChecksum, FrameDecoder}; pub use frame_decoder::{PartialDecode, ResumeInput, ResumeState}; pub use streaming_decoder::StreamingDecoder; +/// Decompressed size a frame declares in its header, as read by +/// [`read_frame_content_size`] without decoding the frame body. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum FrameContentSize { + /// The header carried an explicit `Frame_Content_Size` field (in bytes). + Known(u64), + /// The header did not declare a content size; the true size is only + /// known after decoding (or from out-of-band knowledge). + Unknown, +} + +/// Read the decompressed size a frame declares in its header, without +/// decoding the frame body. +/// +/// Parses only the leading frame header of `src`. Returns +/// [`FrameContentSize::Known`] when the header carries an explicit +/// `Frame_Content_Size`, or [`FrameContentSize::Unknown`] when it does not. +/// This backs the C `ZSTD_getFrameContentSize` entry point, where the two +/// variants map to a concrete size and `ZSTD_CONTENTSIZE_UNKNOWN`. +/// +/// # Errors +/// Returns [`ReadFrameHeaderError`](errors::ReadFrameHeaderError) when `src` +/// is too short to hold a header, carries a bad magic number, or begins with +/// a skippable frame. +/// +/// ```rust +/// use structured_zstd::encoding::{compress_slice_to_vec, CompressionLevel}; +/// use structured_zstd::decoding::{read_frame_content_size, FrameContentSize}; +/// let frame = compress_slice_to_vec(&[42u8; 100], CompressionLevel::Default); +/// assert_eq!(read_frame_content_size(&frame).unwrap(), FrameContentSize::Known(100)); +/// ``` +pub fn read_frame_content_size( + src: &[u8], +) -> Result { + let (header, _consumed) = frame::read_frame_header_with_format(src, false)?; + Ok(if header.fcs_declared() { + FrameContentSize::Known(header.frame_content_size()) + } else { + FrameContentSize::Unknown + }) +} + +/// Error from [`find_frame_compressed_size`]. +#[derive(Debug)] +pub enum FrameSizeError { + /// The frame header could not be parsed. + Header(errors::ReadFrameHeaderError), + /// The buffer ends before the frame's blocks (or trailing checksum) are + /// complete. + Truncated, + /// A block declared the reserved block type, which is invalid per RFC 8878. + ReservedBlock, + /// A block declared a `Block_Size` larger than the frame's + /// `Block_Maximum_Size` (`min(Window_Size, 128 KiB)`), which is invalid per + /// RFC 8878 §3.1.1.2. Accepting it would let a corrupt frame pass a size + /// query and make the no-`Frame_Content_Size` decompressed-bound + /// under-count (each block can regenerate at most `Block_Maximum_Size`). + OversizedBlock, +} + +/// On-disk byte length of the FIRST frame in `src` — magic number, frame +/// header, every block, and the trailing content checksum when present — +/// computed by walking the block headers without decoding any block body. +/// +/// For a skippable frame, returns its full `8 + Frame_Size` length. This backs +/// the C `ZSTD_findFrameCompressedSize` entry point; the returned value is the +/// offset at which a following concatenated frame would begin. +/// +/// # Errors +/// [`FrameSizeError`] when the header is unreadable, the buffer is truncated +/// mid-frame, or a block uses the reserved type. +/// +/// ```rust +/// use structured_zstd::encoding::{compress_slice_to_vec, CompressionLevel}; +/// use structured_zstd::decoding::find_frame_compressed_size; +/// let frame = compress_slice_to_vec(&[5u8; 256], CompressionLevel::Default); +/// assert_eq!(find_frame_compressed_size(&frame).unwrap(), frame.len()); +/// ``` +pub fn find_frame_compressed_size(src: &[u8]) -> Result { + let (header, header_len) = match frame::read_frame_header_with_format(src, false) { + Ok(parsed) => parsed, + // Skippable frame: magic (4) + Frame_Size field (4) + payload. + Err(errors::ReadFrameHeaderError::SkipFrame { length, .. }) => { + return 8usize + .checked_add(length as usize) + .filter(|end| *end <= src.len()) + .ok_or(FrameSizeError::Truncated); + } + Err(e) => return Err(FrameSizeError::Header(e)), + }; + + let walk = walk_blocks(src, header_len as usize, frame_block_size_max(&header))?; + if header.descriptor.content_checksum_flag() { + walk.end + .checked_add(4) + .filter(|end| *end <= src.len()) + .ok_or(FrameSizeError::Truncated) + } else { + Ok(walk.end) + } +} + +/// Result of walking the block sequence of one frame (between the header and +/// the optional trailing checksum). +struct BlockWalk { + /// Offset just past the last block (before any content checksum). + end: usize, + /// Number of blocks in the frame. + count: u64, +} + +/// `Block_Maximum_Size` for the frame: `min(Window_Size, 128 KiB)`. Per RFC +/// 8878 §3.1.1.2 every block's `Block_Size` is bounded by this, and each block +/// regenerates at most this many bytes. Single-segment frames omit the +/// `Window_Descriptor`; their window equals the declared content size. +fn frame_block_size_max(header: &frame::FrameHeader) -> usize { + let window_size = match header.window_descriptor() { + Some(desc) => { + let exponent = u64::from(desc >> 3); + let mantissa = u64::from(desc & 0x7); + let window_base = 1u64 << (10 + exponent); + window_base + (window_base / 8) * mantissa + } + None => header.frame_content_size(), + }; + // The 128 KiB cap keeps the result within usize on every target. + window_size.min(128 * 1024) as usize +} + +/// Walk the block headers of a single frame starting at `start` (the offset of +/// the first block header), validating each fits in `src` and declares a +/// `Block_Size` no larger than `max_block_size` (the frame's +/// `Block_Maximum_Size`). Does not consume the trailing content checksum. +/// Shared by [`find_frame_compressed_size`] and [`frame_decompressed_bound`] so +/// the on-disk-size and block-count views never diverge. +fn walk_blocks( + src: &[u8], + start: usize, + max_block_size: usize, +) -> Result { + let mut offset = start; + let mut count = 0u64; + loop { + // 3-byte block header (RFC 8878 §3.1.1.2): bit0 last-block flag, + // bits1-2 block type, bits3-23 Block_Size. + let hdr = src + .get(offset..offset + 3) + .ok_or(FrameSizeError::Truncated)?; + let raw = u32::from(hdr[0]) | (u32::from(hdr[1]) << 8) | (u32::from(hdr[2]) << 16); + let last_block = (raw & 1) != 0; + let block_type = (raw >> 1) & 0b11; + let block_size = (raw >> 3) as usize; + // On-disk bytes following the header: RLE stores a single byte + // regardless of the run length; Raw/Compressed store Block_Size bytes; + // the reserved type is invalid. + let on_disk = match block_type { + 1 => 1, // RLE + 0 | 2 => block_size, // Raw / Compressed + _ => return Err(FrameSizeError::ReservedBlock), + }; + // RFC 8878 §3.1.1.2: Block_Size MUST NOT exceed Block_Maximum_Size for + // any block type (it bounds both the on-disk Raw/Compressed payload and + // the RLE/Raw regenerated size). Reject rather than accept a corrupt + // declaration that would otherwise pass the size query and let the + // no-FCS bound under-count. + if block_size > max_block_size { + return Err(FrameSizeError::OversizedBlock); + } + offset = offset + .checked_add(3 + on_disk) + .filter(|end| *end <= src.len()) + .ok_or(FrameSizeError::Truncated)?; + count += 1; + if last_block { + break; + } + } + Ok(BlockWalk { end: offset, count }) +} + +/// Upper bound on the decompressed size of the FIRST frame in `src`, without +/// decoding the body. Backs the C `ZSTD_decompressBound` (per-frame term). +/// +/// Returns the exact size when the header declares `Frame_Content_Size`; +/// otherwise a valid (loose) bound of `block_count * block_size_max`, where +/// `block_size_max = min(window_size, 128 KiB)` — every block decompresses to +/// at most that many bytes. Skippable frames contribute `0`. +/// +/// # Errors +/// [`FrameSizeError`] on an unreadable header, truncation, or a reserved block. +pub fn frame_decompressed_bound(src: &[u8]) -> Result { + let (header, header_len) = match frame::read_frame_header_with_format(src, false) { + Ok(parsed) => parsed, + // Skippable frame contributes 0, but its full payload must be present: + // truncation is an error per this function's contract. + Err(errors::ReadFrameHeaderError::SkipFrame { length, .. }) => { + return 8usize + .checked_add(length as usize) + .filter(|end| *end <= src.len()) + .map(|_| 0) + .ok_or(FrameSizeError::Truncated); + } + Err(e) => return Err(FrameSizeError::Header(e)), + }; + + // Walk the blocks (and the optional checksum trailer) so a truncated frame + // is rejected even when Frame_Content_Size is declared — without this the + // declared-FCS path would return a bound for an incomplete buffer. The + // per-frame block maximum both bounds the walk and scales the no-FCS bound. + let block_size_max = frame_block_size_max(&header); + let walk = walk_blocks(src, header_len as usize, block_size_max)?; + if header.descriptor.content_checksum_flag() { + walk.end + .checked_add(4) + .filter(|end| *end <= src.len()) + .ok_or(FrameSizeError::Truncated)?; + } + + if header.fcs_declared() { + return Ok(header.frame_content_size()); + } + // Saturating is intentional here: this is an UPPER bound, so capping at the + // maximum representable value is the correct ceiling for a pathologically + // large frame, not a masked arithmetic bug. Each of `walk.count` blocks + // regenerates at most `block_size_max` bytes (now enforced by `walk_blocks`, + // so the bound can no longer be undercut by an oversized block header). + Ok(walk.count.saturating_mul(block_size_max as u64)) +} + +/// Frame header fields decoded by [`read_frame_header_info`], mirroring the +/// values the C `ZSTD_getFrameHeader` fills into a `ZSTD_FrameHeader`. +#[derive(Copy, Clone, Debug)] +pub struct FrameHeaderInfo { + /// Declared decompressed size, or [`FrameContentSize::Unknown`] when the + /// header omits the `Frame_Content_Size` field. + pub content_size: FrameContentSize, + /// Decoder window size in bytes (the minimum buffer needed to decode the + /// frame). For single-segment frames this equals the content size. + pub window_size: u64, + /// Dictionary id required to decode the frame, if the header carries one. + pub dictionary_id: Option, + /// Whether a 32-bit content checksum trails the frame. + pub content_checksum: bool, + /// Total header length in bytes, including the 4-byte magic number. + pub header_size: usize, +} + +/// Length in bytes of the frame header at the start of `src`, including the +/// 4-byte magic number (the offset at which the first block begins). Backs the +/// C `ZSTD_frameHeaderSize`. +/// +/// # Errors +/// [`ReadFrameHeaderError`](errors::ReadFrameHeaderError) when the header is +/// too short, has a bad magic number, or is a skippable frame. +pub fn frame_header_size(src: &[u8]) -> Result { + let (_header, consumed) = frame::read_frame_header_with_format(src, false)?; + Ok(consumed as usize) +} + +/// Decode the leading frame header fields of `src` without decoding the body. +/// +/// Backs the C `ZSTD_getFrameHeader`. When `magicless` is `true` the 4-byte +/// magic prefix is assumed absent (the `ZSTD_f_zstd1_magicless` format); the +/// caller must know out-of-band that the stream is magicless. The reported +/// [`FrameHeaderInfo::window_size`] is the raw value derived from the header +/// (no maximum-window policy applied here; that bound is enforced at decode +/// time), so callers see the frame's own declared window even when it exceeds +/// a decoder limit. +/// +/// # Errors +/// As [`read_frame_content_size`]. +/// +/// ```rust +/// use structured_zstd::encoding::{compress_slice_to_vec, CompressionLevel}; +/// use structured_zstd::decoding::{read_frame_header_info, FrameContentSize}; +/// let frame = compress_slice_to_vec(&[7u8; 512], CompressionLevel::Default); +/// let info = read_frame_header_info(&frame, false).unwrap(); +/// assert_eq!(info.content_size, FrameContentSize::Known(512)); +/// assert!(info.window_size >= 512); +/// ``` +pub fn read_frame_header_info( + src: &[u8], + magicless: bool, +) -> Result { + let (header, consumed) = frame::read_frame_header_with_format(src, magicless)?; + let content_size = if header.fcs_declared() { + FrameContentSize::Known(header.frame_content_size()) + } else { + FrameContentSize::Unknown + }; + // Compute the window size without the decode-time maximum-window check + // (RFC 8878 §3.1.1.1.2). `window_descriptor()` returns `None` for a + // single-segment frame, where the window equals the content size. + let window_size = match header.window_descriptor() { + Some(desc) => { + let exponent = u64::from(desc >> 3); + let mantissa = u64::from(desc & 0x7); + let window_base = 1u64 << (10 + exponent); + window_base + (window_base / 8) * mantissa + } + None => header.frame_content_size(), + }; + Ok(FrameHeaderInfo { + content_size, + window_size, + dictionary_id: header.dictionary_id(), + content_checksum: header.descriptor.content_checksum_flag(), + header_size: consumed as usize, + }) +} + pub(crate) mod block_decoder; pub(crate) mod buffer_backend; pub(crate) mod decode_buffer; @@ -104,3 +415,207 @@ pub(crate) mod user_slice_buf; #[cfg(feature = "bench_internals")] pub(crate) use self::simd_copy::copy_bytes_overshooting_for_bench; + +#[cfg(test)] +mod frame_inspection_tests { + use super::{ + FrameContentSize, FrameSizeError, find_frame_compressed_size, frame_decompressed_bound, + frame_header_size, read_frame_content_size, read_frame_header_info, + }; + use crate::encoding::{CompressionLevel, compress_slice_to_vec}; + use alloc::vec; + use alloc::vec::Vec; + + fn frame(content: &[u8]) -> Vec { + compress_slice_to_vec(content, CompressionLevel::Default) + } + + /// A hand-built single raw-block frame that omits `Frame_Content_Size` + /// (descriptor `0x00`: FCS_Flag=0, Single_Segment=0) so it carries a + /// Window_Descriptor instead — the only way to exercise the window-size + /// fallback in [`frame_decompressed_bound`] / [`read_frame_header_info`], + /// which the encoder (always declaring FCS) never produces. + fn no_fcs_frame() -> Vec { + vec![ + 0x28, 0xB5, 0x2F, 0xFD, // magic + 0x00, // frame header descriptor: no FCS, multi-segment, no checksum + 0x00, // window descriptor -> windowLog 10 -> 1024 bytes + 0x19, 0x00, 0x00, // block header: last, raw, size 3 + 0xAA, 0xBB, 0xCC, // raw payload + ] + } + + /// As [`no_fcs_frame`] but with the Content_Checksum_flag (descriptor bit 2) + /// set and a 4-byte trailer appended, to cover the checksum-trailer branch. + fn no_fcs_checksum_frame() -> Vec { + vec![ + 0x28, 0xB5, 0x2F, 0xFD, 0x04, // descriptor: checksum flag set + 0x00, 0x19, 0x00, 0x00, 0xAA, 0xBB, 0xCC, // window + block + payload + 0xDE, 0xAD, 0xBE, 0xEF, // content checksum trailer + ] + } + + /// A skippable frame: magic `0x184D2A50`, 4-byte length, then `length` bytes. + fn skippable_frame(payload: &[u8]) -> Vec { + let mut f = vec![0x50, 0x2A, 0x4D, 0x18]; + f.extend_from_slice(&(payload.len() as u32).to_le_bytes()); + f.extend_from_slice(payload); + f + } + + #[test] + fn read_frame_content_size_reports_declared_size() { + let f = frame(&[42u8; 100]); + assert_eq!( + read_frame_content_size(&f).unwrap(), + FrameContentSize::Known(100) + ); + } + + #[test] + fn read_frame_content_size_reports_unknown_without_fcs() { + assert_eq!( + read_frame_content_size(&no_fcs_frame()).unwrap(), + FrameContentSize::Unknown + ); + } + + #[test] + fn read_frame_content_size_errors_on_garbage() { + assert!(read_frame_content_size(&[0xAB; 16]).is_err()); + } + + #[test] + fn find_frame_compressed_size_spans_one_frame_then_the_next() { + let first = frame(&[5u8; 256]); + assert_eq!(find_frame_compressed_size(&first).unwrap(), first.len()); + + let mut two = first.clone(); + two.extend_from_slice(&frame(&[9u8; 50])); + // Still reports only the first frame so a caller can step forward. + assert_eq!(find_frame_compressed_size(&two).unwrap(), first.len()); + } + + #[test] + fn find_frame_compressed_size_measures_skippable_frame() { + let skip = skippable_frame(&[1, 2, 3, 4]); + assert_eq!(find_frame_compressed_size(&skip).unwrap(), skip.len()); + } + + #[test] + fn find_frame_compressed_size_rejects_truncation() { + let f = frame(&[7u8; 512]); + // Drop the trailing block bytes mid-frame. + let err = find_frame_compressed_size(&f[..f.len() - 4]).unwrap_err(); + assert!(matches!(err, FrameSizeError::Truncated)); + } + + #[test] + fn frame_header_size_matches_first_block_offset() { + let f = frame(&[3u8; 2048]); + let hdr = frame_header_size(&f).unwrap(); + assert!((5..=18).contains(&hdr)); + assert!(frame_header_size(&[0u8; 2]).is_err()); + } + + #[test] + fn read_frame_header_info_fills_declared_fields() { + let f = frame(&[7u8; 512]); + let info = read_frame_header_info(&f, false).unwrap(); + assert_eq!(info.content_size, FrameContentSize::Known(512)); + assert!(info.window_size >= 512); + assert_eq!(info.dictionary_id, None); + } + + #[test] + fn read_frame_header_info_derives_window_without_fcs() { + let info = read_frame_header_info(&no_fcs_frame(), false).unwrap(); + assert_eq!(info.content_size, FrameContentSize::Unknown); + assert_eq!(info.window_size, 1024); + } + + #[test] + fn frame_decompressed_bound_returns_declared_size() { + let f = frame(&[4u8; 4096]); + assert_eq!(frame_decompressed_bound(&f).unwrap(), 4096); + } + + #[test] + fn frame_decompressed_bound_uses_block_bound_without_fcs() { + // No declared FCS -> block_count(1) * block_size_max(min(1024,128K)). + assert_eq!(frame_decompressed_bound(&no_fcs_frame()).unwrap(), 1024); + } + + #[test] + fn frame_decompressed_bound_accepts_present_checksum_trailer() { + assert_eq!( + frame_decompressed_bound(&no_fcs_checksum_frame()).unwrap(), + 1024 + ); + } + + #[test] + fn frame_decompressed_bound_rejects_missing_checksum_trailer() { + let mut f = no_fcs_checksum_frame(); + f.truncate(f.len() - 4); // drop the 4-byte trailer the descriptor promises + assert!(matches!( + frame_decompressed_bound(&f).unwrap_err(), + FrameSizeError::Truncated + )); + } + + /// A block header may declare a `Block_Size` larger than the frame's + /// `Block_Maximum_Size` (`min(window, 128 KiB)`). RFC 8878 forbids this; + /// accepting it lets a corrupt frame pass the size query and makes the + /// no-FCS decompressed bound under-count (the raw block regenerates more + /// bytes than `block_count * block_size_max`). Both helpers must reject it. + #[test] + fn size_helpers_reject_oversized_block_header() { + // Window 1024 (WD 0x00) -> Block_Maximum_Size = 1024. Declare a raw + // block of Block_Size 2000 with all 2000 payload bytes present, so the + // failure is the oversized declaration, not truncation. + let block_size = 2000usize; + let raw = ((block_size as u32) << 3) | 1; // last_block flag, Raw type (00) + let mut f = vec![ + 0x28, + 0xB5, + 0x2F, + 0xFD, // magic + 0x00, // descriptor: no FCS, multi-segment, no checksum + 0x00, // window descriptor -> 1024 bytes + (raw & 0xFF) as u8, + ((raw >> 8) & 0xFF) as u8, + ((raw >> 16) & 0xFF) as u8, + ]; + f.resize(f.len() + block_size, 0xAB); + + assert!(matches!( + find_frame_compressed_size(&f).unwrap_err(), + FrameSizeError::OversizedBlock + )); + assert!(matches!( + frame_decompressed_bound(&f).unwrap_err(), + FrameSizeError::OversizedBlock + )); + } + + #[test] + fn frame_decompressed_bound_handles_skippable_frame() { + assert_eq!( + frame_decompressed_bound(&skippable_frame(&[0u8; 8])).unwrap(), + 0 + ); + // A skippable frame whose advertised payload is absent is truncation. + let mut short = skippable_frame(&[0u8; 8]); + short.truncate(short.len() - 2); + assert!(matches!( + frame_decompressed_bound(&short).unwrap_err(), + FrameSizeError::Truncated + )); + } + + #[test] + fn frame_decompressed_bound_errors_on_garbage_header() { + assert!(frame_decompressed_bound(&[0xAB; 16]).is_err()); + } +} diff --git a/zstd/src/decoding/scratch.rs b/zstd/src/decoding/scratch.rs index 930a4b100..56a6090fb 100644 --- a/zstd/src/decoding/scratch.rs +++ b/zstd/src/decoding/scratch.rs @@ -160,6 +160,18 @@ impl DecoderScratch { } } + /// Total heap bytes this scratch holds: the decode-window buffer plus the + /// per-block literal and block-content buffers and the entropy tables. The + /// window dominates and scales with the frame; the rest are bounded by the + /// block maximum and the entropy alphabet. + pub fn workspace_bytes(&self) -> usize { + self.buffer.capacity() + + self.literals_buffer.capacity() + + self.block_content_buffer.capacity() + + self.huf.heap_bytes() + + self.fse.heap_bytes() + } + pub fn reset(&mut self, window_size: usize) { self.offset_hist = [1, 4, 8]; self.literals_buffer.clear(); @@ -281,6 +293,14 @@ impl HuffmanScratch { } } + /// Heap bytes owned by this scratch: the locally-built Huffman table. + /// A `Dict`-sourced table is read through a shared, ref-counted handle + /// (not owned here), so it is excluded, mirroring upstream not charging + /// `refDDict` memory to the decode context. + pub fn heap_bytes(&self) -> usize { + self.table.heap_bytes() + } + /// Live Huffman literals table: the shared dictionary's (zero-copy) /// while the source is still `Dict`, else the locally-built one. pub(crate) fn huf_table(&self) -> &HuffmanTable { @@ -390,6 +410,16 @@ pub struct FSEScratch { } impl FSEScratch { + /// Heap bytes owned by the three locally-built sequence FSE tables + /// (LL/ML/OF). The fixed-size decode arrays are inline (counted by + /// `size_of`); this sums their build-scratch vectors. `Dict`-sourced + /// tables read a shared handle and are not owned here. + pub fn heap_bytes(&self) -> usize { + self.offsets.heap_bytes() + + self.literal_lengths.heap_bytes() + + self.match_lengths.heap_bytes() + } + pub fn new() -> FSEScratch { FSEScratch { offsets: AlignedFSETable::new(MAX_OFFSET_CODE), diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index 0b5b59db1..3977e4475 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -1818,7 +1818,10 @@ impl FrameCompressor { #[cfg(test)] mod tests { - #[cfg(all(feature = "dict_builder", feature = "std"))] + // `format!` is used by ungated tests (e.g. the btlazy2 dict-reuse + // byte-identity test), so the import must not be feature-gated — under + // default features (no `dict_builder`) the gated form left `format!` + // unresolved when the test module is compiled. use alloc::format; use alloc::vec; diff --git a/zstd/src/encoding/mod.rs b/zstd/src/encoding/mod.rs index d310deecb..5cb4e12f3 100644 --- a/zstd/src/encoding/mod.rs +++ b/zstd/src/encoding/mod.rs @@ -198,6 +198,41 @@ pub fn compress_slice_to_vec(source: &[u8], level: CompressionLevel) -> Vec enc.compress_independent_frame(source) } +/// Worst-case compressed-frame size for an input of `src_size` bytes. +/// +/// A destination buffer of this size is always large enough to hold the +/// output of [`compress_slice_to_vec`] (or any single-frame compression) for +/// an input of `src_size` bytes, so a caller sizing a fixed buffer once (the +/// shape the C `ZSTD_compress` entry point needs) never has to grow it. +/// +/// Mirrors the upstream `ZSTD_COMPRESSBOUND` formula exactly: +/// `src_size + (src_size >> 8) + margin`, where `margin` is +/// `(128 KiB - src_size) >> 11` for inputs below 128 KiB and `0` otherwise. +/// The margin guarantees `bound(a) + bound(b) <= bound(a + b)` for blocks of +/// at least 128 KiB, which keeps multi-frame concatenation sizing sound. +/// +/// Saturates at [`usize::MAX`] if the formula would overflow on a +/// pathologically large `src_size` — no allocation that large can exist, so +/// the saturated value is the correct "cannot fit" sentinel rather than a +/// masked wrap. +/// +/// ```rust +/// use structured_zstd::encoding::{compress_bound, compress_slice_to_vec, CompressionLevel}; +/// let data = [7u8; 4096]; +/// assert!(compress_slice_to_vec(&data, CompressionLevel::Default).len() <= compress_bound(data.len())); +/// ``` +pub const fn compress_bound(src_size: usize) -> usize { + const LOWER: usize = 128 * 1024; + let margin = if src_size < LOWER { + (LOWER - src_size) >> 11 + } else { + 0 + }; + src_size + .saturating_add(src_size >> 8) + .saturating_add(margin) +} + /// Compress a byte slice into a fresh `Vec` using fine-grained /// [`CompressionParameters`] (#27) instead of a bare /// [`CompressionLevel`]. @@ -440,3 +475,37 @@ pub enum Sequence<'data> { /// These literals will just be copied at the end of the sequence execution by the decoder Literals { literals: &'data [u8] }, } + +#[cfg(test)] +mod compress_bound_tests { + use super::{CompressionLevel, compress_bound, compress_slice_to_vec}; + + #[test] + fn matches_upstream_formula_below_threshold() { + // src_size + (src_size >> 8) + ((128 KiB - src_size) >> 11). + assert_eq!(compress_bound(0), 64); + assert_eq!(compress_bound(4096), 4096 + 16 + 62); + } + + #[test] + fn drops_margin_at_and_above_threshold() { + let lower = 128 * 1024; + assert_eq!(compress_bound(lower), lower + (lower >> 8)); + assert_eq!(compress_bound(lower + 1), (lower + 1) + ((lower + 1) >> 8)); + } + + #[test] + fn saturates_instead_of_wrapping() { + // No allocation this large can exist; the ceiling is the right sentinel. + assert_eq!(compress_bound(usize::MAX), usize::MAX); + } + + #[test] + fn always_fits_real_compressed_output() { + for len in [0usize, 1, 100, 4096, 200_000] { + let data = alloc::vec![7u8; len]; + let out = compress_slice_to_vec(&data, CompressionLevel::Default); + assert!(out.len() <= compress_bound(len), "len={len}"); + } + } +} diff --git a/zstd/src/fse/fse_decoder.rs b/zstd/src/fse/fse_decoder.rs index 16b8f5684..dde49582a 100644 --- a/zstd/src/fse/fse_decoder.rs +++ b/zstd/src/fse/fse_decoder.rs @@ -300,6 +300,14 @@ impl FSETableImpl { } } + /// Heap bytes owned by this table. The `decode` table is a fixed inline + /// array (counted by `size_of`, not here); only the build-scratch vectors + /// are heap-allocated. + pub fn heap_bytes(&self) -> usize { + self.symbol_spread_buffer.capacity() + + self.symbol_probabilities.capacity() * core::mem::size_of::() + } + /// Live decode entries (`decode[..decode_len]`). #[inline(always)] pub fn decode(&self) -> &[E] { diff --git a/zstd/src/huff0/huff0_decoder.rs b/zstd/src/huff0/huff0_decoder.rs index 93ba2085d..a633ea911 100644 --- a/zstd/src/huff0/huff0_decoder.rs +++ b/zstd/src/huff0/huff0_decoder.rs @@ -337,6 +337,17 @@ pub struct HuffmanTable { } impl HuffmanTable { + /// Heap bytes owned by this table (decode lookup table plus the + /// weight-decoding scratch vectors and the weight-stream FSE table). + pub fn heap_bytes(&self) -> usize { + self.packed_decode.capacity() * core::mem::size_of::() + + self.weights.capacity() + + self.bits.capacity() + + self.bit_ranks.capacity() * core::mem::size_of::() + + self.rank_indexes.capacity() * core::mem::size_of::() + + self.fse_table.heap_bytes() + } + /// Create a new, empty table. pub fn new() -> HuffmanTable { HuffmanTable { From 4445f98b099a831570d7bb8c10cbd47f7932ceab Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Tue, 9 Jun 2026 03:34:21 +0300 Subject: [PATCH 2/3] fix(c-api): OOM-safe context alloc, skippable header size, input preflight - ZSTD_createCCtx/DCtx allocate fallibly (return NULL on OOM instead of aborting the host), with a matching manual free - ZSTD_frameHeaderSize returns the 8-byte skippable header size for a skippable frame instead of prefix_unknown - ZSTD_compress preflights src_size >= MAX_INPUT_SIZE before allocating - ZDICT_finalizeDictionary documents that only dictID is honoured (compressionLevel/notificationLevel accepted for ABI compat) - docs: lib.rs scope lists the ZDICT API; FrameHeaderInfo::header_size clarified for magicless; c_consumer compile note uses -lzstd - ci: retry the upstream-header fetch to ride out transient network blips --- .github/workflows/ci.yml | 2 +- c-api/src/context.rs | 58 +++++++++++++++++++++++++++++++--------- c-api/src/dict.rs | 5 ++++ c-api/src/frame.rs | 8 +++++- c-api/src/lib.rs | 7 ++--- c-api/src/simple.rs | 6 +++++ c-api/tests/c_consumer.c | 5 ++-- zstd/src/decoding/mod.rs | 5 +++- 8 files changed, 76 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59e7e5e30..02315b2a8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -174,7 +174,7 @@ jobs: set -euo pipefail base="https://raw.githubusercontent.com/facebook/zstd/${UPSTREAM_TAG}/lib" for h in zstd.h zdict.h zstd_errors.h; do - curl -fsSL "${base}/${h}" -o "/tmp/${h}.upstream" + curl --retry 5 --retry-all-errors --retry-delay 2 -fsSL "${base}/${h}" -o "/tmp/${h}.upstream" if ! diff -u "/tmp/${h}.upstream" "c-api/include/${h}"; then echo "::error::c-api/include/${h} diverges from upstream ${UPSTREAM_TAG}" exit 1 diff --git a/c-api/src/context.rs b/c-api/src/context.rs index bb2f92074..631bc8df5 100644 --- a/c-api/src/context.rs +++ b/c-api/src/context.rs @@ -15,6 +15,42 @@ use codec::encoding::{CompressionLevel, FrameCompressor}; use crate::error::{ZSTD_ErrorCode, code_for_decoder_error, encode}; use crate::ffi::{in_slice, out_slice}; +/// Heap-allocate `value` fallibly, returning a raw owning pointer or `null` on +/// allocation failure. Unlike `Box::new`, this never aborts the host process on +/// OOM, matching libzstd's `ZSTD_create*` NULL-on-failure contract. Pair every +/// non-null result with [`free_boxed`]. +fn try_box(value: T) -> *mut T { + let layout = core::alloc::Layout::new::(); + // Both context types own a `Vec` / `FrameDecoder`, so they are never + // zero-sized and the allocator path always applies. + debug_assert!(layout.size() != 0); + // SAFETY: `layout` has non-zero size. + let raw = unsafe { std::alloc::alloc(layout) } as *mut T; + if raw.is_null() { + return core::ptr::null_mut(); + } + // SAFETY: `raw` is freshly allocated for `T`'s layout and currently + // uninitialised, so a plain write (no drop of prior contents) is correct. + unsafe { raw.write(value) }; + raw +} + +/// Drop and free a pointer previously returned by [`try_box`]. `null` is a no-op. +/// +/// # Safety +/// `ptr` must be a live, not-yet-freed pointer from [`try_box::`], or `null`. +unsafe fn free_boxed(ptr: *mut T) { + if ptr.is_null() { + return; + } + let layout = core::alloc::Layout::new::(); + // SAFETY: `ptr` came from `try_box::` (same layout) and is still live. + unsafe { + core::ptr::drop_in_place(ptr); + std::alloc::dealloc(ptr as *mut u8, layout); + } +} + /// Opaque compression context. Carries a reusable output buffer so repeated /// `ZSTD_compressCCtx` calls amortise the destination allocation. #[allow(non_camel_case_types)] @@ -29,12 +65,13 @@ pub struct ZSTD_DCtx { decoder: FrameDecoder, } -/// `ZSTD_CCtx* ZSTD_createCCtx(void)`. +/// `ZSTD_CCtx* ZSTD_createCCtx(void)`. Returns `NULL` on allocation failure +/// (never aborts), matching upstream. #[unsafe(no_mangle)] pub extern "C" fn ZSTD_createCCtx() -> *mut ZSTD_CCtx { - Box::into_raw(Box::new(ZSTD_CCtx { + try_box(ZSTD_CCtx { scratch: Vec::new(), - })) + }) } /// `size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)` — frees the context; `NULL` is a @@ -45,9 +82,7 @@ pub extern "C" fn ZSTD_createCCtx() -> *mut ZSTD_CCtx { /// freed, or `NULL`. #[unsafe(no_mangle)] pub unsafe extern "C" fn ZSTD_freeCCtx(cctx: *mut ZSTD_CCtx) -> usize { - if !cctx.is_null() { - drop(unsafe { Box::from_raw(cctx) }); - } + unsafe { free_boxed(cctx) }; 0 } @@ -106,12 +141,13 @@ pub unsafe extern "C" fn ZSTD_compressCCtx( len } -/// `ZSTD_DCtx* ZSTD_createDCtx(void)`. +/// `ZSTD_DCtx* ZSTD_createDCtx(void)`. Returns `NULL` on allocation failure +/// (never aborts), matching upstream. #[unsafe(no_mangle)] pub extern "C" fn ZSTD_createDCtx() -> *mut ZSTD_DCtx { - Box::into_raw(Box::new(ZSTD_DCtx { + try_box(ZSTD_DCtx { decoder: FrameDecoder::new(), - })) + }) } /// `size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)` — frees the context; `NULL` is a @@ -122,9 +158,7 @@ pub extern "C" fn ZSTD_createDCtx() -> *mut ZSTD_DCtx { /// `NULL`. #[unsafe(no_mangle)] pub unsafe extern "C" fn ZSTD_freeDCtx(dctx: *mut ZSTD_DCtx) -> usize { - if !dctx.is_null() { - drop(unsafe { Box::from_raw(dctx) }); - } + unsafe { free_boxed(dctx) }; 0 } diff --git a/c-api/src/dict.rs b/c-api/src/dict.rs index cb423f2ad..afaf1c4ec 100644 --- a/c-api/src/dict.rs +++ b/c-api/src/dict.rs @@ -103,6 +103,11 @@ pub unsafe extern "C" fn ZDICT_trainFromBuffer( /// a full zstd dictionary, writing up to `maxDictSize` bytes into /// `dstDictBuffer`. Returns the dictionary size or an error code. /// +/// Of `parameters`, only `dictID` is honoured (0 derives a compliant ID). The +/// FastCOVER finalizer builds the entropy tables directly from the samples, so +/// `compressionLevel` does not tune them, and `notificationLevel` (builder +/// verbosity) has no effect here; both are accepted for ABI compatibility. +/// /// # Safety /// All buffers valid for their stated lengths; `samplesSizes` valid for /// `nbSamples` entries. diff --git a/c-api/src/frame.rs b/c-api/src/frame.rs index 3cc8f8e4e..4e620af7a 100644 --- a/c-api/src/frame.rs +++ b/c-api/src/frame.rs @@ -24,6 +24,8 @@ const FRAMEHEADERSIZE_MAX: usize = 18; const BLOCKSIZE_MAX: u64 = 1 << 17; /// Base magic of the skippable-frame range; the low nibble is the variant. const SKIPPABLE_MAGIC_BASE: u32 = 0x184D_2A50; +/// `ZSTD_SKIPPABLEHEADERSIZE` = 4-byte magic + 4-byte `Frame_Size`. +const SKIPPABLE_HEADER_SIZE: usize = 8; /// `ZSTD_FrameType_e` — `ZSTD_frame` (0) or `ZSTD_skippableFrame` (1). #[repr(C)] @@ -70,7 +72,11 @@ pub unsafe extern "C" fn ZSTD_frameHeaderSize(src: *const u8, src_size: usize) - let src = unsafe { in_slice(src, src_size) }; match frame_header_size(src) { Ok(size) => size, - Err(ReadFrameHeaderError::BadMagicNumber(_) | ReadFrameHeaderError::SkipFrame { .. }) => { + // A skippable frame's header is its fixed 8-byte prefix (4-byte magic + + // 4-byte Frame_Size), which is what `fill_frame_header` reports for it + // and what upstream returns here, so a caller can step over the frame. + Err(ReadFrameHeaderError::SkipFrame { .. }) => SKIPPABLE_HEADER_SIZE, + Err(ReadFrameHeaderError::BadMagicNumber(_)) => { encode(ZSTD_ErrorCode::ZSTD_error_prefix_unknown) } // A bad frame descriptor is a corrupt frame, not a too-short read: diff --git a/c-api/src/lib.rs b/c-api/src/lib.rs index 635d8137b..609fbe4df 100644 --- a/c-api/src/lib.rs +++ b/c-api/src/lib.rs @@ -6,9 +6,10 @@ //! builds as both a `cdylib` (SONAME `libzstd.so.1`) and a `staticlib`. //! //! Phase 6.1 scope: the synchronous slice of `zstd.h` — the simple one-shot -//! API, the synchronous context API, error-code mapping, and frame content -//! inspection. Streaming, advanced parameters, dictionaries, and the CLI land -//! in later phases. +//! API, the synchronous context API, error-code mapping, frame content +//! inspection, and the stable dictionary-builder API (`ZDICT_*` from +//! `zdict.h`). Streaming, advanced parameters, the experimental +//! `ZDICT_STATIC_LINKING_ONLY` trainers, and the CLI land in later phases. //! //! Every wrapper here is `unsafe extern "C"`; the safety contracts mirror the //! upstream documentation (valid `(ptr, len)` buffers, live context handles). diff --git a/c-api/src/simple.rs b/c-api/src/simple.rs index 71e6db8e2..71c0c9de0 100644 --- a/c-api/src/simple.rs +++ b/c-api/src/simple.rs @@ -88,6 +88,12 @@ pub unsafe extern "C" fn ZSTD_compress( src_size: usize, compression_level: c_int, ) -> usize { + // Preflight an impossible input before building anything: an `src` at or + // above ZSTD_MAX_INPUT_SIZE can never be bounded, so report it up front + // instead of allocating a huge frame just to fail (or abort on OOM). + if src_size >= MAX_INPUT_SIZE { + return encode(ZSTD_ErrorCode::ZSTD_error_srcSize_wrong); + } let src = unsafe { in_slice(src, src_size) }; let level = CompressionLevel::from_level(compression_level); // The bulk encoder aborts via the global allocator on OOM and otherwise diff --git a/c-api/tests/c_consumer.c b/c-api/tests/c_consumer.c index d71df44ef..2b8a88027 100644 --- a/c-api/tests/c_consumer.c +++ b/c-api/tests/c_consumer.c @@ -7,8 +7,9 @@ * surface with a 1 MiB round trip. Exits 0 on success; a non-zero code marks * which check failed. * - * Compiled + run by the `c-abi` CI job: - * cc -Ic-api/include c-api/tests/c_consumer.c -Ltarget/<...> -lstructured_zstd -o consumer + * Compiled + run by the `c-abi` CI job, linking via the canonical drop-in + * name (a `libzstd.so` symlink to the built `libstructured_zstd.so`): + * cc -Ic-api/include c-api/tests/c_consumer.c -Ltarget/<...> -lzstd -o consumer */ #define ZSTD_STATIC_LINKING_ONLY /* expose the experimental frame-inspection API */ #include diff --git a/zstd/src/decoding/mod.rs b/zstd/src/decoding/mod.rs index 42b97e8e4..9e900753a 100644 --- a/zstd/src/decoding/mod.rs +++ b/zstd/src/decoding/mod.rs @@ -282,7 +282,10 @@ pub struct FrameHeaderInfo { pub dictionary_id: Option, /// Whether a 32-bit content checksum trails the frame. pub content_checksum: bool, - /// Total header length in bytes, including the 4-byte magic number. + /// Header length in bytes, measured in the parsed input format: it includes + /// the 4-byte magic number in the default format, but excludes it when + /// parsed as magicless (`read_frame_header_info(.., true)`), since those 4 + /// bytes are not present on the wire in that mode. pub header_size: usize, } From db854d3755ecfff20a9d5449394a7c3d7c4f7edb Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Tue, 9 Jun 2026 03:35:29 +0300 Subject: [PATCH 3/3] docs(c-api): note the std-feature deviation is intentional for the host binding --- c-api/Cargo.toml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/c-api/Cargo.toml b/c-api/Cargo.toml index a632d6f30..8b5518bba 100644 --- a/c-api/Cargo.toml +++ b/c-api/Cargo.toml @@ -25,8 +25,10 @@ crate-type = ["lib", "cdylib", "staticlib"] # staticlib whose `extern "C"` wrappers guard the boundary with # `std::panic::catch_unwind` (a Rust panic must not unwind into C), so the # crate cannot build without std. A default-on `std` feature would advertise a -# `--no-default-features` build that fails to compile. The no_std + alloc -# surface lives in the `codec` crate below. +# `--no-default-features` build that fails to compile. This deliberately +# deviates from the no-std-library feature convention (which targets crates that +# must support a no_std build); a host-only binding has no such build. The +# no_std + alloc surface lives in the `codec` crate below. [dependencies] # The codec is imported under the alias `codec` rather than its own name: this