diff --git a/zstd/examples/decode_loop_z000033.rs b/zstd/examples/decode_loop_z000033.rs index f3647cfc9..814314609 100644 --- a/zstd/examples/decode_loop_z000033.rs +++ b/zstd/examples/decode_loop_z000033.rs @@ -20,6 +20,10 @@ fn main() { let iters: u32 = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(50_000); let mode: &str = args.get(3).map(|s| s.as_str()).unwrap_or("rust"); let corpus_path: Option<&str> = args.get(4).map(|s| s.as_str()); + // 6th arg: "checksum" → encode with content_checksum_flag = 1 so the + // decode loop exercises the post-decode XXH64 verify pass. Lets us + // isolate the checksum cost (flag-on vs flag-off) in one harness. + let checksum: bool = args.get(5).map(|s| s == "checksum").unwrap_or(false); // Source: either a file from disk, or a deterministic 1MB LCG synthetic. let src: Vec = if let Some(path) = corpus_path { @@ -41,14 +45,43 @@ fn main() { // FFI encode once at requested level. let dst_cap = unsafe { zstd_sys::ZSTD_compressBound(src.len()) }; let mut compressed: Vec = vec![0u8; dst_cap]; - let written = unsafe { - zstd_sys::ZSTD_compress( - compressed.as_mut_ptr() as *mut core::ffi::c_void, - dst_cap, - src.as_ptr() as *const core::ffi::c_void, - src.len(), - level, - ) + let written = if checksum { + // ZSTD_compress2 honours CCtx params, including the checksum flag. + let cctx = unsafe { zstd_sys::ZSTD_createCCtx() }; + assert!(!cctx.is_null(), "ZSTD_createCCtx failed"); + unsafe { + zstd_sys::ZSTD_CCtx_setParameter( + cctx, + zstd_sys::ZSTD_cParameter::ZSTD_c_compressionLevel, + level, + ); + zstd_sys::ZSTD_CCtx_setParameter( + cctx, + zstd_sys::ZSTD_cParameter::ZSTD_c_checksumFlag, + 1, + ); + } + let w = unsafe { + zstd_sys::ZSTD_compress2( + cctx, + compressed.as_mut_ptr() as *mut core::ffi::c_void, + dst_cap, + src.as_ptr() as *const core::ffi::c_void, + src.len(), + ) + }; + unsafe { zstd_sys::ZSTD_freeCCtx(cctx) }; + w + } else { + unsafe { + zstd_sys::ZSTD_compress( + compressed.as_mut_ptr() as *mut core::ffi::c_void, + dst_cap, + src.as_ptr() as *const core::ffi::c_void, + src.len(), + level, + ) + } }; assert_eq!( unsafe { zstd_sys::ZSTD_isError(written) }, diff --git a/zstd/src/decoding/simd_copy.rs b/zstd/src/decoding/simd_copy.rs index b6b51d1d1..9d9330f91 100644 --- a/zstd/src/decoding/simd_copy.rs +++ b/zstd/src/decoding/simd_copy.rs @@ -157,6 +157,14 @@ pub mod shape_stats { } } +/// Copy length at or above which [`copy_bytes_overshooting`] hands off to +/// `memcpy` (ERMS `rep movsb` on x86) instead of its chunked SIMD loop. +/// Below this the inline SIMD / overlapping-u64 paths win; above it the +/// copy is bandwidth-bound and `memcpy`'s microcoded bulk store is faster. +/// Picked to sit well above hot literal pushes (1..=32 B) and typical +/// match copies, squarely in raw-block / long-match territory. +const BULK_MEMCPY_THRESHOLD: usize = 2048; + /// Copies at least `copy_at_least` bytes from `src` to `dst`. /// /// This helper may over-copy up to the chunk size of the chosen SIMD/scalar @@ -259,6 +267,23 @@ pub(crate) unsafe fn copy_bytes_overshooting( return; } + // Bulk-copy path: large non-overlapping copies (raw-block payloads, + // long non-overlapping matches) are bandwidth-bound, and on modern + // x86 the microcoded `rep movsb` (ERMS) that `memcpy` lowers to beats + // a hand-rolled 2×32B ymm loop — it issues wider internal stores with + // no per-iteration loop overhead and better hardware prefetch. The + // chunked-SIMD kernels below win only in the small/medium range where + // the `memcpy` call + ERMS startup cost would dominate the few bytes + // actually moved. Above this threshold, hand off to `memcpy`. + if copy_at_least >= BULK_MEMCPY_THRESHOLD { + // SAFETY: by contract `copy_at_least <= min(src.1, dst.1)`, and + // the regions do not overlap, so this reads/writes exactly + // `copy_at_least` bytes within both reported spans (no overshoot). + unsafe { dst.0.copy_from_nonoverlapping(src.0, copy_at_least) }; + debug_assert_eq_copy(src, dst, copy_at_least); + return; + } + // Chunked SIMD fast paths for larger copies. Each branch consults the // appropriate feature-detection mechanism (cached runtime detect under // std, compile-time target_feature otherwise) and falls through on miss