polyval: implement R/F algorithm (#294)

Adapts the Apache 2.0+MIT licensed implementation of the Reduction/Field
algorithm (#290, see https://eprint.iacr.org/2025/2171) from
`hpcrypt-mac`: https://github.com/seceq/hpcrypt/

Sources were taken as of upstream commit 6c22585, then modified to fit
the structure of this crate (e.g. removed buffering).

Benchmarks show for 4 powers-of-H, this algorithm is ~66% faster for
short messages on ARMv8, and ~20% faster for longer messages. It
requires double the memory footprint, however, and for the same size is
faster than our previous Karatsuba + 8 x powers-of-H for short messages,
but slower for longer ones.

This removes the generic parameter, since the implementation we're
adapting isn't currently generic, however it could potentially be
returned if there's a desire for higher performance on long messages.

Additionally support for using a custom init block has been removed, but
could be added back. I was noticing there are some oddities to it like
`Reset` will always use `0` rather than the custom init block. I think
perhaps instead of trying to shoehorn that stuff into `Polyval`/`GHash`
and adding undue complexity there, we should instead expose bits through
`polyval::hazmat` (like the core R/F algorithm) which make it possible
to build your own customized high performance GHASH/POLYVAL-like UHF
that can be tweaked in any way desired.

Author: tarcieri

ghash/benches/ghash.rs

@@ -2,10 +2,7 @@
 
 extern crate test;
 
-use ghash::{
-    GHash,
-    universal_hash::{KeyInit, UniversalHash},
-};
+use ghash::{GHash, universal_hash::UniversalHash};
 use test::Bencher;
 
 // TODO(tarcieri): move this into the `universal-hash` crate

ghash/src/lib.rs

@@ -8,13 +8,11 @@
 
 pub use polyval::universal_hash;
 
-use polyval::{DEFAULT_PARALLELISM, PolyvalGeneric};
+use polyval::Polyval;
 use universal_hash::{
     KeyInit, UhfBackend, UhfClosure, UniversalHash,
-    array::ArraySize,
     common::{BlockSizeUser, KeySizeUser, ParBlocksSizeUser},
     consts::U16,
-    typenum::{Const, ToUInt, U},
 };
 
 #[cfg(feature = "zeroize")]
@@ -33,26 +31,17 @@ pub type Tag = universal_hash::Block<GHash>;
 ///
 /// GHASH is a universal hash function used for message authentication in the AES-GCM authenticated
 /// encryption cipher.
-pub type GHash = GHashGeneric<{ DEFAULT_PARALLELISM }>;
-
-/// **GHASH**: universal hash over GF(2^128) used by AES-GCM.
-///
-/// GHASH is a universal hash function used for message authentication in the AES-GCM authenticated
-/// encryption cipher.
-///
-/// Parameterized on a constant that determines how many blocks to process at once: higher numbers
-/// use more memory, and require more time to re-key, but process data significantly faster.
 #[derive(Clone)]
-pub struct GHashGeneric<const N: usize = { DEFAULT_PARALLELISM }>(PolyvalGeneric<N>);
+pub struct GHash(Polyval);
 
-impl<const N: usize> KeySizeUser for GHashGeneric<N> {
+impl KeySizeUser for GHash {
     type KeySize = U16;
 }
 
-impl<const N: usize> GHashGeneric<N> {
-    /// Initialize GHASH with the given `H` field element and initial block
+impl GHash {
+    /// Initialize GHASH with the given `H` field element as the key.
     #[inline]
-    pub fn new_with_init_block(h: &Key, init_block: u128) -> Self {
+    pub fn new(h: &Key) -> Self {
         let mut h = *h;
         h.reverse();
 
@@ -63,7 +52,7 @@ impl<const N: usize> GHashGeneric<N> {
         h.zeroize();
 
         #[allow(clippy::let_and_return)]
-        let result = GHashGeneric(PolyvalGeneric::new_with_init_block(&h_polyval, init_block));
+        let result = Self(Polyval::new(&h_polyval));
 
         #[cfg(feature = "zeroize")]
         h_polyval.zeroize();
@@ -72,55 +61,51 @@ impl<const N: usize> GHashGeneric<N> {
     }
 }
 
-impl<const N: usize> KeyInit for GHashGeneric<N> {
+impl KeyInit for GHash {
     /// Initialize GHASH with the given `H` field element
     #[inline]
     fn new(h: &Key) -> Self {
-        Self::new_with_init_block(h, 0)
+        Self::new(h)
     }
 }
 
-struct GHashGenericBackend<'b, B: UhfBackend>(&'b mut B);
+struct GHashBackend<'b, B: UhfBackend>(&'b mut B);
 
-impl<B: UhfBackend> BlockSizeUser for GHashGenericBackend<'_, B> {
+impl<B: UhfBackend> BlockSizeUser for GHashBackend<'_, B> {
     type BlockSize = B::BlockSize;
 }
 
-impl<B: UhfBackend> ParBlocksSizeUser for GHashGenericBackend<'_, B> {
+impl<B: UhfBackend> ParBlocksSizeUser for GHashBackend<'_, B> {
     type ParBlocksSize = B::ParBlocksSize;
 }
 
-impl<B: UhfBackend> UhfBackend for GHashGenericBackend<'_, B> {
+impl<B: UhfBackend> UhfBackend for GHashBackend<'_, B> {
     fn proc_block(&mut self, x: &universal_hash::Block<B>) {
         let mut x = x.clone();
         x.reverse();
         self.0.proc_block(&x);
     }
 }
 
-impl<const N: usize> BlockSizeUser for GHashGeneric<N> {
+impl BlockSizeUser for GHash {
     type BlockSize = U16;
 }
 
-impl<const N: usize> UniversalHash for GHashGeneric<N>
-where
-    U<N>: ArraySize,
-    Const<N>: ToUInt,
-{
+impl UniversalHash for GHash {
     fn update_with_backend(&mut self, f: impl UhfClosure<BlockSize = Self::BlockSize>) {
-        struct GHashGenericClosure<C: UhfClosure>(C);
+        struct GHashClosure<C: UhfClosure>(C);
 
-        impl<C: UhfClosure> BlockSizeUser for GHashGenericClosure<C> {
+        impl<C: UhfClosure> BlockSizeUser for GHashClosure<C> {
             type BlockSize = C::BlockSize;
         }
 
-        impl<C: UhfClosure> UhfClosure for GHashGenericClosure<C> {
+        impl<C: UhfClosure> UhfClosure for GHashClosure<C> {
             fn call<B: UhfBackend<BlockSize = Self::BlockSize>>(self, backend: &mut B) {
-                self.0.call(&mut GHashGenericBackend(backend));
+                self.0.call(&mut GHashBackend(backend));
             }
         }
 
-        self.0.update_with_backend(GHashGenericClosure(f));
+        self.0.update_with_backend(GHashClosure(f));
     }
 
     /// Get GHASH output
@@ -132,8 +117,8 @@ where
     }
 }
 
-impl<const N: usize> core::fmt::Debug for GHashGeneric<N> {
+impl core::fmt::Debug for GHash {
     fn fmt(&self, f: &mut core::fmt::Formatter) -> Result<(), core::fmt::Error> {
-        write!(f, "GHashGeneric<{}> {{ ... }}", N)
+        f.debug_tuple("GHash").finish_non_exhaustive()
     }
 }

ghash/tests/lib.rs

@@ -1,7 +1,4 @@
-use ghash::{
-    GHash,
-    universal_hash::{KeyInit, UniversalHash},
-};
+use ghash::{GHash, universal_hash::UniversalHash};
 use hex_literal::hex;
 
 //

polyval/Cargo.toml

@@ -48,6 +48,7 @@ trivial_casts = "warn"
 trivial_numeric_casts = "warn"
 unused_lifetimes = "warn"
 unused_qualifications = "warn"
+unreachable_pub = "warn"
 
 [lints.rust.unexpected_cfgs]
 level = "warn"

polyval/src/backend.rs

@@ -0,0 +1,23 @@
+mod soft;
+
+cpubits::cfg_if! {
+    if #[cfg(all(target_arch = "aarch64", not(polyval_backend = "soft")))] {
+        // PMULL/NEON backend for aarch64
+        mod autodetect;
+        mod neon;
+        pub(crate) use autodetect::State;
+        pub(crate) use neon::InitToken;
+    } else if #[cfg(all(
+        any(target_arch = "x86_64", target_arch = "x86"),
+        not(polyval_backend = "soft")
+    ))] {
+        // CLMUL/AVX2 backend for x86/x86-64
+        mod autodetect;
+        mod avx2;
+        pub(crate) use autodetect::State;
+        pub(crate) use avx2::InitToken;
+    } else {
+        // "soft" pure Rust portable fallback implementation for other targets
+        pub(crate) use soft::{State, InitToken};
+    }
+}

polyval/src/backend/autodetect.rs

@@ -0,0 +1,88 @@
+//! Support for CPU feature autodetection with a portable pure Rust fallback.
+
+use super::{InitToken, soft};
+use crate::{Block, FieldElement, ParBlocks, Tag};
+
+#[cfg(any(target_arch = "x86_64", target_arch = "x86"))]
+use super::avx2 as intrinsics;
+#[cfg(target_arch = "aarch64")]
+use super::neon as intrinsics;
+
+pub(crate) union State {
+    intrinsics: intrinsics::State,
+    soft: soft::State,
+}
+
+impl State {
+    pub(crate) fn new(h: FieldElement, has_intrinsics: InitToken) -> Self {
+        if has_intrinsics.get() {
+            Self {
+                intrinsics: unsafe { intrinsics::State::new(&h.into()) },
+            }
+        } else {
+            Self {
+                soft: soft::State::new(h, soft::InitToken::init()),
+            }
+        }
+    }
+
+    pub(crate) fn proc_block(&mut self, block: &Block, has_intrinsics: InitToken) {
+        if has_intrinsics.get() {
+            unsafe { self.intrinsics.update_block(&block.0) }
+        } else {
+            unsafe { self.soft.proc_block(block, soft::InitToken::init()) }
+        }
+    }
+
+    pub(crate) fn proc_par_blocks(&mut self, par_blocks: &ParBlocks, has_intrinsics: InitToken) {
+        if has_intrinsics.get() {
+            unsafe { self.intrinsics.proc_par_blocks(par_blocks) }
+        } else {
+            unsafe {
+                self.soft
+                    .proc_par_blocks(par_blocks, soft::InitToken::init());
+            }
+        }
+    }
+
+    pub(crate) fn finalize(&self, has_intrinsics: InitToken) -> Tag {
+        if has_intrinsics.get() {
+            unsafe { self.intrinsics.finalize().into() }
+        } else {
+            unsafe { self.soft.finalize(soft::InitToken::init()) }
+        }
+    }
+
+    pub(crate) fn reset(&mut self, has_intrinsics: InitToken) {
+        if has_intrinsics.get() {
+            unsafe { self.intrinsics.reset() }
+        } else {
+            unsafe { self.soft.reset(soft::InitToken::init()) }
+        }
+    }
+
+    pub(crate) fn clone_with_intrinsics(&self, has_intrinsics: InitToken) -> Self {
+        if has_intrinsics.get() {
+            Self {
+                intrinsics: unsafe { self.intrinsics },
+            }
+        } else {
+            Self {
+                soft: unsafe { self.soft },
+            }
+        }
+    }
+
+    #[cfg(feature = "zeroize")]
+    pub(crate) fn zeroize(&mut self, has_intrinsics: InitToken) {
+        if has_intrinsics.get() {
+            unsafe {
+                self.intrinsics.zeroize();
+            }
+        } else {
+            unsafe {
+                self.soft.zeroize(soft::InitToken::init());
+            }
+        }
+    }
+}