feat(loader): support Gemma 4 E4B QAT q4_0 GGUF (#211)

README.md

@@ -52,6 +52,7 @@ greedy decode).
 | Gemma 4 E4B-it Q8 | [unsloth](https://huggingface.co/unsloth/gemma-4-E4B-it-GGUF) | 8 GB | CPU | 5.0 | 5.1 | dense 42-layer gemma4: per-layer head_dim (256 SWA / 512 global), dual-RoPE, KV-share tail (18 layers), 5:1 SWA:global, logit softcap 30, PLE-256 injection (~4.2 GB mmap-resident) |
 | Gemma 4 E4B-it Q8 | (same) | 8 GB | **CUDA** `-g -1 -c 2048` | **3444** | **70.5** | all 42 layers fit at `-c 2048`; KV-share alias + per-layer SWA/global split. Prefill: int8 tensor-core MMQ + tensor-core flash attention + SoA Q8_0 weight repack. Prompts >4096 use a real SWA KV ring on the chunked-flash path (fixes correctness past the 512 window). Decode: dp4a/Q8_1 + CUDA-graph replay; argmax-stable. (llama.cpp ~8475 prefill / ~78 decode) |
 | Gemma 4 E4B-it Q8 | (same) | 8 GB | **CUDA** `-g 22 -c 2048` (hybrid) | 6.7 | 7.0 | 22 GPU + 20 CPU layers. `-g ≤ 22` required so the CPU shared-KV tail can read its own-KV source layers; CPU dense-FFN dominates decode (bandwidth-bound). `SHARPI_CUDA_PROFILE=1` for per-phase breakdown |
+| Gemma 4 E4B-it QAT q4_0 | [google](https://huggingface.co/google/gemma-4-E4B-it-qat-q4_0-gguf) | 5 GB | **CUDA** `-g -1 -c 2048` | **3283** | **100.4** | QAT q4_0 (5.15 GB): **~1.4× decode** vs Q8 at near-identical quality, frees ~3 GB for wider KV/context. Same gemma4 path; the shared-KV tail layers omit `attn_k`/`attn_v`/`attn_k_norm` (loaded conditionally). `download-model.ps1 -Model gemma4-e4b-qat` |
 | Gemma 4 12B-it QAT Q4_0 | [google](https://huggingface.co/google/gemma-4-12B-it-qat-q4_0-gguf) | 7 GB | **CUDA** `-g -1 -c 2048` | **1742** | **54.1** | dense 48-layer gemma4, all bulk weights Q4_0 + tied Q6_K embedding; fits 12 GB full-offload. `attention_k_eq_v` (8 global MQA layers reuse K as V), per-layer KV heads (8 GQA / 1 MQA), pure V-norm. Prefill: Q4_0 int8 tensor-core MMQ over a SoA repack; decode: Q4_0 dp4a/Q8_1, argmax-stable, within ~6% of llama.cpp (57 t/s). **Long context:** `--kv-type bf16` halves / `q8_0` quarters the K/V store (fp32 kernel math, argmax-stable) + a bookkeeping-only host KV cache → **`-c 131072` (128K) within 12 GB** (fp32 `cudaMalloc`-fails at 64K). At 128K, q8_0 keeps the tied embed table resident (~53 t/s) where bf16 spills it (~19 t/s). Default fp32; opt-in `--kv-type bf16\|q8_0` |
 
 _Re-measured 2026-06 from warm sweeps (prefill ~1K ctx, decode near-zero ctx), each after a discarded

src/SharpInference.Engine/ForwardPass.cs

@@ -378,9 +378,17 @@ public ForwardPass(GgufModel model, IComputeBackend backend, ModelHyperparams hp
             if (_hasQkNorm && !hp.UseL2QkNorm)
             {
                 int qNormSize = _perChannelQkNorm ? _numHeads * layerHd : layerHd;
-                int kNormSize = _perChannelQkNorm ? _numKvHeads * layerHd : layerHd;
                 _qNorm[i] = LoadBias($"blk.{i}.attn_q_norm.weight", qNormSize);
-                _kNorm[i] = LoadBias($"blk.{i}.attn_k_norm.weight", kNormSize);
+                // KV-share layers (Gemma 4 shared_kv_layers tail) reuse the source layer's
+                // already-normed K, so they carry no attn_k_norm — the QAT q4_0 GGUF omits it
+                // (the Q8_0 ships a dead, never-read copy). ApplyQkNormLayer passes k=null for
+                // these layers, so _kNorm[i] is never dereferenced; leave it null. Mirrors the
+                // attn_k/attn_v guard above and the CUDA/hybrid loaders (#211).
+                if (!kvShared)
+                {
+                    int kNormSize = _perChannelQkNorm ? _numKvHeads * layerHd : layerHd;
+                    _kNorm[i] = LoadBias($"blk.{i}.attn_k_norm.weight", kNormSize);
+                }
             }
 
             if (_postAttnNorm is not null)

tests/SharpInference.Tests.ForwardPass/Gemma4CpuForwardPassTests.cs

@@ -79,6 +79,45 @@ public void Gemma4_E4B_CpuForward_ProducesNonGarbageLogits()
         AssertCoherentDecode(fwd, eosId, logits, tokens.Length, hp.VocabSize, requireVariety: true);
     }
 
+    [Fact]
+    public void Gemma4_E4B_Q4_0_CpuForward_LoadsWithAbsentSharedKvNorm()
+    {
+        // #211: Google's official E4B QAT q4_0 GGUF omits attn_k/attn_v/attn_k_norm for the
+        // 18 shared-KV tail layers (the Q8_0 ships dead, never-read copies). The CPU loader
+        // used to require attn_k_norm unconditionally and threw
+        //   "Missing bias tensor: blk.24.attn_k_norm.weight".
+        // It now skips the K-norm for KV-share layers (where ApplyQkNormLayer passes k=null),
+        // so the file loads and decodes coherently.
+        var path = FindModelPath("gemma-4-E4B_q4_0-it.gguf");
+        if (path is null) return;   // silent skip — file only present on the dev box
+
+        using var model = GgufModel.Open(path);
+        var hp = ModelHyperparams.FromGgufMetadata(model.Metadata, model);
+
+        // Regression precondition: this file must actually have a KV-share layer whose
+        // attn_k_norm is genuinely absent (else the test wouldn't exercise the fix). Confirm
+        // attn_q_norm is still present for that layer — only the shared layers' K-norm is omitted.
+        Assert.NotNull(hp.KvSourceLayer);
+        int sharedLayer = -1;
+        for (int i = 0; i < hp.KvSourceLayer!.Count; i++)
+            if (hp.KvSourceLayer[i] >= 0) { sharedLayer = i; break; }
+        Assert.True(sharedLayer >= 0, "expected a KV-share layer in the E4B q4_0 GGUF — wrong file?");
+        Assert.Null(model.FindTensor($"blk.{sharedLayer}.attn_k_norm.weight"));
+        Assert.NotNull(model.FindTensor($"blk.{sharedLayer}.attn_q_norm.weight"));
+
+        int bosId = ReadIntMetadata(model, "tokenizer.ggml.bos_token_id", fallback: 2);
+        int eosId = ReadIntMetadata(model, "tokenizer.ggml.eos_token_id", fallback: 1);
+        var tokens = new int[] { bosId, 651, 6037, 576, 6081, 603, 1234, 4567, 8901 };
+
+        using var backend = new CpuBackend();
+        // Pre-#211 this constructor threw on blk.24.attn_k_norm.weight.
+        using var fwd = new SharpInference.Engine.ForwardPass(model, backend, hp);
+
+        var logits = fwd.Prefill(tokens);
+        Assert.Equal(hp.VocabSize, logits.Length);
+        AssertCoherentDecode(fwd, eosId, logits, tokens.Length, hp.VocabSize, requireVariety: true);
+    }
+
     [Fact]
     public void Gemma4_E4B_CpuForward_PleProducesVariety()
     {

tests/SharpInference.Tests.ForwardPass/Gemma4CudaForwardPassTests.cs

@@ -27,20 +27,22 @@ public sealed class Gemma4CudaForwardPassTests
         catch { return null; }
     }
 
-    private static string? FindModelPath()
+    private static string? FindModelPath() => FindModelPath(ModelFile);
+
+    private static string? FindModelPath(string fileName)
     {
         string[] absoluteCandidates =
         {
-            $@"E:\models\{ModelFile}",
-            $@"C:\p\sharpi\models\{ModelFile}",
+            $@"E:\models\{fileName}",
+            $@"C:\p\sharpi\models\{fileName}",
         };
         foreach (var p in absoluteCandidates)
             if (File.Exists(p)) return p;
 
         var dir = Directory.GetCurrentDirectory();
         for (int i = 0; i < 8; i++)
         {
-            var p = Path.Combine(dir, "models", ModelFile);
+            var p = Path.Combine(dir, "models", fileName);
             if (File.Exists(p)) return p;
             var parent = Directory.GetParent(dir);
             if (parent is null) break;
@@ -372,6 +374,60 @@ public void Gemma4_E4B_CudaForward_SnapKvDisabled_NoEvictionEvenOverBudget()
         }
     }
 
+    /// <summary>
+    /// #211: the E4B QAT q4_0 GGUF omits attn_k / attn_v / attn_k_norm for the 18 KV-share
+    /// tail layers (the Q8_0 ships dead copies). CudaForwardPass already guarded the upload
+    /// behind <c>!kvShared</c>; this pins that the full <c>-g -1</c> path loads the file and
+    /// tracks the CPU reference at first-decode argmax. Dev-box-only (silent-skips without the
+    /// file or CUDA), like every other gemma4 test here.
+    /// </summary>
+    [Fact]
+    public void Gemma4_E4B_Q4_0_CudaForward_LoadsAndMatchesCpuArgmax()
+    {
+        using var gpu = TryCreate();
+        if (gpu is null) return;
+        var path = FindModelPath("gemma-4-E4B_q4_0-it.gguf");
+        if (path is null) return;
+
+        using var model = GgufModel.Open(path);
+        var hp = ModelHyperparams.FromGgufMetadata(model.Metadata, model);
+        Assert.NotNull(hp.LayerHeadDim);
+
+        // Precondition: a KV-share layer whose attn_k_norm is genuinely absent (the case that
+        // threw pre-#211 on the CPU pass, and that CudaForwardPass's !kvShared guard skips).
+        Assert.NotNull(hp.KvSourceLayer);
+        int sharedLayer = -1;
+        for (int i = 0; i < hp.KvSourceLayer!.Count; i++)
+            if (hp.KvSourceLayer[i] >= 0) { sharedLayer = i; break; }
+        Assert.True(sharedLayer >= 0, "expected a KV-share layer in the E4B q4_0 GGUF — wrong file?");
+        Assert.Null(model.FindTensor($"blk.{sharedLayer}.attn_k_norm.weight"));
+
+        int bosId = ReadIntMetadata(model, "tokenizer.ggml.bos_token_id", fallback: 2);
+        int eosId = ReadIntMetadata(model, "tokenizer.ggml.eos_token_id", fallback: 1);
+        var tokens = new int[] { bosId, 651, 6037, 576, 6081, 603, 1234, 4567, 8901 };
+
+        // CPU reference (the pass that was fixed) — independent of any GPU kernel.
+        int cpuArgmax;
+        using (var cpuBackend = new CpuBackend())
+        using (var cpuFwd = new SharpInference.Engine.ForwardPass(model, cpuBackend, hp))
+            cpuArgmax = Argmax(cpuFwd.Prefill(tokens));
+
+        using var fwd = new CudaForwardPass(model, gpu, hp, maxContextLength: 512);
+        var logits = fwd.Prefill(tokens);
+        Assert.Equal(hp.VocabSize, logits.Length);
+
+        int nonFinite = 0;
+        for (int i = 0; i < logits.Length; i++)
+            if (!float.IsFinite(logits[i])) nonFinite++;
+        Assert.True(nonFinite == 0, $"{nonFinite} non-finite logits in E4B q4_0 CUDA output.");
+
+        int cudaArgmax = Argmax(logits);
+        if (eosId >= 0)
+            Assert.NotEqual(eosId, cudaArgmax);
+        Assert.True(cpuArgmax == cudaArgmax,
+            $"CPU↔CUDA E4B q4_0 first-argmax disagree: CPU={cpuArgmax} CUDA={cudaArgmax}.");
+    }
+
     private static int[] TopK(ReadOnlySpan<float> logits, int k)
     {
         var result = new int[k];