diff --git a/README.md b/README.md
index a332030..fcc56b0 100644
--- a/README.md
+++ b/README.md
@@ -52,6 +52,7 @@ greedy decode).
| Gemma 4 E4B-it Q8 | [unsloth](https://huggingface.co/unsloth/gemma-4-E4B-it-GGUF) | 8 GB | CPU | 5.0 | 5.1 | dense 42-layer gemma4: per-layer head_dim (256 SWA / 512 global), dual-RoPE, KV-share tail (18 layers), 5:1 SWA:global, logit softcap 30, PLE-256 injection (~4.2 GB mmap-resident) |
| Gemma 4 E4B-it Q8 | (same) | 8 GB | **CUDA** `-g -1 -c 2048` | **3444** | **70.5** | all 42 layers fit at `-c 2048`; KV-share alias + per-layer SWA/global split. Prefill: int8 tensor-core MMQ + tensor-core flash attention + SoA Q8_0 weight repack. Prompts >4096 use a real SWA KV ring on the chunked-flash path (fixes correctness past the 512 window). Decode: dp4a/Q8_1 + CUDA-graph replay; argmax-stable. (llama.cpp ~8475 prefill / ~78 decode) |
| Gemma 4 E4B-it Q8 | (same) | 8 GB | **CUDA** `-g 22 -c 2048` (hybrid) | 6.7 | 7.0 | 22 GPU + 20 CPU layers. `-g ≤ 22` required so the CPU shared-KV tail can read its own-KV source layers; CPU dense-FFN dominates decode (bandwidth-bound). `SHARPI_CUDA_PROFILE=1` for per-phase breakdown |
+| Gemma 4 E4B-it QAT q4_0 | [google](https://huggingface.co/google/gemma-4-E4B-it-qat-q4_0-gguf) | 5 GB | **CUDA** `-g -1 -c 2048` | **3283** | **100.4** | QAT q4_0 (5.15 GB): **~1.4× decode** vs Q8 at near-identical quality, frees ~3 GB for wider KV/context. Same gemma4 path; the shared-KV tail layers omit `attn_k`/`attn_v`/`attn_k_norm` (loaded conditionally). `download-model.ps1 -Model gemma4-e4b-qat` |
| Gemma 4 12B-it QAT Q4_0 | [google](https://huggingface.co/google/gemma-4-12B-it-qat-q4_0-gguf) | 7 GB | **CUDA** `-g -1 -c 2048` | **1742** | **54.1** | dense 48-layer gemma4, all bulk weights Q4_0 + tied Q6_K embedding; fits 12 GB full-offload. `attention_k_eq_v` (8 global MQA layers reuse K as V), per-layer KV heads (8 GQA / 1 MQA), pure V-norm. Prefill: Q4_0 int8 tensor-core MMQ over a SoA repack; decode: Q4_0 dp4a/Q8_1, argmax-stable, within ~6% of llama.cpp (57 t/s). **Long context:** `--kv-type bf16` halves / `q8_0` quarters the K/V store (fp32 kernel math, argmax-stable) + a bookkeeping-only host KV cache → **`-c 131072` (128K) within 12 GB** (fp32 `cudaMalloc`-fails at 64K). At 128K, q8_0 keeps the tied embed table resident (~53 t/s) where bf16 spills it (~19 t/s). Default fp32; opt-in `--kv-type bf16\|q8_0` |
_Re-measured 2026-06 from warm sweeps (prefill ~1K ctx, decode near-zero ctx), each after a discarded
diff --git a/src/SharpInference.Engine/ForwardPass.cs b/src/SharpInference.Engine/ForwardPass.cs
index 543c9b2..9a64068 100644
--- a/src/SharpInference.Engine/ForwardPass.cs
+++ b/src/SharpInference.Engine/ForwardPass.cs
@@ -378,9 +378,17 @@ public ForwardPass(GgufModel model, IComputeBackend backend, ModelHyperparams hp
if (_hasQkNorm && !hp.UseL2QkNorm)
{
int qNormSize = _perChannelQkNorm ? _numHeads * layerHd : layerHd;
- int kNormSize = _perChannelQkNorm ? _numKvHeads * layerHd : layerHd;
_qNorm[i] = LoadBias($"blk.{i}.attn_q_norm.weight", qNormSize);
- _kNorm[i] = LoadBias($"blk.{i}.attn_k_norm.weight", kNormSize);
+ // KV-share layers (Gemma 4 shared_kv_layers tail) reuse the source layer's
+ // already-normed K, so they carry no attn_k_norm — the QAT q4_0 GGUF omits it
+ // (the Q8_0 ships a dead, never-read copy). ApplyQkNormLayer passes k=null for
+ // these layers, so _kNorm[i] is never dereferenced; leave it null. Mirrors the
+ // attn_k/attn_v guard above and the CUDA/hybrid loaders (#211).
+ if (!kvShared)
+ {
+ int kNormSize = _perChannelQkNorm ? _numKvHeads * layerHd : layerHd;
+ _kNorm[i] = LoadBias($"blk.{i}.attn_k_norm.weight", kNormSize);
+ }
}
if (_postAttnNorm is not null)
diff --git a/tests/SharpInference.Tests.ForwardPass/Gemma4CpuForwardPassTests.cs b/tests/SharpInference.Tests.ForwardPass/Gemma4CpuForwardPassTests.cs
index 4f61c3d..b9014a1 100644
--- a/tests/SharpInference.Tests.ForwardPass/Gemma4CpuForwardPassTests.cs
+++ b/tests/SharpInference.Tests.ForwardPass/Gemma4CpuForwardPassTests.cs
@@ -79,6 +79,45 @@ public void Gemma4_E4B_CpuForward_ProducesNonGarbageLogits()
AssertCoherentDecode(fwd, eosId, logits, tokens.Length, hp.VocabSize, requireVariety: true);
}
+ [Fact]
+ public void Gemma4_E4B_Q4_0_CpuForward_LoadsWithAbsentSharedKvNorm()
+ {
+ // #211: Google's official E4B QAT q4_0 GGUF omits attn_k/attn_v/attn_k_norm for the
+ // 18 shared-KV tail layers (the Q8_0 ships dead, never-read copies). The CPU loader
+ // used to require attn_k_norm unconditionally and threw
+ // "Missing bias tensor: blk.24.attn_k_norm.weight".
+ // It now skips the K-norm for KV-share layers (where ApplyQkNormLayer passes k=null),
+ // so the file loads and decodes coherently.
+ var path = FindModelPath("gemma-4-E4B_q4_0-it.gguf");
+ if (path is null) return; // silent skip — file only present on the dev box
+
+ using var model = GgufModel.Open(path);
+ var hp = ModelHyperparams.FromGgufMetadata(model.Metadata, model);
+
+ // Regression precondition: this file must actually have a KV-share layer whose
+ // attn_k_norm is genuinely absent (else the test wouldn't exercise the fix). Confirm
+ // attn_q_norm is still present for that layer — only the shared layers' K-norm is omitted.
+ Assert.NotNull(hp.KvSourceLayer);
+ int sharedLayer = -1;
+ for (int i = 0; i < hp.KvSourceLayer!.Count; i++)
+ if (hp.KvSourceLayer[i] >= 0) { sharedLayer = i; break; }
+ Assert.True(sharedLayer >= 0, "expected a KV-share layer in the E4B q4_0 GGUF — wrong file?");
+ Assert.Null(model.FindTensor($"blk.{sharedLayer}.attn_k_norm.weight"));
+ Assert.NotNull(model.FindTensor($"blk.{sharedLayer}.attn_q_norm.weight"));
+
+ int bosId = ReadIntMetadata(model, "tokenizer.ggml.bos_token_id", fallback: 2);
+ int eosId = ReadIntMetadata(model, "tokenizer.ggml.eos_token_id", fallback: 1);
+ var tokens = new int[] { bosId, 651, 6037, 576, 6081, 603, 1234, 4567, 8901 };
+
+ using var backend = new CpuBackend();
+ // Pre-#211 this constructor threw on blk.24.attn_k_norm.weight.
+ using var fwd = new SharpInference.Engine.ForwardPass(model, backend, hp);
+
+ var logits = fwd.Prefill(tokens);
+ Assert.Equal(hp.VocabSize, logits.Length);
+ AssertCoherentDecode(fwd, eosId, logits, tokens.Length, hp.VocabSize, requireVariety: true);
+ }
+
[Fact]
public void Gemma4_E4B_CpuForward_PleProducesVariety()
{
diff --git a/tests/SharpInference.Tests.ForwardPass/Gemma4CudaForwardPassTests.cs b/tests/SharpInference.Tests.ForwardPass/Gemma4CudaForwardPassTests.cs
index cae69be..46d660f 100644
--- a/tests/SharpInference.Tests.ForwardPass/Gemma4CudaForwardPassTests.cs
+++ b/tests/SharpInference.Tests.ForwardPass/Gemma4CudaForwardPassTests.cs
@@ -27,12 +27,14 @@ public sealed class Gemma4CudaForwardPassTests
catch { return null; }
}
- private static string? FindModelPath()
+ private static string? FindModelPath() => FindModelPath(ModelFile);
+
+ private static string? FindModelPath(string fileName)
{
string[] absoluteCandidates =
{
- $@"E:\models\{ModelFile}",
- $@"C:\p\sharpi\models\{ModelFile}",
+ $@"E:\models\{fileName}",
+ $@"C:\p\sharpi\models\{fileName}",
};
foreach (var p in absoluteCandidates)
if (File.Exists(p)) return p;
@@ -40,7 +42,7 @@ public sealed class Gemma4CudaForwardPassTests
var dir = Directory.GetCurrentDirectory();
for (int i = 0; i < 8; i++)
{
- var p = Path.Combine(dir, "models", ModelFile);
+ var p = Path.Combine(dir, "models", fileName);
if (File.Exists(p)) return p;
var parent = Directory.GetParent(dir);
if (parent is null) break;
@@ -372,6 +374,60 @@ public void Gemma4_E4B_CudaForward_SnapKvDisabled_NoEvictionEvenOverBudget()
}
}
+ ///
+ /// #211: the E4B QAT q4_0 GGUF omits attn_k / attn_v / attn_k_norm for the 18 KV-share
+ /// tail layers (the Q8_0 ships dead copies). CudaForwardPass already guarded the upload
+ /// behind !kvShared; this pins that the full -g -1 path loads the file and
+ /// tracks the CPU reference at first-decode argmax. Dev-box-only (silent-skips without the
+ /// file or CUDA), like every other gemma4 test here.
+ ///
+ [Fact]
+ public void Gemma4_E4B_Q4_0_CudaForward_LoadsAndMatchesCpuArgmax()
+ {
+ using var gpu = TryCreate();
+ if (gpu is null) return;
+ var path = FindModelPath("gemma-4-E4B_q4_0-it.gguf");
+ if (path is null) return;
+
+ using var model = GgufModel.Open(path);
+ var hp = ModelHyperparams.FromGgufMetadata(model.Metadata, model);
+ Assert.NotNull(hp.LayerHeadDim);
+
+ // Precondition: a KV-share layer whose attn_k_norm is genuinely absent (the case that
+ // threw pre-#211 on the CPU pass, and that CudaForwardPass's !kvShared guard skips).
+ Assert.NotNull(hp.KvSourceLayer);
+ int sharedLayer = -1;
+ for (int i = 0; i < hp.KvSourceLayer!.Count; i++)
+ if (hp.KvSourceLayer[i] >= 0) { sharedLayer = i; break; }
+ Assert.True(sharedLayer >= 0, "expected a KV-share layer in the E4B q4_0 GGUF — wrong file?");
+ Assert.Null(model.FindTensor($"blk.{sharedLayer}.attn_k_norm.weight"));
+
+ int bosId = ReadIntMetadata(model, "tokenizer.ggml.bos_token_id", fallback: 2);
+ int eosId = ReadIntMetadata(model, "tokenizer.ggml.eos_token_id", fallback: 1);
+ var tokens = new int[] { bosId, 651, 6037, 576, 6081, 603, 1234, 4567, 8901 };
+
+ // CPU reference (the pass that was fixed) — independent of any GPU kernel.
+ int cpuArgmax;
+ using (var cpuBackend = new CpuBackend())
+ using (var cpuFwd = new SharpInference.Engine.ForwardPass(model, cpuBackend, hp))
+ cpuArgmax = Argmax(cpuFwd.Prefill(tokens));
+
+ using var fwd = new CudaForwardPass(model, gpu, hp, maxContextLength: 512);
+ var logits = fwd.Prefill(tokens);
+ Assert.Equal(hp.VocabSize, logits.Length);
+
+ int nonFinite = 0;
+ for (int i = 0; i < logits.Length; i++)
+ if (!float.IsFinite(logits[i])) nonFinite++;
+ Assert.True(nonFinite == 0, $"{nonFinite} non-finite logits in E4B q4_0 CUDA output.");
+
+ int cudaArgmax = Argmax(logits);
+ if (eosId >= 0)
+ Assert.NotEqual(eosId, cudaArgmax);
+ Assert.True(cpuArgmax == cudaArgmax,
+ $"CPU↔CUDA E4B q4_0 first-argmax disagree: CPU={cpuArgmax} CUDA={cudaArgmax}.");
+ }
+
private static int[] TopK(ReadOnlySpan logits, int k)
{
var result = new int[k];