From 99cfff46ebe0321e3b64f36baf3d82c7221fcf2d Mon Sep 17 00:00:00 2001 From: "Yoshiaki Ueda (bootjp)" Date: Sun, 24 May 2026 22:16:34 +0900 Subject: [PATCH 1/2] feat(encryption): Stage 6D-6c-1 - Applier in-memory accessors for storage envelope state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add `Applier.ActiveStorageKeyID() (uint32, bool)` and `Applier.StorageEnvelopeActive() bool`, backed by `atomic.Uint32` / `atomic.Bool` fields. These are the per-Put closures main.go will thread into `store.WithEncryption` and `store.WithStorageEnvelopeGate` in 6D-6c-2; a ReadSidecar-on-every-Put would serialise the hot path through a JSON parse + fsync barrier. Coherence with the on-disk sidecar is maintained by durable write-then-cache ordering: - `NewApplier` primes both atomics from `ReadSidecar` on construction (best-effort: a missing sidecar is the pre-bootstrap posture and leaves both at zero; a corrupt read surfaces back to the caller so a misconfigured node fails to start instead of silently serving stale-zero state). - `writeBootstrapSidecar`, `writeRotationSidecar`, and the fresh-success branch of `applyEnableStorageEnvelope` call `refreshActiveStateCache(sc)` AFTER the WriteSidecar succeeds. Crash between fsync and atomic store is benign: disk-truth wins and the next startup's prime re-syncs the cache. - The §2.1 #3 stale-DEKID and §2.1 #4 already-active no-op branches intentionally skip the refresh because they do not change `Active.Storage` or `StorageEnvelopeActive` — keeping the invariant "cache changes only when sidecar fields change" explicit. Tests cover pre-bootstrap (0, false), post-bootstrap, post-rotate-dek (re-points active), post-cutover (envelope flips, active unchanged), NewApplier priming from an existing sidecar across simulated process restart, and a concurrent-reads stress test under `-race`. Operator-inert by itself; 6D-6c-2 wires the method values into the storage layer. Refs: docs/design/2026_05_18_partial_6d_enable_storage_envelope.md Self-review (5 lenses): - Data loss: refresh is purely additive after fsync — no write path shortened; corrupt-sidecar reads at construction fail the start. - Concurrency: atomic primitives + -race-clean concurrent-reads test. Two independent atomics is fine because the storage layer consults them independently and "envelope-active ⇒ DEK exists" is already guaranteed by apply ordering. - Performance: hot path drops from JSON parse + fsync barrier to a single atomic load. - Data consistency: durable write-then-cache; no-op branches skip the redundant refresh; §2.1 / §6.4 semantics unchanged. - Test coverage: 5 functional tests + 1 race-stress test, all via the public API. --- ...5_18_partial_6d_enable_storage_envelope.md | 33 +- internal/encryption/applier.go | 86 +++++ internal/encryption/applier_test.go | 313 ++++++++++++++++++ 3 files changed, 425 insertions(+), 7 deletions(-) diff --git a/docs/design/2026_05_18_partial_6d_enable_storage_envelope.md b/docs/design/2026_05_18_partial_6d_enable_storage_envelope.md index a614678a4..9d7aa61cb 100644 --- a/docs/design/2026_05_18_partial_6d_enable_storage_envelope.md +++ b/docs/design/2026_05_18_partial_6d_enable_storage_envelope.md @@ -2,7 +2,7 @@ | Field | Value | |---|---| -| Status | partial — 6D-1 (doc), 6D-2 (startup guards), 6D-3 (capability fan-out helper), 6D-4 (cutover wire + apply dispatch), 6D-5 (storage-layer toggle), 6D-6a (EnableStorageEnvelope server method), 6D-6b (CLI subcommand) shipped; 6D-6c (main.go wiring + integration test) remain | +| Status | partial — 6D-1 (doc), 6D-2 (startup guards), 6D-3 (capability fan-out helper), 6D-4 (cutover wire + apply dispatch), 6D-5 (storage-layer toggle), 6D-6a (EnableStorageEnvelope server method), 6D-6b (CLI subcommand), 6D-6c-1 (Applier in-memory accessors) shipped; 6D-6c-2 (main.go cipher + gate wiring) and 6D-6c-3 (capability fan-out closure + e2e integration test) remain | | Date | 2026-05-18 | | Parent design | [`2026_04_29_partial_data_at_rest_encryption.md`](2026_04_29_partial_data_at_rest_encryption.md) | | Blockers (now satisfied) | 6B (KEK plumbing), 6C-1 / 6C-2 (startup guards), 6C-2d (`ErrSidecarBehindRaftLog` wiring) | @@ -71,14 +71,33 @@ misconfigured shell variable fails fast before the round-trip; the server re-validates as the source of truth. +- **6D-6c-1** (Applier in-memory accessors) — `Applier` grows + `ActiveStorageKeyID() (uint32, bool)` and + `StorageEnvelopeActive() bool` backed by `atomic.Uint32` / + `atomic.Bool` fields, kept coherent with the on-disk sidecar + via durable write-then-cache ordering inside + `writeBootstrapSidecar`, `writeRotationSidecar`, and the + cutover fresh-success branch of `applyEnableStorageEnvelope`. + `NewApplier` primes both atomics from the sidecar on + construction so the storage-layer per-Put closures (wired + in 6D-6c-2) observe correct values before the FSM has + replayed a single entry after restart. Operator-inert by + itself — only consumed once main.go threads the methods + into `store.WithEncryption` and `WithStorageEnvelopeGate` + in 6D-6c-2. + ## Open milestones -- **6D-6c** — main.go production wiring: cipher + WithEncryption - + WithStorageEnvelopeGate threaded from the sidecar, plus the - CapabilityFanout closure bound to the live Raft membership - view. End-to-end integration test exercises a single-node - cluster Bootstrap → EnableStorageEnvelope → Put → read-back- - via-envelope. +- **6D-6c-2** — main.go production wiring: build + `encryption.NewCipher(keystore)` and thread the + `Applier.ActiveStorageKeyID` / `Applier.StorageEnvelopeActive` + method values into `store.WithEncryption` + + `store.WithStorageEnvelopeGate` at shard-group construction. +- **6D-6c-3** — main.go CapabilityFanout closure bound to the + live Raft membership view (etcd engine route snapshot + admin + client DialFunc), and end-to-end integration test exercising + a single-node cluster Bootstrap → EnableStorageEnvelope → + Put → read-back-via-envelope. ## 0. Why this doc exists diff --git a/internal/encryption/applier.go b/internal/encryption/applier.go index f5dbdb356..5a62334be 100644 --- a/internal/encryption/applier.go +++ b/internal/encryption/applier.go @@ -3,6 +3,7 @@ package encryption import ( "bytes" "strconv" + "sync/atomic" "time" "github.com/bootjp/elastickv/internal/encryption/fsmwire" @@ -74,6 +75,30 @@ type Applier struct { keystore *Keystore sidecarPath string now func() time.Time + // activeStorageDEKID is the in-memory mirror of + // sidecar.Active.Storage. The storage layer's encryption + // path (store.WithEncryption) calls a per-Put closure that + // reads this field via ActiveStorageKeyID(); a sidecar + // ReadSidecar() on every Put would serialise hot-path writes + // through a JSON parse + fsync barrier. Apply paths update + // this atomic AFTER the durable WriteSidecar succeeds, so + // a crash leaves disk-truth and cache aligned: on restart + // NewApplier primes the cache from disk before the storage + // layer ever queries it. Zero means "not bootstrapped" — the + // closure surfaces (0, false) and the storage layer writes + // cleartext. + activeStorageDEKID atomic.Uint32 + // storageEnvelopeActive mirrors sidecar.StorageEnvelopeActive + // for the §6.2 cutover gate. Same atomic-vs-sidecar-read + // rationale as activeStorageDEKID above. Lifecycle: + // - false at construction (or primed from disk if a + // previous cutover already fired). + // - flipped to true exactly once by applyEnableStorageEnvelope + // on a fresh-success apply, AFTER WriteSidecar succeeds. + // - never flipped back to false (the cutover is one-way per + // §7.1 Phase 1; rotate-dek under the active envelope keeps + // it true). + storageEnvelopeActive atomic.Bool } // KEKUnwrapper is the abstraction the Applier uses to recover @@ -162,9 +187,67 @@ func NewApplier(registry WriterRegistryStore, opts ...ApplierOption) (*Applier, if a.now == nil { return nil, errors.New("encryption: NewApplier: WithNowFunc(nil) overwrote default time.Now") } + // Prime the in-memory accessors from the on-disk sidecar + // (best-effort: a missing sidecar is the pre-bootstrap + // posture and leaves both atomics at their zero values, + // which is correct). The storage-layer closures may query + // these atomics before the FSM has replayed a single entry + // after restart, so the priming must happen at construction + // rather than lazily on first apply. A read error (corrupt + // JSON, bad version) surfaces back to the caller so a + // misconfigured node fails to start instead of silently + // running with stale-zero state. + if a.sidecarPath != "" { + switch sc, err := ReadSidecar(a.sidecarPath); { + case err == nil: + a.refreshActiveStateCache(sc) + case IsNotExist(err): + // Pre-bootstrap; leave atomics at zero. + default: + return nil, errors.Wrap(err, "encryption: NewApplier: prime in-memory state from sidecar") + } + } return a, nil } +// ActiveStorageKeyID returns the current sidecar.Active.Storage +// DEK id in-memory. Signature matches store.ActiveStorageKeyID +// so main.go can pass `applier.ActiveStorageKeyID` directly into +// `store.WithEncryption(...)` as the per-Put activeKeyID +// closure. A non-zero id with ok=true means the cluster has run +// BootstrapEncryption; zero with ok=false means the cluster is +// still pre-bootstrap and the storage layer should write +// cleartext. +func (a *Applier) ActiveStorageKeyID() (uint32, bool) { + id := a.activeStorageDEKID.Load() + return id, id != 0 +} + +// StorageEnvelopeActive returns the in-memory mirror of +// sidecar.StorageEnvelopeActive. Signature matches +// store.StorageEnvelopeActive so main.go can pass +// `applier.StorageEnvelopeActive` directly into +// `store.WithStorageEnvelopeGate(...)` as the per-Put cutover +// gate (Stage 6D-5). Once true, the storage layer wraps every +// new version in the §4.1 envelope; flips exactly once per +// cluster lifetime when the §7.1 Phase 1 cutover entry applies. +func (a *Applier) StorageEnvelopeActive() bool { + return a.storageEnvelopeActive.Load() +} + +// refreshActiveStateCache copies the relevant sidecar fields +// into the in-memory atomics. Called from NewApplier (prime +// from disk on startup) and from every apply path AFTER a +// successful WriteSidecar (durable write-then-cache ordering +// so a crash leaves disk-truth and the next restart's prime +// in agreement). Holding the mutate-after-WriteSidecar order +// also avoids the storage layer reading a flipped cache while +// the sidecar disk write is still mid-fsync. +func (a *Applier) refreshActiveStateCache(sc *Sidecar) { + a.activeStorageDEKID.Store(sc.Active.Storage) + a.storageEnvelopeActive.Store(sc.StorageEnvelopeActive) +} + // bootstrapAndRotationConfigured reports whether WithKEK, // WithKeystore, and WithSidecarPath have all been supplied. The // three are an indivisible quorum for ApplyBootstrap / @@ -513,6 +596,7 @@ func (a *Applier) writeBootstrapSidecar(raftIdx uint64, p fsmwire.BootstrapPaylo if err := WriteSidecar(a.sidecarPath, sc); err != nil { return errors.Wrap(err, "applier: write sidecar for bootstrap") } + a.refreshActiveStateCache(sc) return nil } @@ -773,6 +857,7 @@ func (a *Applier) applyEnableStorageEnvelope(raftIdx uint64, p fsmwire.RotationP if err := WriteSidecar(a.sidecarPath, sc); err != nil { return errors.Wrap(err, "applier: write sidecar for cutover") } + a.refreshActiveStateCache(sc) return nil } @@ -809,6 +894,7 @@ func (a *Applier) writeRotationSidecar(raftIdx uint64, p fsmwire.RotationPayload if err := WriteSidecar(a.sidecarPath, sc); err != nil { return errors.Wrap(err, "applier: write sidecar for rotation") } + a.refreshActiveStateCache(sc) return nil } diff --git a/internal/encryption/applier_test.go b/internal/encryption/applier_test.go index 7199954a1..6bc58f9e0 100644 --- a/internal/encryption/applier_test.go +++ b/internal/encryption/applier_test.go @@ -1522,3 +1522,316 @@ func TestApplyRotation_RejectsProposerDEKMismatch(t *testing.T) { t.Errorf("err not marked ErrEncryptionApply: %v", err) } } + +// --- Stage 6D-6c-1: in-memory accessor coverage --- +// +// The Applier exposes ActiveStorageKeyID() and StorageEnvelopeActive() +// so main.go can wire them as the per-Put closures into +// store.WithEncryption / store.WithStorageEnvelopeGate without +// ReadSidecar-on-every-Put. Hot-path correctness requires: +// +// - Pre-bootstrap: (0, false) and false. Storage layer must +// write cleartext. +// - Post-bootstrap: (StorageDEKID, true) and false. Encryption +// wraps payloads at-rest using the storage DEK, but the +// §4.1 envelope is still gated off. +// - Post-rotate-dek-storage: returns the NEW storage DEK id, +// and StorageEnvelopeActive unchanged. +// - Post-cutover: ActiveStorageKeyID unchanged, but the gate +// flips to true. +// - NewApplier primes both atomics from the on-disk sidecar so +// a freshly-started process serves correct accessor values +// before the FSM has replayed a single entry. +// - Concurrent readers under -race must observe coherent values +// (atomic.Bool / atomic.Uint32 guarantees). + +func TestApplier_StorageAccessors_PreBootstrap(t *testing.T) { + t.Parallel() + dir := t.TempDir() + sidecarPath := dir + "/keys.json" + app, err := encryption.NewApplier(newMapRegistryStore(), + encryption.WithKEK(&fakeKEK{}), + encryption.WithKeystore(encryption.NewKeystore()), + encryption.WithSidecarPath(sidecarPath), + ) + if err != nil { + t.Fatalf("NewApplier: %v", err) + } + id, ok := app.ActiveStorageKeyID() + if ok || id != 0 { + t.Errorf("ActiveStorageKeyID pre-bootstrap = (%d, %v), want (0, false)", id, ok) + } + if app.StorageEnvelopeActive() { + t.Error("StorageEnvelopeActive pre-bootstrap = true, want false") + } +} + +func TestApplier_StorageAccessors_PostBootstrap(t *testing.T) { + t.Parallel() + reg := newMapRegistryStore() + ks := encryption.NewKeystore() + dir := t.TempDir() + sidecarPath := dir + "/keys.json" + app, err := encryption.NewApplier(reg, + encryption.WithKEK(&fakeKEK{}), + encryption.WithKeystore(ks), + encryption.WithSidecarPath(sidecarPath), + ) + if err != nil { + t.Fatalf("NewApplier: %v", err) + } + if err := app.ApplyBootstrap(100, fsmwire.BootstrapPayload{ + StorageDEKID: 11, WrappedStorage: []byte("s"), + RaftDEKID: 22, WrappedRaft: []byte("r"), + BatchRegistry: []fsmwire.RegistrationPayload{ + {DEKID: 11, FullNodeID: 0xAAAA, LocalEpoch: 0}, + }, + }); err != nil { + t.Fatalf("ApplyBootstrap: %v", err) + } + id, ok := app.ActiveStorageKeyID() + if !ok || id != 11 { + t.Errorf("ActiveStorageKeyID post-bootstrap = (%d, %v), want (11, true)", id, ok) + } + // Cutover gate must NOT flip just because bootstrap landed — + // §7.1 Phase 1 requires an explicit EnableStorageEnvelope + // proposal. + if app.StorageEnvelopeActive() { + t.Error("StorageEnvelopeActive post-bootstrap = true, want false") + } +} + +func TestApplier_StorageAccessors_PostRotateDEK(t *testing.T) { + t.Parallel() + reg := newMapRegistryStore() + ks := encryption.NewKeystore() + dir := t.TempDir() + sidecarPath := dir + "/keys.json" + app, err := encryption.NewApplier(reg, + encryption.WithKEK(&fakeKEK{}), + encryption.WithKeystore(ks), + encryption.WithSidecarPath(sidecarPath), + ) + if err != nil { + t.Fatalf("NewApplier: %v", err) + } + if err := app.ApplyBootstrap(100, fsmwire.BootstrapPayload{ + StorageDEKID: 11, WrappedStorage: []byte("s"), + RaftDEKID: 22, WrappedRaft: []byte("r"), + BatchRegistry: []fsmwire.RegistrationPayload{ + {DEKID: 11, FullNodeID: 0xAAAA, LocalEpoch: 0}, + }, + }); err != nil { + t.Fatalf("ApplyBootstrap: %v", err) + } + // Rotate storage DEK 11 -> 33. + if err := app.ApplyRotation(200, fsmwire.RotationPayload{ + SubTag: fsmwire.RotateSubRotateDEK, + DEKID: 33, + Purpose: fsmwire.PurposeStorage, + Wrapped: []byte("w33"), + ProposerRegistration: fsmwire.RegistrationPayload{DEKID: 33, FullNodeID: 0xAAAA, LocalEpoch: 1}, + }); err != nil { + t.Fatalf("ApplyRotation rotate-dek: %v", err) + } + id, ok := app.ActiveStorageKeyID() + if !ok || id != 33 { + t.Errorf("ActiveStorageKeyID post-rotate = (%d, %v), want (33, true)", id, ok) + } + if app.StorageEnvelopeActive() { + t.Error("StorageEnvelopeActive flipped by rotate-dek, want false (rotate is a key swap, not a cutover)") + } +} + +func TestApplier_StorageAccessors_PostCutover(t *testing.T) { + t.Parallel() + reg := newMapRegistryStore() + ks := encryption.NewKeystore() + dir := t.TempDir() + sidecarPath := dir + "/keys.json" + app, err := encryption.NewApplier(reg, + encryption.WithKEK(&fakeKEK{}), + encryption.WithKeystore(ks), + encryption.WithSidecarPath(sidecarPath), + ) + if err != nil { + t.Fatalf("NewApplier: %v", err) + } + if err := app.ApplyBootstrap(100, fsmwire.BootstrapPayload{ + StorageDEKID: 7, WrappedStorage: []byte("s"), + RaftDEKID: 8, WrappedRaft: []byte("r"), + BatchRegistry: []fsmwire.RegistrationPayload{ + {DEKID: 7, FullNodeID: 0xAAAA, LocalEpoch: 0}, + }, + }); err != nil { + t.Fatalf("ApplyBootstrap: %v", err) + } + if err := app.ApplyRotation(500, fsmwire.RotationPayload{ + SubTag: fsmwire.RotateSubEnableStorageEnvelope, + DEKID: 7, + Purpose: fsmwire.PurposeStorage, + Wrapped: []byte{}, + ProposerRegistration: fsmwire.RegistrationPayload{DEKID: 7, FullNodeID: 0xAAAA, LocalEpoch: 1}, + }); err != nil { + t.Fatalf("ApplyRotation cutover: %v", err) + } + id, ok := app.ActiveStorageKeyID() + if !ok || id != 7 { + t.Errorf("ActiveStorageKeyID post-cutover = (%d, %v), want (7, true) — cutover must not re-point Active", id, ok) + } + if !app.StorageEnvelopeActive() { + t.Error("StorageEnvelopeActive post-cutover = false, want true") + } +} + +// TestApplier_StorageAccessors_PrimedFromExistingSidecar pins the +// startup-recovery contract: a process restart constructs a fresh +// Applier against an already-populated sidecar (the previous +// process's apply landed). The accessors must reflect the on-disk +// state BEFORE the FSM has replayed a single entry, because the +// storage layer's Put path queries them immediately after Pebble +// open. +func TestApplier_StorageAccessors_PrimedFromExistingSidecar(t *testing.T) { + t.Parallel() + dir := t.TempDir() + sidecarPath := dir + "/keys.json" + // First Applier writes a full post-cutover sidecar. + reg := newMapRegistryStore() + first, err := encryption.NewApplier(reg, + encryption.WithKEK(&fakeKEK{}), + encryption.WithKeystore(encryption.NewKeystore()), + encryption.WithSidecarPath(sidecarPath), + ) + if err != nil { + t.Fatalf("NewApplier (first): %v", err) + } + if err := first.ApplyBootstrap(100, fsmwire.BootstrapPayload{ + StorageDEKID: 9, WrappedStorage: []byte("s"), + RaftDEKID: 10, WrappedRaft: []byte("r"), + BatchRegistry: []fsmwire.RegistrationPayload{ + {DEKID: 9, FullNodeID: 0xAAAA, LocalEpoch: 0}, + }, + }); err != nil { + t.Fatalf("first ApplyBootstrap: %v", err) + } + if err := first.ApplyRotation(500, fsmwire.RotationPayload{ + SubTag: fsmwire.RotateSubEnableStorageEnvelope, + DEKID: 9, + Purpose: fsmwire.PurposeStorage, + Wrapped: []byte{}, + ProposerRegistration: fsmwire.RegistrationPayload{DEKID: 9, FullNodeID: 0xAAAA, LocalEpoch: 1}, + }); err != nil { + t.Fatalf("first ApplyRotation cutover: %v", err) + } + + // Second Applier — simulates a process restart. The on-disk + // sidecar holds Active.Storage=9 and StorageEnvelopeActive=true. + // We use a *fresh* mapRegistryStore so this is unambiguously a + // from-disk prime, not state inherited from `first`. + second, err := encryption.NewApplier(newMapRegistryStore(), + encryption.WithKEK(&fakeKEK{}), + encryption.WithKeystore(encryption.NewKeystore()), + encryption.WithSidecarPath(sidecarPath), + ) + if err != nil { + t.Fatalf("NewApplier (second): %v", err) + } + id, ok := second.ActiveStorageKeyID() + if !ok || id != 9 { + t.Errorf("second ActiveStorageKeyID = (%d, %v), want (9, true)", id, ok) + } + if !second.StorageEnvelopeActive() { + t.Error("second StorageEnvelopeActive = false, want true (must prime from disk)") + } +} + +// TestApplier_StorageAccessors_ConcurrentReads exercises the +// atomic.Uint32 / atomic.Bool seam under -race. The accessors are +// called on the hot storage Put path, so a torn read or a missed +// happens-before edge with the apply-side store would be a +// production correctness bug. We run many concurrent reader +// goroutines alongside a single applier goroutine and verify +// readers only ever observe one of the two valid states (zero or +// the post-cutover state) — never a torn intermediate. +func TestApplier_StorageAccessors_ConcurrentReads(t *testing.T) { + t.Parallel() + reg := newMapRegistryStore() + dir := t.TempDir() + sidecarPath := dir + "/keys.json" + app, err := encryption.NewApplier(reg, + encryption.WithKEK(&fakeKEK{}), + encryption.WithKeystore(encryption.NewKeystore()), + encryption.WithSidecarPath(sidecarPath), + ) + if err != nil { + t.Fatalf("NewApplier: %v", err) + } + + const wantDEKID uint32 = 41 + const readers = 8 + const readsPerGoroutine = 2000 + + start := make(chan struct{}) + done := make(chan struct{}, readers) + for i := 0; i < readers; i++ { + go concurrentAccessorReader(t, app, wantDEKID, readsPerGoroutine, start, done) + } + close(start) + if err := app.ApplyBootstrap(100, fsmwire.BootstrapPayload{ + StorageDEKID: wantDEKID, WrappedStorage: []byte("s"), + RaftDEKID: 42, WrappedRaft: []byte("r"), + BatchRegistry: []fsmwire.RegistrationPayload{ + {DEKID: wantDEKID, FullNodeID: 0xAAAA, LocalEpoch: 0}, + }, + }); err != nil { + t.Fatalf("ApplyBootstrap: %v", err) + } + for i := 0; i < readers; i++ { + <-done + } + id, ok := app.ActiveStorageKeyID() + if !ok || id != wantDEKID { + t.Errorf("final ActiveStorageKeyID = (%d, %v), want (%d, true)", id, ok, wantDEKID) + } +} + +// concurrentAccessorReader spins on ActiveStorageKeyID until it +// observes either the pre-bootstrap zero state or the +// post-bootstrap (wantDEKID, true) state. Any other combination +// indicates a torn read across the (uint32 id, bool ok) pair and +// fails the test. Extracted out of +// TestApplier_StorageAccessors_ConcurrentReads to keep that test +// under cyclop's complexity ceiling. +func concurrentAccessorReader( + t *testing.T, + app *encryption.Applier, + wantDEKID uint32, + reads int, + start <-chan struct{}, + done chan<- struct{}, +) { + t.Helper() + defer func() { done <- struct{}{} }() + <-start + for j := 0; j < reads; j++ { + id, ok := app.ActiveStorageKeyID() + if !validAccessorObservation(id, ok, wantDEKID) { + t.Errorf("torn ActiveStorageKeyID read: (%d, %v)", id, ok) + return + } + _ = app.StorageEnvelopeActive() + } +} + +// validAccessorObservation accepts only the two coherent states +// the readers may witness while the bootstrap apply is in flight: +// (0, false) pre-flip and (wantDEKID, true) post-flip. +func validAccessorObservation(id uint32, ok bool, wantDEKID uint32) bool { + if !ok && id == 0 { + return true + } + if ok && id == wantDEKID { + return true + } + return false +} From d547b32853be325d03c934ba7feca71f16697781 Mon Sep 17 00:00:00 2001 From: "Yoshiaki Ueda (bootjp)" Date: Sun, 24 May 2026 23:03:27 +0900 Subject: [PATCH 2/2] fix(encryption): PR821 round-1 - codex P1 shared StateCache across shard appliers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex P1 on PR #821 (review comment r3294696832) flagged a correctness regression in 6D-6c-1's per-Applier atomic fields: in a multi-group deployment, main.go's `buildShardGroups` constructs one Applier per shard with the same sidecar path, but encryption FSM entries apply on exactly one shard's leader (the one whose engine accepted the proposal). Per-Applier atomics left every other shard's storage layer reading the pre-apply zero state forever — silently writing cleartext after bootstrap or skipping the §4.1 envelope wrap after cutover, despite the sidecar having advanced. Fix: introduce `encryption.StateCache` as a process-wide singleton (parallel to the shared `*Keystore`) that the new `WithStateCache` ApplierOption threads into every per-shard Applier. The cache carries the atomic mirrors; Applier delegates to it. main.go (in 6D-6c-2) will: - construct one `StateCache` at startup, - pass it via `WithStateCache` into every per-shard Applier, - pass `cache.ActiveStorageKeyID` / `cache.StorageEnvelopeActive` (NOT the per-shard Applier delegates) into the storage layer's per-Put closures. The Applier-bound accessor methods remain as convenience delegates so the existing single-applier tests keep working unchanged. Tests that omit `WithStateCache` get a private instance auto-installed by `NewApplier`. Regression test: `TestApplier_StorageAccessors_SharedCache_AcrossAppliers` wires two appliers to the same sidecar + shared StateCache, runs ApplyBootstrap on the first and ApplyRotation cutover on the second, then asserts BOTH appliers' accessors AND the shared cache observe the merged final state. Fails on the original per-Applier-atomics design. `TestStateCache_NilSafe` pins the nil-receiver contract so callers in early startup paths get the pre-bootstrap posture rather than a nil-deref panic. Design doc updated: 6D-6c-1 status block records the shared StateCache requirement; 6D-6c-2 entry is updated to instruct main.go to thread `cache.ActiveStorageKeyID` / `cache.StorageEnvelopeActive` into the storage layer (not the per-shard Applier delegates). Self-review (5 lenses): - Data loss: shared cache strengthens, not weakens, the durable-write-then-cache invariant — every shard's storage layer now sees the same on-disk state. - Concurrency: atomic.Store from a single FSM apply goroutine per encryption entry; reads are wait-free from any goroutine. -race-clean. - Performance: unchanged hot path (single atomic load). - Data consistency: closes the cross-shard staleness window that 6D-6c-1 introduced; §2.1 / §6.4 cutover semantics unchanged. - Test coverage: cross-shard regression test + nil-receiver test added. Refs: https://github.com/bootjp/elastickv/pull/821#discussion_r3294696832 --- ...5_18_partial_6d_enable_storage_envelope.md | 50 +++-- internal/encryption/applier.go | 207 +++++++++++++----- internal/encryption/applier_test.go | 128 +++++++++++ 3 files changed, 312 insertions(+), 73 deletions(-) diff --git a/docs/design/2026_05_18_partial_6d_enable_storage_envelope.md b/docs/design/2026_05_18_partial_6d_enable_storage_envelope.md index 9d7aa61cb..4eafa7061 100644 --- a/docs/design/2026_05_18_partial_6d_enable_storage_envelope.md +++ b/docs/design/2026_05_18_partial_6d_enable_storage_envelope.md @@ -71,28 +71,46 @@ misconfigured shell variable fails fast before the round-trip; the server re-validates as the source of truth. -- **6D-6c-1** (Applier in-memory accessors) — `Applier` grows - `ActiveStorageKeyID() (uint32, bool)` and - `StorageEnvelopeActive() bool` backed by `atomic.Uint32` / - `atomic.Bool` fields, kept coherent with the on-disk sidecar - via durable write-then-cache ordering inside +- **6D-6c-1** (Applier in-memory accessors + shared StateCache) + — new exported `encryption.StateCache` type backed by + `atomic.Uint32` / `atomic.Bool` mirrors of `sidecar.Active.Storage` + and `sidecar.StorageEnvelopeActive`. The cache is a **process- + wide singleton** (parallel to the shared `*Keystore`) threaded + into every per-shard `Applier` via the new `WithStateCache` + option. Multi-group encryption FSM entries apply on exactly + one shard's leader, so per-Applier-private atomics would leave + the remaining shards stuck with pre-apply values; the shared + StateCache makes every shard's storage layer observe the update + regardless of which shard ran the apply. Coherence with disk + is maintained by durable write-then-cache ordering inside `writeBootstrapSidecar`, `writeRotationSidecar`, and the cutover fresh-success branch of `applyEnableStorageEnvelope`. - `NewApplier` primes both atomics from the sidecar on - construction so the storage-layer per-Put closures (wired - in 6D-6c-2) observe correct values before the FSM has - replayed a single entry after restart. Operator-inert by - itself — only consumed once main.go threads the methods - into `store.WithEncryption` and `WithStorageEnvelopeGate` - in 6D-6c-2. + `NewApplier` primes the cache from the sidecar on construction + so the storage-layer per-Put closures (wired in 6D-6c-2) + observe correct values before the FSM has replayed a single + entry after restart. `Applier.ActiveStorageKeyID` / + `Applier.StorageEnvelopeActive` remain as delegate methods for + tests and single-applier callers; multi-shard wiring in + 6D-6c-2 must read via `cache.ActiveStorageKeyID` / + `cache.StorageEnvelopeActive` directly. Operator-inert by + itself — only consumed once main.go threads the cache methods + into `store.WithEncryption` and `WithStorageEnvelopeGate` in + 6D-6c-2. ## Open milestones - **6D-6c-2** — main.go production wiring: build - `encryption.NewCipher(keystore)` and thread the - `Applier.ActiveStorageKeyID` / `Applier.StorageEnvelopeActive` - method values into `store.WithEncryption` + - `store.WithStorageEnvelopeGate` at shard-group construction. + `encryption.NewCipher(keystore)` and construct a single + `encryption.StateCache` at startup (parallel to the shared + `*Keystore`). Thread the cache via `WithStateCache` into every + per-shard `Applier` inside `buildShardGroups`, and pass + `cache.ActiveStorageKeyID` / `cache.StorageEnvelopeActive` + (NOT the per-shard `Applier` delegates) into + `store.WithEncryption` + `store.WithStorageEnvelopeGate` for + each shard's PebbleStore. Reading via the StateCache directly + ensures every shard's storage layer sees the post-apply state + regardless of which shard's leader accepted the encryption + proposal. - **6D-6c-3** — main.go CapabilityFanout closure bound to the live Raft membership view (etcd engine route snapshot + admin client DialFunc), and end-to-end integration test exercising diff --git a/internal/encryption/applier.go b/internal/encryption/applier.go index 5a62334be..b45993724 100644 --- a/internal/encryption/applier.go +++ b/internal/encryption/applier.go @@ -64,35 +64,69 @@ type WriterRegistryStore interface { // ErrKEKNotConfigured marker. Stage 6B will swap these for the // real KEK-unwrap + sidecar mutate + keystore install path. // -// The Applier carries no in-memory state of its own; all state -// lives in the supplied WriterRegistryStore. This keeps it safe -// to construct once at FSM startup and share across the lifetime -// of the process — no per-apply allocation, no locks, no leak -// path for stale state across snapshot restore. +// Apart from the shared StateCache pointer (see below), the +// Applier carries no in-memory state of its own; durable state +// lives in the supplied WriterRegistryStore and the on-disk +// sidecar. The StateCache mirrors a small subset of sidecar +// fields the storage hot path consults on every Put — kept +// coherent by durable write-then-cache ordering inside each +// apply path. type Applier struct { registry WriterRegistryStore kek KEKUnwrapper keystore *Keystore sidecarPath string now func() time.Time - // activeStorageDEKID is the in-memory mirror of - // sidecar.Active.Storage. The storage layer's encryption - // path (store.WithEncryption) calls a per-Put closure that - // reads this field via ActiveStorageKeyID(); a sidecar - // ReadSidecar() on every Put would serialise hot-path writes - // through a JSON parse + fsync barrier. Apply paths update - // this atomic AFTER the durable WriteSidecar succeeds, so - // a crash leaves disk-truth and cache aligned: on restart - // NewApplier primes the cache from disk before the storage - // layer ever queries it. Zero means "not bootstrapped" — the - // closure surfaces (0, false) and the storage layer writes - // cleartext. + // stateCache is the process-shared mirror of the sidecar fields + // the storage hot path consults on every Put. See StateCache for + // the full contract; in short, a single instance is owned by + // main.go (parallel to the shared *Keystore) and threaded into + // every per-shard Applier via WithStateCache so that an apply + // landing on shard A's FSM is immediately visible to shard B's + // storage layer. Multi-group encryption applies always land on + // exactly one shard's FSM (the one whose engine accepted the + // proposal), so a per-Applier cache would leave the remaining + // shards stuck with pre-apply atomic values. + // + // Never nil after NewApplier: when WithStateCache is omitted the + // constructor installs a private instance so single-applier + // callers and tests keep working unchanged. + stateCache *StateCache +} + +// StateCache mirrors the sidecar fields the storage hot path needs +// to consult on every Put. Two requirements drive its existence: +// +// 1. ReadSidecar-on-every-Put would serialise the hot path through +// a JSON parse + fsync barrier. atomic.Uint32 / atomic.Bool give +// a wait-free single-load read instead. +// +// 2. In a multi-group deployment, encryption FSM entries apply on +// whichever shard's leader accepted the proposal — not on every +// shard. The per-shard storage layers must still observe the +// updated state, so the cache MUST be a process-shared singleton +// rather than a per-Applier field. main.go constructs one +// StateCache at startup (parallel to the shared *Keystore) and +// threads it into every per-shard Applier via WithStateCache. +// +// Coherence with disk is maintained by **durable write-then-cache** +// ordering: NewApplier primes the cache from ReadSidecar, and every +// apply path calls RefreshFromSidecar AFTER WriteSidecar succeeds. +// A crash between fsync and atomic store is benign because the next +// process start re-primes from disk. +// +// Zero values match the pre-bootstrap posture (no active storage +// DEK, envelope gate off) so a freshly-constructed StateCache is +// safe to use before any apply or prime has run. +type StateCache struct { + // activeStorageDEKID mirrors sidecar.Active.Storage. Zero means + // "not bootstrapped"; readers surface (0, false) and the storage + // layer writes cleartext. activeStorageDEKID atomic.Uint32 // storageEnvelopeActive mirrors sidecar.StorageEnvelopeActive - // for the §6.2 cutover gate. Same atomic-vs-sidecar-read - // rationale as activeStorageDEKID above. Lifecycle: - // - false at construction (or primed from disk if a - // previous cutover already fired). + // for the §6.2 cutover gate. Lifecycle: + // - false at construction (or primed from disk if a previous + // cutover already fired). // - flipped to true exactly once by applyEnableStorageEnvelope // on a fresh-success apply, AFTER WriteSidecar succeeds. // - never flipped back to false (the cutover is one-way per @@ -101,6 +135,58 @@ type Applier struct { storageEnvelopeActive atomic.Bool } +// NewStateCache returns a zero-initialised StateCache. The +// pre-bootstrap posture (Active.Storage=0, StorageEnvelopeActive=false) +// is the correct initial state; RefreshFromSidecar advances it to the +// current sidecar values when one is supplied. +func NewStateCache() *StateCache { return &StateCache{} } + +// RefreshFromSidecar copies the relevant fields out of sc into the +// atomic mirrors. Safe to call concurrently with reads; safe to +// call from multiple goroutines (writers race to the same atomic +// CAS path, but the only writer in production is the FSM apply +// goroutine of the shard that accepted the encryption proposal). +// +// nil sc is a no-op: matches the pre-bootstrap posture where +// ReadSidecar returns IsNotExist. +func (c *StateCache) RefreshFromSidecar(sc *Sidecar) { + if c == nil || sc == nil { + return + } + c.activeStorageDEKID.Store(sc.Active.Storage) + c.storageEnvelopeActive.Store(sc.StorageEnvelopeActive) +} + +// ActiveStorageKeyID returns the current sidecar.Active.Storage DEK +// id. Signature matches store.ActiveStorageKeyID so main.go can pass +// `cache.ActiveStorageKeyID` directly into `store.WithEncryption(...)` +// as the per-Put activeKeyID closure. A non-zero id with ok=true +// means the cluster has run BootstrapEncryption; zero with ok=false +// means the cluster is still pre-bootstrap and the storage layer +// should write cleartext. +func (c *StateCache) ActiveStorageKeyID() (uint32, bool) { + if c == nil { + return 0, false + } + id := c.activeStorageDEKID.Load() + return id, id != 0 +} + +// StorageEnvelopeActive returns the in-memory mirror of +// sidecar.StorageEnvelopeActive. Signature matches +// store.StorageEnvelopeActive so main.go can pass +// `cache.StorageEnvelopeActive` directly into +// `store.WithStorageEnvelopeGate(...)` as the per-Put cutover gate. +// Once true, the storage layer wraps every new version in the §4.1 +// envelope; flips exactly once per cluster lifetime when the §7.1 +// Phase 1 cutover entry applies. +func (c *StateCache) StorageEnvelopeActive() bool { + if c == nil { + return false + } + return c.storageEnvelopeActive.Load() +} + // KEKUnwrapper is the abstraction the Applier uses to recover // cleartext DEK bytes from the wrapped DEK material carried in // BootstrapPayload / RotationPayload. The supplied implementation @@ -157,6 +243,21 @@ func WithNowFunc(now func() time.Time) ApplierOption { return func(a *Applier) { a.now = now } } +// WithStateCache installs a shared StateCache so that an apply +// landing on this Applier (typically the per-shard Applier whose +// FSM accepted the encryption proposal) updates atomics that every +// other Applier in the process reads. main.go owns one StateCache +// for the lifetime of the binary and threads the same pointer into +// every per-shard Applier and into the storage-layer per-Put +// closures. +// +// If WithStateCache is omitted, NewApplier installs a private +// instance — preserves the single-applier ergonomics that tests +// and pre-multi-shard callers rely on. +func WithStateCache(c *StateCache) ApplierOption { + return func(a *Applier) { a.stateCache = c } +} + // NewApplier wires an Applier against the supplied registry store // plus optional KEK / Keystore / sidecar / clock dependencies. // Returns an error if registry is nil so misconfiguration is caught @@ -187,6 +288,13 @@ func NewApplier(registry WriterRegistryStore, opts ...ApplierOption) (*Applier, if a.now == nil { return nil, errors.New("encryption: NewApplier: WithNowFunc(nil) overwrote default time.Now") } + // Install a private StateCache when WithStateCache was not + // supplied so the apply paths and accessors always have a + // non-nil target. Tests rely on this; production main.go is + // expected to thread a shared instance in. + if a.stateCache == nil { + a.stateCache = NewStateCache() + } // Prime the in-memory accessors from the on-disk sidecar // (best-effort: a missing sidecar is the pre-bootstrap // posture and leaves both atomics at their zero values, @@ -200,7 +308,7 @@ func NewApplier(registry WriterRegistryStore, opts ...ApplierOption) (*Applier, if a.sidecarPath != "" { switch sc, err := ReadSidecar(a.sidecarPath); { case err == nil: - a.refreshActiveStateCache(sc) + a.stateCache.RefreshFromSidecar(sc) case IsNotExist(err): // Pre-bootstrap; leave atomics at zero. default: @@ -210,42 +318,27 @@ func NewApplier(registry WriterRegistryStore, opts ...ApplierOption) (*Applier, return a, nil } -// ActiveStorageKeyID returns the current sidecar.Active.Storage -// DEK id in-memory. Signature matches store.ActiveStorageKeyID -// so main.go can pass `applier.ActiveStorageKeyID` directly into -// `store.WithEncryption(...)` as the per-Put activeKeyID -// closure. A non-zero id with ok=true means the cluster has run -// BootstrapEncryption; zero with ok=false means the cluster is -// still pre-bootstrap and the storage layer should write -// cleartext. +// StateCache returns the shared cache this Applier writes to on +// every apply path. main.go wires one StateCache across all +// per-shard Appliers via WithStateCache, but for callers that +// constructed an Applier without supplying one this accessor +// returns the privately-installed instance so tests can still +// reach the atomics directly. +func (a *Applier) StateCache() *StateCache { return a.stateCache } + +// ActiveStorageKeyID delegates to the shared StateCache. Convenience +// for tests and single-applier callers; multi-shard wiring should +// prefer reading StateCache().ActiveStorageKeyID directly so the +// closure target is independent of which shard's Applier received +// the encryption apply. func (a *Applier) ActiveStorageKeyID() (uint32, bool) { - id := a.activeStorageDEKID.Load() - return id, id != 0 + return a.stateCache.ActiveStorageKeyID() } -// StorageEnvelopeActive returns the in-memory mirror of -// sidecar.StorageEnvelopeActive. Signature matches -// store.StorageEnvelopeActive so main.go can pass -// `applier.StorageEnvelopeActive` directly into -// `store.WithStorageEnvelopeGate(...)` as the per-Put cutover -// gate (Stage 6D-5). Once true, the storage layer wraps every -// new version in the §4.1 envelope; flips exactly once per -// cluster lifetime when the §7.1 Phase 1 cutover entry applies. +// StorageEnvelopeActive delegates to the shared StateCache. Same +// rationale as ActiveStorageKeyID above. func (a *Applier) StorageEnvelopeActive() bool { - return a.storageEnvelopeActive.Load() -} - -// refreshActiveStateCache copies the relevant sidecar fields -// into the in-memory atomics. Called from NewApplier (prime -// from disk on startup) and from every apply path AFTER a -// successful WriteSidecar (durable write-then-cache ordering -// so a crash leaves disk-truth and the next restart's prime -// in agreement). Holding the mutate-after-WriteSidecar order -// also avoids the storage layer reading a flipped cache while -// the sidecar disk write is still mid-fsync. -func (a *Applier) refreshActiveStateCache(sc *Sidecar) { - a.activeStorageDEKID.Store(sc.Active.Storage) - a.storageEnvelopeActive.Store(sc.StorageEnvelopeActive) + return a.stateCache.StorageEnvelopeActive() } // bootstrapAndRotationConfigured reports whether WithKEK, @@ -596,7 +689,7 @@ func (a *Applier) writeBootstrapSidecar(raftIdx uint64, p fsmwire.BootstrapPaylo if err := WriteSidecar(a.sidecarPath, sc); err != nil { return errors.Wrap(err, "applier: write sidecar for bootstrap") } - a.refreshActiveStateCache(sc) + a.stateCache.RefreshFromSidecar(sc) return nil } @@ -857,7 +950,7 @@ func (a *Applier) applyEnableStorageEnvelope(raftIdx uint64, p fsmwire.RotationP if err := WriteSidecar(a.sidecarPath, sc); err != nil { return errors.Wrap(err, "applier: write sidecar for cutover") } - a.refreshActiveStateCache(sc) + a.stateCache.RefreshFromSidecar(sc) return nil } @@ -894,7 +987,7 @@ func (a *Applier) writeRotationSidecar(raftIdx uint64, p fsmwire.RotationPayload if err := WriteSidecar(a.sidecarPath, sc); err != nil { return errors.Wrap(err, "applier: write sidecar for rotation") } - a.refreshActiveStateCache(sc) + a.stateCache.RefreshFromSidecar(sc) return nil } diff --git a/internal/encryption/applier_test.go b/internal/encryption/applier_test.go index 6bc58f9e0..ede26b814 100644 --- a/internal/encryption/applier_test.go +++ b/internal/encryption/applier_test.go @@ -1745,6 +1745,134 @@ func TestApplier_StorageAccessors_PrimedFromExistingSidecar(t *testing.T) { } } +// TestApplier_StorageAccessors_SharedCache_AcrossAppliers pins the +// multi-shard correctness invariant: in a multi-group deployment, +// main.go constructs one Applier per shard but encryption FSM +// entries apply on exactly one shard (the one whose engine accepted +// the proposal). The remaining per-shard Appliers must still +// observe the post-apply state, because their per-Put storage +// closures gate writes on the SAME cluster-wide encryption state. +// +// Without a shared StateCache (the original 6D-6c-1 design), each +// Applier carried its own atomics and only the apply-receiving +// shard would surface (id, true) — every other shard's storage +// layer would see (0, false) and silently write cleartext after +// bootstrap, or skip the §4.1 envelope wrap after cutover. This +// test fails on the original per-Applier-atomics design and passes +// once the shared StateCache lands. +// +// Reported as a P1 by chatgpt-codex-connector on PR #821 +// (review comment r3294696832). +func TestApplier_StorageAccessors_SharedCache_AcrossAppliers(t *testing.T) { + t.Parallel() + dir := t.TempDir() + sidecarPath := dir + "/keys.json" + shared := encryption.NewStateCache() + appliers := buildSharedCacheAppliers(t, sidecarPath, shared, 2) + + // Apply Bootstrap on appliers[0] only — models the FSM apply + // landing on shard 0's leader. Shard 1's applier never runs the + // apply path, but its accessors MUST surface the update because + // they read from the same StateCache. + if err := appliers[0].ApplyBootstrap(100, fsmwire.BootstrapPayload{ + StorageDEKID: 17, WrappedStorage: []byte("s"), + RaftDEKID: 18, WrappedRaft: []byte("r"), + BatchRegistry: []fsmwire.RegistrationPayload{ + {DEKID: 17, FullNodeID: 0xAAAA, LocalEpoch: 0}, + }, + }); err != nil { + t.Fatalf("appliers[0].ApplyBootstrap: %v", err) + } + assertAllAppliersAgree(t, appliers, 17, false) + + // Now flip the cutover via appliers[1] (models the cutover entry + // landing on the OTHER shard than the bootstrap). appliers[0] + // must still see the gate flip via the shared cache. + if err := appliers[1].ApplyRotation(500, fsmwire.RotationPayload{ + SubTag: fsmwire.RotateSubEnableStorageEnvelope, + DEKID: 17, + Purpose: fsmwire.PurposeStorage, + Wrapped: []byte{}, + ProposerRegistration: fsmwire.RegistrationPayload{DEKID: 17, FullNodeID: 0xAAAA, LocalEpoch: 1}, + }); err != nil { + t.Fatalf("appliers[1].ApplyRotation cutover: %v", err) + } + assertAllAppliersAgree(t, appliers, 17, true) + + // Reading via the StateCache directly (the API main.go will use + // for the per-Put storage closures in 6D-6c-2) must reflect the + // same state regardless of which Applier ran the apply. + if id, ok := shared.ActiveStorageKeyID(); !ok || id != 17 { + t.Errorf("shared.ActiveStorageKeyID = (%d, %v), want (17, true)", id, ok) + } + if !shared.StorageEnvelopeActive() { + t.Error("shared.StorageEnvelopeActive = false, want true after cutover") + } +} + +// buildSharedCacheAppliers wires N appliers to the same sidecar +// path and the same shared StateCache, modelling main.go's +// buildShardGroups per-shard Applier construction. +func buildSharedCacheAppliers( + t *testing.T, + sidecarPath string, + shared *encryption.StateCache, + n int, +) []*encryption.Applier { + t.Helper() + appliers := make([]*encryption.Applier, n) + for i := range appliers { + a, err := encryption.NewApplier(newMapRegistryStore(), + encryption.WithKEK(&fakeKEK{}), + encryption.WithKeystore(encryption.NewKeystore()), + encryption.WithSidecarPath(sidecarPath), + encryption.WithStateCache(shared), + ) + if err != nil { + t.Fatalf("NewApplier[%d]: %v", i, err) + } + appliers[i] = a + } + return appliers +} + +// assertAllAppliersAgree verifies every applier surfaces the same +// (ActiveStorageKeyID, StorageEnvelopeActive) state — the shared- +// cache invariant for the cross-shard apply propagation. +func assertAllAppliersAgree( + t *testing.T, + appliers []*encryption.Applier, + wantID uint32, + wantEnvelopeActive bool, +) { + t.Helper() + for i, a := range appliers { + id, ok := a.ActiveStorageKeyID() + if !ok || id != wantID { + t.Errorf("appliers[%d].ActiveStorageKeyID = (%d, %v), want (%d, true)", i, id, ok, wantID) + } + if got := a.StorageEnvelopeActive(); got != wantEnvelopeActive { + t.Errorf("appliers[%d].StorageEnvelopeActive = %v, want %v", i, got, wantEnvelopeActive) + } + } +} + +// TestStateCache_NilSafe pins the nil-receiver contract so callers +// that have not yet wired a StateCache (early startup, partial +// option configuration) get the pre-bootstrap posture rather than +// a nil-deref panic. +func TestStateCache_NilSafe(t *testing.T) { + t.Parallel() + var c *encryption.StateCache + if id, ok := c.ActiveStorageKeyID(); ok || id != 0 { + t.Errorf("nil StateCache.ActiveStorageKeyID = (%d, %v), want (0, false)", id, ok) + } + if c.StorageEnvelopeActive() { + t.Error("nil StateCache.StorageEnvelopeActive = true, want false") + } + c.RefreshFromSidecar(&encryption.Sidecar{}) // must not panic +} + // TestApplier_StorageAccessors_ConcurrentReads exercises the // atomic.Uint32 / atomic.Bool seam under -race. The accessors are // called on the hot storage Put path, so a torn read or a missed