Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 21 additions & 1 deletion adapter/encryption_admin.go
Original file line number Diff line number Diff line change
Expand Up @@ -1341,7 +1341,27 @@ func (s *EncryptionAdminServer) proposeEncryptionEntry(ctx context.Context, opco
entry := make([]byte, 0, 1+len(body))
entry = append(entry, opcode)
entry = append(entry, body...)
res, err := s.proposer.Propose(ctx, entry)
// proposeEncryptionEntry composes control-plane entries
// (BootstrapEncryption, EnableStorageEnvelope, the §6E
// EnableRaftEnvelope cutover marker, RotateDEK,
// RegisterEncryptionWriter, etc.). The §7.1 quiescence barrier
// Stage 6E-2d installs on Propose would reject these — most
// trivially, the cutover entry itself would deadlock on its
// own barrier — so admin ops route through the
// barrier-exempt ProposeAdmin path.
//
// ProposeAdmin is barrier-exempt only; the wrap layer above
// the engine (kv.wrappedProposer, when configured) still
// applies its wrap closure to ProposeAdmin payloads, so a
// post-cutover RotateDEK or RegisterEncryptionWriter
// committed at `index > raftEnvelopeCutoverIndex` carries the
// AEAD envelope the §6.3 strict-`>` apply hook expects. The
// EnableRaftEnvelope cutover marker (at `index == cutover`)
// must remain cleartext for strict-`>` to leave it alone;
// today s.proposer is wired to the raw engine (see
// main_encryption_admin.go), so the marker reaches Raft
// without the wrap layer in the path at all.
res, err := s.proposer.ProposeAdmin(ctx, entry)

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Do not send post-cutover admin entries in cleartext

This helper is used by BootstrapEncryption, RotateDEK, and RegisterEncryptionWriter, not just the EnableRaftEnvelope cutover marker. Once the raft-envelope cutover is active, applyNormalEntry unwraps every normal entry with index > raftEnvelopeCutoverIndex before FSM dispatch, while production wiring passes the raw engine as the admin proposer (main_encryption_admin.go:140). Calling ProposeAdmin here therefore makes later admin mutations land as cleartext opcode payloads after cutover, causing raft unwrap failure / halt-apply instead of reaching applyEncryption; only entries guaranteed to land at or before the cutover should bypass wrapping.

Useful? React with 👍 / 👎.

if err != nil {
return 0, proposeErrorToStatus(err, opcode)
}
Expand Down
22 changes: 22 additions & 0 deletions adapter/encryption_admin_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1000,6 +1000,16 @@ func (p *recordingProposer) Propose(_ context.Context, data []byte) (*raftengine
return &raftengine.ProposalResult{CommitIndex: p.commitIndex}, nil
}

// ProposeAdmin records the call into the same slice as Propose so
// existing assertions on p.calls continue to work after the
// adapter/encryption_admin.go production callers (which now route
// every control-plane entry through ProposeAdmin) switched paths.
// In the current build both methods are operationally identical;
// the bookkeeping is unified intentionally.
func (p *recordingProposer) ProposeAdmin(ctx context.Context, data []byte) (*raftengine.ProposalResult, error) {
return p.Propose(ctx, data)
}

// stubLeaderView reports a fixed leadership state. verifyErr lets
// a test simulate a partitioned former leader: State() still
// reports StateLeader but VerifyLeader returns an error because
Expand Down Expand Up @@ -1089,6 +1099,18 @@ func (p *applyingProposer) Propose(ctx context.Context, data []byte) (*raftengin
return res, nil
}

// ProposeAdmin redirects to applyingProposer.Propose (not the
// embedded recordingProposer.Propose) so the applyFn side-effect
// fires for admin-path proposals as well. Without this explicit
// override, Go method resolution would dispatch
// recordingProposer.ProposeAdmin -> recordingProposer.Propose and
// skip the applyingProposer-level Propose body — silently losing
// the apply emulation that EnableStorageEnvelope /
// EnableRaftEnvelope happy-path tests depend on.
func (p *applyingProposer) ProposeAdmin(ctx context.Context, data []byte) (*raftengine.ProposalResult, error) {
return p.Propose(ctx, data)
}

// applyCutover is the §6.4 fresh-success apply effect: flip
// StorageEnvelopeActive to true and stamp the cutover index with
// the apply's Raft index. Used by the EnableStorageEnvelope happy-
Expand Down
8 changes: 8 additions & 0 deletions internal/raftadmin/server_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,10 @@ func (f *fakeEngine) Propose(context.Context, []byte) (*raftengine.ProposalResul
return &raftengine.ProposalResult{}, nil
}

func (f *fakeEngine) ProposeAdmin(ctx context.Context, data []byte) (*raftengine.ProposalResult, error) {
return f.Propose(ctx, data)
}

func (f *fakeEngine) State() raftengine.State {
f.mu.Lock()
defer f.mu.Unlock()
Expand Down Expand Up @@ -312,6 +316,10 @@ func (s stateOnlyEngine) Propose(context.Context, []byte) (*raftengine.ProposalR
return &raftengine.ProposalResult{}, nil
}

func (s stateOnlyEngine) ProposeAdmin(ctx context.Context, data []byte) (*raftengine.ProposalResult, error) {
return s.Propose(ctx, data)
}

func (s stateOnlyEngine) State() raftengine.State { return s.state }

func (s stateOnlyEngine) Leader() raftengine.LeaderInfo { return raftengine.LeaderInfo{} }
Expand Down
37 changes: 37 additions & 0 deletions internal/raftengine/engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,45 @@ type ProposalResult struct {
Response any
}

// Proposer drives a Raft proposal through the engine and returns
// once it has been committed (or the context/engine cancels first).
//
// Two semantically distinct entry points, differing ONLY in the
// §7.1 quiescence-barrier check Stage 6E-2d installs on Propose:
//
// - Propose carries ordinary user-data and control-plane traffic
// that may be paused during a raft-envelope cutover. 6E-2d will
// reject these with ErrEnvelopeCutoverInProgress while the
// barrier is open so the leader cannot admit a fresh entry at
// `index > raftEnvelopeCutoverIndex` mid-installation.
// - ProposeAdmin carries proposals that MUST remain admissible
// across the barrier — the EnableRaftEnvelope cutover entry
// itself (without this exemption the barrier would deadlock on
// its own cutover proposal) and ConfChange-time
// RegisterEncryptionWriter proposals (Stage 7c §3.1, so a new
// member joining mid-barrier can still register its
// writer-registry entry).
//
// ProposeAdmin is NOT a wrap-exemption: a payload-wrap layer
// configured above the engine (kv.wrappedProposer) applies its
// wrap closure to both methods identically. Admin entries that
// land at `index > raftEnvelopeCutoverIndex` (a leader-restart
// registration, a post-cutover RotateDEK, etc.) must carry the
// AEAD envelope the §6.3 strict-`>` apply hook expects; a cleartext
// admin entry above cutover would halt the apply loop on
// unwrap-failure. The lone exception is the EnableRaftEnvelope
// cutover marker (sits at `index == cutover`, strict-`>` leaves it
// alone), which is proposed via a raw engine reference and never
// flows through the wrap layer in the first place.
//
// In the current build the two methods are operationally
// equivalent (the barrier is still 6E-2d work); the distinction at
// the call site is the migration the future barrier requires —
// sites still on Propose would fail closed the moment 6E-2d
// activates the barrier.
type Proposer interface {
Propose(ctx context.Context, data []byte) (*ProposalResult, error)
ProposeAdmin(ctx context.Context, data []byte) (*ProposalResult, error)
}

type LeaderView interface {
Expand Down
28 changes: 28 additions & 0 deletions internal/raftengine/etcd/engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -729,6 +729,34 @@ func (e *Engine) Close() error {
}

func (e *Engine) Propose(ctx context.Context, data []byte) (*raftengine.ProposalResult, error) {
return e.propose(ctx, data)
}

// ProposeAdmin drives a control-plane proposal that must remain
// admissible across the §7.1 quiescence barrier — currently the
// EnableRaftEnvelope cutover entry and ConfChange-time
// RegisterEncryptionWriter proposals (see raftengine.Proposer's
// contract for the full exempt set).
//
// The barrier exemption is the SOLE divergence from Propose: a
// higher-layer wrap (kv.wrappedProposer) applies its wrap closure
// to both methods identically, so admin entries landing above the
// raft-envelope cutover still carry the AEAD envelope the §6.3
// strict-`>` apply hook expects. Only the cutover marker itself is
// cleartext, and it bypasses the wrap layer at the call site
// (raw engine reference), not at the method level.
//
// In the current build ProposeAdmin is operationally identical to
// Propose; Stage 6E-2d adds the barrier check on Propose only.
// The two methods are kept distinct from the outset so the
// migration of call sites (this PR) lands ahead of the behaviour
// change (6E-2d) — calling Propose from an exempt site today is
// silently wrong tomorrow.
func (e *Engine) ProposeAdmin(ctx context.Context, data []byte) (*raftengine.ProposalResult, error) {
return e.propose(ctx, data)
}

func (e *Engine) propose(ctx context.Context, data []byte) (*raftengine.ProposalResult, error) {
if err := contextErr(ctx); err != nil {
return nil, err
}
Expand Down
63 changes: 63 additions & 0 deletions internal/raftengine/etcd/engine_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,69 @@ func (s *countingSnapshotStateMachine) Restore(r io.Reader) error {
return err
}

// TestProposeAdmin_EquivalentToProposeToday pins the Stage 6E-2b
// invariant: ProposeAdmin and Propose commit equivalent entries on
// the current build. Both reach the FSM as the same byte sequence
// and report a non-zero CommitIndex.
//
// This is the bridge property the migration relies on. Stage 6E-2d
// will diverge the two paths — Propose gains a §7.1 quiescence
// barrier check that rejects with ErrEnvelopeCutoverInProgress
// while a cutover is being installed; ProposeAdmin remains
// admissible so the cutover entry and ConfChange-time
// RegisterEncryptionWriter proposals can still land. Until that
// behaviour change ships, callers that switched to ProposeAdmin
// in this PR must see identical results to staying on Propose,
// else the migration would visibly change semantics before
// 6E-2d's protective barrier is in place.
func TestProposeAdmin_EquivalentToProposeToday(t *testing.T) {
fsm := &testStateMachine{}
engine, err := Open(context.Background(), OpenConfig{
NodeID: 1,
LocalID: "n1",
LocalAddress: "127.0.0.1:7011",
DataDir: t.TempDir(),
Bootstrap: true,
StateMachine: fsm,
})
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, engine.Close())
})

require.Equal(t, raftengine.StateLeader, engine.State())

plainRes, err := engine.Propose(context.Background(), []byte("plain"))
require.NoError(t, err)
require.NotNil(t, plainRes)
require.NotZero(t, plainRes.CommitIndex)
require.Equal(t, "plain", plainRes.Response)

adminRes, err := engine.ProposeAdmin(context.Background(), []byte("admin"))
require.NoError(t, err)
require.NotNil(t, adminRes)
require.NotZero(t, adminRes.CommitIndex)
require.Equal(t, "admin", adminRes.Response)
require.Greater(t, adminRes.CommitIndex, plainRes.CommitIndex,
"ProposeAdmin must advance CommitIndex past the preceding Propose, "+
"proving its proposal reaches the engine through the same Raft path")

applied := fsm.Applied()
require.Equal(t, [][]byte{[]byte("plain"), []byte("admin")}, applied,
"both Propose and ProposeAdmin must reach the FSM verbatim and in-order")
}

// Compile-time guard: *Engine must satisfy raftengine.Proposer
// (which Stage 6E-2b extended with ProposeAdmin). Declared at
// package scope so the check fires at test-package compile time
// regardless of which test is selected; wrapping it inside a
// no-op test function would defer the check to a test target
// nobody runs. Removing ProposeAdmin from the interface would
// either leave this assertion dangling (if Engine still has the
// method) or hard-fail the build (if it doesn't) — the latter is
// the failure mode this guard exists to produce.
var _ raftengine.Proposer = (*Engine)(nil)

func TestOpenSingleNodeProposeAndReadIndex(t *testing.T) {
fsm := &testStateMachine{}
engine, err := Open(context.Background(), OpenConfig{
Expand Down
3 changes: 3 additions & 0 deletions kv/coordinator_retry_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ type stubLeaderEngine struct{}
func (stubLeaderEngine) Propose(context.Context, []byte) (*raftengine.ProposalResult, error) {
return &raftengine.ProposalResult{}, nil
}
func (e stubLeaderEngine) ProposeAdmin(ctx context.Context, data []byte) (*raftengine.ProposalResult, error) {
return e.Propose(ctx, data)
}
func (stubLeaderEngine) State() raftengine.State { return raftengine.StateLeader }
func (stubLeaderEngine) Leader() raftengine.LeaderInfo {
return raftengine.LeaderInfo{ID: "self", Address: "127.0.0.1:0"}
Expand Down
6 changes: 6 additions & 0 deletions kv/leader_proxy_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ type stubFollowerEngine struct {
func (s *stubFollowerEngine) Propose(context.Context, []byte) (*raftengine.ProposalResult, error) {
return nil, raftengine.ErrNotLeader
}
func (s *stubFollowerEngine) ProposeAdmin(ctx context.Context, data []byte) (*raftengine.ProposalResult, error) {
return s.Propose(ctx, data)
}
func (s *stubFollowerEngine) State() raftengine.State { return raftengine.StateFollower }
func (s *stubFollowerEngine) Leader() raftengine.LeaderInfo {
return raftengine.LeaderInfo{ID: "leader", Address: s.leaderAddr}
Expand Down Expand Up @@ -162,6 +165,9 @@ func (e *togglingFollowerEngine) setLeader(addr string) {
func (e *togglingFollowerEngine) Propose(context.Context, []byte) (*raftengine.ProposalResult, error) {
return nil, raftengine.ErrNotLeader
}
func (e *togglingFollowerEngine) ProposeAdmin(ctx context.Context, data []byte) (*raftengine.ProposalResult, error) {
return e.Propose(ctx, data)
}
func (e *togglingFollowerEngine) State() raftengine.State { return raftengine.StateFollower }
func (e *togglingFollowerEngine) Leader() raftengine.LeaderInfo {
p := e.addr.Load()
Expand Down
6 changes: 6 additions & 0 deletions kv/lease_read_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,9 @@ func (e *fakeLeaseEngine) Configuration(context.Context) (raftengine.Configurati
func (e *fakeLeaseEngine) Propose(context.Context, []byte) (*raftengine.ProposalResult, error) {
return &raftengine.ProposalResult{}, nil
}
func (e *fakeLeaseEngine) ProposeAdmin(ctx context.Context, data []byte) (*raftengine.ProposalResult, error) {
return e.Propose(ctx, data)
}
func (e *fakeLeaseEngine) Close() error { return nil }
func (e *fakeLeaseEngine) LeaseDuration() time.Duration { return e.leaseDur }
func (e *fakeLeaseEngine) AppliedIndex() uint64 { return e.applied }
Expand Down Expand Up @@ -153,6 +156,9 @@ func (e *nonLeaseEngine) Configuration(context.Context) (raftengine.Configuratio
func (e *nonLeaseEngine) Propose(context.Context, []byte) (*raftengine.ProposalResult, error) {
return &raftengine.ProposalResult{}, nil
}
func (e *nonLeaseEngine) ProposeAdmin(ctx context.Context, data []byte) (*raftengine.ProposalResult, error) {
return e.Propose(ctx, data)
}
func (e *nonLeaseEngine) Close() error { return nil }

// setQuorumAck is a test helper that drives the engine-driven lease
Expand Down
33 changes: 33 additions & 0 deletions kv/raft_payload_wrapper.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,3 +75,36 @@ func (p *wrappedProposer) Propose(ctx context.Context, data []byte) (*raftengine
}
return res, nil
}

// ProposeAdmin applies the configured RaftPayloadWrapper before
// forwarding the payload to the inner ProposeAdmin path. The wrap
// layer is NOT a barrier-exemption concern: it exists so that
// every entry landing at `index > raftEnvelopeCutoverIndex` carries
// an AEAD envelope the §6.3 strict-`>` apply hook can unwrap.
// Admin entries (BootstrapEncryption, RotateDEK,
// RegisterEncryptionWriter, etc.) committed after the cutover are
// no different from user data in that regard — a cleartext admin
// entry above cutover would halt the apply loop on unwrap-failure,
// not be silently passed through.
//
// The lone exception is the EnableRaftEnvelope cutover marker
// itself, which sits exactly at `index == cutover` and must remain
// cleartext for strict-`>` dispatch to leave it alone. That marker
// is proposed via a raw engine reference (adapter/encryption_admin.go
// holds the engine directly as s.proposer), not via the
// wrappedProposer — so this wrap path is never on its way.
//
// ProposeAdmin's only divergence from Propose lives in Stage
// 6E-2d's §7.1 quiescence-barrier check, which is installed on
// Propose only.
func (p *wrappedProposer) ProposeAdmin(ctx context.Context, data []byte) (*raftengine.ProposalResult, error) {
wrapped, err := applyRaftPayloadWrap(p.wrap, data)
if err != nil {
return nil, err
}
res, err := p.inner.ProposeAdmin(ctx, wrapped)
if err != nil {
return nil, errors.Wrap(err, "kv: wrapped propose-admin")
}
return res, nil
}
Loading
Loading