diff --git a/cmd/elastickv-list-routes/main.go b/cmd/elastickv-list-routes/main.go new file mode 100644 index 00000000..3734b3e3 --- /dev/null +++ b/cmd/elastickv-list-routes/main.go @@ -0,0 +1,128 @@ +// elastickv-list-routes prints the cluster's current route catalog as +// JSON. Used by the Composed-1 M5a Jepsen setup-hook verification to +// assert the launch script's --shardRanges actually placed the M5 +// table-route keys on the expected Raft groups before any workload +// op runs. +// +// Per the design doc (docs/design/2026_06_02_proposed_composed1_m5_jepsen_route_shuffle.md +// §3.3), the Jepsen client's setup! shells out to this tool rather +// than re-implementing the gRPC client in Clojure: a JSON contract is +// stable across versions and a future ListRoutes schema change shows +// up as an unmarshal failure rather than as a silent mis-routing +// during a Jepsen run. +// +// Usage: +// +// elastickv-list-routes --address 127.0.0.1:50051 +// +// Output (stdout, one JSON object): +// +// { +// "catalog_version": 7, +// "routes": [ +// {"route_id": 100, "raft_group_id": 1, "start": "...", "end": "...", "state": "ROUTE_STATE_ACTIVE"}, +// ... +// ] +// } +// +// `start` and `end` are base64-encoded raw bytes so any byte +// sequence (including unprintables in the routing keyspace) survives +// the JSON round-trip without quoting issues. Non-zero exit on any +// error so the Jepsen setup-hook sees the failure verbatim. +package main + +import ( + "context" + "encoding/base64" + "encoding/json" + "flag" + "fmt" + "io" + "os" + "time" + + pb "github.com/bootjp/elastickv/proto" + "github.com/cockroachdb/errors" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +const rpcTimeout = 10 * time.Second + +var address = flag.String("address", "127.0.0.1:50051", "gRPC address of an elastickv server (any node works — the Distribution service is replicated)") + +// routeJSON is the on-the-wire shape this tool emits. Field names +// are snake_case to match the proto's wire format; Start/End are +// base64-encoded so arbitrary byte ranges survive JSON. +type routeJSON struct { + RouteID uint64 `json:"route_id"` + RaftGroupID uint64 `json:"raft_group_id"` + Start string `json:"start"` // base64(raw bytes) + End string `json:"end"` // base64(raw bytes); empty == +infinity + State string `json:"state"` +} + +type responseJSON struct { + CatalogVersion uint64 `json:"catalog_version"` + Routes []routeJSON `json:"routes"` +} + +func main() { + flag.Parse() + if err := run(); err != nil { + fmt.Fprintf(os.Stderr, "elastickv-list-routes: %v\n", err) + os.Exit(1) + } +} + +func run() error { + conn, err := grpc.NewClient(*address, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return errors.Wrapf(err, "dial %s", *address) + } + defer func() { + // Mirror cmd/elastickv-split's close pattern: surface close + // errors on stderr without failing the process — by this + // point the response has been printed. + if cerr := conn.Close(); cerr != nil { + fmt.Fprintf(os.Stderr, "elastickv-list-routes: close: %v\n", cerr) + } + }() + + client := pb.NewDistributionClient(conn) + ctx, cancel := context.WithTimeout(context.Background(), rpcTimeout) + defer cancel() + + resp, err := client.ListRoutes(ctx, &pb.ListRoutesRequest{}) + if err != nil { + return errors.Wrap(err, "ListRoutes") + } + return emit(resp, os.Stdout) +} + +// emit serialises resp as JSON to w. Extracted from run() so the +// encoding is testable with bytes.Buffer rather than a real +// temp file (gemini medium on PR #925). io.Writer is the +// idiomatic Go return shape for "I write structured output to +// something." +func emit(resp *pb.ListRoutesResponse, w io.Writer) error { + out := responseJSON{ + CatalogVersion: resp.GetCatalogVersion(), + Routes: make([]routeJSON, 0, len(resp.GetRoutes())), + } + for _, r := range resp.GetRoutes() { + out.Routes = append(out.Routes, routeJSON{ + RouteID: r.GetRouteId(), + RaftGroupID: r.GetRaftGroupId(), + Start: base64.StdEncoding.EncodeToString(r.GetStart()), + End: base64.StdEncoding.EncodeToString(r.GetEnd()), + State: r.GetState().String(), + }) + } + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + if err := enc.Encode(out); err != nil { + return errors.Wrap(err, "encode") + } + return nil +} diff --git a/cmd/elastickv-list-routes/main_test.go b/cmd/elastickv-list-routes/main_test.go new file mode 100644 index 00000000..616c9494 --- /dev/null +++ b/cmd/elastickv-list-routes/main_test.go @@ -0,0 +1,92 @@ +package main + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "testing" + + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" +) + +// TestEmit_Empty checks the JSON shape when no routes exist. +// catalog_version is still emitted, routes is an empty array +// (not nil) so Clojure callers don't have to special-case +// nil-vs-empty. +func TestEmit_Empty(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, emit(&pb.ListRoutesResponse{CatalogVersion: 7}, &buf)) + + var out struct { + CatalogVersion uint64 `json:"catalog_version"` + Routes []any `json:"routes"` + } + require.NoError(t, json.Unmarshal(buf.Bytes(), &out)) + require.Equal(t, uint64(7), out.CatalogVersion) + require.NotNil(t, out.Routes) + require.Empty(t, out.Routes) +} + +// TestEmit_RoundTripsRouteBytes pins the on-the-wire shape against +// the JSON Clojure will parse. Start/End are base64-encoded so any +// byte sequence survives — verified by round-tripping a routing-key +// shape that contains '|' (ASCII 124, outside the base64 alphabet). +func TestEmit_RoundTripsRouteBytes(t *testing.T) { + startBytes := []byte("!ddb|route|table|amVwc2VuX2FwcGVuZF90MQ") + endBytes := []byte("!ddb|route|table|amVwc2VuX2FwcGVuZF90Mw") + + var buf bytes.Buffer + require.NoError(t, emit(&pb.ListRoutesResponse{ + CatalogVersion: 3, + Routes: []*pb.RouteDescriptor{ + { + RouteId: 100, + RaftGroupId: 1, + Start: startBytes, + End: endBytes, + State: pb.RouteState_ROUTE_STATE_ACTIVE, + }, + }, + }, &buf)) + + var out responseJSON + require.NoError(t, json.Unmarshal(buf.Bytes(), &out)) + + require.Equal(t, uint64(3), out.CatalogVersion) + require.Len(t, out.Routes, 1) + require.Equal(t, uint64(100), out.Routes[0].RouteID) + require.Equal(t, uint64(1), out.Routes[0].RaftGroupID) + + // Round-trip the base64-encoded bytes — the load-bearing claim is + // that the Clojure caller decodes back to the exact bytes the + // server holds. + decodedStart, err := base64.StdEncoding.DecodeString(out.Routes[0].Start) + require.NoError(t, err) + require.Equal(t, startBytes, decodedStart) + decodedEnd, err := base64.StdEncoding.DecodeString(out.Routes[0].End) + require.NoError(t, err) + require.Equal(t, endBytes, decodedEnd) + + // State serialises via proto's String() — verify the on-the-wire + // shape so a future enum-name change (e.g. stripped prefix to + // "ACTIVE") is caught here rather than silently parsed by the + // Clojure regex (claude[bot] low on PR #925). + require.Equal(t, "ROUTE_STATE_ACTIVE", out.Routes[0].State) +} + +// TestEmit_EmptyEndDistinguishable verifies that an unset End +// (the +infinity sentinel) round-trips as an empty string rather +// than as a missing field — the Clojure setup-hook relies on this +// to detect the rightmost route in the catalog. +func TestEmit_EmptyEndDistinguishable(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, emit(&pb.ListRoutesResponse{ + Routes: []*pb.RouteDescriptor{ + {RouteId: 1, RaftGroupId: 2, Start: []byte("x"), End: nil}, + }, + }, &buf)) + + require.Contains(t, buf.String(), `"end": ""`, + "empty End must serialise as an explicit empty string so Clojure can detect the +infinity boundary") +} diff --git a/jepsen/src/elastickv/dynamodb_multi_table_workload.clj b/jepsen/src/elastickv/dynamodb_multi_table_workload.clj index b34af6ca..c277f8b4 100644 --- a/jepsen/src/elastickv/dynamodb_multi_table_workload.clj +++ b/jepsen/src/elastickv/dynamodb_multi_table_workload.clj @@ -32,7 +32,8 @@ maintained API and obscure the side-by-side comparability that is the point of running both." (:gen-class) - (:require [clojure.string :as str] + (:require [clojure.java.shell :as shell] + [clojure.string :as str] [cognitect.aws.client.api :as aws] [cognitect.aws.credentials :as creds] [elastickv.cli :as cli] @@ -279,6 +280,90 @@ (ddb-invoke! ddb :TransactWriteItems {:TransactItems (into checks updates)}))) +;; --------------------------------------------------------------------------- +;; M5a setup-hook verification (design doc §3.3) +;; --------------------------------------------------------------------------- + +(def ^:private default-list-routes-bin + "Default path to the cmd/elastickv-list-routes Go helper. Tunable + via (:list-routes-bin opts) so the binary can sit anywhere on disk + when the launch script doesn't put it on PATH." + "elastickv-list-routes") + +(def ^:private default-grpc-port + "Default elastickv server gRPC port. Combined with the test's + first node hostname (or 127.0.0.1 when --local) to form the + --address argument." + 50051) + +(defn- default-grpc-host-port-for + "Returns the default --address for elastickv-list-routes when the + test does NOT pass an explicit :grpc-host-port. Resolves to the + first node's hostname + default port — works in both local mode + (first node is typically 127.0.0.1 or 'n1') and distributed Jepsen + runs where database nodes live on separate hosts (gemini medium + on PR #925). Falls back to 127.0.0.1:50051 only when :nodes is + missing entirely." + [test] + (let [node (first (:nodes test))] + (if node + (str (name node) ":" default-grpc-port) + (str "127.0.0.1:" default-grpc-port)))) + +(defn- distinct-group-ids + "Parses elastickv-list-routes' JSON output and returns the set of + distinct raft_group_id values present. Uses a regex rather than + pulling in a JSON dependency: the CLI emits a stable JSON shape + tested in cmd/elastickv-list-routes/main_test.go, and this hook + only needs a coarse-grained 'how many groups own routes' check. + If a future ListRoutes change introduces a different field name, + the regex returns the empty set and the assertion below fails + loudly — strictly better than silently passing on an unexpected + shape." + [json-str] + (->> (re-seq #"\"raft_group_id\"\s*:\s*(\d+)" json-str) + (map (comp #(Long/parseLong %) second)) + set)) + +(defn- verify-multi-group-routing! + "Asserts the cluster reports >=2 distinct Raft groups in its route + catalog. Shells out to cmd/elastickv-list-routes (the JSON + contract is stable; design doc §3.3). Throws ex-info on any + failure so the Jepsen setup-hook fails fast with a clear error + pointing the operator at the launch-script flag the cluster is + missing. + + opts (read from the test map): + :list-routes-bin — absolute path to the CLI (default \"elastickv-list-routes\"; + assumes PATH or matching launch-script PWD). + :grpc-host-port — --address arg to the CLI; defaults to the + first node's hostname + 50051 so distributed + Jepsen runs work without flag plumbing + (gemini medium on PR #925)." + [test] + (let [bin (or (:list-routes-bin test) default-list-routes-bin) + addr (or (:grpc-host-port test) (default-grpc-host-port-for test)) + result (shell/sh bin "--address" addr)] + (when-not (zero? (:exit result)) + (throw (ex-info (str bin " --address " addr " failed: " (:err result)) + {:exit (:exit result) + :stdout (:out result) + :stderr (:err result)}))) + (let [groups (distinct-group-ids (:out result))] + (when (< (count groups) 2) + (throw (ex-info + (str "M5a multi-group routing precondition failed: only " + (count groups) " distinct Raft group(s) observed in the catalog " + "(expected >=2). Re-launch the cluster with both --raftGroups " + "AND --shardRanges (see scripts/run-jepsen-m5-local.sh) — " + "without both flags --shardRanges collapses every range into " + "the default group 1 and dispatchMultiShardTxn never fires.") + {:groups groups + :bin bin + :address addr + :raw-out (:out result)}))) + groups))) + ;; --------------------------------------------------------------------------- ;; Jepsen client ;; --------------------------------------------------------------------------- @@ -291,7 +376,19 @@ host (or (:dynamo-host test) (name node))] (assoc this :ddb (make-ddb-client host port)))) - (setup! [this _test] + (setup! [_this test] + ;; M5a setup-hook verification per design doc §3.3. Asserts the + ;; launch script's --raftGroups / --shardRanges actually placed + ;; the M5 table-route keys on >=2 distinct Raft groups before any + ;; workload op runs. Fails fast with a clear error so the operator + ;; knows the cluster needs to be relaunched — strictly better than + ;; silently running the workload on a single-shard layout and + ;; reporting "zero G1c" without ever exercising dispatchMultiShardTxn. + ;; + ;; jepsen.client/setup! is invoked exactly once per test (not + ;; per-node like jepsen.db/setup!), so no first-node gating is + ;; required. + (verify-multi-group-routing! test) ;; No DescribeTable poll loop is needed: elastickv's adapter ;; returns TableStatus=ACTIVE synchronously in the CreateTable ;; response (adapter/dynamodb.go:779-783), so the table is @@ -442,6 +539,15 @@ {:name (or (:name opts) "elastickv-dynamodb-append-multi-table") :nodes nodes :db db + ;; Setup-hook verification keys — read by + ;; verify-multi-group-routing! at workload setup! time. + ;; Threaded into the test map (not the workload map) + ;; because jepsen.client/setup! receives the test, not + ;; opts; the M5a launch script passes these via the + ;; --list-routes-bin and --grpc-host-port CLI flags + ;; defined in dynamo-cli-opts below. + :list-routes-bin (:list-routes-bin opts) + :grpc-host-port (:grpc-host-port opts) :dynamo-host (:dynamo-host opts) :os (if local? os/noop debian/os) :net (if local? net/noop net/iptables) @@ -481,7 +587,11 @@ :parse-fn #(Integer/parseInt %)] [nil "--max-txn-length N" "Maximum number of micro-ops per transaction." :default nil - :parse-fn #(Integer/parseInt %)]]) + :parse-fn #(Integer/parseInt %)] + [nil "--list-routes-bin PATH" "Path to the cmd/elastickv-list-routes Go helper used by the workload's setup-hook verification. Defaults to bare-name 'elastickv-list-routes' (assumes PATH lookup); pass an absolute path when invoking from a launch script that builds the binary into a tmp dir." + :default nil] + [nil "--grpc-host-port HOST:PORT" "gRPC --address argument passed to elastickv-list-routes by the setup-hook verification. Default 127.0.0.1:50051 matches scripts/run-jepsen-m5-local.sh's PROC_ADDR." + :default nil]]) (defn- prepare-dynamo-opts "Transform parsed CLI options into the map expected by diff --git a/jepsen/test/elastickv/dynamodb_multi_table_workload_test.clj b/jepsen/test/elastickv/dynamodb_multi_table_workload_test.clj index 4b96bece..493a2510 100644 --- a/jepsen/test/elastickv/dynamodb_multi_table_workload_test.clj +++ b/jepsen/test/elastickv/dynamodb_multi_table_workload_test.clj @@ -35,9 +35,11 @@ ;; routing surfaces in this test rather than as a silent G1c during a ;; Jepsen run. -(def ^:private key->table-idx (var-get #'workload/key->table-idx)) -(def ^:private key->table-name (var-get #'workload/key->table-name)) -(def ^:private key->pk (var-get #'workload/key->pk)) +(def ^:private key->table-idx (var-get #'workload/key->table-idx)) +(def ^:private key->table-name (var-get #'workload/key->table-name)) +(def ^:private key->pk (var-get #'workload/key->pk)) +(def ^:private distinct-group-ids (var-get #'workload/distinct-group-ids)) +(def ^:private default-grpc-host-port-for (var-get #'workload/default-grpc-host-port-for)) (deftest key-routing-distributes-across-all-tables ;; Elle's default key-count is 12. With N=4 tables and the @@ -100,3 +102,64 @@ hits-g2 (some keys-in-group2 tables)] (is hits-g1 "sample 4-key txn must touch at least one table in group 1") (is hits-g2 "sample 4-key txn must touch at least one table in group 2"))) + +;; --------------------------------------------------------------------------- +;; M5a setup-hook verification — distinct-group-ids regex +;; --------------------------------------------------------------------------- + +(deftest distinct-group-ids-extracts-multi-group-routing + ;; The M5a-required shape: >=2 distinct raft_group_id values present. + (let [json (str "{\"catalog_version\":7,\"routes\":[" + "{\"route_id\":100,\"raft_group_id\":1,\"start\":\"\",\"end\":\"\",\"state\":\"ROUTE_STATE_ACTIVE\"}," + "{\"route_id\":101,\"raft_group_id\":2,\"start\":\"\",\"end\":\"\",\"state\":\"ROUTE_STATE_ACTIVE\"}" + "]}")] + (is (= #{1 2} (distinct-group-ids json)) + "two routes on distinct groups must yield #{1 2}"))) + +(deftest distinct-group-ids-collapses-single-group + ;; Failure shape: launch script ran with --shardRanges but no + ;; --raftGroups (or default single-group fallback). + (let [json (str "{\"catalog_version\":1,\"routes\":[" + "{\"route_id\":100,\"raft_group_id\":1,\"start\":\"\",\"end\":\"\",\"state\":\"ROUTE_STATE_ACTIVE\"}," + "{\"route_id\":101,\"raft_group_id\":1,\"start\":\"\",\"end\":\"\",\"state\":\"ROUTE_STATE_ACTIVE\"}" + "]}")] + (is (= #{1} (distinct-group-ids json)) + "two routes on the same group must yield #{1} — verify-multi-group-routing! must reject this"))) + +(deftest distinct-group-ids-empty-on-missing-shape + ;; A future ListRoutes schema change that renames raft_group_id + ;; would make the regex return nothing — the empty set causes + ;; verify-multi-group-routing! to throw, surfacing the schema + ;; drift loudly rather than silently passing. + (is (= #{} (distinct-group-ids "{\"catalog_version\":7,\"routes\":[]}")) + "no routes must yield empty set") + (is (= #{} (distinct-group-ids "{\"unrelated\":true}")) + "missing routes field must yield empty set")) + +(deftest default-grpc-host-port-resolves-from-first-node + ;; Distributed Jepsen runs configure :nodes with real hostnames + ;; (e.g. ["n1" "n2" "n3"]). default-grpc-host-port-for must + ;; derive the --address from the first node so the setup-hook + ;; doesn't punt every distributed run to localhost (gemini + ;; medium on PR #925). + (is (= "n1:50051" (default-grpc-host-port-for {:nodes ["n1" "n2" "n3"]})) + "first node hostname must form the default --address") + (is (= "alpha.internal:50051" + (default-grpc-host-port-for {:nodes ["alpha.internal" "beta.internal"]})) + "FQDN-style nodes must round-trip cleanly") + (is (= "n1:50051" (default-grpc-host-port-for {:nodes [:n1 :n2]})) + "keyword node ids must be coerced via (name)")) + +(deftest default-grpc-host-port-falls-back-on-empty-nodes + ;; The pre-existing 127.0.0.1:50051 default is the right fallback + ;; for test maps with no :nodes key (the workload-builder unit + ;; tests under this file, for one). + (is (= "127.0.0.1:50051" (default-grpc-host-port-for {}))) + (is (= "127.0.0.1:50051" (default-grpc-host-port-for {:nodes []})))) + +(deftest distinct-group-ids-handles-whitespace + ;; The CLI pretty-prints with two-space indent; the regex must + ;; tolerate \s* between the key, colon, and value. + (let [json "{\n \"routes\": [\n {\n \"raft_group_id\": 3,\n \"route_id\": 1\n }\n ]\n}"] + (is (= #{3} (distinct-group-ids json)) + "whitespace and newlines between key/colon/value must be tolerated")))