diff --git a/scripts/run-jepsen-m5-local.sh b/scripts/run-jepsen-m5-local.sh new file mode 100755 index 00000000..4afd2653 --- /dev/null +++ b/scripts/run-jepsen-m5-local.sh @@ -0,0 +1,224 @@ +#!/usr/bin/env bash +# Run the Composed-1 M5 multi-table DynamoDB Jepsen workload against a +# single-process two-group elastickv cluster. +# +# Why this script exists separately from run-jepsen-local.sh: the M5 +# workload requires a multi-Raft-group cluster topology that the +# existing 3-node single-group layout cannot provide. Per the design +# doc (docs/design/2026_06_02_proposed_composed1_m5_jepsen_route_shuffle.md +# §3.3), today's `validateShardRanges` / `buildShardGroups` only +# support a "single process hosts all groups" model — separate +# processes per group fail validation or race on Raft listeners. +# So this script launches ONE process hosting BOTH single-member +# groups, with two Raft listeners (50051, 50054) and one shared +# DynamoDB endpoint (63801). +# +# Trade-off accepted: partition / kill nemeses can't isolate one +# group from the other since they share a process. Only the +# (future) route-shuffle nemesis exercises the cross-group path +# meaningfully under this topology. True distributed multi-group is +# M6+ work — see the parent design doc. +# +# Usage: +# ./scripts/run-jepsen-m5-local.sh # build + start + test +# ./scripts/run-jepsen-m5-local.sh --no-rebuild # skip go build +# ./scripts/run-jepsen-m5-local.sh --no-cluster # reuse running cluster +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +BINARY=/tmp/elastickv4-m5-binary +ROUTE_KEY_BIN=/tmp/elastickv4-m5-route-key +LIST_ROUTES_BIN=/tmp/elastickv4-m5-list-routes +DATA_DIR=/tmp/elastickv-m5-test-run +PID_FILE=/tmp/elastickv-m5-test-run.pid + +# ---- topology: one process, two single-member Raft groups ---- +RAFT_ADDR_G1="127.0.0.1:50051" +RAFT_ADDR_G2="127.0.0.1:50054" +DYNAMO_ADDR="127.0.0.1:63801" +PROC_ADDR="$RAFT_ADDR_G1" # the process's primary gRPC address +RAFT_GROUPS="1=$RAFT_ADDR_G1,2=$RAFT_ADDR_G2" +RAFT_DYNAMO_MAP="$RAFT_ADDR_G1=$DYNAMO_ADDR,$RAFT_ADDR_G2=$DYNAMO_ADDR" + +NO_REBUILD=false +NO_CLUSTER=false +for arg in "$@"; do + case "$arg" in + --no-rebuild) NO_REBUILD=true ;; + --no-cluster) NO_CLUSTER=true ;; + esac +done + +# ---- build (server + route-key + list-routes helpers) ---- +if ! $NO_REBUILD; then + # Pre-flight: cmd/elastickv-list-routes lands in PR #925. If this + # branch is run before #925 merges, `go build` would emit an + # opaque package-not-found error. Surface the cross-PR dependency + # in a machine-readable way (claude[bot] suggestion on PR #924). + if [ ! -d "$REPO_ROOT/cmd/elastickv-list-routes" ]; then + echo "[error] cmd/elastickv-list-routes/ not found in this branch." >&2 + echo " PR #924 depends on PR #925 (setup-hook + list-routes CLI)." >&2 + echo " Merge #925 first, or check out the integrated branch." >&2 + exit 1 + fi + echo "[build] compiling elastickv server..." + cd "$REPO_ROOT" + go build -o "$BINARY" . + echo "[build] compiling elastickv-route-key helper..." + go build -o "$ROUTE_KEY_BIN" ./cmd/elastickv-route-key + echo "[build] compiling elastickv-list-routes helper..." + # Used by the Jepsen workload's setup-hook verification + # (verify-multi-group-routing!). Confirms the cluster booted with + # >=2 distinct Raft groups before any workload op runs. + go build -o "$LIST_ROUTES_BIN" ./cmd/elastickv-list-routes + echo "[build] done -> $BINARY, $ROUTE_KEY_BIN, $LIST_ROUTES_BIN" +fi + +# ---- compute --shardRanges boundary keys ---- +# Multi-table workload uses tables jepsen_append_t{1..4}. Tables 1-2 +# go to group 1, tables 3-4 to group 2. Boundary keys are the +# byte-for-byte route-key encoding of the table names — computed via +# the elastickv-route-key Go helper rather than inlined in shell so +# the base64 encoding stays in sync with adapter/dynamodb.go's +# encodeDynamoSegment (codex P1 #1 on PR #905 ffb9c73f). +# +# Guard: every helper binary must exist before continuing. Runs +# unconditionally — catches both --no-rebuild (helpers expected from +# a previous run) AND a fresh-build environment where a helper +# somehow produced a non-executable. Failing fast with a clear +# remediation message is strictly better than letting `set -e` +# swallow a misleading 'No such file or directory' deeper in the +# script (gemini medium + claude[bot] minor on PR #924). +for bin in "$ROUTE_KEY_BIN" "$LIST_ROUTES_BIN" "$BINARY"; do + if [ ! -x "$bin" ]; then + echo "[error] required helper not found at $bin." >&2 + echo " Re-run without --no-rebuild to compile the helpers." >&2 + exit 1 + fi +done +T1_KEY="$("$ROUTE_KEY_BIN" jepsen_append_t1)" +T3_KEY="$("$ROUTE_KEY_BIN" jepsen_append_t3)" +# Group 1: [T1_KEY, T3_KEY) — tables 1, 2 +# Group 2: [T3_KEY, +inf) — tables 3, 4 +# Keys outside [T1_KEY, +inf) fall through to the default group; this +# workload only writes table-route keys so that range is unused. +SHARD_RANGES="${T1_KEY}:${T3_KEY}=1,${T3_KEY}:=2" +echo "[shard-ranges] $SHARD_RANGES" + +# ---- stop any previously managed cluster ---- +stop_cluster() { + if [ -f "$PID_FILE" ]; then + echo "[cluster] stopping previous cluster..." + while IFS= read -r pid; do + kill "$pid" 2>/dev/null || true + done < "$PID_FILE" + rm -f "$PID_FILE" + fi +} + +# ---- start cluster: ONE process hosting both groups ---- +if ! $NO_CLUSTER; then + # Install the cleanup hook BEFORE starting the cluster so an + # exception during launch (e.g. bind-port collision) still + # tears down the half-started state. EXIT covers normal flow, + # INT/TERM cover user Ctrl-C and CI cancellation. Without + # this the failure path leaks background processes that hold + # the Raft / Dynamo ports for the next run (gemini medium on + # PR #924). + trap stop_cluster EXIT INT TERM + stop_cluster + rm -rf "$DATA_DIR" + mkdir -p "$DATA_DIR" + : > "$PID_FILE" + + echo "[cluster] starting single-process two-group cluster..." + # Notes on flag selection: + # --raftBootstrap : boolean; each group is single-member so no + # peer discovery is needed. --raftBootstrapMembers + # is rejected by resolveBootstrapServers on any + # multi-group process (main.go:735-741) and so + # MUST NOT appear here (codex P2 + claude[bot] + # P2 on PR #905 3ca2a7f7). + # --raftGroups : declares both groups with distinct Raft + # listeners. + # --shardRanges : places t1-t2 in group 1 and t3-t4 in group 2. + # Both flags are required for the multi-group + # contract: --shardRanges alone collapses + # everything to the default group 1 + # (coderabbit Major on PR #905 f92a029e). + # --raftDynamoMap : both Raft addresses point at the same Dynamo + # endpoint since there's only one process. + nohup "$BINARY" \ + --address "$PROC_ADDR" \ + --dynamoAddress "$DYNAMO_ADDR" \ + --redisAddress "" \ + --s3Address "" \ + --sqsAddress "" \ + --metricsAddress "" \ + --pprofAddress "" \ + --raftId "n1" \ + --raftDataDir "${DATA_DIR}/n1" \ + --raftBootstrap \ + --raftGroups "$RAFT_GROUPS" \ + --shardRanges "$SHARD_RANGES" \ + --raftDynamoMap "$RAFT_DYNAMO_MAP" \ + > "${DATA_DIR}/n1.log" 2>&1 & + echo $! >> "$PID_FILE" + + echo "[cluster] waiting for Dynamo endpoint ($DYNAMO_ADDR)..." + for i in $(seq 1 90); do + # Use bash's built-in /dev/tcp probe rather than `nc` so the + # script runs on minimal CI images that may not ship netcat + # (gemini medium on PR #924). + if (echo > /dev/tcp/127.0.0.1/63801) >/dev/null 2>&1; then + echo "[cluster] up after ${i}s" + break + fi + sleep 1 + if [ "$i" -eq 90 ]; then + echo "[cluster] FAILED to start - dumping log:" + tail -n 100 "${DATA_DIR}/n1.log" || true + exit 1 + fi + done +fi + +# ---- run M5 Jepsen multi-table workload ---- +cd "$REPO_ROOT/jepsen" + +# Resolve lein: prefer LEIN env override, then PATH (works on CI), then +# the macOS Homebrew default. Failing to find lein is fatal. +LEIN_BIN="${LEIN:-$(command -v lein || echo /opt/homebrew/bin/lein)}" +if [ ! -x "$LEIN_BIN" ]; then + echo "[jepsen] lein not found (tried \$LEIN, PATH, /opt/homebrew/bin/lein)" >&2 + exit 127 +fi + +echo "[jepsen] running DynamoDB multi-table list-append workload via $LEIN_BIN..." +mkdir -p tmp-home .lein +# --list-routes-bin / --grpc-host-port wire the setup-hook verification +# (verify-multi-group-routing!) at the workload's first setup! call. +# Without them the hook falls back to PATH lookup which fails when +# run from this script's tmp build. +HOME="$(pwd)/tmp-home" LEIN_HOME="$(pwd)/.lein" \ + LEIN_JVM_OPTS="-Duser.home=$(pwd)/tmp-home" \ + "$LEIN_BIN" run -m elastickv.dynamodb-multi-table-workload \ + --local \ + --time-limit 30 \ + --rate 5 \ + --dynamo-port 63801 \ + --list-routes-bin "$LIST_ROUTES_BIN" \ + --grpc-host-port "$PROC_ADDR" \ + || EXIT_CODE=$? + +EXIT_CODE=${EXIT_CODE:-0} + +# ---- teardown ---- +# Cluster shutdown is handled by the `trap stop_cluster EXIT INT TERM` +# installed above the cluster launch. No explicit teardown call is +# needed here; doing so would double-call stop_cluster on success +# (harmless but noisy) and double-call on failure (which is also +# harmless since stop_cluster is idempotent, but the EXIT trap path +# is the canonical one — see gemini medium on PR #924). + +exit "$EXIT_CODE"