CI: Raft consensus testing under failure conditions

.github/workflows/functional.yml

@@ -289,3 +289,85 @@ jobs:
       env:
         PG_IMAGE: postgres:${{ matrix.pg_version }}
       run: docker compose down -v --remove-orphans 2>/dev/null || true
+
+  functional-raft:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    needs: build
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Download orchestrator binary
+      uses: actions/download-artifact@v4
+      with:
+        name: orchestrator-binary
+        path: bin
+
+    - name: Make binary executable
+      run: chmod +x bin/orchestrator
+
+    - name: Start MySQL infrastructure
+      working-directory: tests/functional
+      run: |
+        docker compose up -d mysql1 mysql2 mysql3
+        echo "Waiting for MySQL to be healthy..."
+        timeout 120 bash -c '
+          while true; do
+            HEALTHY=$(docker compose ps --format json 2>/dev/null | python3 -c "
+        import json, sys
+        healthy = 0
+        for line in sys.stdin:
+            svc = json.loads(line)
+            if \"healthy\" in svc.get(\"Status\",\"\").lower():
+                healthy += 1
+        print(healthy)
+        " 2>/dev/null || echo "0")
+            if [ "$HEALTHY" -ge 3 ]; then
+              echo "All 3 MySQL services healthy"
+              exit 0
+            fi
+            sleep 2
+          done
+        ' || { echo "Timeout"; docker compose ps; docker compose logs --tail=30; exit 1; }
+
+    - name: Setup replication
+      run: bash tests/functional/setup-replication.sh
+
+    - name: Run Raft consensus tests
+      run: bash tests/functional/test-raft.sh
+
+    - name: Collect Raft orchestrator logs
+      if: always()
+      working-directory: tests/functional
+      run: |
+        docker compose logs orchestrator-raft1 > /tmp/orchestrator-raft1.log 2>&1 || true
+        docker compose logs orchestrator-raft2 > /tmp/orchestrator-raft2.log 2>&1 || true
+        docker compose logs orchestrator-raft3 > /tmp/orchestrator-raft3.log 2>&1 || true
+
+    - name: Upload Raft orchestrator logs
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: orchestrator-raft-logs
+        path: |
+          /tmp/orchestrator-raft1.log
+          /tmp/orchestrator-raft2.log
+          /tmp/orchestrator-raft3.log
+
+    - name: Collect all docker logs on failure
+      if: failure()
+      working-directory: tests/functional
+      run: docker compose logs > /tmp/docker-compose-raft-logs.txt 2>&1 || true
+
+    - name: Upload docker logs on failure
+      if: failure()
+      uses: actions/upload-artifact@v4
+      with:
+        name: docker-compose-raft-logs
+        path: /tmp/docker-compose-raft-logs.txt
+
+    - name: Cleanup
+      if: always()
+      working-directory: tests/functional
+      run: docker compose down -v --remove-orphans 2>/dev/null || true

go/raft/rel_store.go

@@ -36,15 +36,15 @@ var createQueries = []string{
 			log_index integer,
 			term bigint not null,
 			log_type int not null,
-			data blob not null,
+			data blob,
 			PRIMARY KEY (log_index)
 		)
 	`,
 	`
 		CREATE TABLE IF NOT EXISTS raft_store (
 			store_id integer,
 			store_key varbinary(512) not null,
-			store_value blob not null,
+			store_value blob,
 			PRIMARY KEY (store_id)
 		)
 	`,

tests/functional/docker-compose.yml

@@ -205,6 +205,81 @@ services:
         aliases:
           - orchestrator-pg
 
+  orchestrator-raft1:
+    image: ubuntu:24.04
+    hostname: orchestrator-raft1
+    volumes:
+      - ../../bin/orchestrator:/usr/local/bin/orchestrator:ro
+      - ../../resources:/orchestrator/resources:ro
+      - ./orchestrator-raft1.conf.json:/orchestrator/orchestrator.conf.json:ro
+    command: >
+      bash -c "
+        apt-get update -qq && apt-get install -y -qq curl sqlite3 > /dev/null 2>&1 &&
+        mkdir -p /tmp/raft1 &&
+        cd /orchestrator &&
+        orchestrator -config orchestrator.conf.json http
+      "
+    ports:
+      - "3100:3099"
+    depends_on:
+      mysql1:
+        condition: service_healthy
+    networks:
+      orchnet:
+        ipv4_address: 172.30.0.40
+        aliases:
+          - orchestrator-raft1
+
+  orchestrator-raft2:
+    image: ubuntu:24.04
+    hostname: orchestrator-raft2
+    volumes:
+      - ../../bin/orchestrator:/usr/local/bin/orchestrator:ro
+      - ../../resources:/orchestrator/resources:ro
+      - ./orchestrator-raft2.conf.json:/orchestrator/orchestrator.conf.json:ro
+    command: >
+      bash -c "
+        apt-get update -qq && apt-get install -y -qq curl sqlite3 > /dev/null 2>&1 &&
+        mkdir -p /tmp/raft2 &&
+        cd /orchestrator &&
+        orchestrator -config orchestrator.conf.json http
+      "
+    ports:
+      - "3101:3099"
+    depends_on:
+      mysql1:
+        condition: service_healthy
+    networks:
+      orchnet:
+        ipv4_address: 172.30.0.41
+        aliases:
+          - orchestrator-raft2
+
+  orchestrator-raft3:
+    image: ubuntu:24.04
+    hostname: orchestrator-raft3
+    volumes:
+      - ../../bin/orchestrator:/usr/local/bin/orchestrator:ro
+      - ../../resources:/orchestrator/resources:ro
+      - ./orchestrator-raft3.conf.json:/orchestrator/orchestrator.conf.json:ro
+    command: >
+      bash -c "
+        apt-get update -qq && apt-get install -y -qq curl sqlite3 > /dev/null 2>&1 &&
+        mkdir -p /tmp/raft3 &&
+        cd /orchestrator &&
+        orchestrator -config orchestrator.conf.json http
+      "
+    ports:
+      - "3102:3099"
+    depends_on:
+      mysql1:
+        condition: service_healthy
+    networks:
+      orchnet:
+        ipv4_address: 172.30.0.42
+        aliases:
+          - orchestrator-raft3
+
 networks:
   orchnet:
     driver: bridge

tests/functional/orchestrator-raft1.conf.json

@@ -0,0 +1,25 @@
+{
+  "Debug": true,
+  "ListenAddress": ":3099",
+  "HTTPAdvertise": "http://172.30.0.40:3099",
+  "MySQLTopologyUser": "orchestrator",
+  "MySQLTopologyPassword": "orch_pass",
+  "MySQLOrchestratorHost": "",
+  "MySQLOrchestratorPort": 0,
+  "BackendDB": "sqlite",
+  "SQLite3DataFile": "/tmp/raft1/orchestrator.sqlite3",
+  "DiscoverByShowSlaveHosts": false,
+  "InstancePollSeconds": 5,
+  "RecoveryPeriodBlockSeconds": 10,
+  "RecoverMasterClusterFilters": [".*"],
+  "RecoverIntermediateMasterClusterFilters": [".*"],
+  "AutoPseudoGTID": false,
+  "DetectClusterAliasQuery": "SELECT CONCAT(@@hostname, ':', @@port)",
+  "DetectInstanceAliasQuery": "SELECT CONCAT(@@hostname, ':', @@port)",
+  "PrometheusEnabled": false,
+  "RaftEnabled": true,
+  "RaftDataDir": "/tmp/raft1",
+  "RaftBind": "172.30.0.40",
+  "DefaultRaftPort": 10008,
+  "RaftNodes": ["172.30.0.40", "172.30.0.41", "172.30.0.42"]
+}

tests/functional/orchestrator-raft2.conf.json

@@ -0,0 +1,25 @@
+{
+  "Debug": true,
+  "ListenAddress": ":3099",
+  "HTTPAdvertise": "http://172.30.0.41:3099",
+  "MySQLTopologyUser": "orchestrator",
+  "MySQLTopologyPassword": "orch_pass",
+  "MySQLOrchestratorHost": "",
+  "MySQLOrchestratorPort": 0,
+  "BackendDB": "sqlite",
+  "SQLite3DataFile": "/tmp/raft2/orchestrator.sqlite3",
+  "DiscoverByShowSlaveHosts": false,
+  "InstancePollSeconds": 5,
+  "RecoveryPeriodBlockSeconds": 10,
+  "RecoverMasterClusterFilters": [".*"],
+  "RecoverIntermediateMasterClusterFilters": [".*"],
+  "AutoPseudoGTID": false,
+  "DetectClusterAliasQuery": "SELECT CONCAT(@@hostname, ':', @@port)",
+  "DetectInstanceAliasQuery": "SELECT CONCAT(@@hostname, ':', @@port)",
+  "PrometheusEnabled": false,
+  "RaftEnabled": true,
+  "RaftDataDir": "/tmp/raft2",
+  "RaftBind": "172.30.0.41",
+  "DefaultRaftPort": 10008,
+  "RaftNodes": ["172.30.0.40", "172.30.0.41", "172.30.0.42"]
+}

tests/functional/orchestrator-raft3.conf.json

@@ -0,0 +1,25 @@
+{
+  "Debug": true,
+  "ListenAddress": ":3099",
+  "HTTPAdvertise": "http://172.30.0.42:3099",
+  "MySQLTopologyUser": "orchestrator",
+  "MySQLTopologyPassword": "orch_pass",
+  "MySQLOrchestratorHost": "",
+  "MySQLOrchestratorPort": 0,
+  "BackendDB": "sqlite",
+  "SQLite3DataFile": "/tmp/raft3/orchestrator.sqlite3",
+  "DiscoverByShowSlaveHosts": false,
+  "InstancePollSeconds": 5,
+  "RecoveryPeriodBlockSeconds": 10,
+  "RecoverMasterClusterFilters": [".*"],
+  "RecoverIntermediateMasterClusterFilters": [".*"],
+  "AutoPseudoGTID": false,
+  "DetectClusterAliasQuery": "SELECT CONCAT(@@hostname, ':', @@port)",
+  "DetectInstanceAliasQuery": "SELECT CONCAT(@@hostname, ':', @@port)",
+  "PrometheusEnabled": false,
+  "RaftEnabled": true,
+  "RaftDataDir": "/tmp/raft3",
+  "RaftBind": "172.30.0.42",
+  "DefaultRaftPort": 10008,
+  "RaftNodes": ["172.30.0.40", "172.30.0.41", "172.30.0.42"]
+}