Add functional test CI: Docker MySQL topology + ProxySQL + failover tests

.github/workflows/functional.yml

@@ -0,0 +1,107 @@
+name: Functional Tests
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+  workflow_dispatch:
+
+jobs:
+  functional:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: '1.25.7'
+        cache: true
+
+    - name: Build orchestrator (for CLI tests on host)
+      run: go build -o bin/orchestrator ./go/cmd/orchestrator
+
+    - name: Start test infrastructure (MySQL + ProxySQL + Orchestrator)
+      working-directory: tests/functional
+      run: |
+        docker compose up -d mysql1 mysql2 mysql3 proxysql
+        echo "Waiting for MySQL and ProxySQL to be healthy..."
+        timeout 120 bash -c '
+          while true; do
+            HEALTHY=$(docker compose ps --format json 2>/dev/null | python3 -c "
+        import json, sys
+        healthy = 0
+        for line in sys.stdin:
+            svc = json.loads(line)
+            if \"healthy\" in svc.get(\"Status\",\"\").lower():
+                healthy += 1
+        print(healthy)
+        " 2>/dev/null || echo "0")
+            if [ "$HEALTHY" -ge 4 ]; then
+              echo "All 4 services healthy"
+              exit 0
+            fi
+            sleep 2
+          done
+        ' || { echo "Timeout"; docker compose ps; docker compose logs --tail=30; exit 1; }
+
+    - name: Setup replication
+      run: bash tests/functional/setup-replication.sh
+
+    - name: Start orchestrator in Docker network
+      working-directory: tests/functional
+      run: |
+        docker compose up -d orchestrator
+        echo "Waiting for orchestrator to be ready..."
+        timeout 120 bash -c '
+          while true; do
+            if curl -sf http://localhost:3099/api/clusters > /dev/null 2>&1; then
+              echo "Orchestrator ready"
+              exit 0
+            fi
+            sleep 2
+          done
+        ' || { echo "Orchestrator not ready"; docker compose logs orchestrator --tail=50; exit 1; }
+
+    - name: Run smoke tests
+      run: bash tests/functional/test-smoke.sh
+
+    - name: Run regression tests
+      run: bash tests/functional/test-regression.sh
+
+    - name: Run failover tests
+      run: bash tests/functional/test-failover.sh
+
+    - name: Collect orchestrator logs
+      if: always()
+      working-directory: tests/functional
+      run: docker compose logs orchestrator > /tmp/orchestrator-test.log 2>&1 || true
+
+    - name: Upload orchestrator logs
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: orchestrator-test-logs
+        path: /tmp/orchestrator-test.log
+
+    - name: Collect all docker logs on failure
+      if: failure()
+      working-directory: tests/functional
+      run: docker compose logs > /tmp/docker-compose-logs.txt 2>&1 || true
+
+    - name: Upload docker logs on failure
+      if: failure()
+      uses: actions/upload-artifact@v4
+      with:
+        name: docker-compose-logs
+        path: /tmp/docker-compose-logs.txt
+
+    - name: Cleanup
+      if: always()
+      working-directory: tests/functional
+      run: docker compose down -v --remove-orphans 2>/dev/null || true

tests/functional/docker-compose.yml

@@ -0,0 +1,127 @@
+version: "3.8"
+
+services:
+  mysql1:
+    image: mysql:8.4
+    hostname: mysql1
+    environment:
+      MYSQL_ROOT_PASSWORD: testpass
+    ports:
+      - "13306:3306"
+    volumes:
+      - ./mysql/master.cnf:/etc/mysql/conf.d/repl.cnf
+      - ./mysql/init-master.sql:/docker-entrypoint-initdb.d/init.sql
+    healthcheck:
+      test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-uroot", "-ptestpass"]
+      interval: 5s
+      timeout: 3s
+      retries: 30
+    networks:
+      orchnet:
+        aliases:
+          - mysql1
+
+  mysql2:
+    image: mysql:8.4
+    hostname: mysql2
+    environment:
+      MYSQL_ROOT_PASSWORD: testpass
+    ports:
+      - "13307:3306"
+    volumes:
+      - ./mysql/replica.cnf:/etc/mysql/conf.d/repl.cnf
+      - ./mysql/init-replica.sql:/docker-entrypoint-initdb.d/init.sql
+    depends_on:
+      mysql1:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-uroot", "-ptestpass"]
+      interval: 5s
+      timeout: 3s
+      retries: 30
+    networks:
+      orchnet:
+        aliases:
+          - mysql2
+
+  mysql3:
+    image: mysql:8.4
+    hostname: mysql3
+    environment:
+      MYSQL_ROOT_PASSWORD: testpass
+    ports:
+      - "13308:3306"
+    volumes:
+      - ./mysql/replica2.cnf:/etc/mysql/conf.d/repl.cnf
+      - ./mysql/init-replica.sql:/docker-entrypoint-initdb.d/init.sql
+    depends_on:
+      mysql1:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-uroot", "-ptestpass"]
+      interval: 5s
+      timeout: 3s
+      retries: 30
+    networks:
+      orchnet:
+        aliases:
+          - mysql3
+
+  proxysql:
+    image: proxysql/proxysql:latest
+    hostname: proxysql
+    ports:
+      - "16032:6032"
+      - "16033:6033"
+    volumes:
+      - ./proxysql/proxysql.cnf:/etc/proxysql.cnf
+    depends_on:
+      mysql1:
+        condition: service_healthy
+      mysql2:
+        condition: service_healthy
+      mysql3:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-P6032", "-uradmin", "-pradmin"]
+      interval: 5s
+      timeout: 3s
+      retries: 30
+    networks:
+      orchnet:
+        aliases:
+          - proxysql
+
+  orchestrator:
+    image: ubuntu:24.04
+    hostname: orchestrator
+    volumes:
+      - ../../bin/orchestrator:/usr/local/bin/orchestrator:ro
+      - ../../resources:/orchestrator/resources:ro
+      - ./orchestrator-test.conf.json:/orchestrator/orchestrator.conf.json:ro
+    command: >
+      bash -c "
+        apt-get update -qq && apt-get install -y -qq curl sqlite3 > /dev/null 2>&1 &&
+        rm -f /tmp/orchestrator-test.sqlite3 &&
+        cd /orchestrator &&
+        orchestrator -config orchestrator.conf.json http
+      "
+    ports:
+      - "3099:3099"
+    depends_on:
+      proxysql:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "curl", "-sf", "http://localhost:3099/api/clusters"]
+      interval: 5s
+      timeout: 3s
+      retries: 60
+      start_period: 15s
+    networks:
+      orchnet:
+        aliases:
+          - orchestrator
+
+networks:
+  orchnet:
+    driver: bridge

tests/functional/lib.sh

@@ -0,0 +1,125 @@
+#!/bin/bash
+# Shared test helpers for functional tests
+
+PASS_COUNT=0
+FAIL_COUNT=0
+SKIP_COUNT=0
+ORC_URL="http://localhost:3099"
+
+pass() {
+    echo "  ✅ PASS: $1"
+    ((PASS_COUNT++))
+}
+
+fail() {
+    echo "  ❌ FAIL: $1"
+    [ -n "$2" ] && echo "         $2"
+    ((FAIL_COUNT++))
+}
+
+skip() {
+    echo "  ⚠️  SKIP: $1"
+    ((SKIP_COUNT++))
+}
+
+summary() {
+    echo ""
+    echo "=== RESULTS: $PASS_COUNT passed, $FAIL_COUNT failed, $SKIP_COUNT skipped ==="
+    [ "$FAIL_COUNT" -gt 0 ] && exit 1
+    exit 0
+}
+
+# Test that an HTTP endpoint returns expected status code
+test_endpoint() {
+    local NAME="$1" URL="$2" EXPECT="$3"
+    local CODE
+    CODE=$(curl -s -o /dev/null -w "%{http_code}" "$URL" 2>&1)
+    if [ "$CODE" = "$EXPECT" ]; then
+        pass "$NAME (HTTP $CODE)"
+    else
+        fail "$NAME (HTTP $CODE, expected $EXPECT)"
+    fi
+}
+
+# Test that response body contains a string
+test_body_contains() {
+    local NAME="$1" URL="$2" EXPECT="$3"
+    local BODY
+    BODY=$(curl -s "$URL" 2>&1)
+    if echo "$BODY" | grep -q "$EXPECT"; then
+        pass "$NAME"
+    else
+        fail "$NAME" "Response does not contain '$EXPECT'"
+    fi
+}
+
+# Wait for orchestrator to be ready
+wait_for_orchestrator() {
+    echo "Waiting for orchestrator to be ready..."
+    for i in $(seq 1 30); do
+        if curl -s -o /dev/null "$ORC_URL/api/clusters" 2>/dev/null; then
+            echo "Orchestrator ready after ${i}s"
+            return 0
+        fi
+        sleep 1
+    done
+    echo "Orchestrator not ready after 30s"
+    return 1
+}
+
+# Seed discovery and wait for all instances
+# Sets CLUSTER_NAME as a global variable
+CLUSTER_NAME=""
+discover_topology() {
+    local MASTER_HOST="$1"
+    echo "Seeding discovery with $MASTER_HOST..."
+    curl -s "$ORC_URL/api/discover/$MASTER_HOST/3306" > /dev/null
+
+    # Also seed replicas directly
+    curl -s "$ORC_URL/api/discover/mysql2/3306" > /dev/null 2>&1
+    curl -s "$ORC_URL/api/discover/mysql3/3306" > /dev/null 2>&1
+
+    echo "Waiting for topology discovery..."
+    for i in $(seq 1 60); do
+        # Get the cluster name dynamically
+        CLUSTER_NAME=$(curl -s "$ORC_URL/api/clusters" 2>/dev/null | python3 -c "import json,sys; c=json.load(sys.stdin); print(c[0] if c else '')" 2>/dev/null || echo "")
+        if [ -n "$CLUSTER_NAME" ]; then
+            local COUNT
+            COUNT=$(curl -s "$ORC_URL/api/cluster/$CLUSTER_NAME" 2>/dev/null | python3 -c "import json,sys; print(len(json.load(sys.stdin)))" 2>/dev/null || echo "0")
+            if [ "$COUNT" -ge 3 ] 2>/dev/null; then
+                echo "Full topology discovered (${COUNT} instances, cluster=$CLUSTER_NAME) after ${i}s"
+                return 0
+            fi
+        fi
+        # Re-seed replicas periodically
+        if [ "$((i % 10))" = "0" ]; then
+            curl -s "$ORC_URL/api/discover/mysql2/3306" > /dev/null 2>&1
+            curl -s "$ORC_URL/api/discover/mysql3/3306" > /dev/null 2>&1
+        fi
+        sleep 1
+    done
+    echo "WARNING: Cluster=$CLUSTER_NAME, instances=${COUNT:-0} after 60s"
+    return 1
+}
+
+# Get ProxySQL servers for a hostgroup
+proxysql_servers() {
+    local HG="$1"
+    docker compose -f tests/functional/docker-compose.yml exec -T proxysql \
+        mysql -h127.0.0.1 -P6032 -uradmin -pradmin -Nse \
+        "SELECT hostname, port, status FROM runtime_mysql_servers WHERE hostgroup_id=$HG" 2>/dev/null
+}
+
+# Get MySQL read_only status
+mysql_read_only() {
+    local CONTAINER="$1"
+    docker compose -f tests/functional/docker-compose.yml exec -T "$CONTAINER" \
+        mysql -uroot -ptestpass -Nse "SELECT @@read_only" 2>/dev/null
+}
+
+# Get MySQL replication source
+mysql_source_host() {
+    local CONTAINER="$1"
+    docker compose -f tests/functional/docker-compose.yml exec -T "$CONTAINER" \
+        mysql -uroot -ptestpass -Nse "SHOW REPLICA STATUS\G" 2>/dev/null | grep "Source_Host" | awk '{print $2}'
+}

tests/functional/mysql/init-master.sql

@@ -0,0 +1,9 @@
+-- Orchestrator user with full privileges
+CREATE USER IF NOT EXISTS 'orchestrator'@'%' IDENTIFIED BY 'orch_pass';
+GRANT ALL PRIVILEGES ON *.* TO 'orchestrator'@'%' WITH GRANT OPTION;
+
+-- Replication user
+CREATE USER IF NOT EXISTS 'repl'@'%' IDENTIFIED BY 'repl_pass';
+GRANT REPLICATION SLAVE ON *.* TO 'repl'@'%';
+
+FLUSH PRIVILEGES;

tests/functional/mysql/init-replica.sql

@@ -0,0 +1,8 @@
+-- Orchestrator user
+CREATE USER IF NOT EXISTS 'orchestrator'@'%' IDENTIFIED BY 'orch_pass';
+GRANT ALL PRIVILEGES ON *.* TO 'orchestrator'@'%' WITH GRANT OPTION;
+-- Replication user (needed if this replica gets promoted)
+CREATE USER IF NOT EXISTS 'repl'@'%' IDENTIFIED BY 'repl_pass';
+GRANT REPLICATION SLAVE ON *.* TO 'repl'@'%';
+FLUSH PRIVILEGES;
+-- NOTE: Replication is configured by setup-replication.sh after all containers are up

tests/functional/mysql/master.cnf

@@ -0,0 +1,9 @@
+[mysqld]
+server-id=1
+log-bin=mysql-bin
+binlog-format=ROW
+gtid-mode=ON
+enforce-gtid-consistency=ON
+log-replica-updates=ON
+binlog-row-image=MINIMAL
+report-host=mysql1