Skip to content

Commit 4e3df8a

Browse files
renecannaorenecannao
committed
Add functional test suite with Docker infrastructure and GitHub Actions
Comprehensive functional tests running against real MySQL topology + ProxySQL: Infrastructure (docker-compose.yml): - MySQL 8.4 master + 2 replicas with GTID replication - ProxySQL with writer (HG 10) and reader (HG 20) hostgroups - Auto-configured replication and orchestrator user Test suites: - test-smoke.sh (Tier A): Discovery, API v1/v2, Prometheus, health endpoints, ProxySQL CLI/API, web UI, static files (25+ checks) - test-failover.sh (Tier B): Graceful master takeover with ProxySQL hook validation, hard failover (kill master), recovery audit (12+ checks) - test-regression.sh (Tier C): Full chi router regression, API v2 response format, Prometheus metrics, health endpoints (22+ checks) GitHub Actions workflow (functional.yml): - Triggers on push to master, PRs, and manual dispatch - Starts Docker infrastructure, builds orchestrator, runs all 3 suites - Uploads orchestrator and docker logs as artifacts on failure - 15 minute timeout Closes #46, closes #47, closes #48, closes #49, closes #50
1 parent 7fe2446 commit 4e3df8a

13 files changed

Lines changed: 819 additions & 0 deletions

File tree

‎.github/workflows/functional.yml‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
name: Functional Tests
2+
3+
on:
4+
push:
5+
branches:
6+
- master
7+
pull_request:
8+
branches:
9+
- master
10+
workflow_dispatch:
11+
12+
jobs:
13+
functional:
14+
runs-on: ubuntu-latest
15+
timeout-minutes: 15
16+
17+
steps:
18+
- uses: actions/checkout@v4
19+
20+
- name: Set up Go
21+
uses: actions/setup-go@v5
22+
with:
23+
go-version: '1.25.7'
24+
cache: true
25+
26+
- name: Build orchestrator
27+
run: go build -o bin/orchestrator ./go/cmd/orchestrator
28+
29+
- name: Start test infrastructure
30+
working-directory: tests/functional
31+
run: |
32+
docker compose up -d
33+
echo "Waiting for services to be healthy..."
34+
timeout 120 bash -c '
35+
while true; do
36+
HEALTHY=$(docker compose ps --format json | python3 -c "
37+
import json, sys
38+
healthy = 0
39+
for line in sys.stdin:
40+
svc = json.loads(line)
41+
if \"healthy\" in svc.get(\"Status\",\"\").lower():
42+
healthy += 1
43+
print(healthy)
44+
" 2>/dev/null || echo "0")
45+
if [ "$HEALTHY" -ge 4 ]; then
46+
echo "All services healthy"
47+
exit 0
48+
fi
49+
sleep 2
50+
done
51+
' || { echo "Timeout waiting for services"; docker compose ps; docker compose logs --tail=30; exit 1; }
52+
53+
- name: Verify replication
54+
working-directory: tests/functional
55+
run: |
56+
timeout 30 bash -c '
57+
while true; do
58+
REPL=$(docker compose exec -T mysql2 mysql -uroot -ptestpass -Nse "SHOW REPLICA STATUS\G" 2>/dev/null | grep "Replica_IO_Running: Yes" || true)
59+
if [ -n "$REPL" ]; then
60+
echo "Replication running"
61+
exit 0
62+
fi
63+
sleep 1
64+
done
65+
' || { echo "Replication not running"; exit 1; }
66+
67+
- name: Start orchestrator
68+
run: |
69+
rm -f /tmp/orchestrator-test.sqlite3
70+
bin/orchestrator -config tests/functional/orchestrator-test.conf.json http > /tmp/orchestrator-test.log 2>&1 &
71+
echo "$!" > /tmp/orchestrator-test.pid
72+
sleep 3
73+
74+
- name: Run smoke tests
75+
run: bash tests/functional/test-smoke.sh
76+
77+
- name: Run regression tests
78+
run: bash tests/functional/test-regression.sh
79+
80+
- name: Run failover tests
81+
run: bash tests/functional/test-failover.sh
82+
83+
- name: Upload orchestrator logs
84+
if: always()
85+
uses: actions/upload-artifact@v4
86+
with:
87+
name: orchestrator-test-logs
88+
path: /tmp/orchestrator-test.log
89+
90+
- name: Collect docker logs on failure
91+
if: failure()
92+
working-directory: tests/functional
93+
run: docker compose logs > /tmp/docker-compose-logs.txt 2>&1 || true
94+
95+
- name: Upload docker logs on failure
96+
if: failure()
97+
uses: actions/upload-artifact@v4
98+
with:
99+
name: docker-compose-logs
100+
path: /tmp/docker-compose-logs.txt
101+
102+
- name: Cleanup
103+
if: always()
104+
run: |
105+
kill "$(cat /tmp/orchestrator-test.pid 2>/dev/null)" 2>/dev/null || true
106+
cd tests/functional && docker compose down -v --remove-orphans 2>/dev/null || true

‎tests/functional/docker-compose.yml‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
version: "3.8"
2+
3+
services:
4+
mysql1:
5+
image: mysql:8.4
6+
hostname: mysql1
7+
environment:
8+
MYSQL_ROOT_PASSWORD: testpass
9+
volumes:
10+
- ./mysql/master.cnf:/etc/mysql/conf.d/repl.cnf
11+
- ./mysql/init-master.sql:/docker-entrypoint-initdb.d/init.sql
12+
healthcheck:
13+
test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-uroot", "-ptestpass"]
14+
interval: 5s
15+
timeout: 3s
16+
retries: 30
17+
networks:
18+
- orchnet
19+
20+
mysql2:
21+
image: mysql:8.4
22+
hostname: mysql2
23+
environment:
24+
MYSQL_ROOT_PASSWORD: testpass
25+
volumes:
26+
- ./mysql/replica.cnf:/etc/mysql/conf.d/repl.cnf
27+
- ./mysql/init-replica.sql:/docker-entrypoint-initdb.d/init.sql
28+
depends_on:
29+
mysql1:
30+
condition: service_healthy
31+
healthcheck:
32+
test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-uroot", "-ptestpass"]
33+
interval: 5s
34+
timeout: 3s
35+
retries: 30
36+
networks:
37+
- orchnet
38+
39+
mysql3:
40+
image: mysql:8.4
41+
hostname: mysql3
42+
environment:
43+
MYSQL_ROOT_PASSWORD: testpass
44+
volumes:
45+
- ./mysql/replica.cnf:/etc/mysql/conf.d/repl.cnf
46+
- ./mysql/init-replica.sql:/docker-entrypoint-initdb.d/init.sql
47+
depends_on:
48+
mysql1:
49+
condition: service_healthy
50+
healthcheck:
51+
test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-uroot", "-ptestpass"]
52+
interval: 5s
53+
timeout: 3s
54+
retries: 30
55+
networks:
56+
- orchnet
57+
58+
proxysql:
59+
image: proxysql/proxysql:latest
60+
hostname: proxysql
61+
volumes:
62+
- ./proxysql/proxysql.cnf:/etc/proxysql.cnf
63+
depends_on:
64+
mysql1:
65+
condition: service_healthy
66+
mysql2:
67+
condition: service_healthy
68+
mysql3:
69+
condition: service_healthy
70+
healthcheck:
71+
test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-P6032", "-uradmin", "-pradmin"]
72+
interval: 5s
73+
timeout: 3s
74+
retries: 30
75+
networks:
76+
- orchnet
77+
78+
networks:
79+
orchnet:
80+
driver: bridge

‎tests/functional/lib.sh‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
#!/bin/bash
2+
# Shared test helpers for functional tests
3+
4+
PASS_COUNT=0
5+
FAIL_COUNT=0
6+
SKIP_COUNT=0
7+
ORC_URL="http://localhost:3099"
8+
9+
pass() {
10+
echo " ✅ PASS: $1"
11+
((PASS_COUNT++))
12+
}
13+
14+
fail() {
15+
echo " ❌ FAIL: $1"
16+
[ -n "$2" ] && echo " $2"
17+
((FAIL_COUNT++))
18+
}
19+
20+
skip() {
21+
echo " ⚠️ SKIP: $1"
22+
((SKIP_COUNT++))
23+
}
24+
25+
summary() {
26+
echo ""
27+
echo "=== RESULTS: $PASS_COUNT passed, $FAIL_COUNT failed, $SKIP_COUNT skipped ==="
28+
[ "$FAIL_COUNT" -gt 0 ] && exit 1
29+
exit 0
30+
}
31+
32+
# Test that an HTTP endpoint returns expected status code
33+
test_endpoint() {
34+
local NAME="$1" URL="$2" EXPECT="$3"
35+
local CODE
36+
CODE=$(curl -s -o /dev/null -w "%{http_code}" "$URL" 2>&1)
37+
if [ "$CODE" = "$EXPECT" ]; then
38+
pass "$NAME (HTTP $CODE)"
39+
else
40+
fail "$NAME (HTTP $CODE, expected $EXPECT)"
41+
fi
42+
}
43+
44+
# Test that response body contains a string
45+
test_body_contains() {
46+
local NAME="$1" URL="$2" EXPECT="$3"
47+
local BODY
48+
BODY=$(curl -s "$URL" 2>&1)
49+
if echo "$BODY" | grep -q "$EXPECT"; then
50+
pass "$NAME"
51+
else
52+
fail "$NAME" "Response does not contain '$EXPECT'"
53+
fi
54+
}
55+
56+
# Wait for orchestrator to be ready
57+
wait_for_orchestrator() {
58+
echo "Waiting for orchestrator to be ready..."
59+
for i in $(seq 1 30); do
60+
if curl -s -o /dev/null "$ORC_URL/api/clusters" 2>/dev/null; then
61+
echo "Orchestrator ready after ${i}s"
62+
return 0
63+
fi
64+
sleep 1
65+
done
66+
echo "Orchestrator not ready after 30s"
67+
return 1
68+
}
69+
70+
# Seed discovery and wait for all instances
71+
discover_topology() {
72+
local MASTER_HOST="$1"
73+
echo "Seeding discovery with $MASTER_HOST..."
74+
curl -s "$ORC_URL/api/discover/$MASTER_HOST/3306" > /dev/null
75+
76+
echo "Waiting for topology discovery..."
77+
for i in $(seq 1 60); do
78+
local COUNT
79+
COUNT=$(curl -s "$ORC_URL/api/cluster/$MASTER_HOST:3306" 2>/dev/null | python3 -c "import json,sys; print(len(json.load(sys.stdin)))" 2>/dev/null)
80+
if [ "$COUNT" = "3" ]; then
81+
echo "Full topology discovered (3 instances) after ${i}s"
82+
return 0
83+
fi
84+
# Also try to discover replicas directly
85+
if [ "$i" = "10" ] || [ "$i" = "20" ]; then
86+
curl -s "$ORC_URL/api/discover/mysql2/3306" > /dev/null 2>&1
87+
curl -s "$ORC_URL/api/discover/mysql3/3306" > /dev/null 2>&1
88+
fi
89+
sleep 1
90+
done
91+
echo "WARNING: Only discovered $COUNT instances after 60s"
92+
return 1
93+
}
94+
95+
# Get ProxySQL servers for a hostgroup
96+
proxysql_servers() {
97+
local HG="$1"
98+
docker compose -f tests/functional/docker-compose.yml exec -T proxysql \
99+
mysql -h127.0.0.1 -P6032 -uradmin -pradmin -Nse \
100+
"SELECT hostname, port, status FROM runtime_mysql_servers WHERE hostgroup_id=$HG" 2>/dev/null
101+
}
102+
103+
# Get MySQL read_only status
104+
mysql_read_only() {
105+
local CONTAINER="$1"
106+
docker compose -f tests/functional/docker-compose.yml exec -T "$CONTAINER" \
107+
mysql -uroot -ptestpass -Nse "SELECT @@read_only" 2>/dev/null
108+
}
109+
110+
# Get MySQL replication source
111+
mysql_source_host() {
112+
local CONTAINER="$1"
113+
docker compose -f tests/functional/docker-compose.yml exec -T "$CONTAINER" \
114+
mysql -uroot -ptestpass -Nse "SHOW REPLICA STATUS\G" 2>/dev/null | grep "Source_Host" | awk '{print $2}'
115+
}

‎tests/functional/mysql/init-master.sql‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
-- Orchestrator user with full privileges
2+
CREATE USER IF NOT EXISTS 'orchestrator'@'%' IDENTIFIED BY 'orch_pass';
3+
GRANT ALL PRIVILEGES ON *.* TO 'orchestrator'@'%' WITH GRANT OPTION;
4+
5+
-- Replication user
6+
CREATE USER IF NOT EXISTS 'repl'@'%' IDENTIFIED BY 'repl_pass';
7+
GRANT REPLICATION SLAVE ON *.* TO 'repl'@'%';
8+
9+
FLUSH PRIVILEGES;

‎tests/functional/mysql/init-replica.sql‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
-- Orchestrator user (replicated from master, but define here for safety)
2+
CREATE USER IF NOT EXISTS 'orchestrator'@'%' IDENTIFIED BY 'orch_pass';
3+
GRANT ALL PRIVILEGES ON *.* TO 'orchestrator'@'%' WITH GRANT OPTION;
4+
FLUSH PRIVILEGES;
5+
6+
-- Configure replication to master
7+
CHANGE REPLICATION SOURCE TO
8+
SOURCE_HOST='mysql1',
9+
SOURCE_PORT=3306,
10+
SOURCE_USER='repl',
11+
SOURCE_PASSWORD='repl_pass',
12+
SOURCE_AUTO_POSITION=1;
13+
14+
START REPLICA;

‎tests/functional/mysql/master.cnf‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
[mysqld]
2+
server-id=1
3+
log-bin=mysql-bin
4+
binlog-format=ROW
5+
gtid-mode=ON
6+
enforce-gtid-consistency=ON
7+
log-replica-updates=ON
8+
binlog-row-image=MINIMAL
9+
report-host=mysql1

‎tests/functional/mysql/replica.cnf‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
[mysqld]
2+
server-id=100
3+
log-bin=mysql-bin
4+
binlog-format=ROW
5+
gtid-mode=ON
6+
enforce-gtid-consistency=ON
7+
log-replica-updates=ON
8+
binlog-row-image=MINIMAL
9+
read-only=ON

‎tests/functional/orchestrator-test.conf.json‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
{
2+
"Debug": true,
3+
"ListenAddress": ":3099",
4+
"MySQLTopologyUser": "orchestrator",
5+
"MySQLTopologyPassword": "orch_pass",
6+
"MySQLOrchestratorHost": "",
7+
"MySQLOrchestratorPort": 0,
8+
"BackendDB": "sqlite",
9+
"SQLite3DataFile": "/tmp/orchestrator-test.sqlite3",
10+
"DiscoverByShowSlaveHosts": false,
11+
"InstancePollSeconds": 5,
12+
"RecoveryPeriodBlockSeconds": 10,
13+
"RecoverMasterClusterFilters": [".*"],
14+
"RecoverIntermediateMasterClusterFilters": [".*"],
15+
"AutoPseudoGTID": false,
16+
"DetectClusterAliasQuery": "SELECT CONCAT(@@hostname, ':', @@port)",
17+
"DetectInstanceAliasQuery": "SELECT CONCAT(@@hostname, ':', @@port)",
18+
"ProxySQLAdminAddress": "proxysql",
19+
"ProxySQLAdminPort": 6032,
20+
"ProxySQLAdminUser": "radmin",
21+
"ProxySQLAdminPassword": "radmin",
22+
"ProxySQLWriterHostgroup": 10,
23+
"ProxySQLReaderHostgroup": 20,
24+
"ProxySQLPreFailoverAction": "offline_soft",
25+
"PrometheusEnabled": true
26+
}

0 commit comments

Comments
 (0)