diff --git a/containers/cli-proxy/entrypoint.sh b/containers/cli-proxy/entrypoint.sh index 96afd98fd..615e0499f 100644 --- a/containers/cli-proxy/entrypoint.sh +++ b/containers/cli-proxy/entrypoint.sh @@ -57,9 +57,9 @@ export GIT_SSL_CAINFO="${COMBINED_CA}" echo "[cli-proxy] gh CLI configured to route through DIFC proxy at ${GH_HOST}" # Probe external DIFC proxy liveness before serving agent traffic. -# If this fails, keep retries tightly bounded and fail startup early so -# workflows do not enter long in-agent retry loops for connection-refused errors. -MAX_LIVENESS_ATTEMPTS="${AWF_CLI_PROXY_LIVENESS_ATTEMPTS:-2}" +# Retries with exponential backoff to handle transient startup delays. +# Distinct error messages help distinguish "not yet ready" from "unreachable". +MAX_LIVENESS_ATTEMPTS="${AWF_CLI_PROXY_LIVENESS_ATTEMPTS:-10}" LIVENESS_SLEEP_SECONDS="${AWF_CLI_PROXY_LIVENESS_SLEEP_SECONDS:-1}" LIVENESS_TIMEOUT_SECONDS="${AWF_CLI_PROXY_LIVENESS_TIMEOUT_SECONDS:-5}" ATTEMPT=1 @@ -70,16 +70,29 @@ while [ "$ATTEMPT" -le "$MAX_LIVENESS_ATTEMPTS" ]; do break fi PROBE_EXIT=$? + # Classify the failure for clearer diagnostics: + # ECONNREFUSED (exit 7 for curl, or "connection refused" in gh output) → not yet ready + # Timeout (exit 28 for curl, or "context deadline" in gh output) → unreachable / slow + # Other → unknown / auth error + DIAG_TYPE="unknown" + if echo "${PROBE_ERR}" | grep -qiE "connection refused|ECONNREFUSED"; then + DIAG_TYPE="not-yet-ready (ECONNREFUSED)" + elif [ "${PROBE_EXIT}" -eq 124 ] || echo "${PROBE_ERR}" | grep -qiE "timeout|deadline|timed out"; then + DIAG_TYPE="unreachable (timeout)" + fi if [ "$ATTEMPT" -ge "$MAX_LIVENESS_ATTEMPTS" ]; then - echo "[cli-proxy] ERROR: DIFC proxy liveness probe failed for ${GH_HOST} (gh api exit=${PROBE_EXIT})" + echo "[cli-proxy] ERROR: DIFC proxy liveness probe failed for ${GH_HOST} (gh api exit=${PROBE_EXIT}, diagnosis=${DIAG_TYPE})" if [ -n "${PROBE_ERR}" ]; then echo "[cli-proxy] gh api error: ${PROBE_ERR}" fi echo "[cli-proxy] Failing fast to avoid repeated in-agent retries" exit 1 fi - echo "[cli-proxy] DIFC proxy probe failed (attempt ${ATTEMPT}/${MAX_LIVENESS_ATTEMPTS}), retrying in ${LIVENESS_SLEEP_SECONDS}s..." - sleep "${LIVENESS_SLEEP_SECONDS}" + # Exponential backoff: sleep 1, 2, 4, 8 … seconds (capped at 30s) + SLEEP_SECS=$(( LIVENESS_SLEEP_SECONDS * (1 << (ATTEMPT - 1)) )) + if [ "${SLEEP_SECS}" -gt 30 ]; then SLEEP_SECS=30; fi + echo "[cli-proxy] DIFC proxy probe failed (attempt ${ATTEMPT}/${MAX_LIVENESS_ATTEMPTS}, diagnosis=${DIAG_TYPE}), retrying in ${SLEEP_SECS}s..." + sleep "${SLEEP_SECS}" ATTEMPT=$((ATTEMPT + 1)) done diff --git a/containers/cli-proxy/healthcheck.sh b/containers/cli-proxy/healthcheck.sh index ce2ebad84..8ef4e1eac 100644 --- a/containers/cli-proxy/healthcheck.sh +++ b/containers/cli-proxy/healthcheck.sh @@ -4,5 +4,5 @@ # 2. Verifies the DIFC proxy TCP tunnel is still reachable (fail-fast for Docker-unavailable cases) # so that Docker marks the container unhealthy when the external DIFC proxy goes away, # allowing awf's depends_on: service_healthy to propagate the failure to the agent container. -curl -sf --max-time 3 http://localhost:11000/health > /dev/null && \ +curl -sf --max-time 3 http://127.0.0.1:11000/health > /dev/null && \ timeout 3 bash -c "cat < /dev/null > /dev/tcp/${AWF_DIFC_PROXY_HOST:-host.docker.internal}/${AWF_DIFC_PROXY_PORT:-18443}" 2>/dev/null diff --git a/containers/cli-proxy/server.js b/containers/cli-proxy/server.js index 17c30050b..92202c3b2 100644 --- a/containers/cli-proxy/server.js +++ b/containers/cli-proxy/server.js @@ -353,7 +353,12 @@ if (require.main === module) { }); }); - server.listen(CLI_PROXY_PORT, '0.0.0.0', () => { + // Bind on '::' to accept both IPv4 and IPv6 connections (dual-stack). + // On Linux the default net.ipv6only=0 means '::' also accepts IPv4 traffic, + // so this is equivalent to 0.0.0.0 + [::] in one bind call. This prevents + // health-check failures on dual-stack hosts where Docker resolves `localhost` + // to [::1] but a server listening only on 0.0.0.0 would reject that connection. + server.listen(CLI_PROXY_PORT, '::', () => { accessLog({ event: 'server_start', port: CLI_PROXY_PORT, diff --git a/src/services/cli-proxy-service.test.ts b/src/services/cli-proxy-service.test.ts index 27b4bde2e..8ad7e02d2 100644 --- a/src/services/cli-proxy-service.test.ts +++ b/src/services/cli-proxy-service.test.ts @@ -138,7 +138,7 @@ describe('CLI proxy sidecar (external DIFC proxy)', () => { const result = generateDockerCompose(configWithCliProxy, mockNetworkConfigWithCliProxy); const proxy = result.services['cli-proxy']; expect(proxy.healthcheck).toBeDefined(); - expect((proxy.healthcheck as any).test).toEqual(['CMD', 'curl', '-f', 'http://localhost:11000/health']); + expect((proxy.healthcheck as any).test).toEqual(['CMD', 'curl', '-f', 'http://127.0.0.1:11000/health']); }); it('should drop all capabilities from cli-proxy', () => { diff --git a/src/services/cli-proxy-service.ts b/src/services/cli-proxy-service.ts index b5428b567..6ed894416 100644 --- a/src/services/cli-proxy-service.ts +++ b/src/services/cli-proxy-service.ts @@ -81,7 +81,7 @@ export function buildCliProxyService(params: CliProxyServiceParams): CliProxyBui no_proxy: `localhost,127.0.0.1,::1,host.docker.internal`, }, healthcheck: { - test: ['CMD', 'curl', '-f', `http://localhost:${CLI_PROXY_PORT}/health`], + test: ['CMD', 'curl', '-f', `http://127.0.0.1:${CLI_PROXY_PORT}/health`], interval: '5s', timeout: '3s', retries: 5,