diff --git a/.github/workflows/test-examples.yml b/.github/workflows/test-examples.yml index 078d51cde..147ffc0b9 100644 --- a/.github/workflows/test-examples.yml +++ b/.github/workflows/test-examples.yml @@ -38,6 +38,13 @@ jobs: - name: Install awf globally run: sudo npm link + - name: Build Docker images locally + run: | + # Build agent and squid images from source and tag as GHCR images + # so examples that use default GHCR images get the PR's code + docker build -t ghcr.io/github/gh-aw-firewall/agent:latest containers/agent/ + docker build -t ghcr.io/github/gh-aw-firewall/squid:latest containers/squid/ + - name: Pre-test cleanup run: sudo ./scripts/ci/cleanup.sh diff --git a/containers/agent/entrypoint.sh b/containers/agent/entrypoint.sh index ad454c491..41ec2da6f 100644 --- a/containers/agent/entrypoint.sh +++ b/containers/agent/entrypoint.sh @@ -115,8 +115,28 @@ if [ "${AWF_SSL_BUMP_ENABLED}" = "true" ]; then fi fi -# Setup iptables rules -/usr/local/bin/setup-iptables.sh +# Wait for iptables init container to complete setup +# The awf-iptables-init container shares our network namespace and runs +# setup-iptables.sh, then writes a ready signal file. This ensures the agent +# container NEVER needs NET_ADMIN capability. +echo "[entrypoint] Waiting for iptables initialization from init container..." +INIT_TIMEOUT=300 # 300 * 0.1s = 30 seconds +INIT_ELAPSED=0 +while [ ! -f /tmp/awf-init/ready ]; do + if [ "$INIT_ELAPSED" -ge "$INIT_TIMEOUT" ]; then + echo "[entrypoint][ERROR] Timed out waiting for iptables init container after 30s" + if [ -f /tmp/awf-init/output.log ]; then + echo "[entrypoint] Init container output:" + cat /tmp/awf-init/output.log + else + echo "[entrypoint] No init container output log found" + fi + exit 1 + fi + sleep 0.1 + INIT_ELAPSED=$((INIT_ELAPSED + 1)) +done +echo "[entrypoint] iptables initialization complete" # Run API proxy health checks (verifies credential isolation and connectivity) # This must run AFTER iptables setup (which allows api-proxy traffic) but BEFORE user command @@ -275,15 +295,19 @@ runuser -u awfuser -- git config --global --add safe.directory '*' 2>/dev/null | echo "[entrypoint] ==================================" # Determine which capabilities to drop -# - CAP_NET_ADMIN is always dropped (prevents iptables bypass) +# - CAP_NET_ADMIN is NOT present (never granted to agent container - iptables setup +# is handled by the separate awf-iptables-init container) # - CAP_SYS_CHROOT is dropped when chroot mode is enabled (prevents user code from using chroot) # - CAP_SYS_ADMIN is dropped when chroot mode is enabled (was needed for mounting procfs) if [ "${AWF_CHROOT_ENABLED}" = "true" ]; then - CAPS_TO_DROP="cap_net_admin,cap_sys_chroot,cap_sys_admin" - echo "[entrypoint] Chroot mode enabled - dropping CAP_NET_ADMIN, CAP_SYS_CHROOT, and CAP_SYS_ADMIN" + CAPS_TO_DROP="cap_sys_chroot,cap_sys_admin" + echo "[entrypoint] Chroot mode enabled - dropping CAP_SYS_CHROOT and CAP_SYS_ADMIN" else - CAPS_TO_DROP="cap_net_admin" - echo "[entrypoint] Dropping CAP_NET_ADMIN capability" + # In non-chroot mode, no capabilities need to be dropped + # NET_ADMIN is never granted (init container handles iptables) + # SYS_CHROOT and SYS_ADMIN are only needed/dropped in chroot mode + CAPS_TO_DROP="" + echo "[entrypoint] No capabilities to drop (NET_ADMIN never granted to agent)" fi # Function to unset sensitive tokens from the entrypoint's environment @@ -650,7 +674,12 @@ else # SECURITY: Run agent command in background, then unset tokens from parent shell # This prevents tokens from being accessible via /proc/1/environ after agent starts # The one-shot-token library caches tokens in the agent process, so agent can still read them - capsh --drop=$CAPS_TO_DROP -- -c "exec gosu awfuser $(printf '%q ' "$@")" & + if [ -n "$CAPS_TO_DROP" ]; then + capsh --drop=$CAPS_TO_DROP -- -c "exec gosu awfuser $(printf '%q ' "$@")" & + else + # No capabilities to drop - just switch to unprivileged user + gosu awfuser "$@" & + fi AGENT_PID=$! # Wait for agent to initialize and cache tokens (5 seconds) diff --git a/containers/agent/setup-iptables.sh b/containers/agent/setup-iptables.sh index d54d02507..4e3991f90 100644 --- a/containers/agent/setup-iptables.sh +++ b/containers/agent/setup-iptables.sh @@ -44,13 +44,21 @@ SQUID_PORT="${SQUID_PROXY_PORT:-3128}" echo "[iptables] Squid proxy: ${SQUID_HOST}:${SQUID_PORT}" # Resolve Squid hostname to IP -# Use awk's NR to get first line to avoid host binary dependency in chroot mode -SQUID_IP=$(getent hosts "$SQUID_HOST" | awk 'NR==1 { print $1 }') -if [ -z "$SQUID_IP" ]; then - echo "[iptables] ERROR: Could not resolve Squid proxy hostname: $SQUID_HOST" - exit 1 +# If SQUID_HOST is already a valid IPv4 address, use it directly (no DNS lookup needed). +# This is important for the init container which passes a direct IP via SQUID_PROXY_HOST +# because getent hosts with an IP does a reverse DNS lookup that fails in Docker. +if is_valid_ipv4 "$SQUID_HOST"; then + SQUID_IP="$SQUID_HOST" + echo "[iptables] Squid host is already an IP address: $SQUID_IP" +else + # Use awk's NR to get first line to avoid host binary dependency in chroot mode + SQUID_IP=$(getent hosts "$SQUID_HOST" | awk 'NR==1 { print $1 }') + if [ -z "$SQUID_IP" ]; then + echo "[iptables] ERROR: Could not resolve Squid proxy hostname: $SQUID_HOST" + exit 1 + fi + echo "[iptables] Squid IP resolved to: $SQUID_IP" fi -echo "[iptables] Squid IP resolved to: $SQUID_IP" # Clear existing NAT rules (both IPv4 and IPv6) iptables -t nat -F OUTPUT 2>/dev/null || true diff --git a/scripts/ci/cleanup.sh b/scripts/ci/cleanup.sh index 690a41194..cd921717d 100755 --- a/scripts/ci/cleanup.sh +++ b/scripts/ci/cleanup.sh @@ -12,7 +12,7 @@ echo "===========================================" # First, explicitly remove containers by name (handles orphaned containers) echo "Removing awf containers by name..." -docker rm -f awf-squid awf-agent awf-api-proxy 2>/dev/null || true +docker rm -f awf-squid awf-agent awf-iptables-init awf-api-proxy 2>/dev/null || true # Cleanup diagnostic test containers echo "Stopping docker compose services..." diff --git a/src/docker-manager.test.ts b/src/docker-manager.test.ts index 062387221..ccbb6e04f 100644 --- a/src/docker-manager.test.ts +++ b/src/docker-manager.test.ts @@ -727,11 +727,12 @@ describe('docker-manager', () => { expect(volumes).toContain(`${homeDir}/.copilot:/host${homeDir}/.copilot:rw`); }); - it('should add SYS_CHROOT and SYS_ADMIN capabilities', () => { + it('should add SYS_CHROOT and SYS_ADMIN capabilities but NOT NET_ADMIN', () => { const result = generateDockerCompose(mockConfig, mockNetworkConfig); const agent = result.services.agent; - expect(agent.cap_add).toContain('NET_ADMIN'); + // NET_ADMIN is NOT on the agent - it's on the iptables-init container + expect(agent.cap_add).not.toContain('NET_ADMIN'); expect(agent.cap_add).toContain('SYS_CHROOT'); // SYS_ADMIN is needed to mount procfs at /host/proc for dynamic /proc/self/exe expect(agent.cap_add).toContain('SYS_ADMIN'); @@ -1062,14 +1063,37 @@ describe('docker-manager', () => { expect(depends['squid-proxy'].condition).toBe('service_healthy'); }); - it('should add NET_ADMIN capability to agent for iptables setup', () => { - // NET_ADMIN is required at container start for setup-iptables.sh - // The capability is dropped before user command execution via capsh - // (see containers/agent/entrypoint.sh) + it('should NOT add NET_ADMIN to agent (handled by iptables-init container)', () => { + // NET_ADMIN is NOT granted to the agent container. + // iptables setup is performed by the awf-iptables-init service which shares + // the agent's network namespace. const result = generateDockerCompose(mockConfig, mockNetworkConfig); const agent = result.services.agent; - expect(agent.cap_add).toContain('NET_ADMIN'); + expect(agent.cap_add).not.toContain('NET_ADMIN'); + }); + + it('should add iptables-init service with NET_ADMIN capability', () => { + const result = generateDockerCompose(mockConfig, mockNetworkConfig); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const initService = result.services['iptables-init'] as any; + + expect(initService).toBeDefined(); + expect(initService.container_name).toBe('awf-iptables-init'); + expect(initService.cap_add).toEqual(['NET_ADMIN', 'NET_RAW']); + expect(initService.cap_drop).toEqual(['ALL']); + expect(initService.network_mode).toBe('service:agent'); + expect(initService.depends_on).toEqual({ + 'agent': { condition: 'service_healthy' }, + }); + // Entrypoint is overridden to bypass agent's entrypoint.sh (which has init wait loop) + expect(initService.entrypoint).toEqual(['/bin/bash']); + expect(initService.command).toEqual([ + '-c', + '/usr/local/bin/setup-iptables.sh > /tmp/awf-init/output.log 2>&1 && touch /tmp/awf-init/ready', + ]); + expect(initService.security_opt).toBeUndefined(); + expect(initService.restart).toBe('no'); }); it('should apply container hardening measures', () => { @@ -1419,7 +1443,7 @@ describe('docker-manager', () => { expect(result.services.agent.working_dir).toBe('/custom/workdir'); // Verify other config is still present expect(result.services.agent.container_name).toBe('awf-agent'); - expect(result.services.agent.cap_add).toContain('NET_ADMIN'); + expect(result.services.agent.cap_add).toContain('SYS_CHROOT'); }); it('should handle empty string containerWorkDir by not setting working_dir', () => { @@ -2336,7 +2360,7 @@ describe('docker-manager', () => { expect(mockExecaFn).toHaveBeenCalledWith( 'docker', - ['rm', '-f', 'awf-squid', 'awf-agent', 'awf-api-proxy'], + ['rm', '-f', 'awf-squid', 'awf-agent', 'awf-iptables-init', 'awf-api-proxy'], { reject: false } ); }); diff --git a/src/docker-manager.ts b/src/docker-manager.ts index 65c1afe8d..592abbbcb 100644 --- a/src/docker-manager.ts +++ b/src/docker-manager.ts @@ -543,6 +543,12 @@ export function generateDockerCompose( // Only mount the workspace directory ($GITHUB_WORKSPACE or current working directory) // to prevent access to credential files in $HOME const workspaceDir = process.env.GITHUB_WORKSPACE || process.cwd(); + // Create init-signal directory for iptables init container coordination + const initSignalDir = path.join(config.workDir, 'init-signal'); + if (!fs.existsSync(initSignalDir)) { + fs.mkdirSync(initSignalDir, { recursive: true }); + } + const agentVolumes: string[] = [ // Essential mounts that are always included '/tmp:/tmp:rw', @@ -551,6 +557,8 @@ export function generateDockerCompose( `${workspaceDir}:${workspaceDir}:rw`, // Mount agent logs directory to workDir for persistence `${config.workDir}/agent-logs:${effectiveHome}/.copilot/logs:rw`, + // Init signal volume for iptables init container coordination + `${initSignalDir}:/tmp/awf-init:rw`, ]; // Volume mounts for chroot /host to work properly with host binaries @@ -936,13 +944,15 @@ export function generateDockerCompose( condition: 'service_healthy', }, }, - // NET_ADMIN is required for iptables setup in entrypoint.sh. + // SECURITY: NET_ADMIN is NOT granted to the agent container. + // iptables setup is performed by the awf-iptables-init service which shares + // the agent's network namespace via network_mode: "service:agent". // SYS_CHROOT is required for chroot operations. // SYS_ADMIN is required to mount procfs at /host/proc (required for // dynamic /proc/self/exe resolution needed by .NET CLR and other runtimes). - // Security: All capabilities are dropped before running user commands - // via 'capsh --drop=cap_net_admin,cap_sys_chroot,cap_sys_admin' in entrypoint.sh. - cap_add: ['NET_ADMIN', 'SYS_CHROOT', 'SYS_ADMIN'], + // Security: SYS_CHROOT and SYS_ADMIN are dropped before running user commands + // via 'capsh --drop=cap_sys_chroot,cap_sys_admin' in entrypoint.sh. + cap_add: ['SYS_CHROOT', 'SYS_ADMIN'], // Drop capabilities to reduce attack surface (security hardening) cap_drop: [ 'NET_RAW', // Prevents raw socket creation (iptables bypass attempts) @@ -967,6 +977,17 @@ export function generateDockerCompose( cpu_shares: 1024, // Default CPU share stdin_open: true, tty: config.tty || false, // Use --tty flag, default to false for clean logs + // Healthcheck ensures the agent process is alive and its PID is visible in /proc + // before the iptables-init container tries to join via network_mode: service:agent. + // Without this, there's a race where the init container tries to look up the agent's + // PID in /proc/PID/ns/net before the kernel has made it visible. + healthcheck: { + test: ['CMD-SHELL', 'true'], + interval: '1s', + timeout: '1s', + retries: 3, + start_period: '1s', + }, // Escape $ with $$ for Docker Compose variable interpolation command: ['/bin/bash', '-c', config.agentCommand.replace(/\$/g, '$$$$')], }; @@ -1031,10 +1052,75 @@ export function generateDockerCompose( agentService.image = agentImage; } + // Pre-set API proxy IP in environment before the init container definition. + // The init container's environment object captures values at definition time, + // so AWF_API_PROXY_IP must be set before the init container is defined. + // Without this, the init container gets an empty AWF_API_PROXY_IP and + // setup-iptables.sh never adds ACCEPT rules for the API proxy, blocking connectivity. + if (config.enableApiProxy && networkConfig.proxyIp) { + environment.AWF_API_PROXY_IP = networkConfig.proxyIp; + } + + // SECURITY: iptables init container - sets up NAT rules in a separate container + // that shares the agent's network namespace but NEVER gives NET_ADMIN to the agent. + // This eliminates the window where the agent holds NET_ADMIN during startup. + const iptablesInitService: any = { + container_name: 'awf-iptables-init', + // Share agent's network namespace so iptables rules apply to agent's traffic + network_mode: 'service:agent', + // Only mount the init signal volume and the iptables setup script + volumes: [ + `${initSignalDir}:/tmp/awf-init:rw`, + ], + environment: { + // Pass through environment variables needed by setup-iptables.sh + // IMPORTANT: setup-iptables.sh reads SQUID_PROXY_HOST/PORT (not AWF_ prefixed). + // Use the direct IP address since the init container (network_mode: service:agent) + // may not have DNS resolution for compose service names. + SQUID_PROXY_HOST: `${networkConfig.squidIp}`, + SQUID_PROXY_PORT: String(SQUID_PORT), + AWF_DNS_SERVERS: environment.AWF_DNS_SERVERS || '', + AWF_BLOCKED_PORTS: environment.AWF_BLOCKED_PORTS || '', + AWF_ENABLE_HOST_ACCESS: environment.AWF_ENABLE_HOST_ACCESS || '', + AWF_API_PROXY_IP: environment.AWF_API_PROXY_IP || '', + AWF_DOH_PROXY_IP: environment.AWF_DOH_PROXY_IP || '', + AWF_SSL_BUMP_ENABLED: environment.AWF_SSL_BUMP_ENABLED || '', + AWF_SSL_BUMP_INTERCEPT_PORT: environment.AWF_SSL_BUMP_INTERCEPT_PORT || '', + }, + depends_on: { + 'agent': { + condition: 'service_healthy', + }, + }, + // NET_ADMIN is required for iptables rule manipulation. + // NET_RAW is required by iptables for netfilter socket operations. + cap_add: ['NET_ADMIN', 'NET_RAW'], + cap_drop: ['ALL'], + // Override entrypoint to bypass the agent's entrypoint.sh, which contains an + // "init container wait" loop that would deadlock (the init container waiting for itself). + // The init container only needs to run setup-iptables.sh directly. + entrypoint: ['/bin/bash'], + // Run setup-iptables.sh then signal readiness; log output to shared volume for diagnostics + command: ['-c', '/usr/local/bin/setup-iptables.sh > /tmp/awf-init/output.log 2>&1 && touch /tmp/awf-init/ready'], + // Resource limits (init container exits quickly) + mem_limit: '128m', + pids_limit: 50, + // Restart policy: never restart (init container runs once) + restart: 'no', + }; + + // Use the same image/build as the agent container for the iptables init service + if (agentService.image) { + iptablesInitService.image = agentService.image; + } else if (agentService.build) { + iptablesInitService.build = agentService.build; + } + // API Proxy sidecar service (Node.js) - optionally deployed const services: Record = { 'squid-proxy': squidService, 'agent': agentService, + 'iptables-init': iptablesInitService, }; // Add Node.js API proxy sidecar if enabled @@ -1444,7 +1530,7 @@ export async function startContainers(workDir: string, allowedDomains: string[], // This handles orphaned containers from failed/interrupted previous runs logger.debug('Removing any existing containers with conflicting names...'); try { - await execa('docker', ['rm', '-f', 'awf-squid', 'awf-agent', 'awf-api-proxy'], { + await execa('docker', ['rm', '-f', 'awf-squid', 'awf-agent', 'awf-iptables-init', 'awf-api-proxy'], { reject: false, }); } catch {