From e63e4be87dd212c49831d6cd8d8b7cc013dba89f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 26 Jun 2026 13:43:06 +0000 Subject: [PATCH 1/4] Initial plan From 81be8da17e136c1ed577d4fed32ddae8100f08b5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 26 Jun 2026 14:27:46 +0000 Subject: [PATCH 2/4] Initial plan Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .github/workflows/ace-editor.lock.yml | 4 ++-- .github/workflows/blog-auditor.lock.yml | 4 ++-- .github/workflows/cli-consistency-checker.lock.yml | 4 ++-- .github/workflows/cli-version-checker.lock.yml | 4 ++-- .github/workflows/copilot-pr-merged-report.lock.yml | 4 ++-- .github/workflows/daily-team-evolution-insights.lock.yml | 4 ++-- .github/workflows/dev.lock.yml | 4 ++-- .github/workflows/example-permissions-warning.lock.yml | 4 ++-- .github/workflows/gpclean.lock.yml | 4 ++-- .github/workflows/mcp-inspector.lock.yml | 4 ++-- 10 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ace-editor.lock.yml b/.github/workflows/ace-editor.lock.yml index 2064fd45062..371d60f9ef3 100644 --- a/.github/workflows/ace-editor.lock.yml +++ b/.github/workflows/ace-editor.lock.yml @@ -165,9 +165,9 @@ jobs: const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_aw_info.cjs'); await main(core, context); - name: Enforce strict mode policy - if: ${{ contains(toJSON(vars), '"GH_AW_POLICY_STRICT":') }} + if: ${{ vars.GH_AW_POLICY_STRICT == 'true' }} run: | - echo "::error::GH_AW_POLICY_STRICT is set but this workflow was not compiled in strict mode. Recompile with --strict or strict: true." + echo "::error::GH_AW_POLICY_STRICT=true but this workflow was not compiled in strict mode. Recompile with --strict or strict: true." exit 1 - name: Restore daily AIC usage cache id: restore-daily-aic-cache diff --git a/.github/workflows/blog-auditor.lock.yml b/.github/workflows/blog-auditor.lock.yml index 5aecf272a27..6ce887e0e69 100644 --- a/.github/workflows/blog-auditor.lock.yml +++ b/.github/workflows/blog-auditor.lock.yml @@ -163,9 +163,9 @@ jobs: const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_aw_info.cjs'); await main(core, context); - name: Enforce strict mode policy - if: ${{ contains(toJSON(vars), '"GH_AW_POLICY_STRICT":') }} + if: ${{ vars.GH_AW_POLICY_STRICT == 'true' }} run: | - echo "::error::GH_AW_POLICY_STRICT is set but this workflow was not compiled in strict mode. Recompile with --strict or strict: true." + echo "::error::GH_AW_POLICY_STRICT=true but this workflow was not compiled in strict mode. Recompile with --strict or strict: true." exit 1 - name: Restore daily AIC usage cache id: restore-daily-aic-cache diff --git a/.github/workflows/cli-consistency-checker.lock.yml b/.github/workflows/cli-consistency-checker.lock.yml index ad81c26e5a0..17f192a5bba 100644 --- a/.github/workflows/cli-consistency-checker.lock.yml +++ b/.github/workflows/cli-consistency-checker.lock.yml @@ -155,9 +155,9 @@ jobs: const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_aw_info.cjs'); await main(core, context); - name: Enforce strict mode policy - if: ${{ contains(toJSON(vars), '"GH_AW_POLICY_STRICT":') }} + if: ${{ vars.GH_AW_POLICY_STRICT == 'true' }} run: | - echo "::error::GH_AW_POLICY_STRICT is set but this workflow was not compiled in strict mode. Recompile with --strict or strict: true." + echo "::error::GH_AW_POLICY_STRICT=true but this workflow was not compiled in strict mode. Recompile with --strict or strict: true." exit 1 - name: Restore daily AIC usage cache id: restore-daily-aic-cache diff --git a/.github/workflows/cli-version-checker.lock.yml b/.github/workflows/cli-version-checker.lock.yml index 060fb146e3d..5521b918250 100644 --- a/.github/workflows/cli-version-checker.lock.yml +++ b/.github/workflows/cli-version-checker.lock.yml @@ -160,9 +160,9 @@ jobs: const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_aw_info.cjs'); await main(core, context); - name: Enforce strict mode policy - if: ${{ contains(toJSON(vars), '"GH_AW_POLICY_STRICT":') }} + if: ${{ vars.GH_AW_POLICY_STRICT == 'true' }} run: | - echo "::error::GH_AW_POLICY_STRICT is set but this workflow was not compiled in strict mode. Recompile with --strict or strict: true." + echo "::error::GH_AW_POLICY_STRICT=true but this workflow was not compiled in strict mode. Recompile with --strict or strict: true." exit 1 - name: Restore daily AIC usage cache id: restore-daily-aic-cache diff --git a/.github/workflows/copilot-pr-merged-report.lock.yml b/.github/workflows/copilot-pr-merged-report.lock.yml index 8b275aab671..1412de095a8 100644 --- a/.github/workflows/copilot-pr-merged-report.lock.yml +++ b/.github/workflows/copilot-pr-merged-report.lock.yml @@ -160,9 +160,9 @@ jobs: const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_aw_info.cjs'); await main(core, context); - name: Enforce strict mode policy - if: ${{ contains(toJSON(vars), '"GH_AW_POLICY_STRICT":') }} + if: ${{ vars.GH_AW_POLICY_STRICT == 'true' }} run: | - echo "::error::GH_AW_POLICY_STRICT is set but this workflow was not compiled in strict mode. Recompile with --strict or strict: true." + echo "::error::GH_AW_POLICY_STRICT=true but this workflow was not compiled in strict mode. Recompile with --strict or strict: true." exit 1 - name: Restore daily AIC usage cache id: restore-daily-aic-cache diff --git a/.github/workflows/daily-team-evolution-insights.lock.yml b/.github/workflows/daily-team-evolution-insights.lock.yml index 34daf45a20a..4766f5759ee 100644 --- a/.github/workflows/daily-team-evolution-insights.lock.yml +++ b/.github/workflows/daily-team-evolution-insights.lock.yml @@ -161,9 +161,9 @@ jobs: const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_aw_info.cjs'); await main(core, context); - name: Enforce strict mode policy - if: ${{ contains(toJSON(vars), '"GH_AW_POLICY_STRICT":') }} + if: ${{ vars.GH_AW_POLICY_STRICT == 'true' }} run: | - echo "::error::GH_AW_POLICY_STRICT is set but this workflow was not compiled in strict mode. Recompile with --strict or strict: true." + echo "::error::GH_AW_POLICY_STRICT=true but this workflow was not compiled in strict mode. Recompile with --strict or strict: true." exit 1 - name: Restore daily AIC usage cache id: restore-daily-aic-cache diff --git a/.github/workflows/dev.lock.yml b/.github/workflows/dev.lock.yml index 3364736d814..8212f7b262e 100644 --- a/.github/workflows/dev.lock.yml +++ b/.github/workflows/dev.lock.yml @@ -178,9 +178,9 @@ jobs: const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_aw_info.cjs'); await main(core, context); - name: Enforce strict mode policy - if: ${{ contains(toJSON(vars), '"GH_AW_POLICY_STRICT":') }} + if: ${{ vars.GH_AW_POLICY_STRICT == 'true' }} run: | - echo "::error::GH_AW_POLICY_STRICT is set but this workflow was not compiled in strict mode. Recompile with --strict or strict: true." + echo "::error::GH_AW_POLICY_STRICT=true but this workflow was not compiled in strict mode. Recompile with --strict or strict: true." exit 1 - name: Restore daily AIC usage cache id: restore-daily-aic-cache diff --git a/.github/workflows/example-permissions-warning.lock.yml b/.github/workflows/example-permissions-warning.lock.yml index 917a60564df..582b40ee80a 100644 --- a/.github/workflows/example-permissions-warning.lock.yml +++ b/.github/workflows/example-permissions-warning.lock.yml @@ -154,9 +154,9 @@ jobs: const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_aw_info.cjs'); await main(core, context); - name: Enforce strict mode policy - if: ${{ contains(toJSON(vars), '"GH_AW_POLICY_STRICT":') }} + if: ${{ vars.GH_AW_POLICY_STRICT == 'true' }} run: | - echo "::error::GH_AW_POLICY_STRICT is set but this workflow was not compiled in strict mode. Recompile with --strict or strict: true." + echo "::error::GH_AW_POLICY_STRICT=true but this workflow was not compiled in strict mode. Recompile with --strict or strict: true." exit 1 - name: Restore daily AIC usage cache id: restore-daily-aic-cache diff --git a/.github/workflows/gpclean.lock.yml b/.github/workflows/gpclean.lock.yml index 53269f18b23..dd210b0d32a 100644 --- a/.github/workflows/gpclean.lock.yml +++ b/.github/workflows/gpclean.lock.yml @@ -161,9 +161,9 @@ jobs: const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_aw_info.cjs'); await main(core, context); - name: Enforce strict mode policy - if: ${{ contains(toJSON(vars), '"GH_AW_POLICY_STRICT":') }} + if: ${{ vars.GH_AW_POLICY_STRICT == 'true' }} run: | - echo "::error::GH_AW_POLICY_STRICT is set but this workflow was not compiled in strict mode. Recompile with --strict or strict: true." + echo "::error::GH_AW_POLICY_STRICT=true but this workflow was not compiled in strict mode. Recompile with --strict or strict: true." exit 1 - name: Restore daily AIC usage cache id: restore-daily-aic-cache diff --git a/.github/workflows/mcp-inspector.lock.yml b/.github/workflows/mcp-inspector.lock.yml index 082429d7dfd..ee2e51d5a3c 100644 --- a/.github/workflows/mcp-inspector.lock.yml +++ b/.github/workflows/mcp-inspector.lock.yml @@ -206,9 +206,9 @@ jobs: const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_aw_info.cjs'); await main(core, context); - name: Enforce strict mode policy - if: ${{ contains(toJSON(vars), '"GH_AW_POLICY_STRICT":') }} + if: ${{ vars.GH_AW_POLICY_STRICT == 'true' }} run: | - echo "::error::GH_AW_POLICY_STRICT is set but this workflow was not compiled in strict mode. Recompile with --strict or strict: true." + echo "::error::GH_AW_POLICY_STRICT=true but this workflow was not compiled in strict mode. Recompile with --strict or strict: true." exit 1 - name: Restore daily AIC usage cache id: restore-daily-aic-cache From 572b66a0e7d614b6d8c672f78dd8855a15b65cd6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 26 Jun 2026 14:33:22 +0000 Subject: [PATCH 3/4] fix: treat post-completion MCP relaunch failure as non-fatal; flag removed denied domains as anomalies Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- actions/setup/js/log_parser_bootstrap.cjs | 33 +++++++++ .../setup/js/log_parser_bootstrap.test.cjs | 72 +++++++++++++++++++ pkg/cli/audit_diff.go | 10 +++ pkg/cli/audit_diff_test.go | 34 +++++++++ 4 files changed, 149 insertions(+) diff --git a/actions/setup/js/log_parser_bootstrap.cjs b/actions/setup/js/log_parser_bootstrap.cjs index 00955b6a894..0d4ba36fe5b 100644 --- a/actions/setup/js/log_parser_bootstrap.cjs +++ b/actions/setup/js/log_parser_bootstrap.cjs @@ -72,6 +72,32 @@ async function runLogParser(options) { return count; } + /** + * Checks whether the agent ran at least one turn to completion. + * Used to distinguish a post-completion MCP relaunch failure (non-fatal) + * from a startup failure where the agent never ran (fatal). + * + * Handles both log formats: + * - Legacy format (Codex, Copilot, etc.): { type: "result", num_turns: N } + * - Copilot event format (Claude): { type: "session.result", data: { numTurns: N } } + * + * @param {Array|null|undefined} entries + * @returns {boolean} + */ + function agentRanToCompletion(entries) { + if (!entries || !Array.isArray(entries) || entries.length === 0) { + return false; + } + return entries.some(e => { + if (!e || typeof e !== "object") return false; + // Legacy format + if (e.type === "result" && typeof e.num_turns === "number" && e.num_turns > 0) return true; + // Copilot event format (Claude) + if (e.type === "session.result" && e.data && typeof e.data.numTurns === "number" && e.data.numTurns > 0) return true; + return false; + }); + } + try { const logPath = process.env.GH_AW_AGENT_OUTPUT; if (!logPath) { @@ -309,6 +335,13 @@ async function runLogParser(options) { const failedServers = mcpFailures.join(", "); if (safeOutputEntriesCount > 0) { core.warning(`MCP server(s) failed to launch (${failedServers}), but agent completed with ${safeOutputEntriesCount} safe output ${safeOutputEntriesCount === 1 ? "entry" : "entries"}`); + } else if (agentRanToCompletion(logEntries)) { + // The agent ran turns to completion even though an MCP server failed to launch. + // This is a post-completion relaunch/health-probe failure — the MCP server was + // healthy during execution (the agent used it throughout the run) and the failure + // occurred after the work was done. Treat as non-fatal so genuine task success + // is not masked by a transient infrastructure event. + core.warning(`MCP server(s) failed to launch (${failedServers}), but agent completed turns — treating as non-fatal post-completion relaunch`); } else { core.setFailed(`${ERR_API}: MCP server(s) failed to launch: ${failedServers}`); } diff --git a/actions/setup/js/log_parser_bootstrap.test.cjs b/actions/setup/js/log_parser_bootstrap.test.cjs index fe50fdd57ed..3038a033b0f 100644 --- a/actions/setup/js/log_parser_bootstrap.test.cjs +++ b/actions/setup/js/log_parser_bootstrap.test.cjs @@ -131,6 +131,78 @@ describe("log_parser_bootstrap.cjs", () => { fs.unlinkSync(safeOutputsFile), fs.rmdirSync(tmpDir)); }), + it("should warn (non-fatal) when MCP fails but agent ran turns (legacy result entry)", () => { + const tmpDir = fs.mkdtempSync(path.join(__dirname, "test-")); + const logFile = path.join(tmpDir, "test.log"); + try { + fs.writeFileSync(logFile, "content"); + process.env.GH_AW_AGENT_OUTPUT = logFile; + // No safe outputs — simulates a workflow that uses GitHub MCP directly (e.g. creates a + // discussion) without writing safe-output file entries. + const mockParseLog = vi.fn().mockReturnValue({ + markdown: "## Result\n", + mcpFailures: ["github"], + maxTurnsHit: false, + logEntries: [ + { type: "system", subtype: "init", model: "claude-3-7-sonnet" }, + { type: "assistant", message: { content: [{ type: "text", text: "Analysis complete" }] } }, + { type: "result", num_turns: 34, duration_ms: 60000 }, + ], + }); + runLogParser({ parseLog: mockParseLog, parserName: "TestParser" }); + expect(mockCore.warning).toHaveBeenCalledWith("MCP server(s) failed to launch (github), but agent completed turns — treating as non-fatal post-completion relaunch"); + expect(mockCore.setFailed).not.toHaveBeenCalled(); + } finally { + fs.unlinkSync(logFile); + fs.rmdirSync(tmpDir); + } + }), + it("should warn (non-fatal) when MCP fails but agent ran turns (Copilot event session.result)", () => { + const tmpDir = fs.mkdtempSync(path.join(__dirname, "test-")); + const logFile = path.join(tmpDir, "test.log"); + try { + fs.writeFileSync(logFile, "content"); + process.env.GH_AW_AGENT_OUTPUT = logFile; + // Copilot event format entries (as returned by parse_claude_log.cjs via + // convertLegacyLogEntriesToCopilotEvents): session.result with data.numTurns. + const mockParseLog = vi.fn().mockReturnValue({ + markdown: "## Result\n", + mcpFailures: ["github"], + maxTurnsHit: false, + logEntries: [ + { type: "session.init", data: { model: "claude-opus-4-5" } }, + { type: "assistant.message", data: { content: "Done" } }, + { type: "session.result", data: { numTurns: 34, durationMs: 60000 } }, + ], + }); + runLogParser({ parseLog: mockParseLog, parserName: "Claude" }); + expect(mockCore.warning).toHaveBeenCalledWith("MCP server(s) failed to launch (github), but agent completed turns — treating as non-fatal post-completion relaunch"); + expect(mockCore.setFailed).not.toHaveBeenCalled(); + } finally { + fs.unlinkSync(logFile); + fs.rmdirSync(tmpDir); + } + }), + it("should still fail when MCP fails and agent ran no turns", () => { + const tmpDir = fs.mkdtempSync(path.join(__dirname, "test-")); + const logFile = path.join(tmpDir, "test.log"); + try { + fs.writeFileSync(logFile, "content"); + process.env.GH_AW_AGENT_OUTPUT = logFile; + // logEntries has no result entry — agent never ran any turns (startup failure). + const mockParseLog = vi.fn().mockReturnValue({ + markdown: "## Result\n", + mcpFailures: ["github"], + maxTurnsHit: false, + logEntries: [{ type: "system", subtype: "init" }], + }); + runLogParser({ parseLog: mockParseLog, parserName: "TestParser" }); + expect(mockCore.setFailed).toHaveBeenCalledWith(`${ERR_API}: MCP server(s) failed to launch: github`); + } finally { + fs.unlinkSync(logFile); + fs.rmdirSync(tmpDir); + } + }), it("should handle max-turns limit reached", () => { const tmpDir = fs.mkdtempSync(path.join(__dirname, "test-")), logFile = path.join(tmpDir, "test.log"); diff --git a/pkg/cli/audit_diff.go b/pkg/cli/audit_diff.go index ace68ffb3af..16e9fb11e8c 100644 --- a/pkg/cli/audit_diff.go +++ b/pkg/cli/audit_diff.go @@ -129,6 +129,16 @@ func computeFirewallDiff(run1ID, run2ID int64, run1, run2 *FirewallAnalysis) *Fi Run1Blocked: stats1.Blocked, Run1Status: classifyFirewallDomainStatus(stats1), } + // Anomaly: the removed domain was denied in the base run. This indicates a + // transient firewall block that prevented the agent from reaching an MCP server + // (e.g. awmg-mcpg:8080) — even though the domain is absent from the comparison + // run (and therefore looks "normal"), its prior denial is worth surfacing so + // post-completion relaunch failures are detectable in audit diffs. + if stats1.Blocked > 0 { + entry.IsAnomaly = true + entry.AnomalyNote = "denied in base run — absent from comparison run" + anomalyCount++ + } diff.RemovedDomains = append(diff.RemovedDomains, entry) } else { // Domain exists in both runs - check for changes diff --git a/pkg/cli/audit_diff_test.go b/pkg/cli/audit_diff_test.go index 99feb73f73f..dbdd8c6c1c6 100644 --- a/pkg/cli/audit_diff_test.go +++ b/pkg/cli/audit_diff_test.go @@ -76,6 +76,40 @@ func TestComputeFirewallDiff_RemovedDomains(t *testing.T) { assert.Equal(t, "allowed", diff.RemovedDomains[0].Run1Status, "Domain was allowed in run 1") assert.Equal(t, 8, diff.RemovedDomains[0].Run1Allowed, "Domain had 8 allowed requests") assert.Equal(t, 1, diff.Summary.RemovedDomainCount, "Summary should show 1 removed domain") + // An allowed-only removed domain must NOT be an anomaly. + assert.False(t, diff.RemovedDomains[0].IsAnomaly, "Allowed removed domain should not be an anomaly") + assert.False(t, diff.Summary.HasAnomalies, "No anomalies expected for an allowed removed domain") +} + +// TestComputeFirewallDiff_RemovedDeniedDomain verifies that a domain which was denied in +// the base run but is absent from the comparison run is flagged as an anomaly. This +// covers the false-red scenario where awmg-mcpg:8080 is blocked in the failed run but +// is simply absent (no traffic) in the green run — the block should still be surfaced. +func TestComputeFirewallDiff_RemovedDeniedDomain(t *testing.T) { + run1 := &FirewallAnalysis{ + RequestsByDomain: map[string]DomainRequestStats{ + "api.github.com:443": {Allowed: 10, Blocked: 0}, + "awmg-mcpg:8080": {Allowed: 0, Blocked: 1}, + }, + } + run2 := &FirewallAnalysis{ + RequestsByDomain: map[string]DomainRequestStats{ + "api.github.com:443": {Allowed: 10, Blocked: 0}, + }, + } + + diff := computeFirewallDiff(100, 200, run1, run2) + + assert.Len(t, diff.RemovedDomains, 1, "Should have 1 removed domain") + entry := diff.RemovedDomains[0] + assert.Equal(t, "awmg-mcpg:8080", entry.Domain) + assert.Equal(t, "removed", entry.Status) + assert.Equal(t, "denied", entry.Run1Status, "Domain was denied in run 1") + assert.Equal(t, 1, entry.Run1Blocked, "Domain had 1 blocked request") + assert.True(t, entry.IsAnomaly, "Denied removed domain should be an anomaly") + assert.NotEmpty(t, entry.AnomalyNote, "Anomaly note should be set") + assert.True(t, diff.Summary.HasAnomalies, "Summary should report anomalies") + assert.Equal(t, 1, diff.Summary.AnomalyCount, "Should have 1 anomaly") } func TestComputeFirewallDiff_StatusChanges(t *testing.T) { From 4cf653241bd4f79677c73e881948d137418e460d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 26 Jun 2026 14:37:31 +0000 Subject: [PATCH 4/4] fix: clarify agentRanToCompletion JSDoc to note it checks for at least one turn Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- actions/setup/js/log_parser_bootstrap.cjs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/actions/setup/js/log_parser_bootstrap.cjs b/actions/setup/js/log_parser_bootstrap.cjs index 0d4ba36fe5b..21bb562feda 100644 --- a/actions/setup/js/log_parser_bootstrap.cjs +++ b/actions/setup/js/log_parser_bootstrap.cjs @@ -73,9 +73,13 @@ async function runLogParser(options) { } /** - * Checks whether the agent ran at least one turn to completion. - * Used to distinguish a post-completion MCP relaunch failure (non-fatal) - * from a startup failure where the agent never ran (fatal). + * Returns true if the log entries show the agent ran at least one turn. + * + * "At least one turn" is used (rather than "all work finished") because the + * log only records the turn count, not whether every intended task succeeded. + * The check is sufficient to distinguish a post-completion MCP relaunch + * failure (the agent was already executing) from a startup failure where the + * MCP never launched and the agent ran zero turns. * * Handles both log formats: * - Legacy format (Codex, Copilot, etc.): { type: "result", num_turns: N }