Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 30 additions & 11 deletions actions/setup/js/check_daily_aic_workflow_guardrail.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ const RATE_LIMIT_RESERVE = 100;
const REQUEST_OVERHEAD_BUDGET = MAX_WORKFLOW_RUN_PAGES + 4;
const ESTIMATED_API_OPERATIONS_PER_RUN = 2;
const INTEGER_FORMATTER = new Intl.NumberFormat("en-US");
const DAILY_AIC_GUARDRAIL_DOCS_URL = "https://github.github.com/gh-aw/reference/cost-management/";

/**
* @returns {Promise<import("@actions/artifact").DefaultArtifactClient>}
Expand Down Expand Up @@ -55,6 +56,15 @@ function logDailyGuardrail(message, details) {
core.info(formatDailyGuardrailLogMessage(message, details));
}

/**
* @param {unknown} error
* @returns {boolean}
*/
function isMissingArtifactDownloadError(error) {
const message = getErrorMessage(error).toLowerCase();
return message.includes("unable to download artifact") && message.includes("artifact not found");
}

/**
* @returns {boolean}
*/
Expand Down Expand Up @@ -285,16 +295,16 @@ async function appendDailyAICSummary(workflowName, actorLogin, threshold, counte
*
* Error handling: all GitHub API interactions after the initial guard checks are wrapped
* in a top-level try-catch. Any unexpected error (network failure, permission error, etc.)
* is logged as a warning and the function returns cleanly with `daily_effective_workflow_exceeded`
* is logged as a warning and the function returns cleanly with `daily_ai_credits_exceeded`
* left at its default value of `"false"`. This design ensures the step never fails the
* activation job — a guardrail error results in a safe bypass (agent allowed to run) rather
* than a confusing workflow failure that blocks the agent entirely.
*/
async function main() {
core.setOutput("daily_effective_workflow_exceeded", "false");
core.setOutput("daily_effective_workflow_total_effective_tokens", "");
core.setOutput("daily_effective_workflow_total_ai_credits", "");
core.setOutput("daily_effective_workflow_threshold", "");
core.setOutput("daily_ai_credits_exceeded", "false");
core.setOutput("daily_ai_credits_total_effective_tokens", "");
core.setOutput("daily_ai_credits_total_ai_credits", "");
core.setOutput("daily_ai_credits_threshold", "");
const threshold = parsePositiveCompactNumber(process.env.GH_AW_MAX_DAILY_AI_CREDITS);
if (threshold <= 0) {
return;
Expand All @@ -312,7 +322,7 @@ async function main() {

// Wrap all GitHub API interactions in a top-level try-catch so that transient API
// errors, permission failures, or unexpected exceptions never fail the activation
// job step. A failure here would leave `daily_effective_workflow_exceeded` at its
// job step. A failure here would leave `daily_ai_credits_exceeded` at its
// default "false" value, which is the safe fallback: the agent is allowed to run
// and the guardrail is effectively bypassed for this invocation rather than causing
// a confusing workflow failure.
Expand Down Expand Up @@ -445,13 +455,19 @@ async function main() {
countedRunIds: countedRuns.map(item => item.id),
});
} catch (error) {
if (isMissingArtifactDownloadError(error)) {
logDailyGuardrail("Skipping run because usage artifact was not found at download time", {
runId: run.id,
});
continue;
}
core.warning(`Failed to inspect token usage for run ${run.id}: ${getErrorMessage(error)}`);
}
}

core.setOutput("daily_effective_workflow_total_effective_tokens", String(totalAIC));
core.setOutput("daily_effective_workflow_total_ai_credits", String(totalAIC));
core.setOutput("daily_effective_workflow_threshold", String(threshold));
core.setOutput("daily_ai_credits_total_effective_tokens", String(totalAIC));
core.setOutput("daily_ai_credits_total_ai_credits", String(totalAIC));
core.setOutput("daily_ai_credits_threshold", String(threshold));

/** @type {{candidateRunsCount:number,inspectedRunsCount:number,truncatedByRateLimit:boolean}} */
const summaryMeta = {
Expand Down Expand Up @@ -490,9 +506,11 @@ async function main() {
return;
}

core.setOutput("daily_effective_workflow_exceeded", "true");
core.setOutput("daily_ai_credits_exceeded", "true");
await appendDailyAICSummary(workflowName, actorLogin, threshold, countedRuns, rateLimit, summaryMeta);
core.warning(`Daily workflow AIC guardrail exceeded for ${workflowName}: ${totalAIC}/${threshold}.`);
const exceededMessage = `Daily workflow AIC guardrail exceeded for ${workflowName}: ${totalAIC}/${threshold}. See ${DAILY_AIC_GUARDRAIL_DOCS_URL}`;
core.error(exceededMessage);
core.setFailed(exceededMessage);

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This will skip the conclusion job that files the issue.

} catch (error) {
// Treat any unexpected error as a non-blocking skip so the step never fails the
// activation job. The output stays at the default "false", allowing the agent to
Expand All @@ -505,6 +523,7 @@ module.exports = {
main,
shouldSkipDailyAICGuardrail,
matchesGuardrailArtifactName,
isMissingArtifactDownloadError,
findJSONLFiles,
sumAICFromUsageJSONLFiles,
calculateDailyAICStats,
Expand Down
115 changes: 113 additions & 2 deletions actions/setup/js/check_daily_aic_workflow_guardrail.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,11 @@ describe("check_daily_aic_workflow_guardrail", () => {
expect(exports.matchesGuardrailArtifactName("activation")).toBe(false);
});

it("recognizes artifact-not-found download errors as non-actionable", () => {
expect(exports.isMissingArtifactDownloadError(new Error("Unable to download artifact(s): Artifact not found for name: usage"))).toBe(true);
expect(exports.isMissingArtifactDownloadError(new Error("network timeout"))).toBe(false);
});

it("sums AI Credits across multiple JSONL files and usage attributes", () => {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "daily-guardrail-token-usage-"));
const nestedDir = path.join(tmpDir, "nested");
Expand Down Expand Up @@ -173,7 +178,7 @@ describe("check_daily_aic_workflow_guardrail", () => {

it("main() does not fail the step when GitHub API calls throw", async () => {
// Simulate a scenario where the GitHub API throws during workflow run lookup.
// The step should catch the error and NOT rethrow it, keeping daily_effective_workflow_exceeded at "false".
// The step should catch the error and NOT rethrow it, keeping daily_ai_credits_exceeded at "false".
const coreOutputs = {};
const coreWarnings = [];
const mockCore = {
Expand Down Expand Up @@ -219,7 +224,7 @@ describe("check_daily_aic_workflow_guardrail", () => {
// Should resolve without throwing even though the API calls throw
await expect(exports.main()).resolves.toBeUndefined();
// The default "false" output must be set
expect(coreOutputs["daily_effective_workflow_exceeded"]).toBe("false");
expect(coreOutputs["daily_ai_credits_exceeded"]).toBe("false");
// A warning must be emitted describing the error
expect(coreWarnings.some(w => /unexpected error.*skipped/i.test(w))).toBe(true);
} finally {
Expand Down Expand Up @@ -323,4 +328,110 @@ describe("check_daily_aic_workflow_guardrail", () => {
delete process.env.GH_AW_GITHUB_TOKEN;
}
});

it("main() marks the step as failed with docs link when daily guardrail is exceeded", async () => {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "daily-guardrail-download-"));
fs.writeFileSync(path.join(tmpDir, "usage.jsonl"), JSON.stringify({ aic: 50 }), "utf8");

vi.resetModules();
vi.doMock("@actions/artifact", () => ({
DefaultArtifactClient: class {
async listArtifacts() {
return { artifacts: [{ id: 1, name: "usage" }] };
}
async downloadArtifact() {
return { downloadPath: tmpDir };
}
},
}));

const mod = await import("./check_daily_aic_workflow_guardrail.cjs");
const guardrail = mod.default || mod;

const coreOutputs = {};
const coreErrors = [];
const coreFailures = [];
const mockCore = {
setOutput: (key, value) => {
coreOutputs[key] = value;
},
info: () => {},
warning: () => {},
error: msg => coreErrors.push(msg),
setFailed: msg => coreFailures.push(msg),
summary: {
addDetails: function () {
return this;
},
write: async () => {},
},
};

const mockGithub = {
rest: {
rateLimit: {
get: async () => ({
data: {
resources: {
core: { limit: 5000, remaining: 4995, used: 5, reset: Math.floor(Date.now() / 1000) + 3600 },
},
},
headers: {},
}),
},
actions: {
getWorkflowRun: async () => ({
data: {
workflow_id: 777,
actor: { login: "octocat" },
triggering_actor: { login: "octocat" },
},
headers: {},
}),
listWorkflowRuns: async () => ({
data: {
workflow_runs: [
{
id: 1234,
html_url: "https://example.test/runs/1234",
created_at: new Date().toISOString(),
conclusion: "success",
},
],
},
headers: {},
}),
},
},
};

const mockContext = {
repo: { owner: "test-owner", repo: "test-repo" },
runId: 99,
};

global.core = mockCore;
global.github = mockGithub;
global.context = mockContext;

process.env.GH_AW_WORKFLOW_NAME = "N+1 Crusher";
process.env.GH_AW_MAX_DAILY_AI_CREDITS = "10";
process.env.GH_AW_GITHUB_TOKEN = "fake-token";

try {
await expect(guardrail.main()).resolves.toBeUndefined();
expect(coreOutputs["daily_ai_credits_exceeded"]).toBe("true");
expect(coreErrors[0]).toContain("Daily workflow AIC guardrail exceeded for N+1 Crusher: 50/10.");
expect(coreErrors[0]).toContain("https://github.github.com/gh-aw/reference/cost-management/");
expect(coreFailures[0]).toBe(coreErrors[0]);
} finally {
vi.doUnmock("@actions/artifact");
delete global.core;
delete global.github;
delete global.context;
delete process.env.GH_AW_WORKFLOW_NAME;
delete process.env.GH_AW_MAX_DAILY_AI_CREDITS;
delete process.env.GH_AW_GITHUB_TOKEN;
}
});
});
12 changes: 6 additions & 6 deletions actions/setup/js/handle_agent_failure.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,7 @@ function buildFailureMatchCategories(options) {
if (options.hasAppTokenMintingFailed) categories.push("app_token_minting_failed");
if (options.hasLockdownCheckFailed) categories.push("lockdown_check_failed");
if (options.hasStaleLockFileFailed) categories.push("stale_lock_file_failed");
if (options.hasDailyAICExceeded) categories.push("daily_effective_workflow_exceeded");
if (options.hasDailyAICExceeded) categories.push("daily_ai_credits_exceeded");

if (options.agentConclusion === "failure" && !options.isTimedOut) {
categories.push("agent_failure");
Expand Down Expand Up @@ -2323,9 +2323,9 @@ async function main() {
// stored in the compiled .lock.yml no longer matches the source .md file.
// The agent is skipped in this case; the conclusion job runs to surface remediation guidance.
const hasStaleLockFileFailed = process.env.GH_AW_STALE_LOCK_FILE_FAILED === "true";
const hasDailyAICExceeded = process.env.GH_AW_DAILY_EFFECTIVE_WORKFLOW_EXCEEDED === "true";
const dailyAICTotal = process.env.GH_AW_DAILY_EFFECTIVE_WORKFLOW_TOTAL_EFFECTIVE_TOKENS || "";
const dailyAICThreshold = process.env.GH_AW_DAILY_EFFECTIVE_WORKFLOW_THRESHOLD || "";
const hasDailyAICExceeded = process.env.GH_AW_DAILY_AI_CREDITS_EXCEEDED === "true";
const dailyAICTotal = process.env.GH_AW_DAILY_AI_CREDITS_TOTAL_EFFECTIVE_TOKENS || "";
const dailyAICThreshold = process.env.GH_AW_DAILY_AI_CREDITS_THRESHOLD || "";
// Cache-memory availability flag — set when cache-memory is configured for the workflow.
// Used to detect cache-miss misconfigurations reported by the agent.
const cacheMemoryEnabled = process.env.GH_AW_CACHE_MEMORY_ENABLED === "true";
Expand Down Expand Up @@ -2827,7 +2827,7 @@ async function main() {
app_token_minting_failed_context: appTokenMintingFailedContext,
lockdown_check_failed_context: lockdownCheckFailedContext,
stale_lock_file_failed_context: staleLockFileFailedContext,
daily_effective_workflow_exceeded_context: dailyAICExceededContext,
daily_ai_credits_exceeded_context: dailyAICExceededContext,
};

// Render the comment template
Expand Down Expand Up @@ -3054,7 +3054,7 @@ async function main() {
app_token_minting_failed_context: appTokenMintingFailedContext,
lockdown_check_failed_context: lockdownCheckFailedContext,
stale_lock_file_failed_context: staleLockFileFailedContext,
daily_effective_workflow_exceeded_context: dailyAICExceededContext,
daily_ai_credits_exceeded_context: dailyAICExceededContext,
};

// Render the issue template
Expand Down
2 changes: 1 addition & 1 deletion actions/setup/md/agent_failure_comment.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
Agent job [{run_id}]({run_url}) failed.

{secret_verification_context}{credential_auth_error_context}{inference_access_error_context}{mcp_policy_error_context}{model_not_supported_error_context}{ai_credits_rate_limit_error_context}{app_token_minting_failed_context}{lockdown_check_failed_context}{stale_lock_file_failed_context}{daily_effective_workflow_exceeded_context}{assignment_errors_context}{assign_copilot_failure_context}{create_discussion_errors_context}{code_push_failure_context}{repo_memory_validation_context}{push_repo_memory_failure_context}{missing_data_context}{missing_tool_context}{permission_denied_context}{tool_denials_exceeded_context}{report_incomplete_context}{missing_safe_outputs_context}{engine_failure_context}{timeout_context}{fork_context}
{secret_verification_context}{credential_auth_error_context}{inference_access_error_context}{mcp_policy_error_context}{model_not_supported_error_context}{ai_credits_rate_limit_error_context}{app_token_minting_failed_context}{lockdown_check_failed_context}{stale_lock_file_failed_context}{daily_ai_credits_exceeded_context}{assignment_errors_context}{assign_copilot_failure_context}{create_discussion_errors_context}{code_push_failure_context}{repo_memory_validation_context}{push_repo_memory_failure_context}{missing_data_context}{missing_tool_context}{permission_denied_context}{tool_denials_exceeded_context}{report_incomplete_context}{missing_safe_outputs_context}{engine_failure_context}{timeout_context}{fork_context}
2 changes: 1 addition & 1 deletion actions/setup/md/agent_failure_issue.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
**Branch:** {branch}
**Run:** {run_url}{pull_request_info}

{secret_verification_context}{credential_auth_error_context}{inference_access_error_context}{mcp_policy_error_context}{model_not_supported_error_context}{ai_credits_rate_limit_error_context}{app_token_minting_failed_context}{lockdown_check_failed_context}{stale_lock_file_failed_context}{daily_effective_workflow_exceeded_context}{assignment_errors_context}{assign_copilot_failure_context}{create_discussion_errors_context}{code_push_failure_context}{repo_memory_validation_context}{push_repo_memory_failure_context}{missing_data_context}{missing_tool_context}{permission_denied_context}{tool_denials_exceeded_context}{report_incomplete_context}{missing_safe_outputs_context}{engine_failure_context}{timeout_context}{fork_context}
{secret_verification_context}{credential_auth_error_context}{inference_access_error_context}{mcp_policy_error_context}{model_not_supported_error_context}{ai_credits_rate_limit_error_context}{app_token_minting_failed_context}{lockdown_check_failed_context}{stale_lock_file_failed_context}{daily_ai_credits_exceeded_context}{assignment_errors_context}{assign_copilot_failure_context}{create_discussion_errors_context}{code_push_failure_context}{repo_memory_validation_context}{push_repo_memory_failure_context}{missing_data_context}{missing_tool_context}{permission_denied_context}{tool_denials_exceeded_context}{report_incomplete_context}{missing_safe_outputs_context}{engine_failure_context}{timeout_context}{fork_context}

### Action Required

Expand Down
2 changes: 1 addition & 1 deletion actions/setup/md/daily_workflow_aic_exceeded.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ Commit and push the updated `.lock.yml` file.

The `max-daily-ai-credits` frontmatter option sets a per-workflow spending cap measured in *AI Credits* across the 24-hour window before the current run. The cap is scoped to the repository and workflow — it aggregates usage across all runs of this workflow regardless of who triggered them.

When the aggregated AI Credits usage across all completed runs of this workflow in the last 24 hours exceeds the threshold, the activation job sets the `daily_effective_workflow_exceeded` output to `true` and the agent job is skipped for that run. The conclusion job still runs and creates this report.
When the aggregated AI Credits usage across all completed runs of this workflow in the last 24 hours exceeds the threshold, the activation job sets the `daily_ai_credits_exceeded` output to `true` and the agent job is skipped for that run. The conclusion job still runs and creates this report.

The guardrail is evaluated at activation time, not retrospectively, so a single very large run that pushes usage over the threshold only blocks *subsequent* runs in the same window — it does not cancel a run that is already in progress.

Expand Down
6 changes: 3 additions & 3 deletions pkg/workflow/compiler_activation_job_builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -197,9 +197,9 @@ func (c *Compiler) addActivationFeedbackAndValidationSteps(ctx *activationJobBui
c.maybeAddActivationAppTokenMintStep(ctx)
if hasMaxDailyAICGuardrail(data) {
ctx.steps = append(ctx.steps, c.buildActivationDailyAICGuardrailStep(data)...)
ctx.outputs["daily_effective_workflow_exceeded"] = "${{ steps.daily-effective-workflow-guardrail.outputs.daily_effective_workflow_exceeded == 'true' }}"
ctx.outputs["daily_effective_workflow_total_effective_tokens"] = "${{ steps.daily-effective-workflow-guardrail.outputs.daily_effective_workflow_total_effective_tokens || '' }}"
ctx.outputs["daily_effective_workflow_threshold"] = "${{ steps.daily-effective-workflow-guardrail.outputs.daily_effective_workflow_threshold || '' }}"
ctx.outputs["daily_ai_credits_exceeded"] = "${{ steps.daily-effective-workflow-guardrail.outputs.daily_ai_credits_exceeded == 'true' }}"
ctx.outputs["daily_ai_credits_total_effective_tokens"] = "${{ steps.daily-effective-workflow-guardrail.outputs.daily_ai_credits_total_effective_tokens || '' }}"
ctx.outputs["daily_ai_credits_threshold"] = "${{ steps.daily-effective-workflow-guardrail.outputs.daily_ai_credits_threshold || '' }}"
}
c.addActivationReactionStep(ctx)
c.addActivationSecretValidationStep(ctx)
Expand Down
2 changes: 1 addition & 1 deletion pkg/workflow/compiler_main_job.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ func (c *Compiler) buildMainJob(data *WorkflowData, activationJobCreated bool) (
// we keep the condition on the agent job
}
if activationJobCreated && hasMaxDailyAICGuardrail(data) {
guard := &ExpressionNode{Expression: fmt.Sprintf("needs.%s.outputs.daily_effective_workflow_exceeded != 'true'", constants.ActivationJobName)}
guard := &ExpressionNode{Expression: fmt.Sprintf("needs.%s.outputs.daily_ai_credits_exceeded != 'true'", constants.ActivationJobName)}
if jobCondition == "" {
jobCondition = RenderCondition(guard)
} else {
Expand Down
Loading