Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions actions/setup/js/messages.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ describe("messages.cjs", () => {
delete process.env.GH_AW_WORKFLOW_ID;
delete process.env.GH_AW_EFFECTIVE_TOKENS;
delete process.env.GH_AW_AIC;
delete process.env.GH_AW_AGENT_AIC;
delete process.env.GH_AW_THREAT_DETECTION_AIC;
delete process.env.GH_AW_DETECTION_CONCLUSION;
delete process.env.GH_AW_DETECTION_REASON;
// Point GH_AW_PROMPTS_DIR to the source md/ directory so getPromptPath()
Expand Down Expand Up @@ -410,6 +412,20 @@ describe("messages.cjs", () => {
expect(result).toBe("> Generated by [Test Workflow](https://github.com/test/repo/actions/runs/123) · 0.125 AIC");
});

it("should render separate agent and threat-detection AI Credits entries in the default footer", async () => {
process.env.GH_AW_AGENT_AIC = "0.125";
process.env.GH_AW_THREAT_DETECTION_AIC = "0.025";

const { getFooterMessage } = await import("./messages.cjs");

const result = getFooterMessage({
workflowName: "Test Workflow",
runUrl: "https://github.com/test/repo/actions/runs/123",
});

expect(result).toBe("> Generated by [Test Workflow](https://github.com/test/repo/actions/runs/123) · agent 0.125 AIC · threat-detection 0.025 AIC");
});

it("should not include effective tokens when GH_AW_EFFECTIVE_TOKENS is not set and not passed in context", async () => {
delete process.env.GH_AW_EFFECTIVE_TOKENS;

Expand Down Expand Up @@ -471,6 +487,23 @@ describe("messages.cjs", () => {
expect(result).toBe("> Custom: [Test Workflow](https://github.com/test/repo/actions/runs/123) · 1.25 AIC");
});

it("should expose separate AI Credits entries in custom footer templates", async () => {
process.env.GH_AW_AGENT_AIC = "1.25";
process.env.GH_AW_THREAT_DETECTION_AIC = "0.25";
process.env.GH_AW_SAFE_OUTPUT_MESSAGES = JSON.stringify({
footer: "> Custom: [{workflow_name}]({run_url}){ai_credits_suffix}",
});

const { getFooterMessage } = await import("./messages.cjs");

const result = getFooterMessage({
workflowName: "Test Workflow",
runUrl: "https://github.com/test/repo/actions/runs/123",
});

expect(result).toBe("> Custom: [Test Workflow](https://github.com/test/repo/actions/runs/123) · agent 1.25 AIC · threat-detection 0.25 AIC");
});

it("should include AI Credits next to effective_tokens_suffix in generated custom footers", async () => {
process.env.GH_AW_EFFECTIVE_TOKENS = "5000";
process.env.GH_AW_AIC = "1.25";
Expand Down
84 changes: 74 additions & 10 deletions actions/setup/js/messages_footer.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -53,17 +53,65 @@ function getEffectiveTokensFromEnv(modelName) {
}

/**
* Read AI Credits from GH_AW_AIC and return the raw value, formatted value, and suffix.
* @returns {{ aiCredits: number|undefined, aiCreditsFormatted: string|undefined, aiCreditsSuffix: string }}
* @param {string|undefined} raw
* @returns {number|undefined}
*/
function getAICFromEnv() {
const raw = process.env.GH_AW_AIC;
function parsePositiveAIC(raw) {
const parsed = raw ? Number.parseFloat(raw) : NaN;
if (Number.isFinite(parsed) && parsed > 0) {
const aiCreditsFormatted = formatAIC(parsed);
return { aiCredits: parsed, aiCreditsFormatted, aiCreditsSuffix: aiCreditsFormatted ? ` · ${aiCreditsFormatted} AIC` : "" };
}
return { aiCredits: undefined, aiCreditsFormatted: undefined, aiCreditsSuffix: "" };
return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
}

/**
* @param {string} label
* @param {number|undefined} value
* @returns {{ value: number|undefined, formatted: string|undefined, suffix: string }}
*/
function buildAICEntry(label, value) {
const formatted = typeof value === "number" ? formatAIC(value) : undefined;
return {
value,
formatted,
suffix: formatted ? ` · ${label} ${formatted} AIC` : "",
};
}

/**
* Read AI Credits from the environment and return the total, separate entries when
* threat detection consumed credits, and the pre-formatted footer suffix.
* @returns {{
* aiCredits: number|undefined,
* aiCreditsFormatted: string|undefined,
* aiCreditsSuffix: string,
* agentAiCredits: number|undefined,
* agentAiCreditsFormatted: string|undefined,
* agentAiCreditsSuffix: string,
* threatDetectionAiCredits: number|undefined,
* threatDetectionAiCreditsFormatted: string|undefined,
* threatDetectionAiCreditsSuffix: string
* }}
*/
function getAICFromEnv() {
const totalAIC = parsePositiveAIC(process.env.GH_AW_AIC);
const agentAIC = parsePositiveAIC(process.env.GH_AW_AGENT_AIC);
const threatDetectionAIC = parsePositiveAIC(process.env.GH_AW_THREAT_DETECTION_AIC);
const agentEntry = buildAICEntry("agent", agentAIC);
const threatDetectionEntry = buildAICEntry("threat-detection", threatDetectionAIC);
const useBreakdown = threatDetectionEntry.suffix.length > 0;
const aiCredits = useBreakdown ? (agentAIC || 0) + (threatDetectionAIC || 0) : typeof totalAIC === "number" ? totalAIC : agentAIC;
const aiCreditsFormatted = typeof aiCredits === "number" ? formatAIC(aiCredits) : undefined;
const aiCreditsSuffix = useBreakdown ? `${agentEntry.suffix}${threatDetectionEntry.suffix}` : aiCreditsFormatted ? ` · ${aiCreditsFormatted} AIC` : "";

Comment on lines +93 to +103
return {
aiCredits,
aiCreditsFormatted,
aiCreditsSuffix,
agentAiCredits: agentEntry.value,
agentAiCreditsFormatted: agentEntry.formatted,
agentAiCreditsSuffix: agentEntry.suffix,
threatDetectionAiCredits: threatDetectionEntry.value,
threatDetectionAiCreditsFormatted: threatDetectionEntry.formatted,
threatDetectionAiCreditsSuffix: threatDetectionEntry.suffix,
};
}

/**
Expand Down Expand Up @@ -122,7 +170,17 @@ function getFooterMessage(ctx) {
// over GH_AW_ENGINE_MODEL, which may be a user-supplied alias (e.g. "agent").
const resolvedModelName = ctx.model || resolveActualModelName();
const { effectiveTokens: envEffectiveTokens, effectiveTokensFormatted: envEffectiveTokensFormatted, effectiveTokensSuffix: envEffectiveTokensSuffix } = getEffectiveTokensFromEnv(resolvedModelName);
const { aiCredits: envAIC, aiCreditsFormatted: envAICFormatted, aiCreditsSuffix: envAICSuffix } = getAICFromEnv();
const {
aiCredits: envAIC,
aiCreditsFormatted: envAICFormatted,
aiCreditsSuffix: envAICSuffix,
agentAiCredits,
agentAiCreditsFormatted,
agentAiCreditsSuffix,
threatDetectionAiCredits,
threatDetectionAiCreditsFormatted,
threatDetectionAiCreditsSuffix,
} = getAICFromEnv();
const effectiveTokens = ctx.effectiveTokens ?? envEffectiveTokens;
const aiCredits = ctx.aiCredits ?? envAIC;

Expand Down Expand Up @@ -172,6 +230,12 @@ function getFooterMessage(ctx) {
effectiveTokensSuffix: finalEffectiveTokensSuffix,
aiCreditsFormatted,
aiCreditsSuffix,
agentAiCredits,
agentAiCreditsFormatted,
agentAiCreditsSuffix,
threatDetectionAiCredits,
threatDetectionAiCreditsFormatted,
threatDetectionAiCreditsSuffix,
});

// Use custom footer template if configured (no automatic suffix appended)
Expand Down
14 changes: 13 additions & 1 deletion actions/setup/js/parse_token_usage.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ const TOKEN_USAGE_AUDIT_PATH = "/tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy
const TOKEN_USAGE_PATH = "/tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl";
const TOKEN_USAGE_PATHS = [TOKEN_USAGE_AUDIT_PATH, TOKEN_USAGE_PATH];
const AGENT_USAGE_PATH = "/tmp/gh-aw/agent_usage.json";
const DEFAULT_SUMMARY_TITLE = "Token Usage";

/**
* Returns readable, non-empty token usage files, skipping paths that error.
Expand Down Expand Up @@ -82,6 +83,15 @@ function readDedupedTokenUsage(paths) {
return dedupedLines.join("\n");
}

/**
* Returns the token usage summary title for the current job.
* @returns {string}
*/
function getSummaryTitle() {
const title = process.env.GH_AW_TOKEN_USAGE_SUMMARY_TITLE;
return title && title.trim() ? title.trim() : DEFAULT_SUMMARY_TITLE;
}

/**
* Main function to parse token usage and write the step summary.
*/
Expand All @@ -104,7 +114,7 @@ async function main() {

const markdown = generateTokenUsageSummary(summary);
if (markdown.length > 0) {
core.summary.addDetails("Token Usage", "\n\n" + markdown);
core.summary.addDetails(getSummaryTitle(), "\n\n" + markdown);
}

await core.summary.write();
Expand Down Expand Up @@ -162,10 +172,12 @@ if (typeof module !== "undefined" && module.exports) {
getReadableTokenUsagePaths,
extractRequestId,
readDedupedTokenUsage,
getSummaryTitle,
TOKEN_USAGE_AUDIT_PATH,
TOKEN_USAGE_PATH,
TOKEN_USAGE_PATHS,
AGENT_USAGE_PATH,
DEFAULT_SUMMARY_TITLE,
};
}

Expand Down
41 changes: 40 additions & 1 deletion actions/setup/js/parse_token_usage.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ const fs = require("fs");
const path = require("path");
const os = require("os");

const { main, getReadableTokenUsagePaths, extractRequestId, readDedupedTokenUsage, TOKEN_USAGE_AUDIT_PATH, TOKEN_USAGE_PATH, TOKEN_USAGE_PATHS, AGENT_USAGE_PATH } = require("./parse_token_usage.cjs");
const { main, getReadableTokenUsagePaths, extractRequestId, readDedupedTokenUsage, getSummaryTitle, TOKEN_USAGE_AUDIT_PATH, TOKEN_USAGE_PATH, TOKEN_USAGE_PATHS, AGENT_USAGE_PATH, DEFAULT_SUMMARY_TITLE } = require("./parse_token_usage.cjs");

describe("parse_token_usage", () => {
const singleEntry = JSON.stringify({
Expand Down Expand Up @@ -39,6 +39,10 @@ describe("parse_token_usage", () => {
test("AGENT_USAGE_PATH points to agent_usage.json", () => {
expect(AGENT_USAGE_PATH).toBe("/tmp/gh-aw/agent_usage.json");
});

test("DEFAULT_SUMMARY_TITLE points to Token Usage", () => {
expect(DEFAULT_SUMMARY_TITLE).toBe("Token Usage");
});
});

describe("main function", () => {
Expand All @@ -51,6 +55,7 @@ describe("parse_token_usage", () => {

beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "parse-token-usage-test-"));
delete process.env.GH_AW_TOKEN_USAGE_SUMMARY_TITLE;

mockCore = {
info: vi.fn(),
Expand Down Expand Up @@ -159,6 +164,30 @@ describe("parse_token_usage", () => {
expect(mockCore.info).toHaveBeenCalledWith(expect.stringContaining("Token usage summary appended"));
});

test("uses custom summary title when configured", async () => {
process.env.GH_AW_TOKEN_USAGE_SUMMARY_TITLE = "Threat Detection Token Usage";

fs.existsSync = vi.fn(p => {
if (p === TOKEN_USAGE_PATH) return true;
if (p === TOKEN_USAGE_AUDIT_PATH) return false;
return originalExistsSync(p);
});
fs.statSync = vi.fn(p => {
if (p === TOKEN_USAGE_PATH) return { size: singleEntry.length };
if (p === TOKEN_USAGE_AUDIT_PATH) return { size: 0 };
return originalStatSync(p);
});
fs.readFileSync = vi.fn((p, enc) => {
if (p === TOKEN_USAGE_PATH) return singleEntry;
if (p === TOKEN_USAGE_AUDIT_PATH) return "";
return originalReadFileSync(p, enc);
});

await main();

expect(mockCore.summary.addDetails).toHaveBeenCalledWith("Threat Detection Token Usage", expect.stringContaining("| Alias |"));
});

test("writes agent_usage.json with aggregated token totals including effective_tokens and primary_model", async () => {
const agentUsageFile = path.join(tmpDir, "agent_usage.json");

Expand Down Expand Up @@ -456,5 +485,15 @@ describe("parse_token_usage", () => {
expect(deduped).toContain('"request_id":"req-3"');
expect(deduped.match(/"request_id":"req-1"/g)).toHaveLength(1);
});

test("getSummaryTitle returns trimmed env title", () => {
process.env.GH_AW_TOKEN_USAGE_SUMMARY_TITLE = " Threat Detection Token Usage ";
expect(getSummaryTitle()).toBe("Threat Detection Token Usage");
});

test("getSummaryTitle falls back to default title", () => {
delete process.env.GH_AW_TOKEN_USAGE_SUMMARY_TITLE;
expect(getSummaryTitle()).toBe("Token Usage");
});
});
});
2 changes: 2 additions & 0 deletions pkg/workflow/compiler_safe_outputs_job.go
Original file line number Diff line number Diff line change
Expand Up @@ -691,6 +691,7 @@ func (c *Compiler) buildJobLevelSafeOutputEnvVars(data *WorkflowData, workflowID
// An empty/missing value is handled gracefully by getEffectiveTokensFromEnv() in messages_footer.cjs.
envVars["GH_AW_EFFECTIVE_TOKENS"] = fmt.Sprintf("${{ needs.%s.outputs.effective_tokens }}", constants.AgentJobName)
envVars["GH_AW_AIC"] = fmt.Sprintf("${{ needs.%s.outputs.aic }}", constants.AgentJobName)
envVars["GH_AW_AGENT_AIC"] = fmt.Sprintf("${{ needs.%s.outputs.aic }}", constants.AgentJobName)

// Add slash command metadata so safe output handlers can render run-again footer hints.
if len(data.Command) > 0 {
Expand Down Expand Up @@ -735,6 +736,7 @@ func (c *Compiler) buildJobLevelSafeOutputEnvVars(data *WorkflowData, workflowID
if IsDetectionJobEnabled(data.SafeOutputs) {
envVars["GH_AW_DETECTION_CONCLUSION"] = fmt.Sprintf("${{ needs.%s.outputs.detection_conclusion }}", constants.DetectionJobName)
envVars["GH_AW_DETECTION_REASON"] = fmt.Sprintf("${{ needs.%s.outputs.detection_reason }}", constants.DetectionJobName)
envVars["GH_AW_THREAT_DETECTION_AIC"] = fmt.Sprintf("${{ needs.%s.outputs.aic }}", constants.DetectionJobName)
}

return envVars
Expand Down
18 changes: 18 additions & 0 deletions pkg/workflow/detection_success_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,9 @@ Create an issue.
if !strings.Contains(yaml, "detection_reason:") {
t.Error("Detection job missing detection_reason output")
}
if !strings.Contains(yaml, "aic:") {
t.Error("Detection job missing aic output")
}
Comment on lines +68 to +70

// Check that the detection conclusion step has GH_AW_DETECTION_CONTINUE_ON_ERROR env var
if !strings.Contains(detectionSection, "GH_AW_DETECTION_CONTINUE_ON_ERROR:") {
Expand All @@ -83,6 +86,15 @@ Create an issue.
if !strings.Contains(detectionSection, "require('${{ runner.temp }}/gh-aw/actions/parse_threat_detection_results.cjs')") {
t.Error("Detection conclusion step doesn't use require to load parse_threat_detection_results.cjs")
}
if !strings.Contains(detectionSection, "id: parse_detection_token_usage") {
t.Error("Detection job missing parse_detection_token_usage step")
}
if !strings.Contains(detectionSection, "GH_AW_TOKEN_USAGE_SUMMARY_TITLE: Threat Detection Token Usage") {
t.Error("Detection token usage step missing threat detection summary title")
}
if !strings.Contains(detectionSection, "require('${{ runner.temp }}/gh-aw/actions/parse_token_usage.cjs')") {
t.Error("Detection token usage step doesn't use require to load parse_token_usage.cjs")
}

// Check that setupGlobals is called
if !strings.Contains(yaml, "setupGlobals(core, github, context, exec, io, getOctokit)") {
Expand Down Expand Up @@ -147,6 +159,12 @@ Create outputs.
if !strings.Contains(yaml, "needs.detection.result == 'success'") {
t.Error("Safe output jobs don't check detection result via detection job result")
}
if !strings.Contains(yaml, "GH_AW_AGENT_AIC: ${{ needs.agent.outputs.aic }}") {
t.Error("Safe output jobs should receive agent AI Credits separately")
}
if !strings.Contains(yaml, "GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }}") {
t.Error("Safe output jobs should receive threat-detection AI Credits separately")
}
}

// TestDetectionRunsStepInConclusionJob verifies that when threat detection is enabled,
Expand Down
29 changes: 27 additions & 2 deletions pkg/workflow/threat_detection.go
Original file line number Diff line number Diff line change
Expand Up @@ -307,10 +307,13 @@ func (c *Compiler) buildDetectionJobSteps(data *WorkflowData) []string {
steps = append(steps, c.buildCustomThreatDetectionSteps(data.SafeOutputs.ThreatDetection.PostSteps)...)
}

// Step 9: Upload detection-artifact
// Step 9: Parse threat-detection token usage for step summary and downstream footer rendering.
steps = append(steps, c.buildDetectionTokenUsageSummaryStep(data)...)

// Step 10: Upload detection-artifact
steps = append(steps, c.buildUploadDetectionLogStep(data)...)

// Step 10: Parse results, log extensively, and set job conclusion (single JS step)
// Step 11: Parse results, log extensively, and set job conclusion (single JS step)
steps = append(steps, c.buildDetectionConclusionStep(data)...)

threatLog.Printf("Generated %d detection job step lines", len(steps))
Expand Down Expand Up @@ -505,6 +508,27 @@ func (c *Compiler) buildDetectionConclusionStep(data *WorkflowData) []string {
return steps
}

// buildDetectionTokenUsageSummaryStep creates a step that parses threat-detection
// firewall token usage, appends a separate table to the detection job summary,
// and exposes AI Credits for downstream jobs.
func (c *Compiler) buildDetectionTokenUsageSummaryStep(data *WorkflowData) []string {
return []string{
" - name: Parse threat detection token usage for step summary\n",
" id: parse_detection_token_usage\n",
" if: always()\n",
" continue-on-error: true\n",
fmt.Sprintf(" uses: %s\n", getCachedActionPin("actions/github-script", data)),
" env:\n",
" GH_AW_TOKEN_USAGE_SUMMARY_TITLE: Threat Detection Token Usage\n",
" with:\n",
" script: |\n",
" const { setupGlobals } = require('" + SetupActionDestination + "/setup_globals.cjs');\n",
" setupGlobals(core, github, context, exec, io, getOctokit);\n",
" const { main } = require('" + SetupActionDestination + "/parse_token_usage.cjs');\n",
" await main();\n",
}
}

// buildThreatDetectionAnalysisStep creates the main threat analysis step
func (c *Compiler) buildThreatDetectionAnalysisStep(data *WorkflowData) []string {
var steps []string
Expand Down Expand Up @@ -920,6 +944,7 @@ func (c *Compiler) buildDetectionJob(data *WorkflowData) (*Job, error) {
"detection_success": "${{ steps.detection_conclusion.outputs.success }}",
"detection_conclusion": "${{ steps.detection_conclusion.outputs.conclusion }}",
"detection_reason": "${{ steps.detection_conclusion.outputs.reason }}",
"aic": "${{ steps.parse_detection_token_usage.outputs.aic }}",
}

// Detection job depends on agent job and activation job (for trace ID)
Expand Down
Loading