diff --git a/.github/workflows/objective-impact-report.lock.yml b/.github/workflows/objective-impact-report.lock.yml index 22611759b8e..fab0babc804 100644 --- a/.github/workflows/objective-impact-report.lock.yml +++ b/.github/workflows/objective-impact-report.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"d2bca470c27b25256be1071d07254e340589733d170b6c9e9f475369fdf2cf2a","body_hash":"a7ad7feeba080f3bfbe37fdb8d59feb9320d4e5efb9d38579f2ee66116e91c3b","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.60"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"d2bca470c27b25256be1071d07254e340589733d170b6c9e9f475369fdf2cf2a","body_hash":"bdcc3de08e69ba565b30c9c238fd38da492c12b1843309770c4a717570da8aa5","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.60"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/cache/save","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/checkout","sha":"df4cb1c069e1874edd31b4311f1884172cec0e10","version":"v6.0.3"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.2","digest":"sha256:f88e5b17b6b7a600117bc121114d6ce2155c88c983c0c939c5df884f730fa1d6","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.2@sha256:f88e5b17b6b7a600117bc121114d6ce2155c88c983c0c939c5df884f730fa1d6"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.2","digest":"sha256:ee39841d980878ebbb87592903b06d31a1af500c71525c9616f7e8e2a27041a4","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.2@sha256:ee39841d980878ebbb87592903b06d31a1af500c71525c9616f7e8e2a27041a4"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.2","digest":"sha256:2e3a717e5f19a654cd9a2263beb52012b56bcb68562ec5ae2e42f9d156b49591","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.2@sha256:2e3a717e5f19a654cd9a2263beb52012b56bcb68562ec5ae2e42f9d156b49591"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.25","digest":"sha256:c10331ad17668ef89f38f5e356678788a40b0cd5fef96e8f92e1d9c1de47cbaa","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.3.25@sha256:c10331ad17668ef89f38f5e356678788a40b0cd5fef96e8f92e1d9c1de47cbaa"},{"image":"ghcr.io/github/github-mcp-server:v1.1.2","digest":"sha256:30197479d8036c7811892bc07e06f9a05c9ef3cdd79bc59f256d50647f95788c","pinned_image":"ghcr.io/github/github-mcp-server:v1.1.2@sha256:30197479d8036c7811892bc07e06f9a05c9ef3cdd79bc59f256d50647f95788c"}]} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # diff --git a/.github/workflows/objective-impact-report.md b/.github/workflows/objective-impact-report.md index e0a0ee5bf35..986b1a6b0c3 100644 --- a/.github/workflows/objective-impact-report.md +++ b/.github/workflows/objective-impact-report.md @@ -19,21 +19,20 @@ safe-outputs: ## Goal -Test whether Impact Efficiency is a more meaningful signal than accepted outcome counts alone. +POC: Test whether Impact Efficiency is a more meaningful signal than accepted outcome counts alone. + +Focus only on **pull request outcomes** and **safe output outcomes** (issues created or closed via the safe-output mechanism). Other outcome types are excluded from this POC because their acceptance criteria are not yet well-defined and most remain pending. Use this model: ```text -Outcome = recorded work item produced by a GitHub Agentic Workflow run -Objective = issue/epic/work item linked to the outcome +Outcome = a PR or safe-output issue produced by a GitHub Agentic Workflow run Objective Value = numeric value from the repository objective-mapping configuration applied to traced root labels Outcome Indicator = 1 for accepted/delivered outcomes, 0 otherwise Outcome Value = Outcome Indicator × Objective Value Impact Efficiency = Σ Outcome Value / AI Credits ``` -Treat an outcome as one recorded result item produced by a GitHub Agentic Workflow run (for example, a PR change, completed fix, or report action), which may later be accepted or not accepted. -Use workflow run outputs/artifacts and linked GitHub objects (issues, PRs, comments, discussions) as the outcome source of truth. Treat AI Credits as total model-credit cost consumed by the workflow runs that produced the analyzed outcomes. When available, use deterministic precomputed run data that already includes each run's `aic` field. Prefer existing gh-aw outputs that already surface `aic`, such as pre-downloaded `gh aw logs --json` data or audit/log artifacts derived from the same run summaries. @@ -58,7 +57,12 @@ If a run's `aic` field is missing or null, treat it as `0` and count it as missi ## Scope -Analyze workflow outcomes and linked objectives from the last 180 days. +Analyze only the following outcome types from the last 180 days: + +- **Pull request outcomes**: PRs created by GitHub Agentic Workflow runs. Accepted = merged. Rejected = closed without merge. Skip open (pending) PRs. +- **Safe output outcomes**: issues created or closed by workflow runs via the safe-output mechanism. Accepted = issue successfully created/closed. Skip any with unresolved state. + +Exclude all other outcome types (direct issue outcomes, comments, discussions, etc.). These are omitted because their acceptance criteria are incomplete and most are left pending, which would distort the metric. ## Objective value mapping @@ -81,36 +85,29 @@ Objective Labels = mapping.GetObjectiveLabels(root_labels) Do not invent fallback scoring rules such as milestone bonuses, project bonuses, or priority-to-points heuristics when the mapping file is present. -```text -Examples of mapped labels in this repository include campaign, security, observability, testing, automation, and other configured objective labels. -``` - If a traced root object has no labels that exist in the mapping, mark the outcome as `unmapped`. ## Outcome association rules -For each workflow outcome, follow the implemented root-tracing behavior: +For each in-scope outcome, follow the implemented root-tracing behavior: 1. For pull-request outcomes, trace the PR to its linked closing issue and use that root issue's labels. -2. If PR root tracing fails, or for direct issue outcomes, use labels on the issue itself. -3. Record the traced root URL when one is found so the report preserves an audit trail. -4. If no mapped objective labels can be found, mark the outcome as `unmapped`, exclude it from `Σ Outcome Value`, and report it separately. - -Prefer precomputed outcome evaluation data when available. Do not re-derive a different mapping model inside the report. +2. If PR root tracing fails, use labels on the PR itself. +3. For safe-output issue outcomes, use labels on the safe-output issue itself. +4. Record the traced root URL when one is found so the report preserves an audit trail. +5. If no mapped objective labels can be found, mark the outcome as `unmapped`, exclude it from `Σ Outcome Value`, and report it separately. ## Computation -For each outcome: +For each in-scope outcome: ```text -Outcome Indicator = 1 for accepted/delivered outcome, 0 for rejected, abandoned, or incomplete outcome +Outcome Indicator: + PR outcome: 1 if merged, 0 if closed without merge (open PRs excluded) + Safe output outcome: 1 if successfully created/closed, 0 otherwise Outcome Value = Outcome Indicator × Objective Value ``` -Treat pending-review outcomes as `Outcome Indicator = 0` until explicitly accepted. - -Accepted/delivered outcome means the intended result was accepted in GitHub state (for example: merged PR, closed issue with completion signal, or explicit accepted status in the workflow outcome record). - Then compute: ```text @@ -146,7 +143,8 @@ The report must include: ### Summary -- Outcomes analyzed +- PR outcomes analyzed (merged / closed / excluded open) +- Safe output outcomes analyzed - Objectives mapped - Unmapped outcomes - Accepted outcome count @@ -156,18 +154,13 @@ The report must include: ### Top outcomes by outcome value -| Outcome | Associated objective | Objective value signals | Objective Value | Outcome Value | +| Outcome | Type | Associated objective | Objective Value | Outcome Value | |---|---|---|---:|---:| -### Top objectives by delivered value - -| Objective Label | Delivered Outcome Value | Attempted Outcome Value | Delivered Outcomes | Efficiency | -|---|---:|---:|---:|---:| - ### Unmapped outcomes -| Outcome | Reason objective was not mapped | -|---|---| +| Outcome | Type | Reason objective was not mapped | +|---|---|---| ### Interpretation @@ -182,9 +175,8 @@ Explain which one better reflects meaningful delivered value relative to cost. Mention missing or weak links in: -- outcome-to-objective association -- root tracing and linked-object coverage -- label mapping coverage in `.github/objective-mapping.json` +- PR root tracing and linked-closing-issue coverage +- safe-output issue label mapping coverage in `.github/objective-mapping.json` - AI Credits availability State whether AI Credits came from deterministic precomputed data or from a live fallback path. diff --git a/.github/workflows/portfolio-analyst.lock.yml b/.github/workflows/portfolio-analyst.lock.yml index 888178d2290..2803881983b 100644 --- a/.github/workflows/portfolio-analyst.lock.yml +++ b/.github/workflows/portfolio-analyst.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"17cad040856577e7c1d0faa853da74a0bdb9ef65f883f409665de04a9aa7ccf7","body_hash":"503a3f5c7193d58f3b8226cc872886506447cddb86046c0b1d0858493bd0b063","strict":true,"agent_id":"claude","engine_versions":{"claude":"2.1.168"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"17cad040856577e7c1d0faa853da74a0bdb9ef65f883f409665de04a9aa7ccf7","body_hash":"78306c33e4469a8641fc9e753730950f28615d4bb190961d71d32b6eb288ad6e","strict":true,"agent_id":"claude","engine_versions":{"claude":"2.1.168"}} # gh-aw-manifest: {"version":1,"secrets":["ANTHROPIC_API_KEY","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN","GRAFANA_SERVICE_ACCOUNT_TOKEN","GRAFANA_URL","SENTRY_ACCESS_TOKEN","SENTRY_OPENAI_API_KEY"],"actions":[{"repo":"actions/cache/restore","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/cache/save","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/checkout","sha":"df4cb1c069e1874edd31b4311f1884172cec0e10","version":"v6.0.3"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/setup-python","sha":"a309ff8b426b58ec0e2a45f0f869d46889d02405","version":"v6.2.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.2","digest":"sha256:f88e5b17b6b7a600117bc121114d6ce2155c88c983c0c939c5df884f730fa1d6","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.2@sha256:f88e5b17b6b7a600117bc121114d6ce2155c88c983c0c939c5df884f730fa1d6"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.2","digest":"sha256:ee39841d980878ebbb87592903b06d31a1af500c71525c9616f7e8e2a27041a4","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.2@sha256:ee39841d980878ebbb87592903b06d31a1af500c71525c9616f7e8e2a27041a4"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.2","digest":"sha256:2e3a717e5f19a654cd9a2263beb52012b56bcb68562ec5ae2e42f9d156b49591","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.2@sha256:2e3a717e5f19a654cd9a2263beb52012b56bcb68562ec5ae2e42f9d156b49591"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.25","digest":"sha256:c10331ad17668ef89f38f5e356678788a40b0cd5fef96e8f92e1d9c1de47cbaa","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.3.25@sha256:c10331ad17668ef89f38f5e356678788a40b0cd5fef96e8f92e1d9c1de47cbaa"},{"image":"ghcr.io/github/github-mcp-server:v1.1.2","digest":"sha256:30197479d8036c7811892bc07e06f9a05c9ef3cdd79bc59f256d50647f95788c","pinned_image":"ghcr.io/github/github-mcp-server:v1.1.2@sha256:30197479d8036c7811892bc07e06f9a05c9ef3cdd79bc59f256d50647f95788c"},{"image":"grafana/mcp-grafana","digest":"sha256:60a4e3a417a69eeb864a72052c53b4aa4466ff3577d6ef9bacc671f4b77d7090","pinned_image":"grafana/mcp-grafana@sha256:60a4e3a417a69eeb864a72052c53b4aa4466ff3577d6ef9bacc671f4b77d7090"},{"image":"node:lts-alpine","digest":"sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14","pinned_image":"node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14"}]} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md #