diff --git a/.github/workflows/cjs.yml b/.github/workflows/cjs.yml index 6d23498f3b4..c2a43be4b51 100644 --- a/.github/workflows/cjs.yml +++ b/.github/workflows/cjs.yml @@ -6,6 +6,7 @@ on: paths: - 'actions/setup/js/**' - 'actions/setup/md/**' + - 'pkg/cli/data/models.json' - 'scripts/**/*.js' - 'Makefile' - '.github/workflows/ci.yml' @@ -15,6 +16,7 @@ on: paths: - 'actions/setup/js/**' - 'actions/setup/md/**' + - 'pkg/cli/data/models.json' - 'scripts/**/*.js' - 'Makefile' - '.github/workflows/ci.yml' @@ -155,6 +157,9 @@ jobs: - name: Check JSON formatting run: make fmt-check-json + - name: Validate models.json mirror consistency + run: make validate-models-json-sync + artifact-integration: name: Artifact API Integration runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 99dfa1c477c..88a141ab5ec 100644 --- a/Makefile +++ b/Makefile @@ -900,6 +900,24 @@ refresh-models-json: cp "$$tmp" actions/setup/js/models.json; \ echo "✓ Refreshed pkg/cli/data/models.json and actions/setup/js/models.json (catalog providers: anthropic, openai, github-copilot)" +.PHONY: validate-models-json-sync +validate-models-json-sync: + @echo "Validating models.json mirror consistency..." + @set -e; \ + command -v jq >/dev/null 2>&1 || { echo "jq is required for validate-models-json-sync"; exit 1; }; \ + tmp1=$$(mktemp); tmp2=$$(mktemp); \ + trap 'rm -f "$$tmp1" "$$tmp2"' EXIT; \ + jq -S . pkg/cli/data/models.json > "$$tmp1"; \ + jq -S . actions/setup/js/models.json > "$$tmp2"; \ + cmp -s "$$tmp1" "$$tmp2" || { \ + echo "models.json mirrors diverged:"; \ + echo " pkg/cli/data/models.json and actions/setup/js/models.json must remain semantically identical after key sorting."; \ + echo " Run 'make refresh-models-json' to synchronize the mirrors."; \ + diff -u "$$tmp1" "$$tmp2" || true; \ + exit 1; \ + } + @echo "✓ models.json mirrors are synchronized" + # Check file sizes and function counts .PHONY: check-file-sizes check-file-sizes: diff --git a/docs/src/content/docs/specs/ai-credits-specification.md b/docs/src/content/docs/specs/ai-credits-specification.md index 8f14f40c203..f50b4432283 100644 --- a/docs/src/content/docs/specs/ai-credits-specification.md +++ b/docs/src/content/docs/specs/ai-credits-specification.md @@ -40,7 +40,8 @@ This document is governed by the GitHub Agentic Workflows project specifications 10. [Per-Run AI Credits Budget](#10-per-run-ai-credits-budget) 11. [Appendices](#appendices) 12. [References](#references) -13. [Change Log](#change-log) +13. [Sync Notes](#sync-notes) +14. [Change Log](#change-log) --- @@ -249,7 +250,9 @@ A conforming implementation MUST define and enforce behavior for catalog synchro 3. **Missing required fields**: When a catalog entry is missing required fields (`input` or `output` cost values), the implementation MUST treat that entry as invalid and MUST NOT compute AIC using zero or undefined costs for that model. The entry MUST be skipped or flagged, and a diagnostic MUST be emitted. -4. **Catalog version mismatch**: When the two required catalog paths (`pkg/cli/data/models.json` and `actions/setup/js/models.json`) diverge in content after a sync operation, the sync tooling/CI gate MUST treat this as a sync failure and MUST fail the refresh operation until consistency is restored. +4. **Catalog version mismatch**: When the two required catalog paths (`pkg/cli/data/models.json` and `actions/setup/js/models.json`) diverge in content after a sync operation, the sync tooling/CI gate MUST treat this as a sync failure. The refresh operation MUST fail until consistency is restored. In the gh-aw repository: + - CI MUST run `make validate-models-json-sync` (or an equivalent normalized-content comparison gate). + - That gate MUST fail whenever these mirror files cease to represent the same JSON dataset. --- @@ -541,6 +544,25 @@ Pricing catalogs are configuration inputs. Implementations MUST: --- + +## Sync Notes + +The canonical gh-aw mirror files for this specification are: + +- `pkg/cli/data/models.json` +- `actions/setup/js/models.json` + +The repository validation target for mirror consistency is `make validate-models-json-sync`. + +This specification MUST be revalidated when any of the following occurs: + +1. A new model or provider entry is added to either mirror. +2. A pricing value or billing-related field changes in either mirror. +3. This specification is incremented to a new minor version. + +--- + + ## Change Log ### Version 1.4.0 (2026-06-09) diff --git a/docs/src/content/docs/specs/bash-command-parser-specification.md b/docs/src/content/docs/specs/bash-command-parser-specification.md index deee6f64fba..661bc5503b6 100644 --- a/docs/src/content/docs/specs/bash-command-parser-specification.md +++ b/docs/src/content/docs/specs/bash-command-parser-specification.md @@ -36,6 +36,8 @@ This document is a draft extracted from the current production parser behavior a 10. [Testing Strategies](#10-testing-strategies) 11. [Security Considerations](#11-security-considerations) 12. [References](#12-references) +13. [Sync Notes](#sync-notes) +14. [Change Log](#change-log) --- @@ -332,3 +334,30 @@ A Class I (Integration Consumer) conforming implementation MUST NOT treat any pa - Copilot SDK Shell Permission Integration (implementation-defined binding) - Repository Conformance Vectors (implementation-defined location) + +--- + + +## Sync Notes + +The canonical machine-readable vector collection for this specification is `specs/test-vectors/bash-command-parser/`. + +For version `1.1.0`, the canonical seed files are: + +- `specs/test-vectors/bash-command-parser/v1.1.0-model-based.json` +- `specs/test-vectors/bash-command-parser/v1.1.0-verification.json` + +These vectors MUST be revalidated whenever either of the following occurs: + +1. This specification is incremented to a new version. +2. The parser behavior changes in `actions/setup/js/bash_command_parser.cjs`, including updates to `splitOnPipelineOperators`, `extractCommandName`, or `extractCommandNamesFromPipeline`. + +--- + + +## Change Log + +### Version 1.1.0 (2026-07-06 maintenance update) + +- Added canonical versioned machine-readable vector files at `specs/test-vectors/bash-command-parser/v1.1.0-model-based.json` and `specs/test-vectors/bash-command-parser/v1.1.0-verification.json`. +- Added Sync Notes identifying the canonical vector path and the required revalidation triggers for parser behavior and specification version changes. diff --git a/docs/src/content/docs/specs/copilot-sdk-driver-specification.md b/docs/src/content/docs/specs/copilot-sdk-driver-specification.md index 442edb91c97..f8f8eba443b 100644 --- a/docs/src/content/docs/specs/copilot-sdk-driver-specification.md +++ b/docs/src/content/docs/specs/copilot-sdk-driver-specification.md @@ -36,7 +36,8 @@ This document is governed by the GitHub Agentic Workflows project specifications 8. [Compliance Testing](#8-compliance-testing) 9. [Appendices](#9-appendices) 10. [References](#10-references) -11. [Change Log](#11-change-log) +11. [Sync Notes](#sync-notes) +12. [Change Log](#change-log) --- @@ -181,12 +182,14 @@ A conforming driver implementation MUST treat `COPILOT_SDK_SEND_TIMEOUT_MS` as m ### 4.5 Tool-Denials Guardrail Environment Variable `GH_AW_MAX_TOOL_DENIALS` controls the catastrophic tool-denials -guardrail in SDK mode. A conforming driver implementation SHOULD count +guardrail in SDK mode. A conforming driver implementation MUST count repeated tool refusals (permission denials), and MUST stop inference once the configured threshold is reached. When unset, non-numeric, or non-positive, implementations MUST apply the default value (`5`). +This counting requirement tightens the specification to match the existing gh-aw reference implementation in `actions/setup/js/copilot_sdk_session.cjs` and the current `copilot_sdk_driver.test.cjs` guardrail coverage. It does not change gh-aw runtime behavior, but implementations that previously treated denial counting as optional would need to align with this clarified conformance requirement. + ### 4.6 TypeScript Example (Non-Normative) Prerequisite: install [`@github/copilot-sdk`](https://www.npmjs.com/package/@github/copilot-sdk) in the runtime where this example executes. @@ -495,7 +498,26 @@ A conforming implementation SHOULD: --- -## 11. Change Log + +## Sync Notes + +The canonical gh-aw harness implementation for this specification is centered in: + +- `actions/setup/js/copilot_sdk_driver.cjs` +- `actions/setup/js/copilot_sdk_session.cjs` +- `actions/setup/js/copilot_harness.cjs` +- `actions/setup/js/copilot_sdk_driver.test.cjs` + +This specification MUST be revalidated whenever any of the following occurs: + +1. The standalone environment-variable contract changes. +2. The permission-request handling or tool-denial guardrail semantics change. +3. Harness token propagation or secret-isolation behavior changes. + +--- + + +## Change Log ### Version 1.0.2 (Draft Specification) diff --git a/specs/security-architecture-spec-summary.md b/specs/security-architecture-spec-summary.md index 078e4d02dd6..9e858db1339 100644 --- a/specs/security-architecture-spec-summary.md +++ b/specs/security-architecture-spec-summary.md @@ -203,10 +203,10 @@ Summary version **1.0.0** corresponds to the minimum validated `.lock.yml` compi | Update summary to reflect v1.0.2 CTR-012 work | ✅ Done (2026-05-10) | Appendix count updated; security architecture remains at version 1.0.0 | | Audit "Next Steps" for stale v1.0.0 tasks | ✅ Done (2026-05-10) | This table replaces the stale untracked list | | Add spec-to-lock sync note for security summary consumers | ✅ Done (2026-06-25) | Added "Spec-to-Lock Sync (v1.0.0)" section mapping summary version to validated `.lock.yml` behaviors | -| Track pre_activation note from validation doc | ⏳ Pending | Add an explicit note tying role validation to the separate `pre_activation` job in `specs/security-architecture-spec.md` | -| Track detection job naming note from validation doc | ⏳ Pending | Add an Appendix D example that names the `detection` job explicitly as the runtime threat-detection layer | -| Track conclusion job note from validation doc | ⏳ Pending | Document optional cleanup/reporting jobs such as `conclusion` without making them normative | -| Audit trusted-users runtime enforcement coverage | ⏳ Pending | `trusted-users` runtime enforcement is not yet called out in Sections 8-9 of `specs/security-architecture-spec.md`; decide whether to document it there or defer to the companion GitHub MCP access-control spec | +| Track pre_activation note from validation doc | ✅ Done (2026-07-06) | Added an explicit PM-11 note tying runtime membership validation to the separate `pre_activation` job in `specs/security-architecture-spec.md` | +| Track detection job naming note from validation doc | ✅ Done (2026-07-06) | Appendix D now names the `detection` job explicitly as the runtime threat-detection layer | +| Track conclusion job note from validation doc | ✅ Done (2026-07-06) | Documented the optional `conclusion` job as non-normative cleanup/reporting guidance | +| Audit trusted-users runtime enforcement coverage | ✅ Done (2026-07-06) | Sections 8-9 now document runtime `trusted-users` enforcement scope directly in this spec summary (membership checks gate privileged runtime access) | ## Versioning diff --git a/specs/security-architecture-spec-validation.md b/specs/security-architecture-spec-validation.md index 75f1c35d782..574e41add9a 100644 --- a/specs/security-architecture-spec-validation.md +++ b/specs/security-architecture-spec-validation.md @@ -1,7 +1,7 @@ # Security Architecture Specification Validation **Document**: Validation of `security-architecture-spec.md` against compiled `.lock.yml` files -**Date**: May 15, 2026 +**Date**: July 6, 2026 **Validator**: GitHub Copilot Agent **Scope**: Cross-reference specification requirements with actual implementation @@ -9,7 +9,7 @@ ## Executive Summary -✅ **VALIDATION RESULT**: The specification accurately reflects the implementation in compiled `.lock.yml` files and JavaScript implementation (revalidated on 2026-05-15). +✅ **VALIDATION RESULT**: The specification accurately reflects the implementation in compiled `.lock.yml` files and JavaScript implementation (revalidated on 2026-07-06). All major security architecture claims in the specification have been verified against actual workflow implementations: - ✅ Job architecture (activation, agent, safe_outputs) @@ -21,6 +21,7 @@ All major security architecture claims in the specification have been verified a - ✅ Action pinning to SHAs - ✅ Timestamp validation at runtime - ✅ Concurrency control (context-aware grouping with cancel-in-progress) +- ✅ Documentation maintenance follow-ups for `pre_activation`, `detection`, optional `conclusion`, and companion `trusted-users` references remain aligned with implementation behavior --- @@ -248,6 +249,19 @@ pre_activation: --- +### 4a. Documentation Maintenance Follow-ups (2026-07-06) + +The July 2026 maintenance pass rechecked the documentation-only clarifications requested by the summary tracker against the same compiled workflow behavior: + +- The new PM-11 note is consistent with the verified `pre_activation -> activation` gate shown above. +- Appendix D now names the compiled `detection` job explicitly as the runtime threat-detection layer that gates `safe_outputs`. +- The optional `conclusion` job remains a non-normative cleanup/reporting example and is not required by the compiled architecture. +- `trusted-users` runtime enforcement continues to live in the companion GitHub MCP access-control specifications and tests; the new Section 9 note accurately scopes that behavior without changing the top-level threat-detection contract. + +**Status**: ✅ **VERIFIED** - The specification clarifications match current implementation and companion-spec boundaries. + +--- + ### 5. Threat Detection Layer (Section 9 - TD-01, TD-04) **Specification Claim**: diff --git a/specs/security-architecture-spec.md b/specs/security-architecture-spec.md index 835433b3f7f..a43886a9850 100644 --- a/specs/security-architecture-spec.md +++ b/specs/security-architecture-spec.md @@ -617,6 +617,8 @@ roles: all # Least restrictive **PM-11**: Role checks MUST be performed at runtime using membership validation. +The separate `pre_activation` job defined in Section 7.6.1 is the normative runtime mechanism that satisfies **PM-11**. Role validation MUST complete in `pre_activation` before the `activation` job begins. Implementations MUST NOT replace this gate with a later best-effort check in `activation`, `agent`, or `safe_outputs`. + **PM-12**: Failed role checks MUST cancel workflow execution with a warning message. #### 7.6.1 Pre-Activation Pattern @@ -746,6 +748,8 @@ The sandbox isolation layer provides process-level and container-level isolation The threat detection layer analyzes AI agent output for security threats before safe output jobs execute. +> Note: Repository-specific GitHub-tool access-control policies such as `trusted-users` are complementary runtime controls, but they are specified in the companion GitHub MCP access-control documents (`scratchpad/github-mcp-access-control-specification.md` and `scratchpad/guard-policies-specification.md`), not in the threat-detection requirements below. + ### 9.2 Threat Detection Requirements **TD-01**: A conforming implementation with complete conformance MUST provide automated threat detection. @@ -1336,6 +1340,8 @@ conclusion (cleanup, summary — optional) The `detection` job acts as a security gate: `safe_outputs` only runs when `needs.detection.outputs.success == 'true'`. +> Note: The optional `conclusion` job shown above is non-normative. Implementations MAY add cleanup or reporting jobs after `safe_outputs`, but conformance does not require a `conclusion` job and such jobs MUST NOT weaken the required isolation or permission boundaries. + ### Appendix B: Sanitization Examples #### Example 1: @Mention Neutralization @@ -1533,6 +1539,8 @@ safe-outputs: **Behavior**: AI detection + TruffleHog scan before PR creation. +In compiled workflows, this pattern yields a dedicated `detection` job that serves as the runtime threat-detection layer between the `agent` job and any `safe_outputs` write operations. + #### Example 4: Concurrency Control ```yaml diff --git a/specs/test-vectors/bash-command-parser/v1.1.0-model-based.json b/specs/test-vectors/bash-command-parser/v1.1.0-model-based.json new file mode 100644 index 00000000000..a84d174d2d9 --- /dev/null +++ b/specs/test-vectors/bash-command-parser/v1.1.0-model-based.json @@ -0,0 +1,129 @@ +{ + "version": "1.1.0", + "source": "model-based", + "canonicalPath": "specs/test-vectors/bash-command-parser/", + "spec": "docs/src/content/docs/specs/bash-command-parser-specification.md", + "vectors": [ + { + "id": "BP-MB-S-001", + "category": "S-CORE", + "requirement": "§6.1 S-CORE-1", + "function": "splitOnPipelineOperators", + "note": "Intentionally includes a literal newline to cover the top-level newline split case in the minimal suite.", + "input": "echo a && printf b || cat file | sed 's/x/y/' ; pwd\necho done", + "expected": ["echo a", "printf b", "cat file", "sed 's/x/y/'", "pwd", "echo done"] + }, + { + "id": "BP-MB-S-002", + "category": "S-CORE", + "requirement": "§6.1 S-CORE-2", + "function": "splitOnPipelineOperators", + "input": "echo 'a && b || c | d ; e'", + "expected": ["echo 'a && b || c | d ; e'"] + }, + { + "id": "BP-MB-S-003", + "category": "S-CORE", + "requirement": "§6.1 S-CORE-3", + "function": "splitOnPipelineOperators", + "input": "echo \"a && b || c | d ; e\"", + "expected": ["echo \"a && b || c | d ; e\""] + }, + { + "id": "BP-MB-S-004", + "category": "S-CORE", + "requirement": "§6.1 S-CORE-4", + "function": "splitOnPipelineOperators", + "input": "echo $(printf 'a && b | c ; d') && date", + "expected": ["echo $(printf 'a && b | c ; d')", "date"] + }, + { + "id": "BP-MB-E-001", + "category": "E-CORE", + "requirement": "§6.1 E-CORE-1", + "function": "extractCommandName", + "input": "git status", + "expected": "git" + }, + { + "id": "BP-MB-E-002", + "category": "E-CORE", + "requirement": "§6.1 E-CORE-2", + "function": "extractCommandName", + "input": "FOO=1 BAR='two words' env | grep FOO", + "expected": "env" + }, + { + "id": "BP-MB-E-003", + "category": "E-CORE", + "requirement": "§6.1 E-CORE-3", + "function": "extractCommandName", + "input": "2>&1", + "expected": null + }, + { + "id": "BP-MB-E-004", + "category": "E-CORE", + "requirement": "§6.1 E-CORE-4 (clause keyword branch)", + "function": "extractCommandName", + "input": "then cat file", + "expected": "cat" + }, + { + "id": "BP-MB-E-005", + "category": "E-CORE", + "requirement": "§6.1 E-CORE-4 (structural keyword branch)", + "function": "extractCommandName", + "input": "if true; then echo done; fi", + "expected": null + }, + { + "id": "BP-MB-E-006", + "category": "E-CORE", + "requirement": "§6.1 E-CORE-5", + "function": "extractCommandName", + "input": "! { printf '%s' ok; }", + "expected": "printf" + }, + { + "id": "BP-MB-P-001", + "category": "P-CORE", + "requirement": "§6.1 P-CORE-1", + "function": "extractCommandNamesFromPipeline", + "input": "FOO=1 env && grep FOO file || awk '{print $1}' file | sort ; uniq", + "expected": ["env", "grep", "awk", "sort", "uniq"] + }, + { + "id": "BP-MB-P-002", + "category": "P-CORE", + "requirement": "§6.1 P-CORE-2", + "function": "extractCommandNamesFromPipeline", + "input": "2>/dev/null ; ls /tmp", + "expected": ["ls"] + }, + { + "id": "BP-MB-P-003", + "category": "P-CORE", + "requirement": "§6.1 P-CORE-3", + "function": "extractCommandNamesFromPipeline", + "input": "echo a && echo b | echo c ; printf done", + "expected": ["echo", "printf"] + }, + { + "id": "BP-MB-P-004", + "category": "P-CORE", + "requirement": "§6.1 P-CORE-4 (blank input branch)", + "function": "extractCommandNamesFromPipeline", + "input": "", + "expected": [] + }, + { + "id": "BP-MB-P-005", + "category": "P-CORE", + "requirement": "§6.1 P-CORE-4 (non-string branch)", + "function": "extractCommandNamesFromPipeline", + "input": null, + "expected": [] + } + ] +} diff --git a/specs/test-vectors/bash-command-parser/v1.1.0-verification.json b/specs/test-vectors/bash-command-parser/v1.1.0-verification.json new file mode 100644 index 00000000000..cdf7f1ad2e3 --- /dev/null +++ b/specs/test-vectors/bash-command-parser/v1.1.0-verification.json @@ -0,0 +1,57 @@ +{ + "version": "1.1.0", + "source": "verification", + "canonicalPath": "specs/test-vectors/bash-command-parser/", + "spec": "docs/src/content/docs/specs/bash-command-parser-specification.md", + "vectors": [ + { + "id": "BP-VF-R-001", + "category": "R-CORE", + "requirement": "§6.1 R-CORE-1", + "function": "splitOnPipelineOperators", + "note": "Intentionally contains an unterminated double quote to verify malformed-input tolerance without throwing.", + "input": "echo \"unterminated && text", + "expected": ["echo \"unterminated && text"], + "nonThrowing": true + }, + { + "id": "BP-VF-R-002", + "category": "R-CORE", + "requirement": "§6.1 R-CORE-2", + "function": "splitOnPipelineOperators", + "input": "echo $(printf '%s' foo && cat", + "expected": ["echo $(printf '%s' foo && cat"], + "nonThrowing": true + }, + { + "id": "BP-VF-MR-001", + "category": "MR", + "requirement": "§8.1", + "function": "splitOnPipelineOperators", + "relation": "whitespace-invariance", + "left": "ls /tmp&&cat file", + "right": " ls /tmp && cat file ", + "expected": ["ls /tmp", "cat file"] + }, + { + "id": "BP-VF-MR-002", + "category": "MR", + "requirement": "§8.2", + "function": "extractCommandNamesFromPipeline", + "relation": "quoted-operator-shielding", + "left": "echo \"a && b\"", + "right": "echo 'a && b'", + "expected": ["echo"] + }, + { + "id": "BP-VF-MR-003", + "category": "MR", + "requirement": "§8.5", + "function": "extractCommandNamesFromPipeline", + "relation": "duplicate-collapse-invariance", + "left": "echo a && cat b && echo c", + "right": "echo a && cat b", + "expected": ["echo", "cat"] + } + ] +}