From 876f288792313914947ec34a5f97441b9c953a7b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 9 Jul 2026 02:40:31 +0000 Subject: [PATCH 1/3] Initial plan From 0f47524989042ba233e98e24e1ec0b5e1e7e5661 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 9 Jul 2026 02:52:00 +0000 Subject: [PATCH 2/3] feat: add CI guard to prevent model allowlist drift MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the three-part fix from issue #6024: 1. Single source of truth comment — SUPPORTED_COPILOT_MODELS in src/copilot-model.ts now documents exactly which related files must be updated when adding or retiring a model. 2. CI guard — new src/copilot-model-catalog-sync.test.ts fails if any model added to ai-credits-pricing.js is missing from SUPPORTED_COPILOT_MODELS and not explicitly excluded. This catches the class of bug fixed in #5831 (mai-code-1-flash) before it ships. 3. Pointer comment in ai-credits-pricing.js — directs contributors to add new Copilot CLI models to SUPPORTED_COPILOT_MODELS as well. testHelpers export added to copilot-model.ts (@internal / ts-prune-ignore-next) to give the sync test access to the internal Set. --- containers/api-proxy/ai-credits-pricing.js | 6 ++ src/copilot-model-catalog-sync.test.ts | 104 +++++++++++++++++++++ src/copilot-model.ts | 22 +++++ 3 files changed, 132 insertions(+) create mode 100644 src/copilot-model-catalog-sync.test.ts diff --git a/containers/api-proxy/ai-credits-pricing.js b/containers/api-proxy/ai-credits-pricing.js index 32a962891..589c8dfee 100644 --- a/containers/api-proxy/ai-credits-pricing.js +++ b/containers/api-proxy/ai-credits-pricing.js @@ -3,6 +3,12 @@ // Curated per-model pricing in dollars per 1M tokens. // These provider-agnostic aliases take precedence over the bundled models.dev // catalog fallback. +// +// SYNC NOTE: When adding a new Copilot CLI completion model here, also add it +// to SUPPORTED_COPILOT_MODELS in src/copilot-model.ts (the host-side validator). +// If you don't, AWF will reject the COPILOT_MODEL value with a misleading +// "retired or unsupported" error before any container starts. +// A CI guard in src/copilot-model-catalog-sync.test.ts enforces this invariant. module.exports = Object.freeze({ 'gpt-5-mini': { input: 0.25, cachedInput: 0.025, cacheWrite: null, output: 2.00 }, 'gpt-5-codex-mini': { input: 0.25, cachedInput: 0.025, cacheWrite: null, output: 2.00 }, diff --git a/src/copilot-model-catalog-sync.test.ts b/src/copilot-model-catalog-sync.test.ts new file mode 100644 index 000000000..7bd7af6b4 --- /dev/null +++ b/src/copilot-model-catalog-sync.test.ts @@ -0,0 +1,104 @@ +/** + * CI guard: verifies that SUPPORTED_COPILOT_MODELS and the api-proxy pricing + * catalog stay in sync. + * + * When a new Copilot CLI completion model is added to + * containers/api-proxy/ai-credits-pricing.js, it must ALSO appear in + * SUPPORTED_COPILOT_MODELS in src/copilot-model.ts. Without that, AWF rejects + * the COPILOT_MODEL value with a misleading "retired or unsupported" error + * before containers even start (as happened with mai-code-1-flash in #5831). + * + * When adding a new entry to ai-credits-pricing.js, you must either: + * (a) Add the model to SUPPORTED_COPILOT_MODELS in src/copilot-model.ts, OR + * (b) Add it to INTENTIONALLY_EXCLUDED_FROM_SUPPORTED below with a comment + * explaining why it is not a Copilot CLI completion model. + */ + +import * as path from 'path'; +import { testHelpers } from './copilot-model'; + +/** Mirror of the separator normalisation in copilot-model.ts. */ +function normalizeSeparators(s: string): string { + return s.replace(/[._]/g, '-').toLowerCase(); +} + +/** + * Models present in ai-credits-pricing.js that are intentionally absent from + * SUPPORTED_COPILOT_MODELS. These are either non-completion models (e.g. + * embeddings), BYOK/OpenRouter-only models, or older versioned variants that + * the Copilot CLI model picker does not surface. + * + * Add an entry here (with a comment) only when the model truly should NOT be + * settable via COPILOT_MODEL. + */ +const INTENTIONALLY_EXCLUDED_FROM_SUPPORTED = new Set([ + // ── Embedding models — cannot be used as COPILOT_MODEL ──────────────────── + 'text-embedding-3-small', // text embedding, not a chat/completion model + 'text-embedding-ada-002', // text embedding, not a chat/completion model + + // ── Older Claude Opus 4 versions ────────────────────────────────────────── + // Only the latest revision (claude-opus-4.8) is exposed in the Copilot CLI + // model picker; older versioned slugs are priced for BYOK/billing purposes. + 'claude-opus-4-5', + 'claude-opus-4-6', + 'claude-opus-4-7', + + // ── Unversioned Claude Sonnet 4 base alias ──────────────────────────────── + // Use explicit versioned models (claude-sonnet-4.5, claude-sonnet-4.6) instead. + 'claude-sonnet-4', + + // ── Gemini models not yet available via the Copilot CLI model picker ────── + 'gemini-2.5-pro', // older generation + 'gemini-3-flash', // not surfaced in Copilot CLI + 'gemini-3.1-flash', // not surfaced in Copilot CLI (different from gemini-3.1-pro-preview) + 'gemini-3.1-pro', // not surfaced in Copilot CLI (different from gemini-3.1-pro-preview) + + // ── GPT variants not in the Copilot CLI model picker ───────────────────── + 'gpt-5-codex-mini', // internal variant; not a public Copilot CLI model + 'gpt-5.4-nano', // not yet surfaced in Copilot CLI + + // ── Internal / experimental models ─────────────────────────────────────── + 'raptor-mini', // internal model; not a public Copilot CLI model +]); + +describe('SUPPORTED_COPILOT_MODELS ↔ ai-credits-pricing catalog sync', () => { + const pricingPath = path.resolve( + __dirname, + '..', + 'containers', + 'api-proxy', + 'ai-credits-pricing.js', + ); + + // eslint-disable-next-line @typescript-eslint/no-require-imports + const pricing = require(pricingPath) as Record; + const pricingModels = Object.keys(pricing); + + it('every Copilot CLI model in ai-credits-pricing.js appears in SUPPORTED_COPILOT_MODELS', () => { + // Build a separator-normalised view of the supported set so that + // claude-haiku-4-5 (pricing) matches claude-haiku-4.5 (SUPPORTED). + const normalizedSupported = new Set( + [...testHelpers.supportedCopilotModels].map(m => normalizeSeparators(m)), + ); + + const missing: string[] = []; + for (const model of pricingModels) { + if (INTENTIONALLY_EXCLUDED_FROM_SUPPORTED.has(model)) continue; + if (!normalizedSupported.has(normalizeSeparators(model))) { + missing.push(model); + } + } + + if (missing.length > 0) { + const modelList = missing.map(m => ` '${m}'`).join('\n'); + throw new Error( + `Found ${missing.length} model(s) in containers/api-proxy/ai-credits-pricing.js ` + + `that are missing from SUPPORTED_COPILOT_MODELS in src/copilot-model.ts:\n` + + `${modelList}\n\n` + + `If this is a new Copilot CLI completion model, add it to SUPPORTED_COPILOT_MODELS.\n` + + `If it is NOT a Copilot CLI model, add it to INTENTIONALLY_EXCLUDED_FROM_SUPPORTED ` + + `in src/copilot-model-catalog-sync.test.ts with a comment explaining why.`, + ); + } + }); +}); diff --git a/src/copilot-model.ts b/src/copilot-model.ts index 0971676e4..6a7b2a35e 100644 --- a/src/copilot-model.ts +++ b/src/copilot-model.ts @@ -25,6 +25,22 @@ const RETIRED_COPILOT_MODEL_ALIASES: Record = { 'gpt-5-codex': 'gpt-5.3-codex', }; +/** + * Single source of truth for all Copilot CLI completion models that AWF accepts + * as a `COPILOT_MODEL` value. The host-side validator rejects any value not in + * this set (or its separator-normalised form) before containers start. + * + * When **adding** a new Copilot CLI model here, also update: + * - `containers/api-proxy/ai-credits-pricing.js` (AI-credits billing) + * - `docs/model-api-mapping.json` (endpoint routing, OpenAI/Anthropic only) + * + * When **retiring** a model, move it to `RETIRED_COPILOT_MODEL_ALIASES` (below) + * and mirror the change in `containers/api-proxy/guards/retired-model-guard.js`. + * + * A CI guard in `src/copilot-model-catalog-sync.test.ts` fails if any model + * present in `ai-credits-pricing.js` is missing from this set (and is not + * explicitly listed as a non-CLI model in the test's exclusion set). + */ const SUPPORTED_COPILOT_MODELS = new Set([ 'gpt-4', 'gpt-4.1', @@ -91,6 +107,12 @@ function levenshtein(a: string, b: string): number { return dp[a.length][b.length]; } +/** @internal Exposed for unit tests — catalog sync guard uses this to verify pricing parity. */ +// ts-prune-ignore-next +export const testHelpers = { + supportedCopilotModels: SUPPORTED_COPILOT_MODELS, +}; + export function validateCopilotModel(rawModel: string): CopilotModelValidationResult { const trimmed = rawModel.trim(); if (!trimmed) { From a0dd6740e6ceaf34e85767d7d5f8d2f0ad34a3c4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 9 Jul 2026 03:07:35 +0000 Subject: [PATCH 3/3] fix: address review feedback on copilot-model test helpers - Export testHelpers.supportedCopilotModels as a ReadonlySet defensive copy to prevent accidental mutation of the production singleton in tests - Update normalizeSeparators() comment to accurately describe full canonicalization (lowercase + separator replacement), not just separator normalization --- src/copilot-model-catalog-sync.test.ts | 2 +- src/copilot-model.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/copilot-model-catalog-sync.test.ts b/src/copilot-model-catalog-sync.test.ts index 7bd7af6b4..e55e338e5 100644 --- a/src/copilot-model-catalog-sync.test.ts +++ b/src/copilot-model-catalog-sync.test.ts @@ -17,7 +17,7 @@ import * as path from 'path'; import { testHelpers } from './copilot-model'; -/** Mirror of the separator normalisation in copilot-model.ts. */ +/** Mirrors the full canonicalization in copilot-model.ts: lowercase then replace separators. */ function normalizeSeparators(s: string): string { return s.replace(/[._]/g, '-').toLowerCase(); } diff --git a/src/copilot-model.ts b/src/copilot-model.ts index 6a7b2a35e..fca9578e5 100644 --- a/src/copilot-model.ts +++ b/src/copilot-model.ts @@ -110,7 +110,7 @@ function levenshtein(a: string, b: string): number { /** @internal Exposed for unit tests — catalog sync guard uses this to verify pricing parity. */ // ts-prune-ignore-next export const testHelpers = { - supportedCopilotModels: SUPPORTED_COPILOT_MODELS, + supportedCopilotModels: new Set(SUPPORTED_COPILOT_MODELS) as ReadonlySet, }; export function validateCopilotModel(rawModel: string): CopilotModelValidationResult {