diff --git a/containers/api-proxy/ai-credits-pricing.js b/containers/api-proxy/ai-credits-pricing.js index 32a962891..589c8dfee 100644 --- a/containers/api-proxy/ai-credits-pricing.js +++ b/containers/api-proxy/ai-credits-pricing.js @@ -3,6 +3,12 @@ // Curated per-model pricing in dollars per 1M tokens. // These provider-agnostic aliases take precedence over the bundled models.dev // catalog fallback. +// +// SYNC NOTE: When adding a new Copilot CLI completion model here, also add it +// to SUPPORTED_COPILOT_MODELS in src/copilot-model.ts (the host-side validator). +// If you don't, AWF will reject the COPILOT_MODEL value with a misleading +// "retired or unsupported" error before any container starts. +// A CI guard in src/copilot-model-catalog-sync.test.ts enforces this invariant. module.exports = Object.freeze({ 'gpt-5-mini': { input: 0.25, cachedInput: 0.025, cacheWrite: null, output: 2.00 }, 'gpt-5-codex-mini': { input: 0.25, cachedInput: 0.025, cacheWrite: null, output: 2.00 }, diff --git a/src/copilot-model-catalog-sync.test.ts b/src/copilot-model-catalog-sync.test.ts new file mode 100644 index 000000000..e55e338e5 --- /dev/null +++ b/src/copilot-model-catalog-sync.test.ts @@ -0,0 +1,104 @@ +/** + * CI guard: verifies that SUPPORTED_COPILOT_MODELS and the api-proxy pricing + * catalog stay in sync. + * + * When a new Copilot CLI completion model is added to + * containers/api-proxy/ai-credits-pricing.js, it must ALSO appear in + * SUPPORTED_COPILOT_MODELS in src/copilot-model.ts. Without that, AWF rejects + * the COPILOT_MODEL value with a misleading "retired or unsupported" error + * before containers even start (as happened with mai-code-1-flash in #5831). + * + * When adding a new entry to ai-credits-pricing.js, you must either: + * (a) Add the model to SUPPORTED_COPILOT_MODELS in src/copilot-model.ts, OR + * (b) Add it to INTENTIONALLY_EXCLUDED_FROM_SUPPORTED below with a comment + * explaining why it is not a Copilot CLI completion model. + */ + +import * as path from 'path'; +import { testHelpers } from './copilot-model'; + +/** Mirrors the full canonicalization in copilot-model.ts: lowercase then replace separators. */ +function normalizeSeparators(s: string): string { + return s.replace(/[._]/g, '-').toLowerCase(); +} + +/** + * Models present in ai-credits-pricing.js that are intentionally absent from + * SUPPORTED_COPILOT_MODELS. These are either non-completion models (e.g. + * embeddings), BYOK/OpenRouter-only models, or older versioned variants that + * the Copilot CLI model picker does not surface. + * + * Add an entry here (with a comment) only when the model truly should NOT be + * settable via COPILOT_MODEL. + */ +const INTENTIONALLY_EXCLUDED_FROM_SUPPORTED = new Set([ + // ── Embedding models — cannot be used as COPILOT_MODEL ──────────────────── + 'text-embedding-3-small', // text embedding, not a chat/completion model + 'text-embedding-ada-002', // text embedding, not a chat/completion model + + // ── Older Claude Opus 4 versions ────────────────────────────────────────── + // Only the latest revision (claude-opus-4.8) is exposed in the Copilot CLI + // model picker; older versioned slugs are priced for BYOK/billing purposes. + 'claude-opus-4-5', + 'claude-opus-4-6', + 'claude-opus-4-7', + + // ── Unversioned Claude Sonnet 4 base alias ──────────────────────────────── + // Use explicit versioned models (claude-sonnet-4.5, claude-sonnet-4.6) instead. + 'claude-sonnet-4', + + // ── Gemini models not yet available via the Copilot CLI model picker ────── + 'gemini-2.5-pro', // older generation + 'gemini-3-flash', // not surfaced in Copilot CLI + 'gemini-3.1-flash', // not surfaced in Copilot CLI (different from gemini-3.1-pro-preview) + 'gemini-3.1-pro', // not surfaced in Copilot CLI (different from gemini-3.1-pro-preview) + + // ── GPT variants not in the Copilot CLI model picker ───────────────────── + 'gpt-5-codex-mini', // internal variant; not a public Copilot CLI model + 'gpt-5.4-nano', // not yet surfaced in Copilot CLI + + // ── Internal / experimental models ─────────────────────────────────────── + 'raptor-mini', // internal model; not a public Copilot CLI model +]); + +describe('SUPPORTED_COPILOT_MODELS ↔ ai-credits-pricing catalog sync', () => { + const pricingPath = path.resolve( + __dirname, + '..', + 'containers', + 'api-proxy', + 'ai-credits-pricing.js', + ); + + // eslint-disable-next-line @typescript-eslint/no-require-imports + const pricing = require(pricingPath) as Record; + const pricingModels = Object.keys(pricing); + + it('every Copilot CLI model in ai-credits-pricing.js appears in SUPPORTED_COPILOT_MODELS', () => { + // Build a separator-normalised view of the supported set so that + // claude-haiku-4-5 (pricing) matches claude-haiku-4.5 (SUPPORTED). + const normalizedSupported = new Set( + [...testHelpers.supportedCopilotModels].map(m => normalizeSeparators(m)), + ); + + const missing: string[] = []; + for (const model of pricingModels) { + if (INTENTIONALLY_EXCLUDED_FROM_SUPPORTED.has(model)) continue; + if (!normalizedSupported.has(normalizeSeparators(model))) { + missing.push(model); + } + } + + if (missing.length > 0) { + const modelList = missing.map(m => ` '${m}'`).join('\n'); + throw new Error( + `Found ${missing.length} model(s) in containers/api-proxy/ai-credits-pricing.js ` + + `that are missing from SUPPORTED_COPILOT_MODELS in src/copilot-model.ts:\n` + + `${modelList}\n\n` + + `If this is a new Copilot CLI completion model, add it to SUPPORTED_COPILOT_MODELS.\n` + + `If it is NOT a Copilot CLI model, add it to INTENTIONALLY_EXCLUDED_FROM_SUPPORTED ` + + `in src/copilot-model-catalog-sync.test.ts with a comment explaining why.`, + ); + } + }); +}); diff --git a/src/copilot-model.ts b/src/copilot-model.ts index 0971676e4..fca9578e5 100644 --- a/src/copilot-model.ts +++ b/src/copilot-model.ts @@ -25,6 +25,22 @@ const RETIRED_COPILOT_MODEL_ALIASES: Record = { 'gpt-5-codex': 'gpt-5.3-codex', }; +/** + * Single source of truth for all Copilot CLI completion models that AWF accepts + * as a `COPILOT_MODEL` value. The host-side validator rejects any value not in + * this set (or its separator-normalised form) before containers start. + * + * When **adding** a new Copilot CLI model here, also update: + * - `containers/api-proxy/ai-credits-pricing.js` (AI-credits billing) + * - `docs/model-api-mapping.json` (endpoint routing, OpenAI/Anthropic only) + * + * When **retiring** a model, move it to `RETIRED_COPILOT_MODEL_ALIASES` (below) + * and mirror the change in `containers/api-proxy/guards/retired-model-guard.js`. + * + * A CI guard in `src/copilot-model-catalog-sync.test.ts` fails if any model + * present in `ai-credits-pricing.js` is missing from this set (and is not + * explicitly listed as a non-CLI model in the test's exclusion set). + */ const SUPPORTED_COPILOT_MODELS = new Set([ 'gpt-4', 'gpt-4.1', @@ -91,6 +107,12 @@ function levenshtein(a: string, b: string): number { return dp[a.length][b.length]; } +/** @internal Exposed for unit tests — catalog sync guard uses this to verify pricing parity. */ +// ts-prune-ignore-next +export const testHelpers = { + supportedCopilotModels: new Set(SUPPORTED_COPILOT_MODELS) as ReadonlySet, +}; + export function validateCopilotModel(rawModel: string): CopilotModelValidationResult { const trimmed = rawModel.trim(); if (!trimmed) {