diff --git a/actions/setup/js/model_costs.cjs b/actions/setup/js/model_costs.cjs index 03bb914cd33..9b9f3535dcf 100644 --- a/actions/setup/js/model_costs.cjs +++ b/actions/setup/js/model_costs.cjs @@ -102,6 +102,7 @@ function providerIncludesCacheReadsInInput(provider) { case "openai": case "azure-openai": case "azure_openai": + case "github-copilot": // proxies OpenAI and Anthropic models that bundle cache reads in input return true; default: return false; diff --git a/actions/setup/js/model_costs.test.cjs b/actions/setup/js/model_costs.test.cjs index 0d025b39729..e2a2db410e1 100644 --- a/actions/setup/js/model_costs.test.cjs +++ b/actions/setup/js/model_costs.test.cjs @@ -132,4 +132,78 @@ describe("model_costs.cjs", () => { }); expect(aic).toBeGreaterThan(0); }); + + it("subtracts cache reads from input for github-copilot provider (§3.5, no double-charge)", async () => { + // github-copilot proxies OpenAI and Anthropic models, which bundle cache-read tokens + // inside the reported input total. §3.5 requires the cache reads to be subtracted + // from input_tokens before the input price is applied, preventing double-charging. + // + // Pricing used in this fixture: + // input: $0.000003 / token + // output: $0.000015 / token + // cache_read: $0.0000003 / token + // + // With 1000 input, 200 output, 400 cache_read: + // net input = 1000 − 400 = 600 + // cost = 600×0.000003 + 200×0.000015 + 400×0.0000003 = 0.0018 + 0.003 + 0.00012 = 0.00492 + // AIC = 0.00492 / 0.01 = 0.492 + writeModelsFixture({ + "github-copilot": { + models: { + "claude-sonnet-4.6": { + cost: { + input: "0.000003", + output: "0.000015", + cache_read: "0.0000003", + }, + }, + }, + }, + }); + + const { computeInferenceAIC } = await import("./model_costs.cjs"); + + for (const provider of ["github-copilot", "github_models", "github", "copilot"]) { + const aic = computeInferenceAIC({ + provider, + model: "claude-sonnet-4.6", + inputTokens: 1000, + outputTokens: 200, + cacheReadTokens: 400, + cacheWriteTokens: 0, + }); + expect(aic).toBeCloseTo(0.492, 6); + } + }); + + it("clamps net input to 0 when cache_read exceeds input (§3.5 boundary)", async () => { + writeModelsFixture({ + "github-copilot": { + models: { + "claude-sonnet-4.6": { + cost: { + input: "0.000003", + output: "0.000015", + cache_read: "0.0000003", + }, + }, + }, + }, + }); + + const { computeInferenceAIC } = await import("./model_costs.cjs"); + const aic = computeInferenceAIC({ + provider: "github-copilot", + model: "claude-sonnet-4.6", + inputTokens: 1000, + outputTokens: 200, + cacheReadTokens: 1200, + cacheWriteTokens: 0, + }); + + // net input = max(1000 − 1200, 0) = 0 + // cost = 0×0.000003 + 200×0.000015 + 1200×0.0000003 = 0.00336 + // AIC = 0.00336 / 0.01 = 0.336 + expect(aic).toBeCloseTo(0.336, 6); + }); }); diff --git a/docs/src/content/docs/specs/ai-credits-specification.md b/docs/src/content/docs/specs/ai-credits-specification.md index 46db13b3ae5..8f14f40c203 100644 --- a/docs/src/content/docs/specs/ai-credits-specification.md +++ b/docs/src/content/docs/specs/ai-credits-specification.md @@ -147,6 +147,15 @@ If a model entry omits optional price fields, implementations MUST apply the fol For providers that include cache-read tokens in total input tokens, implementations MUST subtract `cache_read_tokens` from `input_tokens` before applying input price and MUST NOT double-charge cache-read usage. +The following providers are known to bundle cache-read tokens in the reported input total and MUST have §3.5 applied: + +| Provider (normalized) | Notes | +|-----------------------|-------| +| `anthropic` | Direct Anthropic API | +| `openai` | Direct OpenAI API | +| `azure-openai` / `azure_openai` | Azure-hosted OpenAI | +| `github-copilot` (and aliases `github`, `copilot`, `github_models`) | GitHub Copilot proxy — proxies both OpenAI and Anthropic models, which bundle cache-read tokens in input | + ### 3.6 Aggregation For grouped runs (for example, episodes), implementations MUST aggregate AIC by summing per-invocation AIC values. diff --git a/pkg/cli/effective_tokens.go b/pkg/cli/effective_tokens.go index f6390ce4ad0..7b8de2f6cb1 100644 --- a/pkg/cli/effective_tokens.go +++ b/pkg/cli/effective_tokens.go @@ -37,8 +37,11 @@ func providerIncludesCacheReadsInInput(normalizedProvider string) bool { // the provider field. // We include both "azure-openai" and "azure_openai" to handle observed // provider naming variants in historical logs. + // Callers should pass the catalog-normalized provider so canonical aliases like + // "github", "copilot", and "github_models" collapse to "github-copilot" + // before this check. switch normalizedProvider { - case "", "anthropic", "openai", "azure-openai", "azure_openai": + case "", "anthropic", "openai", "azure-openai", "azure_openai", "github-copilot": return true default: return false diff --git a/pkg/cli/model_costs.go b/pkg/cli/model_costs.go index e9d8d1624de..cc4f2370381 100644 --- a/pkg/cli/model_costs.go +++ b/pkg/cli/model_costs.go @@ -167,7 +167,7 @@ func computeModelInferenceCostUSD(provider, model string, inputTokens, outputTok input := inputTokens cacheRead := cacheReadTokens - if cacheRead > 0 && providerIncludesCacheReadsInInput(strings.ToLower(strings.TrimSpace(provider))) { + if cacheRead > 0 && providerIncludesCacheReadsInInput(normalizeCatalogProvider(provider)) { input = max(inputTokens-cacheReadTokens, 0) } diff --git a/pkg/cli/model_costs_test.go b/pkg/cli/model_costs_test.go index 14fc158c400..5c65cf64a27 100644 --- a/pkg/cli/model_costs_test.go +++ b/pkg/cli/model_costs_test.go @@ -63,6 +63,41 @@ func TestComputeModelInferenceAICCopilotAlias(t *testing.T) { assert.InDelta(t, aicViaGitHubCopilot, aicViaCopilot, 1e-9, "copilot and github-copilot should yield identical AIC") } +func TestComputeModelInferenceAICGitHubCopilotCacheReadDeduction(t *testing.T) { + // github-copilot (and its aliases) proxies OpenAI and Anthropic models, which bundle + // cache-read tokens inside the reported input total (§3.5). Cache reads MUST be + // subtracted from input_tokens before applying the input price so that they are not + // double-charged. + // + // Pricing for github-copilot/claude-sonnet-4.6: + // input: $0.000003/token + // output: $0.000015/token + // cache_read: $0.0000003/token + // + // With 1000 input, 200 output, 400 cache_read (no cache_write, no reasoning): + // net input = 1000 − 400 = 600 + // cost = 600×0.000003 + 200×0.000015 + 400×0.0000003 = 0.0018 + 0.003 + 0.00012 = 0.00492 + // AIC = 0.00492 / 0.01 = 0.492 + const wantAIC = 0.492 + + for _, provider := range []string{"github-copilot", "github_models", "github", "copilot"} { + t.Run(provider, func(t *testing.T) { + aic := computeModelInferenceAIC(provider, "claude-sonnet-4.6", 1000, 200, 400, 0, 0) + assert.InDelta(t, wantAIC, aic, 1e-9, + "provider=%q: cache reads must not be double-charged (§3.5)", provider) + }) + } +} + +func TestComputeModelInferenceAICGitHubCopilotNoCacheRead(t *testing.T) { + // With no cache reads, github-copilot pricing should match anthropic exactly: + // net input remains inputTokens, so no subtraction is applied. + aicViaGitHubCopilot := computeModelInferenceAIC("github-copilot", "claude-sonnet-4.6", 1000, 200, 0, 0, 0) + aicViaAnthropic := computeModelInferenceAIC("anthropic", "claude-sonnet-4.6", 1000, 200, 0, 0, 0) + assert.InDelta(t, aicViaAnthropic, aicViaGitHubCopilot, 1e-9, + "zero cache reads must not alter the charged input token count") +} + func TestFindOrFetchModelPricing_EmbeddedModelReturnsNil(t *testing.T) { // claude-sonnet-4.6 is in the embedded catalog; FindOrFetchModelPricing should return // (nil, false) so the lock.yml overlay does not duplicate what models.json already has.