Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions actions/setup/js/model_costs.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ function providerIncludesCacheReadsInInput(provider) {
case "openai":
case "azure-openai":
case "azure_openai":
case "github-copilot": // proxies OpenAI and Anthropic models that bundle cache reads in input
return true;
default:
return false;
Expand Down
74 changes: 74 additions & 0 deletions actions/setup/js/model_costs.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -132,4 +132,78 @@ describe("model_costs.cjs", () => {
});
expect(aic).toBeGreaterThan(0);
});

it("subtracts cache reads from input for github-copilot provider (§3.5, no double-charge)", async () => {
// github-copilot proxies OpenAI and Anthropic models, which bundle cache-read tokens
// inside the reported input total. §3.5 requires the cache reads to be subtracted
// from input_tokens before the input price is applied, preventing double-charging.
//
// Pricing used in this fixture:
// input: $0.000003 / token
// output: $0.000015 / token
// cache_read: $0.0000003 / token
//
// With 1000 input, 200 output, 400 cache_read:
// net input = 1000 − 400 = 600
// cost = 600×0.000003 + 200×0.000015 + 400×0.0000003 = 0.0018 + 0.003 + 0.00012 = 0.00492
// AIC = 0.00492 / 0.01 = 0.492
writeModelsFixture({
"github-copilot": {
models: {
"claude-sonnet-4.6": {
cost: {
input: "0.000003",
output: "0.000015",
cache_read: "0.0000003",
},
},
},
},
});

const { computeInferenceAIC } = await import("./model_costs.cjs");

for (const provider of ["github-copilot", "github_models", "github", "copilot"]) {
const aic = computeInferenceAIC({
provider,
model: "claude-sonnet-4.6",
inputTokens: 1000,
outputTokens: 200,
cacheReadTokens: 400,
cacheWriteTokens: 0,
});
expect(aic).toBeCloseTo(0.492, 6);

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[/tdd] The JS test covers all four alias inputs (line 166) which is excellent. One missing boundary: cacheReadTokens > inputTokens. The Go implementation uses max(inputTokens-cacheReadTokens, 0) to clamp negative values, but the JS implementation uses Math.max(input - cacheRead, 0) (line 204 of model_costs.cjs). A test with cacheReadTokens: 1200 (> inputTokens: 1000) would confirm the clamp is working and prevent silent negative-cost bugs.

💡 Suggested boundary test
it('clamps net input to 0 when cache_read exceeds input (§3.5 boundary)', async () => {
  // ... same fixture setup ...
  const aic = computeInferenceAIC({
    provider: 'github-copilot',
    model: 'claude-sonnet-4.6',
    inputTokens: 1000,
    outputTokens: 200,
    cacheReadTokens: 1200, // exceeds input
    cacheWriteTokens: 0,
  });
  // net input = max(1000-1200, 0) = 0
  // cost = 0 * 0.000003 + 200 * 0.000015 + 1200 * 0.0000003 = 0.003 + 0.00036 = 0.00336
  expect(aic).toBeCloseTo(0.336, 6);
});

@copilot please address this.

}
});

it("clamps net input to 0 when cache_read exceeds input (§3.5 boundary)", async () => {
writeModelsFixture({
"github-copilot": {
models: {
"claude-sonnet-4.6": {
cost: {
input: "0.000003",
output: "0.000015",
cache_read: "0.0000003",
},
},
},
},
});

const { computeInferenceAIC } = await import("./model_costs.cjs");
const aic = computeInferenceAIC({
provider: "github-copilot",
model: "claude-sonnet-4.6",
inputTokens: 1000,
outputTokens: 200,
cacheReadTokens: 1200,
cacheWriteTokens: 0,
});

// net input = max(1000 − 1200, 0) = 0
// cost = 0×0.000003 + 200×0.000015 + 1200×0.0000003 = 0.00336
// AIC = 0.00336 / 0.01 = 0.336
expect(aic).toBeCloseTo(0.336, 6);
});
});
9 changes: 9 additions & 0 deletions docs/src/content/docs/specs/ai-credits-specification.md
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,15 @@ If a model entry omits optional price fields, implementations MUST apply the fol

For providers that include cache-read tokens in total input tokens, implementations MUST subtract `cache_read_tokens` from `input_tokens` before applying input price and MUST NOT double-charge cache-read usage.

The following providers are known to bundle cache-read tokens in the reported input total and MUST have §3.5 applied:

| Provider (normalized) | Notes |
|-----------------------|-------|
| `anthropic` | Direct Anthropic API |
| `openai` | Direct OpenAI API |
| `azure-openai` / `azure_openai` | Azure-hosted OpenAI |
| `github-copilot` (and aliases `github`, `copilot`, `github_models`) | GitHub Copilot proxy — proxies both OpenAI and Anthropic models, which bundle cache-read tokens in input |

### 3.6 Aggregation

For grouped runs (for example, episodes), implementations MUST aggregate AIC by summing per-invocation AIC values.
Expand Down
5 changes: 4 additions & 1 deletion pkg/cli/effective_tokens.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,11 @@ func providerIncludesCacheReadsInInput(normalizedProvider string) bool {
// the provider field.
// We include both "azure-openai" and "azure_openai" to handle observed
// provider naming variants in historical logs.
// Callers should pass the catalog-normalized provider so canonical aliases like
// "github", "copilot", and "github_models" collapse to "github-copilot"
// before this check.
switch normalizedProvider {
case "", "anthropic", "openai", "azure-openai", "azure_openai":
case "", "anthropic", "openai", "azure-openai", "azure_openai", "github-copilot":
return true
default:
return false
Expand Down
2 changes: 1 addition & 1 deletion pkg/cli/model_costs.go
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ func computeModelInferenceCostUSD(provider, model string, inputTokens, outputTok

input := inputTokens
cacheRead := cacheReadTokens
if cacheRead > 0 && providerIncludesCacheReadsInInput(strings.ToLower(strings.TrimSpace(provider))) {
if cacheRead > 0 && providerIncludesCacheReadsInInput(normalizeCatalogProvider(provider)) {
input = max(inputTokens-cacheReadTokens, 0)
}

Expand Down
35 changes: 35 additions & 0 deletions pkg/cli/model_costs_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,41 @@ func TestComputeModelInferenceAICCopilotAlias(t *testing.T) {
assert.InDelta(t, aicViaGitHubCopilot, aicViaCopilot, 1e-9, "copilot and github-copilot should yield identical AIC")
}

func TestComputeModelInferenceAICGitHubCopilotCacheReadDeduction(t *testing.T) {
// github-copilot (and its aliases) proxies OpenAI and Anthropic models, which bundle
// cache-read tokens inside the reported input total (§3.5). Cache reads MUST be
// subtracted from input_tokens before applying the input price so that they are not
// double-charged.
//
// Pricing for github-copilot/claude-sonnet-4.6:
// input: $0.000003/token
// output: $0.000015/token
// cache_read: $0.0000003/token
//
// With 1000 input, 200 output, 400 cache_read (no cache_write, no reasoning):
// net input = 1000 − 400 = 600
// cost = 600×0.000003 + 200×0.000015 + 400×0.0000003 = 0.0018 + 0.003 + 0.00012 = 0.00492
// AIC = 0.00492 / 0.01 = 0.492
const wantAIC = 0.492

for _, provider := range []string{"github-copilot", "github_models", "github", "copilot"} {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[/tdd] The test iterates over all four provider aliases in a single it block in the JS test, but in Go the loop is inside TestComputeModelInferenceAICGitHubCopilotCacheReadDeduction which is good. However, the existing tests TestComputeModelInferenceAICGitHubModels and TestComputeModelInferenceAICCopilotAlias (lines 49–64) exercise the alias normalization path without cache reads. They do not cover the combination of alias + cache-read deduction — the new test catches that gap.

Consider also adding a negative test: verify that with cacheReadTokens=0, the AIC matches plain input pricing (no unintended subtraction) to prevent future off-by-one regressions at the zero boundary.

💡 Suggested negative test skeleton
func TestComputeModelInferenceAICGitHubCopilotNoCacheRead(t *testing.T) {
    // With no cache reads, net input == inputTokens; no subtraction should occur.
    aicNone := computeModelInferenceAIC("github-copilot", "claude-sonnet-4.6", 1000, 200, 0, 0, 0)
    aicAnthropic := computeModelInferenceAIC("anthropic", "claude-sonnet-4.6", 1000, 200, 0, 0, 0)
    assert.InDelta(t, aicAnthropic, aicNone, 1e-9, "zero cache reads must not alter input token count")
}

@copilot please address this.

t.Run(provider, func(t *testing.T) {
aic := computeModelInferenceAIC(provider, "claude-sonnet-4.6", 1000, 200, 400, 0, 0)
assert.InDelta(t, wantAIC, aic, 1e-9,
"provider=%q: cache reads must not be double-charged (§3.5)", provider)
})
}
}

func TestComputeModelInferenceAICGitHubCopilotNoCacheRead(t *testing.T) {
// With no cache reads, github-copilot pricing should match anthropic exactly:
// net input remains inputTokens, so no subtraction is applied.
aicViaGitHubCopilot := computeModelInferenceAIC("github-copilot", "claude-sonnet-4.6", 1000, 200, 0, 0, 0)
aicViaAnthropic := computeModelInferenceAIC("anthropic", "claude-sonnet-4.6", 1000, 200, 0, 0, 0)
assert.InDelta(t, aicViaAnthropic, aicViaGitHubCopilot, 1e-9,
"zero cache reads must not alter the charged input token count")
}

func TestFindOrFetchModelPricing_EmbeddedModelReturnsNil(t *testing.T) {
// claude-sonnet-4.6 is in the embedded catalog; FindOrFetchModelPricing should return
// (nil, false) so the lock.yml overlay does not duplicate what models.json already has.
Expand Down
Loading