From 7defaea8a013348cbe94248c3147f282a7fc47cd Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Tue, 31 Mar 2026 21:54:42 -0400 Subject: [PATCH 1/3] test(app): route prompt e2e through mock llm --- packages/app/e2e/prompt/prompt.spec.ts | 181 ++++++++++------- packages/opencode/test/lib/llm-server.ts | 243 +++++++++++++++++++++++ 2 files changed, 352 insertions(+), 72 deletions(-) diff --git a/packages/app/e2e/prompt/prompt.spec.ts b/packages/app/e2e/prompt/prompt.spec.ts index 1acf17f5bf44..80c05ce6620b 100644 --- a/packages/app/e2e/prompt/prompt.spec.ts +++ b/packages/app/e2e/prompt/prompt.spec.ts @@ -1,44 +1,66 @@ -import fs from "node:fs/promises" -import path from "node:path" import { test, expect } from "../fixtures" import { promptSelector } from "../selectors" import { sessionIDFromUrl } from "../actions" -import { createSdk } from "../utils" -async function config(dir: string, url: string) { - await fs.writeFile( - path.join(dir, "opencode.json"), - JSON.stringify({ - $schema: "https://opencode.ai/config.json", - enabled_providers: ["e2e-llm"], - provider: { - "e2e-llm": { - name: "E2E LLM", - npm: "@ai-sdk/openai-compatible", - env: [], - models: { - "test-model": { - name: "Test Model", - tool_call: true, - limit: { context: 128000, output: 32000 }, - }, - }, - options: { - apiKey: "test-key", - baseURL: url, - }, - }, - }, - agent: { - build: { - model: "e2e-llm/test-model", - }, - }, - }), - ) +const mdl = { providerID: "openai", modelID: "gpt-5.3-chat-latest" } + +async function pickModel(page: Parameters[0]["page"], value: { providerID: string; modelID: string }) { + await expect + .poll( + () => + page.evaluate(() => { + const win = window as Window & { + __opencode_e2e?: { + model?: { + controls?: { + setModel?: (value: { providerID: string; modelID: string } | undefined) => void + } + } + } + } + return !!win.__opencode_e2e?.model?.controls?.setModel + }), + { timeout: 30_000 }, + ) + .toBe(true) + + await page.evaluate((value) => { + const win = window as Window & { + __opencode_e2e?: { + model?: { + controls?: { + setModel?: (value: { providerID: string; modelID: string } | undefined) => void + } + } + } + } + const fn = win.__opencode_e2e?.model?.controls?.setModel + if (!fn) throw new Error("Model e2e model control is not enabled") + fn(value) + }, value) + + await expect + .poll( + () => + page.evaluate(() => { + const win = window as Window & { + __opencode_e2e?: { + model?: { + current?: { + model?: { providerID: string; modelID: string } + } + } + } + } + const model = win.__opencode_e2e?.model?.current?.model + return model ? `${model.providerID}/${model.modelID}` : null + }), + { timeout: 30_000 }, + ) + .toBe(`${value.providerID}/${value.modelID}`) } -test("can send a prompt and receive a reply", async ({ page, llm, withProject }) => { +test("can send a prompt and receive a reply", async ({ page, llm, sdk, gotoSession }) => { test.setTimeout(120_000) const pageErrors: string[] = [] @@ -47,50 +69,65 @@ test("can send a prompt and receive a reply", async ({ page, llm, withProject }) } page.on("pageerror", onPageError) + const prev = await sdk.global.config.get().then((res) => res.data ?? {}) + try { - await withProject( - async (project) => { - const sdk = createSdk(project.directory) - const token = `E2E_OK_${Date.now()}` + await sdk.global.config.update({ + config: { + ...prev, + model: `${mdl.providerID}/${mdl.modelID}`, + enabled_providers: ["openai"], + provider: { + ...prev.provider, + openai: { + ...prev.provider?.openai, + options: { + ...prev.provider?.openai?.options, + apiKey: "test-key", + baseURL: llm.url, + }, + }, + }, + }, + }) - await llm.text(token) - await project.gotoSession() + const token = `E2E_OK_${Date.now()}` + await llm.text("E2E Title") + await llm.text(token) + await gotoSession() + await pickModel(page, mdl) - const prompt = page.locator(promptSelector) - await prompt.click() - await page.keyboard.type(`Reply with exactly: ${token}`) - await page.keyboard.press("Enter") + const prompt = page.locator(promptSelector) + await prompt.click() + await page.keyboard.type(`Reply with exactly: ${token}`) + await page.keyboard.press("Enter") - await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 }) + await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 }) - const sessionID = (() => { - const id = sessionIDFromUrl(page.url()) - if (!id) throw new Error(`Failed to parse session id from url: ${page.url()}`) - return id - })() - project.trackSession(sessionID) + const sessionID = (() => { + const id = sessionIDFromUrl(page.url()) + if (!id) throw new Error(`Failed to parse session id from url: ${page.url()}`) + return id + })() - await expect - .poll( - async () => { - const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? []) - return messages - .filter((m) => m.info.role === "assistant") - .flatMap((m) => m.parts) - .filter((p) => p.type === "text") - .map((p) => p.text) - .join("\n") - }, - { timeout: 30_000 }, - ) - .toContain(token) - }, - { - model: { providerID: "e2e-llm", modelID: "test-model" }, - setup: (dir) => config(dir, llm.url), - }, - ) + await expect.poll(() => llm.calls()).toBeGreaterThanOrEqual(2) + + await expect + .poll( + async () => { + const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? []) + return messages + .filter((m) => m.info.role === "assistant") + .flatMap((m) => m.parts) + .filter((p) => p.type === "text") + .map((p) => p.text) + .join("\n") + }, + { timeout: 30_000 }, + ) + .toContain(token) } finally { + await sdk.global.config.update({ config: prev }) page.off("pageerror", onPageError) } diff --git a/packages/opencode/test/lib/llm-server.ts b/packages/opencode/test/lib/llm-server.ts index 8e7365d97f4b..d651140b313a 100644 --- a/packages/opencode/test/lib/llm-server.ts +++ b/packages/opencode/test/lib/llm-server.ts @@ -119,6 +119,225 @@ function bytes(input: Iterable) { return Stream.fromIterable([...input].map(line)).pipe(Stream.encodeText) } +function created(model: string) { + return { + type: "response.created", + sequence_number: 1, + response: { + id: "resp_test", + created_at: Math.floor(Date.now() / 1000), + model, + service_tier: null, + }, + } +} + +function completed(input: { seq: number; usage?: Usage }) { + return { + type: "response.completed", + sequence_number: input.seq, + response: { + incomplete_details: null, + service_tier: null, + usage: { + input_tokens: input.usage?.input ?? 0, + input_tokens_details: { cached_tokens: null }, + output_tokens: input.usage?.output ?? 0, + output_tokens_details: { reasoning_tokens: null }, + }, + }, + } +} + +function responses(item: Sse, model: string) { + let seq = 1 + let msg: string | undefined + let reason: string | undefined + let call: + | { + id: string + item: string + name: string + args: string + } + | undefined + let usage: Usage | undefined + const lines: unknown[] = [created(model)] + + const all = [...item.head, ...item.tail] + for (const part of all) { + if (!part || typeof part !== "object") continue + if (!("choices" in part) || !Array.isArray(part.choices)) continue + const choice = part.choices[0] + if (!choice || typeof choice !== "object") continue + const delta = "delta" in choice && choice.delta && typeof choice.delta === "object" ? choice.delta : undefined + + if (delta && "content" in delta && typeof delta.content === "string") { + msg ||= "msg_1" + if ( + !lines.some( + (item) => + typeof item === "object" && + item && + "type" in item && + item.type === "response.output_item.added" && + "item" in item && + item.item && + typeof item.item === "object" && + "id" in item.item && + item.item.id === msg, + ) + ) { + seq += 1 + lines.push({ + type: "response.output_item.added", + sequence_number: seq, + output_index: 0, + item: { type: "message", id: msg }, + }) + } + seq += 1 + lines.push({ + type: "response.output_text.delta", + sequence_number: seq, + item_id: msg, + delta: delta.content, + logprobs: null, + }) + } + + if (delta && "reasoning_content" in delta && typeof delta.reasoning_content === "string") { + reason ||= "rs_1" + if ( + !lines.some( + (item) => + typeof item === "object" && + item && + "type" in item && + item.type === "response.output_item.added" && + "item" in item && + item.item && + typeof item.item === "object" && + "id" in item.item && + item.item.id === reason, + ) + ) { + seq += 1 + lines.push({ + type: "response.output_item.added", + sequence_number: seq, + output_index: 0, + item: { type: "reasoning", id: reason, encrypted_content: null }, + }) + seq += 1 + lines.push({ + type: "response.reasoning_summary_part.added", + sequence_number: seq, + item_id: reason, + summary_index: 0, + }) + } + seq += 1 + lines.push({ + type: "response.reasoning_summary_text.delta", + sequence_number: seq, + item_id: reason, + summary_index: 0, + delta: delta.reasoning_content, + }) + } + + if (delta && "tool_calls" in delta && Array.isArray(delta.tool_calls)) { + for (const tool of delta.tool_calls) { + if (!tool || typeof tool !== "object") continue + const fn = "function" in tool && tool.function && typeof tool.function === "object" ? tool.function : undefined + const id = "id" in tool && typeof tool.id === "string" ? tool.id : call?.id + const name = fn && "name" in fn && typeof fn.name === "string" ? fn.name : call?.name + const args = fn && "arguments" in fn && typeof fn.arguments === "string" ? fn.arguments : "" + if (!id || !name) continue + if (!call) { + call = { id, item: "fc_1", name, args: "" } + seq += 1 + lines.push({ + type: "response.output_item.added", + sequence_number: seq, + output_index: 0, + item: { + type: "function_call", + id: call.item, + call_id: id, + name, + arguments: "", + status: "in_progress", + }, + }) + } + call.args += args + if (args) { + seq += 1 + lines.push({ + type: "response.function_call_arguments.delta", + sequence_number: seq, + output_index: 0, + item_id: call.item, + delta: args, + }) + } + } + } + + if ("usage" in part && part.usage && typeof part.usage === "object") { + const raw = part.usage as Record + if (typeof raw.prompt_tokens === "number" && typeof raw.completion_tokens === "number") { + usage = { input: raw.prompt_tokens, output: raw.completion_tokens } + } + } + } + + if (msg) { + seq += 1 + lines.push({ + type: "response.output_item.done", + sequence_number: seq, + output_index: 0, + item: { type: "message", id: msg }, + }) + } + if (reason) { + seq += 1 + lines.push({ + type: "response.output_item.done", + sequence_number: seq, + output_index: 0, + item: { type: "reasoning", id: reason, encrypted_content: null }, + }) + } + if (call && !item.hang && !item.error) { + seq += 1 + lines.push({ + type: "response.output_item.done", + sequence_number: seq, + output_index: 0, + item: { + type: "function_call", + id: call.item, + call_id: call.id, + name: call.name, + arguments: call.args, + status: "completed", + }, + }) + } + if (!item.hang && !item.error) lines.push(completed({ seq: seq + 1, usage })) + return { ...item, head: lines, tail: [] } satisfies Sse +} + +function modelFrom(body: unknown) { + if (!body || typeof body !== "object") return "test-model" + if (!("model" in body) || typeof body.model !== "string") return "test-model" + return body.model +} + function send(item: Sse) { const head = bytes(item.head) const tail = bytes([...item.tail, ...(item.hang || item.error ? [] : [done])]) @@ -358,6 +577,9 @@ export class TestLLMServer extends ServiceMap.Service ({}))) + hits = [ + ...hits, + { + url: new URL(req.originalUrl, "http://localhost"), + body: body && typeof body === "object" ? (body as Record) : {}, + }, + ] + yield* notify() + if (next.type === "sse") return send(responses(next, modelFrom(body))) + return fail(next) + }), + ) + yield* server.serve(router.asHttpEffect()) return TestLLMServer.of({ From 5d5e170c020c6033b6addc763aecd8c99fbe0366 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Tue, 31 Mar 2026 22:10:31 -0400 Subject: [PATCH 2/3] test: simplify mock llm responses renderer --- packages/opencode/test/lib/llm-server.ts | 415 ++++++++++++----------- 1 file changed, 225 insertions(+), 190 deletions(-) diff --git a/packages/opencode/test/lib/llm-server.ts b/packages/opencode/test/lib/llm-server.ts index d651140b313a..fb84f1175ae8 100644 --- a/packages/opencode/test/lib/llm-server.ts +++ b/packages/opencode/test/lib/llm-server.ts @@ -8,6 +8,13 @@ export type Usage = { input: number; output: number } type Line = Record +type Flow = + | { type: "text"; text: string } + | { type: "reason"; text: string } + | { type: "tool-start"; id: string; name: string } + | { type: "tool-args"; text: string } + | { type: "usage"; usage: Usage } + type Hit = { url: URL body: Record @@ -119,7 +126,7 @@ function bytes(input: Iterable) { return Stream.fromIterable([...input].map(line)).pipe(Stream.encodeText) } -function created(model: string) { +function responseCreated(model: string) { return { type: "response.created", sequence_number: 1, @@ -132,7 +139,7 @@ function created(model: string) { } } -function completed(input: { seq: number; usage?: Usage }) { +function responseCompleted(input: { seq: number; usage?: Usage }) { return { type: "response.completed", sequence_number: input.seq, @@ -149,10 +156,168 @@ function completed(input: { seq: number; usage?: Usage }) { } } +function responseMessage(id: string, seq: number) { + return { + type: "response.output_item.added", + sequence_number: seq, + output_index: 0, + item: { type: "message", id }, + } +} + +function responseText(id: string, text: string, seq: number) { + return { + type: "response.output_text.delta", + sequence_number: seq, + item_id: id, + delta: text, + logprobs: null, + } +} + +function responseMessageDone(id: string, seq: number) { + return { + type: "response.output_item.done", + sequence_number: seq, + output_index: 0, + item: { type: "message", id }, + } +} + +function responseReason(id: string, seq: number) { + return { + type: "response.output_item.added", + sequence_number: seq, + output_index: 0, + item: { type: "reasoning", id, encrypted_content: null }, + } +} + +function responseReasonPart(id: string, seq: number) { + return { + type: "response.reasoning_summary_part.added", + sequence_number: seq, + item_id: id, + summary_index: 0, + } +} + +function responseReasonText(id: string, text: string, seq: number) { + return { + type: "response.reasoning_summary_text.delta", + sequence_number: seq, + item_id: id, + summary_index: 0, + delta: text, + } +} + +function responseReasonDone(id: string, seq: number) { + return { + type: "response.output_item.done", + sequence_number: seq, + output_index: 0, + item: { type: "reasoning", id, encrypted_content: null }, + } +} + +function responseTool(id: string, item: string, name: string, seq: number) { + return { + type: "response.output_item.added", + sequence_number: seq, + output_index: 0, + item: { + type: "function_call", + id: item, + call_id: id, + name, + arguments: "", + status: "in_progress", + }, + } +} + +function responseToolArgs(id: string, text: string, seq: number) { + return { + type: "response.function_call_arguments.delta", + sequence_number: seq, + output_index: 0, + item_id: id, + delta: text, + } +} + +function responseToolDone(tool: { id: string; item: string; name: string; args: string }, seq: number) { + return { + type: "response.output_item.done", + sequence_number: seq, + output_index: 0, + item: { + type: "function_call", + id: tool.item, + call_id: tool.id, + name: tool.name, + arguments: tool.args, + status: "completed", + }, + } +} + +function choices(part: unknown) { + if (!part || typeof part !== "object") return + if (!("choices" in part) || !Array.isArray(part.choices)) return + const choice = part.choices[0] + if (!choice || typeof choice !== "object") return + return choice +} + +function flow(item: Sse) { + const out: Flow[] = [] + for (const part of [...item.head, ...item.tail]) { + const choice = choices(part) + const delta = + choice && "delta" in choice && choice.delta && typeof choice.delta === "object" ? choice.delta : undefined + + if (delta && "content" in delta && typeof delta.content === "string") { + out.push({ type: "text", text: delta.content }) + } + + if (delta && "reasoning_content" in delta && typeof delta.reasoning_content === "string") { + out.push({ type: "reason", text: delta.reasoning_content }) + } + + if (delta && "tool_calls" in delta && Array.isArray(delta.tool_calls)) { + for (const tool of delta.tool_calls) { + if (!tool || typeof tool !== "object") continue + const fn = "function" in tool && tool.function && typeof tool.function === "object" ? tool.function : undefined + if ("id" in tool && typeof tool.id === "string" && fn && "name" in fn && typeof fn.name === "string") { + out.push({ type: "tool-start", id: tool.id, name: fn.name }) + } + if (fn && "arguments" in fn && typeof fn.arguments === "string" && fn.arguments) { + out.push({ type: "tool-args", text: fn.arguments }) + } + } + } + + if (part && typeof part === "object" && "usage" in part && part.usage && typeof part.usage === "object") { + const raw = part.usage as Record + if (typeof raw.prompt_tokens === "number" && typeof raw.completion_tokens === "number") { + out.push({ + type: "usage", + usage: { input: raw.prompt_tokens, output: raw.completion_tokens }, + }) + } + } + } + return out +} + function responses(item: Sse, model: string) { let seq = 1 let msg: string | undefined let reason: string | undefined + let hasMsg = false + let hasReason = false let call: | { id: string @@ -162,173 +327,66 @@ function responses(item: Sse, model: string) { } | undefined let usage: Usage | undefined - const lines: unknown[] = [created(model)] + const lines: unknown[] = [responseCreated(model)] - const all = [...item.head, ...item.tail] - for (const part of all) { - if (!part || typeof part !== "object") continue - if (!("choices" in part) || !Array.isArray(part.choices)) continue - const choice = part.choices[0] - if (!choice || typeof choice !== "object") continue - const delta = "delta" in choice && choice.delta && typeof choice.delta === "object" ? choice.delta : undefined - - if (delta && "content" in delta && typeof delta.content === "string") { - msg ||= "msg_1" - if ( - !lines.some( - (item) => - typeof item === "object" && - item && - "type" in item && - item.type === "response.output_item.added" && - "item" in item && - item.item && - typeof item.item === "object" && - "id" in item.item && - item.item.id === msg, - ) - ) { + for (const part of flow(item)) { + if (part.type === "text") { + msg ??= "msg_1" + if (!hasMsg) { + hasMsg = true seq += 1 - lines.push({ - type: "response.output_item.added", - sequence_number: seq, - output_index: 0, - item: { type: "message", id: msg }, - }) + lines.push(responseMessage(msg, seq)) } seq += 1 - lines.push({ - type: "response.output_text.delta", - sequence_number: seq, - item_id: msg, - delta: delta.content, - logprobs: null, - }) + lines.push(responseText(msg, part.text, seq)) + continue } - if (delta && "reasoning_content" in delta && typeof delta.reasoning_content === "string") { + if (part.type === "reason") { reason ||= "rs_1" - if ( - !lines.some( - (item) => - typeof item === "object" && - item && - "type" in item && - item.type === "response.output_item.added" && - "item" in item && - item.item && - typeof item.item === "object" && - "id" in item.item && - item.item.id === reason, - ) - ) { + if (!hasReason) { + hasReason = true seq += 1 - lines.push({ - type: "response.output_item.added", - sequence_number: seq, - output_index: 0, - item: { type: "reasoning", id: reason, encrypted_content: null }, - }) + lines.push(responseReason(reason, seq)) seq += 1 - lines.push({ - type: "response.reasoning_summary_part.added", - sequence_number: seq, - item_id: reason, - summary_index: 0, - }) + lines.push(responseReasonPart(reason, seq)) } seq += 1 - lines.push({ - type: "response.reasoning_summary_text.delta", - sequence_number: seq, - item_id: reason, - summary_index: 0, - delta: delta.reasoning_content, - }) + lines.push(responseReasonText(reason, part.text, seq)) + continue } - if (delta && "tool_calls" in delta && Array.isArray(delta.tool_calls)) { - for (const tool of delta.tool_calls) { - if (!tool || typeof tool !== "object") continue - const fn = "function" in tool && tool.function && typeof tool.function === "object" ? tool.function : undefined - const id = "id" in tool && typeof tool.id === "string" ? tool.id : call?.id - const name = fn && "name" in fn && typeof fn.name === "string" ? fn.name : call?.name - const args = fn && "arguments" in fn && typeof fn.arguments === "string" ? fn.arguments : "" - if (!id || !name) continue - if (!call) { - call = { id, item: "fc_1", name, args: "" } - seq += 1 - lines.push({ - type: "response.output_item.added", - sequence_number: seq, - output_index: 0, - item: { - type: "function_call", - id: call.item, - call_id: id, - name, - arguments: "", - status: "in_progress", - }, - }) - } - call.args += args - if (args) { - seq += 1 - lines.push({ - type: "response.function_call_arguments.delta", - sequence_number: seq, - output_index: 0, - item_id: call.item, - delta: args, - }) - } - } + if (part.type === "tool-start") { + call ||= { id: part.id, item: "fc_1", name: part.name, args: "" } + seq += 1 + lines.push(responseTool(call.id, call.item, call.name, seq)) + continue } - if ("usage" in part && part.usage && typeof part.usage === "object") { - const raw = part.usage as Record - if (typeof raw.prompt_tokens === "number" && typeof raw.completion_tokens === "number") { - usage = { input: raw.prompt_tokens, output: raw.completion_tokens } - } + if (part.type === "tool-args") { + if (!call) continue + call.args += part.text + seq += 1 + lines.push(responseToolArgs(call.item, part.text, seq)) + continue } + + usage = part.usage } if (msg) { seq += 1 - lines.push({ - type: "response.output_item.done", - sequence_number: seq, - output_index: 0, - item: { type: "message", id: msg }, - }) + lines.push(responseMessageDone(msg, seq)) } if (reason) { seq += 1 - lines.push({ - type: "response.output_item.done", - sequence_number: seq, - output_index: 0, - item: { type: "reasoning", id: reason, encrypted_content: null }, - }) + lines.push(responseReasonDone(reason, seq)) } if (call && !item.hang && !item.error) { seq += 1 - lines.push({ - type: "response.output_item.done", - sequence_number: seq, - output_index: 0, - item: { - type: "function_call", - id: call.item, - call_id: call.id, - name: call.name, - arguments: call.args, - status: "completed", - }, - }) + lines.push(responseToolDone(call, seq)) } - if (!item.hang && !item.error) lines.push(completed({ seq: seq + 1, usage })) + if (!item.hang && !item.error) lines.push(responseCompleted({ seq: seq + 1, usage })) return { ...item, head: lines, tail: [] } satisfies Sse } @@ -512,6 +570,13 @@ function item(input: Item | Reply) { return input instanceof Reply ? input.item() : input } +function hit(url: string, body: unknown) { + return { + url: new URL(url, "http://localhost"), + body: body && typeof body === "object" ? (body as Record) : {}, + } satisfies Hit +} + namespace TestLLMServer { export interface Service { readonly url: string @@ -561,54 +626,24 @@ export class TestLLMServer extends ServiceMap.Service ({}))) - hits = [ - ...hits, - { - url: new URL(req.originalUrl, "http://localhost"), - body: body && typeof body === "object" ? (body as Record) : {}, - }, - ] - yield* notify() - if (req.originalUrl.endsWith("/v1/responses") && next.type === "sse") { - return send(responses(next, modelFrom(body))) - } - if (next.type === "sse" && next.reset) { - yield* reset(next) - return HttpServerResponse.empty() - } - if (next.type === "sse") return send(next) - return fail(next) - }), - ) - - yield* router.add( - "POST", - "/v1/responses", - Effect.gen(function* () { - const req = yield* HttpServerRequest.HttpServerRequest - const next = pull() - if (!next) return HttpServerResponse.text("unexpected request", { status: 500 }) - const body = yield* req.json.pipe(Effect.orElseSucceed(() => ({}))) - hits = [ - ...hits, - { - url: new URL(req.originalUrl, "http://localhost"), - body: body && typeof body === "object" ? (body as Record) : {}, - }, - ] - yield* notify() - if (next.type === "sse") return send(responses(next, modelFrom(body))) - return fail(next) - }), - ) + const handle = Effect.fn("TestLLMServer.handle")(function* (mode: "chat" | "responses") { + const req = yield* HttpServerRequest.HttpServerRequest + const next = pull() + if (!next) return HttpServerResponse.text("unexpected request", { status: 500 }) + const body = yield* req.json.pipe(Effect.orElseSucceed(() => ({}))) + hits = [...hits, hit(req.originalUrl, body)] + yield* notify() + if (next.type !== "sse") return fail(next) + if (mode === "responses") return send(responses(next, modelFrom(body))) + if (next.reset) { + yield* reset(next) + return HttpServerResponse.empty() + } + return send(next) + }) + + yield* router.add("POST", "/v1/chat/completions", handle("chat")) + yield* router.add("POST", "/v1/responses", handle("responses")) yield* server.serve(router.asHttpEffect()) From e5a4c164e000b23a9f7ae0d538f9898fbff892d2 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Tue, 31 Mar 2026 23:17:50 -0400 Subject: [PATCH 3/3] test(app): revert unstable mock prompt routing --- packages/app/e2e/prompt/prompt.spec.ts | 181 ++++++++++--------------- 1 file changed, 72 insertions(+), 109 deletions(-) diff --git a/packages/app/e2e/prompt/prompt.spec.ts b/packages/app/e2e/prompt/prompt.spec.ts index 80c05ce6620b..1acf17f5bf44 100644 --- a/packages/app/e2e/prompt/prompt.spec.ts +++ b/packages/app/e2e/prompt/prompt.spec.ts @@ -1,66 +1,44 @@ +import fs from "node:fs/promises" +import path from "node:path" import { test, expect } from "../fixtures" import { promptSelector } from "../selectors" import { sessionIDFromUrl } from "../actions" +import { createSdk } from "../utils" -const mdl = { providerID: "openai", modelID: "gpt-5.3-chat-latest" } - -async function pickModel(page: Parameters[0]["page"], value: { providerID: string; modelID: string }) { - await expect - .poll( - () => - page.evaluate(() => { - const win = window as Window & { - __opencode_e2e?: { - model?: { - controls?: { - setModel?: (value: { providerID: string; modelID: string } | undefined) => void - } - } - } - } - return !!win.__opencode_e2e?.model?.controls?.setModel - }), - { timeout: 30_000 }, - ) - .toBe(true) - - await page.evaluate((value) => { - const win = window as Window & { - __opencode_e2e?: { - model?: { - controls?: { - setModel?: (value: { providerID: string; modelID: string } | undefined) => void - } - } - } - } - const fn = win.__opencode_e2e?.model?.controls?.setModel - if (!fn) throw new Error("Model e2e model control is not enabled") - fn(value) - }, value) - - await expect - .poll( - () => - page.evaluate(() => { - const win = window as Window & { - __opencode_e2e?: { - model?: { - current?: { - model?: { providerID: string; modelID: string } - } - } - } - } - const model = win.__opencode_e2e?.model?.current?.model - return model ? `${model.providerID}/${model.modelID}` : null - }), - { timeout: 30_000 }, - ) - .toBe(`${value.providerID}/${value.modelID}`) +async function config(dir: string, url: string) { + await fs.writeFile( + path.join(dir, "opencode.json"), + JSON.stringify({ + $schema: "https://opencode.ai/config.json", + enabled_providers: ["e2e-llm"], + provider: { + "e2e-llm": { + name: "E2E LLM", + npm: "@ai-sdk/openai-compatible", + env: [], + models: { + "test-model": { + name: "Test Model", + tool_call: true, + limit: { context: 128000, output: 32000 }, + }, + }, + options: { + apiKey: "test-key", + baseURL: url, + }, + }, + }, + agent: { + build: { + model: "e2e-llm/test-model", + }, + }, + }), + ) } -test("can send a prompt and receive a reply", async ({ page, llm, sdk, gotoSession }) => { +test("can send a prompt and receive a reply", async ({ page, llm, withProject }) => { test.setTimeout(120_000) const pageErrors: string[] = [] @@ -69,65 +47,50 @@ test("can send a prompt and receive a reply", async ({ page, llm, sdk, gotoSessi } page.on("pageerror", onPageError) - const prev = await sdk.global.config.get().then((res) => res.data ?? {}) - try { - await sdk.global.config.update({ - config: { - ...prev, - model: `${mdl.providerID}/${mdl.modelID}`, - enabled_providers: ["openai"], - provider: { - ...prev.provider, - openai: { - ...prev.provider?.openai, - options: { - ...prev.provider?.openai?.options, - apiKey: "test-key", - baseURL: llm.url, - }, - }, - }, - }, - }) - - const token = `E2E_OK_${Date.now()}` - await llm.text("E2E Title") - await llm.text(token) - await gotoSession() - await pickModel(page, mdl) + await withProject( + async (project) => { + const sdk = createSdk(project.directory) + const token = `E2E_OK_${Date.now()}` - const prompt = page.locator(promptSelector) - await prompt.click() - await page.keyboard.type(`Reply with exactly: ${token}`) - await page.keyboard.press("Enter") + await llm.text(token) + await project.gotoSession() - await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 }) + const prompt = page.locator(promptSelector) + await prompt.click() + await page.keyboard.type(`Reply with exactly: ${token}`) + await page.keyboard.press("Enter") - const sessionID = (() => { - const id = sessionIDFromUrl(page.url()) - if (!id) throw new Error(`Failed to parse session id from url: ${page.url()}`) - return id - })() + await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 }) - await expect.poll(() => llm.calls()).toBeGreaterThanOrEqual(2) + const sessionID = (() => { + const id = sessionIDFromUrl(page.url()) + if (!id) throw new Error(`Failed to parse session id from url: ${page.url()}`) + return id + })() + project.trackSession(sessionID) - await expect - .poll( - async () => { - const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? []) - return messages - .filter((m) => m.info.role === "assistant") - .flatMap((m) => m.parts) - .filter((p) => p.type === "text") - .map((p) => p.text) - .join("\n") - }, - { timeout: 30_000 }, - ) - .toContain(token) + await expect + .poll( + async () => { + const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? []) + return messages + .filter((m) => m.info.role === "assistant") + .flatMap((m) => m.parts) + .filter((p) => p.type === "text") + .map((p) => p.text) + .join("\n") + }, + { timeout: 30_000 }, + ) + .toContain(token) + }, + { + model: { providerID: "e2e-llm", modelID: "test-model" }, + setup: (dir) => config(dir, llm.url), + }, + ) } finally { - await sdk.global.config.update({ config: prev }) page.off("pageerror", onPageError) }