From 548e5f61ade94c40b5701b70f65a5dbbbba1f0b7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 19 Jun 2026 19:10:24 +0000 Subject: [PATCH] sync: update AgentMark packages Updated 14 files from upstream. Changed files: .nx/version-plans/multimodal-blob-offload.md packages/api-schemas/src/schemas/experiments.ts packages/api-types/src/types.ts packages/prompt-core/src/webhook-runner.ts packages/prompt-core/test/webhook-runner.test.ts packages/ui-components/src/sections/traces/trace-drawer/hooks/use-selected-span-io.ts packages/ui-components/src/sections/traces/trace-drawer/span-info/tabs/hooks/use-span-prompts.ts packages/ui-components/src/sections/traces/trace-drawer/span-info/tabs/input-output-tab/input-output-tab.tsx packages/ui-components/src/sections/traces/trace-drawer/span-info/tabs/input-output-tab/offloaded-fields.tsx packages/ui-components/src/sections/traces/trace-drawer/trace-drawer-provider.tsx packages/ui-components/src/sections/traces/types/index.ts packages/ui-components/test/span-info/offloaded-fields.test.tsx packages/ui-components/test/span-info/use-span-prompts-blobrefs.test.tsx packages/ui-components/test/span-info/use-span-prompts.test.ts --- .nx/version-plans/multimodal-blob-offload.md | 28 +++ .../api-schemas/src/schemas/experiments.ts | 4 + packages/api-types/src/types.ts | 13 ++ packages/prompt-core/src/webhook-runner.ts | 30 ++- .../prompt-core/test/webhook-runner.test.ts | 57 ++++++ .../hooks/use-selected-span-io.ts | 1 + .../span-info/tabs/hooks/use-span-prompts.ts | 5 + .../input-output-tab/input-output-tab.tsx | 4 +- .../input-output-tab/offloaded-fields.tsx | 161 ++++++++++++++++ .../trace-drawer/trace-drawer-provider.tsx | 6 + .../src/sections/traces/types/index.ts | 9 + .../test/span-info/offloaded-fields.test.tsx | 181 ++++++++++++++++++ .../use-span-prompts-blobrefs.test.tsx | 91 +++++++++ .../test/span-info/use-span-prompts.test.ts | 11 ++ 14 files changed, 598 insertions(+), 3 deletions(-) create mode 100644 .nx/version-plans/multimodal-blob-offload.md create mode 100644 packages/ui-components/src/sections/traces/trace-drawer/span-info/tabs/input-output-tab/offloaded-fields.tsx create mode 100644 packages/ui-components/test/span-info/offloaded-fields.test.tsx create mode 100644 packages/ui-components/test/span-info/use-span-prompts-blobrefs.test.tsx diff --git a/.nx/version-plans/multimodal-blob-offload.md b/.nx/version-plans/multimodal-blob-offload.md new file mode 100644 index 00000000..3b6c9329 --- /dev/null +++ b/.nx/version-plans/multimodal-blob-offload.md @@ -0,0 +1,28 @@ +--- +'@agentmark-ai/api-types': minor +'@agentmark-ai/api-schemas': minor +'@agentmark-ai/prompt-core': minor +'@agentmark-ai/ui-components': minor +--- + +Size-driven blob offload for trace I/O (multimodal output support). + +Oversized span fields (image/audio/large text output, large inputs, tool calls) +are lifted to object storage at ingest; ClickHouse keeps an 8KB inline preview +plus a `BlobRefs` pointer, so the 128KB queue-message limit never truncates a +generation. Full-fidelity consumers fetch the full value back on demand. + +- **api-types**: `Span` / `SpanIO` gain an optional `blobRefs` (JSON array of + offloaded-field pointers); `ExperimentItemSummary` gains an optional + `blobRefs` so the experiment-detail path can rehydrate offloaded item I/O. + All additive — existing consumers are unaffected. +- **api-schemas**: `ExperimentItemSummarySchema` gains an optional `blobRefs` + (the gateway rehydrates the full value into `input`/`output` before + responding, so consumers may ignore it). +- **prompt-core**: the webhook runner records image/speech generation output via + `setSpanOutput` (the `agentmark.output` attribute) so generated media is + captured on the span and offloaded like any other oversized field. +- **ui-components**: the trace drawer's Input/Output tab renders every offloaded + field — image/audio inline (data URIs), full text/JSON otherwise — fetched on + demand via the host-provided `fetchBlob`; `OutputObject` is deduped when + `Output` is also offloaded. diff --git a/packages/api-schemas/src/schemas/experiments.ts b/packages/api-schemas/src/schemas/experiments.ts index 6ed3d7ab..c5d38aac 100644 --- a/packages/api-schemas/src/schemas/experiments.ts +++ b/packages/api-schemas/src/schemas/experiments.ts @@ -72,6 +72,10 @@ export const ExperimentItemSummarySchema = z.object({ tokens: z.number(), model: z.string(), scores: z.array(ExperimentItemScoreSchema), + // Present (carried from ClickHouse) only when this item's input/output was + // offloaded to object storage at ingest. The gateway rehydrates the full + // value into `input`/`output` before responding, so consumers can ignore it. + blobRefs: z.string().optional(), }); export const ExperimentDetailSchema = ExperimentSummarySchema.extend({ diff --git a/packages/api-types/src/types.ts b/packages/api-types/src/types.ts index c77ebc91..41f089d1 100644 --- a/packages/api-types/src/types.ts +++ b/packages/api-types/src/types.ts @@ -451,6 +451,13 @@ export interface Span { spanKind: string; serviceName: string; promptName: string | null; + /** + * JSON array of pointers to oversized field payloads offloaded to object + * storage: `[{field,blob_id,size}]`. Empty/absent when nothing was offloaded. + * When present, the matching inline column (input/output/...) holds only a + * preview; the full value is fetched from storage by blob_id. + */ + blobRefs?: string; } /** @@ -461,6 +468,12 @@ export interface SpanIO { output: string; outputObject: string | null; toolCalls: string | null; + /** + * JSON array of pointers to oversized field payloads offloaded to object + * storage: `[{field,blob_id,size}]`. When present, the inline columns hold + * previews and the full values are fetched from storage by blob_id. + */ + blobRefs?: string; /** * Custom per-span metadata (raw map; reserved namespaces stripped at the API * boundary). Optional on the internal type so existing producers/mocks need diff --git a/packages/prompt-core/src/webhook-runner.ts b/packages/prompt-core/src/webhook-runner.ts index e058454c..d4a8e595 100644 --- a/packages/prompt-core/src/webhook-runner.ts +++ b/packages/prompt-core/src/webhook-runner.ts @@ -202,6 +202,24 @@ function setSpanInput(ctx: SpanLike, input: unknown): void { } } +/** + * Record the prompt's output on the span as `agentmark.output` (the key the + * normalizer reads as the output fallback). For image/speech prompts this is + * what carries the generated media (`{mimeType, base64}`) into the trace so + * the gateway can lift it to object storage at ingest — without it the + * generated media is never captured on a span. + */ +function setSpanOutput(ctx: SpanLike, output: unknown): void { + try { + ctx.setAttribute( + "agentmark.output", + typeof output === "string" ? output : JSON.stringify(output) + ); + } catch { + /* tracing must never break the run */ + } +} + /** * Record the prompt's configured model as `gen_ai.request.model` on the * prompt span. The runner reads it from frontmatter (adapter-agnostic), so @@ -887,7 +905,11 @@ export class WebhookRunner< span: ctx, promptName: frontmatter.name, }; - return executeImage(input, ctxExec); + const exec = await executeImage(input, ctxExec); + // result is Array<{ mimeType, base64 }> — captured so the gateway can + // lift it to object storage (see utils/media-extraction.ts). + setSpanOutput(ctx, (exec as { result?: unknown }).result); + return exec; } ); return { ...(await result), traceId } as WebhookPromptResponse; @@ -926,7 +948,11 @@ export class WebhookRunner< span: ctx, promptName: frontmatter.name, }; - return executeSpeech(input, ctxExec); + const exec = await executeSpeech(input, ctxExec); + // result is { mimeType, base64, format } — captured so the gateway can + // lift it to object storage (see utils/media-extraction.ts). + setSpanOutput(ctx, (exec as { result?: unknown }).result); + return exec; } ); return { ...(await result), traceId } as WebhookPromptResponse; diff --git a/packages/prompt-core/test/webhook-runner.test.ts b/packages/prompt-core/test/webhook-runner.test.ts index e33f30a2..c6096a07 100644 --- a/packages/prompt-core/test/webhook-runner.test.ts +++ b/packages/prompt-core/test/webhook-runner.test.ts @@ -20,6 +20,7 @@ import type { DatasetStreamChunk, SpanLike, ExperimentItemSpanHook, + PromptSpanHook, } from "../src/index"; import { WebhookRunner } from "../src/webhook-runner"; @@ -613,6 +614,62 @@ describe("WebhookRunner — image/speech runPrompt", () => { ); }); + // ── setSpanOutput: capture the generated media on the span ────────────────── + // Without this the image/audio output is never recorded on a span, so the + // gateway has nothing to offload to object storage and the trace viewer has + // nothing to render. + function makeCapturingPromptHook() { + const attrs: Record = {}; + const span: SpanLike = { traceId: "", setAttribute: (k, v) => { attrs[k] = v; } }; + const hook = (async (_p: unknown, fn: (s: SpanLike) => Promise) => { + const result = await fn(span); + return { result, traceId: span.traceId }; + }) as PromptSpanHook; + return { hook, attrs }; + } + + it("captures the image result onto the span as agentmark.output (JSON array)", async () => { + const { hook, attrs } = makeCapturingPromptHook(); + const media = [{ mimeType: "image/png", base64: "iVBOR" }]; + const exec = makeMediaExecutor({ image: async () => ({ type: "image", result: media }) }); + const runner = new WebhookRunner(makeClient([]), exec, { promptSpanHook: hook }); + + await runner.runPrompt(IMAGE_AST); + + expect(attrs["agentmark.output"]).toBe(JSON.stringify(media)); + expect(attrs["gen_ai.request.model"]).toBe("test"); // setSpanModel still runs + }); + + it("captures the speech result onto the span as agentmark.output", async () => { + const { hook, attrs } = makeCapturingPromptHook(); + const audio = { mimeType: "audio/mpeg", base64: "SUQz", format: "mp3" }; + const exec = makeMediaExecutor({ speech: async () => ({ type: "speech", result: audio }) }); + const runner = new WebhookRunner(makeClient([]), exec, { promptSpanHook: hook }); + + await runner.runPrompt(SPEECH_AST); + + expect(attrs["agentmark.output"]).toBe(JSON.stringify(audio)); + }); + + it("does not break the run when setAttribute throws (tracing is best-effort)", async () => { + const span: SpanLike = { + traceId: "", + setAttribute: () => { throw new Error("span backend down"); }, + }; + const hook = (async (_p: unknown, fn: (s: SpanLike) => Promise) => { + const result = await fn(span); + return { result, traceId: span.traceId }; + }) as PromptSpanHook; + const exec = makeMediaExecutor({ + image: async () => ({ type: "image", result: [{ mimeType: "image/png", base64: "x" }] }), + }); + const runner = new WebhookRunner(makeClient([]), exec, { promptSpanHook: hook }); + + // setSpanInput/Model/Output all swallow errors — the run still returns. + const res: any = await runner.runPrompt(IMAGE_AST); + expect(res.type).toBe("image"); + }); + it("throws 'Invalid prompt' when frontmatter declares no recognized config", async () => { const exec = makeMediaExecutor({}); const runner = new WebhookRunner(makeClient([]), exec); diff --git a/packages/ui-components/src/sections/traces/trace-drawer/hooks/use-selected-span-io.ts b/packages/ui-components/src/sections/traces/trace-drawer/hooks/use-selected-span-io.ts index 60226a75..b6c4b54a 100644 --- a/packages/ui-components/src/sections/traces/trace-drawer/hooks/use-selected-span-io.ts +++ b/packages/ui-components/src/sections/traces/trace-drawer/hooks/use-selected-span-io.ts @@ -22,6 +22,7 @@ export const mergeSpanIO = ( output: io.output, outputObject: io.outputObject, toolCalls: io.toolCalls, + blobRefs: io.blobRefs, } as SpanData["data"], }; }; diff --git a/packages/ui-components/src/sections/traces/trace-drawer/span-info/tabs/hooks/use-span-prompts.ts b/packages/ui-components/src/sections/traces/trace-drawer/span-info/tabs/hooks/use-span-prompts.ts index 33594cec..eb9ef8e8 100644 --- a/packages/ui-components/src/sections/traces/trace-drawer/span-info/tabs/hooks/use-span-prompts.ts +++ b/packages/ui-components/src/sections/traces/trace-drawer/span-info/tabs/hooks/use-span-prompts.ts @@ -18,6 +18,8 @@ interface UseSpanPromptsResult { objectResponse?: any; documents?: RetrievalDocumentView[]; } | null; + /** JSON array of offloaded field pointers from the effective (IO-hydrated) span. */ + blobRefs?: string; isLoading: boolean; isLoadingIO: boolean; } @@ -319,6 +321,9 @@ export const useSpanPrompts = (): UseSpanPromptsResult => { return { prompts, outputData, + // From the effective span so it reflects the lazily-hydrated IO (the raw + // selectedSpan has no blobRefs until merge) — drives OffloadedOutput. + blobRefs: effectiveSpan?.data?.blobRefs, isLoading: !selectedSpan, isLoadingIO, }; diff --git a/packages/ui-components/src/sections/traces/trace-drawer/span-info/tabs/input-output-tab/input-output-tab.tsx b/packages/ui-components/src/sections/traces/trace-drawer/span-info/tabs/input-output-tab/input-output-tab.tsx index cbc2faa4..4bab4b1f 100644 --- a/packages/ui-components/src/sections/traces/trace-drawer/span-info/tabs/input-output-tab/input-output-tab.tsx +++ b/packages/ui-components/src/sections/traces/trace-drawer/span-info/tabs/input-output-tab/input-output-tab.tsx @@ -1,6 +1,7 @@ import { useSpanPrompts } from "../hooks/use-span-prompts"; import { PromptList } from "./prompt-list"; import { OutputDisplay } from "./output-display"; +import { OffloadedFields } from "./offloaded-fields"; import { TabPanel } from "@mui/lab"; import { Alert, Box, Skeleton } from "@mui/material"; import { useSpanInfoContext } from "../../span-info-provider"; @@ -15,7 +16,7 @@ const IOLoadingSkeleton = () => ( ); export const InputOutputTab = () => { - const { prompts, outputData, isLoadingIO } = useSpanPrompts(); + const { prompts, outputData, isLoadingIO, blobRefs } = useSpanPrompts(); const { span } = useSpanInfoContext(); return ( @@ -39,6 +40,7 @@ export const InputOutputTab = () => { <> + {blobRefs && } )} {span.data.statusMessage && ( diff --git a/packages/ui-components/src/sections/traces/trace-drawer/span-info/tabs/input-output-tab/offloaded-fields.tsx b/packages/ui-components/src/sections/traces/trace-drawer/span-info/tabs/input-output-tab/offloaded-fields.tsx new file mode 100644 index 00000000..851c0898 --- /dev/null +++ b/packages/ui-components/src/sections/traces/trace-drawer/span-info/tabs/input-output-tab/offloaded-fields.tsx @@ -0,0 +1,161 @@ +import { useEffect, useMemo, useState } from "react"; +import { Box, Skeleton, Typography } from "@mui/material"; +import { useTraceDrawerContext } from "../../../trace-drawer-provider"; + +/** Pointer recorded in SpanData.data.blobRefs for an offloaded field. */ +interface BlobRef { + field: string; + blob_id: string; + size: number; +} + +interface MediaItem { + mime: string; + base64: string; +} + +/** Human label per offloaded field. Also gates which fields we render. */ +const FIELD_LABEL: Record = { + Input: "Full input", + Output: "Full output", + OutputObject: "Full output object", + ToolCalls: "Full tool calls", +}; + +const FIELD_ORDER = ["Input", "Output", "OutputObject", "ToolCalls"]; + +/** + * Walk an arbitrary parsed payload and collect every `{ mimeType|mediaType, + * base64 }` leaf — covers image generation (array of items) and speech + * generation (single item), wherever they sit in the payload shape. + */ +function collectMedia(value: unknown, acc: MediaItem[] = []): MediaItem[] { + if (!value || typeof value !== "object") return acc; + if (Array.isArray(value)) { + for (const v of value) collectMedia(v, acc); + return acc; + } + const obj = value as Record; + const mime = obj.mimeType ?? obj.mediaType; + if (typeof mime === "string" && typeof obj.base64 === "string") { + acc.push({ mime, base64: obj.base64 }); + return acc; + } + for (const v of Object.values(obj)) collectMedia(v, acc); + return acc; +} + +/** + * Renders ONE offloaded field's full value, fetched on demand from object + * storage via the host `fetchBlob`. Image/audio render inline; everything else + * (large text, JSON object, tool calls) renders as full text. + */ +const OffloadedField = ({ field, blobId }: { field: string; blobId: string }) => { + const { fetchBlob } = useTraceDrawerContext(); + const [state, setState] = useState<{ loading: boolean; content?: string; error?: boolean }>({ + loading: false, + }); + + useEffect(() => { + if (!blobId || !fetchBlob) return; + let cancelled = false; + setState({ loading: true }); + fetchBlob(blobId) + .then((content) => { + if (cancelled) return; + setState({ loading: false, content: content ?? undefined, error: content == null }); + }) + .catch(() => { + if (!cancelled) setState({ loading: false, error: true }); + }); + return () => { + cancelled = true; + }; + }, [blobId, fetchBlob]); + + let body: React.ReactNode; + if (state.loading) { + body = ; + } else if (state.error || state.content == null) { + body = ( + + Could not load the full {(FIELD_LABEL[field] ?? field).replace(/^Full /, "")}. + + ); + } else { + let media: MediaItem[] = []; + try { + media = collectMedia(JSON.parse(state.content)); + } catch { + /* not JSON — show as text */ + } + body = + media.length > 0 ? ( + media.map((m, i) => + m.mime.startsWith("image/") ? ( + {`${field} + ) : m.mime.startsWith("audio/") ? ( +