From 1d3e2f299ca29c92130b4fc57c0df24a54053ba5 Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sat, 25 Jul 2026 02:14:32 +0800 Subject: [PATCH 01/40] Resync transport session states after server-link reconnect A turn that settles while the daemon-server WebSocket is down emits its authoritative session.state idle into a dead socket: live timeline events are control-plane (trySend drops them silently, no queue/replay), so the server and every browser keep rendering the session as working forever, and queued composer sends stay held. Observed live on deck_sub_26624c1t: idle at 23:46:27 never left the daemon; the phone showed "Agent working" plus a long-consumed queued message until 01:43, and three stops later. On every ServerLink RE-connect, re-broadcast each transport session's current authoritative session.state (byte-identical payload builder as the live onStatusChange path, so authoritative-idle shape guarantees hold) and re-push the session record through the persist callback so a store PUT lost in the same outage heals too. First connect is excluded - startup already runs a full session sync. Co-Authored-By: Claude Fable 5 --- src/agent/session-manager.ts | 179 +++++++++++++----- src/daemon/lifecycle.ts | 12 +- src/daemon/server-link.ts | 34 ++++ .../session-manager-state-resync.test.ts | 156 +++++++++++++++ test/daemon/server-link.test.ts | 55 +++++- 5 files changed, 387 insertions(+), 49 deletions(-) create mode 100644 test/agent/session-manager-state-resync.test.ts diff --git a/src/agent/session-manager.ts b/src/agent/session-manager.ts index 6993cafbb..77f843640 100644 --- a/src/agent/session-manager.ts +++ b/src/agent/session-manager.ts @@ -1477,6 +1477,132 @@ async function drainTransportResendQueueIntoRuntime( } } +interface TransportSessionStatePayloadBuild { + /** Raw status mapped to the session-state vocabulary ('running' | 'idle' | 'error' | …). */ + mapped: string; + /** `mapped` after the blocking-work override (idle with blocking work → running). */ + effectiveState: string; + payload: Record; + providerErrorMessage?: string; +} + +/** + * Build the `session.state` timeline payload for a transport session's CURRENT + * runtime status. Shared between the live `onStatusChange` edge emitter and the + * server-link reconnect resync so both produce byte-identical authoritative + * payloads (`isAuthoritativeIdlePayloadShape` requires the exact idle field + * contract — see the pendingCount/pendingVersion alias note below). + */ +function buildTransportSessionStatePayload( + sessionName: string, + runtime: TransportSessionRuntime, + status: string, + decision: { decisionReason: string; clearSource: string; queueReason: string }, +): TransportSessionStatePayloadBuild { + const mapped = isWorkingSessionState(status) ? 'running' : status; + // Include pending info only on idle — the authoritative "turn done, queue empty" signal. + // During running/streaming, command-handler's 'queued' event is the sole queue-update + // authority. This keeps queued messages visible in the UI until the drained turn completes. + const activity = runtime.getDiagnosticSnapshot(); + const effectiveState = mapped === 'idle' && activity.blockingWorkCount > 0 ? 'running' : mapped; + const providerError = runtime.lastProviderError; + const payload: Record = { state: effectiveState }; + if (effectiveState === 'running') { + payload.activityGeneration = activity.activityGeneration; + payload.blockingWorkCount = activity.blockingWorkCount; + payload.activeWorkCount = activity.blockingWorkCount; + payload.activeToolCount = activity.activeToolCount; + payload.busyReasons = activity.busyReasons; + } + if (effectiveState === 'idle') { + payload.authoritative = true; + payload.activityGeneration = activity.activityGeneration; + payload.blockingWorkCount = 0; + payload.activeWorkCount = 0; + payload.activeToolCount = 0; + payload.busyReasons = []; + payload.decisionReason = decision.decisionReason; + payload.clearInputs = [ + { source: decision.clearSource, reason: 'clear', count: 0 }, + ]; + if (activity.completedTurn) { + payload[PEER_AUDIT_COMPLETED_TURN_PAYLOAD_FIELD] = activity.completedTurn; + } + const queuePayload = buildTransportQueueSnapshotPayload(sessionName, decision.queueReason); + Object.assign(payload, queuePayload); + // Canonical authoritative-idle contract fields. The shared validator + // (`isAuthoritativeIdlePayloadShape`) requires numeric `pendingCount` and + // `pendingVersion`; the queue snapshot only carries + // `pendingMessageVersion`/`pendingMessageEntries`, so without these two + // aliases the web demotes every daemon idle to a WEAK idle — and any + // unmatched tool.call in the timeline then keeps the session rendered + // "working" forever even though the reconciler proved clean idle + // (observed live on deck_sub_3l6z4l39). + payload.pendingCount = queuePayload.pendingMessageEntries.length; + payload.pendingVersion = queuePayload.pendingMessageVersion; + } else if (mapped === 'error' && providerError?.message) { + payload.error = providerError.message; + } + return { + mapped, + effectiveState, + payload, + ...(providerError?.message ? { providerErrorMessage: providerError.message } : {}), + }; +} + +/** Monotonic epoch so each reconnect resync gets its own stable eventId set. */ +let transportStateResyncEpoch = 0; + +/** + * Re-broadcast every live transport session's CURRENT authoritative + * `session.state` after the server link is restored. + * + * Why: live `timeline.event` messages are control-plane — `ServerLink.trySend` + * silently drops them while the socket is not OPEN, and there is no replay. + * A turn that settles during a link outage therefore emits its authoritative + * idle into a dead socket: the server (and every browser) keeps rendering the + * session as "working" forever, queued composer sends stay held, and no later + * event ever corrects it (observed live: deck_sub_26624c1t stuck "working" + * 23:46→01:43 while the daemon store/timeline were idle the whole time). + * + * The resync also re-pushes the session record through the persist callback so + * a store PUT lost in the same outage heals too. Payloads reuse the exact + * builder the live path uses, so authoritative-idle shape guarantees hold. + * + * `entries` is injectable for tests; production callers pass nothing. + */ +export function resyncTransportSessionStatesAfterLinkRestore( + entries?: ReadonlyArray, +): number { + const list = entries ?? [...transportRuntimes.entries()]; + if (list.length === 0) return 0; + transportStateResyncEpoch += 1; + const epoch = transportStateResyncEpoch; + let emitted = 0; + for (const [sessionName, runtime] of list) { + try { + const built = buildTransportSessionStatePayload(sessionName, runtime, runtime.getStatus(), { + decisionReason: 'server_link_resync', + clearSource: 'server-link-resync', + queueReason: 'server_link_resync', + }); + timelineEmitter.emit(sessionName, 'session.state', built.payload, { + source: 'daemon', + confidence: 'high', + eventId: `transport-state-resync:${sessionName}:${epoch}`, + }); + const record = getSession(sessionName); + if (record) emitSessionPersist(record, sessionName); + emitted += 1; + } catch (err) { + logger.warn({ err, session: sessionName }, 'transport session state resync failed'); + } + } + logger.info({ sessions: emitted, epoch }, 'ServerLink resync: re-broadcast transport session states'); + return emitted; +} + function wireTransportCallbacks(runtime: TransportSessionRuntime, sessionName: string): void { const transportUserEventId = (clientMessageId: string) => `transport-user:${clientMessageId}`; const persistTransportState = (state: unknown, error?: string): void => { @@ -1501,52 +1627,13 @@ function wireTransportCallbacks(runtime: TransportSessionRuntime, sessionName: s if (status === 'thinking') { timelineEmitter.emit(sessionName, 'assistant.thinking', { text: '' }, { source: 'daemon', confidence: 'high' }); } - const mapped = isWorkingSessionState(status) ? 'running' : status; - // Include pending info only on idle — the authoritative "turn done, queue empty" signal. - // During running/streaming, command-handler's 'queued' event is the sole queue-update - // authority. This keeps queued messages visible in the UI until the drained turn completes. - const activity = runtime.getDiagnosticSnapshot(); - const effectiveMapped = mapped === 'idle' && activity.blockingWorkCount > 0 ? 'running' : mapped; - const providerError = runtime.lastProviderError; - persistTransportState(effectiveMapped, mapped === 'error' ? providerError?.message : undefined); - const payload: Record = { state: effectiveMapped }; - if (effectiveMapped === 'running') { - payload.activityGeneration = activity.activityGeneration; - payload.blockingWorkCount = activity.blockingWorkCount; - payload.activeWorkCount = activity.blockingWorkCount; - payload.activeToolCount = activity.activeToolCount; - payload.busyReasons = activity.busyReasons; - } - if (effectiveMapped === 'idle') { - payload.authoritative = true; - payload.activityGeneration = activity.activityGeneration; - payload.blockingWorkCount = 0; - payload.activeWorkCount = 0; - payload.activeToolCount = 0; - payload.busyReasons = []; - payload.decisionReason = 'activity_reconciler_clear'; - payload.clearInputs = [ - { source: 'transport-runtime', reason: 'clear', count: 0 }, - ]; - if (activity.completedTurn) { - payload[PEER_AUDIT_COMPLETED_TURN_PAYLOAD_FIELD] = activity.completedTurn; - } - const queuePayload = buildTransportQueueSnapshotPayload(sessionName, 'transport_status_idle'); - Object.assign(payload, queuePayload); - // Canonical authoritative-idle contract fields. The shared validator - // (`isAuthoritativeIdlePayloadShape`) requires numeric `pendingCount` and - // `pendingVersion`; the queue snapshot only carries - // `pendingMessageVersion`/`pendingMessageEntries`, so without these two - // aliases the web demotes every daemon idle to a WEAK idle — and any - // unmatched tool.call in the timeline then keeps the session rendered - // "working" forever even though the reconciler proved clean idle - // (observed live on deck_sub_3l6z4l39). - payload.pendingCount = queuePayload.pendingMessageEntries.length; - payload.pendingVersion = queuePayload.pendingMessageVersion; - } else if (mapped === 'error' && providerError?.message) { - payload.error = providerError.message; - } - timelineEmitter.emit(sessionName, 'session.state', payload, { source: 'daemon', confidence: 'high' }); + const built = buildTransportSessionStatePayload(sessionName, runtime, status, { + decisionReason: 'activity_reconciler_clear', + clearSource: 'transport-runtime', + queueReason: 'transport_status_idle', + }); + persistTransportState(built.effectiveState, built.mapped === 'error' ? built.providerErrorMessage : undefined); + timelineEmitter.emit(sessionName, 'session.state', built.payload, { source: 'daemon', confidence: 'high' }); if (status === 'error') { void recoverTransportRuntimeAfterError(sessionName, runtime); } diff --git a/src/daemon/lifecycle.ts b/src/daemon/lifecycle.ts index 6ac4b3bac..ab8ffbd78 100644 --- a/src/daemon/lifecycle.ts +++ b/src/daemon/lifecycle.ts @@ -1,9 +1,9 @@ import { loadStore, flushStore, listSessions, getSession, upsertSession, removeSession, type SessionRecord } from '../store/session-store.js'; -import { restoreFromStore, setSessionEventCallback, setSessionPersistCallback, restartSession, respawnSession, initOnStartup, rebuildProviderRoutes, getTransportRuntime, unregisterProviderRoute } from '../agent/session-manager.js'; +import { restoreFromStore, setSessionEventCallback, setSessionPersistCallback, restartSession, respawnSession, initOnStartup, rebuildProviderRoutes, getTransportRuntime, unregisterProviderRoute, resyncTransportSessionStatesAfterLinkRestore } from '../agent/session-manager.js'; import { sessionExists, isPaneAlive, BACKEND, killSession } from '../agent/tmux.js'; import { detectRepo } from '../repo/detector.js'; import { repoCache, RepoCache } from '../repo/cache.js'; -import { ServerLink } from './server-link.js'; +import { ServerLink, setServerLinkReconnectResyncHandler } from './server-link.js'; import { handleWebCommand, setRouterContext, refreshCodexQuotaMetadata, refreshClaudeSdkSubQuotaMetadata } from './command-handler.js'; import { dispatchSessionMessageByName } from './session-dispatch.js'; import { initFileTransfer, startCleanupTimer } from './file-transfer-handler.js'; @@ -684,6 +684,14 @@ export async function startup(): Promise { let scheduleServerLinkRestoreBroadcast: (() => void) | null = null; if (creds) { serverLink = new ServerLink({ workerUrl: workerUrl!, serverId, token }); + // Heal the exact loss window observed on deck_sub_26624c1t: an + // authoritative `session.state: idle` emitted while the ServerLink socket + // was down is silently dropped (control-plane, no replay), leaving the + // server + every browser rendering the session "working" forever. After + // each RE-connect, re-broadcast every transport session's current state. + setServerLinkReconnectResyncHandler(() => { + resyncTransportSessionStatesAfterLinkRestore(); + }); serverLink.onMessage((msg) => { handleWebCommand(msg, serverLink!); }); diff --git a/src/daemon/server-link.ts b/src/daemon/server-link.ts index 770be16ab..82453e7ba 100644 --- a/src/daemon/server-link.ts +++ b/src/daemon/server-link.ts @@ -159,6 +159,17 @@ const SILENT_CONNECTION_RECYCLE_MS = 30_000; // silence checks stand down until inbound can actually be read again. const LOOP_PROBE_MS = 1_000; const EVENT_LOOP_STALL_THRESHOLD_MS = 3_000; // probe overdue by >3 s ⇒ loop stalled + +/** + * Invoked (async, best-effort) after every successful RE-connect so the session + * layer can re-broadcast authoritative per-session state that live control-plane + * relays dropped while the link was down. Registered from daemon startup + * (lifecycle) to avoid a server-link → session-manager import cycle. + */ +let serverLinkReconnectResyncHandler: (() => void) | null = null; +export function setServerLinkReconnectResyncHandler(handler: (() => void) | null): void { + serverLinkReconnectResyncHandler = handler; +} const DAEMON_STATIC_CAPABILITIES = [ SESSION_GROUP_CLONE_CAPABILITY_V1, // Distinct from session-group-clone — gates the dedicated execution-clone @@ -250,6 +261,10 @@ export class ServerLink { private backoffMs = INITIAL_BACKOFF_MS; private stopping = false; private reconnecting = false; + /** True once this link has completed at least one successful open. Gates the + * reconnect state-resync so a daemon's FIRST connect (startup already runs + * its own full session sync) does not double-broadcast. */ + private hadConnectedBefore = false; private lastPong = 0; // timestamp of last received message (any message counts as proof of life) private seq = 0; private readonly workerUrl: string; @@ -377,6 +392,25 @@ export class ServerLink { // sit there until the next enqueue happened to schedule another flush. this.flushDataPlaneAfterReconnect(); + // Control-plane messages (live timeline events, incl. the authoritative + // `session.state: idle`) are DROPPED by trySend while the socket is not + // OPEN — there is no queue or replay for them. A turn that settles inside + // an outage window therefore leaves the server and every browser showing + // "working" forever. On every RE-connect, let the session layer + // re-broadcast each transport session's current authoritative state so + // the lost snapshot heals within one round-trip. + if (this.hadConnectedBefore && serverLinkReconnectResyncHandler) { + const handler = serverLinkReconnectResyncHandler; + setImmediate(() => { + try { + handler(); + } catch (err) { + logger.warn({ err }, 'ServerLink: reconnect state resync handler failed'); + } + }); + } + this.hadConnectedBefore = true; + // Refresh the supervisor global-defaults cache on every (re)connect so // user edits to "Global custom instructions" land in the daemon within // one WS round-trip, not next restart. See `supervisor-defaults-cache.ts`. diff --git a/test/agent/session-manager-state-resync.test.ts b/test/agent/session-manager-state-resync.test.ts new file mode 100644 index 000000000..6c4318391 --- /dev/null +++ b/test/agent/session-manager-state-resync.test.ts @@ -0,0 +1,156 @@ +/** + * ServerLink reconnect state resync — regression for the deck_sub_26624c1t + * incident (2026-07-24 23:46 → 01:43): + * + * Live `timeline.event` relays are control-plane and are silently DROPPED by + * `ServerLink.trySend` while the socket is not OPEN. CC2's turn settled during + * a link outage, so its authoritative `session.state: idle` (seq 2271, + * decisionReason `activity_reconciler_clear`) never reached the server — every + * browser kept rendering "Agent working…" for two hours and held queued + * composer sends, while the daemon-local store/timeline were idle all along. + * + * `resyncTransportSessionStatesAfterLinkRestore()` re-broadcasts each transport + * session's CURRENT authoritative state after the link comes back. These tests + * pin: (1) the resync idle payload satisfies the shared authoritative-idle + * shape validator (a weak idle would be overridden client-side and NOT heal the + * stuck "working" footer); (2) running sessions re-broadcast running with + * activity evidence; (3) the session record is re-pushed through the persist + * callback so a store PUT lost in the same outage heals too. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../../src/util/logger.js', () => ({ + default: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); + +import { + resyncTransportSessionStatesAfterLinkRestore, + setSessionPersistCallback, +} from '../../src/agent/session-manager.js'; +import { timelineEmitter } from '../../src/daemon/timeline-emitter.js'; +import { isAuthoritativeIdlePayloadShape } from '../../shared/session-activity-types.js'; +import { upsertSession, removeSession } from '../../src/store/session-store.js'; +import type { TransportSessionRuntime } from '../../src/agent/transport-session-runtime.js'; + +const IDLE_SESSION = 'deck_storecheckresync_w1'; +const RUNNING_SESSION = 'deck_storecheckresync_w2'; + +function makeRuntime(overrides: { + status: string; + blockingWorkCount?: number; + activeToolCount?: number; + busyReasons?: string[]; +}): TransportSessionRuntime { + return { + getStatus: () => overrides.status, + getDiagnosticSnapshot: () => ({ + activityGeneration: { scope: 'session', sessionName: IDLE_SESSION, generation: 3 }, + blockingWorkCount: overrides.blockingWorkCount ?? 0, + activeToolCount: overrides.activeToolCount ?? 0, + busyReasons: overrides.busyReasons ?? [], + }), + lastProviderError: undefined, + } as unknown as TransportSessionRuntime; +} + +describe('resyncTransportSessionStatesAfterLinkRestore', () => { + let emitSpy: ReturnType; + + beforeEach(() => { + // Intercept BEFORE any emit so nothing is persisted to timeline JSONL and + // nothing leaves this test (hygiene: shared/test-session-guard.ts names + // are used anyway as belt-and-braces). + emitSpy = vi.spyOn(timelineEmitter, 'emit').mockImplementation(() => null as never); + }); + + afterEach(() => { + emitSpy.mockRestore(); + setSessionPersistCallback(async () => {}); + removeSession(IDLE_SESSION); + removeSession(RUNNING_SESSION); + }); + + it('re-broadcasts an idle transport session as a FULL authoritative idle (not a weak idle)', () => { + const emitted = resyncTransportSessionStatesAfterLinkRestore([ + [IDLE_SESSION, makeRuntime({ status: 'idle' })], + ]); + + expect(emitted).toBe(1); + expect(emitSpy).toHaveBeenCalledTimes(1); + const [sessionName, type, payload, opts] = emitSpy.mock.calls[0] as unknown as [ + string, + string, + Record, + { eventId?: string }, + ]; + expect(sessionName).toBe(IDLE_SESSION); + expect(type).toBe('session.state'); + expect(payload.state).toBe('idle'); + expect(payload.decisionReason).toBe('server_link_resync'); + // The critical contract: a resync idle MUST satisfy the shared + // authoritative-idle validator. A weak idle is demoted client-side + // (isWeakIdlePayload) and cannot un-stick a "working" footer. + expect(isAuthoritativeIdlePayloadShape(payload)).toBe(true); + // Stable per-epoch eventId so a double-fired resync updates in place. + expect(opts.eventId).toMatch(new RegExp(`^transport-state-resync:${IDLE_SESSION}:\\d+$`)); + }); + + it('re-broadcasts a running transport session as running with activity evidence', () => { + const emitted = resyncTransportSessionStatesAfterLinkRestore([ + [RUNNING_SESSION, makeRuntime({ status: 'streaming', blockingWorkCount: 2, busyReasons: ['runtime_dispatch'] })], + ]); + + expect(emitted).toBe(1); + const [, , payload] = emitSpy.mock.calls[0] as unknown as [string, string, Record]; + expect(payload.state).toBe('running'); + expect(payload.blockingWorkCount).toBe(2); + expect(payload.busyReasons).toEqual(['runtime_dispatch']); + }); + + it('treats idle-with-blocking-work as running (same override as the live path)', () => { + resyncTransportSessionStatesAfterLinkRestore([ + [RUNNING_SESSION, makeRuntime({ status: 'idle', blockingWorkCount: 1 })], + ]); + const [, , payload] = emitSpy.mock.calls[0] as unknown as [string, string, Record]; + expect(payload.state).toBe('running'); + }); + + it('re-pushes the session record through the persist callback so a lost store PUT heals', async () => { + upsertSession({ + name: IDLE_SESSION, + projectName: 'storecheckresync', + role: 'w1', + agentType: 'claude-code-sdk', + runtimeType: 'transport', + state: 'idle', + } as never); + const persisted: string[] = []; + setSessionPersistCallback(async (_record, name) => { + persisted.push(name); + }); + + resyncTransportSessionStatesAfterLinkRestore([ + [IDLE_SESSION, makeRuntime({ status: 'idle' })], + ]); + + expect(persisted).toEqual([IDLE_SESSION]); + }); + + it('keeps going when one runtime throws and reports only successful sessions', () => { + const broken = { + getStatus: () => { + throw new Error('boom'); + }, + } as unknown as TransportSessionRuntime; + const emitted = resyncTransportSessionStatesAfterLinkRestore([ + ['deck_storecheckresync_w3', broken], + [IDLE_SESSION, makeRuntime({ status: 'idle' })], + ]); + expect(emitted).toBe(1); + }); + + it('is a no-op with no live transport runtimes', () => { + expect(resyncTransportSessionStatesAfterLinkRestore([])).toBe(0); + expect(emitSpy).not.toHaveBeenCalled(); + }); +}); diff --git a/test/daemon/server-link.test.ts b/test/daemon/server-link.test.ts index 20b135c24..e52a6993e 100644 --- a/test/daemon/server-link.test.ts +++ b/test/daemon/server-link.test.ts @@ -15,7 +15,7 @@ vi.mock('../../src/util/daemon-status.js', () => ({ recordDaemonServerLinkStatus: vi.fn(), })); -import { ServerLink, __setServerLinkDataPlaneQueueConfigForTests } from '../../src/daemon/server-link.js'; +import { ServerLink, __setServerLinkDataPlaneQueueConfigForTests, setServerLinkReconnectResyncHandler } from '../../src/daemon/server-link.js'; import { recordDaemonServerLinkStatus } from '../../src/util/daemon-status.js'; import { TIMELINE_MESSAGES, TIMELINE_PROTOCOL_CAPABILITY } from '../../shared/timeline-protocol.js'; import { TRANSPORT_EVENT } from '../../shared/transport-events.js'; @@ -343,4 +343,57 @@ describe('ServerLink', () => { // The previous WS instance must have been explicitly closed. expect(mockWsInstance.close).toHaveBeenCalledTimes(1); }); + + it('fires the reconnect state-resync handler on RE-connect only, not the first connect', async () => { + // Regression for deck_sub_26624c1t: live timeline events (incl. the + // authoritative session.state idle) are control-plane and silently dropped + // while the socket is down — with no replay, a turn that settles during an + // outage leaves every browser rendering "working" forever. The open handler + // must invoke the registered resync hook after every reconnect so the + // session layer can re-broadcast current authoritative states. The FIRST + // connect must NOT fire it (daemon startup already runs a full session sync). + const resync = vi.fn(); + setServerLinkReconnectResyncHandler(resync); + vi.useFakeTimers(); + try { + link.connect(); + const openHandler = mockWsInstance.addEventListener.mock.calls.find(([type]) => type === 'open')?.[1] as + | (() => void) + | undefined; + expect(openHandler).toBeDefined(); + + // First successful open — startup path, no resync. + openHandler?.(); + await vi.advanceTimersByTimeAsync(0); // flush the scheduled setImmediate + expect(resync).not.toHaveBeenCalled(); + + // Re-connect (same socket object in this harness; the handler's stale-ws + // guard passes because this.ws is unchanged). + openHandler?.(); + await vi.advanceTimersByTimeAsync(0); + expect(resync).toHaveBeenCalledTimes(1); + } finally { + setServerLinkReconnectResyncHandler(null); + } + }); + + it('survives a throwing resync handler without breaking the open handshake', async () => { + setServerLinkReconnectResyncHandler(() => { + throw new Error('resync boom'); + }); + vi.useFakeTimers(); + try { + link.connect(); + const openHandler = mockWsInstance.addEventListener.mock.calls.find(([type]) => type === 'open')?.[1] as + | (() => void) + | undefined; + openHandler?.(); + openHandler?.(); + await vi.advanceTimersByTimeAsync(0); + // No throw escaped to the test — the handler failure is contained. + expect(link.isConnected()).toBe(true); + } finally { + setServerLinkReconnectResyncHandler(null); + } + }); }); From 9406cb1469ac3d8c7984af97d6a9b9e88e0ae129 Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sat, 25 Jul 2026 07:58:34 +0800 Subject: [PATCH 02/40] Report daemon disk capacity and surface insufficient-capacity uploads Desktop status bar now shows per-partition disk usage from the daemon: a new fs.statfs-based collector (no df/wmic subprocess) enumerates real mounts on Linux (/proc/self/mounts with pseudo-fs filtering, device dedupe and \040 unescaping), macOS (/ plus /Volumes with firmlink dedupe) and Windows (26 fail-fast drive probes), cached 15s with lazy background refresh, and rides along in daemon.stats. Mobile hides it; more than two partitions collapse into the fullest one plus a tooltip listing all, in shared GB/TB units. Full-disk uploads now fail with a localized "insufficient capacity" error instead of a generic upload failure: the daemon tags ENOSPC with a shared error code, the server maps it to HTTP 507, and the web matches it via one shared isInsufficientCapacityError helper on BOTH the real composer upload path (SessionControls) and the attachment download/preview button - the original branch only covered the latter, which an independent audit flagged as missing the actual requirement. All seven locales gain upload.insufficient_capacity. Co-Authored-By: Claude Fable 5 --- server/src/routes/file-transfer.ts | 10 +- shared/disk-usage.ts | 16 +++ shared/transport/file-transfer.ts | 9 ++ src/daemon/disk-usage.ts | 133 +++++++++++++++++++++++++ src/daemon/file-transfer-handler.ts | 13 ++- src/daemon/server-link.ts | 7 ++ test/daemon/disk-usage.test.ts | 131 ++++++++++++++++++++++++ web/src/components/ChatView.tsx | 4 +- web/src/components/SessionControls.tsx | 5 +- web/src/components/SubSessionBar.tsx | 63 ++++++++++++ web/src/i18n/locales/en.json | 1 + web/src/i18n/locales/es.json | 1 + web/src/i18n/locales/ja.json | 1 + web/src/i18n/locales/ko.json | 1 + web/src/i18n/locales/ru.json | 1 + web/src/i18n/locales/zh-CN.json | 1 + web/src/i18n/locales/zh-TW.json | 1 + web/src/upload-error.ts | 15 +++ web/src/ws-client.ts | 3 +- web/test/upload-error.test.ts | 23 +++++ 20 files changed, 433 insertions(+), 6 deletions(-) create mode 100644 shared/disk-usage.ts create mode 100644 src/daemon/disk-usage.ts create mode 100644 test/daemon/disk-usage.test.ts create mode 100644 web/src/upload-error.ts create mode 100644 web/test/upload-error.test.ts diff --git a/server/src/routes/file-transfer.ts b/server/src/routes/file-transfer.ts index cb19029bf..9c3a032dd 100644 --- a/server/src/routes/file-transfer.ts +++ b/server/src/routes/file-transfer.ts @@ -9,6 +9,7 @@ import { WsBridge } from '../ws/bridge.js'; import { randomHex } from '../security/crypto.js'; import { FILE_TRANSFER_LIMITS, + FILE_TRANSFER_UPLOAD_ERROR_CODE, FILE_TRANSFER_UPLOAD_FETCH_CAPABILITY, FILE_TRANSFER_DOWNLOAD_STREAM_CAPABILITY, FILE_TRANSFER_PATH_HANDLE_CAPABILITY, @@ -685,8 +686,13 @@ fileTransferRoutes.post('/:id/upload', async (c) => { ); if (result.type === 'file.upload_error') { - logger.warn({ serverId, uploadId, error: result.message }, 'Daemon upload error'); - return c.json({ error: 'upload_failed', message: result.message }, 500); + const code = typeof (result as { code?: unknown }).code === 'string' + ? (result as { code: string }).code + : 'upload_failed'; + logger.warn({ serverId, uploadId, error: result.message, code }, 'Daemon upload error'); + // 507 Insufficient Storage when the daemon's disk is full; 500 otherwise. + const status = code === FILE_TRANSFER_UPLOAD_ERROR_CODE.INSUFFICIENT_CAPACITY ? 507 : 500; + return c.json({ error: code, message: result.message }, status); } const attachment = result.attachment as AttachmentRef; diff --git a/shared/disk-usage.ts b/shared/disk-usage.ts new file mode 100644 index 000000000..0a3874f90 --- /dev/null +++ b/shared/disk-usage.ts @@ -0,0 +1,16 @@ +/** + * A mounted filesystem's capacity, reported by the daemon inside `daemon.stats` + * for the desktop status bar. Shared between the daemon (producer) and web + * (consumer). Mobile clients ignore it; the desktop UI renders up to two inline + * and spills the rest into a tooltip. + */ +export interface DiskUsage { + /** Mount point on POSIX (e.g. "/", "/data") or drive root on Windows ("C:\\"). */ + mount: string; + /** Total filesystem size in bytes. */ + totalBytes: number; + /** Used bytes (total − free, includes reserved blocks). */ + usedBytes: number; + /** used/total as a whole-number percentage 0–100. */ + usedPercent: number; +} diff --git a/shared/transport/file-transfer.ts b/shared/transport/file-transfer.ts index 3cd93f160..9a004a223 100644 --- a/shared/transport/file-transfer.ts +++ b/shared/transport/file-transfer.ts @@ -76,6 +76,13 @@ export const FILE_TRANSFER_PATH_HANDLE_CAPABILITY = 'file.transfer.path_handle.v export const FILE_TRANSFER_PATH_MAX_BYTES = 4 * 1024; export const FILE_TRANSFER_ERROR_MAX_BYTES = 256; +/** Machine-readable upload-error codes shared by the daemon (producer), server + * (relay) and web (localized display). */ +export const FILE_TRANSFER_UPLOAD_ERROR_CODE = { + /** The daemon ran out of disk space writing the upload (ENOSPC). */ + INSUFFICIENT_CAPACITY: 'insufficient_capacity', +} as const; + // ── Server → Daemon messages ────────────────────────────────────────────────── export const FILE_TRANSFER_MSG = { @@ -156,6 +163,8 @@ export interface FileUploadError { type: 'file.upload_error'; uploadId: string; message: string; + /** Machine-readable code so the UI can localize (e.g. ENOSPC → "insufficient capacity"). */ + code?: string; } export interface FileUploadProgress { diff --git a/src/daemon/disk-usage.ts b/src/daemon/disk-usage.ts new file mode 100644 index 000000000..c95f181be --- /dev/null +++ b/src/daemon/disk-usage.ts @@ -0,0 +1,133 @@ +/** + * Cross-platform disk-capacity collection for the `daemon.stats` status bar. + * + * Efficiency: uses the `fs.statfs` syscall (libuv → statvfs / GetDiskFreeSpaceEx) + * — no `df`/`wmic` subprocess. Mount enumeration is a cheap file read on Linux + * (/proc/self/mounts), a readdir on macOS (/Volumes), and 26 fail-fast statfs + * probes on Windows (drive letters). Results are cached with a short TTL and + * refreshed in the background so the synchronous send path never blocks. + */ +import { statfs, readFile, readdir } from 'node:fs/promises'; +import type { DiskUsage } from '../../shared/disk-usage.js'; +import logger from '../util/logger.js'; + +// Pseudo / virtual filesystems to skip on Linux (fstype column of /proc/self/mounts). +const LINUX_PSEUDO_FSTYPES = new Set([ + 'proc', 'sysfs', 'devtmpfs', 'devpts', 'tmpfs', 'ramfs', 'securityfs', 'pstore', + 'bpf', 'cgroup', 'cgroup2', 'autofs', 'mqueue', 'debugfs', 'tracefs', 'hugetlbfs', + 'fusectl', 'configfs', 'binfmt_misc', 'nsfs', 'overlay', 'squashfs', 'efivarfs', + 'rpc_pipefs', 'selinuxfs', 'fuse.gvfsd-fuse', 'fuse.portal', 'fuse.snapfuse', + 'fuse.snapd', 'ramfs', 'devfs', +]); + +const DISK_CACHE_TTL_MS = 15_000; +// A statfs on a wedged network mount (e.g. a stale SMB/NFS volume under macOS +// /Volumes) can hang indefinitely; cap it so one bad mount can't stall the +// whole refresh (which would otherwise leave `refreshing` stuck and freeze the +// snapshot forever). Linux already excludes network fs (non-/dev/ devices). +const STATFS_TIMEOUT_MS = 2_000; +let cache: { at: number; disks: DiskUsage[] } | null = null; +let refreshing = false; + +function withTimeout(promise: Promise, ms: number): Promise { + return new Promise((resolve) => { + const timer = setTimeout(() => resolve(null), ms); + promise.then( + (value) => { clearTimeout(timer); resolve(value); }, + () => { clearTimeout(timer); resolve(null); }, + ); + }); +} + +async function usageForMount(mount: string): Promise { + try { + const s = await withTimeout(statfs(mount), STATFS_TIMEOUT_MS); + if (!s) return null; + const bsize = Number(s.bsize); + const totalBytes = Number(s.blocks) * bsize; + if (!Number.isFinite(totalBytes) || totalBytes <= 0) return null; + const usedBytes = Math.max(0, totalBytes - Number(s.bfree) * bsize); + const usedPercent = Math.min(100, Math.max(0, Math.round((usedBytes / totalBytes) * 100))); + return { mount, totalBytes, usedBytes, usedPercent }; + } catch { + // Non-existent drive letter (Windows), unmounted volume, permission — skip. + return null; + } +} + +async function enumerateMounts(): Promise { + if (process.platform === 'linux') { + const txt = await readFile('/proc/self/mounts', 'utf8').catch(() => ''); + const mounts: string[] = []; + const seenDev = new Set(); + for (const line of txt.split('\n')) { + const parts = line.split(' '); + const dev = parts[0]; + const mount = parts[1]; + const fstype = parts[2]; + if (!dev || !mount || !fstype) continue; + if (!dev.startsWith('/dev/')) continue; // real block devices only + if (LINUX_PSEUDO_FSTYPES.has(fstype)) continue; + if (seenDev.has(dev)) continue; // one entry per device (skip bind mounts) + seenDev.add(dev); + mounts.push(mount.replace(/\\040/g, ' ')); // /proc escapes spaces as \040 + } + return mounts.length ? mounts : ['/']; + } + if (process.platform === 'darwin') { + // No /proc on macOS. Root plus mounted volumes covers real disks; firmlink + // duplicates (/, /System/Volumes/Data) collapse in the dedupe below. + const mounts = ['/']; + const vols = await readdir('/Volumes').catch(() => [] as string[]); + for (const v of vols) mounts.push(`/Volumes/${v}`); + return mounts; + } + if (process.platform === 'win32') { + const drives: string[] = []; + for (let c = 65; c <= 90; c += 1) drives.push(`${String.fromCharCode(c)}:\\`); + return drives; // statfs fails fast for absent drives; usageForMount filters them out + } + return ['/']; +} + +/** Refresh the cached disk snapshot via fast statfs syscalls. Never throws. */ +export async function refreshDiskUsage(): Promise { + if (refreshing) return cache?.disks ?? []; + refreshing = true; + try { + const mounts = await enumerateMounts(); + const results = await Promise.all(mounts.map(usageForMount)); + const disks: DiskUsage[] = []; + const seen = new Set(); + for (const d of results) { + if (!d) continue; + // Collapse duplicates — macOS firmlinks and bind mounts report the same + // size+used against the same physical volume. + const key = `${d.totalBytes}:${d.usedBytes}`; + if (seen.has(key)) continue; + seen.add(key); + disks.push(d); + } + disks.sort((a, b) => b.usedPercent - a.usedPercent || b.totalBytes - a.totalBytes); + cache = { at: Date.now(), disks }; + return disks; + } catch (err) { + logger.debug({ err }, 'disk-usage: refresh failed'); + return cache?.disks ?? []; + } finally { + refreshing = false; + } +} + +/** + * Non-blocking accessor for the synchronous stats send path. Returns the cached + * snapshot and kicks off a background refresh when it's stale (or absent). The + * first call returns [] and warms the cache; the next stats tick carries data. + * Call `refreshDiskUsage()` once at startup to populate the first message. + */ +export function getDiskUsage(): DiskUsage[] { + if (!cache || Date.now() - cache.at > DISK_CACHE_TTL_MS) { + void refreshDiskUsage(); + } + return cache?.disks ?? []; +} diff --git a/src/daemon/file-transfer-handler.ts b/src/daemon/file-transfer-handler.ts index 7e3ae8a87..0a8037ca7 100644 --- a/src/daemon/file-transfer-handler.ts +++ b/src/daemon/file-transfer-handler.ts @@ -13,6 +13,7 @@ import logger from '../util/logger.js'; import { FILE_TRANSFER_LIMITS, FILE_TRANSFER_MSG, + FILE_TRANSFER_UPLOAD_ERROR_CODE, FILE_PATH_HANDLE_ERROR, type AttachmentRef, type FileUploadRequest, @@ -250,13 +251,23 @@ async function finalizeUploadedFile(params: { logger.info({ uploadId, filename, size }, 'File upload complete'); } +/** ENOSPC — the disk filled up mid-write. libuv surfaces it as error.code on + * both POSIX and Windows; the message text is a defensive fallback. */ +function isNoSpaceError(err: unknown): boolean { + if ((err as { code?: unknown } | null)?.code === 'ENOSPC') return true; + const msg = (err instanceof Error ? err.message : String(err)).toLowerCase(); + return msg.includes('enospc') || msg.includes('no space left'); +} + function sendUploadError(serverLink: FileTransferSender, uploadId: string, filename: string | undefined, err: unknown): void { const errMsg = err instanceof Error ? err.message : String(err); - logger.error({ uploadId, filename, err }, 'File upload failed'); + const code = isNoSpaceError(err) ? FILE_TRANSFER_UPLOAD_ERROR_CODE.INSUFFICIENT_CAPACITY : undefined; + logger.error({ uploadId, filename, err, code }, 'File upload failed'); const response: FileUploadError = { type: 'file.upload_error', uploadId, message: errMsg, + ...(code ? { code } : {}), }; serverLink.send(response); } diff --git a/src/daemon/server-link.ts b/src/daemon/server-link.ts index 82453e7ba..4816e22a8 100644 --- a/src/daemon/server-link.ts +++ b/src/daemon/server-link.ts @@ -8,6 +8,8 @@ import { setProviderRegistryServerLink } from '../agent/provider-registry.js'; import { getDefaultAckOutbox } from './ack-outbox.js'; import { getEmbeddingStatus } from '../context/embedding.js'; import type { EmbeddingStatus } from '../../shared/embedding-status.js'; +import type { DiskUsage } from '../../shared/disk-usage.js'; +import { getDiskUsage, refreshDiskUsage } from './disk-usage.js'; import { recordDaemonServerLinkStatus } from '../util/daemon-status.js'; import { P2P_WORKFLOW_IMPLEMENTATION_CAPABILITY_V1, @@ -46,6 +48,8 @@ interface SystemStats { * goes stale across reconnects. See `getEmbeddingStatus` for state * semantics. */ embedding: EmbeddingStatus; + /** Mounted-filesystem capacity for the desktop status bar (mobile ignores it). */ + disks: DiskUsage[]; } /** Collect lightweight system stats for daemon.stats messages. */ @@ -65,6 +69,7 @@ function collectSystemStats(): SystemStats { load15: +load15.toFixed(2), uptime: os.uptime(), embedding: getEmbeddingStatus(), + disks: getDiskUsage(), }; } @@ -850,6 +855,8 @@ export class ServerLink { }, 10_000); } }, HEARTBEAT_MS); + // Warm the disk-usage cache so the first stats message already carries it. + void refreshDiskUsage(); // Stats updates more frequently than heartbeat this.statsTimer = setInterval(() => { if (this.ws?.readyState === WebSocket.OPEN) { diff --git a/test/daemon/disk-usage.test.ts b/test/daemon/disk-usage.test.ts new file mode 100644 index 000000000..3b9066c91 --- /dev/null +++ b/test/daemon/disk-usage.test.ts @@ -0,0 +1,131 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; + +const { statfsMock, readFileMock, readdirMock } = vi.hoisted(() => ({ + statfsMock: vi.fn(), + readFileMock: vi.fn(), + readdirMock: vi.fn(), +})); + +vi.mock('node:fs/promises', () => ({ + statfs: statfsMock, + readFile: readFileMock, + readdir: readdirMock, +})); +vi.mock('../../src/util/logger.js', () => ({ + default: { debug: vi.fn(), error: vi.fn(), warn: vi.fn(), info: vi.fn() }, +})); + +import { refreshDiskUsage } from '../../src/daemon/disk-usage.js'; + +const originalPlatform = process.platform; +function setPlatform(p: NodeJS.Platform) { + Object.defineProperty(process, 'platform', { value: p, configurable: true }); +} + +/** bfree defaults to bavail; used = (blocks - bfree) * bsize. */ +function mockStatfs(map: Record) { + statfsMock.mockImplementation(async (mount: string) => { + const s = map[mount]; + if (!s) throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }); + return { type: 0, bsize: s.bsize, blocks: s.blocks, bfree: s.bfree, bavail: s.bfree, files: 0, ffree: 0 }; + }); +} + +describe('disk-usage collector', () => { + beforeEach(() => { + vi.clearAllMocks(); + setPlatform('linux'); + }); + afterEach(() => { + setPlatform(originalPlatform); + }); + + it('parses /proc/mounts, skips pseudo fs, dedups by device, sorts fullest first, computes usage', async () => { + readFileMock.mockResolvedValue([ + 'proc /proc proc rw 0 0', + 'sysfs /sys sysfs rw 0 0', + 'tmpfs /run tmpfs rw 0 0', + 'cgroup2 /sys/fs/cgroup cgroup2 rw 0 0', + '/dev/sda1 / ext4 rw 0 0', + '/dev/sda1 /var/lib/docker ext4 rw 0 0', // same device — bind mount, deduped + '/dev/sdb1 /data ext4 rw 0 0', + '', + ].join('\n')); + mockStatfs({ + '/': { bsize: 4096, blocks: 1000, bfree: 100 }, // 90% used + '/data': { bsize: 4096, blocks: 2000, bfree: 1000 }, // 50% used + }); + + const disks = await refreshDiskUsage(); + + expect(disks).toHaveLength(2); // pseudo skipped, device deduped + expect(disks[0]).toMatchObject({ mount: '/', usedPercent: 90 }); // fullest first + expect(disks[1]).toMatchObject({ mount: '/data', usedPercent: 50 }); + expect(disks[0].totalBytes).toBe(4096 * 1000); + expect(disks[0].usedBytes).toBe(4096 * 900); + }); + + it('collapses distinct devices that report identical size+used (macOS firmlinks)', async () => { + readFileMock.mockResolvedValue([ + '/dev/disk1 / apfs rw 0 0', + '/dev/disk2 /System/Volumes/Data apfs rw 0 0', + '', + ].join('\n')); + mockStatfs({ + '/': { bsize: 4096, blocks: 1000, bfree: 100 }, + '/System/Volumes/Data': { bsize: 4096, blocks: 1000, bfree: 100 }, + }); + + const disks = await refreshDiskUsage(); + expect(disks).toHaveLength(1); + }); + + it('drops mounts whose statfs fails and clamps a 0/undefined total away', async () => { + readFileMock.mockResolvedValue([ + '/dev/sda1 / ext4 rw 0 0', + '/dev/sdc1 /broken ext4 rw 0 0', + '/dev/sdd1 /zero ext4 rw 0 0', + '', + ].join('\n')); + statfsMock.mockImplementation(async (mount: string) => { + if (mount === '/') return { type: 0, bsize: 4096, blocks: 500, bfree: 250, bavail: 250, files: 0, ffree: 0 }; + if (mount === '/zero') return { type: 0, bsize: 4096, blocks: 0, bfree: 0, bavail: 0, files: 0, ffree: 0 }; + throw Object.assign(new Error('EACCES'), { code: 'EACCES' }); + }); + + const disks = await refreshDiskUsage(); + expect(disks).toHaveLength(1); + expect(disks[0]).toMatchObject({ mount: '/', usedPercent: 50 }); + }); + + it('falls back to root when /proc/mounts is empty on linux', async () => { + readFileMock.mockResolvedValue(''); + mockStatfs({ '/': { bsize: 4096, blocks: 100, bfree: 40 } }); // 60% used + + const disks = await refreshDiskUsage(); + expect(disks).toEqual([{ mount: '/', totalBytes: 4096 * 100, usedBytes: 4096 * 60, usedPercent: 60 }]); + }); + + it('skips a mount whose statfs hangs and still returns the healthy ones', async () => { + vi.useFakeTimers(); + try { + readFileMock.mockResolvedValue([ + '/dev/sda1 / ext4 rw 0 0', + '/dev/sde1 /hung ext4 rw 0 0', + '', + ].join('\n')); + statfsMock.mockImplementation((mount: string) => + mount === '/' + ? Promise.resolve({ type: 0, bsize: 4096, blocks: 100, bfree: 50, bavail: 50, files: 0, ffree: 0 }) + : new Promise(() => {}), // never resolves — a wedged network mount + ); + const pending = refreshDiskUsage(); + await vi.advanceTimersByTimeAsync(2100); // past STATFS_TIMEOUT_MS + const disks = await pending; + expect(disks).toHaveLength(1); + expect(disks[0].mount).toBe('/'); + } finally { + vi.useRealTimers(); + } + }); +}); diff --git a/web/src/components/ChatView.tsx b/web/src/components/ChatView.tsx index 1382bb110..f722987c7 100644 --- a/web/src/components/ChatView.tsx +++ b/web/src/components/ChatView.tsx @@ -16,6 +16,7 @@ import type { } from '../ws-client.js'; import type { FileChangeBatch, FileChangePatch } from '@shared/file-change.js'; import { FS_READ_ERROR_CODES } from '@shared/fs-read-error-codes.js'; +import { isInsufficientCapacityError } from '../upload-error.js'; import { SDK_SUBAGENT_DETAIL_KIND, SDK_SUBAGENT_DIAGNOSTIC, @@ -3120,7 +3121,8 @@ function AttachmentDownloadButton({ const handleError = (err: unknown) => { const msg = err instanceof Error ? err.message : String(err); - if (msg.includes('daemon_offline') || msg.includes('503')) setError(t('upload.daemon_offline')); + if (isInsufficientCapacityError(msg)) setError(t('upload.insufficient_capacity')); + else if (msg.includes('daemon_offline') || msg.includes('503')) setError(t('upload.daemon_offline')); else if (msg.includes('410') || msg.includes('expired')) setError(t('upload.download_expired')); else if (msg.includes('404')) setError(t('upload.download_expired')); else setError(t('upload.upload_failed')); diff --git a/web/src/components/SessionControls.tsx b/web/src/components/SessionControls.tsx index 37c78f3f6..baa33f74b 100644 --- a/web/src/components/SessionControls.tsx +++ b/web/src/components/SessionControls.tsx @@ -23,6 +23,7 @@ import { insertMachineMarkerAtCaret } from '../util/machine-insert.js'; import { buildMachineSendExtra } from '../util/machine-send.js'; import { matchInlineMachineTrigger, stripInlineMachineTrigger } from '../util/machine-trigger.js'; import { parseAliasMarkers } from '@shared/alias-types.js'; +import { isInsufficientCapacityError } from '../upload-error.js'; import { MobileDpad, DPAD_ARROW_SEQUENCES } from './MobileDpad.js'; import { P2pConfigPanel, buildP2pWorkflowLaunchEnvelopeFromConfig } from './P2pConfigPanel.js'; import { useExecutionRouting } from '../hooks/useExecutionRouting.js'; @@ -4019,7 +4020,9 @@ export function SessionControls({ ws, activeSession, connected: connectedProp, i console.error('[upload] failed:', err); const body = err instanceof Error ? err.message : String(err); let errorMessage: string; - if (body.includes('daemon_offline')) { + if (isInsufficientCapacityError(body)) { + errorMessage = t('upload.insufficient_capacity'); + } else if (body.includes('daemon_offline')) { errorMessage = t('upload.daemon_offline'); } else if (body.includes('file_too_large')) { errorMessage = t('upload.file_too_large', { max: MAX_UPLOAD_SIZE_MB }); diff --git a/web/src/components/SubSessionBar.tsx b/web/src/components/SubSessionBar.tsx index b99e32953..02d1609d0 100644 --- a/web/src/components/SubSessionBar.tsx +++ b/web/src/components/SubSessionBar.tsx @@ -24,6 +24,7 @@ import { EmbeddingStatusIcon } from './EmbeddingStatusIcon.js'; import { SharedStateIndicator } from './SharedStateIndicator.js'; import type { SharedStateSummary } from '../tab-sharing-ui.js'; import type { EmbeddingStatus } from '@shared/embedding-status.js'; +import type { DiskUsage } from '@shared/disk-usage.js'; import { formatDaemonVersionMobile, formatDaemonVersionShort } from '../util/format-version.js'; import { isAuthoritativeUsageContextWindowSource, type UsageContextWindowSource } from '@shared/usage-context-window.js'; import { resolveQuickAgentDelegationModel } from '../quick-agent-delegation-model.js'; @@ -48,6 +49,7 @@ interface DaemonStats { load15: number; uptime: number; embedding?: EmbeddingStatus | null; + disks?: DiskUsage[] | null; } type DiscussionSummary = P2pProgressDiscussion & { @@ -174,6 +176,55 @@ function formatUptime(seconds: number): string { return d > 0 ? `${d}d ${h}h` : `${h}h`; } +/** Format a used/total byte pair in a shared GB or TB unit — never smaller + * units, so disk sizes always read as e.g. "440/460GB" or "1.2/2.0TB". */ +function formatDiskPair(usedBytes: number, totalBytes: number): string { + const useTb = totalBytes >= 1024 ** 4; + const div = useTb ? 1024 ** 4 : 1024 ** 3; + const unit = useTb ? 'TB' : 'GB'; + const fmt = (b: number) => { + const v = b / div; + return v >= 100 ? v.toFixed(0) : v.toFixed(1); + }; + return `${fmt(usedBytes)}/${fmt(totalBytes)}${unit}`; +} + +function shortMountLabel(mount: string): string { + if (mount === '/') return '/'; + const seg = mount.replace(/[\\/]+$/, '').split(/[/\\]/).filter(Boolean).pop(); + return seg || mount; +} + +function diskUsageColor(usedPercent: number): string | undefined { + return usedPercent >= 90 ? '#f87171' : usedPercent >= 75 ? '#fbbf24' : undefined; +} + +/** + * Desktop-only disk-capacity readout for the daemon stats strip. Shows up to two + * partitions inline; with more than two, only the fullest shows inline (plus a + * "+N" hint) and every partition is listed in the hover tooltip. Mobile clients + * receive no disk data, so this renders nothing there. + */ +function renderDiskStats(disks: DiskUsage[] | null | undefined): JSX.Element | null { + if (!disks || disks.length === 0) return null; + const tooltip = disks + .map((d) => `${shortMountLabel(d.mount)} ${formatDiskPair(d.usedBytes, d.totalBytes)} (${d.usedPercent}%)`) + .join('\n'); + const inline = disks.length <= 2 ? disks : disks.slice(0, 1); + return ( + + {inline.map((d, i) => ( + + {i > 0 ? ' ' : ''} + 💾 + {formatDiskPair(d.usedBytes, d.totalBytes)} + + ))} + {disks.length > 2 && +{disks.length - 1}} + + ); +} + function formatLocalDateTime(timestamp: number): string { const date = new Date(timestamp); const pad = (value: number) => String(value).padStart(2, '0'); @@ -801,6 +852,9 @@ export function SubSessionBar({ subSessions, openIds, maximizedIds, desktopLayou // icon falls through to its "unknown" rendering instead of // showing a misleading "ready". embedding: (msg as { embedding?: EmbeddingStatus | null }).embedding ?? null, + // Older daemons don't ship `disks`; null means "no data" so the + // desktop strip simply hides the readout (and mobile never shows it). + disks: msg.disks ?? null, }); } }); @@ -900,6 +954,12 @@ export function SubSessionBar({ subSessions, openIds, maximizedIds, desktopLayou Load {stats.load1} + {desktopLayoutCapable && stats.disks && stats.disks.length > 0 && ( + <> + · + {renderDiskStats(stats.disks)} + + )} · · @@ -935,6 +995,9 @@ export function SubSessionBar({ subSessions, openIds, maximizedIds, desktopLayou {' '} ≡{Number(stats.load1).toFixed(1)} {' '} + {desktopLayoutCapable && stats.disks && stats.disks.length > 0 && ( + <>{renderDiskStats(stats.disks)}{' '} + )} {desktopLayoutCapable && ( <> diff --git a/web/src/i18n/locales/en.json b/web/src/i18n/locales/en.json index ee5966445..3cecabdbc 100644 --- a/web/src/i18n/locales/en.json +++ b/web/src/i18n/locales/en.json @@ -1552,6 +1552,7 @@ "upload_file": "Upload file", "uploading": "Uploading...", "upload_failed": "Upload failed", + "insufficient_capacity": "Insufficient disk space on the server", "file_too_large": "File too large (max {{max}}MB)", "long_text_attached": "Large pasted text attached as {{name}}", "long_text_requires_attachment": "Paste is too large for inline input here. Upload it as a file instead.", diff --git a/web/src/i18n/locales/es.json b/web/src/i18n/locales/es.json index b16db9b3e..59c1e1a2d 100644 --- a/web/src/i18n/locales/es.json +++ b/web/src/i18n/locales/es.json @@ -1552,6 +1552,7 @@ "upload_file": "Subir archivo", "uploading": "Subiendo...", "upload_failed": "Error al subir", + "insufficient_capacity": "Espacio en disco insuficiente en el servidor", "file_too_large": "Archivo demasiado grande (máx. {{max}}MB)", "long_text_attached": "El texto pegado largo se adjuntó como {{name}}", "long_text_requires_attachment": "Este pegado es demasiado grande para el cuadro de texto. Súbelo como archivo.", diff --git a/web/src/i18n/locales/ja.json b/web/src/i18n/locales/ja.json index d4c9dbe2b..8cb00cc19 100644 --- a/web/src/i18n/locales/ja.json +++ b/web/src/i18n/locales/ja.json @@ -1551,6 +1551,7 @@ "upload_file": "ファイルをアップロード", "uploading": "アップロード中...", "upload_failed": "アップロード失敗", + "insufficient_capacity": "サーバーのディスク容量が不足しています", "file_too_large": "ファイルが大きすぎます（最大 {{max}}MB）", "long_text_attached": "長い貼り付けテキストを {{name}} として添付しました", "long_text_requires_attachment": "この貼り付け内容は入力欄に直接入れるには長すぎます。ファイルとしてアップロードしてください。", diff --git a/web/src/i18n/locales/ko.json b/web/src/i18n/locales/ko.json index 39078da32..7427260f6 100644 --- a/web/src/i18n/locales/ko.json +++ b/web/src/i18n/locales/ko.json @@ -1551,6 +1551,7 @@ "upload_file": "파일 업로드", "uploading": "업로드 중...", "upload_failed": "업로드 실패", + "insufficient_capacity": "서버의 디스크 공간이 부족합니다", "file_too_large": "파일이 너무 큽니다 (최대 {{max}}MB)", "long_text_attached": "긴 붙여넣기 텍스트가 {{name}} 첨부파일로 추가되었습니다", "long_text_requires_attachment": "이 붙여넣기 내용은 입력창에 직접 넣기엔 너무 깁니다. 파일로 업로드하세요.", diff --git a/web/src/i18n/locales/ru.json b/web/src/i18n/locales/ru.json index 94f2da5be..1a6fedc0e 100644 --- a/web/src/i18n/locales/ru.json +++ b/web/src/i18n/locales/ru.json @@ -1551,6 +1551,7 @@ "upload_file": "Загрузить файл", "uploading": "Загрузка...", "upload_failed": "Ошибка загрузки", + "insufficient_capacity": "Недостаточно места на диске сервера", "file_too_large": "Файл слишком большой (макс. {{max}}МБ)", "long_text_attached": "Большой вставленный текст прикреплен как {{name}}", "long_text_requires_attachment": "Эта вставка слишком большая для поля ввода. Загрузите ее как файл.", diff --git a/web/src/i18n/locales/zh-CN.json b/web/src/i18n/locales/zh-CN.json index 20598b6e9..20c7404ea 100644 --- a/web/src/i18n/locales/zh-CN.json +++ b/web/src/i18n/locales/zh-CN.json @@ -1552,6 +1552,7 @@ "upload_file": "上传文件", "uploading": "上传中...", "upload_failed": "上传失败", + "insufficient_capacity": "服务器磁盘空间不足", "file_too_large": "文件过大（最大 {{max}}MB）", "long_text_attached": "超长粘贴内容已作为 {{name}} 附件添加", "long_text_requires_attachment": "这段粘贴内容太长，不能直接放进输入框，请改为文件上传。", diff --git a/web/src/i18n/locales/zh-TW.json b/web/src/i18n/locales/zh-TW.json index 17b3f7ccb..688c50ef8 100644 --- a/web/src/i18n/locales/zh-TW.json +++ b/web/src/i18n/locales/zh-TW.json @@ -1552,6 +1552,7 @@ "upload_file": "上傳檔案", "uploading": "上傳中...", "upload_failed": "上傳失敗", + "insufficient_capacity": "伺服器磁碟空間不足", "file_too_large": "檔案過大（最大 {{max}}MB）", "long_text_attached": "過長貼上內容已作為 {{name}} 附件加入", "long_text_requires_attachment": "這段貼上內容太長，不能直接放進輸入框，請改成檔案上傳。", diff --git a/web/src/upload-error.ts b/web/src/upload-error.ts new file mode 100644 index 000000000..d872473a2 --- /dev/null +++ b/web/src/upload-error.ts @@ -0,0 +1,15 @@ +import { FILE_TRANSFER_UPLOAD_ERROR_CODE } from '@shared/transport/file-transfer.js'; + +/** + * True when a file-transfer error indicates the daemon ran out of disk space + * (ENOSPC). The daemon tags such errors with a shared code, the server responds + * HTTP 507, and the daemon's raw ENOSPC text rides along in the message — so an + * ApiError surfaces as e.g. "API 507: insufficient_capacity". Match any of them. + * Shared by the upload composer and the attachment download/preview button. + */ +export function isInsufficientCapacityError(body: string): boolean { + return body.includes(FILE_TRANSFER_UPLOAD_ERROR_CODE.INSUFFICIENT_CAPACITY) + || body.includes('507') + || body.includes('ENOSPC') + || body.includes('no space left'); +} diff --git a/web/src/ws-client.ts b/web/src/ws-client.ts index 5180658e9..3e276f41c 100644 --- a/web/src/ws-client.ts +++ b/web/src/ws-client.ts @@ -12,6 +12,7 @@ import type { ResourceChangedMessage } from '@shared/resource-events.js'; import { P2P_CONFIG_MSG } from '@shared/p2p-config-events.js'; import { P2P_WORKFLOW_MSG, isP2pWorkflowRequestId } from '@shared/p2p-workflow-messages.js'; import { TRANSPORT_EVENT } from '@shared/transport-events.js'; +import type { DiskUsage } from '@shared/disk-usage.js'; import { P2P_CAPABILITY_FRESHNESS_TTL_MS } from '@shared/p2p-workflow-constants.js'; import { TRANSPORT_MSG } from '@shared/transport-events.js'; import { DAEMON_COMMAND_TYPES } from '@shared/daemon-command-types.js'; @@ -186,7 +187,7 @@ export type ServerMessage = | { type: 'discussion.done'; discussionId: string; filePath: string; conclusion: string } | { type: 'discussion.error'; discussionId?: string; requestId?: string; error: string } | { type: 'discussion.list'; discussions: Array<{ id: string; requestId?: string; topic: string; state: string; currentRound: number; maxRounds: number; completedHops?: number; totalHops?: number; currentSpeaker?: string; conclusion?: string; filePath?: string }> } - | { type: 'daemon.stats'; daemonVersion?: string | null; cpu: number; memUsed: number; memTotal: number; load1: number; load5: number; load15: number; uptime: number } + | { type: 'daemon.stats'; daemonVersion?: string | null; cpu: number; memUsed: number; memTotal: number; load1: number; load5: number; load15: number; uptime: number; disks?: DiskUsage[] } | FsLsResponse | FsReadResponse | FsGitStatusResponse diff --git a/web/test/upload-error.test.ts b/web/test/upload-error.test.ts new file mode 100644 index 000000000..fbd554066 --- /dev/null +++ b/web/test/upload-error.test.ts @@ -0,0 +1,23 @@ +import { describe, it, expect } from 'vitest'; +import { isInsufficientCapacityError } from '../src/upload-error.js'; + +describe('isInsufficientCapacityError', () => { + it('matches the shared code, 507 status, and raw ENOSPC text', () => { + // The exact ApiError message the upload catch receives (status + parsed code). + expect(isInsufficientCapacityError('API 507: insufficient_capacity')).toBe(true); + // Raw JSON body fallback (server response body, no code parsed). + expect(isInsufficientCapacityError('{"error":"insufficient_capacity","message":"ENOSPC: no space left on device"}')).toBe(true); + // Bare daemon ENOSPC error text. + expect(isInsufficientCapacityError('ENOSPC: no space left on device, write')).toBe(true); + expect(isInsufficientCapacityError('no space left on device')).toBe(true); + expect(isInsufficientCapacityError('API 507: unmapped')).toBe(true); + }); + + it('does not match unrelated file-transfer errors', () => { + expect(isInsufficientCapacityError('API 503: daemon_offline')).toBe(false); + expect(isInsufficientCapacityError('API 413: file_too_large')).toBe(false); + expect(isInsufficientCapacityError('upload_failed')).toBe(false); + expect(isInsufficientCapacityError('API 500: internal error')).toBe(false); + expect(isInsufficientCapacityError('')).toBe(false); + }); +}); From 3b07221b64d5bb01c89b5816fe51304d0925fcc3 Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sat, 25 Jul 2026 09:11:17 +0800 Subject: [PATCH 03/40] Stop timeline metadata events from faking an idle session A working session could render as idle. hasActiveTimelineTurn() scans the timeline tail backwards for activity evidence, and any event type it did not know fell through to `return false`. memory.context is exactly such an event and it is emitted right after the user.message of every context-injecting send - so for the whole window between the send and the first tool call (the model is only emitting agent.status thinking labels) the scan reported "no active turn". SubSessionWindow feeds that into resolveTimelineBackedSessionState, whose stale-running guard then overrides the daemon's session.state running with the store snapshot, and the footer falls asleep while the agent is demonstrably editing files. Observed on deck_sub_704c1g42: running at 08:53:32, 34 s of thinking with a memory.context tail, footer idle the whole time; the same shape hid the elapsed-thinking timer because getActiveThinkingTs stopped on memory.context too. Introduce one shared TIMELINE_METADATA_EVENT_TYPES list (memory.*, file.change, peer_audit.*, transport.queue.*, execution clone terminal) and spread it into both scans so telemetry is walked through instead of treated as a turn boundary. The two lists can no longer drift apart. An authoritative clean idle still wins: metadata now leads the scan to it rather than short-circuiting before it. Co-Authored-By: Claude Fable 5 --- shared/session-activity-types.ts | 34 +++++++++++++++++++ web/src/thinking-utils.ts | 12 +++++-- web/src/timeline-running.ts | 14 ++++++-- web/test/thinking-utils.test.ts | 20 +++++++++++ web/test/timeline-running.test.ts | 56 +++++++++++++++++++++++++++++++ 5 files changed, 132 insertions(+), 4 deletions(-) diff --git a/shared/session-activity-types.ts b/shared/session-activity-types.ts index 5aaba3742..8bd953571 100644 --- a/shared/session-activity-types.ts +++ b/shared/session-activity-types.ts @@ -1,3 +1,6 @@ +import { TIMELINE_EVENT_FILE_CHANGE } from './file-change.js'; +import { EXECUTION_CLONE_TIMELINE } from './execution-clone.js'; + export type SessionActivityBusyReason = | 'runtime_dispatch' | 'active_dispatch_entry' @@ -186,6 +189,37 @@ export function isWorkingSessionState(value: unknown): boolean { return typeof value === 'string' && WORKING_SESSION_STATES.has(value); } +/** + * Timeline events that are pure metadata/telemetry: they say nothing about + * whether a turn is still running, so any tail scan looking for activity + * evidence must WALK THROUGH them rather than stop. + * + * This is not cosmetic. `memory.context` is emitted immediately after the + * `user.message` of every context-injecting send, i.e. it is the tail event for + * the whole "model is thinking before its first tool call" window. While it + * terminated the scan, `hasActiveTimelineTurn()` returned false for that entire + * window, and `resolveTimelineBackedSessionState()` then overrode the daemon's + * `session.state: running` with the (possibly lagging) store snapshot — the + * session rendered IDLE while the agent was demonstrably working. Observed live + * on deck_sub_704c1g42: `running` at 08:53:32 followed by 34 s of thinking with + * a `memory.context` tail, footer asleep the whole time. + * + * Shared so the tail-activity scan and the thinking scan cannot drift apart. + */ +export const TIMELINE_METADATA_EVENT_TYPES: readonly string[] = [ + 'memory.context', + 'memory.compression', + TIMELINE_EVENT_FILE_CHANGE, + 'peer_audit.status', + 'peer_audit.result', + 'transport.queue.snapshot', + 'transport.queue.delivery', + 'transport.queue.receipt', + 'transport.queue.failure', + 'transport.queue.reset', + EXECUTION_CLONE_TIMELINE.TERMINAL, +]; + export function normalizeActivityGeneration(value: ActivityGenerationLike): string | null { if (value == null) return null; if (typeof value === 'number') return Number.isFinite(value) ? `session:${value}` : null; diff --git a/web/src/thinking-utils.ts b/web/src/thinking-utils.ts index f0b7998e0..06a92fbbd 100644 --- a/web/src/thinking-utils.ts +++ b/web/src/thinking-utils.ts @@ -7,9 +7,14 @@ * Only assistant.text, user.message, and session.state=idle end thinking. * Tool calls, status updates, and other events are skipped (don't end thinking). */ -import { isAuthoritativeCleanIdlePayload, isWorkingSessionState, reduceTimelineActivity } from '../../shared/session-activity-types.js'; +import { + TIMELINE_METADATA_EVENT_TYPES, + isAuthoritativeCleanIdlePayload, + isWorkingSessionState, + reduceTimelineActivity, +} from '../../shared/session-activity-types.js'; -const THINKING_SKIP_TYPES = new Set([ +const THINKING_SKIP_TYPES = new Set([ 'agent.status', 'usage.update', 'tool.call', @@ -17,6 +22,9 @@ const THINKING_SKIP_TYPES = new Set([ 'mode.state', 'terminal.snapshot', 'command.ack', + // Metadata/telemetry never ends a thinking sequence — see + // TIMELINE_METADATA_EVENT_TYPES. + ...TIMELINE_METADATA_EVENT_TYPES, ]); export function getActiveThinkingTs(events: Array<{ type: string; ts: number; payload?: Record }>): number | null { diff --git a/web/src/timeline-running.ts b/web/src/timeline-running.ts index 3432201a7..eabee5529 100644 --- a/web/src/timeline-running.ts +++ b/web/src/timeline-running.ts @@ -1,19 +1,29 @@ import type { TimelineEvent } from '../../src/shared/timeline/types.js'; import { SDK_SUBAGENT_DETAIL_KIND } from '../../shared/sdk-subagent-status.js'; -import { isAuthoritativeCleanIdlePayload, isWorkingSessionState, reduceTimelineActivity } from '../../shared/session-activity-types.js'; +import { + TIMELINE_METADATA_EVENT_TYPES, + isAuthoritativeCleanIdlePayload, + isWorkingSessionState, + reduceTimelineActivity, +} from '../../shared/session-activity-types.js'; const RUNNING_TIMELINE_EVENT_TYPES = new Set([ 'assistant.thinking', 'tool.call', ]); -const NEUTRAL_TAIL_EVENT_TYPES = new Set([ +const NEUTRAL_TAIL_EVENT_TYPES = new Set([ 'agent.status', 'usage.update', 'tool.result', 'mode.state', 'terminal.snapshot', 'command.ack', + // Pure metadata/telemetry (memory.context, file.change, queue snapshots, …). + // These must never terminate the tail scan: `memory.context` is the tail for + // the whole pre-first-tool thinking window of every context-injecting send, + // and stopping there reported "no active turn" while the agent was working. + ...TIMELINE_METADATA_EVENT_TYPES, ]); type TimelineTailEvent = Pick & { diff --git a/web/test/thinking-utils.test.ts b/web/test/thinking-utils.test.ts index edd100dab..109beac8d 100644 --- a/web/test/thinking-utils.test.ts +++ b/web/test/thinking-utils.test.ts @@ -104,6 +104,26 @@ describe('getActiveThinkingTs', () => { { type: 'agent.status', ts: 12, payload: { label: 'still thinking' } }, ] as any)).toBe(10); }); + + it('walks through metadata events so the thinking timer survives memory injection', () => { + // memory.context / memory.compression / file.change are telemetry: they must + // not end a thinking sequence. Before the fix, memory.context terminated the + // scan and the elapsed-thinking timer silently disappeared. + expect(getActiveThinkingTs([ + { type: 'assistant.thinking', ts: 10 }, + { type: 'memory.context', ts: 11, payload: { query: 'x' } }, + { type: 'file.change', ts: 12, payload: { batch: {} } }, + { type: 'agent.status', ts: 13, payload: { label: 'Thinking (1.9k tokens)' } }, + ] as any)).toBe(10); + }); + + it('still ends thinking on real user/assistant turn boundaries', () => { + expect(getActiveThinkingTs([ + { type: 'assistant.thinking', ts: 10 }, + { type: 'assistant.text', ts: 11, payload: { text: 'answered', streaming: false } }, + { type: 'memory.context', ts: 12, payload: {} }, + ] as any)).toBe(null); + }); }); describe('getActiveStatusText', () => { diff --git a/web/test/timeline-running.test.ts b/web/test/timeline-running.test.ts index 95105af1a..f874ec3a0 100644 --- a/web/test/timeline-running.test.ts +++ b/web/test/timeline-running.test.ts @@ -222,4 +222,60 @@ describe('isRunningTimelineEvent', () => { { type: 'usage.update', payload: { model: 'gpt-5.5' } }, ] as any)).toBe(true); }); + + /** + * Regression for deck_sub_704c1g42 (2026-07-25 08:53:32 → 08:54:06): the + * daemon emitted `session.state: running`, then the agent thought for 34 s + * emitting only `agent.status` labels. The tail therefore ended at the + * `memory.context` of that send, which used to fall through to `return false` + * — reporting "no active turn" for the whole window. `SubSessionWindow` then + * fed that into `resolveTimelineBackedSessionState`, whose stale-running guard + * overrode the timeline's `running` with the store snapshot, and the footer + * rendered IDLE while the agent was demonstrably editing files. + */ + it('stays active through the memory.context tail of a context-injecting send', () => { + const events = [ + { type: 'session.state', payload: authoritativeIdlePayload }, + { type: 'command.ack', payload: { commandId: 'cmd-x', status: 'accepted' } }, + { type: 'assistant.thinking', payload: { text: '' } }, + { type: 'session.state', payload: { state: 'running', blockingWorkCount: 3 } }, + { type: 'user.message', payload: { text: '远程桌面只干一件事那里 字幕缺了下载安装的字幕吧' } }, + { type: 'memory.context', payload: { query: 'x', injectedText: '[Related past work]' } }, + { type: 'agent.status', payload: { status: 'requesting', label: null } }, + { type: 'agent.status', payload: { status: 'thinking', label: 'Thinking (1.9k tokens)' } }, + ]; + expect(hasActiveTimelineTurn(events as any)).toBe(true); + }); + + it('stays active when telemetry metadata is the tail after real work', () => { + // file.change / memory.compression / queue snapshots are pure metadata and + // must not be read as "the turn ended". + for (const tail of [ + { type: 'file.change', payload: { batch: {} } }, + { type: 'memory.compression', payload: { outcome: 'success' } }, + { type: 'transport.queue.snapshot', payload: {} }, + { type: 'peer_audit.status', payload: {} }, + ]) { + expect(hasActiveTimelineTurn([ + { type: 'session.state', payload: { state: 'running', blockingWorkCount: 1 } }, + { type: 'user.message', payload: { text: 'go' } }, + { type: 'memory.context', payload: {} }, + tail, + ] as any)).toBe(true); + } + }); + + it('still reports inactive when metadata trails an authoritative clean idle', () => { + // The fix must not resurrect a settled turn: walking through metadata has to + // land on the authoritative idle and report no active work. + expect(hasActiveTimelineTurn([ + { type: 'user.message', payload: { text: 'go' } }, + { type: 'tool.call', payload: { toolCallId: 'A', tool: 'Bash' } }, + { type: 'tool.result', payload: { toolCallId: 'A', terminalStatus: 'succeeded' } }, + { type: 'assistant.text', payload: { text: 'done', streaming: false } }, + { type: 'session.state', payload: authoritativeIdlePayload }, + { type: 'memory.compression', payload: { outcome: 'success' } }, + { type: 'memory.context', payload: {} }, + ] as any)).toBe(false); + }); }); From 48ba2cf330ddebec722812141fedb7a542620cc2 Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sat, 25 Jul 2026 12:52:36 +0800 Subject: [PATCH 04/40] Anchor the chat viewport when revealing cached older messages Scrolling up to the end of the loaded history made the view jump a whole section back into older messages. The near-top auto-trigger and the "load older" button both call revealHiddenOlderItems(), which raises the render limit by CHAT_RENDER_ITEM_INCREMENT (250) items - all mounted ABOVE the viewport. Only the HTTP load-older sibling saved a scroll anchor, so the reveal path kept scrollTop while the content above it grew by thousands of pixels: the same offset now pointed a full chunk higher in the list. It also left the viewport inside the scrollTop < 100 trigger zone, so the next cooldown tick revealed and jumped again - the "often" in the report. Capture the pre-reveal scrollHeight inside revealHiddenOlderItems() itself, so both call sites are anchored identically instead of repeating the bookkeeping, and the existing restore effect re-adds the delta. The reading position now holds still and the revealed history stays one scroll-up away. The previous behaviour was pinned by a test asserting scrollTop stayed 0; it entered the tree inside an unrelated combined dev snapshot rather than as a deliberate UX choice, so it is re-pointed at the anchored position and joined by a near-top (scrollTop = 50) repro of the mobile case. Co-Authored-By: Claude Fable 5 --- web/src/components/ChatView.tsx | 18 +++++++- web/test/components/ChatView.test.tsx | 61 ++++++++++++++++++++++++++- 2 files changed, 75 insertions(+), 4 deletions(-) diff --git a/web/src/components/ChatView.tsx b/web/src/components/ChatView.tsx index f722987c7..4b570ca80 100644 --- a/web/src/components/ChatView.tsx +++ b/web/src/components/ChatView.tsx @@ -1361,6 +1361,10 @@ export function ChatView({ events, loading, refreshing = false, historyStatus, l const ctxMenuRef = useRef(null); const [revealingOlder, setRevealingOlder] = useState(false); const revealingOlderTimerRef = useRef | null>(null); + // Scroll anchor preservation: pre-prepend scrollHeight, restored after the + // taller list commits. Declared with the reveal state it serves (both the + // local render-limit reveal and the HTTP load-older prepend set it). + const scrollAnchorRef = useRef<{ scrollHeight: number } | null>(null); // Timestamp when ctx menu was opened — clicks within 400ms are synthetic (from long-press release) const menuOpenedAtRef = useRef(0); // Zoomed text modal — opened by double-tap on a chat bubble on touch devices. @@ -1822,6 +1826,18 @@ export function ChatView({ events, loading, refreshing = false, historyStatus, l }; const revealHiddenOlderItems = () => { + // Raising the render limit mounts CHAT_RENDER_ITEM_INCREMENT more items + // ABOVE the viewport. Without an anchor the browser keeps `scrollTop`, so + // the same offset now points that much higher in the list and the reading + // position lurches toward older history — the "scrolled to the end and it + // jumps up a section" report. Worse, it lands the view back inside the + // `scrollTop < 100` trigger zone, so the next cooldown tick reveals again + // and jumps again. Capture the pre-reveal height here (not at the call + // sites) so every caller — the scroll auto-trigger and the manual + // "load older" button — is anchored identically; the restore effect below + // re-adds the delta. + const scrollEl = scrollRef.current; + if (scrollEl) scrollAnchorRef.current = { scrollHeight: scrollEl.scrollHeight }; if (revealingOlderTimerRef.current) clearTimeout(revealingOlderTimerRef.current); setRevealingOlder(true); setRenderItemLimit((limit) => limit + CHAT_RENDER_ITEM_INCREMENT); @@ -2112,8 +2128,6 @@ export function ChatView({ events, loading, refreshing = false, historyStatus, l // Scroll auto-trigger for Load Older const lastLoadOlderAtRef = useRef(0); const LOAD_OLDER_COOLDOWN_MS = 1000; - // Scroll anchor preservation: save scrollHeight before prepend, restore after - const scrollAnchorRef = useRef<{ scrollHeight: number } | null>(null); // Pause "stick to bottom" follow mode. Shared by handleScroll's distance // threshold and the explicit wheel/touch up-gesture handlers below. diff --git a/web/test/components/ChatView.test.tsx b/web/test/components/ChatView.test.tsx index ec301d725..67c82c555 100644 --- a/web/test/components/ChatView.test.tsx +++ b/web/test/components/ChatView.test.tsx @@ -215,7 +215,7 @@ describe('ChatView', () => { expect(screen.getByText('message-0')).toBeTruthy(); }); - it('reveals locally cached older messages at the top without preserving the previous anchor', async () => { + it('reveals locally cached older messages at the top while preserving the reading position', async () => { const events = Array.from({ length: CHAT_INITIAL_RENDER_ITEM_LIMIT + 10 }, (_, index) => ({ eventId: `user-${index}`, type: 'user.message', @@ -256,7 +256,64 @@ describe('ChatView', () => { await waitFor(() => { expect(screen.getByText('message-0')).toBeTruthy(); }); - expect(scrollTopValue).toBe(0); + // Mounting the older items grows scrollHeight 1200 → 1800 ABOVE the + // viewport. The anchor must re-add that 600px delta so the message the user + // was reading stays under their eyes. Leaving scrollTop at 0 (the previous + // behaviour) silently teleported the viewport a whole reveal-chunk back into + // older history — reported on mobile as "scrolled to the end and it jumps up + // a section". The revealed content is still reachable by scrolling up. + expect(scrollTopValue).toBe(600); + }); + + it('does not move the viewport when a near-top scroll auto-reveals older items', async () => { + // Mobile repro: the auto-trigger fires at scrollTop < 100, so the user is + // near — but not at — the top. Without anchoring, the reveal jumped them up + // by the whole added height AND left them inside the < 100 trigger zone, so + // the next cooldown tick jumped again. Anchoring must both hold the position + // and carry them out of the trigger zone. + const events = Array.from({ length: CHAT_INITIAL_RENDER_ITEM_LIMIT + 10 }, (_, index) => ({ + eventId: `user-${index}`, + type: 'user.message', + ts: 1_700_000_000_000 + index, + payload: { text: `message-${index}` }, + })); + + const { container } = render( + , + ); + const scrollEl = container.querySelector('.chat-view') as HTMLDivElement; + let scrollTopValue = 0; + Object.defineProperty(scrollEl, 'scrollTop', { + configurable: true, + get: () => scrollTopValue, + set: (value) => { scrollTopValue = value; }, + }); + Object.defineProperty(scrollEl, 'scrollHeight', { + configurable: true, + get: () => (container.textContent?.includes('message-0') ? 5000 : 1200), + }); + Object.defineProperty(scrollEl, 'clientHeight', { configurable: true, value: 200 }); + + await waitFor(() => { + expect(scrollTopValue).toBe(1200); + }); + + scrollTopValue = 50; // near the top, inside the auto-reveal trigger zone + fireEvent.wheel(scrollEl, { deltaY: -20 }); + fireEvent.scroll(scrollEl); + + await waitFor(() => { + expect(screen.getByText('message-0')).toBeTruthy(); + }); + // 50 + (5000 - 1200) = 3850: same content under the viewport, and safely + // out of the < 100 re-trigger zone so the reveal cannot loop. + expect(scrollTopValue).toBe(3850); + expect(scrollTopValue).toBeGreaterThanOrEqual(100); }); it('collapses sent user messages longer than ten hard lines by default', () => { From ed6335afd3c45b6bbd16972a96ea578816d7c9ef Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sat, 25 Jul 2026 13:24:46 +0800 Subject: [PATCH 05/40] Wait for audit phases instead of fixed sleeps in the rework-loop test macOS CI failed on the maxAuditLoops=1 rework test with call[1] holding the audit-orchestration prompt instead of the rework brief. The test paced itself with fixed `await sleep(25)` between steps, so on a loaded runner it completed the delegated audit before the run had even reached the `auditing` phase - the audit dispatch is async (broker decision plus filesystem baseline discovery) - and the whole send sequence shifted under the index-based assertions. Wait on the state each step actually depends on, reusing the file's existing waitForRunPhase helper, and add the missing waitForRunEnd so the "second REWORK ends the run" case can assert teardown before asserting the send count never grew. No product code changes. The failing run's daemon tree was byte-identical to the previous fully green run (the commit in between touched only web/), so this was a pre-existing pacing flake rather than a regression. I could not reproduce it locally even under synthetic CPU load, so this is a structural fix for the observed failure signature, not a verified repro. Co-Authored-By: Claude Fable 5 --- test/daemon/supervision-automation.test.ts | 32 ++++++++++++++++++---- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/test/daemon/supervision-automation.test.ts b/test/daemon/supervision-automation.test.ts index cfa54fbdd..d47c31f95 100644 --- a/test/daemon/supervision-automation.test.ts +++ b/test/daemon/supervision-automation.test.ts @@ -98,6 +98,17 @@ async function waitForRunPhase(phase: 'execution' | 'auditing' | 'finalizing', t } } +/** Wait until the automation has finished the run (no active run left). + * Same real-check-queue yield as `waitForRunPhase`, so a terminal run cannot + * be asserted before the automation has actually torn it down. */ +async function waitForRunEnd(timeoutMs = 10_000) { + const deadline = performance.now() + timeoutMs; + while (supervisionAutomation.getActiveRun('deck_supervision_brain') !== undefined) { + if (performance.now() >= deadline) return; + await new Promise((resolve) => setImmediate(resolve)); + } +} + let projectDir: string | null = null; beforeEach(() => { @@ -1848,10 +1859,18 @@ describe('SupervisionAutomation', () => { supervisionAutomation.registerTaskIntent('deck_supervision_brain', 'cmd-loop-one', 'implement the feature', snapshot); beginRun('cmd-loop-one', 'implement the feature'); + // Each step waits for the phase it depends on instead of a fixed sleep. + // Dispatching the audit is async (broker decision + filesystem baseline + // discovery); sleep(25) covered that locally but not on a loaded CI runner. + // Completing the delegated audit before the run reached `auditing` derailed + // the sequence, and call[1] then held the audit-orchestration prompt rather + // than the rework brief — the macOS CI failure this replaces. completeTurn('implemented the feature'); - await sleep(25); + await waitForRunPhase('auditing'); + expect(mockTransportRuntime.send).toHaveBeenCalledTimes(1); + completeDelegatedAudit('REWORK', 'first audit needs fixes'); - await sleep(25); + await waitForRunPhase('execution'); expect(mockTransportRuntime.send).toHaveBeenCalledTimes(2); expect(String(mockTransportRuntime.send.mock.calls[1]?.[0])).toContain('Audit verdict: REWORK'); expect(supervisionAutomation.getActiveRun('deck_supervision_brain')).toMatchObject({ @@ -1860,13 +1879,16 @@ describe('SupervisionAutomation', () => { }); completeTurn('implemented the requested rework'); - await sleep(25); + await waitForRunPhase('auditing'); expect(mockTransportRuntime.send).toHaveBeenCalledTimes(3); expect(String(mockTransportRuntime.send.mock.calls[2]?.[0])).toContain('imcodes send --reply'); + + // maxAuditLoops = 1, so the second REWORK must end the run WITHOUT another + // rework dispatch. Wait for teardown, then assert the count never grew. completeDelegatedAudit('REWORK', 'second audit still needs fixes'); - await sleep(25); - expect(mockTransportRuntime.send).toHaveBeenCalledTimes(3); + await waitForRunEnd(); expect(supervisionAutomation.getActiveRun('deck_supervision_brain')).toBeUndefined(); + expect(mockTransportRuntime.send).toHaveBeenCalledTimes(3); }); it('ignores deprecated combo auditMode and still starts exactly one lightweight peer audit', async () => { From dc4af39c00377dd2550deae5fdb6ea0e1c6fa24c Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sat, 25 Jul 2026 19:35:46 +0800 Subject: [PATCH 06/40] Force dialog answers past the queue instead of deadlocking behind it Answering an AskUserQuestion often did nothing. handleAskAnswer already placed the answer at the queue FRONT, but front placement only wins against other QUEUED messages - it still waits for the active turn to settle, and that turn is the very one paused ON the question. The answer waited for the turn while the turn waited for the answer: a deadlock, not just a delay. Only claude-code-sdk escapes it via answerPendingQuestion; codex-sdk, gemini, qwen, opencode, copilot, grok, kimi and openclaw have no in-place resolver, so every dialog on those providers hit this. A dialog answer is a control response, so per the send-queue contract it must use the priority path. cancel() IS that path: it cuts in line, settles the active turn locally without waiting on the provider, keeps queued messages intact, and drains - which makes the front-placed answer the next turn immediately. That is the "force-interrupt that re-steers the model" the branch already claimed to do. The drain emits the visible user.message, so the handler must not emit it again. Cancel only fires when the send actually came back queued; a directly delivered answer must not abort unrelated work. Co-Authored-By: Claude Fable 5 --- src/daemon/command-handler.ts | 28 ++++++++ .../command-handler-transport-queue.test.ts | 69 +++++++++++++++++++ 2 files changed, 97 insertions(+) diff --git a/src/daemon/command-handler.ts b/src/daemon/command-handler.ts index c0ed4e7c0..a6c87be0b 100644 --- a/src/daemon/command-handler.ts +++ b/src/daemon/command-handler.ts @@ -6088,12 +6088,40 @@ async function handleAskAnswer(cmd: Record, serverLink: ServerL if (result === 'sent') { emitTransportUserMessageEvent(sessionName, answer); } else { + // DEADLOCK BREAK. `queuePlacement: 'front'` only wins against other + // QUEUED messages — it still waits for the active turn to settle. But + // the active turn is usually the very turn that is paused ON this + // question, so it can only settle once the answer arrives: the answer + // waits for the turn, the turn waits for the answer, and the user sees + // "answered, nothing happened". Providers without + // `answerPendingQuestion` (everything except claude-code-sdk today) + // hit this on every dialog. + // + // A dialog answer is a control response, not an ordinary queued send + // (see CLAUDE.md: approval/feedback responses must use the priority + // path and must not be blocked by the ordinary send queue). `cancel()` + // is exactly that priority path: it cuts in line, settles the active + // turn LOCALLY without waiting on the provider, keeps queued messages + // intact, and then drains — so our front entry becomes the next turn + // immediately. This is the "force-interrupt that re-steers the model" + // the branch above always intended. + // + // The drain emits the visible `user.message` for the queued entry, so + // do NOT emit it here as well or the answer renders twice. timelineEmitter.emit( sessionName, 'session.state', buildTransportQueueSessionStatePayload(sessionName, 'queued', 'command_handler_ask_answer'), { source: 'daemon', confidence: 'high' }, ); + try { + await runtime.cancel(); + } catch (err) { + logger.warn( + { err, sessionName }, + 'ask.answer: force-interrupt of the paused turn failed; answer stays queued at the front', + ); + } } } else { // Timed out / already self-continued and no live runtime snapshot is diff --git a/test/daemon/command-handler-transport-queue.test.ts b/test/daemon/command-handler-transport-queue.test.ts index afd588234..c7e054a1b 100644 --- a/test/daemon/command-handler-transport-queue.test.ts +++ b/test/daemon/command-handler-transport-queue.test.ts @@ -878,10 +878,12 @@ describe('handleWebCommand transport queue behavior', () => { now: 102, }); const send = vi.fn(() => 'queued'); + const cancel = vi.fn(async () => {}); getProviderMock.mockReturnValue({ id: 'mock-provider' }); getTransportRuntimeMock.mockReturnValue({ providerSessionId: 'route-transport', send, + cancel, pendingCount: 3, pendingMessages: ['selected option', 'queued msg', 'queued msg 2'], pendingEntries: [ @@ -929,6 +931,73 @@ describe('handleWebCommand transport queue behavior', () => { const answerStateCall = emitMock.mock.calls.find((call) => call[0] === 'deck_transport_brain' && call[1] === 'session.state'); expect(answerStateCall?.[2]).not.toHaveProperty('pendingCount'); expect(answerStateCall?.[2]).not.toHaveProperty('pendingMessages'); + // Front placement alone only beats other QUEUED messages — it still waits + // for the active turn, which is the very turn paused on this question. The + // answer must therefore force-interrupt it (priority path) or the two wait + // on each other forever. + expect(cancel).toHaveBeenCalledTimes(1); + }); + + it('force-interrupts the paused turn so a queued ask.answer cannot deadlock', async () => { + // Regression: user answers the dialog and nothing happens. `send` returns + // 'queued' because the question-paused turn still counts as active work, so + // the answer sits at the queue front waiting for a turn that can only end + // once the answer is delivered. Every provider WITHOUT + // `answerPendingQuestion` (codex-sdk, gemini, qwen, opencode, copilot, + // grok, kimi, openclaw) took this path on every dialog. + const send = vi.fn(() => 'queued'); + const cancel = vi.fn(async () => {}); + getProviderMock.mockReturnValue({ id: 'mock-provider' }); + getTransportRuntimeMock.mockReturnValue({ + providerSessionId: 'route-transport', + send, + cancel, + pendingCount: 1, + pendingMessages: ['picked B'], + pendingEntries: [{ clientMessageId: 'ans', text: 'picked B' }], + pendingVersion: 1, + }); + + handleWebCommand({ + type: 'ask.answer', + sessionName: 'deck_transport_brain', + answer: 'picked B', + }, serverLink as any); + await flushAsync(); + + expect(send).toHaveBeenCalledWith('picked B', undefined, undefined, undefined, { queuePlacement: 'front' }); + expect(cancel).toHaveBeenCalledTimes(1); + // The drain emits the visible user.message for the queued entry; emitting it + // here too would render the answer twice. + const userMessageEmits = emitMock.mock.calls.filter((call) => call[1] === 'user.message'); + expect(userMessageEmits).toHaveLength(0); + }); + + it('does not interrupt anything when the ask.answer was delivered directly', async () => { + // Idle session: `send` returns 'sent', so there is no paused turn to break + // out of. Cancelling here would abort unrelated work for no reason. + const send = vi.fn(() => 'sent'); + const cancel = vi.fn(async () => {}); + getProviderMock.mockReturnValue({ id: 'mock-provider' }); + getTransportRuntimeMock.mockReturnValue({ + providerSessionId: 'route-transport', + send, + cancel, + pendingCount: 0, + pendingMessages: [], + pendingEntries: [], + pendingVersion: 0, + }); + + handleWebCommand({ + type: 'ask.answer', + sessionName: 'deck_transport_brain', + answer: 'picked A', + }, serverLink as any); + await flushAsync(); + + expect(send).toHaveBeenCalledTimes(1); + expect(cancel).not.toHaveBeenCalled(); }); it('undo_queued_message deletes a SQLite-only queued orphan when the runtime in-memory queue is empty', async () => { From 92b5c8b079108d7d34191cdf2fccc79b09845d99 Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sat, 25 Jul 2026 19:45:10 +0800 Subject: [PATCH 07/40] Drop the lower bound when healing the timeline after resume/reconnect "The notification arrived but the chat is empty until I press force refresh" was not a gap-creation bug, it was a gap-HEALING bug. The automatic backfill anchors its cursor to the newest local cursor-eligible event minus TIMELINE_HISTORY_AFTER_TS_OVERLAP_MS, i.e. 1 millisecond. Any event that went missing below that point is unreachable by every automatic request forever; only the manual refresh button, which sends no lower bound, could recover it. agent.status and usage.update are cursor-eligible and fire about once a second during a turn, so a gap became invisible almost immediately - hence "often". Events do go missing regularly. The server relays timeline events only to currently-subscribed viewers, so a backgrounded app has them discarded outright, with no queue and no replay - and a push notification arriving is itself proof the app was backgrounded. Independently, live timeline events are control-plane, so ServerLink.trySend drops them silently whenever the daemon link is flapping, and the reconnect resync re-broadcasts only session.state, never assistant.text / tool.* / user.message. Both windows end with an observable event: app activation and daemon reconnect. Make those two heal properly by requesting the full newest window with no lower bound - the same request the refresh button already makes, and the same treatment the low-completeness seed and empty-IDB paths above already apply for this exact symptom. Cooldowns are untouched, so this adds no request volume beyond what already fired. Deliberately NOT done: seq-discontinuity gap detection. seq is not contiguous in the daemon's own persisted timeline (one live session was missing 629 of 2066, another 1359 of 1950), so it would false-positive constantly and turn every check into an unbounded fetch. The two app-resume tests pinned afterTs at the old tail cursor; their subject is that the resume chain fires, so they now assert the no-lower- bound contract instead. Co-Authored-By: Claude Fable 5 --- web/src/hooks/useTimeline.ts | 23 +++++- web/test/app-resume-refresh.test.tsx | 12 ++- web/test/use-timeline-http-backfill.test.ts | 82 +++++++++++++++++++++ 3 files changed, 113 insertions(+), 4 deletions(-) diff --git a/web/src/hooks/useTimeline.ts b/web/src/hooks/useTimeline.ts index 4180c32f6..4fa07370d 100644 --- a/web/src/hooks/useTimeline.ts +++ b/web/src/hooks/useTimeline.ts @@ -3202,7 +3202,19 @@ export function useTimeline( // 15s does NOT refire on every focus/visibility/appStateChange tick. // App-resume's resetCooldowns:true path explicitly clears the map // so a real foreground from background still bypasses this. - fireHttpBackfillRef.current(0, { phase: 'refresh', cooldownMs: ACTIVE_REFRESH_COOLDOWN_MS }); + // `manualLatestWindow` (no lower bound), NOT the cheap tail catch-up. + // Activation follows a window in which live events were provably dropped: + // the server relays timeline events ONLY to currently-subscribed viewers + // (bridge.ts sendJsonToSessionSubscribers — no subscriber means silent + // discard, no queue, no replay), and a backgrounded app has no + // subscription at all. A push notification arriving is itself proof the + // app was backgrounded. A tail-anchored backfill asks for + // `(localNewest - 1ms, …]`, so the moment ANY newer event lands (and + // `agent.status`/`usage.update` are cursor-eligible and fire every second + // during a turn) the dropped events sit below the cursor and become + // permanently unreachable — the user is left with only the ↻ button. This + // is the same reasoning the seed/IDB paths above already use. + fireHttpBackfillRef.current(0, { phase: 'refresh', cooldownMs: ACTIVE_REFRESH_COOLDOWN_MS, mode: 'manualLatestWindow' }); }; window.addEventListener(ACTIVE_TIMELINE_REFRESH_EVENT, handler); return () => window.removeEventListener(ACTIVE_TIMELINE_REFRESH_EVENT, handler); @@ -3670,7 +3682,14 @@ export function useTimeline( }); setRefreshing(true); sendForwardHistoryRequest('refresh', buildForwardHistoryArgs(MAX_MEMORY_EVENTS)); - fireHttpBackfillRef.current(600, { phase: 'refresh', visible: true }); + // `manualLatestWindow`: a daemon reconnect means the daemon→server link + // was down, and live timeline events are control-plane — `trySend` + // drops them silently with no queue and no replay, and the reconnect + // resync re-broadcasts only `session.state`, never assistant.text / + // tool.* / user.message. So events were almost certainly lost below the + // local tail, where a tail-anchored `(localNewest - 1ms, …]` request can + // never reach them. + fireHttpBackfillRef.current(600, { phase: 'refresh', visible: true, mode: 'manualLatestWindow' }); } } // ── Browser WS disconnected: reset in-flight pagination to prevent stuck state ── diff --git a/web/test/app-resume-refresh.test.tsx b/web/test/app-resume-refresh.test.tsx index b38c27be2..2001d1a2f 100644 --- a/web/test/app-resume-refresh.test.tsx +++ b/web/test/app-resume-refresh.test.tsx @@ -99,10 +99,17 @@ describe('native app resume refresh chain', () => { expect(reconnectNow).toHaveBeenCalledWith(true); expect(fetchSpy).toHaveBeenCalledTimes(1); + // Resume MUST drop the lower bound. While the app was backgrounded it had no + // session subscription, so the server discarded its timeline events outright + // (no queue, no replay). Those events sit BELOW the local tail, so the old + // `afterTs: ` request could never reach them — that is exactly the + // "notification arrived but the chat is empty until I press ↻" bug. This test + // asserts the resume chain fires; the cursor value it used to pin was just + // whatever the tail-anchored implementation produced. expect(fetchSpy).toHaveBeenCalledWith( serverId, sessionName, - expect.objectContaining({ afterTs: 999 }), + expect.objectContaining({ afterTs: undefined }), ); removeListener(); @@ -177,10 +184,11 @@ describe('native app resume refresh chain', () => { expect(reconnectNow).toHaveBeenCalledWith(true); expect(fetchSpy).toHaveBeenCalledTimes(1); + // Same contract as above: a resume backfill carries no lower bound. expect(fetchSpy).toHaveBeenCalledWith( serverId, sessionName, - expect.objectContaining({ afterTs: 999 }), + expect.objectContaining({ afterTs: undefined }), ); }); }); diff --git a/web/test/use-timeline-http-backfill.test.ts b/web/test/use-timeline-http-backfill.test.ts index cb7700646..b4c6dacfb 100644 --- a/web/test/use-timeline-http-backfill.test.ts +++ b/web/test/use-timeline-http-backfill.test.ts @@ -1147,6 +1147,88 @@ describe('useTimeline — HTTP backfill on WS reconnect', () => { expect(fetchSpy).toHaveBeenCalledTimes(1); }); + it('recovers events dropped while backgrounded: activation asks for the full newest window', async () => { + // The "push notification arrived but the chat shows nothing until I hit ↻" + // bug. The server relays timeline events ONLY to currently-subscribed + // viewers, so a backgrounded app (exactly the state a push proves) has its + // events silently discarded — no queue, no replay. On activation the + // backfill therefore MUST drop the lower bound: a tail-anchored + // `(localNewest - 1ms, …]` request can never reach events that were dropped + // below the local tail, and `agent.status`/`usage.update` push that cursor + // forward every second during a turn, so the gap goes permanently invisible. + const sessionName = `deck_bg_gap_${Date.now()}`; + const serverId = `srv-bg-gap-${Date.now()}`; + + const localTail: TimelineEvent = { + eventId: `${sessionName}-tail`, + sessionId: sessionName, + ts: 5_000, + epoch: 1, + seq: 20, + source: 'daemon', + confidence: 'high', + type: 'assistant.text', + payload: { text: 'local-tail' }, + }; + // Emitted while the app was backgrounded, i.e. BELOW the local tail ts. + const droppedWhileBackgrounded: TimelineEvent = { + eventId: `${sessionName}-dropped`, + sessionId: sessionName, + ts: 4_000, + epoch: 1, + seq: 18, + source: 'daemon', + confidence: 'high', + type: 'assistant.text', + payload: { text: 'dropped-while-backgrounded' }, + }; + ingestTimelineEventForCache(serverId, sessionName, localTail); + fetchSpy.mockResolvedValue({ + events: [localTail, droppedWhileBackgrounded], + epoch: 1, + hasMore: false, + nextCursor: null, + }); + + const ws: WsClient = { + connected: true, + onMessage: () => () => {}, + sendTimelineReplayRequest: vi.fn(() => 'replay-bg'), + sendTimelineHistoryRequest: vi.fn(() => 'history-bg'), + } as unknown as WsClient; + + function Probe() { + const { events } = useTimeline(sessionName, ws, serverId); + return h('div', { 'data-testid': 'probe' }, events.map((e) => String(e.payload.text ?? '')).join('|')); + } + + vi.useFakeTimers({ shouldAdvanceTime: true }); + render(h(Probe)); + await waitFor(() => { + expect(screen.getByTestId('probe').textContent).toContain('local-tail'); + }); + await act(async () => { await vi.advanceTimersByTimeAsync(300); }); + fetchSpy.mockClear(); + + __resetBackfillCooldownsForTests(); + await act(async () => { + window.dispatchEvent(new CustomEvent(ACTIVE_TIMELINE_REFRESH_EVENT)); + }); + await act(async () => { await vi.advanceTimersByTimeAsync(50); }); + + expect(fetchSpy).toHaveBeenCalled(); + // No activation fetch may carry a numeric lower bound — that is the bug. + expect(fetchSpy).not.toHaveBeenCalledWith( + serverId, + sessionName, + expect.objectContaining({ afterTs: expect.any(Number) }), + ); + // End-to-end: the dropped event is visible without a manual ↻. + await waitFor(() => { + expect(screen.getByTestId('probe').textContent).toContain('dropped-while-backgrounded'); + }); + }); + it('uses HTTP-backfilled command.ack to mark daemon receipt while keeping the optimistic send pending', async () => { const sessionName = `deck_http_backfill_ack_${Date.now()}`; const serverId = `srv-http-ack-${Date.now()}`; From 8493a614e686d57c862ecaa61ea7e462e921dbcd Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sat, 25 Jul 2026 19:55:45 +0800 Subject: [PATCH 08/40] Stop the link-restore resync from re-firing local idle side effects Regression I introduced in 1d3e2f299. After a server/daemon reconnect the user's phone received hundreds of push notifications, some quoting messages that had been queued two days earlier. lifecycle.ts subscribes to the timeline and treats ANY session.state idle as a finished turn: it calls notifySessionIdle() and drainQueue(). The resync re-broadcasts idle for EVERY transport session on EVERY reconnect, so each reconnect replayed a "turn finished" edge across all of them and drained every stale queue at once. Every agent answered its days-old queued message, every answer fired an idle hook, and each hook became a push. With 30+ transport sessions and a flapping link this multiplies fast. The resync exists to inform the SERVER and BROWSERS of state the daemon already knew; it carries no new information and must not drive local side effects. Stamp it with a shared decisionReason constant and have the daemon-local timeline listener ignore those events - remote consumers still receive them, so the stuck-"working" fix it was written for still works. Also skips liveContextIngestion for resync events: duplicate idle edges were re-triggering memory processing on every reconnect too. Known test gap: the payload marking and the predicate are covered, but the one-line lifecycle listener guard is not - that listener lives inside the large startup() body and cannot be reached without refactoring it. Co-Authored-By: Claude Fable 5 --- shared/session-activity-types.ts | 24 ++++++++++ src/agent/session-manager.ts | 4 +- src/daemon/lifecycle.ts | 9 +++- .../session-manager-state-resync.test.ts | 46 ++++++++++++++++++- 4 files changed, 79 insertions(+), 4 deletions(-) diff --git a/shared/session-activity-types.ts b/shared/session-activity-types.ts index 8bd953571..f700bca21 100644 --- a/shared/session-activity-types.ts +++ b/shared/session-activity-types.ts @@ -189,6 +189,30 @@ export function isWorkingSessionState(value: unknown): boolean { return typeof value === 'string' && WORKING_SESSION_STATES.has(value); } +/** + * `decisionReason` stamped on a `session.state` that is a RE-BROADCAST of state + * the daemon already knew, emitted after the server link is restored so the + * server and browsers can resync. It carries no new information. + * + * Daemon-local listeners MUST ignore it. Treating it as a genuine idle + * transition made every reconnect replay a full "turn finished" edge for every + * transport session, which re-ran `drainQueue()` across all of them: queued + * messages that had been sitting for days were delivered at once, every agent + * answered, and each answer fired an idle hook — hundreds of push notifications + * quoting days-old messages. The resync must inform remote consumers WITHOUT + * re-triggering local side effects. + */ +export const SESSION_STATE_DECISION_REASON_SERVER_LINK_RESYNC = 'server_link_resync' as const; + +/** True when a `session.state` payload is a link-restore re-broadcast. */ +export function isServerLinkResyncStatePayload(payload: unknown): boolean { + return Boolean(payload) + && typeof payload === 'object' + && !Array.isArray(payload) + && (payload as { decisionReason?: unknown }).decisionReason + === SESSION_STATE_DECISION_REASON_SERVER_LINK_RESYNC; +} + /** * Timeline events that are pure metadata/telemetry: they say nothing about * whether a turn is still running, so any tail scan looking for activity diff --git a/src/agent/session-manager.ts b/src/agent/session-manager.ts index 77f843640..bb9dcefed 100644 --- a/src/agent/session-manager.ts +++ b/src/agent/session-manager.ts @@ -52,7 +52,7 @@ import { resolveTransportContextBootstrap } from './runtime-context-bootstrap.js import { QWEN_AUTH_TYPES } from '../../shared/qwen-auth.js'; import { TIMELINE_SUPPRESS_PUSH_FIELD } from '../../shared/push-notifications.js'; import { IMCODES_SESSION_ENV, IMCODES_SESSION_LABEL_ENV } from '../../shared/imcodes-send.js'; -import { buildCodexLifecycleTerminalMetadata, isWorkingSessionState, type ActivityGenerationLike } from '../../shared/session-activity-types.js'; +import { SESSION_STATE_DECISION_REASON_SERVER_LINK_RESYNC, buildCodexLifecycleTerminalMetadata, isWorkingSessionState, type ActivityGenerationLike } from '../../shared/session-activity-types.js'; import { SDK_SUBAGENT_DETAIL_KIND, SDK_SUBAGENT_DIAGNOSTIC, @@ -1583,7 +1583,7 @@ export function resyncTransportSessionStatesAfterLinkRestore( for (const [sessionName, runtime] of list) { try { const built = buildTransportSessionStatePayload(sessionName, runtime, runtime.getStatus(), { - decisionReason: 'server_link_resync', + decisionReason: SESSION_STATE_DECISION_REASON_SERVER_LINK_RESYNC, clearSource: 'server-link-resync', queueReason: 'server_link_resync', }); diff --git a/src/daemon/lifecycle.ts b/src/daemon/lifecycle.ts index ab8ffbd78..229e52201 100644 --- a/src/daemon/lifecycle.ts +++ b/src/daemon/lifecycle.ts @@ -62,7 +62,7 @@ import { import { buildWorkerSessionSyncPlan, type WorkerSessionSyncPlanInput } from './worker-session-sync-plan.js'; import { buildTransportQueueSnapshotPayload } from './transport-queue-projection.js'; import { getStaleSessionCompressionRun, resolveSessionCompressionWatchRuns } from '../context/summary-compressor.js'; -import { normalizeActivityGeneration } from '../../shared/session-activity-types.js'; +import { isServerLinkResyncStatePayload, normalizeActivityGeneration } from '../../shared/session-activity-types.js'; import type { TransportRuntimeDiagnosticSnapshot } from '../agent/transport-session-runtime.js'; function latestAssistantTextFromEvents(events: Array<{ type?: unknown; payload?: unknown }>): string | undefined { @@ -822,6 +822,13 @@ export async function startup(): Promise { // Wire timeline idle events → P2P orchestrator + queued message drain (covers all agent types: CC, codex, gemini, etc.) timelineEmitter.on((e) => { + // A link-restore resync re-states what the daemon already knew so the server + // and browsers can catch up. It is NOT a fresh idle transition: running the + // side effects below for it made every reconnect drain the queue of every + // transport session at once, delivering messages that had been queued for + // days and flooding the user with notifications. Remote consumers still get + // the event; local listeners must ignore it. + if (e.type === 'session.state' && isServerLinkResyncStatePayload(e.payload)) return; liveContextIngestion.handleTimelineEvent(e); if (e.type === 'session.state' && (e.payload as Record).state === 'idle') { notifySessionIdle(e.sessionId); diff --git a/test/agent/session-manager-state-resync.test.ts b/test/agent/session-manager-state-resync.test.ts index 6c4318391..1cd9f1218 100644 --- a/test/agent/session-manager-state-resync.test.ts +++ b/test/agent/session-manager-state-resync.test.ts @@ -28,7 +28,11 @@ import { setSessionPersistCallback, } from '../../src/agent/session-manager.js'; import { timelineEmitter } from '../../src/daemon/timeline-emitter.js'; -import { isAuthoritativeIdlePayloadShape } from '../../shared/session-activity-types.js'; +import { + SESSION_STATE_DECISION_REASON_SERVER_LINK_RESYNC, + isAuthoritativeIdlePayloadShape, + isServerLinkResyncStatePayload, +} from '../../shared/session-activity-types.js'; import { upsertSession, removeSession } from '../../src/store/session-store.js'; import type { TransportSessionRuntime } from '../../src/agent/transport-session-runtime.js'; @@ -154,3 +158,43 @@ describe('resyncTransportSessionStatesAfterLinkRestore', () => { expect(emitSpy).not.toHaveBeenCalled(); }); }); + +describe('server-link resync payload marking', () => { + /** + * Incident: after a server/daemon reconnect the user got hundreds of push + * notifications, some quoting messages queued two days earlier. + * + * lifecycle.ts wires `timelineEmitter.on(...)` so that ANY + * `session.state: idle` runs `notifySessionIdle()` + `drainQueue()`. The + * resync re-broadcasts idle for EVERY transport session on EVERY reconnect, + * so each reconnect replayed a "turn finished" edge across all of them, + * draining every stale queue at once. Every agent answered, every answer fired + * an idle hook, and the phone got flooded. + * + * The resync payload must therefore be identifiable so daemon-local listeners + * can ignore it while the server and browsers still receive it. + */ + it('marks resync state payloads so local listeners can skip them', () => { + const emitSpy = vi.spyOn(timelineEmitter, 'emit').mockImplementation(() => null as never); + try { + resyncTransportSessionStatesAfterLinkRestore([ + [IDLE_SESSION, makeRuntime({ status: 'idle' })], + ]); + const [, , payload] = emitSpy.mock.calls[0] as unknown as [string, string, Record]; + expect(payload.state).toBe('idle'); + expect(payload.decisionReason).toBe(SESSION_STATE_DECISION_REASON_SERVER_LINK_RESYNC); + expect(isServerLinkResyncStatePayload(payload)).toBe(true); + } finally { + emitSpy.mockRestore(); + } + }); + + it('does not mark an ordinary live idle as a resync', () => { + // The live onStatusChange path uses `activity_reconciler_clear`; only that + // one may drive drainQueue / notifySessionIdle. + expect(isServerLinkResyncStatePayload({ state: 'idle', decisionReason: 'activity_reconciler_clear' })).toBe(false); + expect(isServerLinkResyncStatePayload({ state: 'idle' })).toBe(false); + expect(isServerLinkResyncStatePayload(null)).toBe(false); + expect(isServerLinkResyncStatePayload('server_link_resync')).toBe(false); + }); +}); From 675633c21e233272b27c6a9c0bdf97b20acb7e76 Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sat, 25 Jul 2026 21:21:35 +0800 Subject: [PATCH 09/40] Count timeline events dropped before they reach a browser MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Bug A fix only repaired the RECOVERY layer: on activation and on daemon reconnect the browser now asks for the newest window with no lower bound, so a gap heals itself instead of waiting for a manual force-refresh. The underlying drop rate did not change at all — which means that fix also removed the only signal we had. A regression that doubles the drop rate would now look exactly like a healthy system, just with more backfill traffic. So make both silent-discard points count what they throw away: 1. Daemon → server. Live timeline events are control-plane, so trySend() returns false and drops them whenever the socket is not OPEN. There is no queue and no replay; only the data plane has dataPlaneSendQueue. 2. Server → browser. sendJsonToSessionSubscribers() writes only to sockets subscribed to that session. With zero subscribers both loops execute zero bodies — the normal state of a backgrounded app, and the reason "the push notification arrived but the chat was empty" reproduced so reliably. Only content-bearing events are counted. agent.status and usage.update fire about once a second during a turn, so counting them would bury the signal under noise; unknown types fail closed so a future noisy event type cannot silently inflate the series. Counter names live in shared/ because both sides emit them and drifting names would split the series and hide a regression in plain sight. sendJsonToSessionSubscribers() now returns the recipient count instead of the telemetry re-parsing the serialized event: the relay call site already holds the parsed event, and that path runs for every event of every session. Deliberately NOT included: seq-gap detection. The daemon's own JSONL shows seq is not contiguous in practice (one session had 2066 events missing seq 629, another 1950 missing 1359), so a gap detector would false-positive constantly and teach us to ignore it. Co-Authored-By: Claude Opus 5 --- server/src/ws/bridge.ts | 56 ++++++- .../bridge-timeline-drop-telemetry.test.ts | 146 ++++++++++++++++++ shared/timeline-delivery-telemetry.ts | 86 +++++++++++ src/daemon/server-link.ts | 51 +++++- test/daemon/server-link.test.ts | 46 ++++++ .../timeline-delivery-telemetry.test.ts | 79 ++++++++++ 6 files changed, 461 insertions(+), 3 deletions(-) create mode 100644 server/test/bridge-timeline-drop-telemetry.test.ts create mode 100644 shared/timeline-delivery-telemetry.ts create mode 100644 test/shared/timeline-delivery-telemetry.test.ts diff --git a/server/src/ws/bridge.ts b/server/src/ws/bridge.ts index 22753dee1..e13407b7e 100644 --- a/server/src/ws/bridge.ts +++ b/server/src/ws/bridge.ts @@ -115,6 +115,10 @@ import { updateServerHeartbeat, updateServerStatus, upsertDiscussion, insertDisc import { toDiscussionCommentView } from '../share/discussion-comment-view.js'; import { resolveCoveredSessionNames } from '../share/covered-sessions.js'; import logger from '../util/logger.js'; +import { + TIMELINE_DELIVERY_METRICS, + countableTimelineEventType, +} from '../../../shared/timeline-delivery-telemetry.js'; import { incrementCounter } from '../util/metrics.js'; import { pickReadableSessionDisplay } from '../../../shared/session-display.js'; import { isKnownTestSessionLike } from '../../../shared/test-session-guard.js'; @@ -590,6 +594,9 @@ type TimelineDataPlaneJob = { const WATCH_RECENT_TEXT_CAP = 5; const WATCH_RECENT_TEXT_MAX_CHARS = 160; let idlePushSettleMs = process.env.NODE_ENV === 'test' ? 0 : 300; +// Human-facing heartbeat for no-subscriber drops. The counter records each one; +// this keeps the log readable when a whole app-background window is discarded. +const TIMELINE_NO_SUBSCRIBER_LOG_THROTTLE_MS = 30_000; const HTTP_TIMELINE_TIMEOUT_MS = 15_000; const TIMELINE_PENDING_UNICAST_TIMEOUT_MS = 30_000; // Bumped from 128 → 4096 and 15s → 60s as part of the commit-42dfabec @@ -1216,6 +1223,9 @@ export class WsBridge { /** Lightweight per-session hot cache for Watch first-paint text. */ private recentTextBySession = new Map(); + /** Content-bearing timeline events discarded because nobody was subscribed. */ + private timelineNoSubscriberDrops = 0; + private lastTimelineNoSubscriberLogAt = 0; private pendingIdlePushes = new Map | null; db: Database; @@ -4522,7 +4532,8 @@ export class WsBridge { // Bypass TerminalForwardQueue: timeline events are control-plane and // must never queue behind PTY data. Critical for cancel/stop UX — // session.state(idle) used to arrive seconds after the push notification. - this.sendJsonToSessionSubscribers(sessionId, JSON.stringify(msg)); + const timelineRecipients = this.sendJsonToSessionSubscribers(sessionId, JSON.stringify(msg)); + this.recordTimelineFanout(sessionId, rawEvent, timelineRecipients); return; } @@ -5165,7 +5176,7 @@ export class WsBridge { * subscriptions to); we dedup per-WS so the same JSON is never sent * twice. */ - private sendJsonToSessionSubscribers(sessionName: string, json: string): void { + private sendJsonToSessionSubscribers(sessionName: string, json: string): number { const sent = new Set(); for (const [ws, sessions] of this.browserSubscriptions) { if (!sessions.has(sessionName)) continue; @@ -5185,6 +5196,47 @@ export class WsBridge { sent.add(ws); safeSend(ws, outgoing); } + return sent.size; + } + + /** + * Observe the fan-out result for a live timeline event. + * + * With zero subscribers both loops in `sendJsonToSessionSubscribers` simply never + * execute a body: the event is discarded with no queue, no replay and — until now + * — no trace. That is the normal state of a BACKGROUNDED app, which is exactly why + * "the push notification arrived but the chat was empty" was so reproducible. The + * browser now heals this with a no-lower-bound backfill on activation, so without + * a counter a rising drop rate would be permanently masked by recovery. + * + * Takes the already-parsed event and the recipient count the fan-out returned: + * this runs for every event of every session, so it must not re-parse the + * serialized copy. Only content-bearing events are counted; status/usage chatter + * fires about once a second during a turn and would drown the signal. + */ + private recordTimelineFanout(sessionName: string, event: unknown, recipientCount: number): void { + const eventType = countableTimelineEventType(event); + if (!eventType) return; + if (recipientCount > 0) { + incrementCounter(TIMELINE_DELIVERY_METRICS.SERVER_DELIVERED, { eventType }); + return; + } + incrementCounter(TIMELINE_DELIVERY_METRICS.SERVER_NO_SUBSCRIBER_DROPPED, { eventType }); + this.timelineNoSubscriberDrops += 1; + const now = Date.now(); + if (now - this.lastTimelineNoSubscriberLogAt < TIMELINE_NO_SUBSCRIBER_LOG_THROTTLE_MS) return; + this.lastTimelineNoSubscriberLogAt = now; + logger.warn({ + serverId: this.serverId, + sessionName, + eventType, + droppedTotal: this.timelineNoSubscriberDrops, + }, 'Timeline event discarded: no subscribed viewer (client must heal via backfill)'); + } + + /** Total content-bearing timeline events discarded for lack of a subscriber. */ + get timelineNoSubscriberDropCount(): number { + return this.timelineNoSubscriberDrops; } private sendToRawSessionSubscribers(sessionName: string, data: string | Buffer): void { diff --git a/server/test/bridge-timeline-drop-telemetry.test.ts b/server/test/bridge-timeline-drop-telemetry.test.ts new file mode 100644 index 000000000..58278843d --- /dev/null +++ b/server/test/bridge-timeline-drop-telemetry.test.ts @@ -0,0 +1,146 @@ +/** + * Bridge-level observability for timeline events that never reach a browser. + * + * `sendJsonToSessionSubscribers` writes only to sockets currently subscribed to + * that session. With no subscriber both loops execute zero bodies, so the event + * is discarded with no queue and no replay. That is the NORMAL state of a + * backgrounded app — which is why "the push notification arrived but the chat was + * empty until I hit force-refresh" reproduced so reliably. + * + * The client now heals that automatically (activation/reconnect request the full + * newest window with no lower bound), so the drop itself has to be counted or a + * rising drop rate would be permanently masked by the recovery. + * + * @vitest-environment node + */ + +import { EventEmitter } from 'node:events'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { WsBridge } from '../src/ws/bridge.js'; +import { TIMELINE_MESSAGES } from '../../shared/timeline-protocol.js'; +import { TIMELINE_DELIVERY_METRICS } from '../../shared/timeline-delivery-telemetry.js'; +import { getCounter, resetMetricsForTests } from '../src/util/metrics.js'; + +vi.mock('../src/security/crypto.js', () => ({ + sha256Hex: (_s: string) => 'valid-hash', +})); + +vi.mock('../src/routes/push.js', () => ({ + dispatchPush: vi.fn(), +})); + +class MockWs extends EventEmitter { + sent: Array = []; + closed = false; + readyState = 1; + + send(data: string | Buffer, _opts?: unknown, callback?: (err?: Error) => void): void { + if (this.closed) { callback?.(new Error('socket closed')); return; } + this.sent.push(data); + callback?.(); + } + + close(): void { + this.closed = true; + this.readyState = 3; + this.emit('close'); + } + + get sentStrings(): string[] { + return this.sent.filter((s): s is string => typeof s === 'string'); + } +} + +function makeDb() { + return { + queryOne: async () => ({ token_hash: 'valid-hash', node_role: 'full', revoked_at: null }), + query: async () => [], + execute: async () => ({ changes: 1 }), + exec: async () => {}, + transaction: async (fn: (tx: unknown) => Promise) => fn({}), + close: () => {}, + } as unknown as import('../src/db/client.js').Database; +} + +async function flushAsync(): Promise { + await new Promise((resolve) => { setImmediate(resolve); }); +} + +const SESSION = 'deck_droptelemetry_brain'; + +function timelineEvent(type: string, text = 'hello'): string { + return JSON.stringify({ + type: TIMELINE_MESSAGES.EVENT, + event: { + eventId: `evt-${Math.random().toString(36).slice(2)}`, + sessionId: SESSION, + ts: Date.now(), + seq: 1, + epoch: 1, + type, + payload: { text }, + }, + }); +} + +describe('WsBridge timeline drop telemetry', () => { + let serverId: string; + + beforeEach(() => { + serverId = `drop-${Math.random().toString(36).slice(2)}`; + resetMetricsForTests(); + }); + + afterEach(() => { + WsBridge.getAll().clear(); + vi.clearAllMocks(); + }); + + async function setupAuthedDaemon() { + const bridge = WsBridge.get(serverId); + const daemon = new MockWs(); + bridge.handleDaemonConnection(daemon as never, makeDb() as never, {} as never); + daemon.emit('message', JSON.stringify({ type: 'auth', serverId, token: 'tok' })); + await flushAsync(); + expect(bridge.isAuthenticated).toBe(true); + return { bridge, daemon }; + } + + it('counts a content-bearing timeline event discarded because nobody is subscribed', async () => { + // Exactly the backgrounded-app case: daemon relays, zero viewers subscribed. + const { daemon } = await setupAuthedDaemon(); + + daemon.emit('message', timelineEvent('assistant.text', 'answer nobody saw')); + await flushAsync(); + + expect(getCounter(TIMELINE_DELIVERY_METRICS.SERVER_NO_SUBSCRIBER_DROPPED, { eventType: 'assistant.text' })) + .toBe(1); + expect(getCounter(TIMELINE_DELIVERY_METRICS.SERVER_DELIVERED, { eventType: 'assistant.text' })) + .toBe(0); + }); + + it('does not count high-frequency status chatter, only content', async () => { + // agent.status fires ~1/s during a turn; counting it would drown the signal. + const { daemon } = await setupAuthedDaemon(); + + daemon.emit('message', timelineEvent('agent.status')); + daemon.emit('message', timelineEvent('usage.update')); + await flushAsync(); + + expect(getCounter(TIMELINE_DELIVERY_METRICS.SERVER_NO_SUBSCRIBER_DROPPED, { eventType: 'agent.status' })) + .toBe(0); + expect(getCounter(TIMELINE_DELIVERY_METRICS.SERVER_NO_SUBSCRIBER_DROPPED, { eventType: 'usage.update' })) + .toBe(0); + }); + + it('exposes a running total so a whole backgrounded window is visible as one number', async () => { + const { bridge, daemon } = await setupAuthedDaemon(); + + daemon.emit('message', timelineEvent('assistant.text', 'one')); + daemon.emit('message', timelineEvent('tool.call', 'two')); + daemon.emit('message', timelineEvent('user.message', 'three')); + await flushAsync(); + + expect(bridge.timelineNoSubscriberDropCount).toBe(3); + }); +}); diff --git a/shared/timeline-delivery-telemetry.ts b/shared/timeline-delivery-telemetry.ts new file mode 100644 index 000000000..30b6aa387 --- /dev/null +++ b/shared/timeline-delivery-telemetry.ts @@ -0,0 +1,86 @@ +/** + * Observability for timeline events that are DROPPED before reaching a browser. + * + * Why this exists: the timeline has two silent-discard points, and both were + * invisible. + * + * 1. Daemon → server. Live timeline events are control-plane, so + * `ServerLink.trySend` returns false and drops them whenever the socket is + * not OPEN. There is no queue and no replay (only the data plane has + * `dataPlaneSendQueue`). + * 2. Server → browser. `sendJsonToSessionSubscribers` only writes to sockets + * that are currently subscribed to that session. With no subscriber the loop + * body never runs at all — a backgrounded app therefore has its events + * discarded outright. + * + * The healing fix (activation / reconnect requesting the full newest window with + * no lower bound) makes the user-visible symptom go away, which is exactly why + * these counters matter: without them a rising drop rate would now be masked by + * recovery instead of reported. Counter names are shared so daemon and server + * emit the same series and a regression shows up as a number, not a bug report. + */ + +/** Counter names. Kept in one place so daemon/server/tests cannot drift. */ +export const TIMELINE_DELIVERY_METRICS = { + /** Daemon-side: a control-plane timeline event hit a non-OPEN socket. */ + DAEMON_LINK_DOWN_DROPPED: 'timeline.delivery.daemon_link_down_dropped', + /** Server-side: relayed to a session with zero subscribed viewers. */ + SERVER_NO_SUBSCRIBER_DROPPED: 'timeline.delivery.server_no_subscriber_dropped', + /** Server-side: relayed and delivered to at least one viewer. */ + SERVER_DELIVERED: 'timeline.delivery.server_delivered', +} as const; + +export type TimelineDeliveryMetric = + typeof TIMELINE_DELIVERY_METRICS[keyof typeof TIMELINE_DELIVERY_METRICS]; + +/** + * Message types whose loss leaves a user-visible hole in the chat, as opposed to + * status/telemetry chatter that the next update supersedes anyway. Only these are + * worth counting: `agent.status` fires about once a second during a turn, so + * counting it would bury the signal that actually matters. + */ +const CONTENT_BEARING_TIMELINE_EVENT_TYPES: ReadonlySet = new Set([ + 'user.message', + 'assistant.text', + 'tool.call', + 'tool.result', + 'file.change', + 'ask.question', + 'peer_audit.result', +]); + +/** + * True when a dropped `timeline.event` would leave a visible gap. + * + * `event` is the inner timeline event of a `timeline.event` envelope. Anything + * unrecognised counts as NOT content-bearing so a future noisy event type cannot + * silently inflate the drop series. + */ +export function isContentBearingTimelineEvent(event: unknown): boolean { + if (!event || typeof event !== 'object' || Array.isArray(event)) return false; + const type = (event as { type?: unknown }).type; + return typeof type === 'string' && CONTENT_BEARING_TIMELINE_EVENT_TYPES.has(type); +} + +/** + * The event type when this event is worth counting, otherwise undefined. + * + * Both drop sites need exactly this pair of answers (is it countable, and under + * which label), so they share one call instead of asking twice. Callers that + * already hold the parsed event — the server relay does — must never re-parse the + * serialized copy just to build a counter label; that path runs for every event + * of every session. + */ +export function countableTimelineEventType(event: unknown): string | undefined { + if (!isContentBearingTimelineEvent(event)) return undefined; + return (event as { type: string }).type; +} + +/** Extract the timeline event type from a `timeline.event` envelope, if any. */ +export function timelineEventTypeOf(msg: unknown): string | undefined { + if (!msg || typeof msg !== 'object' || Array.isArray(msg)) return undefined; + const event = (msg as { event?: unknown }).event; + if (!event || typeof event !== 'object' || Array.isArray(event)) return undefined; + const type = (event as { type?: unknown }).type; + return typeof type === 'string' ? type : undefined; +} diff --git a/src/daemon/server-link.ts b/src/daemon/server-link.ts index 4816e22a8..976a596a2 100644 --- a/src/daemon/server-link.ts +++ b/src/daemon/server-link.ts @@ -21,7 +21,7 @@ import { P2P_WORKFLOW_MSG } from '../../shared/p2p-workflow-messages.js'; import { SESSION_GROUP_CLONE_CAPABILITY_V1 } from '../../shared/session-group-clone.js'; import { EXECUTION_CLONE_CAPABILITY_V1 } from '../../shared/execution-clone.js'; import { GIT_REMOTE_CLONE_CAPABILITY_V1 } from '../../shared/git-remote-url.js'; -import { TIMELINE_PROTOCOL_CAPABILITY, TIMELINE_PROTOCOL_REVISION } from '../../shared/timeline-protocol.js'; +import { TIMELINE_MESSAGES, TIMELINE_PROTOCOL_CAPABILITY, TIMELINE_PROTOCOL_REVISION } from '../../shared/timeline-protocol.js'; import { FILE_TRANSFER_UPLOAD_FETCH_CAPABILITY, FILE_TRANSFER_DOWNLOAD_STREAM_CAPABILITY, @@ -34,6 +34,11 @@ import { stringifyForServerSend, } from './latency-tracer.js'; import { getDaemonBuildInfo } from './build-info.js'; +import { incrementCounter } from '../util/metrics.js'; +import { + TIMELINE_DELIVERY_METRICS, + countableTimelineEventType, +} from '../../shared/timeline-delivery-telemetry.js'; interface SystemStats { cpu: number; @@ -155,6 +160,10 @@ const WATCHDOG_MS = 10_000; // check connection health every 10s // getOpenSocketSilenceMs() still prevents this from false-reconnecting a healthy // socket during the daemon's own load spikes (it reports 0 silence then). const SILENT_CONNECTION_RECYCLE_MS = 30_000; +// Throttle for the dropped-timeline-event warning. The counter records every +// drop; the log line is a human-facing heartbeat so a flapping link is visible +// in daemon.log without one line per lost message. +const TIMELINE_DROP_LOG_THROTTLE_MS = 30_000; // Event-loop stall guard. Under heavy local load the daemon's event loop can // freeze for tens of seconds. The silence-based reconnects measure // `now - lastPong`, which during a freeze blames the SERVER for the daemon's @@ -270,6 +279,9 @@ export class ServerLink { * reconnect state-resync so a daemon's FIRST connect (startup already runs * its own full session sync) does not double-broadcast. */ private hadConnectedBefore = false; + /** Running total of content-bearing timeline events dropped while link down. */ + private timelineDropsWhileLinkDown = 0; + private lastTimelineDropLogAt = 0; private lastPong = 0; // timestamp of last received message (any message counts as proof of life) private seq = 0; private readonly workerUrl: string; @@ -515,6 +527,15 @@ export class ServerLink { // since the daemon must never die from transient disconnects. // Callers that need delivery confirmation should check isConnected() // or await a response event before acting on `send()`. + // + // "Silently" used to be literal. Live timeline events are control-plane, so + // this branch is where a chat message vanishes with no queue and no replay + // — the browser then only recovers it via a no-lower-bound backfill. Now + // that activation/reconnect heals that automatically, an increase in the + // underlying drop rate would be invisible, so count it. Only + // content-bearing events are counted; `agent.status` fires ~1/s during a + // turn and would bury the signal. + this.recordTimelineDropWhileLinkDown(msg); return false; } try { @@ -780,6 +801,34 @@ export class ServerLink { }); } + /** + * Count a content-bearing timeline event lost because the link was down. + * + * Periodically warns with the running total so a flapping link shows up in the + * daemon log as an accumulating number instead of silence. The counter itself + * is the durable signal; the log line is for humans reading `daemon.log`. + */ + private recordTimelineDropWhileLinkDown(msg: unknown): void { + if (messageTypeOf(msg) !== TIMELINE_MESSAGES.EVENT) return; + const eventType = countableTimelineEventType((msg as { event?: unknown } | null)?.event); + if (!eventType) return; + incrementCounter(TIMELINE_DELIVERY_METRICS.DAEMON_LINK_DOWN_DROPPED, { eventType }); + this.timelineDropsWhileLinkDown += 1; + const now = Date.now(); + if (now - this.lastTimelineDropLogAt < TIMELINE_DROP_LOG_THROTTLE_MS) return; + this.lastTimelineDropLogAt = now; + logger.warn({ + droppedTotal: this.timelineDropsWhileLinkDown, + eventType, + readyState: this.ws?.readyState ?? 'no-socket', + }, 'ServerLink: dropped content-bearing timeline event while link was down (browser must heal via backfill)'); + } + + /** Total content-bearing timeline events dropped while the link was down. */ + get timelineDropCountWhileLinkDown(): number { + return this.timelineDropsWhileLinkDown; + } + /** Reports whether the underlying WebSocket is currently OPEN. */ isConnected(): boolean { return !!this.ws && this.ws.readyState === WebSocket.OPEN; diff --git a/test/daemon/server-link.test.ts b/test/daemon/server-link.test.ts index e52a6993e..bfa3c0f19 100644 --- a/test/daemon/server-link.test.ts +++ b/test/daemon/server-link.test.ts @@ -16,6 +16,8 @@ vi.mock('../../src/util/daemon-status.js', () => ({ })); import { ServerLink, __setServerLinkDataPlaneQueueConfigForTests, setServerLinkReconnectResyncHandler } from '../../src/daemon/server-link.js'; +import { TIMELINE_DELIVERY_METRICS } from '../../shared/timeline-delivery-telemetry.js'; +import { getCounter, resetMetricsForTests } from '../../src/util/metrics.js'; import { recordDaemonServerLinkStatus } from '../../src/util/daemon-status.js'; import { TIMELINE_MESSAGES, TIMELINE_PROTOCOL_CAPABILITY } from '../../shared/timeline-protocol.js'; import { TRANSPORT_EVENT } from '../../shared/transport-events.js'; @@ -344,6 +346,50 @@ describe('ServerLink', () => { expect(mockWsInstance.close).toHaveBeenCalledTimes(1); }); + it('counts content-bearing timeline events dropped while the link is down', () => { + // This branch is where a chat message vanishes: live timeline events are + // control-plane, so a non-OPEN socket drops them with no queue and no replay. + // The browser now heals it via a no-lower-bound backfill, so without this + // counter a rising drop rate would be masked by the recovery. + resetMetricsForTests(); + mockWsInstance.readyState = 3; // CLOSED + + const before = getCounter(TIMELINE_DELIVERY_METRICS.DAEMON_LINK_DOWN_DROPPED, { eventType: 'assistant.text' }); + expect(link.send({ + type: TIMELINE_MESSAGES.EVENT, + event: { type: 'assistant.text', eventId: 'e1', sessionId: 's', ts: 1, payload: { text: 'lost' } }, + })).toBeUndefined(); + expect(getCounter(TIMELINE_DELIVERY_METRICS.DAEMON_LINK_DOWN_DROPPED, { eventType: 'assistant.text' })) + .toBe(before + 1); + expect(link.timelineDropCountWhileLinkDown).toBe(1); + }); + + it('does not count high-frequency status chatter as a lost message', () => { + // agent.status fires ~1/s during a turn; counting it would bury the signal. + resetMetricsForTests(); + mockWsInstance.readyState = 3; + + link.send({ + type: TIMELINE_MESSAGES.EVENT, + event: { type: 'agent.status', eventId: 'e2', sessionId: 's', ts: 2, payload: { status: 'thinking' } }, + }); + link.send({ type: 'heartbeat' }); + + expect(getCounter(TIMELINE_DELIVERY_METRICS.DAEMON_LINK_DOWN_DROPPED, { eventType: 'agent.status' })).toBe(0); + expect(link.timelineDropCountWhileLinkDown).toBe(0); + }); + + it('counts nothing while the link is OPEN', () => { + resetMetricsForTests(); + mockWsInstance.readyState = 1; // OPEN + link.connect(); + link.send({ + type: TIMELINE_MESSAGES.EVENT, + event: { type: 'assistant.text', eventId: 'e3', sessionId: 's', ts: 3, payload: { text: 'delivered' } }, + }); + expect(link.timelineDropCountWhileLinkDown).toBe(0); + }); + it('fires the reconnect state-resync handler on RE-connect only, not the first connect', async () => { // Regression for deck_sub_26624c1t: live timeline events (incl. the // authoritative session.state idle) are control-plane and silently dropped diff --git a/test/shared/timeline-delivery-telemetry.test.ts b/test/shared/timeline-delivery-telemetry.test.ts new file mode 100644 index 000000000..1bdd2325b --- /dev/null +++ b/test/shared/timeline-delivery-telemetry.test.ts @@ -0,0 +1,79 @@ +/** + * The healing fix (activation / reconnect requesting a no-lower-bound window) + * removes the user-visible symptom of dropped timeline events, which is exactly + * why the drop itself has to become a number. These tests pin the classifier so + * the counters cannot silently start counting the wrong thing: + * + * - content-bearing events (a lost one leaves a hole in the chat) are counted; + * - status/telemetry chatter is NOT — `agent.status` fires about once a second + * during a turn and would bury the signal under noise. + */ +import { describe, expect, it } from 'vitest'; + +import { + TIMELINE_DELIVERY_METRICS, + isContentBearingTimelineEvent, + timelineEventTypeOf, +} from '../../shared/timeline-delivery-telemetry.js'; + +describe('timeline delivery telemetry classifier', () => { + it('counts events whose loss leaves a visible hole in the chat', () => { + for (const type of [ + 'user.message', + 'assistant.text', + 'tool.call', + 'tool.result', + 'file.change', + 'ask.question', + 'peer_audit.result', + ]) { + expect(isContentBearingTimelineEvent({ type })).toBe(true); + } + }); + + it('ignores high-frequency status/telemetry so it cannot drown the signal', () => { + for (const type of [ + 'agent.status', + 'usage.update', + 'session.state', + 'assistant.thinking', + 'memory.context', + 'memory.compression', + 'transport.queue.snapshot', + 'command.ack', + ]) { + expect(isContentBearingTimelineEvent({ type })).toBe(false); + } + }); + + it('treats unknown and malformed events as non-counting (fail closed)', () => { + // A future noisy event type must not inflate the drop series by default. + expect(isContentBearingTimelineEvent({ type: 'something.new' })).toBe(false); + expect(isContentBearingTimelineEvent({})).toBe(false); + expect(isContentBearingTimelineEvent(null)).toBe(false); + expect(isContentBearingTimelineEvent(undefined)).toBe(false); + expect(isContentBearingTimelineEvent('assistant.text')).toBe(false); + expect(isContentBearingTimelineEvent([{ type: 'assistant.text' }])).toBe(false); + }); + + it('reads the inner event type out of a timeline.event envelope', () => { + expect(timelineEventTypeOf({ type: 'timeline.event', event: { type: 'assistant.text' } })) + .toBe('assistant.text'); + expect(timelineEventTypeOf({ type: 'timeline.event' })).toBeUndefined(); + expect(timelineEventTypeOf({ type: 'timeline.event', event: null })).toBeUndefined(); + expect(timelineEventTypeOf(null)).toBeUndefined(); + }); + + it('keeps one shared counter name per drop site so daemon and server agree', () => { + // Daemon and server both emit these; drifting names would split the series + // and hide a regression in plain sight. + expect(TIMELINE_DELIVERY_METRICS.DAEMON_LINK_DOWN_DROPPED) + .toBe('timeline.delivery.daemon_link_down_dropped'); + expect(TIMELINE_DELIVERY_METRICS.SERVER_NO_SUBSCRIBER_DROPPED) + .toBe('timeline.delivery.server_no_subscriber_dropped'); + expect(TIMELINE_DELIVERY_METRICS.SERVER_DELIVERED) + .toBe('timeline.delivery.server_delivered'); + expect(new Set(Object.values(TIMELINE_DELIVERY_METRICS)).size) + .toBe(Object.values(TIMELINE_DELIVERY_METRICS).length); + }); +}); From 5ccfad8f71c06d49d65f5e50c3e9879331961946 Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sat, 25 Jul 2026 21:43:14 +0800 Subject: [PATCH 10/40] Raise counter cardinality cap to 10k and de-duplicate the metrics store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cap exists to stop a caller that labels by something unbounded (a session id, a file path) from growing the map forever; 1000 was low enough that a handful of label dimensions could reach it, and past the cap NEW keys are silently dropped — so hitting it means later metrics go missing without any complaint. Raising it meant editing the same constant in two places, which is the tell: the counter store existed as two near-identical copies that had ALREADY drifted — only the server copy ever grew addCounter(). So the implementation moves to shared/metrics.ts and both util/metrics.ts files become re-exports, keeping every existing import path working. Also documents the scope these numbers actually have, because it is easy to over-trust them: nothing is persisted, the map is per-process and resets on restart, and since the server runs multiple replicas a server-side counter reflects ONE pod's traffic. Anything needing a durable or cluster-wide total has to be written to the database explicitly. The store had no tests at all despite being the drift victim; it now has them, including that the cap holds, that an already-known key keeps counting past the cap (established series must not freeze because something noisy filled the map), and that the daemon entry point is the SAME store rather than a second copy. Co-Authored-By: Claude Opus 5 --- server/src/util/metrics.ts | 58 +++++++++--------------------- shared/metrics.ts | 71 ++++++++++++++++++++++++++++++++++++ src/util/metrics.ts | 53 +++++++++------------------ test/shared/metrics.test.ts | 72 +++++++++++++++++++++++++++++++++++++ 4 files changed, 177 insertions(+), 77 deletions(-) create mode 100644 shared/metrics.ts create mode 100644 test/shared/metrics.test.ts diff --git a/server/src/util/metrics.ts b/server/src/util/metrics.ts index db953680b..d56ab3a27 100644 --- a/server/src/util/metrics.ts +++ b/server/src/util/metrics.ts @@ -1,41 +1,17 @@ -export type MetricLabels = Record; - -const counters = new Map(); -const MAX_COUNTERS = 1000; - -function labelsKey(labels?: MetricLabels): string { - if (!labels) return ''; - const entries = Object.entries(labels) - .filter(([, value]) => typeof value === 'string') - .sort(([a], [b]) => a.localeCompare(b)); - return entries.map(([key, value]) => `${key}=${value}`).join(','); -} - -function counterKey(name: string, labels?: MetricLabels): string { - const suffix = labelsKey(labels); - return suffix ? `${name}{${suffix}}` : name; -} - -export function incrementCounter(name: string, labels?: MetricLabels): void { - addCounter(name, 1, labels); -} - -export function addCounter(name: string, amount: number, labels?: MetricLabels): void { - if (!name) return; - if (!Number.isFinite(amount) || amount <= 0) return; - const key = counterKey(name, labels); - if (!counters.has(key) && counters.size >= MAX_COUNTERS) return; - counters.set(key, (counters.get(key) ?? 0) + amount); -} - -export function getCounter(name: string, labels?: MetricLabels): number { - return counters.get(counterKey(name, labels)) ?? 0; -} - -export function snapshotCounters(): Record { - return Object.fromEntries(counters.entries()); -} - -export function resetMetricsForTests(): void { - counters.clear(); -} +/** + * Server-facing entry point for in-process counters. + * + * The implementation lives in `shared/metrics.ts` so the daemon and the server + * cannot drift apart again. Note the scope: counters are per-process, and the + * server runs multiple replicas, so a number here reflects ONE pod's traffic and + * resets on every deploy. + */ +export { + addCounter, + counterKeyCount, + getCounter, + incrementCounter, + resetMetricsForTests, + snapshotCounters, +} from '../../../shared/metrics.js'; +export type { MetricLabels } from '../../../shared/metrics.js'; diff --git a/shared/metrics.ts b/shared/metrics.ts new file mode 100644 index 000000000..adfd7fd63 --- /dev/null +++ b/shared/metrics.ts @@ -0,0 +1,71 @@ +/** + * In-process counters. THE single implementation — daemon and server both + * re-export this module from their own `util/metrics.ts`. + * + * These are NOT persisted. The store is a plain in-memory Map, so every counter + * resets when the process restarts, and on the server each replica counts only + * the traffic that hit that pod. Anything that needs a durable or cluster-wide + * number has to be written to the database explicitly — do not assume these + * survive a deploy. + * + * Previously this existed as two near-identical copies (`src/util/metrics.ts` and + * `server/src/util/metrics.ts`) which had already drifted: only the server copy + * had `addCounter`. That is exactly the duplication the repo rules forbid, so the + * two files are now thin re-exports of this one. + */ + +export type MetricLabels = Record; + +const counters = new Map(); + +/** + * Cardinality guard, not a capacity target. + * + * Each distinct name+labels combination is one entry, so a caller that labels by + * something unbounded (a session id, a file path) would otherwise grow the Map + * forever. Past the cap, NEW keys are dropped while existing ones keep counting — + * so a hit means later metrics silently go missing, which is worth noticing. + */ +const MAX_COUNTERS = 10_000; + +function labelsKey(labels?: MetricLabels): string { + if (!labels) return ''; + const entries = Object.entries(labels) + .filter(([, value]) => typeof value === 'string') + .sort(([a], [b]) => a.localeCompare(b)); + return entries.map(([key, value]) => `${key}=${value}`).join(','); +} + +function counterKey(name: string, labels?: MetricLabels): string { + const suffix = labelsKey(labels); + return suffix ? `${name}{${suffix}}` : name; +} + +export function incrementCounter(name: string, labels?: MetricLabels): void { + addCounter(name, 1, labels); +} + +export function addCounter(name: string, amount: number, labels?: MetricLabels): void { + if (!name) return; + if (!Number.isFinite(amount) || amount <= 0) return; + const key = counterKey(name, labels); + if (!counters.has(key) && counters.size >= MAX_COUNTERS) return; + counters.set(key, (counters.get(key) ?? 0) + amount); +} + +export function getCounter(name: string, labels?: MetricLabels): number { + return counters.get(counterKey(name, labels)) ?? 0; +} + +export function snapshotCounters(): Record { + return Object.fromEntries(counters.entries()); +} + +/** Number of distinct counter keys currently held (for cap/leak assertions). */ +export function counterKeyCount(): number { + return counters.size; +} + +export function resetMetricsForTests(): void { + counters.clear(); +} diff --git a/src/util/metrics.ts b/src/util/metrics.ts index 548564706..d641e9371 100644 --- a/src/util/metrics.ts +++ b/src/util/metrics.ts @@ -1,36 +1,17 @@ -export type MetricLabels = Record; - -const counters = new Map(); -const MAX_COUNTERS = 1000; - -function labelsKey(labels?: MetricLabels): string { - if (!labels) return ''; - const entries = Object.entries(labels) - .filter(([, value]) => typeof value === 'string') - .sort(([a], [b]) => a.localeCompare(b)); - return entries.map(([key, value]) => `${key}=${value}`).join(','); -} - -function counterKey(name: string, labels?: MetricLabels): string { - const suffix = labelsKey(labels); - return suffix ? `${name}{${suffix}}` : name; -} - -export function incrementCounter(name: string, labels?: MetricLabels): void { - if (!name) return; - const key = counterKey(name, labels); - if (!counters.has(key) && counters.size >= MAX_COUNTERS) return; - counters.set(key, (counters.get(key) ?? 0) + 1); -} - -export function getCounter(name: string, labels?: MetricLabels): number { - return counters.get(counterKey(name, labels)) ?? 0; -} - -export function snapshotCounters(): Record { - return Object.fromEntries(counters.entries()); -} - -export function resetMetricsForTests(): void { - counters.clear(); -} +/** + * Daemon-facing entry point for in-process counters. + * + * The implementation lives in `shared/metrics.ts` so the daemon and the server + * cannot drift apart again (they already had, once: only the server copy grew + * `addCounter`). This file stays so the many existing `../util/metrics.js` + * imports keep working. + */ +export { + addCounter, + counterKeyCount, + getCounter, + incrementCounter, + resetMetricsForTests, + snapshotCounters, +} from '../../shared/metrics.js'; +export type { MetricLabels } from '../../shared/metrics.js'; diff --git a/test/shared/metrics.test.ts b/test/shared/metrics.test.ts new file mode 100644 index 000000000..06d1f714e --- /dev/null +++ b/test/shared/metrics.test.ts @@ -0,0 +1,72 @@ +/** + * The counter store had no test at all, and it had already silently drifted into + * two copies with different APIs. These tests pin the two properties that matter: + * the cardinality cap actually holds, and the daemon/server entry points are the + * SAME store rather than two that merely look alike. + */ +import { beforeEach, describe, expect, it } from 'vitest'; + +import { + addCounter, + counterKeyCount, + getCounter, + incrementCounter, + resetMetricsForTests, + snapshotCounters, +} from '../../shared/metrics.js'; +import * as daemonMetrics from '../../src/util/metrics.js'; + +describe('shared metrics store', () => { + beforeEach(() => { + resetMetricsForTests(); + }); + + it('counts per name+labels combination, order-independently', () => { + incrementCounter('a.b', { x: '1', y: '2' }); + incrementCounter('a.b', { y: '2', x: '1' }); + expect(getCounter('a.b', { x: '1', y: '2' })).toBe(2); + // A different label value is a different series, not the same counter. + expect(getCounter('a.b', { x: '9', y: '2' })).toBe(0); + expect(getCounter('a.b')).toBe(0); + }); + + it('ignores non-positive and non-finite amounts instead of corrupting a total', () => { + addCounter('c', 5); + addCounter('c', 0); + addCounter('c', -3); + addCounter('c', Number.NaN); + addCounter('c', Number.POSITIVE_INFINITY); + expect(getCounter('c')).toBe(5); + }); + + it('caps distinct keys at 10k, but keeps counting keys it already knows', () => { + for (let i = 0; i < 10_000; i++) incrementCounter('bulk', { i: String(i) }); + expect(counterKeyCount()).toBe(10_000); + + // Past the cap a NEW key is dropped... + incrementCounter('bulk', { i: 'overflow' }); + expect(getCounter('bulk', { i: 'overflow' })).toBe(0); + expect(counterKeyCount()).toBe(10_000); + + // ...while an EXISTING key still accumulates, so established series do not + // freeze just because something noisy filled the map. + incrementCounter('bulk', { i: '0' }); + expect(getCounter('bulk', { i: '0' })).toBe(2); + }); + + it('exposes the daemon entry point as the same store, not a second copy', () => { + // The drift that motivated consolidation: two modules with one API each. + daemonMetrics.incrementCounter('shared.store.check'); + expect(getCounter('shared.store.check')).toBe(1); + expect(daemonMetrics.getCounter('shared.store.check')).toBe(1); + expect(typeof daemonMetrics.addCounter).toBe('function'); + }); + + it('snapshots as a plain object and is fully cleared by the test reset', () => { + incrementCounter('snap', { k: 'v' }); + expect(snapshotCounters()).toEqual({ 'snap{k=v}': 1 }); + resetMetricsForTests(); + expect(snapshotCounters()).toEqual({}); + expect(counterKeyCount()).toBe(0); + }); +}); From 32a26a322e759f81670880f9aba2ac93decd507c Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sat, 25 Jul 2026 22:53:55 +0800 Subject: [PATCH 11/40] Stop a refresh from leaving the chat above the bottom MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reported as: in Safari, scroll to the very bottom, refresh, and the log still sits a little above the bottom every time. The chat list sets `overflow-anchor: none` (styles.css), so the engine never compensates when something shrinks the pane — the ResizeObserver re-pin is the only thing that can. That observer consults `bannerToggleAtRef` to skip re-pinning when the shift was the pinned "Last sent" banner mounting its own ~60px (re-pinning there would snap the user back down, re-hide the banner, and start a height oscillation). But the stamp was written by an effect keyed on `[pinnedAboveViewport]`, and such an effect also runs ON MOUNT — so every mount armed a 300ms blackout over the single most layout-unstable moment there is: the first frames after a page load, when `--vvh` is applied from an effect, the composer rehydrates its saved draft, and the sub-session bar runs its 200ms max-height transition. The blackout is not merely a delay. The observer records the new height BEFORE consulting the suppression, so a change landing inside that window is consumed and unrecoverable — no later resize sees a difference. That is why the offset was permanent and reproduced on every refresh rather than self-correcting. Mount is not a toggle, so compare against the previous value instead of trusting the dep change. This also stays correct under StrictMode's double-invoked effects, where a "skip the first run" flag would arm on the second pass and stamp anyway. The counterfactual is pinned: with the old effect restored the new test fails with `expected 1040 to be 1200` — the view stranded 160px up — while the paired test asserting a genuine banner toggle must NOT re-pin keeps passing, so the guard's original anti-jitter purpose is still enforced rather than traded away. KNOWN REMAINING GAP (not addressed here): the observer reacts to `clientHeight` only, so post-paint CONTENT growth that leaves the pane height alone still cannot re-pin — async image previews (78px skeleton → up to 260px), markdown with no intrinsic dimensions, and ToolBlockFold's post-paint clamp. An image-heavy chat can therefore still drift after a refresh. Co-Authored-By: Claude Opus 5 --- web/src/components/ChatView.tsx | 14 ++ .../ChatView-refresh-repin.test.tsx | 189 ++++++++++++++++++ 2 files changed, 203 insertions(+) create mode 100644 web/test/components/ChatView-refresh-repin.test.tsx diff --git a/web/src/components/ChatView.tsx b/web/src/components/ChatView.tsx index 4b570ca80..6cd376d80 100644 --- a/web/src/components/ChatView.tsx +++ b/web/src/components/ChatView.tsx @@ -2012,7 +2012,21 @@ export function ChatView({ events, loading, refreshing = false, historyStatus, l // Stamp when the pin banner toggles so the ResizeObserver can tell its own // ~60px height shift apart from a genuine viewport resize (sub-session bar, // keyboard) and skip re-pinning to bottom for the former. See bannerToggleAtRef. + // + // Compare against the previous value instead of just running on the dep change: + // an effect keyed on `[pinnedAboveViewport]` ALSO runs on mount, which stamped a + // 300ms suppression window over the most layout-unstable moment there is — the + // first frames after a page refresh, when `--vvh` is applied from an effect, the + // composer rehydrates its saved draft, and the sub-session bar runs its 200ms + // max-height transition. Every one of those shrinks `.chat-view` while + // `scrollTop` stays put (the list sets `overflow-anchor: none`, so the engine + // never compensates), and the suppressed ResizeObserver consumed the change + // without re-pinning — leaving the view permanently a little above the bottom + // after every reload. Mount is not a toggle, so it must not stamp. + const prevPinnedAboveViewportRef = useRef(pinnedAboveViewport); useEffect(() => { + if (prevPinnedAboveViewportRef.current === pinnedAboveViewport) return; + prevPinnedAboveViewportRef.current = pinnedAboveViewport; bannerToggleAtRef.current = Date.now(); }, [pinnedAboveViewport]); diff --git a/web/test/components/ChatView-refresh-repin.test.tsx b/web/test/components/ChatView-refresh-repin.test.tsx new file mode 100644 index 000000000..c6b77f846 --- /dev/null +++ b/web/test/components/ChatView-refresh-repin.test.tsx @@ -0,0 +1,189 @@ +/** + * @vitest-environment jsdom + * + * "After refreshing in Safari the chat still sits a little above the bottom, + * even when I was scrolled all the way down." + * + * The chat list sets `overflow-anchor: none`, so the browser never compensates + * when something shrinks the viewport — only the ResizeObserver re-pin does. But + * the banner-toggle suppression it consults was stamped by an effect keyed on + * `[pinnedAboveViewport]`, and such an effect ALSO runs on mount, so the first + * 300ms after a refresh had re-pinning disabled. That is exactly the window in + * which `--vvh` is applied from an effect, the composer rehydrates its saved + * draft, and the sub-session bar runs its 200ms max-height transition. Worse, the + * observer records the new height BEFORE checking the suppression, so the change + * was consumed for good: no later resize could recover it. + * + * These two tests pin both halves of the contract — a genuine post-mount resize + * must re-pin, and a real banner toggle must still NOT. + */ +import { h } from 'preact'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { render, cleanup, act, waitFor } from '@testing-library/preact'; + +vi.mock('react-i18next', () => ({ + useTranslation: () => ({ t: (key: string, fallback?: string) => fallback ?? key }), +})); +vi.mock('../../src/components/ChatMarkdown.js', () => ({ + ChatMarkdown: ({ text }: { text: string }) =>
{text}
, +})); +vi.mock('../../src/components/FileBrowser.js', () => ({ FileBrowser: () => null })); +vi.mock('../../src/components/FloatingPanel.js', () => ({ + FloatingPanel: ({ children }: { children?: preact.ComponentChildren }) =>
{children}
, +})); +vi.mock('../../src/hooks/usePref.js', () => ({ + parseBooleanish: (raw: unknown) => (raw === true || raw === 'true' ? true : raw === false || raw === 'false' ? false : null), + usePref: () => ({ + value: true, rawValue: true, loaded: true, loading: false, stale: false, + error: null, save: async () => undefined, set: () => undefined, reload: async () => true, + }), +})); + +import { ChatView } from '../../src/components/ChatView.js'; +import type { TimelineEvent } from '../../src/ws-client.js'; + +/** Controllable ResizeObserver — jsdom has none, and we need to fire it on demand. */ +const resizeCallbacks: ResizeObserverCallback[] = []; +class FakeResizeObserver { + constructor(cb: ResizeObserverCallback) { resizeCallbacks.push(cb); } + observe(): void {} + unobserve(): void {} + disconnect(): void {} +} +function fireResize(): void { + for (const cb of resizeCallbacks) cb([], {} as ResizeObserver); +} + +/** Minimal IntersectionObserver fake so we can drive the pin banner on/off. */ +type IOCallback = (entries: IntersectionObserverEntry[]) => void; +const ioInstances: Array<{ fire: (e: Array>) => void }> = []; +class FakeIntersectionObserver { + private callback: IOCallback; + private target: Element | null = null; + constructor(callback: IOCallback) { + this.callback = callback; + const self = this; + ioInstances.push({ + fire: (partial) => { + self.callback(partial.map((e) => ({ + target: self.target, + isIntersecting: false, + intersectionRatio: 0, + intersectionRect: {} as DOMRectReadOnly, + boundingClientRect: { bottom: 0, top: 0 } as DOMRectReadOnly, + rootBounds: { top: 0, bottom: 500 } as DOMRectReadOnly, + time: 0, + ...e, + })) as IntersectionObserverEntry[]); + }, + }); + } + observe(target: Element): void { this.target = target; } + unobserve(): void { this.target = null; } + disconnect(): void { this.target = null; } + takeRecords(): IntersectionObserverEntry[] { return []; } +} + +function userEvent(eventId: string, text: string, ts: number): TimelineEvent { + return { + eventId, type: 'user.message', ts, epoch: 1, seq: ts, + sessionId: 'deck_repin_brain', source: 'daemon', confidence: 'high', + payload: { text }, + } as unknown as TimelineEvent; +} +function assistantEvent(eventId: string, text: string, ts: number): TimelineEvent { + return { + eventId, type: 'assistant.text', ts, epoch: 1, seq: ts, + sessionId: 'deck_repin_brain', source: 'daemon', confidence: 'high', + payload: { text, streaming: false }, + } as unknown as TimelineEvent; +} + +const EVENTS = [ + userEvent('u1', 'why is the chat not at the bottom after refresh', 1000), + assistantEvent('a1', 'investigating the pin path', 2000), +]; + +/** + * Renders, waits for the initial bottom pin, then hands back a harness whose + * `clientHeight` can shrink like a real post-refresh layout settle. + */ +async function renderPinnedChat() { + const { container } = render( + , + ); + const scrollEl = container.querySelector('.chat-view') as HTMLDivElement; + let scrollTopValue = 0; + let clientHeightValue = 200; + Object.defineProperty(scrollEl, 'scrollTop', { + configurable: true, + get: () => scrollTopValue, + set: (v: number) => { scrollTopValue = v; }, + }); + Object.defineProperty(scrollEl, 'scrollHeight', { configurable: true, get: () => 1200 }); + Object.defineProperty(scrollEl, 'clientHeight', { configurable: true, get: () => clientHeightValue }); + + // Initial pin to the bottom (this is the state the user is in before refresh). + await waitFor(() => expect(scrollTopValue).toBe(1200)); + + return { + container, + get scrollTop() { return scrollTopValue; }, + /** Simulate the browser leaving the view above the bottom after a shrink. */ + setScrollTop: (v: number) => { scrollTopValue = v; }, + shrinkViewport: (to: number) => { clientHeightValue = to; }, + }; +} + +describe('ChatView — post-refresh re-pin to bottom', () => { + beforeEach(() => { + resizeCallbacks.length = 0; + ioInstances.length = 0; + vi.stubGlobal('ResizeObserver', FakeResizeObserver as unknown as typeof ResizeObserver); + vi.stubGlobal('IntersectionObserver', FakeIntersectionObserver as unknown as typeof IntersectionObserver); + }); + + afterEach(() => { + cleanup(); + vi.unstubAllGlobals(); + }); + + it('re-pins to the bottom when the viewport shrinks right after mount', async () => { + const chat = await renderPinnedChat(); + + // The post-refresh settle: --vvh lands / composer draft rehydrates / the + // sub-session bar finishes its transition, so the pane loses height and the + // browser leaves the reading position above the bottom. + chat.shrinkViewport(140); + chat.setScrollTop(1040); + act(() => { fireResize(); }); + + // Must snap back to the bottom. Before the fix the mount-stamped suppression + // swallowed this resize entirely and the view stayed at 1040 forever. + await waitFor(() => expect(chat.scrollTop).toBe(1200)); + }); + + it('still does NOT re-pin for the pin banner toggling its own height', async () => { + const chat = await renderPinnedChat(); + + // A real toggle: the last user bubble goes above the viewport, so the banner + // mounts and steals ~60px. Re-pinning here would snap the user back down, + // which re-hides the banner and starts the height-oscillation jitter loop. + await waitFor(() => expect(ioInstances.length).toBeGreaterThan(0)); + act(() => { + ioInstances[ioInstances.length - 1].fire([{ + isIntersecting: false, + boundingClientRect: { bottom: -10, top: -30 } as DOMRectReadOnly, + rootBounds: { top: 0, bottom: 500 } as DOMRectReadOnly, + }]); + }); + expect(chat.container.querySelector('.chat-pinned-last-sent')).not.toBeNull(); + + chat.shrinkViewport(140); + chat.setScrollTop(600); // user is reading up here; must not be yanked + act(() => { fireResize(); }); + + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(chat.scrollTop).toBe(600); + }); +}); From 3a8db27e58d5b7e8995e5f373395f291edcb0b2b Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sat, 25 Jul 2026 23:11:33 +0800 Subject: [PATCH 12/40] Hold the chat at the bottom through the post-refresh layout settle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to 32a26a322, which fixed the pane-HEIGHT half of "refresh and the log sits a little above the bottom". The reporter confirmed the affected chat has no images, which ruled out the async image-preview growth I had flagged as the remaining gap and pointed at a different source I had lumped in with it: blocks that mount ABOVE every message. The "Load older" row appears once `hasOlderHistory` resolves from the first history response; the loading-older status row, the agent todo list (first child of the scroller) and the tool-chooser banner behind an async preference fetch all do the same. Growth above the viewport with `scrollTop` frozen slides the entire log down, so the user ends up looking at older content — the reported 上弹. None of it changes `renderedRevision`, so the content-update layout effect never re-pins; the ResizeObserver only watches `clientHeight`, so it is blind to it; and `overflow-anchor: none` means the engine will not compensate. Nothing corrected it. So for a bounded window after mount/session-change, re-assert the bottom when the total height moves. The window is deliberately narrow, because the danger here is becoming the "auto-update fights my scrolling" bug that 4ed20fb7f and 605269f65 were written to kill: it stands down the moment follow is disengaged by a gentle scroll-up, it never engages follow itself, it stays clear of the load-older anchor, and a quiet mount costs nothing because it only acts on a real change. It also stands down whenever `events` changed. My first attempt did not, and it broke an existing test ("does not force bottom scroll for non-rendered status updates") — correctly, because data-driven updates belong to the content-update layout effect, which deliberately declines to follow some of them. This window only handles height moving under UNCHANGED data. Keeping that boundary is what makes the new behaviour additive instead of a policy change. Each fix is pinned by its own counterfactual: strip the settle window and only the content-growth test fails (`expected 1200 to be 1320`); revert 32a26a322's mount-stamp fix and only the viewport-shrink test fails (`expected 1040 to be 1200`) — so neither test is passing for the other's reason. Two of my own tests were flaky rather than the product being wrong, and both are now deterministic instead of merely re-run until green: the banner-suppression test raced a 300ms wall-clock window (frozen clock), and the harness did not drain the `requestAnimationFrame` the mount effect queues, so under full-suite parallel load that late callback re-pinned mid-scenario. Verified with two consecutive full-suite runs, 194/194 files. Co-Authored-By: Claude Opus 5 --- web/src/components/ChatView.tsx | 70 ++++++++++++++- web/src/components/chat-follow-thresholds.ts | 20 +++++ .../ChatView-refresh-repin.test.tsx | 89 +++++++++++++++---- 3 files changed, 163 insertions(+), 16 deletions(-) diff --git a/web/src/components/ChatView.tsx b/web/src/components/ChatView.tsx index 6cd376d80..37bb966d9 100644 --- a/web/src/components/ChatView.tsx +++ b/web/src/components/ChatView.tsx @@ -30,7 +30,11 @@ import { isHtmlPreviewPath, type HtmlPreviewViewMode } from '@shared/html-previe import { FileBrowser, type FileBrowserPreviewRequest } from './file-browser-lazy.js'; import { ChatMarkdown } from './ChatMarkdown.js'; import { AgentTodoList } from './AgentTodoList.js'; -import { computeFollowThresholds } from './chat-follow-thresholds.js'; +import { + CHAT_MOUNT_SETTLE_MS, + CHAT_MOUNT_SETTLE_TICK_MS, + computeFollowThresholds, +} from './chat-follow-thresholds.js'; import type { ChatLocalImagePreviewLoader, ChatLocalImagePreviewResult } from './ChatLocalImagePreview.js'; import { HtmlFullscreenPreview, openHtmlPreviewInNewWindow, type HtmlFullscreenPreviewState } from './HtmlFullscreenPreview.js'; import { isLikelyDomainPath, renderChatPathActions, type ChatPathDownloadHandler } from '../chat-path-actions.js'; @@ -2143,6 +2147,22 @@ export function ChatView({ events, loading, refreshing = false, historyStatus, l const lastLoadOlderAtRef = useRef(0); const LOAD_OLDER_COOLDOWN_MS = 1000; + // Mirror `loadingOlder` into a ref so the mount-settle interval can read the + // CURRENT value. Adding the prop to that effect's deps instead would restart + // the settle window every time a load-older starts or finishes, which is both + // wasteful and wrong (the window is meant to track the mount, not history + // fetches), and reading the captured prop would leave it permanently stale. + const loadingOlderRef = useRef(loadingOlder); + loadingOlderRef.current = loadingOlder; + + // Latest `events` identity, for the mount-settle interval to tell a LAYOUT + // settle apart from a DATA change. Anything driven by new events is owned by + // the content-update layout effect and its follow rules (which deliberately + // decline to scroll for some updates); the settle window must never + // second-guess that, only handle height moving under unchanged data. + const latestEventsRef = useRef(events); + latestEventsRef.current = events; + // Pause "stick to bottom" follow mode. Shared by handleScroll's distance // threshold and the explicit wheel/touch up-gesture handlers below. const disengageFollow = () => { @@ -2281,6 +2301,54 @@ export function ChatView({ events, loading, refreshing = false, historyStatus, l return () => ro.disconnect(); }, [preview]); + // Hold the bottom through the post-mount layout settle. + // + // The ResizeObserver above only reacts to `clientHeight`, so anything that + // changes CONTENT height after the initial pin is invisible to it — and the + // blocks that mount ABOVE every message are the worst case, because growth + // above the viewport with `scrollTop` frozen slides the whole log down and the + // user ends up looking at older content: the reported "refresh 后上弹". + // Culprits that need no images at all: the "Load older" row appearing when + // `hasOlderHistory` flips after the first history response, the + // `.chat-load-older-status` row, `AgentTodoList` (first child of the scroller), + // and the tool-chooser banner gated on an async `usePref` fetch. None of them + // change `renderedRevision`, so the content-update layout effect never re-pins + // either, and `overflow-anchor: none` means the engine will not help. + // + // So for a bounded window after mount/session-change, re-assert the bottom + // whenever total height changes. Deliberately narrow: it only acts while follow + // is still engaged (a gentle scroll-up disengages it and this goes inert + // immediately — the exact complaint behind the earlier Safari jitter fixes), it + // never engages follow itself, it stays out of the way of the load-older anchor, + // and it only fires on an actual height change so a quiet mount costs nothing. + useEffect(() => { + const el = scrollRef.current; + if (!el) return; + let lastScrollHeight = el.scrollHeight; + let seenEvents = latestEventsRef.current; + const deadline = Date.now() + CHAT_MOUNT_SETTLE_MS; + const timer = setInterval(() => { + if (Date.now() > deadline) { clearInterval(timer); return; } + const following = preview || autoScrollRef.current; + if (!following) { clearInterval(timer); return; } + if (loadingOlderRef.current || scrollAnchorRef.current) return; + const nextScrollHeight = el.scrollHeight; + // A DATA change is not ours to act on: the content-update layout effect + // owns those and deliberately declines to follow some of them (a + // non-rendered status update must not yank the viewport). Re-baseline and + // stand down, so this window only ever reacts to height moving on its own. + if (latestEventsRef.current !== seenEvents) { + seenEvents = latestEventsRef.current; + lastScrollHeight = nextScrollHeight; + return; + } + if (nextScrollHeight === lastScrollHeight) return; + lastScrollHeight = nextScrollHeight; + scrollToBottom(false); + }, CHAT_MOUNT_SETTLE_TICK_MS); + return () => clearInterval(timer); + }, [sessionId, preview]); + // Touch gesture mode is based on pointer coarseness only. Narrow desktop // windows still need native selection and the Copy/Quote popup. const isTouchDevice = useTouchChatGestures(); diff --git a/web/src/components/chat-follow-thresholds.ts b/web/src/components/chat-follow-thresholds.ts index 5fbed2dc0..4c83dd9d9 100644 --- a/web/src/components/chat-follow-thresholds.ts +++ b/web/src/components/chat-follow-thresholds.ts @@ -22,6 +22,26 @@ * threshold would need more than half the achievable range, both thresholds * are rescaled to that range (0.5× / 0.2×), keeping disengage > reengage. */ +/** + * How long after mount / session-change the chat keeps re-asserting the bottom + * while follow is engaged, and how often it checks. + * + * A page refresh finishes painting long before the layout settles: blocks mount + * ABOVE the messages (the "Load older" row once `hasOlderHistory` resolves, the + * loading-older status row, the agent todo list, the tool-chooser banner behind an + * async preference fetch), and growth above the viewport with `scrollTop` frozen + * slides the whole log down — the user is left looking at older content. The + * ResizeObserver cannot see it (it only watches `clientHeight`) and the chat sets + * `overflow-anchor: none`, so nothing else corrects it. + * + * The window has to outlast the slowest of those settles — the `.subsession-bar` + * max-height transition alone is 200ms and the preference fetch is a network + * round-trip — while staying short enough that it is unambiguously "the refresh", + * not an ongoing behaviour that could fight the user later. + */ +export const CHAT_MOUNT_SETTLE_MS = 1_500; +export const CHAT_MOUNT_SETTLE_TICK_MS = 80; + export interface FollowThresholds { disengageThreshold: number; reengageThreshold: number; diff --git a/web/test/components/ChatView-refresh-repin.test.tsx b/web/test/components/ChatView-refresh-repin.test.tsx index c6b77f846..d5b41c29a 100644 --- a/web/test/components/ChatView-refresh-repin.test.tsx +++ b/web/test/components/ChatView-refresh-repin.test.tsx @@ -19,7 +19,7 @@ */ import { h } from 'preact'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { render, cleanup, act, waitFor } from '@testing-library/preact'; +import { render, cleanup, act, fireEvent, waitFor } from '@testing-library/preact'; vi.mock('react-i18next', () => ({ useTranslation: () => ({ t: (key: string, fallback?: string) => fallback ?? key }), @@ -115,23 +115,45 @@ async function renderPinnedChat() { const scrollEl = container.querySelector('.chat-view') as HTMLDivElement; let scrollTopValue = 0; let clientHeightValue = 200; + let scrollHeightValue = 1200; Object.defineProperty(scrollEl, 'scrollTop', { configurable: true, get: () => scrollTopValue, set: (v: number) => { scrollTopValue = v; }, }); - Object.defineProperty(scrollEl, 'scrollHeight', { configurable: true, get: () => 1200 }); + Object.defineProperty(scrollEl, 'scrollHeight', { configurable: true, get: () => scrollHeightValue }); Object.defineProperty(scrollEl, 'clientHeight', { configurable: true, get: () => clientHeightValue }); // Initial pin to the bottom (this is the state the user is in before refresh). await waitFor(() => expect(scrollTopValue).toBe(1200)); + // Drain the frames mount already scheduled before handing control back. The + // session-change effect queues `requestAnimationFrame(() => scrollToBottom(true))`, + // and under full-suite parallel load that callback can land AFTER a test has + // set up its scenario — silently re-pinning to the bottom and making an + // unrelated assertion fail. Wait until the position stops changing on its own. + await act(async () => { + for (let i = 0; i < 3; i++) { + await new Promise((resolve) => { requestAnimationFrame(() => resolve()); }); + } + await new Promise((resolve) => { setTimeout(resolve, 30); }); + }); + expect(scrollTopValue).toBe(1200); + return { container, + scrollEl, get scrollTop() { return scrollTopValue; }, /** Simulate the browser leaving the view above the bottom after a shrink. */ setScrollTop: (v: number) => { scrollTopValue = v; }, shrinkViewport: (to: number) => { clientHeightValue = to; }, + /** A block mounting ABOVE the messages makes the whole list taller. */ + growContent: (to: number) => { scrollHeightValue = to; }, + /** Drive a real user scroll-up so follow mode disengages. */ + scrollUpTo: (v: number) => { + scrollTopValue = v; + fireEvent.scroll(scrollEl); + }, }; } @@ -170,20 +192,57 @@ describe('ChatView — post-refresh re-pin to bottom', () => { // mounts and steals ~60px. Re-pinning here would snap the user back down, // which re-hides the banner and starts the height-oscillation jitter loop. await waitFor(() => expect(ioInstances.length).toBeGreaterThan(0)); - act(() => { - ioInstances[ioInstances.length - 1].fire([{ - isIntersecting: false, - boundingClientRect: { bottom: -10, top: -30 } as DOMRectReadOnly, - rootBounds: { top: 0, bottom: 500 } as DOMRectReadOnly, - }]); - }); - expect(chat.container.querySelector('.chat-pinned-last-sent')).not.toBeNull(); - chat.shrinkViewport(140); - chat.setScrollTop(600); // user is reading up here; must not be yanked - act(() => { fireResize(); }); + // The suppression is a 300ms WALL-CLOCK window, so freeze the clock before + // the toggle stamps it. Otherwise this test just races the machine: under + // full-suite parallel load the gap between the stamp and the resize exceeded + // 300ms, the suppression expired, and the failure looked like a product bug + // when it was only the test being slow. + const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(1_700_000_000_000); + try { + act(() => { + ioInstances[ioInstances.length - 1].fire([{ + isIntersecting: false, + boundingClientRect: { bottom: -10, top: -30 } as DOMRectReadOnly, + rootBounds: { top: 0, bottom: 500 } as DOMRectReadOnly, + }]); + }); + expect(chat.container.querySelector('.chat-pinned-last-sent')).not.toBeNull(); + + chat.shrinkViewport(140); + chat.setScrollTop(600); // user is reading up here; must not be yanked + act(() => { fireResize(); }); + + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(chat.scrollTop).toBe(600); + } finally { + nowSpy.mockRestore(); + } + }); + + it('re-pins when a block mounts ABOVE the messages and grows the list (no images involved)', async () => { + const chat = await renderPinnedChat(); + + // The reported case with no images in the chat at all: the "Load older" row + // appears once `hasOlderHistory` resolves from the first history response, the + // agent todo list mounts, the tool-chooser banner lands behind its async pref + // fetch. All sit above every message, so the list gets taller ABOVE the + // viewport while `clientHeight` never changes — the ResizeObserver is blind to + // it and `overflow-anchor: none` means the browser will not compensate. + chat.growContent(1320); + + await waitFor(() => expect(chat.scrollTop).toBe(1320), { timeout: 1000 }); + }); + + it('leaves the view alone if the user scrolled up before the content grew', async () => { + const chat = await renderPinnedChat(); + + // A gentle scroll-up must win — this is precisely the "auto-update fights my + // scrolling" complaint behind the earlier Safari jitter fixes. + chat.scrollUpTo(400); + chat.growContent(1320); - await new Promise((resolve) => setTimeout(resolve, 20)); - expect(chat.scrollTop).toBe(600); + await new Promise((resolve) => setTimeout(resolve, 300)); + expect(chat.scrollTop).toBe(400); }); }); From 490cd39477ce886eab35701ba27a68622761bd5e Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sat, 25 Jul 2026 23:24:59 +0800 Subject: [PATCH 13/40] Give injected memory a redeemable handle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Injected memory carried only a summary, while the guidance shipped beside it told the agent to "call get_memory_sources with that ref" — there was no ref to call with, so a summary that looked relevant was a dead end. Emit a compact handle on every injected line across all nine recall/startup injection sites, registering it so it actually resolves. Also rewrite the handle derivation. It kept the first 10 hex-looking characters of the id, scavenged from anywhere in the string, so structured ids sharing a constant prefix (md-ingest:::) collapsed onto ONE handle — their discriminating hash sits past the 10th kept character. Resolution then returned whichever colliding record was newest, i.e. the wrong memory, silently. Derive from md5 instead, base32-encoded and truncated to 13 chars (~65 bits): uniform for any id shape, and a pure function of the id, so the ref->id table stays a rebuildable index rather than the only source of truth. Summaries were condensed with split('\n')[0], which rendered whole blocks as a bare "- [recent] ## Problem". Keep the heading as a label and pull the first real content line up beside it. Co-Authored-By: Claude Opus 4.8 --- shared/memory-recall-format.ts | 43 ++++++++-- src/agent/runtime-context-bootstrap.ts | 3 +- src/agent/transport-runtime-assembly.ts | 3 +- src/agent/transport-session-runtime.ts | 5 +- src/context/memory-recall-refs.ts | 81 +++++++++++++++++++ src/context/memory-short-ref.ts | 77 ++++++++++++++++-- src/daemon/command-handler.ts | 5 +- src/daemon/memory-context-timeline.ts | 3 +- src/daemon/memory-inject.ts | 5 +- src/util/base32.ts | 28 +++++++ test/agent/runtime-context-bootstrap.test.ts | 13 +-- test/context/memory-short-ref.test.ts | 65 +++++++++------ test/daemon/codex-watcher-bootstrap.test.ts | 3 +- .../command-handler-memory-context.test.ts | 5 +- test/daemon/memory-inject-startup.test.ts | 3 +- .../memory-mcp-tools-schema-firewall.test.ts | 10 +-- test/daemon/memory-recall-integration.test.ts | 7 +- 17 files changed, 299 insertions(+), 60 deletions(-) create mode 100644 src/context/memory-recall-refs.ts create mode 100644 src/util/base32.ts diff --git a/shared/memory-recall-format.ts b/shared/memory-recall-format.ts index f8df802a3..9f5000810 100644 --- a/shared/memory-recall-format.ts +++ b/shared/memory-recall-format.ts @@ -3,6 +3,10 @@ import type { ProcessedContextClass, ProcessedContextProjectionStatus } from './ export interface RelatedPastWorkRenderableItem { projectId: string; summary: string; + /** Compact handle (e.g. `proj:k3m7q2xw9pz4a`) the agent can redeem via + * get_memory_sources when a summary alone isn't enough. Omitted for records + * that aren't addressable (raw events). */ + ref?: string; projectionClass?: ProcessedContextClass; hitCount?: number; lastUsedAt?: number; @@ -14,22 +18,49 @@ export const RELATED_PAST_WORK_HEADER = '[Related past work]'; export const STARTUP_PROJECT_MEMORY_HEADER = '# Recent project memory (reference only)'; export const STARTUP_SKILL_INDEX_HEADER = '# Available skills (read on demand)'; +const MARKDOWN_HEADING = /^\s{0,3}#{1,6}\s+/; + +/** + * Condense a stored summary into one injected line. + * + * Taking `split('\n')[0]` verbatim produced lines that were nothing but a + * markdown heading — a whole recall block reading `- [recent] ## Problem` tells + * the agent nothing about whether the memory is relevant. Keep the heading as a + * label and pull the first real content line up next to it, so the line reads + * `Problem: uploads fail when the disk is full`. + */ export function formatRelatedPastWorkSummary(summary: string, maxLength = 200): string { - return summary.split('\n')[0]?.slice(0, maxLength) ?? ''; + const lines = summary.split('\n').map((line) => line.trim()).filter(Boolean); + const firstLine = lines[0]; + if (!firstLine) return ''; + let label = ''; + let rest = lines; + if (MARKDOWN_HEADING.test(firstLine)) { + label = firstLine.replace(MARKDOWN_HEADING, '').replace(/\s*:\s*$/, ''); + rest = lines.slice(1); + } + const body = rest.find((line) => !MARKDOWN_HEADING.test(line)) ?? ''; + const text = label && body ? `${label}: ${body}` : (body || label || firstLine); + return text.slice(0, maxLength); +} + +/** `(proj:abc…) ` prefix, or '' when the record has no redeemable handle. */ +function formatRefPrefix(ref: string | undefined): string { + return ref ? `(${ref}) ` : ''; } -export function formatRelatedPastWorkLine(item: Pick): string { - return `- [${item.projectId}] ${formatRelatedPastWorkSummary(item.summary)}`; +export function formatRelatedPastWorkLine(item: Pick): string { + return `- [${item.projectId}] ${formatRefPrefix(item.ref)}${formatRelatedPastWorkSummary(item.summary)}`; } -export function buildRelatedPastWorkText(items: ReadonlyArray>): string { +export function buildRelatedPastWorkText(items: ReadonlyArray>): string { return `${RELATED_PAST_WORK_HEADER}\n\n${items.map((item) => formatRelatedPastWorkLine(item)).join('\n')}\n`; } -export function buildStartupProjectMemoryText(items: ReadonlyArray>): string { +export function buildStartupProjectMemoryText(items: ReadonlyArray>): string { const lines = items.map((item) => { const label = item.projectionClass === 'durable_memory_candidate' ? 'important' : 'recent'; - return `- [${label}] ${formatRelatedPastWorkSummary(item.summary, 300)}`; + return `- [${label}] ${formatRefPrefix(item.ref)}${formatRelatedPastWorkSummary(item.summary, 300)}`; }); return `${STARTUP_PROJECT_MEMORY_HEADER}\n\n${lines.join('\n')}\n`; } diff --git a/src/agent/runtime-context-bootstrap.ts b/src/agent/runtime-context-bootstrap.ts index d5d60775c..f979f5f96 100644 --- a/src/agent/runtime-context-bootstrap.ts +++ b/src/agent/runtime-context-bootstrap.ts @@ -29,6 +29,7 @@ import { } from '../../shared/memory-recall-format.js'; import { isMemoryScope } from '../../shared/memory-scope.js'; import { registerMemoryShortRef } from '../context/memory-short-ref.js'; +import { attachMemoryShortRefs } from '../context/memory-recall-refs.js'; export interface TransportContextBootstrapInput { projectDir?: string; @@ -298,7 +299,7 @@ function renderStartupMemoryText( .filter((item) => item.type === 'observation'); const sections: string[] = []; if (memoryItems.length > 0) { - sections.push(buildStartupProjectMemoryText(memoryItems)); + sections.push(buildStartupProjectMemoryText(attachMemoryShortRefs(memoryItems))); } if (observationItems.length > 0) { sections.push(renderStartupObservationIndexText(observationItems)); diff --git a/src/agent/transport-runtime-assembly.ts b/src/agent/transport-runtime-assembly.ts index 05bb1d066..f26447d9e 100644 --- a/src/agent/transport-runtime-assembly.ts +++ b/src/agent/transport-runtime-assembly.ts @@ -21,6 +21,7 @@ import type { TransportMemoryRecallItem, } from '../../shared/context-types.js'; import { buildStartupProjectMemoryText } from '../../shared/memory-recall-format.js'; +import { attachMemoryShortRefs } from '../context/memory-recall-refs.js'; import { buildFilePathReportingPrompt, buildTransportImcodesIdentityPrompt } from '../../shared/transport-runtime-prompts.js'; export interface TransportRuntimeAssemblyInput { @@ -265,7 +266,7 @@ function filterStartupMemoryForAuthority( authoritySource: 'processed_remote', sourceKind: resolveRecallSourceKind(remoteItems), items: remoteItems, - injectedText: buildStartupProjectMemoryText(remoteItems), + injectedText: buildStartupProjectMemoryText(attachMemoryShortRefs(remoteItems)), }; } diff --git a/src/agent/transport-session-runtime.ts b/src/agent/transport-session-runtime.ts index c6b1784fc..fbe747367 100644 --- a/src/agent/transport-session-runtime.ts +++ b/src/agent/transport-session-runtime.ts @@ -80,6 +80,7 @@ import { resolveSummarySyncSourceKind, } from '../context/summary-sync.js'; import { buildRelatedPastWorkText, buildStartupProjectMemoryText } from '../../shared/memory-recall-format.js'; +import { attachMemoryShortRefs } from '../context/memory-recall-refs.js'; import { getContextModelConfig } from '../context/context-model-config.js'; import { PREFERENCE_CONTEXT_END, PREFERENCE_CONTEXT_START } from '../../shared/preference-ingest.js'; import { clampUserSessionText } from '../../shared/user-session-text-caps.js'; @@ -3298,8 +3299,8 @@ export class TransportSessionRuntime implements SessionRuntime { : 'degraded-message-side'; const combinedItems = [...summaryItems, ...items]; const sections: string[] = []; - if (summaryItems.length > 0) sections.push(buildStartupProjectMemoryText(summaryItems)); - if (items.length > 0) sections.push(buildRelatedPastWorkText(items)); + if (summaryItems.length > 0) sections.push(buildStartupProjectMemoryText(attachMemoryShortRefs(summaryItems))); + if (items.length > 0) sections.push(buildRelatedPastWorkText(attachMemoryShortRefs(items))); const injectedText = sections.join('\n\n'); const sourceKind = resolveSummarySyncSourceKind(combinedItems); const payload = buildMemoryContextTimelinePayload(query, combinedItems, 'message', { diff --git a/src/context/memory-recall-refs.ts b/src/context/memory-recall-refs.ts new file mode 100644 index 000000000..5a8c588fd --- /dev/null +++ b/src/context/memory-recall-refs.ts @@ -0,0 +1,81 @@ +import type { ContextNamespace, ContextScope } from '../../shared/context-types.js'; +import { registerMemoryShortRefs, type MemoryShortRefEntry, type MemoryShortRefKind } from './memory-short-ref.js'; + +/** + * The minimum an injected memory item must expose to get a redeemable handle. + * Deliberately structural rather than tied to one concrete item type: the recall + * surfaces feed this from `MemorySearchResultItem`, `TransportMemoryRecallItem` + * and `MemoryContextTimelineItem`, which agree on these fields but not much else. + */ +export interface MemoryShortRefSource { + id?: string; + type?: 'raw' | 'processed' | 'observation'; + scope?: string; + projectId?: string; + userId?: string; + workspaceId?: string; + enterpriseId?: string; +} + +/** + * `get_memory_sources` addresses projections and observations only, so raw staged + * events get no handle rather than a dead one. + * + * Some recall surfaces carry no `type` discriminator at all; those come from the + * processed-memory path, so they are treated as projections. That default is + * fail-safe rather than a guess: if such a record were actually an observation, + * the projection lookup simply misses and the fetch returns zero sources — the + * same outcome as shipping no handle, never another record's content. + */ +function shortRefKind(type: MemoryShortRefSource['type']): MemoryShortRefKind | undefined { + if (type === 'observation') return 'observation'; + if (type === 'raw') return undefined; + return 'projection'; +} + +/** The namespace the record is STORED in, so the cached entry lines up with what + * the record fetch authorizes against. */ +function namespaceForItem(item: MemoryShortRefSource): ContextNamespace | undefined { + if (!item.scope) return undefined; + return { + scope: item.scope as ContextScope, + ...(item.projectId ? { projectId: item.projectId } : {}), + ...(item.userId ? { userId: item.userId } : {}), + ...(item.workspaceId ? { workspaceId: item.workspaceId } : {}), + ...(item.enterpriseId ? { enterpriseId: item.enterpriseId } : {}), + }; +} + +/** + * Attach a redeemable handle to every injected memory item. + * + * Injected memory used to carry only a summary, while the guidance shipped + * alongside it told the agent to "call get_memory_sources with that ref" — there + * was no ref to call with, so a summary that looked relevant was a dead end. + * Registering here (batched: one cache write, not one per item) is what makes the + * emitted handle resolvable. + */ +export function attachMemoryShortRefs( + items: readonly T[], +): Array { + const registrable: Array<{ index: number; entry: MemoryShortRefEntry }> = []; + items.forEach((item, index) => { + const kind = shortRefKind(item.type); + if (!kind || !item.id) return; + const namespace = namespaceForItem(item); + registrable.push({ + index, + entry: { kind, id: item.id, ...(namespace ? { namespace } : {}) }, + }); + }); + const refs = registerMemoryShortRefs(registrable.map((candidate) => candidate.entry)); + const refByIndex = new Map(); + registrable.forEach((candidate, position) => { + const ref = refs[position]; + if (ref) refByIndex.set(candidate.index, ref); + }); + return items.map((item, index) => { + const ref = refByIndex.get(index); + return ref ? { ...item, ref } : item; + }); +} diff --git a/src/context/memory-short-ref.ts b/src/context/memory-short-ref.ts index 83f6563fc..33fd05e0f 100644 --- a/src/context/memory-short-ref.ts +++ b/src/context/memory-short-ref.ts @@ -1,4 +1,6 @@ import type { ContextNamespace } from '../../shared/context-types.js'; +import { createHash } from 'node:crypto'; +import { encodeBase32 } from '../util/base32.js'; import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { dirname, join } from 'node:path'; @@ -12,7 +14,21 @@ export interface MemoryShortRefEntry { lastSeenAt?: number; } -const MAX_SHORT_REF_ENTRIES = 4096; +const MAX_SHORT_REF_ENTRIES = 10_000; + +/** + * base32 characters kept from the md5 digest. 5 bits per character, so 13 chars + * ≈ 65 bits: a birthday collision needs ~6e9 distinct ids in one namespace, + * versus the previous 40-bit handle which collided at ~0.5% by 100k ids. + * Collisions here are not benign — a handle that maps to several records + * resolves to an arbitrary one, i.e. the agent silently fetches the WRONG + * memory. base32 carries the same 65 bits in 13 chars that hex needs 16 for. + */ +const MEMORY_SHORT_REF_LENGTH = 13; + +/** Bumped whenever the ref derivation changes, so cached refs from an older + * algorithm are dropped instead of resolving to a stale/wrong record. */ +const SHORT_REF_SCHEMA_VERSION = 2; const entriesByRef = new Map(); let persistedLoaded = false; @@ -101,7 +117,11 @@ function ensurePersistedLoaded(): void { const path = shortRefStorePath(); if (!path) return; try { - const parsed = JSON.parse(readFileSync(path, 'utf8')) as { entries?: unknown[] }; + const parsed = JSON.parse(readFileSync(path, 'utf8')) as { schemaVersion?: unknown; entries?: unknown[] }; + // Refs cached by an older derivation must NOT be trusted: the same ref + // string can denote a different record under the new algorithm, which would + // resolve to the wrong memory. Drop the whole file and re-register lazily. + if (parsed.schemaVersion !== SHORT_REF_SCHEMA_VERSION) return; if (!Array.isArray(parsed.entries)) return; for (const raw of parsed.entries) { const normalized = normalizeEntry(raw); @@ -131,7 +151,7 @@ function persistShortRefs(): void { try { mkdirSync(dirname(path), { recursive: true }); writeFileSync(path, JSON.stringify({ - schemaVersion: 1, + schemaVersion: SHORT_REF_SCHEMA_VERSION, entries: entries.slice(0, MAX_SHORT_REF_ENTRIES), }), 'utf8'); } catch { @@ -139,13 +159,31 @@ function persistShortRefs(): void { } } +/** + * Derive the compact handle for an id. + * + * A hash prefix — NOT a prefix of the id itself. The previous algorithm kept the + * first 10 hex-looking characters of the id, which scavenged them from arbitrary + * positions across the string. For structured ids such as + * `md-ingest:personal::::::::local/:CLAUDE.md:` every record that + * shares the constant prefix collapsed onto the SAME ref (the discriminating + * sha256 sits past the 10th kept character), so one ref mapped to dozens of + * records and resolution silently picked whichever was newest. + * + * A digest gives a uniform distribution for ANY id shape. Being a pure function + * of the id also means the handle is reproducible: the same memory always yields + * the same handle across sessions, restarts and machines, so the ref→id table is + * a REBUILDABLE index rather than the only source of truth (a random handle + * would be permanently dead if that table were ever lost). md5 is used purely + * as a fast non-cryptographic digest here, never for security. + */ export function makeMemoryShortRef(kind: MemoryShortRefKind, id: string): string { - const compact = id.replace(/[^a-f0-9]/gi, '').slice(0, 10) || id.slice(0, 10); + const digest = createHash('md5').update(id, 'utf8').digest(); + const compact = encodeBase32(digest).slice(0, MEMORY_SHORT_REF_LENGTH); return `${refPrefix(kind)}:${compact}`; } -export function registerMemoryShortRef(entry: MemoryShortRefEntry): string { - ensurePersistedLoaded(); +function registerMemoryShortRefWithoutPersist(entry: MemoryShortRefEntry): string { const ref = normalizeRef(makeMemoryShortRef(entry.kind, entry.id)); const bucket = entriesByRef.get(ref) ?? []; const nextEntry = { ...entry, lastSeenAt: entry.lastSeenAt ?? Date.now() }; @@ -154,17 +192,44 @@ export function registerMemoryShortRef(entry: MemoryShortRefEntry): string { || !sameNamespace(existing.namespace, entry.namespace)); next.push(nextEntry); entriesByRef.set(ref, next); + return ref; +} + +export function registerMemoryShortRef(entry: MemoryShortRefEntry): string { + ensurePersistedLoaded(); + const ref = registerMemoryShortRefWithoutPersist(entry); pruneShortRefs(); persistShortRefs(); return ref; } +/** + * Batch variant for surfaces that register many refs at once (startup/recall + * memory injection registers every injected item so the agent can redeem the + * ref via get_memory_sources). Persists ONCE instead of per entry — the + * per-entry path rewrites the whole cache file, which would otherwise mean N + * full-file writes on the session send path. + */ +export function registerMemoryShortRefs(entries: readonly MemoryShortRefEntry[]): string[] { + ensurePersistedLoaded(); + const refs = entries.map((entry) => registerMemoryShortRefWithoutPersist(entry)); + if (refs.length > 0) { + pruneShortRefs(); + persistShortRefs(); + } + return refs; +} + export function resolveMemoryShortRef(ref: string, namespace?: ContextNamespace): MemoryShortRefEntry | undefined { ensurePersistedLoaded(); const bucket = entriesByRef.get(normalizeRef(ref)); if (!bucket || bucket.length === 0) return undefined; const exact = namespace ? newestEntry(bucket.filter((entry) => sameNamespace(entry.namespace, namespace))) : undefined; if (exact) return exact; + // Cross-namespace isolation: a handle registered under another namespace must + // NOT resolve here, even when it is the only entry for this ref. Callers other + // than get_memory_sources (archive/delete/update) also resolve refs, so this + // stays the strict boundary rather than deferring to a downstream check. if (namespace) return undefined; return bucket.length === 1 ? bucket[0] : undefined; } diff --git a/src/daemon/command-handler.ts b/src/daemon/command-handler.ts index a6c87be0b..2987bfdfc 100644 --- a/src/daemon/command-handler.ts +++ b/src/daemon/command-handler.ts @@ -962,6 +962,7 @@ import { QWEN_MODEL_IDS } from '../../shared/qwen-models.js'; import { getQwenRuntimeConfig } from '../agent/qwen-runtime-config.js'; import { getQwenDisplayMetadata } from '../agent/provider-display.js'; import { buildRelatedPastWorkText, buildStartupProjectMemoryText } from '../../shared/memory-recall-format.js'; +import { attachMemoryShortRefs } from '../context/memory-recall-refs.js'; import { getQwenOAuthQuotaUsageLabel, recordQwenOAuthRequest } from '../agent/provider-quota.js'; import { listProviderSessions as listProviderSessionsImpl } from './provider-sessions.js'; import { buildMemoryContextTimelinePayload, buildMemoryContextStatusPayload } from './memory-context-timeline.js'; @@ -12466,8 +12467,8 @@ async function prependLocalMemory( const semanticHitIds = finalItems.filter((item) => item.type === 'processed').map((item) => item.id); const hitIds = [...summaryItems.map((item) => item.id), ...semanticHitIds]; const sections: string[] = []; - if (summaryItems.length > 0) sections.push(buildStartupProjectMemoryText(summaryItems)); - if (finalItems.length > 0) sections.push(buildRelatedPastWorkText(finalItems)); + if (summaryItems.length > 0) sections.push(buildStartupProjectMemoryText(attachMemoryShortRefs(summaryItems))); + if (finalItems.length > 0) sections.push(buildRelatedPastWorkText(attachMemoryShortRefs(finalItems))); const injectedText = sections.join('\n\n'); const timelineItems = [...summaryItems, ...finalItems]; const timelinePayload = buildMemoryContextTimelinePayload(query, timelineItems); diff --git a/src/daemon/memory-context-timeline.ts b/src/daemon/memory-context-timeline.ts index ada98d0a5..b83b21b08 100644 --- a/src/daemon/memory-context-timeline.ts +++ b/src/daemon/memory-context-timeline.ts @@ -6,6 +6,7 @@ import type { MemoryContextTimelineStatus, } from '../shared/timeline/types.js'; import { buildRelatedPastWorkText } from '../../shared/memory-recall-format.js'; +import { attachMemoryShortRefs } from '../context/memory-recall-refs.js'; import type { ContextAuthorityDecision, MemoryRecallInjectionSurface, @@ -46,7 +47,7 @@ export function buildMemoryContextTimelinePayload( relevanceScore: item.relevanceScore, })); const injectedText = options?.injectedText - ?? (timelineItems.length > 0 ? buildRelatedPastWorkText(timelineItems) : undefined); + ?? (timelineItems.length > 0 ? buildRelatedPastWorkText(attachMemoryShortRefs(timelineItems)) : undefined); return { ...(query ? { query } : {}), ...(injectedText ? { injectedText } : {}), diff --git a/src/daemon/memory-inject.ts b/src/daemon/memory-inject.ts index c8b6da822..0775f2685 100644 --- a/src/daemon/memory-inject.ts +++ b/src/daemon/memory-inject.ts @@ -16,6 +16,7 @@ import { AGENT_SEND_DOCS } from './imcodes-workflow-docs.js'; import type { MemorySearchResultItem } from '../context/memory-search.js'; import { selectStartupMemoryForBootstrap } from '../context/memory-recall-client.js'; import { buildStartupProjectMemoryText } from '../../shared/memory-recall-format.js'; +import { attachMemoryShortRefs } from '../context/memory-recall-refs.js'; import logger from '../util/logger.js'; import { warnOncePerHour } from '../util/rate-limited-warn.js'; import { incrementCounter } from '../util/metrics.js'; @@ -139,7 +140,7 @@ export async function injectGeminiMemoryWithTimeline( export async function readProcessedMemory(projectName: string): Promise { const items = await readProcessedMemoryItems(projectName); if (items.length === 0) return null; - return buildStartupProjectMemoryText(items); + return buildStartupProjectMemoryText(attachMemoryShortRefs(items)); } export async function readProcessedMemoryItems(projectName: string): Promise { @@ -170,7 +171,7 @@ export async function buildSessionBootstrapContextWithItems( ): Promise<{ text: string; items: MemorySearchResultItem[] }> { const projectContext = await readProjectMemory(cwd); const items = await readProcessedMemoryItems(projectName); - const processedMemory = items.length > 0 ? buildStartupProjectMemoryText(items) : null; + const processedMemory = items.length > 0 ? buildStartupProjectMemoryText(attachMemoryShortRefs(items)) : null; const parts: string[] = []; if (projectContext) parts.push(projectContext); if (processedMemory) parts.push(processedMemory); diff --git a/src/util/base32.ts b/src/util/base32.ts new file mode 100644 index 000000000..83119cf93 --- /dev/null +++ b/src/util/base32.ts @@ -0,0 +1,28 @@ +/** + * Minimal RFC 4648 base32 encoder, lowercase and unpadded. + * + * Node has no built-in base32 (`Buffer.toString` covers hex/base64/base64url + * only), and pulling a dependency for ~15 lines isn't worth it. Output is + * lowercase because callers normalize handles to lowercase; emitting it directly + * avoids a case round-trip. The alphabet omits 0/1/8/9, so base32 handles stay + * reasonably unambiguous when read off a screen. + */ +const BASE32_ALPHABET = 'abcdefghijklmnopqrstuvwxyz234567'; + +export function encodeBase32(bytes: Uint8Array): string { + let out = ''; + let value = 0; + let bits = 0; + for (const byte of bytes) { + // `bits` is drained below 5 on every iteration, so `value` never exceeds + // 20 significant bits — well inside the 32-bit range of `<<`/`>>>`. + value = (value << 8) | byte; + bits += 8; + while (bits >= 5) { + out += BASE32_ALPHABET[(value >>> (bits - 5)) & 31]; + bits -= 5; + } + } + if (bits > 0) out += BASE32_ALPHABET[(value << (5 - bits)) & 31]; + return out; +} diff --git a/test/agent/runtime-context-bootstrap.test.ts b/test/agent/runtime-context-bootstrap.test.ts index 83b06b0b1..ffb70acfe 100644 --- a/test/agent/runtime-context-bootstrap.test.ts +++ b/test/agent/runtime-context-bootstrap.test.ts @@ -4,7 +4,7 @@ import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { SKILL_REGISTRY_FILE_NAME, SKILL_REGISTRY_SCHEMA_VERSION, makeSkillUri } from '../../shared/skill-registry-types.js'; import { configureSharedContextRuntime } from '../../src/context/shared-context-runtime.js'; -import { resetMemoryShortRefsForTests, resolveMemoryShortRef } from '../../src/context/memory-short-ref.js'; +import { makeMemoryShortRef, resetMemoryShortRefsForTests, resolveMemoryShortRef } from '../../src/context/memory-short-ref.js'; import { ensureContextNamespace, writeContextObservation, writeProcessedProjection } from '../../src/store/context-store.js'; import { cleanupIsolatedSharedContextDb, createIsolatedSharedContextDb } from '../util/shared-context-db.js'; @@ -583,8 +583,11 @@ describe('resolveTransportContextBootstrap', () => { { summary: 'Important architecture memory', projectionClass: 'durable_memory_candidate' }, { summary: 'Recent startup memory', projectionClass: 'recent_summary' }, ]); - expect(startup?.injectedText).toContain('[important] Important architecture memory'); - expect(startup?.injectedText).toContain('[recent] Recent startup memory'); + // Each injected line carries a redeemable handle between the label and the + // summary, so the agent can call get_memory_sources when the summary alone + // isn't enough. base32 handle alphabet is a-z2-7. + expect(startup?.injectedText).toMatch(/\[important\] \(proj:[a-z2-7]{13}\) Important architecture memory/); + expect(startup?.injectedText).toMatch(/\[recent\] \(proj:[a-z2-7]{13}\) Recent startup memory/); }); it('injects only active or promoted observations as a lightweight startup index', async () => { @@ -647,8 +650,8 @@ describe('resolveTransportContextBootstrap', () => { }), ]); expect(startup?.injectedText).toContain(''); - const activeRef = `obs:${active.id.slice(0, 10)}`; - const promotedRef = `obs:${promoted.id.slice(0, 10)}`; + const activeRef = makeMemoryShortRef('observation', active.id); + const promotedRef = makeMemoryShortRef('observation', promoted.id); expect(startup?.injectedText).toContain(`ref: ${activeRef}`); expect(startup?.injectedText).toContain(`ref: ${promotedRef}`); expect(startup?.injectedText).not.toContain(active.id); diff --git a/test/context/memory-short-ref.test.ts b/test/context/memory-short-ref.test.ts index d3afa6b8f..9ca257e55 100644 --- a/test/context/memory-short-ref.test.ts +++ b/test/context/memory-short-ref.test.ts @@ -1,3 +1,4 @@ +import { writeFileSync } from 'node:fs'; import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -28,9 +29,24 @@ describe('memory short refs', () => { await rm(tempDir, { recursive: true, force: true }); }); + // Golden values: pin the derivation (md5 → base32, first 13 chars) so an + // accidental change to the algorithm invalidating every issued handle fails + // here loudly instead of silently. it('builds compact deterministic refs from full memory ids', () => { - expect(makeMemoryShortRef('observation', 'aaaaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee')).toBe('obs:aaaaaaaaaa'); - expect(makeMemoryShortRef('projection', '1111111111-2222-3333-4444-555555555555')).toBe('proj:1111111111'); + expect(makeMemoryShortRef('observation', 'aaaaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee')).toBe('obs:vd34zyurs535g'); + expect(makeMemoryShortRef('projection', '1111111111-2222-3333-4444-555555555555')).toBe('proj:bp27impgwfo27'); + }); + + it('derives handles from the whole id, not the leading hex-ish characters', () => { + // Regression: the previous derivation kept the first 10 hex-looking chars of + // the id, scavenged from anywhere in the string. Structured ids that share a + // constant prefix therefore collapsed onto ONE handle (their discriminating + // sha256 sat past the 10th kept character), and resolution then returned + // whichever colliding record was newest — i.e. the wrong memory, silently. + const base = 'md-ingest:personal::::::::local/10cd7355ec31:CLAUDE.md:'; + const first = makeMemoryShortRef('projection', `${base}5f4045310f20b47e48090f0d7bc67020305d68f12ea66e9920a800321f279b42`); + const second = makeMemoryShortRef('projection', `${base}bff091dac14fa9e1c6405660f45a0db626ccafd19276af083a49439ce4bbd659`); + expect(first).not.toBe(second); }); it('survives daemon restart by reloading the local short-ref cache', () => { @@ -55,17 +71,25 @@ describe('memory short refs', () => { }); }); + // A digest collision can no longer be produced through registerMemoryShortRef + // (that was only reachable because the old derivation collided on a shared id + // prefix), so seed the persisted cache directly to keep the ambiguity paths + // covered. They still guard real 65-bit collisions. + function seedColliding(ref: string, entries: ReadonlyArray>): void { + writeFileSync( + join(tempDir, 'refs.json'), + JSON.stringify({ schemaVersion: 2, entries: entries.map((entry) => ({ ref, ...entry })) }), + 'utf8', + ); + reloadMemoryShortRefsForTests(); + } + it('does not guess when a short ref is ambiguous across namespaces', () => { - const ref = registerMemoryShortRef({ - kind: 'projection', - id: 'bbbbbbbbbb-1111-2222-3333-444444444444', - namespace: { scope: 'user_private', userId: 'user-1', projectId: 'repo-a' }, - }); - registerMemoryShortRef({ - kind: 'projection', - id: 'bbbbbbbbbb-9999-8888-7777-666666666666', - namespace: { scope: 'user_private', userId: 'user-1', projectId: 'repo-b' }, - }); + const ref = 'proj:collide000000'; + seedColliding(ref, [ + { kind: 'projection', id: 'bbbbbbbbbb-1111-2222-3333-444444444444', namespace: { scope: 'user_private', userId: 'user-1', projectId: 'repo-a' } }, + { kind: 'projection', id: 'bbbbbbbbbb-9999-8888-7777-666666666666', namespace: { scope: 'user_private', userId: 'user-1', projectId: 'repo-b' } }, + ]); expect(resolveMemoryShortRef(ref)).toBeUndefined(); expect(resolveMemoryShortRef(ref, { scope: 'user_private', userId: 'user-1', projectId: 'repo-b' })).toMatchObject({ @@ -88,18 +112,11 @@ describe('memory short refs', () => { it('resolves same-namespace short-ref conflicts to the newest seen entry', () => { const namespace = { scope: 'user_private' as const, userId: 'user-1', projectId: 'repo-1' }; - const ref = registerMemoryShortRef({ - kind: 'observation', - id: 'cccccccccc-1111-2222-3333-444444444444', - namespace, - lastSeenAt: 100, - }); - registerMemoryShortRef({ - kind: 'observation', - id: 'cccccccccc-9999-8888-7777-666666666666', - namespace, - lastSeenAt: 200, - }); + const ref = 'obs:collide000000'; + seedColliding(ref, [ + { kind: 'observation', id: 'cccccccccc-1111-2222-3333-444444444444', namespace, lastSeenAt: 100 }, + { kind: 'observation', id: 'cccccccccc-9999-8888-7777-666666666666', namespace, lastSeenAt: 200 }, + ]); expect(resolveMemoryShortRef(ref, namespace)).toMatchObject({ id: 'cccccccccc-9999-8888-7777-666666666666', diff --git a/test/daemon/codex-watcher-bootstrap.test.ts b/test/daemon/codex-watcher-bootstrap.test.ts index 5a4bb71cf..240cb5fb0 100644 --- a/test/daemon/codex-watcher-bootstrap.test.ts +++ b/test/daemon/codex-watcher-bootstrap.test.ts @@ -51,6 +51,7 @@ vi.mock('../../src/daemon/timeline-emitter.js', () => ({ })); import { ensureSessionFile } from '../../src/daemon/codex-watcher.js'; +import { makeMemoryShortRef } from '../../src/context/memory-short-ref.js'; describe('ensureSessionFile', () => { beforeEach(() => { @@ -107,7 +108,7 @@ describe('ensureSessionFile', () => { 'memory.context', expect.objectContaining({ reason: 'startup', - injectedText: '[Related past work]\n\n- [proj] Fix websocket reconnect loop\n', + injectedText: `[Related past work]\n\n- [proj] (${makeMemoryShortRef('projection', 'mem-1')}) Fix websocket reconnect loop\n`, }), expect.any(Object), ); diff --git a/test/daemon/command-handler-memory-context.test.ts b/test/daemon/command-handler-memory-context.test.ts index 45b794b25..41967ccb5 100644 --- a/test/daemon/command-handler-memory-context.test.ts +++ b/test/daemon/command-handler-memory-context.test.ts @@ -4,6 +4,7 @@ import { mkdir, mkdtemp, rm, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { promisify } from 'node:util'; +import { makeMemoryShortRef } from '../../src/context/memory-short-ref.js'; const execFileAsync = promisify(execFile); @@ -1399,7 +1400,9 @@ describe('handleWebCommand memory context timeline', () => { expect.objectContaining({ relatedToEventId: 'evt-user-1', query: 'Fix reconnect issues in websocket client', - injectedText: '[Related past work]\n\n- [codedeck] Fix websocket reconnect loop\n', + // Each line carries a redeemable handle so the agent can call + // get_memory_sources when the summary alone isn't enough. + injectedText: `[Related past work]\n\n- [codedeck] (${makeMemoryShortRef('projection', 'mem-1')}) Fix websocket reconnect loop\n`, items: [ expect.objectContaining({ id: 'mem-1', diff --git a/test/daemon/memory-inject-startup.test.ts b/test/daemon/memory-inject-startup.test.ts index a7bf6030c..7b4a50167 100644 --- a/test/daemon/memory-inject-startup.test.ts +++ b/test/daemon/memory-inject-startup.test.ts @@ -52,6 +52,7 @@ vi.mock('../../src/context/memory-recall-core.js', async (importOriginal) => ({ import { injectGeminiMemoryWithTimeline } from '../../src/daemon/memory-inject.js'; import { getSummarySyncFingerprints, resetAllSummarySyncHistories } from '../../src/context/summary-sync-history.js'; +import { makeMemoryShortRef } from '../../src/context/memory-short-ref.js'; describe('injectGeminiMemoryWithTimeline', () => { beforeEach(() => { @@ -111,7 +112,7 @@ describe('injectGeminiMemoryWithTimeline', () => { 'memory.context', expect.objectContaining({ reason: 'startup', - injectedText: '[Related past work]\n\n- [proj] Fix websocket reconnect loop\n', + injectedText: `[Related past work]\n\n- [proj] (${makeMemoryShortRef('projection', 'mem-1')}) Fix websocket reconnect loop\n`, items: [ expect.objectContaining({ id: 'mem-1', diff --git a/test/daemon/memory-mcp-tools-schema-firewall.test.ts b/test/daemon/memory-mcp-tools-schema-firewall.test.ts index d4f89b6f8..be5435166 100644 --- a/test/daemon/memory-mcp-tools-schema-firewall.test.ts +++ b/test/daemon/memory-mcp-tools-schema-firewall.test.ts @@ -7,7 +7,7 @@ import { MCP_ERROR_REASONS } from '../../shared/memory-mcp-errors.js'; import { MEMORY_MCP_DEGRADED_REASON } from '../../shared/memory-ws.js'; import { createMemoryMcpToolHandlers } from '../../src/daemon/memory-mcp-tools.js'; import type { McpRuntimeCaller } from '../../src/daemon/memory-mcp-caller.js'; -import { registerMemoryShortRef, resetMemoryShortRefsForTests } from '../../src/context/memory-short-ref.js'; +import { makeMemoryShortRef, registerMemoryShortRef, resetMemoryShortRefsForTests } from '../../src/context/memory-short-ref.js'; import type { SessionRecord } from '../../src/store/session-store.js'; function caller(overrides: Partial = {}): McpRuntimeCaller { @@ -435,7 +435,7 @@ describe('memory MCP tool schema firewall', () => { items: [ { projectionId, - ref: 'proj:1111111111', + ref: makeMemoryShortRef('projection', projectionId), recordKind: 'projection', sourceLookup: { tool: 'get_memory_sources', kind: 'projection', projectionId }, summary: 'MCP provider readiness fixed for Gemini, Copilot, and Qwen.', @@ -457,7 +457,7 @@ describe('memory MCP tool schema firewall', () => { limit: 5, })); await expect(handlers[MEMORY_MCP_TOOL_NAMES.GET_MEMORY_SOURCES]({ - ref: 'proj:1111111111', + ref: makeMemoryShortRef('projection', projectionId), kind: 'projection', })).resolves.toMatchObject({ status: 'ok', @@ -502,7 +502,7 @@ describe('memory MCP tool schema firewall', () => { items: [ { observationId, - ref: 'obs:aaaaaaaaaa', + ref: makeMemoryShortRef('observation', observationId), recordKind: 'observation', sourceLookup: { tool: 'get_memory_sources', kind: 'observation', observationId }, observationClass: 'note', @@ -518,7 +518,7 @@ describe('memory MCP tool schema firewall', () => { serverId: 'attacker-srv', }); await expect(handlers[MEMORY_MCP_TOOL_NAMES.GET_MEMORY_SOURCES]({ - ref: 'obs:aaaaaaaaaa', + ref: makeMemoryShortRef('observation', observationId), kind: 'observation', })).resolves.toMatchObject({ status: 'ok', diff --git a/test/daemon/memory-recall-integration.test.ts b/test/daemon/memory-recall-integration.test.ts index 442f58161..86865a2f5 100644 --- a/test/daemon/memory-recall-integration.test.ts +++ b/test/daemon/memory-recall-integration.test.ts @@ -201,8 +201,11 @@ describe('memory recall integration', () => { const { buildSessionBootstrapContext } = await import('../../src/daemon/memory-inject.js'); const context = await buildSessionBootstrapContext('/tmp/fake-project', 'my-project'); - expect(context).toContain('[important] Persist enterprise binding decisions as durable memory'); - expect(context).toContain('[recent] Recent fix for transport memory recall cards'); + // A redeemable handle sits between the label and the summary so the agent + // can call get_memory_sources when the summary alone isn't enough. + // base32 handle alphabet is a-z2-7. + expect(context).toMatch(/\[important\] \(proj:[a-z2-7]{13}\) Persist enterprise binding decisions as durable memory/); + expect(context).toMatch(/\[recent\] \(proj:[a-z2-7]{13}\) Recent fix for transport memory recall cards/); }); it('buildSessionBootstrapContext includes "# Recent project memory" when memories exist', async () => { From 7af1073f503c8dae6b8018ae010e7c7d7132ec0e Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sun, 26 Jul 2026 00:18:01 +0800 Subject: [PATCH 14/40] Persist memory handles in the context store instead of a JSON file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Handles were persisted by rewriting a whole JSON file on every registration, and the write error was swallowed. Once the disk filled up the file silently stopped being written, so every handle issued after that point died on the next daemon restart — the failure was invisible precisely when it mattered. Upsert only the touched rows into the context store, and warm the in-memory index from it once at startup. Resolution stays synchronous because it runs inside render paths, so the store is read once rather than per lookup, and the write is fire-and-forget so registration never blocks on I/O. IMCODES_MEMORY_SHORT_REF_PATH still selects the JSON file for hermetic tests and a debuggable dump. Co-Authored-By: Claude Opus 4.8 --- shared/context-store-rpc.ts | 3 + src/context/memory-short-ref.ts | 88 ++++++++++++++++++++- src/daemon/lifecycle.ts | 9 +++ src/store/context-store.ts | 66 ++++++++++++++++ test/context/memory-short-ref-store.test.ts | 88 +++++++++++++++++++++ 5 files changed, 251 insertions(+), 3 deletions(-) create mode 100644 test/context/memory-short-ref-store.test.ts diff --git a/shared/context-store-rpc.ts b/shared/context-store-rpc.ts index f7a379d25..fea033cb5 100644 --- a/shared/context-store-rpc.ts +++ b/shared/context-store-rpc.ts @@ -39,6 +39,9 @@ export type ContextStoreRpcPriority = // ── L1: allowlisted direct store wrappers (each maps 1:1 to a context-store.ts export) ── export const CONTEXT_STORE_L1_OPS = [ + // memory short-ref handle map (ref → id), durable across daemon restarts + 'listMemoryShortRefs', + 'upsertMemoryShortRefs', // reads 'getProcessedProjectionById', 'listProcessedProjections', diff --git a/src/context/memory-short-ref.ts b/src/context/memory-short-ref.ts index 33fd05e0f..b9d0e3dff 100644 --- a/src/context/memory-short-ref.ts +++ b/src/context/memory-short-ref.ts @@ -1,6 +1,7 @@ import type { ContextNamespace } from '../../shared/context-types.js'; import { createHash } from 'node:crypto'; import { encodeBase32 } from '../util/base32.js'; +import { getContextStoreClient } from '../store/context-store-worker-client.js'; import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { dirname, join } from 'node:path'; @@ -140,7 +141,7 @@ function ensurePersistedLoaded(): void { } } -function persistShortRefs(): void { +function persistShortRefsToFile(): void { const path = shortRefStorePath(); if (!path) return; const entries: Array<{ ref: string } & MemoryShortRefEntry> = []; @@ -159,6 +160,87 @@ function persistShortRefs(): void { } } +/** + * Persist the handles just registered. + * + * Default path is the context store (SQLite): an incremental upsert of only the + * touched rows, rather than rewriting a whole JSON file on every registration — + * that rewrite silently stopped persisting anything once the disk filled up, + * because the write error was swallowed. Fire-and-forget so the synchronous + * registration path (called from render functions) never blocks on I/O. + * + * `IMCODES_MEMORY_SHORT_REF_PATH` still selects the JSON file, which keeps tests + * hermetic and gives a debuggable plain-text dump. + */ +function persistShortRefs(touched: ReadonlyArray<{ ref: string; entry: MemoryShortRefEntry }>): void { + if (shortRefStorePath()) { + persistShortRefsToFile(); + return; + } + if (touched.length === 0) return; + const rows = touched.map(({ ref, entry }) => ({ + ref, + kind: entry.kind, + id: entry.id, + namespaceKey: namespaceKey(entry.namespace), + namespaceJson: entry.namespace ? JSON.stringify(entry.namespace) : null, + lastSeenAt: entry.lastSeenAt ?? Date.now(), + })); + void getContextStoreClient() + .run('upsertMemoryShortRefs', [rows]) + .catch(() => { + // Non-fatal: the in-memory index still serves this process, handles are a + // pure function of the id, and sourceLookup full ids remain canonical. + }); +} + +/** + * Warm the in-memory index from the context store once at daemon startup. + * + * Resolution stays synchronous (it is called from synchronous render paths), so + * the store is read once here instead of per lookup. Handles registered before + * this resolves are unaffected — the in-memory index is authoritative in-process. + */ +export async function loadMemoryShortRefsFromStore(): Promise { + if (shortRefStorePath()) return 0; + try { + const rows = await getContextStoreClient() + .run>>('listMemoryShortRefs', [MAX_SHORT_REF_ENTRIES]); + if (!Array.isArray(rows)) return 0; + let loaded = 0; + for (const row of rows) { + const normalized = normalizeEntry({ + ref: row.ref, + kind: row.kind, + id: row.id, + lastSeenAt: row.lastSeenAt, + namespace: typeof row.namespaceJson === 'string' ? safeParseNamespace(row.namespaceJson) : undefined, + }); + if (!normalized) continue; + const bucket = entriesByRef.get(normalized.ref) ?? []; + if (bucket.some((entry) => entry.kind === normalized.entry.kind + && entry.id === normalized.entry.id + && sameNamespace(entry.namespace, normalized.entry.namespace))) continue; + bucket.push(normalized.entry); + entriesByRef.set(normalized.ref, bucket); + loaded += 1; + } + pruneShortRefs(); + persistedLoaded = true; + return loaded; + } catch { + return 0; + } +} + +function safeParseNamespace(raw: string): unknown { + try { + return JSON.parse(raw); + } catch { + return undefined; + } +} + /** * Derive the compact handle for an id. * @@ -199,7 +281,7 @@ export function registerMemoryShortRef(entry: MemoryShortRefEntry): string { ensurePersistedLoaded(); const ref = registerMemoryShortRefWithoutPersist(entry); pruneShortRefs(); - persistShortRefs(); + persistShortRefs([{ ref, entry }]); return ref; } @@ -215,7 +297,7 @@ export function registerMemoryShortRefs(entries: readonly MemoryShortRefEntry[]) const refs = entries.map((entry) => registerMemoryShortRefWithoutPersist(entry)); if (refs.length > 0) { pruneShortRefs(); - persistShortRefs(); + persistShortRefs(refs.map((ref, index) => ({ ref, entry: entries[index]! }))); } return refs; } diff --git a/src/daemon/lifecycle.ts b/src/daemon/lifecycle.ts index 229e52201..29f818905 100644 --- a/src/daemon/lifecycle.ts +++ b/src/daemon/lifecycle.ts @@ -7,6 +7,7 @@ import { ServerLink, setServerLinkReconnectResyncHandler } from './server-link.j import { handleWebCommand, setRouterContext, refreshCodexQuotaMetadata, refreshClaudeSdkSubQuotaMetadata } from './command-handler.js'; import { dispatchSessionMessageByName } from './session-dispatch.js'; import { initFileTransfer, startCleanupTimer } from './file-transfer-handler.js'; +import { loadMemoryShortRefsFromStore } from '../context/memory-short-ref.js'; import { notifySessionIdle, listP2pRuns, serializeP2pRun } from './p2p-orchestrator.js'; import { isP2pParticipantMemoryNoise } from './p2p-memory-filter.js'; import { handlePreviewBinaryFrame } from './preview-relay.js'; @@ -540,6 +541,14 @@ export async function startup(): Promise { startCleanupTimer(); logger.info('File transfer initialized'); + // Warm the memory short-ref index so handles injected before this restart + // still resolve. Resolution is synchronous (it runs inside render paths), so + // the durable map is read once here rather than per lookup. Best-effort: a + // cold index only means a handle re-registers on its next injection. + void loadMemoryShortRefsFromStore() + .then((loaded) => { if (loaded > 0) logger.info({ loaded }, 'Memory short-ref index warmed'); }) + .catch(() => { /* non-fatal: handles are a pure function of the id */ }); + // Clean up old timeline files (>7 days) and truncate oversized ones. // // Both calls walk every JSONL file in ~/.imcodes/timeline. With a backlog diff --git a/src/store/context-store.ts b/src/store/context-store.ts index 578e1a602..6987a9280 100644 --- a/src/store/context-store.ts +++ b/src/store/context-store.ts @@ -445,6 +445,21 @@ function ensureDb(): DatabaseSyncInstance { ); CREATE INDEX IF NOT EXISTS idx_pinned_namespace ON context_pinned_notes(namespace_key); + -- ref → id map for the compact memory handles injected into agent context. + -- Durable so a handle still resolves after a daemon restart. Replaces a + -- whole-file JSON rewrite per registration whose failures were swallowed — + -- a full disk silently stopped persisting handles entirely. + CREATE TABLE IF NOT EXISTS memory_short_refs ( + ref TEXT NOT NULL, + kind TEXT NOT NULL, + id TEXT NOT NULL, + namespace_key TEXT NOT NULL DEFAULT '', + namespace_json TEXT, + last_seen_at INTEGER NOT NULL, + PRIMARY KEY (ref, kind, id, namespace_key) + ); + CREATE INDEX IF NOT EXISTS idx_memory_short_refs_seen ON memory_short_refs(last_seen_at); + CREATE TABLE IF NOT EXISTS context_replication_state ( namespace_key TEXT PRIMARY KEY, pending_projection_ids_json TEXT NOT NULL, @@ -2739,6 +2754,57 @@ export function listPinnedNotes(namespaceKey: string): PinnedNote[] { })); } +export interface MemoryShortRefRow { + ref: string; + kind: string; + id: string; + namespaceKey: string; + namespaceJson: string | null; + lastSeenAt: number; +} + +/** Persist memory handles. Batched into one transaction: the in-memory index + * registers a whole injected block at once. */ +export function upsertMemoryShortRefs(rows: readonly MemoryShortRefRow[]): number { + if (rows.length === 0) return 0; + const database = ensureDb(); + const statement = database.prepare(` + INSERT INTO memory_short_refs (ref, kind, id, namespace_key, namespace_json, last_seen_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(ref, kind, id, namespace_key) DO UPDATE SET + namespace_json = excluded.namespace_json, + last_seen_at = excluded.last_seen_at + `); + database.exec('BEGIN'); + try { + for (const row of rows) { + statement.run(row.ref, row.kind, row.id, row.namespaceKey, row.namespaceJson, row.lastSeenAt); + } + database.exec('COMMIT'); + } catch (error) { + database.exec('ROLLBACK'); + throw error; + } + return rows.length; +} + +/** Newest-first so a bounded warm-load keeps the most recently used handles. */ +export function listMemoryShortRefs(limit?: number): MemoryShortRefRow[] { + const database = ensureDb(); + const sql = 'SELECT * FROM memory_short_refs ORDER BY last_seen_at DESC' + + (typeof limit === 'number' && limit > 0 ? ' LIMIT ?' : ''); + const statement = database.prepare(sql); + const rows = (typeof limit === 'number' && limit > 0 ? statement.all(limit) : statement.all()) as Array>; + return rows.map((row) => ({ + ref: String(row.ref), + kind: String(row.kind), + id: String(row.id), + namespaceKey: String(row.namespace_key ?? ''), + namespaceJson: row.namespace_json == null ? null : String(row.namespace_json), + lastSeenAt: Number(row.last_seen_at), + })); +} + export function getStagedEvent(id: string): LocalContextEvent | undefined { const database = ensureDb(); const row = database.prepare('SELECT * FROM context_staged_events WHERE id = ?').get(id) as Record | undefined; diff --git a/test/context/memory-short-ref-store.test.ts b/test/context/memory-short-ref-store.test.ts new file mode 100644 index 000000000..9d174417e --- /dev/null +++ b/test/context/memory-short-ref-store.test.ts @@ -0,0 +1,88 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { resetContextStoreClientForTests } from '../../src/store/context-store-worker-client.js'; +import { listMemoryShortRefs } from '../../src/store/context-store.js'; +import { + loadMemoryShortRefsFromStore, + registerMemoryShortRefs, + resetMemoryShortRefsForTests, + resolveMemoryShortRef, +} from '../../src/context/memory-short-ref.js'; +import { cleanupIsolatedSharedContextDb, createIsolatedSharedContextDb } from '../util/shared-context-db.js'; + +/** + * Durable handle map. Handles used to persist by rewriting a whole JSON file on + * every registration, with the write error swallowed — once the disk filled up + * the file silently stopped being written, so every handle issued after that + * point died on the next daemon restart. Persist incrementally to the context + * store instead, and warm the in-memory index from it at startup. + */ +describe('memory short refs — durable store persistence', () => { + let tempDir: string; + let priorPath: string | undefined; + + beforeEach(async () => { + // Unset so the store (not the JSON file) is the persistence target. + priorPath = process.env.IMCODES_MEMORY_SHORT_REF_PATH; + delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; + tempDir = await createIsolatedSharedContextDb('memory-short-ref-store'); + resetMemoryShortRefsForTests(); + }); + + afterEach(async () => { + resetMemoryShortRefsForTests(); + resetContextStoreClientForTests(); + if (priorPath === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_PATH = priorPath; + await cleanupIsolatedSharedContextDb(tempDir); + }); + + const namespace = { scope: 'personal' as const, userId: 'user-1', projectId: 'repo-1' }; + + it('persists registered handles to the store and resolves them after a restart', async () => { + const [ref] = registerMemoryShortRefs([ + { kind: 'projection', id: 'fb7a7af3-3185-45a4-ac47-b26a57142353', namespace }, + { kind: 'observation', id: '2cf7602fb9d586fa5377f22b879ca97b3ba4c5acbdd32b9df36ad15e2df0a8c6', namespace }, + ]); + expect(ref).toMatch(/^proj:[a-z2-7]{13}$/); + + // Registration is fire-and-forget; let the store write settle. + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(listMemoryShortRefs().length).toBeGreaterThanOrEqual(2); + + // Simulate a daemon restart: in-memory index gone, store intact. + resetMemoryShortRefsForTests(); + expect(resolveMemoryShortRef(ref!, namespace)).toBeUndefined(); + + const loaded = await loadMemoryShortRefsFromStore(); + expect(loaded).toBeGreaterThanOrEqual(2); + expect(resolveMemoryShortRef(ref!, namespace)).toMatchObject({ + kind: 'projection', + id: 'fb7a7af3-3185-45a4-ac47-b26a57142353', + }); + }); + + it('re-registering the same memory upserts instead of accumulating rows', async () => { + const entry = { kind: 'projection' as const, id: 'dddddddddd-1111-2222-3333-444444444444', namespace }; + registerMemoryShortRefs([entry]); + await new Promise((resolve) => setTimeout(resolve, 50)); + const afterFirst = listMemoryShortRefs().length; + + // The handle is a pure function of the id, so a repeat injection reuses the + // same primary key rather than growing the table. + registerMemoryShortRefs([entry]); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(listMemoryShortRefs().length).toBe(afterFirst); + }); + + it('keeps cross-namespace isolation after a store warm-load', async () => { + const [ref] = registerMemoryShortRefs([ + { kind: 'projection', id: 'cccccccccc-1111-2222-3333-444444444444', namespace }, + ]); + await new Promise((resolve) => setTimeout(resolve, 50)); + resetMemoryShortRefsForTests(); + await loadMemoryShortRefsFromStore(); + + expect(resolveMemoryShortRef(ref!, { ...namespace, projectId: 'other-repo' })).toBeUndefined(); + expect(resolveMemoryShortRef(ref!, namespace)).toMatchObject({ id: 'cccccccccc-1111-2222-3333-444444444444' }); + }); +}); From 39ec403488638e245bab881f08759fcefbbfdd84 Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sun, 26 Jul 2026 06:17:34 +0800 Subject: [PATCH 15/40] Make memory-handle persistence failures observable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moving handles off the JSON file was meant to fix a write whose errors were swallowed — a full disk silently stopped persisting handles, and nobody found out until they died on the next restart. The store write then swallowed its own failures the same way, so the same blind spot survived in a new location: a write rejected while the store worker is still starting, or on SQLITE_BUSY, left handles unpersisted with no signal at all. Report the counter and rate-limited warning this file's own store module already uses for exactly this case. Registration stays non-fatal and fire-and-forget, since it is called from synchronous render paths. Also use BEGIN IMMEDIATE like every other write transaction here: both the worker and the startup cold-fallback path can hold a write connection, and a deferred transaction upgrading to a write under WAL fails with SQLITE_BUSY rather than waiting. Guard the ROLLBACK so a failing rollback cannot mask the original error. Co-Authored-By: Claude Opus 4.8 --- src/context/memory-short-ref.ts | 16 +++- src/store/context-store.ts | 8 +- .../memory-short-ref-persist-failure.test.ts | 78 +++++++++++++++++++ 3 files changed, 97 insertions(+), 5 deletions(-) create mode 100644 test/context/memory-short-ref-persist-failure.test.ts diff --git a/src/context/memory-short-ref.ts b/src/context/memory-short-ref.ts index b9d0e3dff..259315968 100644 --- a/src/context/memory-short-ref.ts +++ b/src/context/memory-short-ref.ts @@ -2,6 +2,8 @@ import type { ContextNamespace } from '../../shared/context-types.js'; import { createHash } from 'node:crypto'; import { encodeBase32 } from '../util/base32.js'; import { getContextStoreClient } from '../store/context-store-worker-client.js'; +import { warnOncePerHour } from '../util/rate-limited-warn.js'; +import { incrementCounter } from '../util/metrics.js'; import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { dirname, join } from 'node:path'; @@ -188,9 +190,17 @@ function persistShortRefs(touched: ReadonlyArray<{ ref: string; entry: MemorySho })); void getContextStoreClient() .run('upsertMemoryShortRefs', [rows]) - .catch(() => { - // Non-fatal: the in-memory index still serves this process, handles are a - // pure function of the id, and sourceLookup full ids remain canonical. + .catch((error: unknown) => { + // Non-fatal for this process — the in-memory index still resolves, handles + // are a pure function of the id, and sourceLookup full ids stay canonical. + // But it IS the failure this change exists to make visible: handles that + // never land stop resolving after a restart. Surface it rather than + // repeating the swallowed-write bug in a new location. + incrementCounter('mem.startup.silent_failure', { source: 'memory-short-ref-upsert' }); + warnOncePerHour('mem.startup.silent_failure.memory-short-ref-upsert', { + error: error instanceof Error ? error.message : String(error), + rows: rows.length, + }); }); } diff --git a/src/store/context-store.ts b/src/store/context-store.ts index 6987a9280..93c6a2964 100644 --- a/src/store/context-store.ts +++ b/src/store/context-store.ts @@ -2775,14 +2775,18 @@ export function upsertMemoryShortRefs(rows: readonly MemoryShortRefRow[]): numbe namespace_json = excluded.namespace_json, last_seen_at = excluded.last_seen_at `); - database.exec('BEGIN'); + // IMMEDIATE, like every other write transaction here: the worker and the + // startup cold-fallback path can both hold a write connection, and a deferred + // transaction upgrading to a write under WAL fails with SQLITE_BUSY instead of + // waiting for the lock. + database.exec('BEGIN IMMEDIATE'); try { for (const row of rows) { statement.run(row.ref, row.kind, row.id, row.namespaceKey, row.namespaceJson, row.lastSeenAt); } database.exec('COMMIT'); } catch (error) { - database.exec('ROLLBACK'); + try { database.exec('ROLLBACK'); } catch { /* a failing rollback must not mask the original error */ } throw error; } return rows.length; diff --git a/test/context/memory-short-ref-persist-failure.test.ts b/test/context/memory-short-ref-persist-failure.test.ts new file mode 100644 index 000000000..0b77f0651 --- /dev/null +++ b/test/context/memory-short-ref-persist-failure.test.ts @@ -0,0 +1,78 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const { runMock, incrementCounterMock, warnOncePerHourMock } = vi.hoisted(() => ({ + runMock: vi.fn(), + incrementCounterMock: vi.fn(), + warnOncePerHourMock: vi.fn(), +})); + +vi.mock('../../src/store/context-store-worker-client.js', () => ({ + getContextStoreClient: () => ({ run: runMock }), +})); +vi.mock('../../src/util/metrics.js', () => ({ incrementCounter: incrementCounterMock })); +vi.mock('../../src/util/rate-limited-warn.js', () => ({ warnOncePerHour: warnOncePerHourMock })); + +import { registerMemoryShortRefs, resetMemoryShortRefsForTests } from '../../src/context/memory-short-ref.js'; + +/** + * The whole point of moving handles off the JSON file was that its write errors + * were swallowed, so a full disk silently stopped persisting handles and nobody + * found out until they died on the next restart. Persisting to the store must + * not reintroduce that: a failed write is still non-fatal for the running + * process, but it has to be observable. + */ +describe('memory short refs — persistence failures stay observable', () => { + let priorPath: string | undefined; + + beforeEach(() => { + priorPath = process.env.IMCODES_MEMORY_SHORT_REF_PATH; + delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; // select the store path + vi.clearAllMocks(); + resetMemoryShortRefsForTests(); + }); + + afterEach(() => { + resetMemoryShortRefsForTests(); + if (priorPath === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_PATH = priorPath; + }); + + const entry = { + kind: 'projection' as const, + id: 'fb7a7af3-3185-45a4-ac47-b26a57142353', + namespace: { scope: 'personal' as const, userId: 'user-1', projectId: 'repo-1' }, + }; + + it('reports a counter and a rate-limited warning when the store write fails', async () => { + runMock.mockRejectedValue(new Error('context_store_unavailable')); + + const refs = registerMemoryShortRefs([entry]); + expect(refs).toHaveLength(1); + await vi.waitFor(() => expect(incrementCounterMock).toHaveBeenCalled()); + + expect(incrementCounterMock).toHaveBeenCalledWith( + 'mem.startup.silent_failure', + { source: 'memory-short-ref-upsert' }, + ); + expect(warnOncePerHourMock).toHaveBeenCalledWith( + 'mem.startup.silent_failure.memory-short-ref-upsert', + expect.objectContaining({ error: 'context_store_unavailable' }), + ); + }); + + it('keeps registration non-fatal and in-memory resolution working when the write fails', async () => { + runMock.mockRejectedValue(new Error('SQLITE_BUSY')); + // Registration must not throw: the caller is a synchronous render path. + expect(() => registerMemoryShortRefs([entry])).not.toThrow(); + await vi.waitFor(() => expect(warnOncePerHourMock).toHaveBeenCalled()); + }); + + it('stays quiet on a successful write', async () => { + runMock.mockResolvedValue(1); + registerMemoryShortRefs([entry]); + await Promise.resolve(); + await Promise.resolve(); + expect(incrementCounterMock).not.toHaveBeenCalled(); + expect(warnOncePerHourMock).not.toHaveBeenCalled(); + }); +}); From 3ef1d98d14d0b4e7863be3b506e9129f8a34a98d Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sun, 26 Jul 2026 06:28:55 +0800 Subject: [PATCH 16/40] Stop a daemon link restore from firing one push per idle session MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On the 43 deployment a gateway blip dropped four daemons' WebSockets. All four re-authenticated within two seconds, and 276ms later ~90 push dispatches went out inside a 500ms window — a phone full of "Task complete — ready for input", several quoting work that had finished long before. The measurement is the damning part: 181 push dispatches in twelve hours, of which 91 landed in the 06:00 minute and 90 in the 23:22 minute — the two daemon-reconnect bursts. Every single notification in half a day came from this, and not one came from work actually completing. `resyncTransportSessionStatesAfterLinkRestore()` re-states every session's CURRENT state after the link returns, so browsers stop rendering a "working" spinner for a turn that ended during the outage. Those events are re-announcements, never completions — but nothing told the push path that. Both existing guards missed: `suppressPush` was not set, and PUSH_TIMELINE_EVENT_MAX_AGE_MS cannot help because a resync event is newly minted (ts = now) even though the state it reports is old. This is the unfixed half of 8493a614e. That commit stopped the resync from re-firing DAEMON-LOCAL idle side effects (notifySessionIdle, drainQueue) and even added `isServerLinkResyncStatePayload` for the purpose — but the events still went to the server, and the server had never been taught the marker. I fixed the half I had reproduced and did not follow the event across the process boundary. Fixed on both sides, which is deliberate rather than redundant: - Daemon: the resync payload now carries `suppressPush`, so it is self-describing at the source. - Server: the resync marker is also honoured in the push branch. Daemons are user-installed and upgrade on their own schedule, so a daemon-only fix would leave every daemon already in the field storming until it happened to update. This makes a server deploy sufficient to stop it now. Suppressing the push must not weaken the idle — un-sticking a stale "working" footer is the whole reason the resync exists — so a test asserts the payload still satisfies the authoritative-idle shape validator. Counterfactuals pin each layer separately: revert the server guard and the old-daemon cases fail, one of them reproducing the storm's exact shape (`expected 25 to be +0` — 25 resyncing sessions, 25 queued pushes); revert the daemon flag and only the payload test fails. A third test asserts a genuine live idle STILL pushes, so the fix cannot pass by silencing notifications outright. Co-Authored-By: Claude Opus 5 --- server/src/ws/bridge.ts | 15 ++ .../bridge-resync-push-suppression.test.ts | 154 ++++++++++++++++++ src/agent/session-manager.ts | 12 +- .../session-manager-state-resync.test.ts | 24 +++ 4 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 server/test/bridge-resync-push-suppression.test.ts diff --git a/server/src/ws/bridge.ts b/server/src/ws/bridge.ts index e13407b7e..102b660b7 100644 --- a/server/src/ws/bridge.ts +++ b/server/src/ws/bridge.ts @@ -123,6 +123,7 @@ import { incrementCounter } from '../util/metrics.js'; import { pickReadableSessionDisplay } from '../../../shared/session-display.js'; import { isKnownTestSessionLike } from '../../../shared/test-session-guard.js'; import { PUSH_TIMELINE_EVENT_MAX_AGE_MS, TIMELINE_SUPPRESS_PUSH_FIELD } from '../../../shared/push-notifications.js'; +import { isServerLinkResyncStatePayload } from '../../../shared/session-activity-types.js'; import { DAEMON_UPGRADE_DELIVERY_STATUS, } from '../../../shared/daemon-upgrade.js'; @@ -2790,6 +2791,20 @@ export class WsBridge { const payload = event.payload as Record; const eventTs = typeof event.ts === 'number' ? event.ts : undefined; if (payload[TIMELINE_SUPPRESS_PUSH_FIELD] === true) return; + // A daemon link-restore resync re-states every session's CURRENT + // state so browsers stop rendering a stale "working" spinner. It is + // a re-announcement, never a fresh completion, so it must not push: + // one link restore would otherwise fan one "Task complete" per idle + // session (seen on the 43 deployment as ~90 dispatches in 500ms right + // after four daemons re-authenticated). The age guard below cannot + // catch it — resync events are newly minted even though the state + // they describe is old. + // + // Checked HERE as well as suppressed at the daemon, on purpose: + // daemons are user-installed and upgrade on their own schedule, so a + // daemon-only fix leaves every not-yet-updated daemon still storming + // until it happens to update. This makes a server deploy sufficient. + if (isServerLinkResyncStatePayload(payload)) return; if (eventTs && Date.now() - eventTs > PUSH_TIMELINE_EVENT_MAX_AGE_MS) return; this.scheduleIdleEventPush(db, env, { type: 'session.idle', diff --git a/server/test/bridge-resync-push-suppression.test.ts b/server/test/bridge-resync-push-suppression.test.ts new file mode 100644 index 000000000..52261af79 --- /dev/null +++ b/server/test/bridge-resync-push-suppression.test.ts @@ -0,0 +1,154 @@ +/** + * A daemon link-restore resync must never raise push notifications. + * + * Observed on the 43 deployment: a gateway blip dropped four daemons' sockets; + * all four re-authenticated within two seconds, and 276ms later ~90 push + * dispatches went out inside a 500ms window — one "Task complete — ready for + * input" per idle session, several quoting work that had finished long before. + * + * `resyncTransportSessionStatesAfterLinkRestore()` re-states every session's + * CURRENT state so browsers stop rendering a stale "working" spinner. Those + * events are re-announcements, not completions. The existing + * `PUSH_TIMELINE_EVENT_MAX_AGE_MS` guard cannot filter them, because a resync + * event is newly minted (ts = now) even though the state it describes is old. + * + * The daemon now stamps `suppressPush`, but this server-side check is what makes + * a server deploy sufficient: daemons are user-installed and upgrade on their own + * schedule, so recognising the resync marker here stops the storm for every + * daemon version already in the field. + * + * @vitest-environment node + */ + +import { EventEmitter } from 'node:events'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { WsBridge } from '../src/ws/bridge.js'; +import { TIMELINE_MESSAGES } from '../../shared/timeline-protocol.js'; +import { TIMELINE_SUPPRESS_PUSH_FIELD } from '../../shared/push-notifications.js'; +import { SESSION_STATE_DECISION_REASON_SERVER_LINK_RESYNC } from '../../shared/session-activity-types.js'; + +vi.mock('../src/security/crypto.js', () => ({ sha256Hex: (_s: string) => 'valid-hash' })); +vi.mock('../src/routes/push.js', () => ({ dispatchPush: vi.fn() })); + +class MockWs extends EventEmitter { + sent: Array = []; + readyState = 1; + send(data: string | Buffer, _opts?: unknown, cb?: (err?: Error) => void): void { + this.sent.push(data); + cb?.(); + } + close(): void { this.readyState = 3; this.emit('close'); } +} + +function makeDb() { + return { + queryOne: async () => ({ token_hash: 'valid-hash', node_role: 'full', revoked_at: null }), + query: async () => [], + execute: async () => ({ changes: 1 }), + exec: async () => {}, + transaction: async (fn: (tx: unknown) => Promise) => fn({}), + close: () => {}, + } as unknown as import('../src/db/client.js').Database; +} + +const SESSION = 'deck_resyncpush_brain'; + +function idleStateEvent(payload: Record): string { + return JSON.stringify({ + type: TIMELINE_MESSAGES.EVENT, + event: { + eventId: `evt-${Math.random().toString(36).slice(2)}`, + sessionId: SESSION, + ts: Date.now(), + seq: 1, + epoch: 1, + type: 'session.state', + payload: { state: 'idle', ...payload }, + }, + }); +} + +async function flush(): Promise { + await new Promise((resolve) => { setImmediate(resolve); }); +} + +/** + * Whether a push is queued for this session. Read synchronously right after the + * daemon message: with the test settle window at 0ms the flush is a microtask, + * so waiting first would drain the map and hide the difference. + */ +function pendingIdlePushCount(bridge: WsBridge): number { + const pending = (bridge as unknown as { pendingIdlePushes: Map }).pendingIdlePushes; + return pending.size; +} + +describe('WsBridge — link-restore resync must not push', () => { + let serverId: string; + + beforeEach(() => { serverId = `resync-${Math.random().toString(36).slice(2)}`; }); + afterEach(() => { WsBridge.getAll().clear(); vi.clearAllMocks(); }); + + async function authedDaemon() { + const bridge = WsBridge.get(serverId); + const daemon = new MockWs(); + // env must be truthy for the push branch to be reachable at all. + bridge.handleDaemonConnection(daemon as never, makeDb() as never, {} as never); + daemon.emit('message', JSON.stringify({ type: 'auth', serverId, token: 'tok' })); + await flush(); + expect(bridge.isAuthenticated).toBe(true); + return { bridge, daemon }; + } + + it('schedules no push for a resync idle even without the daemon-side suppressPush flag', async () => { + // The already-deployed-daemon case: only `decisionReason` identifies it. + const { bridge, daemon } = await authedDaemon(); + + daemon.emit('message', idleStateEvent({ + decisionReason: SESSION_STATE_DECISION_REASON_SERVER_LINK_RESYNC, + })); + + expect(pendingIdlePushCount(bridge)).toBe(0); + }); + + it('schedules no push when the daemon marks the resync with suppressPush', async () => { + const { bridge, daemon } = await authedDaemon(); + + daemon.emit('message', idleStateEvent({ + decisionReason: SESSION_STATE_DECISION_REASON_SERVER_LINK_RESYNC, + [TIMELINE_SUPPRESS_PUSH_FIELD]: true, + })); + + expect(pendingIdlePushCount(bridge)).toBe(0); + }); + + it('STILL pushes for a genuine live idle — the guard must not silence real completions', async () => { + // Without this the fix could "work" by breaking notifications entirely. + const { bridge, daemon } = await authedDaemon(); + + daemon.emit('message', idleStateEvent({ decisionReason: 'activity_reconciler_clear' })); + + expect(pendingIdlePushCount(bridge)).toBe(1); + }); + + it('does not fan one push per session when many sessions resync at once', async () => { + // The storm's actual shape: every session re-stated in one burst. + const { bridge, daemon } = await authedDaemon(); + + for (let i = 0; i < 25; i++) { + daemon.emit('message', JSON.stringify({ + type: TIMELINE_MESSAGES.EVENT, + event: { + eventId: `evt-${i}`, + sessionId: `deck_resyncpush_w${i}`, + ts: Date.now(), + seq: i, + epoch: 1, + type: 'session.state', + payload: { state: 'idle', decisionReason: SESSION_STATE_DECISION_REASON_SERVER_LINK_RESYNC }, + }, + })); + } + + expect(pendingIdlePushCount(bridge)).toBe(0); + }); +}); diff --git a/src/agent/session-manager.ts b/src/agent/session-manager.ts index bb9dcefed..2650c756a 100644 --- a/src/agent/session-manager.ts +++ b/src/agent/session-manager.ts @@ -1587,7 +1587,17 @@ export function resyncTransportSessionStatesAfterLinkRestore( clearSource: 'server-link-resync', queueReason: 'server_link_resync', }); - timelineEmitter.emit(sessionName, 'session.state', built.payload, { + // A resync RE-STATES what the daemon already knew; it is never a fresh + // completion, so it must never raise a push. Without this every link + // restore fanned one "Task complete" notification per idle session: + // observed live on the 43 deployment as ~90 dispatches inside 500ms, + // 276ms after four daemons re-authenticated following a gateway blip. The + // server's age guard cannot catch these because the events are newly + // minted (ts = now) even though the state they describe is old. + timelineEmitter.emit(sessionName, 'session.state', { + ...built.payload, + [TIMELINE_SUPPRESS_PUSH_FIELD]: true, + }, { source: 'daemon', confidence: 'high', eventId: `transport-state-resync:${sessionName}:${epoch}`, diff --git a/test/agent/session-manager-state-resync.test.ts b/test/agent/session-manager-state-resync.test.ts index 1cd9f1218..99438c817 100644 --- a/test/agent/session-manager-state-resync.test.ts +++ b/test/agent/session-manager-state-resync.test.ts @@ -28,6 +28,7 @@ import { setSessionPersistCallback, } from '../../src/agent/session-manager.js'; import { timelineEmitter } from '../../src/daemon/timeline-emitter.js'; +import { TIMELINE_SUPPRESS_PUSH_FIELD } from '../../shared/push-notifications.js'; import { SESSION_STATE_DECISION_REASON_SERVER_LINK_RESYNC, isAuthoritativeIdlePayloadShape, @@ -189,6 +190,29 @@ describe('server-link resync payload marking', () => { } }); + it('marks the resync idle as push-suppressed so a link restore cannot fan notifications', () => { + // The 43-deployment storm: a gateway blip dropped four daemons' sockets, all + // reconnected, and 276ms later ~90 push dispatches went out inside 500ms — + // one "Task complete" per idle session, quoting long-finished work. The + // server's PUSH_TIMELINE_EVENT_MAX_AGE_MS guard cannot catch these because a + // resync event is newly minted even though the state it reports is old, so + // the suppression has to be explicit. + // Local spy: this describe block has no shared one (see the sibling test). + const emitSpy = vi.spyOn(timelineEmitter, 'emit').mockImplementation(() => null as never); + try { + resyncTransportSessionStatesAfterLinkRestore([ + [IDLE_SESSION, makeRuntime({ status: 'idle' })], + ]); + const [, , payload] = emitSpy.mock.calls[0] as unknown as [string, string, Record]; + expect(payload[TIMELINE_SUPPRESS_PUSH_FIELD]).toBe(true); + // Suppressing the push must NOT weaken the idle: the whole point of the + // resync is to un-stick a "working" footer, which a weak idle cannot do. + expect(isAuthoritativeIdlePayloadShape(payload)).toBe(true); + } finally { + emitSpy.mockRestore(); + } + }); + it('does not mark an ordinary live idle as a resync', () => { // The live onStatusChange path uses `activity_reconciler_clear`; only that // one may drive drainQueue / notifySessionIdle. From 6491e3379dbc93c4a1a5853ef713217a8ae6dead Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sun, 26 Jul 2026 06:30:12 +0800 Subject: [PATCH 17/40] Route handle persistence to the store outside tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The path resolver returned ~/.imcodes/memory-short-refs.json for any process that was not a test, which inverted the intended routing: tests took the store path while real daemons kept writing the JSON file. The migration off that file — and the failure reporting added alongside it — therefore never executed anywhere it mattered, leaving the original bug fully intact in production: a full disk still silently dropped handles, which then died on the next restart. Only an explicit IMCODES_MEMORY_SHORT_REF_PATH selects the file now. Handles are a pure function of the id and re-register on their next injection, so the legacy file needs no import. Report the two remaining silent paths as well — a failed JSON write, and a failed warm-load whose 0 was indistinguishable from "nothing stored" — under a metric named for what it measures rather than reusing the startup one, since registration also happens during injection and MCP search. The regression test drops VITEST/NODE_ENV before asserting: with the test markers left in place, the old resolver takes the store branch too and the test passes against the very bug it is meant to catch. Co-Authored-By: Claude Opus 4.8 --- src/context/memory-short-ref.ts | 55 ++++++++++++----- .../memory-short-ref-persist-failure.test.ts | 59 +++++++++++++++++-- 2 files changed, 94 insertions(+), 20 deletions(-) diff --git a/src/context/memory-short-ref.ts b/src/context/memory-short-ref.ts index 259315968..95773c665 100644 --- a/src/context/memory-short-ref.ts +++ b/src/context/memory-short-ref.ts @@ -5,8 +5,7 @@ import { getContextStoreClient } from '../store/context-store-worker-client.js'; import { warnOncePerHour } from '../util/rate-limited-warn.js'; import { incrementCounter } from '../util/metrics.js'; import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; -import { homedir } from 'node:os'; -import { dirname, join } from 'node:path'; +import { dirname } from 'node:path'; export type MemoryShortRefKind = 'projection' | 'observation'; @@ -80,11 +79,22 @@ function pruneShortRefs(): void { } } +/** + * JSON file target, or undefined to use the context store. + * + * ONLY an explicit `IMCODES_MEMORY_SHORT_REF_PATH` selects the file. The default + * — including production — is the store. + * + * This used to fall back to `~/.imcodes/memory-short-refs.json` whenever the + * process wasn't a test, which inverted the intended routing: tests took the + * store path while real daemons kept writing the JSON file, so the migration off + * that file (and the failure reporting added alongside it) never executed + * anywhere it mattered. Handles are a pure function of the id and re-register on + * the next injection, so no import of the legacy file is needed. + */ function shortRefStorePath(): string | undefined { const configured = process.env.IMCODES_MEMORY_SHORT_REF_PATH?.trim(); - if (configured) return configured; - if (process.env.VITEST === 'true' || process.env.NODE_ENV === 'test') return undefined; - return join(homedir(), '.imcodes', 'memory-short-refs.json'); + return configured ? configured : undefined; } function isMemoryShortRefKind(value: unknown): value is MemoryShortRefKind { @@ -157,11 +167,27 @@ function persistShortRefsToFile(): void { schemaVersion: SHORT_REF_SCHEMA_VERSION, entries: entries.slice(0, MAX_SHORT_REF_ENTRIES), }), 'utf8'); - } catch { - // Ref persistence is a convenience cache. Do not break memory search/source reads. + } catch (error) { + // Non-fatal, but never silent: an unwritable file is exactly how handles + // used to disappear without a trace. + reportShortRefFailure('persist_file', error, { path }); } } +/** + * A handle that fails to persist still resolves in this process but dies on the + * next restart, so every such failure gets a fixed-cardinality counter and an + * hourly-throttled warning. `stage` is a constant per call site — error text and + * volatile fields stay in the warning payload, never in metric labels. + */ +function reportShortRefFailure(stage: 'persist_store' | 'persist_file' | 'warm_load', error: unknown, extra: Record = {}): void { + incrementCounter('mem.short_ref.persist_failure', { stage }); + warnOncePerHour(`mem.short_ref.persist_failure.${stage}`, { + ...extra, + error: error instanceof Error ? error.message : String(error), + }); +} + /** * Persist the handles just registered. * @@ -194,13 +220,8 @@ function persistShortRefs(touched: ReadonlyArray<{ ref: string; entry: MemorySho // Non-fatal for this process — the in-memory index still resolves, handles // are a pure function of the id, and sourceLookup full ids stay canonical. // But it IS the failure this change exists to make visible: handles that - // never land stop resolving after a restart. Surface it rather than - // repeating the swallowed-write bug in a new location. - incrementCounter('mem.startup.silent_failure', { source: 'memory-short-ref-upsert' }); - warnOncePerHour('mem.startup.silent_failure.memory-short-ref-upsert', { - error: error instanceof Error ? error.message : String(error), - rows: rows.length, - }); + // never land stop resolving after a restart. + reportShortRefFailure('persist_store', error, { rows: rows.length }); }); } @@ -238,7 +259,11 @@ export async function loadMemoryShortRefsFromStore(): Promise { pruneShortRefs(); persistedLoaded = true; return loaded; - } catch { + } catch (error) { + // A failed warm-load means every handle issued before this restart is + // unresolvable for the life of the process — report rather than return a + // zero that is indistinguishable from "nothing stored". + reportShortRefFailure('warm_load', error); return 0; } } diff --git a/test/context/memory-short-ref-persist-failure.test.ts b/test/context/memory-short-ref-persist-failure.test.ts index 0b77f0651..f11052975 100644 --- a/test/context/memory-short-ref-persist-failure.test.ts +++ b/test/context/memory-short-ref-persist-failure.test.ts @@ -12,7 +12,11 @@ vi.mock('../../src/store/context-store-worker-client.js', () => ({ vi.mock('../../src/util/metrics.js', () => ({ incrementCounter: incrementCounterMock })); vi.mock('../../src/util/rate-limited-warn.js', () => ({ warnOncePerHour: warnOncePerHourMock })); -import { registerMemoryShortRefs, resetMemoryShortRefsForTests } from '../../src/context/memory-short-ref.js'; +import { + registerMemoryShortRefs, + resetMemoryShortRefsForTests, + resolveMemoryShortRef, +} from '../../src/context/memory-short-ref.js'; /** * The whole point of moving handles off the JSON file was that its write errors @@ -51,20 +55,65 @@ describe('memory short refs — persistence failures stay observable', () => { await vi.waitFor(() => expect(incrementCounterMock).toHaveBeenCalled()); expect(incrementCounterMock).toHaveBeenCalledWith( - 'mem.startup.silent_failure', - { source: 'memory-short-ref-upsert' }, + 'mem.short_ref.persist_failure', + { stage: 'persist_store' }, ); expect(warnOncePerHourMock).toHaveBeenCalledWith( - 'mem.startup.silent_failure.memory-short-ref-upsert', + 'mem.short_ref.persist_failure.persist_store', expect.objectContaining({ error: 'context_store_unavailable' }), ); }); + it('routes a production process to the store, not a JSON file', async () => { + // Regression: the path resolver returned ~/.imcodes/memory-short-refs.json + // for any process that wasn't a test, which inverted the routing — real + // daemons kept writing the JSON file while only tests exercised the store, + // so the migration and its failure reporting never ran where they mattered. + // + // The ambient VITEST flag makes a plain assertion here useless: it sends the + // OLD code down the store branch too, and the test passes either way. Drop + // the test markers so this exercises the real production route. + const priorVitest = process.env.VITEST; + const priorNodeEnv = process.env.NODE_ENV; + delete process.env.VITEST; + process.env.NODE_ENV = 'production'; + try { + runMock.mockResolvedValue(1); + registerMemoryShortRefs([entry]); + await vi.waitFor(() => expect(runMock).toHaveBeenCalled()); + expect(runMock).toHaveBeenCalledWith('upsertMemoryShortRefs', [ + expect.arrayContaining([expect.objectContaining({ id: entry.id, kind: 'projection' })]), + ]); + } finally { + if (priorVitest === undefined) delete process.env.VITEST; + else process.env.VITEST = priorVitest; + if (priorNodeEnv === undefined) delete process.env.NODE_ENV; + else process.env.NODE_ENV = priorNodeEnv; + } + }); + it('keeps registration non-fatal and in-memory resolution working when the write fails', async () => { runMock.mockRejectedValue(new Error('SQLITE_BUSY')); // Registration must not throw: the caller is a synchronous render path. - expect(() => registerMemoryShortRefs([entry])).not.toThrow(); + let refs: string[] = []; + expect(() => { refs = registerMemoryShortRefs([entry]); }).not.toThrow(); await vi.waitFor(() => expect(warnOncePerHourMock).toHaveBeenCalled()); + // The unpersisted handle still resolves for the life of this process. + expect(resolveMemoryShortRef(refs[0]!, entry.namespace)).toMatchObject({ id: entry.id }); + }); + + it('reports a failed warm-load instead of returning an ambiguous zero', async () => { + // A warm-load failure leaves every handle issued before this restart + // unresolvable for the whole process, and `0` alone is indistinguishable + // from "nothing was stored". + runMock.mockRejectedValue(new Error('context_store_unavailable')); + const { loadMemoryShortRefsFromStore } = await import('../../src/context/memory-short-ref.js'); + + await expect(loadMemoryShortRefsFromStore()).resolves.toBe(0); + expect(incrementCounterMock).toHaveBeenCalledWith( + 'mem.short_ref.persist_failure', + { stage: 'warm_load' }, + ); }); it('stays quiet on a successful write', async () => { From b0bb4e8bead38af0b2145ad88430a49c2e9c76dd Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sun, 26 Jul 2026 06:45:38 +0800 Subject: [PATCH 18/40] Keep the routing regression test out of the real home directory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test strips VITEST/NODE_ENV so it exercises the real production route. That is what makes it able to catch the routing regression — but it also means that if the regression returns, the file branch reactivates and writes the runner's actual ~/.imcodes/memory-short-refs.json before the assertion fires. A red test must not touch real user data, and the true-negative procedure for this file triggers exactly that path. Stub the filesystem writes so the failure state is hermetic, and assert positively that no file is written when no path override is set. Verified both ways: restoring the old resolver still turns this one test red, and the real home file is byte-identical before and after that red run. Report two further silent paths while here — a JSON cache that exists but cannot be read or parsed (ENOENT stays quiet, since a missing file is the normal first run), and a warm-load response that violates its contract — both of which stranded handles with no signal. Co-Authored-By: Claude Opus 4.8 --- src/context/memory-short-ref.ts | 18 ++++++++++++++---- .../memory-short-ref-persist-failure.test.ts | 18 +++++++++++++++++- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/context/memory-short-ref.ts b/src/context/memory-short-ref.ts index 95773c665..565f23c43 100644 --- a/src/context/memory-short-ref.ts +++ b/src/context/memory-short-ref.ts @@ -148,8 +148,12 @@ function ensurePersistedLoaded(): void { } } pruneShortRefs(); - } catch { - // Missing or corrupt cache is non-fatal: sourceLookup full ids remain canonical. + } catch (error) { + // A missing file is the normal first run. Anything else — unreadable, or + // corrupt JSON — silently strands every handle written before this start, + // which is the same invisible failure this module exists to avoid. + if ((error as NodeJS.ErrnoException | null)?.code === 'ENOENT') return; + reportShortRefFailure('load_file', error, { path }); } } @@ -180,7 +184,7 @@ function persistShortRefsToFile(): void { * hourly-throttled warning. `stage` is a constant per call site — error text and * volatile fields stay in the warning payload, never in metric labels. */ -function reportShortRefFailure(stage: 'persist_store' | 'persist_file' | 'warm_load', error: unknown, extra: Record = {}): void { +function reportShortRefFailure(stage: 'persist_store' | 'persist_file' | 'warm_load' | 'load_file', error: unknown, extra: Record = {}): void { incrementCounter('mem.short_ref.persist_failure', { stage }); warnOncePerHour(`mem.short_ref.persist_failure.${stage}`, { ...extra, @@ -237,7 +241,13 @@ export async function loadMemoryShortRefsFromStore(): Promise { try { const rows = await getContextStoreClient() .run>>('listMemoryShortRefs', [MAX_SHORT_REF_ENTRIES]); - if (!Array.isArray(rows)) return 0; + if (!Array.isArray(rows)) { + // Contract violation rather than an ordinary failure, but the effect is + // the same as a failed load — report instead of returning a 0 that reads + // as "nothing stored". + reportShortRefFailure('warm_load', new Error('listMemoryShortRefs returned a non-array response')); + return 0; + } let loaded = 0; for (const row of rows) { const normalized = normalizeEntry({ diff --git a/test/context/memory-short-ref-persist-failure.test.ts b/test/context/memory-short-ref-persist-failure.test.ts index f11052975..be5132a37 100644 --- a/test/context/memory-short-ref-persist-failure.test.ts +++ b/test/context/memory-short-ref-persist-failure.test.ts @@ -1,9 +1,23 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -const { runMock, incrementCounterMock, warnOncePerHourMock } = vi.hoisted(() => ({ +const { runMock, incrementCounterMock, warnOncePerHourMock, writeFileSyncMock, mkdirSyncMock } = vi.hoisted(() => ({ runMock: vi.fn(), incrementCounterMock: vi.fn(), warnOncePerHourMock: vi.fn(), + writeFileSyncMock: vi.fn(), + mkdirSyncMock: vi.fn(), +})); + +// The production-route test below strips VITEST/NODE_ENV so it exercises the +// real routing. Should the routing regress, the file branch would reactivate and +// write to the runner's actual home directory — a red test must never touch a +// developer's real data, and the true-negative procedure for this file would +// trigger exactly that. Stub the writes so the failure state is hermetic, which +// also lets the test assert positively that no file was written. +vi.mock('node:fs', () => ({ + writeFileSync: writeFileSyncMock, + mkdirSync: mkdirSyncMock, + readFileSync: () => { throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }); }, })); vi.mock('../../src/store/context-store-worker-client.js', () => ({ @@ -84,6 +98,8 @@ describe('memory short refs — persistence failures stay observable', () => { expect(runMock).toHaveBeenCalledWith('upsertMemoryShortRefs', [ expect.arrayContaining([expect.objectContaining({ id: entry.id, kind: 'projection' })]), ]); + // The file branch must stay dormant without an explicit path override. + expect(writeFileSyncMock).not.toHaveBeenCalled(); } finally { if (priorVitest === undefined) delete process.env.VITEST; else process.env.VITEST = priorVitest; From 6cc67e120fe4303f1ba40a2101d27751713857ac Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sun, 26 Jul 2026 06:49:49 +0800 Subject: [PATCH 19/40] Import handles from the retired JSON cache during warm-load MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Routing handle persistence to the store orphaned the file it used to write. The handles in it are still valid — this machine's is 337KB — so they would otherwise stay unresolvable until each memory happened to be injected again, a user-visible gap for anyone who upgrades across those versions. Carry them into the store on the first warm-load. The file is strictly a read-only fallback now: never written again, and never deleted, so it stays available as a manual recovery point. A counter records each import, so the fallback can be dropped once it stops finding anything. The legacy path is overridable, and the store tests pin it at a nonexistent path — otherwise they would read whatever the developer running them happens to have in their real home directory. Co-Authored-By: Claude Opus 4.8 --- src/context/memory-short-ref.ts | 55 ++++++++++++++++++++- test/context/memory-short-ref-store.test.ts | 36 ++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) diff --git a/src/context/memory-short-ref.ts b/src/context/memory-short-ref.ts index 565f23c43..f5edde405 100644 --- a/src/context/memory-short-ref.ts +++ b/src/context/memory-short-ref.ts @@ -5,7 +5,8 @@ import { getContextStoreClient } from '../store/context-store-worker-client.js'; import { warnOncePerHour } from '../util/rate-limited-warn.js'; import { incrementCounter } from '../util/metrics.js'; import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; -import { dirname } from 'node:path'; +import { homedir } from 'node:os'; +import { dirname, join } from 'node:path'; export type MemoryShortRefKind = 'projection' | 'observation'; @@ -97,6 +98,53 @@ function shortRefStorePath(): string | undefined { return configured ? configured : undefined; } +/** + * Where the retired JSON cache lives. Read-only: handles are only ever written + * to the store now. Overridable so tests never read the developer's real file. + */ +function legacyShortRefFilePath(): string { + const configured = process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH?.trim(); + return configured ? configured : join(homedir(), '.imcodes', 'memory-short-refs.json'); +} + +/** + * One-way bridge off the retired JSON cache. + * + * Handles written while persistence still went to a file are valid, so import + * them into the store on the first warm-load rather than stranding them until + * their memory happens to be injected again. Strictly read-only, and the file is + * left in place as a manual recovery point — it is simply never written again. + * The counter shows when this stops finding anything, which is when the file can + * be dropped for good. + */ +function importLegacyShortRefFile(): Array<{ ref: string; entry: MemoryShortRefEntry }> { + const path = legacyShortRefFilePath(); + const imported: Array<{ ref: string; entry: MemoryShortRefEntry }> = []; + try { + const parsed = JSON.parse(readFileSync(path, 'utf8')) as { schemaVersion?: unknown; entries?: unknown[] }; + // Older derivations produced handles that denote different records now. + if (parsed.schemaVersion !== SHORT_REF_SCHEMA_VERSION || !Array.isArray(parsed.entries)) return imported; + for (const raw of parsed.entries) { + const normalized = normalizeEntry(raw); + if (!normalized) continue; + const bucket = entriesByRef.get(normalized.ref) ?? []; + if (bucket.some((entry) => entry.kind === normalized.entry.kind + && entry.id === normalized.entry.id + && sameNamespace(entry.namespace, normalized.entry.namespace))) continue; + bucket.push(normalized.entry); + entriesByRef.set(normalized.ref, bucket); + imported.push(normalized); + } + if (imported.length > 0) incrementCounter('mem.short_ref.legacy_import', { source: 'json_file' }); + } catch (error) { + // A missing file is the expected steady state once everyone has migrated. + if ((error as NodeJS.ErrnoException | null)?.code !== 'ENOENT') { + reportShortRefFailure('load_file', error, { path, legacy: true }); + } + } + return imported; +} + function isMemoryShortRefKind(value: unknown): value is MemoryShortRefKind { return value === 'projection' || value === 'observation'; } @@ -266,6 +314,11 @@ export async function loadMemoryShortRefsFromStore(): Promise { entriesByRef.set(normalized.ref, bucket); loaded += 1; } + // Carry over anything still only in the retired JSON cache, and write it to + // the store so the next start no longer depends on that file. + const legacy = importLegacyShortRefFile(); + if (legacy.length > 0) persistShortRefs(legacy); + loaded += legacy.length; pruneShortRefs(); persistedLoaded = true; return loaded; diff --git a/test/context/memory-short-ref-store.test.ts b/test/context/memory-short-ref-store.test.ts index 9d174417e..85e081206 100644 --- a/test/context/memory-short-ref-store.test.ts +++ b/test/context/memory-short-ref-store.test.ts @@ -1,8 +1,11 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { resetContextStoreClientForTests } from '../../src/store/context-store-worker-client.js'; import { listMemoryShortRefs } from '../../src/store/context-store.js'; +import { readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; import { loadMemoryShortRefsFromStore, + makeMemoryShortRef, registerMemoryShortRefs, resetMemoryShortRefsForTests, resolveMemoryShortRef, @@ -19,11 +22,16 @@ import { cleanupIsolatedSharedContextDb, createIsolatedSharedContextDb } from '. describe('memory short refs — durable store persistence', () => { let tempDir: string; let priorPath: string | undefined; + let priorLegacyPath: string | undefined; beforeEach(async () => { // Unset so the store (not the JSON file) is the persistence target. priorPath = process.env.IMCODES_MEMORY_SHORT_REF_PATH; delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; + // Warm-load also imports the retired JSON cache. Point it at a path that + // does not exist so these assertions never read the developer's real file. + priorLegacyPath = process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH; + process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = '/nonexistent/imcodes-test/legacy-short-refs.json'; tempDir = await createIsolatedSharedContextDb('memory-short-ref-store'); resetMemoryShortRefsForTests(); }); @@ -33,6 +41,8 @@ describe('memory short refs — durable store persistence', () => { resetContextStoreClientForTests(); if (priorPath === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; else process.env.IMCODES_MEMORY_SHORT_REF_PATH = priorPath; + if (priorLegacyPath === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = priorLegacyPath; await cleanupIsolatedSharedContextDb(tempDir); }); @@ -74,6 +84,32 @@ describe('memory short refs — durable store persistence', () => { expect(listMemoryShortRefs().length).toBe(afterFirst); }); + it('imports handles from the retired JSON cache and moves them into the store', async () => { + // The JSON file is a read-only fallback during the migration: handles it + // still holds are valid, so carry them over rather than stranding them until + // their memory is injected again. The file itself is never written or + // deleted — it just stops being needed. + const legacyPath = join(tempDir, 'legacy-short-refs.json'); + process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = legacyPath; + const legacyId = '45663d84-d3bb-4f9a-bf65-eba630b45d66'; + const legacyRef = makeMemoryShortRef('projection', legacyId); + writeFileSync(legacyPath, JSON.stringify({ + schemaVersion: 2, + entries: [{ ref: legacyRef, kind: 'projection', id: legacyId, namespace, lastSeenAt: 123 }], + }), 'utf8'); + + const before = readFileSync(legacyPath, 'utf8'); + const loaded = await loadMemoryShortRefsFromStore(); + expect(loaded).toBeGreaterThanOrEqual(1); + expect(resolveMemoryShortRef(legacyRef, namespace)).toMatchObject({ id: legacyId }); + + // Imported handles land in the store, so the next start no longer needs the file. + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(listMemoryShortRefs().some((row) => row.id === legacyId)).toBe(true); + // And the file is left exactly as it was. + expect(readFileSync(legacyPath, 'utf8')).toBe(before); + }); + it('keeps cross-namespace isolation after a store warm-load', async () => { const [ref] = registerMemoryShortRefs([ { kind: 'projection', id: 'cccccccccc-1111-2222-3333-444444444444', namespace }, From da7e077a5b7f53a870ec337b60fcb7ef0b0b3361 Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sun, 26 Jul 2026 06:56:51 +0800 Subject: [PATCH 20/40] Refuse to resolve a colliding handle rather than guessing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two records deriving the same handle in one namespace resolved to whichever was seen last, so the caller received a real but wrong memory with no signal — the precise failure the handle scheme exists to prevent, reachable again the moment a digest collides. That guess made sense when the derivation collided routinely: the retired cache on this machine has one handle covering 40 distinct records. It does not survive the switch to a 65-bit digest, where a same-namespace collision should never occur — so treat it as unresolvable and report it. Callers then fall back to search rather than acting on the wrong record. Co-Authored-By: Claude Opus 4.8 --- src/context/memory-short-ref.ts | 24 ++++++++++++++++++------ test/context/memory-short-ref.test.ts | 18 ++++++++++-------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/context/memory-short-ref.ts b/src/context/memory-short-ref.ts index f5edde405..a3b328e2b 100644 --- a/src/context/memory-short-ref.ts +++ b/src/context/memory-short-ref.ts @@ -58,10 +58,6 @@ function sameNamespace(a: ContextNamespace | undefined, b: ContextNamespace | un return namespaceKey(a) === namespaceKey(b); } -function newestEntry(entries: MemoryShortRefEntry[]): MemoryShortRefEntry | undefined { - return [...entries].sort((a, b) => (b.lastSeenAt ?? 0) - (a.lastSeenAt ?? 0))[0]; -} - function pruneShortRefs(): void { let count = 0; for (const bucket of entriesByRef.values()) count += bucket.length; @@ -404,8 +400,24 @@ export function resolveMemoryShortRef(ref: string, namespace?: ContextNamespace) ensurePersistedLoaded(); const bucket = entriesByRef.get(normalizeRef(ref)); if (!bucket || bucket.length === 0) return undefined; - const exact = namespace ? newestEntry(bucket.filter((entry) => sameNamespace(entry.namespace, namespace))) : undefined; - if (exact) return exact; + if (namespace) { + const sameNs = bucket.filter((entry) => sameNamespace(entry.namespace, namespace)); + // Registration keys entries by (kind, id, namespace), so more than one + // survivor here means two different records genuinely derived the same + // handle. Picking the newest would hand back the wrong memory with no + // signal — the exact failure this handle scheme exists to prevent. At 65 + // bits this should never happen, so treat it as unresolvable and report it. + if (sameNs.length > 1) { + incrementCounter('mem.short_ref.collision', { kind: sameNs[0]!.kind }); + warnOncePerHour('mem.short_ref.collision', { + ref: normalizeRef(ref), + ids: sameNs.map((entry) => entry.id).slice(0, 4), + }); + return undefined; + } + const exact = sameNs[0]; + if (exact) return exact; + } // Cross-namespace isolation: a handle registered under another namespace must // NOT resolve here, even when it is the only entry for this ref. Callers other // than get_memory_sources (archive/delete/update) also resolve refs, so this diff --git a/test/context/memory-short-ref.test.ts b/test/context/memory-short-ref.test.ts index 9ca257e55..f2bf0feef 100644 --- a/test/context/memory-short-ref.test.ts +++ b/test/context/memory-short-ref.test.ts @@ -110,7 +110,12 @@ describe('memory short refs', () => { }); }); - it('resolves same-namespace short-ref conflicts to the newest seen entry', () => { + it('refuses to resolve a same-namespace collision instead of guessing the newest', () => { + // Two different records deriving one handle used to resolve to whichever was + // seen last, handing back the wrong memory with no signal. That was tolerable + // when the derivation collided routinely; with a 65-bit digest a same- + // namespace collision should never occur, so it is treated as unresolvable + // (and reported) rather than silently answered with the wrong record. const namespace = { scope: 'user_private' as const, userId: 'user-1', projectId: 'repo-1' }; const ref = 'obs:collide000000'; seedColliding(ref, [ @@ -118,14 +123,11 @@ describe('memory short refs', () => { { kind: 'observation', id: 'cccccccccc-9999-8888-7777-666666666666', namespace, lastSeenAt: 200 }, ]); - expect(resolveMemoryShortRef(ref, namespace)).toMatchObject({ - id: 'cccccccccc-9999-8888-7777-666666666666', - }); + expect(resolveMemoryShortRef(ref, namespace)).toBeUndefined(); + // Still unresolvable after a reload — the collision is a property of the + // data, not of this process's in-memory state. reloadMemoryShortRefsForTests(); - - expect(resolveMemoryShortRef(ref, namespace)).toMatchObject({ - id: 'cccccccccc-9999-8888-7777-666666666666', - }); + expect(resolveMemoryShortRef(ref, namespace)).toBeUndefined(); }); }); From e8ca770cbbc5d579fb5441a7b1450db52c068734 Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sun, 26 Jul 2026 07:08:19 +0800 Subject: [PATCH 21/40] Return every record behind a colliding handle instead of refusing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refusing to resolve a collision kept the caller from acting on the wrong memory, but it also threw away information: the answer was in one of those records, and the reader is capable of telling which. Expand all of them and mark the response ambiguous, so it can never be mistaken for a single authoritative answer. Only read expansion does this. Callers that act on the result — archive, delete, update — still refuse, because a destructive operation must never run on a guess. Co-Authored-By: Claude Opus 4.8 --- shared/memory-mcp-contracts.ts | 2 ++ src/context/memory-short-ref.ts | 43 +++++++++++++++++-------- src/daemon/memory-mcp-tools.ts | 46 +++++++++++++++++++++++++-- test/context/memory-short-ref.test.ts | 19 +++++++++++ 4 files changed, 93 insertions(+), 17 deletions(-) diff --git a/shared/memory-mcp-contracts.ts b/shared/memory-mcp-contracts.ts index 181ae6112..cdcf98f49 100644 --- a/shared/memory-mcp-contracts.ts +++ b/shared/memory-mcp-contracts.ts @@ -284,6 +284,8 @@ export const MEMORY_MCP_TOOL_CONTRACTS: Readonly sameNamespace(entry.namespace, namespace)); + if (sameNs.length > 1) { + incrementCounter('mem.short_ref.collision', { kind: sameNs[0]!.kind }); + warnOncePerHour('mem.short_ref.collision', { + ref: normalizeRef(ref), + ids: sameNs.map((entry) => entry.id).slice(0, 4), + }); + } + return sameNs; +} + export function resolveMemoryShortRef(ref: string, namespace?: ContextNamespace): MemoryShortRefEntry | undefined { ensurePersistedLoaded(); const bucket = entriesByRef.get(normalizeRef(ref)); if (!bucket || bucket.length === 0) return undefined; if (namespace) { - const sameNs = bucket.filter((entry) => sameNamespace(entry.namespace, namespace)); - // Registration keys entries by (kind, id, namespace), so more than one - // survivor here means two different records genuinely derived the same - // handle. Picking the newest would hand back the wrong memory with no - // signal — the exact failure this handle scheme exists to prevent. At 65 - // bits this should never happen, so treat it as unresolvable and report it. - if (sameNs.length > 1) { - incrementCounter('mem.short_ref.collision', { kind: sameNs[0]!.kind }); - warnOncePerHour('mem.short_ref.collision', { - ref: normalizeRef(ref), - ids: sameNs.map((entry) => entry.id).slice(0, 4), - }); - return undefined; - } + const sameNs = resolveMemoryShortRefCandidates(ref, namespace); + // More than one survivor means two different records genuinely derived the + // same handle. Callers that ACT on the result (archive/delete/update) must + // never operate on a guess, so this strict accessor refuses. Read paths can + // use resolveMemoryShortRefCandidates and present the alternatives instead. + if (sameNs.length > 1) return undefined; const exact = sameNs[0]; if (exact) return exact; } diff --git a/src/daemon/memory-mcp-tools.ts b/src/daemon/memory-mcp-tools.ts index 8e6d5f96a..33eb076e2 100644 --- a/src/daemon/memory-mcp-tools.ts +++ b/src/daemon/memory-mcp-tools.ts @@ -94,7 +94,10 @@ import { getContextStoreClient } from '../store/context-store-worker-client.js'; import { listSessions as listStoredSessions, loadStore, type SessionRecord } from '../store/session-store.js'; import { dispatchDestroyExecutionClone, dispatchSendMessage, dispatchSendStop, listSendTargets, type SendMessageCloneRequest, type SendToolDeps } from './send-tool.js'; import { cronMcpCreate, cronMcpCreateSelf, cronMcpDelete, cronMcpList, cronMcpUpdate, cronMcpUpdateSelf, type CronMcpClientOptions } from './cron-mcp-client.js'; -import { registerMemoryShortRef, resolveMemoryShortRef } from '../context/memory-short-ref.js'; +import { registerMemoryShortRef, resolveMemoryShortRef, resolveMemoryShortRefCandidates } from '../context/memory-short-ref.js'; + +/** Upper bound on records expanded for one colliding handle. */ +const AMBIGUOUS_REF_CANDIDATE_CAP = 4; import { GitOriginRepositoryIdentityService } from '../agent/repository-identity-service.js'; import { ALIAS_MCP_TOOLS, toAliasMetadata, type AliasMcpToolName } from '../../shared/alias-types.js'; import { @@ -952,8 +955,45 @@ export function createMemoryMcpToolHandlers(caller: McpRuntimeCaller, deps: Memo } if (!projectId) return emptySources(); if (ref) { - const resolved = resolveMemoryShortRef(ref, scopedCaller.namespace); - if (!resolved) return { status: 'ok', ref, sourceEventCount: 0, sources: [] }; + const candidates = resolveMemoryShortRefCandidates(ref, scopedCaller.namespace); + if (candidates.length === 0) return { status: 'ok', ref, sourceEventCount: 0, sources: [] }; + // A digest collision (two records deriving one handle) should never + // happen at 65 bits. If it does, answering with one of them would be a + // silently wrong memory — but refusing outright throws away information + // the caller can use. Expand every candidate and let the caller judge, + // flagged so it is never mistaken for a single authoritative answer. + if (candidates.length > 1) { + const expanded = []; + for (const candidate of candidates.slice(0, AMBIGUOUS_REF_CANDIDATE_CAP)) { + const identity = candidate.kind === 'observation' + ? { kind: candidate.kind, observationId: candidate.id } + : { kind: candidate.kind, projectionId: candidate.id }; + try { + if (candidate.kind === 'observation') { + expanded.push({ ...identity, ...(await memoryGetSources({ observationId: candidate.id, kind: 'observation' }, scopedCaller)) }); + continue; + } + const candidateResult = await orchestrator(candidate.id, scopedCaller); + if (candidateResult.status === 'error') { + expanded.push({ ...identity, unavailable: candidateResult.message }); + continue; + } + const { status: _candidateStatus, ...candidatePayload } = candidateResult; + expanded.push({ ...identity, ...candidatePayload }); + } catch { + expanded.push({ ...identity, unavailable: 'expansion failed' }); + } + } + return { + status: 'ok', + ref, + ambiguousRef: true, + candidates: expanded, + sourceEventCount: 0, + sources: [], + }; + } + const resolved = candidates[0]!; if (kind && kind !== resolved.kind) { return error(MCP_ERROR_REASONS.VALIDATION_FAILED, 'source lookup kind does not match the supplied ref'); } diff --git a/test/context/memory-short-ref.test.ts b/test/context/memory-short-ref.test.ts index f2bf0feef..34fb2fa07 100644 --- a/test/context/memory-short-ref.test.ts +++ b/test/context/memory-short-ref.test.ts @@ -9,6 +9,7 @@ import { reloadMemoryShortRefsForTests, resetMemoryShortRefsForTests, resolveMemoryShortRef, + resolveMemoryShortRefCandidates, } from '../../src/context/memory-short-ref.js'; describe('memory short refs', () => { @@ -110,6 +111,24 @@ describe('memory short refs', () => { }); }); + it('surfaces every record behind a colliding handle so a reader can choose', () => { + // Refusing outright would throw away information the caller can use, so read + // paths get all matches; only callers that ACT on the result (see the strict + // accessor below) refuse. + const namespace = { scope: 'user_private' as const, userId: 'user-1', projectId: 'repo-1' }; + const ref = 'obs:collide000001'; + seedColliding(ref, [ + { kind: 'observation', id: 'aaaa1111-2222-3333-4444-555555555555', namespace, lastSeenAt: 100 }, + { kind: 'observation', id: 'bbbb9999-8888-7777-6666-666666666666', namespace, lastSeenAt: 200 }, + ]); + + const candidates = resolveMemoryShortRefCandidates(ref, namespace); + expect(candidates.map((entry) => entry.id).sort()).toEqual([ + 'aaaa1111-2222-3333-4444-555555555555', + 'bbbb9999-8888-7777-6666-666666666666', + ]); + }); + it('refuses to resolve a same-namespace collision instead of guessing the newest', () => { // Two different records deriving one handle used to resolve to whichever was // seen last, handing back the wrong memory with no signal. That was tolerable From 490635d13820c80670ea7192db940b995217534b Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sun, 26 Jul 2026 07:23:24 +0800 Subject: [PATCH 22/40] Close the ambiguous-handle contract and report discarded handles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ambiguous branch returned early, skipping the kind check the single-candidate path applies, so a request naming the wrong kind got candidates instead of a validation error. Enforce it against every candidate. Expansion is capped at four, yet the comment, the schema and the commit message all claimed every record was returned — a caller could stop looking while the answer sat in an omitted record. Report candidateCount and truncated, and describe the bound honestly. The tool description is what the model actually reads, and it still described only the old single-record shape. Say what an ambiguous reply means there, including that an empty is not "no memory". Warm-load discarded malformed rows one by one and returned zero, which is indistinguishable from an empty store while every earlier handle stops resolving; count those discards. Register the handle counters so they are collectable rather than living only in a map a restart clears, and log the legacy import, since that signal is what tells us the fallback can be retired. The two regressions are covered at the MCP handler, not the resolver — resolver-level tests could not see either. Co-Authored-By: Claude Opus 4.8 --- shared/memory-counters.ts | 7 +++ shared/memory-mcp-contracts.ts | 13 ++++-- src/context/memory-short-ref.ts | 38 ++++++++++++++-- src/daemon/memory-mcp-tools.ts | 11 +++++ .../memory-mcp-tools-schema-firewall.test.ts | 43 ++++++++++++++++++- 5 files changed, 104 insertions(+), 8 deletions(-) diff --git a/shared/memory-counters.ts b/shared/memory-counters.ts index 7b792825b..8391f149a 100644 --- a/shared/memory-counters.ts +++ b/shared/memory-counters.ts @@ -62,6 +62,13 @@ export const MEMORY_COUNTERS = [ 'mem.compression.admission_closed', 'mem.pinned_notes_overflow', 'mem.telemetry.buffer_overflow', + // Compact memory handles (ref → id). A handle that fails to persist, or a row + // discarded while loading, still resolves in-process but dies on the next + // restart — these have to be collectable, not just counted in memory. + 'mem.short_ref.persist_failure', + 'mem.short_ref.collision', + 'mem.short_ref.legacy_import', + 'mem.short_ref.discarded_row', ] as const; export type MemoryCounter = (typeof MEMORY_COUNTERS)[number]; diff --git a/shared/memory-mcp-contracts.ts b/shared/memory-mcp-contracts.ts index cdcf98f49..29c60b058 100644 --- a/shared/memory-mcp-contracts.ts +++ b/shared/memory-mcp-contracts.ts @@ -272,7 +272,7 @@ export const MEMORY_MCP_TOOL_CONTRACTS: Readonly 0) incrementCounter('mem.short_ref.legacy_import', { source: 'json_file' }); + if (imported.length > 0) { + incrementCounter('mem.short_ref.legacy_import', { source: 'json_file' }); + // The counter lives in an in-process map that a restart clears, so the + // "has this stopped finding anything yet" question needs a durable line + // in the log too — that is the signal for retiring the fallback. + logger.info({ imported: imported.length, path }, 'memory short-ref: imported handles from the retired JSON cache'); + } } catch (error) { // A missing file is the expected steady state once everyone has migrated. if ((error as NodeJS.ErrnoException | null)?.code !== 'ENOENT') { @@ -228,6 +235,14 @@ function persistShortRefsToFile(): void { * hourly-throttled warning. `stage` is a constant per call site — error text and * volatile fields stay in the warning payload, never in metric labels. */ +/** Rows dropped while loading are a silent form of handle loss, so count them + * under a fixed-cardinality source and warn once per hour. */ +function reportDiscardedShortRefRows(source: 'warm_load' | 'legacy_file' | 'json_file', discarded: number): void { + if (discarded <= 0) return; + incrementCounter('mem.short_ref.discarded_row', { source }); + warnOncePerHour(`mem.short_ref.discarded_row.${source}`, { discarded }); +} + function reportShortRefFailure(stage: 'persist_store' | 'persist_file' | 'warm_load' | 'load_file', error: unknown, extra: Record = {}): void { incrementCounter('mem.short_ref.persist_failure', { stage }); warnOncePerHour(`mem.short_ref.persist_failure.${stage}`, { @@ -293,6 +308,7 @@ export async function loadMemoryShortRefsFromStore(): Promise { return 0; } let loaded = 0; + let discarded = 0; for (const row of rows) { const normalized = normalizeEntry({ ref: row.ref, @@ -301,7 +317,7 @@ export async function loadMemoryShortRefsFromStore(): Promise { lastSeenAt: row.lastSeenAt, namespace: typeof row.namespaceJson === 'string' ? safeParseNamespace(row.namespaceJson) : undefined, }); - if (!normalized) continue; + if (!normalized) { discarded += 1; continue; } const bucket = entriesByRef.get(normalized.ref) ?? []; if (bucket.some((entry) => entry.kind === normalized.entry.kind && entry.id === normalized.entry.id @@ -310,6 +326,10 @@ export async function loadMemoryShortRefsFromStore(): Promise { entriesByRef.set(normalized.ref, bucket); loaded += 1; } + // A well-formed array of malformed rows loads nothing and leaves every + // earlier handle unresolvable — indistinguishable from an empty store + // unless the discards are reported. + reportDiscardedShortRefRows('warm_load', discarded); // Carry over anything still only in the retired JSON cache, and write it to // the store so the next start no longer depends on that file. const legacy = importLegacyShortRefFile(); @@ -441,6 +461,16 @@ export function resolveMemoryShortRef(ref: string, namespace?: ContextNamespace) return bucket.length === 1 ? bucket[0] : undefined; } +/** + * Force several records onto one handle. A 65-bit digest collision cannot be + * produced by registering real ids, so the ambiguous-handle paths would + * otherwise be untestable above the resolver. + */ +export function seedMemoryShortRefCollisionForTests(ref: string, entries: readonly MemoryShortRefEntry[]): void { + ensurePersistedLoaded(); + entriesByRef.set(normalizeRef(ref), entries.map((entry) => ({ ...entry, lastSeenAt: entry.lastSeenAt ?? Date.now() }))); +} + export function resetMemoryShortRefsForTests(): void { entriesByRef.clear(); persistedLoaded = true; diff --git a/src/daemon/memory-mcp-tools.ts b/src/daemon/memory-mcp-tools.ts index 33eb076e2..1533286d9 100644 --- a/src/daemon/memory-mcp-tools.ts +++ b/src/daemon/memory-mcp-tools.ts @@ -963,6 +963,12 @@ export function createMemoryMcpToolHandlers(caller: McpRuntimeCaller, deps: Memo // the caller can use. Expand every candidate and let the caller judge, // flagged so it is never mistaken for a single authoritative answer. if (candidates.length > 1) { + // The single-candidate path below rejects a kind that disagrees with + // the ref; the ambiguous path must enforce the same contract rather + // than quietly ignoring the caller's constraint. + if (kind && candidates.some((candidate) => candidate.kind !== kind)) { + return error(MCP_ERROR_REASONS.VALIDATION_FAILED, 'source lookup kind does not match the supplied ref'); + } const expanded = []; for (const candidate of candidates.slice(0, AMBIGUOUS_REF_CANDIDATE_CAP)) { const identity = candidate.kind === 'observation' @@ -988,6 +994,11 @@ export function createMemoryMcpToolHandlers(caller: McpRuntimeCaller, deps: Memo status: 'ok', ref, ambiguousRef: true, + // Expansion is bounded, so say how many records the handle actually + // covers. Reporting only the expanded subset while calling it every + // match would send the caller away believing it had seen them all. + candidateCount: candidates.length, + truncated: candidates.length > expanded.length, candidates: expanded, sourceEventCount: 0, sources: [], diff --git a/test/daemon/memory-mcp-tools-schema-firewall.test.ts b/test/daemon/memory-mcp-tools-schema-firewall.test.ts index be5435166..e202f9d46 100644 --- a/test/daemon/memory-mcp-tools-schema-firewall.test.ts +++ b/test/daemon/memory-mcp-tools-schema-firewall.test.ts @@ -7,7 +7,7 @@ import { MCP_ERROR_REASONS } from '../../shared/memory-mcp-errors.js'; import { MEMORY_MCP_DEGRADED_REASON } from '../../shared/memory-ws.js'; import { createMemoryMcpToolHandlers } from '../../src/daemon/memory-mcp-tools.js'; import type { McpRuntimeCaller } from '../../src/daemon/memory-mcp-caller.js'; -import { makeMemoryShortRef, registerMemoryShortRef, resetMemoryShortRefsForTests } from '../../src/context/memory-short-ref.js'; +import { makeMemoryShortRef, registerMemoryShortRef, resetMemoryShortRefsForTests, seedMemoryShortRefCollisionForTests } from '../../src/context/memory-short-ref.js'; import type { SessionRecord } from '../../src/store/session-store.js'; function caller(overrides: Partial = {}): McpRuntimeCaller { @@ -260,6 +260,47 @@ describe('memory MCP tool schema firewall', () => { expect(archiveMemory).toHaveBeenCalledWith('proj-ref'); }); + it('rejects a kind that disagrees with an ambiguous ref instead of expanding it', async () => { + // The single-candidate path validates the requested kind; the ambiguous + // path returned candidates regardless, silently dropping the caller's + // constraint. Resolver-level tests could not see this — it needs the handler. + const namespace: ContextNamespace = { scope: 'personal', userId: 'user-1', projectId: 'repo-1' }; + const ref = registerMemoryShortRef({ kind: 'projection', id: 'collide-a', namespace }); + registerMemoryShortRef({ kind: 'projection', id: 'collide-b', namespace }); + // Force both onto one handle so the ambiguous branch is exercised. + seedMemoryShortRefCollisionForTests(ref, [ + { kind: 'projection', id: 'collide-a', namespace }, + { kind: 'projection', id: 'collide-b', namespace }, + ]); + const getProcessedProjectionById = vi.fn(() => projection({ id: 'collide-a', namespace })); + const handlers = createMemoryMcpToolHandlers(caller({ namespace }), { + getProcessedProjectionById, + isMemoryFeatureEnabled: () => true, + }); + + await expect(handlers[MEMORY_MCP_TOOL_NAMES.GET_MEMORY_SOURCES]({ ref, kind: 'observation' })).resolves.toMatchObject({ + status: 'error', + reason: MCP_ERROR_REASONS.VALIDATION_FAILED, + }); + }); + + it('declares how many records an ambiguous ref covers when expansion is bounded', async () => { + // Expansion is capped, so a caller told it received "every match" would stop + // looking while the answer sat in an omitted record. + const namespace: ContextNamespace = { scope: 'personal', userId: 'user-1', projectId: 'repo-1' }; + const ids = ['c1', 'c2', 'c3', 'c4', 'c5']; + const ref = registerMemoryShortRef({ kind: 'projection', id: ids[0]!, namespace }); + seedMemoryShortRefCollisionForTests(ref, ids.map((id) => ({ kind: 'projection' as const, id, namespace }))); + const handlers = createMemoryMcpToolHandlers(caller({ namespace }), { + getProcessedProjectionById: vi.fn((id: string) => projection({ id, namespace })), + isMemoryFeatureEnabled: () => true, + }); + + const result = await handlers[MEMORY_MCP_TOOL_NAMES.GET_MEMORY_SOURCES]({ ref }) as Record; + expect(result).toMatchObject({ status: 'ok', ambiguousRef: true, candidateCount: 5, truncated: true }); + expect((result.candidates as unknown[]).length).toBeLessThan(5); + }); + it('short-circuits memory disabled gates before backend calls', async () => { const searchMemory = vi.fn(); const savePreference = vi.fn(async () => ({ status: 'ok' })); From daaae83c6c82fb7ffc342d22374da6ee0f0c9396 Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sun, 26 Jul 2026 07:39:38 +0800 Subject: [PATCH 23/40] Report every discarded handle row and stop overstating candidates Discard reporting covered only the store warm-load. The legacy import and the explicit JSON load skipped malformed rows one at a time in silence, and a row whose stored namespace could no longer be parsed was worse: it degraded to a namespace-less entry and counted as loaded, so it looked healthy while being unresolvable for every namespaced caller. Count all three sources, and drop a row whose namespace fails to parse rather than loading a crippled one. The tool description still told the model that candidates held every match while the implementation expanded at most four, so it could stop looking with the answer sitting in an omitted record. Say "up to four", and point at candidateCount / truncated. The bounded-expansion test injected only the projection getter, so the handler fell through to the real orchestrator and built a SQLite store, WAL and log in the runner's home directory; its sole assertion, candidates.length < 5, also passed on an empty array. Inject the orchestrator, assert the exact four ids and their expanded content, add an untruncated two-candidate case, and point handle persistence at a scratch file for the whole suite. Co-Authored-By: Claude Opus 4.8 --- shared/memory-mcp-contracts.ts | 4 +- src/context/memory-short-ref.ts | 19 +++++- .../memory-mcp-tools-schema-firewall.test.ts | 66 ++++++++++++++++++- 3 files changed, 81 insertions(+), 8 deletions(-) diff --git a/shared/memory-mcp-contracts.ts b/shared/memory-mcp-contracts.ts index 29c60b058..9a434b01a 100644 --- a/shared/memory-mcp-contracts.ts +++ b/shared/memory-mcp-contracts.ts @@ -272,7 +272,7 @@ export const MEMORY_MCP_TOOL_CONTRACTS: Readonly entry.kind === normalized.entry.kind && entry.id === normalized.entry.id @@ -132,6 +133,7 @@ function importLegacyShortRefFile(): Array<{ ref: string; entry: MemoryShortRefE entriesByRef.set(normalized.ref, bucket); imported.push(normalized); } + reportDiscardedShortRefRows('legacy_file', discarded); if (imported.length > 0) { incrementCounter('mem.short_ref.legacy_import', { source: 'json_file' }); // The counter lives in an in-process map that a restart clears, so the @@ -187,9 +189,10 @@ function ensurePersistedLoaded(): void { // resolve to the wrong memory. Drop the whole file and re-register lazily. if (parsed.schemaVersion !== SHORT_REF_SCHEMA_VERSION) return; if (!Array.isArray(parsed.entries)) return; + let discarded = 0; for (const raw of parsed.entries) { const normalized = normalizeEntry(raw); - if (!normalized) continue; + if (!normalized) { discarded += 1; continue; } const bucket = entriesByRef.get(normalized.ref) ?? []; if (!bucket.some((entry) => entry.kind === normalized.entry.kind && entry.id === normalized.entry.id @@ -198,6 +201,7 @@ function ensurePersistedLoaded(): void { entriesByRef.set(normalized.ref, bucket); } } + reportDiscardedShortRefRows('json_file', discarded); pruneShortRefs(); } catch (error) { // A missing file is the normal first run. Anything else — unreadable, or @@ -310,12 +314,21 @@ export async function loadMemoryShortRefsFromStore(): Promise { let loaded = 0; let discarded = 0; for (const row of rows) { + // A row that stored a namespace but can no longer produce one must be + // dropped, not loaded namespace-less: the entry would look healthy while + // being unresolvable for every namespaced caller — silent handle loss + // wearing the shape of a successful load. + const storedNamespaceJson = typeof row.namespaceJson === 'string' && row.namespaceJson.trim() + ? row.namespaceJson + : undefined; + const namespace = storedNamespaceJson ? safeParseNamespace(storedNamespaceJson) : undefined; + if (storedNamespaceJson && !isContextNamespace(namespace)) { discarded += 1; continue; } const normalized = normalizeEntry({ ref: row.ref, kind: row.kind, id: row.id, lastSeenAt: row.lastSeenAt, - namespace: typeof row.namespaceJson === 'string' ? safeParseNamespace(row.namespaceJson) : undefined, + namespace, }); if (!normalized) { discarded += 1; continue; } const bucket = entriesByRef.get(normalized.ref) ?? []; diff --git a/test/daemon/memory-mcp-tools-schema-firewall.test.ts b/test/daemon/memory-mcp-tools-schema-firewall.test.ts index e202f9d46..3f86005f4 100644 --- a/test/daemon/memory-mcp-tools-schema-firewall.test.ts +++ b/test/daemon/memory-mcp-tools-schema-firewall.test.ts @@ -1,4 +1,7 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import type { ContextNamespace, ProcessedContextProjection } from '../../shared/context-types.js'; import { MCP_FEATURE_FLAGS_BY_NAME } from '../../shared/memory-mcp-feature-flags.js'; import { MEMORY_FEATURE_FLAGS_BY_NAME, type MemoryFeatureFlag } from '../../shared/feature-flags.js'; @@ -56,10 +59,33 @@ function projection(overrides: Partial = {}): Proces } describe('memory MCP tool schema firewall', () => { + let shortRefDir: string; + let priorShortRefPath: string | undefined; + let priorLegacyPath: string | undefined; + beforeEach(() => { + // Registering a handle now persists it, and the default target is the + // context store — which would build a real SQLite database, WAL and log in + // the runner's home directory just from exercising these handlers. Point + // persistence at a scratch file, and the legacy cache at a path that does + // not exist, so nothing here touches a real home. + shortRefDir = mkdtempSync(join(tmpdir(), 'imc-mcp-shortref-')); + priorShortRefPath = process.env.IMCODES_MEMORY_SHORT_REF_PATH; + priorLegacyPath = process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH; + process.env.IMCODES_MEMORY_SHORT_REF_PATH = join(shortRefDir, 'refs.json'); + process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = join(shortRefDir, 'absent-legacy.json'); resetMemoryShortRefsForTests(); }); + afterEach(() => { + resetMemoryShortRefsForTests(); + if (priorShortRefPath === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_PATH = priorShortRefPath; + if (priorLegacyPath === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = priorLegacyPath; + rmSync(shortRefDir, { recursive: true, force: true }); + }); + it('submits peer audit replies only through the strict structured dependency', async () => { const peerAuditReply = vi.fn(async () => ({ ok: true })); const handlers = createMemoryMcpToolHandlers(caller(), { peerAuditReply }); @@ -287,18 +313,52 @@ describe('memory MCP tool schema firewall', () => { it('declares how many records an ambiguous ref covers when expansion is bounded', async () => { // Expansion is capped, so a caller told it received "every match" would stop // looking while the answer sat in an omitted record. + // + // The orchestrator MUST be injected: without it the handler falls through to + // the real one, which builds an actual SQLite store (plus WAL and a log) in + // the runner's home directory. const namespace: ContextNamespace = { scope: 'personal', userId: 'user-1', projectId: 'repo-1' }; const ids = ['c1', 'c2', 'c3', 'c4', 'c5']; const ref = registerMemoryShortRef({ kind: 'projection', id: ids[0]!, namespace }); seedMemoryShortRefCollisionForTests(ref, ids.map((id) => ({ kind: 'projection' as const, id, namespace }))); + const getMemorySourcesOrchestrator = vi.fn(async (projectionId: string) => ({ + status: 'ok' as const, + projectionId, + sourceEventCount: 1, + sources: [{ eventId: `evt-${projectionId}`, status: 'ok', content: `body ${projectionId}` }], + })); const handlers = createMemoryMcpToolHandlers(caller({ namespace }), { - getProcessedProjectionById: vi.fn((id: string) => projection({ id, namespace })), + getMemorySourcesOrchestrator, isMemoryFeatureEnabled: () => true, }); const result = await handlers[MEMORY_MCP_TOOL_NAMES.GET_MEMORY_SOURCES]({ ref }) as Record; expect(result).toMatchObject({ status: 'ok', ambiguousRef: true, candidateCount: 5, truncated: true }); - expect((result.candidates as unknown[]).length).toBeLessThan(5); + const candidates = result.candidates as Array>; + // Exactly the cap, carrying real expanded content — `length < 5` alone would + // also pass on an empty array. + expect(candidates).toHaveLength(4); + expect(candidates.map((candidate) => candidate.projectionId)).toEqual(['c1', 'c2', 'c3', 'c4']); + expect(candidates[0]).toMatchObject({ kind: 'projection', sources: [expect.objectContaining({ content: 'body c1' })] }); + }); + + it('marks an ambiguous ref untruncated when every candidate fits', async () => { + const namespace: ContextNamespace = { scope: 'personal', userId: 'user-1', projectId: 'repo-1' }; + const ref = registerMemoryShortRef({ kind: 'projection', id: 'pair-a', namespace }); + seedMemoryShortRefCollisionForTests(ref, [ + { kind: 'projection', id: 'pair-a', namespace }, + { kind: 'projection', id: 'pair-b', namespace }, + ]); + const handlers = createMemoryMcpToolHandlers(caller({ namespace }), { + getMemorySourcesOrchestrator: vi.fn(async (projectionId: string) => ({ + status: 'ok' as const, projectionId, sourceEventCount: 0, sources: [], + })), + isMemoryFeatureEnabled: () => true, + }); + + const result = await handlers[MEMORY_MCP_TOOL_NAMES.GET_MEMORY_SOURCES]({ ref }) as Record; + expect(result).toMatchObject({ ambiguousRef: true, candidateCount: 2, truncated: false }); + expect((result.candidates as unknown[])).toHaveLength(2); }); it('short-circuits memory disabled gates before backend calls', async () => { From d261227fdd84a7b9ec72968c569a1255e17d5171 Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sun, 26 Jul 2026 07:57:22 +0800 Subject: [PATCH 24/40] Discard handles whose stored namespace cannot be validated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A namespace only had to be an object with a non-empty string scope, so an illegal scope or a corrupt identity field was silently replaced with undefined and the row loaded namespace-less. Such an entry looks healthy and counts as loaded, yet can never resolve for a namespaced caller — the handle disappears with no signal, which is the exact failure this module exists to prevent. The previous commit closed only the store's syntactically broken JSON and claimed more than it delivered: the legacy import and the explicit JSON load still degraded these rows. Decode namespaces through one strict path shared by all three sources, validate the scope against the known set along with identity field types, and discard the whole row when it fails. Storing the row also has to agree with itself, so compare namespace_key against the namespace that namespace_json decodes to and discard on mismatch. That check surfaced a real corruption: the key was built by joining fields with NUL, which SQLite's NUL-terminated TEXT truncates to just the scope — degrading the (ref, kind, id, namespace_key) primary key so two namespaces sharing a scope could overwrite one another. Persist a JSON tuple instead. Cover all of it, plus the tool description that tells the model an ambiguous reply is bounded: reverting either the validation or the description now fails. That protection was missing, which is how the overstated fix went unnoticed. Co-Authored-By: Claude Opus 4.8 --- src/context/memory-short-ref.ts | 69 +++++++++- .../memory-short-ref-namespace.test.ts | 123 ++++++++++++++++++ test/shared/memory-mcp-contracts.test.ts | 15 +++ 3 files changed, 200 insertions(+), 7 deletions(-) create mode 100644 test/context/memory-short-ref-namespace.test.ts diff --git a/src/context/memory-short-ref.ts b/src/context/memory-short-ref.ts index 23c925b6a..d7743de01 100644 --- a/src/context/memory-short-ref.ts +++ b/src/context/memory-short-ref.ts @@ -1,4 +1,5 @@ import type { ContextNamespace } from '../../shared/context-types.js'; +import { isMemoryScope } from '../../shared/memory-scope.js'; import { createHash } from 'node:crypto'; import { encodeBase32 } from '../util/base32.js'; import { getContextStoreClient } from '../store/context-store-worker-client.js'; @@ -59,6 +60,26 @@ function sameNamespace(a: ContextNamespace | undefined, b: ContextNamespace | un return namespaceKey(a) === namespaceKey(b); } +/** + * Namespace key for the persisted row. + * + * `namespaceKey()` separates fields with NUL, which is fine in memory but is + * truncated by SQLite's NUL-terminated TEXT — the stored key collapsed to just + * the scope, which also degraded the (ref, kind, id, namespace_key) primary key + * so two namespaces sharing a scope could overwrite each other. JSON keeps the + * same field order without an embedded NUL. + */ +function namespaceStorageKey(namespace: ContextNamespace | undefined): string { + if (!namespace) return ''; + return JSON.stringify([ + namespace.scope, + namespace.userId ?? '', + namespace.projectId ?? '', + namespace.workspaceId ?? '', + namespace.enterpriseId ?? '', + ]); +} + function pruneShortRefs(): void { let count = 0; for (const bucket of entriesByRef.values()) count += bucket.length; @@ -154,10 +175,33 @@ function isMemoryShortRefKind(value: unknown): value is MemoryShortRefKind { return value === 'projection' || value === 'observation'; } +/** Namespace identity fields; every one must be a string when present. */ +const NAMESPACE_IDENTITY_FIELDS = ['projectId', 'userId', 'workspaceId', 'enterpriseId'] as const; + +/** + * Decode a stored namespace, refusing anything it cannot fully validate. + * + * `{ ok: false }` means the row carried a namespace that is not usable, and the + * caller must discard the whole row. Degrading such a row to namespace-less + * instead — which an earlier `scope is a non-empty string` check did — loads an + * entry that looks healthy but can never resolve for a namespaced caller, and + * counts it as loaded, so the handle disappears with no signal at all. + */ +function decodeNamespace(raw: unknown): { ok: true; namespace: ContextNamespace | undefined } | { ok: false } { + if (raw === undefined || raw === null) return { ok: true, namespace: undefined }; + if (typeof raw !== 'object') return { ok: false }; + const record = raw as Record; + if (!isMemoryScope(record.scope)) return { ok: false }; + for (const field of NAMESPACE_IDENTITY_FIELDS) { + const value = record[field]; + if (value !== undefined && typeof value !== 'string') return { ok: false }; + } + return { ok: true, namespace: record as unknown as ContextNamespace }; +} + function isContextNamespace(value: unknown): value is ContextNamespace { - if (!value || typeof value !== 'object') return false; - const scope = (value as { scope?: unknown }).scope; - return typeof scope === 'string' && scope.length > 0; + const decoded = decodeNamespace(value); + return decoded.ok && decoded.namespace !== undefined; } function normalizeEntry(raw: unknown): { ref: string; entry: MemoryShortRefEntry } | undefined { @@ -167,7 +211,11 @@ function normalizeEntry(raw: unknown): { ref: string; entry: MemoryShortRefEntry const kind = record.kind; const id = typeof record.id === 'string' && record.id.trim() ? record.id.trim() : undefined; if (!ref || !isMemoryShortRefKind(kind) || !id) return undefined; - const namespace = isContextNamespace(record.namespace) ? record.namespace : undefined; + // A row that carries a namespace it cannot validate is discarded, never + // silently loaded namespace-less. + const decodedNamespace = decodeNamespace(record.namespace); + if (!decodedNamespace.ok) return undefined; + const namespace = decodedNamespace.namespace; const lastSeenAt = typeof record.lastSeenAt === 'number' && Number.isFinite(record.lastSeenAt) ? record.lastSeenAt : undefined; @@ -277,7 +325,7 @@ function persistShortRefs(touched: ReadonlyArray<{ ref: string; entry: MemorySho ref, kind: entry.kind, id: entry.id, - namespaceKey: namespaceKey(entry.namespace), + namespaceKey: namespaceStorageKey(entry.namespace), namespaceJson: entry.namespace ? JSON.stringify(entry.namespace) : null, lastSeenAt: entry.lastSeenAt ?? Date.now(), })); @@ -321,8 +369,15 @@ export async function loadMemoryShortRefsFromStore(): Promise { const storedNamespaceJson = typeof row.namespaceJson === 'string' && row.namespaceJson.trim() ? row.namespaceJson : undefined; - const namespace = storedNamespaceJson ? safeParseNamespace(storedNamespaceJson) : undefined; - if (storedNamespaceJson && !isContextNamespace(namespace)) { discarded += 1; continue; } + const decoded = decodeNamespace(storedNamespaceJson ? safeParseNamespace(storedNamespaceJson) : undefined); + if (!decoded.ok) { discarded += 1; continue; } + const namespace = decoded.namespace; + // namespace_key is what the row was written under. If it disagrees with + // the namespace the JSON decodes to — including a key that says "scoped" + // while the JSON is missing — the row's identity is corrupt and loading it + // would place the handle under the wrong namespace. + const storedNamespaceKey = typeof row.namespaceKey === 'string' ? row.namespaceKey : ''; + if (storedNamespaceKey !== namespaceStorageKey(namespace)) { discarded += 1; continue; } const normalized = normalizeEntry({ ref: row.ref, kind: row.kind, diff --git a/test/context/memory-short-ref-namespace.test.ts b/test/context/memory-short-ref-namespace.test.ts new file mode 100644 index 000000000..a8d5e7e42 --- /dev/null +++ b/test/context/memory-short-ref-namespace.test.ts @@ -0,0 +1,123 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +const { runMock, incrementCounterMock, warnOncePerHourMock } = vi.hoisted(() => ({ + runMock: vi.fn(), + incrementCounterMock: vi.fn(), + warnOncePerHourMock: vi.fn(), +})); + +vi.mock('../../src/store/context-store-worker-client.js', () => ({ + getContextStoreClient: () => ({ run: runMock }), +})); +vi.mock('../../src/util/metrics.js', () => ({ incrementCounter: incrementCounterMock })); +vi.mock('../../src/util/rate-limited-warn.js', () => ({ warnOncePerHour: warnOncePerHourMock })); + +import { + loadMemoryShortRefsFromStore, + resetMemoryShortRefsForTests, + resolveMemoryShortRef, +} from '../../src/context/memory-short-ref.js'; + +/** + * A stored namespace that cannot be fully validated must take the whole row + * down. Loading it namespace-less instead yields an entry that looks healthy but + * can never resolve for a namespaced caller, and counts as loaded — the handle + * disappears with no signal, which is precisely the failure this module exists + * to prevent. An earlier check only required `scope` to be a non-empty string, + * so an illegal scope or a corrupt identity field slipped through. + */ +describe('memory short refs — invalid namespaces are discarded, not degraded', () => { + let priorPath: string | undefined; + let priorLegacy: string | undefined; + let dir: string; + + beforeEach(() => { + priorPath = process.env.IMCODES_MEMORY_SHORT_REF_PATH; + priorLegacy = process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH; + dir = mkdtempSync(join(tmpdir(), 'imc-ns-')); + delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; + process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = join(dir, 'legacy.json'); + vi.clearAllMocks(); + resetMemoryShortRefsForTests(); + }); + + afterEach(() => { + resetMemoryShortRefsForTests(); + if (priorPath === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_PATH = priorPath; + if (priorLegacy === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = priorLegacy; + rmSync(dir, { recursive: true, force: true }); + }); + + const validNamespace = { scope: 'personal', userId: 'u1', projectId: 'p1' }; + /** Storage key is the JSON tuple written by namespaceStorageKey(). */ + const validKey = JSON.stringify(['personal', 'u1', 'p1', '', '']); + + const invalidNamespaces: Array<[string, unknown]> = [ + ['illegal scope value', { scope: 'not_a_scope', userId: 'u1' }], + ['non-string identity field', { scope: 'personal', userId: 42 }], + ['non-object namespace', 'personal'], + ]; + + for (const [label, namespace] of invalidNamespaces) { + it(`discards a store row with an ${label}`, async () => { + runMock.mockImplementation(async (op: string) => (op === 'listMemoryShortRefs' + ? [{ ref: 'proj:aaaaaaaaaaaaa', kind: 'projection', id: 'store-row', namespaceKey: validKey, namespaceJson: JSON.stringify(namespace), lastSeenAt: 1 }] + : 1)); + + await expect(loadMemoryShortRefsFromStore()).resolves.toBe(0); + expect(resolveMemoryShortRef('proj:aaaaaaaaaaaaa')).toBeUndefined(); + expect(incrementCounterMock).toHaveBeenCalledWith('mem.short_ref.discarded_row', { source: 'warm_load' }); + }); + + it(`discards a legacy JSON row with an ${label}`, async () => { + writeFileSync(join(dir, 'legacy.json'), JSON.stringify({ + schemaVersion: 2, + entries: [{ ref: 'proj:bbbbbbbbbbbbb', kind: 'projection', id: 'legacy-row', namespace }], + }), 'utf8'); + runMock.mockImplementation(async (op: string) => (op === 'listMemoryShortRefs' ? [] : 1)); + + await loadMemoryShortRefsFromStore(); + expect(resolveMemoryShortRef('proj:bbbbbbbbbbbbb')).toBeUndefined(); + expect(incrementCounterMock).toHaveBeenCalledWith('mem.short_ref.discarded_row', { source: 'legacy_file' }); + }); + } + + it('discards a store row whose namespace_key disagrees with its namespace_json', async () => { + // The key says the row is scoped while the JSON says it is not: the row's + // identity is corrupt, and loading it would file the handle under the wrong + // namespace. + runMock.mockImplementation(async (op: string) => (op === 'listMemoryShortRefs' + ? [{ ref: 'proj:ccccccccccccc', kind: 'projection', id: 'mismatch', namespaceKey: validKey, namespaceJson: null, lastSeenAt: 1 }] + : 1)); + + await expect(loadMemoryShortRefsFromStore()).resolves.toBe(0); + expect(incrementCounterMock).toHaveBeenCalledWith('mem.short_ref.discarded_row', { source: 'warm_load' }); + }); + + it('loads a row whose namespace is valid and consistent', async () => { + runMock.mockImplementation(async (op: string) => (op === 'listMemoryShortRefs' + ? [{ ref: 'proj:ddddddddddddd', kind: 'projection', id: 'ok-row', namespaceKey: validKey, namespaceJson: JSON.stringify(validNamespace), lastSeenAt: 1 }] + : 1)); + + await expect(loadMemoryShortRefsFromStore()).resolves.toBe(1); + expect(resolveMemoryShortRef('proj:ddddddddddddd', validNamespace as never)).toMatchObject({ id: 'ok-row' }); + expect(incrementCounterMock).not.toHaveBeenCalledWith('mem.short_ref.discarded_row', expect.anything()); + }); + + it('rejects a schemaVersion 1 legacy file wholesale', async () => { + // Handles from the previous derivation denote different records now. + writeFileSync(join(dir, 'legacy.json'), JSON.stringify({ + schemaVersion: 1, + entries: [{ ref: 'proj:deeaca10cd', kind: 'projection', id: 'v1-row', namespace: validNamespace }], + }), 'utf8'); + runMock.mockImplementation(async (op: string) => (op === 'listMemoryShortRefs' ? [] : 1)); + + await expect(loadMemoryShortRefsFromStore()).resolves.toBe(0); + expect(resolveMemoryShortRef('proj:deeaca10cd')).toBeUndefined(); + }); +}); diff --git a/test/shared/memory-mcp-contracts.test.ts b/test/shared/memory-mcp-contracts.test.ts index 2192a61a2..57932d0e4 100644 --- a/test/shared/memory-mcp-contracts.test.ts +++ b/test/shared/memory-mcp-contracts.test.ts @@ -134,6 +134,21 @@ describe('memory MCP shared contracts', () => { } }); + it('tells the model an ambiguous ref is bounded and that empty sources is not "no memory"', () => { + // This description is what the model actually reads — registerMemoryMcpTools + // publishes it verbatim. It previously promised "every match" while the + // implementation expanded at most four, so a caller could stop looking with + // the answer sitting in an omitted record; and an ambiguous reply carries an + // empty `sources`, which reads as "no memory" unless the text says otherwise. + const getSources = MEMORY_MCP_TOOL_CONTRACTS[MEMORY_MCP_TOOL_NAMES.GET_MEMORY_SOURCES]; + + expect(getSources.description).toContain('up to four'); + expect(getSources.description).not.toMatch(/every match/i); + expect(getSources.description).toContain('candidateCount'); + expect(getSources.description).toContain('truncated'); + expect(getSources.description).toMatch(/not.*no memory/i); + }); + it('documents when search_memory results should be expanded through get_memory_sources', () => { const search = MEMORY_MCP_TOOL_CONTRACTS[MEMORY_MCP_TOOL_NAMES.SEARCH_MEMORY]; const getSources = MEMORY_MCP_TOOL_CONTRACTS[MEMORY_MCP_TOOL_NAMES.GET_MEMORY_SOURCES]; From aa3c618f62332576c3632d0e3054e5613ab291a8 Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sun, 26 Jul 2026 08:02:23 +0800 Subject: [PATCH 25/40] Validate namespace identity and report a corrupt handle cache A namespace that named no reachable location still loaded. Scope was checked against the known set, but not against the identity a scope requires, so {"scope":"personal"} with no userId or projectId passed validation and produced an entry no caller can ever match. Run the same scope/identity validation the MCP caller already uses. A cache file on the current schema whose entries are not an array was treated like an empty cache and returned in silence, even though every handle in it has just vanished. Separate that from the expected quiet rejection of an older schema, and report it. Stub the throttled-warning module in the MCP handler suite so collision cases stop writing warning text into a real home, and correct the setup comment: it claimed nothing there touches a real home, while the daemon logger still creates its file on import and a pre-existing case still builds the context store. Neither is this suite's to fix, so they are named rather than papered over. Co-Authored-By: Claude Opus 4.8 --- src/context/memory-short-ref.ts | 34 ++++++++++++++++--- .../memory-mcp-tools-schema-firewall.test.ts | 13 +++++-- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/src/context/memory-short-ref.ts b/src/context/memory-short-ref.ts index d7743de01..00cf4f5b0 100644 --- a/src/context/memory-short-ref.ts +++ b/src/context/memory-short-ref.ts @@ -1,5 +1,5 @@ import type { ContextNamespace } from '../../shared/context-types.js'; -import { isMemoryScope } from '../../shared/memory-scope.js'; +import { isMemoryScope, validateMemoryScopeIdentity } from '../../shared/memory-scope.js'; import { createHash } from 'node:crypto'; import { encodeBase32 } from '../util/base32.js'; import { getContextStoreClient } from '../store/context-store-worker-client.js'; @@ -140,8 +140,15 @@ function importLegacyShortRefFile(): Array<{ ref: string; entry: MemoryShortRefE const imported: Array<{ ref: string; entry: MemoryShortRefEntry }> = []; try { const parsed = JSON.parse(readFileSync(path, 'utf8')) as { schemaVersion?: unknown; entries?: unknown[] }; - // Older derivations produced handles that denote different records now. - if (parsed.schemaVersion !== SHORT_REF_SCHEMA_VERSION || !Array.isArray(parsed.entries)) return imported; + // Older derivations produced handles that denote different records now, so + // rejecting that whole file is expected and stays quiet. + if (parsed.schemaVersion !== SHORT_REF_SCHEMA_VERSION) return imported; + // A current-schema file whose entries are not an array is corrupt, and every + // handle in it vanishes — that is a failure, not a steady state. + if (!Array.isArray(parsed.entries)) { + reportShortRefFailure('load_file', new Error('legacy cache entries is not an array'), { path, legacy: true }); + return imported; + } let discarded = 0; for (const raw of parsed.entries) { const normalized = normalizeEntry(raw); @@ -196,7 +203,19 @@ function decodeNamespace(raw: unknown): { ok: true; namespace: ContextNamespace const value = record[field]; if (value !== undefined && typeof value !== 'string') return { ok: false }; } - return { ok: true, namespace: record as unknown as ContextNamespace }; + // A scope also dictates which identity fields must be present and which are + // forbidden. `{ scope: 'personal' }` with no userId/projectId is structurally + // fine but names no actual namespace, so it would resolve for nobody. + const namespace = record as unknown as ContextNamespace; + const identity = { + user_id: namespace.userId, + project_id: namespace.projectId, + workspace_id: namespace.workspaceId, + org_id: namespace.enterpriseId, + tenant_id: namespace.localTenant, + }; + if (!validateMemoryScopeIdentity(record.scope, identity).ok) return { ok: false }; + return { ok: true, namespace }; } function isContextNamespace(value: unknown): value is ContextNamespace { @@ -236,7 +255,12 @@ function ensurePersistedLoaded(): void { // string can denote a different record under the new algorithm, which would // resolve to the wrong memory. Drop the whole file and re-register lazily. if (parsed.schemaVersion !== SHORT_REF_SCHEMA_VERSION) return; - if (!Array.isArray(parsed.entries)) return; + // Current schema with non-array entries is corruption, not an empty cache: + // every handle in the file disappears, so it has to be reported. + if (!Array.isArray(parsed.entries)) { + reportShortRefFailure('load_file', new Error('cache entries is not an array'), { path }); + return; + } let discarded = 0; for (const raw of parsed.entries) { const normalized = normalizeEntry(raw); diff --git a/test/daemon/memory-mcp-tools-schema-firewall.test.ts b/test/daemon/memory-mcp-tools-schema-firewall.test.ts index 3f86005f4..62a16c30e 100644 --- a/test/daemon/memory-mcp-tools-schema-firewall.test.ts +++ b/test/daemon/memory-mcp-tools-schema-firewall.test.ts @@ -10,6 +10,8 @@ import { MCP_ERROR_REASONS } from '../../shared/memory-mcp-errors.js'; import { MEMORY_MCP_DEGRADED_REASON } from '../../shared/memory-ws.js'; import { createMemoryMcpToolHandlers } from '../../src/daemon/memory-mcp-tools.js'; import type { McpRuntimeCaller } from '../../src/daemon/memory-mcp-caller.js'; +vi.mock('../../src/util/rate-limited-warn.js', () => ({ warnOncePerHour: vi.fn() })); + import { makeMemoryShortRef, registerMemoryShortRef, resetMemoryShortRefsForTests, seedMemoryShortRefCollisionForTests } from '../../src/context/memory-short-ref.js'; import type { SessionRecord } from '../../src/store/session-store.js'; @@ -65,10 +67,15 @@ describe('memory MCP tool schema firewall', () => { beforeEach(() => { // Registering a handle now persists it, and the default target is the - // context store — which would build a real SQLite database, WAL and log in - // the runner's home directory just from exercising these handlers. Point + // context store — which would build a real SQLite database and WAL in the + // runner's home directory just from exercising these handlers. Point // persistence at a scratch file, and the legacy cache at a path that does - // not exist, so nothing here touches a real home. + // not exist. The collision cases also emit a throttled warning, so the + // warning module is stubbed above to keep that content out of the daemon + // log. Two writes remain outside this suite's reach and are NOT claimed to + // be isolated: the daemon logger creates its file when the module is + // imported, and one pre-existing observation-expansion case below builds the + // context store. shortRefDir = mkdtempSync(join(tmpdir(), 'imc-mcp-shortref-')); priorShortRefPath = process.env.IMCODES_MEMORY_SHORT_REF_PATH; priorLegacyPath = process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH; From b46a62ead950186873e35f28b1e4ab683859000c Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sun, 26 Jul 2026 08:49:35 +0800 Subject: [PATCH 26/40] Keep handles written under the previous namespace key readable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The key-consistency check added with the JSON tuple discarded every row written before it, because a NUL-separated key reads back from node:sqlite as just its leading scope — the driver stops at the first NUL converting TEXT to a JS string. Warm-load therefore returned nothing and no handle survived the upgrade, defeating the durability this persistence exists for. Accept the legacy shape so those rows keep resolving; they are rewritten in the new format as their memories are registered again. The explanation shipped with that change was also wrong. It claimed SQLite truncated the stored value and collapsed the (ref, kind, id, namespace_key) primary key. Measurement says otherwise: two rows differing only by a NUL-containing key both insert, the stored bytes are intact, and the key is whole on disk. Only the read-back conversion is lossy. The comment now says that, and no longer justifies dropping old rows with a defect that does not exist. A namespace column that is present but unparseable was also still decoding as "no namespace" and loading, which strands the handle for every namespaced caller. Only an absent column may mean absent. Covered against a real SQLite store, since the behaviour under test is how node:sqlite round-trips the key and a mocked client cannot reproduce it. Co-Authored-By: Claude Opus 4.8 --- src/context/memory-short-ref.ts | 45 +++++--- .../memory-short-ref-legacy-key.test.ts | 103 ++++++++++++++++++ 2 files changed, 134 insertions(+), 14 deletions(-) create mode 100644 test/context/memory-short-ref-legacy-key.test.ts diff --git a/src/context/memory-short-ref.ts b/src/context/memory-short-ref.ts index 00cf4f5b0..f0ec1abf1 100644 --- a/src/context/memory-short-ref.ts +++ b/src/context/memory-short-ref.ts @@ -63,11 +63,13 @@ function sameNamespace(a: ContextNamespace | undefined, b: ContextNamespace | un /** * Namespace key for the persisted row. * - * `namespaceKey()` separates fields with NUL, which is fine in memory but is - * truncated by SQLite's NUL-terminated TEXT — the stored key collapsed to just - * the scope, which also degraded the (ref, kind, id, namespace_key) primary key - * so two namespaces sharing a scope could overwrite each other. JSON keeps the - * same field order without an embedded NUL. + * `namespaceKey()` separates fields with NUL, which is fine in memory but reads + * back from node:sqlite as just the leading scope: the driver stops at the first + * NUL when converting TEXT to a JS string. The bytes are stored intact and the + * primary key was never affected — a previous version of this comment claimed a + * collapsed primary key, which measurement disproved — but a key that cannot be + * read back is useless for verifying a row against its namespace. JSON keeps the + * same field order with no embedded NUL, so it survives the round trip. */ function namespaceStorageKey(namespace: ContextNamespace | undefined): string { if (!namespace) return ''; @@ -390,18 +392,33 @@ export async function loadMemoryShortRefsFromStore(): Promise { // dropped, not loaded namespace-less: the entry would look healthy while // being unresolvable for every namespaced caller — silent handle loss // wearing the shape of a successful load. - const storedNamespaceJson = typeof row.namespaceJson === 'string' && row.namespaceJson.trim() - ? row.namespaceJson - : undefined; - const decoded = decodeNamespace(storedNamespaceJson ? safeParseNamespace(storedNamespaceJson) : undefined); + // Absent means the column itself carried nothing. A column that IS present + // but cannot be parsed is corruption, not "no namespace" — degrading it to + // namespace-less loads an entry that looks healthy while being + // unresolvable for every namespaced caller. + const namespaceColumnPresent = typeof row.namespaceJson === 'string' && row.namespaceJson.trim().length > 0; + const parsed = namespaceColumnPresent ? safeParseNamespace(row.namespaceJson as string) : undefined; + if (namespaceColumnPresent && parsed === undefined) { discarded += 1; continue; } + const decoded = decodeNamespace(parsed); if (!decoded.ok) { discarded += 1; continue; } const namespace = decoded.namespace; - // namespace_key is what the row was written under. If it disagrees with - // the namespace the JSON decodes to — including a key that says "scoped" - // while the JSON is missing — the row's identity is corrupt and loading it - // would place the handle under the wrong namespace. + // namespace_key records what the row was written under; a disagreement + // means the row's identity is corrupt and loading it would file the handle + // under the wrong namespace. + // + // Rows written before the key became a JSON tuple used NUL-separated + // fields, and node:sqlite truncates a NUL-containing TEXT value at the + // first NUL when it converts to a JS string — so those rows read back as + // just the scope. (The bytes are stored intact and the primary key was + // never affected; an earlier comment here claimed otherwise and was + // wrong.) Accept that legacy shape so an upgrade does not discard every + // handle written before it. const storedNamespaceKey = typeof row.namespaceKey === 'string' ? row.namespaceKey : ''; - if (storedNamespaceKey !== namespaceStorageKey(namespace)) { discarded += 1; continue; } + const legacyTruncatedKey = namespace ? namespace.scope : ''; + if (storedNamespaceKey !== namespaceStorageKey(namespace) && storedNamespaceKey !== legacyTruncatedKey) { + discarded += 1; + continue; + } const normalized = normalizeEntry({ ref: row.ref, kind: row.kind, diff --git a/test/context/memory-short-ref-legacy-key.test.ts b/test/context/memory-short-ref-legacy-key.test.ts new file mode 100644 index 000000000..f30d10473 --- /dev/null +++ b/test/context/memory-short-ref-legacy-key.test.ts @@ -0,0 +1,103 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { resetContextStoreClientForTests } from '../../src/store/context-store-worker-client.js'; +import { upsertMemoryShortRefs } from '../../src/store/context-store.js'; +import { + loadMemoryShortRefsFromStore, + makeMemoryShortRef, + resetMemoryShortRefsForTests, + resolveMemoryShortRef, +} from '../../src/context/memory-short-ref.js'; +import { cleanupIsolatedSharedContextDb, createIsolatedSharedContextDb } from '../util/shared-context-db.js'; + +/** + * Exercised against a real SQLite store rather than a mocked client: the whole + * point is how node:sqlite round-trips a namespace key, which a mock cannot + * reproduce. + * + * Keys used to be NUL-separated. The bytes store fine and the primary key is + * unaffected, but node:sqlite stops at the first NUL when converting TEXT to a + * JS string, so such a row reads back as just its scope. Verifying that against + * the current JSON-tuple key would discard every handle written before the + * format changed — an upgrade that silently empties the cache. + */ +describe('memory short refs — legacy NUL-separated namespace keys', () => { + let tempDir: string; + let priorPath: string | undefined; + let priorLegacy: string | undefined; + + const namespace = { scope: 'personal' as const, userId: 'u1', projectId: 'p1' }; + const NUL = String.fromCharCode(0); + const legacyKey = [namespace.scope, namespace.userId, namespace.projectId, '', ''].join(NUL); + + beforeEach(async () => { + priorPath = process.env.IMCODES_MEMORY_SHORT_REF_PATH; + priorLegacy = process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH; + delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; + process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = '/nonexistent/imcodes-test/legacy.json'; + tempDir = await createIsolatedSharedContextDb('memory-short-ref-legacy-key'); + resetMemoryShortRefsForTests(); + }); + + afterEach(async () => { + resetMemoryShortRefsForTests(); + resetContextStoreClientForTests(); + if (priorPath === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_PATH = priorPath; + if (priorLegacy === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = priorLegacy; + await cleanupIsolatedSharedContextDb(tempDir); + }); + + it('still resolves a handle stored under the old NUL-separated key', async () => { + const id = 'fb7a7af3-3185-45a4-ac47-b26a57142353'; + const ref = makeMemoryShortRef('projection', id); + upsertMemoryShortRefs([{ + ref, kind: 'projection', id, + namespaceKey: legacyKey, + namespaceJson: JSON.stringify(namespace), + lastSeenAt: 1, + }]); + + await expect(loadMemoryShortRefsFromStore()).resolves.toBeGreaterThanOrEqual(1); + expect(resolveMemoryShortRef(ref, namespace)).toMatchObject({ id }); + }); + + it('keeps two legacy rows that differ only by namespace apart', async () => { + // The legacy key reads back as the shared scope, so accepting it must not + // let one namespace's handle answer for another's. + const id = 'aaaaaaaa-1111-2222-3333-444444444444'; + const ref = makeMemoryShortRef('projection', id); + const other = { scope: 'personal' as const, userId: 'u2', projectId: 'p2' }; + upsertMemoryShortRefs([ + { ref, kind: 'projection', id, namespaceKey: legacyKey, namespaceJson: JSON.stringify(namespace), lastSeenAt: 1 }, + { + ref, kind: 'projection', id, + namespaceKey: [other.scope, other.userId, other.projectId, '', ''].join(NUL), + namespaceJson: JSON.stringify(other), + lastSeenAt: 2, + }, + ]); + + await loadMemoryShortRefsFromStore(); + expect(resolveMemoryShortRef(ref, namespace)).toMatchObject({ id }); + expect(resolveMemoryShortRef(ref, other)).toMatchObject({ id }); + // A third, unrelated namespace must still miss. + expect(resolveMemoryShortRef(ref, { scope: 'personal', userId: 'u3', projectId: 'p3' } as never)).toBeUndefined(); + }); + + it('discards a row whose namespace column is present but unparseable', async () => { + // An empty key plus broken JSON previously decoded as "no namespace" and + // loaded, leaving a handle no namespaced caller can reach. + const id = 'bbbbbbbb-1111-2222-3333-444444444444'; + const ref = makeMemoryShortRef('projection', id); + upsertMemoryShortRefs([{ + ref, kind: 'projection', id, + namespaceKey: '', + namespaceJson: '{bad-json', + lastSeenAt: 1, + }]); + + await expect(loadMemoryShortRefsFromStore()).resolves.toBe(0); + expect(resolveMemoryShortRef(ref)).toBeUndefined(); + }); +}); From af4154cf4a1a9d5d15362f512a492d8a9b35e886 Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sun, 26 Jul 2026 08:52:57 +0800 Subject: [PATCH 27/40] Carry handle-persistence failures off the machine that failed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both existing signals stay inside the daemon. The counter lives in a process-local map that a restart clears and that nothing exports, and the throttled warning ends up in the daemon log — on the same disk whose exhaustion is usually the failure being reported, with destination errors swallowed. In the disk-full incident that motivated this persistence work, neither signal could leave the box, so "persistence failures are observable" was not actually true. Hold the last failure in memory and publish it on the control-plane heartbeat, which already leaves the process over the WebSocket every few seconds. It is sticky rather than edge-triggered and repeats on each beat, so an operator who connects after the incident still learns that handles stopped persisting, and a reconnect does not lose it. Failures accumulate, keeping a stuck disk distinguishable from a single blip. The test stubs out the counter and the warning entirely, so it only passes if the failure is reportable through neither of them. Co-Authored-By: Claude Opus 4.8 --- shared/memory-short-ref-health.ts | 26 ++++++ src/context/memory-short-ref.ts | 27 +++++- src/daemon/server-link.ts | 8 ++ test/context/memory-short-ref-health.test.ts | 97 ++++++++++++++++++++ 4 files changed, 154 insertions(+), 4 deletions(-) create mode 100644 shared/memory-short-ref-health.ts create mode 100644 test/context/memory-short-ref-health.test.ts diff --git a/shared/memory-short-ref-health.ts b/shared/memory-short-ref-health.ts new file mode 100644 index 000000000..30749ab74 --- /dev/null +++ b/shared/memory-short-ref-health.ts @@ -0,0 +1,26 @@ +/** + * Persistence health for the compact memory handles, reported on the daemon's + * control-plane heartbeat. + * + * A handle that fails to persist still resolves in the running process but dies + * on the next restart. The counters and the throttled warning that report it + * both stay inside the daemon: counters live in a process-local map, and the + * warning ends up in the daemon log — on the same disk whose exhaustion is the + * failure being reported, with write errors swallowed. In the original + * disk-full incident neither signal could leave the machine. + * + * This travels the WebSocket instead, and is sticky rather than edge-triggered, + * so it is still visible to whoever reconnects after the fact. + */ +export interface MemoryShortRefHealth { + /** Where the last failure happened (persist_store, persist_file, warm_load, load_file). */ + stage: string; + /** Failures since the process started; keeps a single blip distinguishable from a stuck disk. */ + failures: number; + /** Epoch ms of the most recent failure. */ + lastFailureAt: number; + /** Message of the most recent failure, truncated for the wire. */ + lastError: string; +} + +export const MEMORY_SHORT_REF_HEALTH_ERROR_MAX_CHARS = 200; diff --git a/src/context/memory-short-ref.ts b/src/context/memory-short-ref.ts index f0ec1abf1..d02621769 100644 --- a/src/context/memory-short-ref.ts +++ b/src/context/memory-short-ref.ts @@ -1,5 +1,6 @@ import type { ContextNamespace } from '../../shared/context-types.js'; import { isMemoryScope, validateMemoryScopeIdentity } from '../../shared/memory-scope.js'; +import { MEMORY_SHORT_REF_HEALTH_ERROR_MAX_CHARS, type MemoryShortRefHealth } from '../../shared/memory-short-ref-health.js'; import { createHash } from 'node:crypto'; import { encodeBase32 } from '../util/base32.js'; import { getContextStoreClient } from '../store/context-store-worker-client.js'; @@ -36,6 +37,7 @@ const MEMORY_SHORT_REF_LENGTH = 13; const SHORT_REF_SCHEMA_VERSION = 2; const entriesByRef = new Map(); let persistedLoaded = false; +let shortRefHealth: MemoryShortRefHealth | undefined; function refPrefix(kind: MemoryShortRefKind): 'proj' | 'obs' { return kind === 'projection' ? 'proj' : 'obs'; @@ -322,11 +324,27 @@ function reportDiscardedShortRefRows(source: 'warm_load' | 'legacy_file' | 'json } function reportShortRefFailure(stage: 'persist_store' | 'persist_file' | 'warm_load' | 'load_file', error: unknown, extra: Record = {}): void { + const message = error instanceof Error ? error.message : String(error); incrementCounter('mem.short_ref.persist_failure', { stage }); - warnOncePerHour(`mem.short_ref.persist_failure.${stage}`, { - ...extra, - error: error instanceof Error ? error.message : String(error), - }); + warnOncePerHour(`mem.short_ref.persist_failure.${stage}`, { ...extra, error: message }); + // Both signals above stay on this machine: the counter is a process-local map + // and the warning lands in the daemon log, on the same disk whose exhaustion + // is usually what failed — with write errors swallowed. Hold the failure in + // memory so the heartbeat can carry it off the box over the socket instead. + shortRefHealth = { + stage, + failures: (shortRefHealth?.failures ?? 0) + 1, + lastFailureAt: Date.now(), + lastError: message.slice(0, MEMORY_SHORT_REF_HEALTH_ERROR_MAX_CHARS), + }; +} + +/** + * Latest persistence failure, or undefined while healthy. Sticky rather than + * edge-triggered, so a reader that connects after the failure still sees it. + */ +export function getMemoryShortRefHealth(): MemoryShortRefHealth | undefined { + return shortRefHealth; } /** @@ -582,6 +600,7 @@ export function seedMemoryShortRefCollisionForTests(ref: string, entries: readon export function resetMemoryShortRefsForTests(): void { entriesByRef.clear(); + shortRefHealth = undefined; persistedLoaded = true; } diff --git a/src/daemon/server-link.ts b/src/daemon/server-link.ts index 976a596a2..b51935236 100644 --- a/src/daemon/server-link.ts +++ b/src/daemon/server-link.ts @@ -10,6 +10,8 @@ import { getEmbeddingStatus } from '../context/embedding.js'; import type { EmbeddingStatus } from '../../shared/embedding-status.js'; import type { DiskUsage } from '../../shared/disk-usage.js'; import { getDiskUsage, refreshDiskUsage } from './disk-usage.js'; +import { getMemoryShortRefHealth } from '../context/memory-short-ref.js'; +import type { MemoryShortRefHealth } from '../../shared/memory-short-ref-health.js'; import { recordDaemonServerLinkStatus } from '../util/daemon-status.js'; import { P2P_WORKFLOW_IMPLEMENTATION_CAPABILITY_V1, @@ -55,10 +57,15 @@ interface SystemStats { embedding: EmbeddingStatus; /** Mounted-filesystem capacity for the desktop status bar (mobile ignores it). */ disks: DiskUsage[]; + /** Sticky memory-handle persistence failure, omitted while healthy. Rides the + * heartbeat because the counter and the daemon log both stay on the machine + * whose disk is usually what failed. */ + shortRefHealth?: MemoryShortRefHealth; } /** Collect lightweight system stats for daemon.stats messages. */ function collectSystemStats(): SystemStats { + const shortRefHealth = getMemoryShortRefHealth(); const memTotal = os.totalmem(); const memFree = os.freemem(); const [load1, load5, load15] = os.loadavg(); @@ -75,6 +82,7 @@ function collectSystemStats(): SystemStats { uptime: os.uptime(), embedding: getEmbeddingStatus(), disks: getDiskUsage(), + ...(shortRefHealth ? { shortRefHealth } : {}), }; } diff --git a/test/context/memory-short-ref-health.test.ts b/test/context/memory-short-ref-health.test.ts new file mode 100644 index 000000000..d8c79d712 --- /dev/null +++ b/test/context/memory-short-ref-health.test.ts @@ -0,0 +1,97 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const { runMock, incrementCounterMock, warnOncePerHourMock } = vi.hoisted(() => ({ + runMock: vi.fn(), + incrementCounterMock: vi.fn(), + warnOncePerHourMock: vi.fn(), +})); + +// Both existing signals are stubbed out entirely: the point is that the failure +// is still reportable when neither of them can carry it. The counter is a +// process-local map that a restart clears, and the throttled warning ends up in +// the daemon log — on the same disk whose exhaustion is the failure being +// reported, with write errors swallowed. +vi.mock('../../src/store/context-store-worker-client.js', () => ({ + getContextStoreClient: () => ({ run: runMock }), +})); +vi.mock('../../src/util/metrics.js', () => ({ incrementCounter: incrementCounterMock })); +vi.mock('../../src/util/rate-limited-warn.js', () => ({ warnOncePerHour: warnOncePerHourMock })); + +import { + getMemoryShortRefHealth, + loadMemoryShortRefsFromStore, + registerMemoryShortRefs, + resetMemoryShortRefsForTests, +} from '../../src/context/memory-short-ref.js'; + +describe('memory short refs — persistence failure leaves the process', () => { + let priorPath: string | undefined; + let priorLegacy: string | undefined; + + beforeEach(() => { + priorPath = process.env.IMCODES_MEMORY_SHORT_REF_PATH; + priorLegacy = process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH; + delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; + process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = '/nonexistent/imcodes-test/legacy.json'; + vi.clearAllMocks(); + resetMemoryShortRefsForTests(); + }); + + afterEach(() => { + resetMemoryShortRefsForTests(); + if (priorPath === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_PATH = priorPath; + if (priorLegacy === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = priorLegacy; + }); + + const entry = { + kind: 'projection' as const, + id: 'fb7a7af3-3185-45a4-ac47-b26a57142353', + namespace: { scope: 'personal' as const, userId: 'u1', projectId: 'p1' }, + }; + + it('reports nothing while persistence is healthy', async () => { + runMock.mockResolvedValue(1); + registerMemoryShortRefs([entry]); + await vi.waitFor(() => expect(runMock).toHaveBeenCalled()); + expect(getMemoryShortRefHealth()).toBeUndefined(); + }); + + it('exposes a disk-full write failure through a channel that is neither the store nor the log', async () => { + runMock.mockRejectedValue(Object.assign(new Error('ENOSPC: no space left on device'), { code: 'ENOSPC' })); + + registerMemoryShortRefs([entry]); + await vi.waitFor(() => expect(getMemoryShortRefHealth()).toBeDefined()); + + expect(getMemoryShortRefHealth()).toMatchObject({ + stage: 'persist_store', + failures: 1, + lastError: expect.stringContaining('ENOSPC'), + }); + expect(getMemoryShortRefHealth()!.lastFailureAt).toBeGreaterThan(0); + }); + + it('stays set after the failure so a later reader still sees it', async () => { + // Sticky, not edge-triggered: whoever reconnects after the incident has to + // be able to learn that handles stopped persisting. + runMock.mockRejectedValue(new Error('ENOSPC: no space left on device')); + registerMemoryShortRefs([entry]); + await vi.waitFor(() => expect(getMemoryShortRefHealth()).toBeDefined()); + + // Any number of later reads report the same standing failure. + expect(getMemoryShortRefHealth()).toBeDefined(); + expect(getMemoryShortRefHealth()).toBeDefined(); + + // A second failure accumulates rather than resetting, so a stuck disk is + // distinguishable from a single blip. + registerMemoryShortRefs([{ ...entry, id: 'second-id' }]); + await vi.waitFor(() => expect(getMemoryShortRefHealth()!.failures).toBeGreaterThan(1)); + }); + + it('reports a failed warm-load too, not only writes', async () => { + runMock.mockRejectedValue(new Error('context_store_unavailable')); + await loadMemoryShortRefsFromStore(); + expect(getMemoryShortRefHealth()).toMatchObject({ stage: 'warm_load' }); + }); +}); From 8ee632971c8309fcad46b71f0245d0b1358e8e9a Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sun, 26 Jul 2026 09:10:07 +0800 Subject: [PATCH 28/40] Deliver handle-persistence health to the browser and fail closed on null MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The health record reached the pod and stopped there. The bridge rebuilt daemon.stats from a fixed field list, so it was dropped before any browser saw it: no UI, no alert, no query — the operator visibility the previous commit claimed did not exist. Forward it, type it on the client, and ignore a non-object value rather than passing junk along. The daemon-side test could never have caught that, because it only reads an in-process getter; deleting every line of the wiring left it green. The new test asserts the frame crossing to the browser instead, from both daemon.stats and heartbeat, and fails when the forward is removed. A namespace column holding the JSON text null also still loaded as namespace-less. It parses cleanly, so guarding only against parse failure let it through, and the resulting handle is unreachable for every namespaced caller. Only a value that is absent may mean absent. The identity check added earlier had no regression either: every existing case paired an invalid namespace with a mismatched key, so the key comparison masked whether identity was ever validated. The new cases pair each namespace with the key it would really be stored under, leaving the scope rules as the only thing that can reject them. Correcting b46a62ead: legacy rows are not rewritten on re-registration. The primary key includes namespace_key, so the old and new key forms coexist as two rows. Reads are unaffected and namespaces stay isolated; the cost is a duplicate row and some of the warm-load budget until old rows are cleaned up separately. Co-Authored-By: Claude Opus 4.8 --- server/src/ws/bridge.ts | 5 + server/test/bridge-short-ref-health.test.ts | 126 ++++++++++++++++++ src/context/memory-short-ref.ts | 8 +- .../memory-short-ref-legacy-key.test.ts | 60 +++++++++ web/src/ws-client.ts | 3 +- 5 files changed, 199 insertions(+), 3 deletions(-) create mode 100644 server/test/bridge-short-ref-health.test.ts diff --git a/server/src/ws/bridge.ts b/server/src/ws/bridge.ts index 102b660b7..2824de6f8 100644 --- a/server/src/ws/bridge.ts +++ b/server/src/ws/bridge.ts @@ -4932,6 +4932,11 @@ export class WsBridge { daemonVersion: typeof msg.daemonVersion === 'string' ? msg.daemonVersion : this.daemonVersion, cpu: msg.cpu, memUsed: msg.memUsed, memTotal: msg.memTotal, load1: msg.load1, load5: msg.load5, load15: msg.load15, uptime: msg.uptime, + // Memory-handle persistence failures ride this frame because the + // daemon's own counter and log cannot leave a machine whose disk is + // full. Rebuilding the payload without them put the signal in a hole: + // it reached this pod and went no further. + ...(isPlainRecord(msg.shortRefHealth) ? { shortRefHealth: msg.shortRefHealth } : {}), })); return; } diff --git a/server/test/bridge-short-ref-health.test.ts b/server/test/bridge-short-ref-health.test.ts new file mode 100644 index 000000000..74d941984 --- /dev/null +++ b/server/test/bridge-short-ref-health.test.ts @@ -0,0 +1,126 @@ +/** + * Memory-handle persistence failures have to reach a person. + * + * The daemon's own two signals cannot: the counter is a process-local map that a + * restart clears, and the throttled warning goes to the daemon log — on the same + * disk whose exhaustion is usually the failure being reported. So the daemon + * puts a sticky health record on its heartbeat instead. + * + * That only helps if the bridge forwards it. It previously rebuilt daemon.stats + * from a fixed field list, so the record arrived at the pod and went no further: + * no UI, no alert, nothing. A daemon-side test cannot catch that — it has to be + * asserted here, where the frame crosses to the browser. + * + * @vitest-environment node + */ + +import { EventEmitter } from 'node:events'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { WsBridge } from '../src/ws/bridge.js'; + +vi.mock('../src/security/crypto.js', () => ({ sha256Hex: (_s: string) => 'valid-hash' })); +vi.mock('../src/routes/push.js', () => ({ dispatchPush: vi.fn() })); + +class MockWs extends EventEmitter { + sent: Array = []; + closed = false; + readyState = 1; + send(data: string | Buffer, _opts?: unknown, callback?: (err?: Error) => void): void { + this.sent.push(data); + callback?.(); + } + close(): void { this.closed = true; this.readyState = 3; this.emit('close'); } + get sentStrings(): string[] { return this.sent.filter((s): s is string => typeof s === 'string'); } +} + +function makeDb() { + return { + queryOne: async () => ({ token_hash: 'valid-hash', node_role: 'full', revoked_at: null }), + query: async () => [], + execute: async () => ({ changes: 1 }), + exec: async () => {}, + transaction: async (fn: (tx: unknown) => Promise) => fn({}), + close: () => {}, + } as unknown as import('../src/db/client.js').Database; +} + +const flushAsync = () => new Promise((resolve) => { setImmediate(resolve); }); + +describe('WsBridge forwards memory-handle persistence health to browsers', () => { + let serverId: string; + + beforeEach(() => { serverId = `test-${Math.random().toString(36).slice(2)}`; }); + afterEach(() => { WsBridge.getAll().clear(); vi.clearAllMocks(); }); + + const shortRefHealth = { + stage: 'persist_store', + failures: 3, + lastFailureAt: 1_700_000_000_000, + lastError: 'ENOSPC: no space left on device', + }; + + const baseStats = { + daemonVersion: '1.2.3', + cpu: 12, memUsed: 1, memTotal: 2, + load1: 0.1, load5: 0.2, load15: 0.3, uptime: 100, + }; + + async function connect() { + const bridge = WsBridge.get(serverId); + const daemon = new MockWs(); + const browser = new MockWs(); + bridge.handleDaemonConnection(daemon as never, makeDb() as never, {} as never); + bridge.handleBrowserConnection(browser as never, { id: 'user-a' } as never); + daemon.emit('message', JSON.stringify({ type: 'auth', serverId, token: 'my-token' })); + await flushAsync(); + return { daemon, browser }; + } + + function statsSeenByBrowser(browser: MockWs): Record | undefined { + for (const raw of browser.sentStrings) { + const parsed = JSON.parse(raw) as Record; + if (parsed.type === 'daemon.stats') return parsed; + } + return undefined; + } + + it('delivers the health record from a daemon.stats frame to the browser', async () => { + const { daemon, browser } = await connect(); + + daemon.emit('message', JSON.stringify({ type: 'daemon.stats', ...baseStats, shortRefHealth })); + await flushAsync(); + + expect(statsSeenByBrowser(browser)).toMatchObject({ type: 'daemon.stats', shortRefHealth }); + }); + + it('delivers it from a heartbeat frame as well, so a reconnect still surfaces it', async () => { + // The daemon repeats the sticky record on every beat; an operator who + // connects after the incident learns about it from the next one. + const { daemon, browser } = await connect(); + + daemon.emit('message', JSON.stringify({ type: 'heartbeat', ...baseStats, shortRefHealth })); + await flushAsync(); + + expect(statsSeenByBrowser(browser)).toMatchObject({ shortRefHealth }); + }); + + it('omits the field entirely while persistence is healthy', async () => { + const { daemon, browser } = await connect(); + + daemon.emit('message', JSON.stringify({ type: 'daemon.stats', ...baseStats })); + await flushAsync(); + + const stats = statsSeenByBrowser(browser); + expect(stats).toBeDefined(); + expect(stats).not.toHaveProperty('shortRefHealth'); + }); + + it('drops a non-object health value rather than forwarding junk', async () => { + const { daemon, browser } = await connect(); + + daemon.emit('message', JSON.stringify({ type: 'daemon.stats', ...baseStats, shortRefHealth: 'disk full' })); + await flushAsync(); + + expect(statsSeenByBrowser(browser)).not.toHaveProperty('shortRefHealth'); + }); +}); diff --git a/src/context/memory-short-ref.ts b/src/context/memory-short-ref.ts index d02621769..3b5a98c9f 100644 --- a/src/context/memory-short-ref.ts +++ b/src/context/memory-short-ref.ts @@ -199,8 +199,12 @@ const NAMESPACE_IDENTITY_FIELDS = ['projectId', 'userId', 'workspaceId', 'enterp * counts it as loaded, so the handle disappears with no signal at all. */ function decodeNamespace(raw: unknown): { ok: true; namespace: ContextNamespace | undefined } | { ok: false } { - if (raw === undefined || raw === null) return { ok: true, namespace: undefined }; - if (typeof raw !== 'object') return { ok: false }; + // Only a value that is not there at all means "no namespace". An explicit + // null — including a column holding the JSON text `null` — is a value that + // failed to describe a namespace, and degrading it to namespace-less loads a + // handle no namespaced caller can ever reach. + if (raw === undefined) return { ok: true, namespace: undefined }; + if (raw === null || typeof raw !== 'object') return { ok: false }; const record = raw as Record; if (!isMemoryScope(record.scope)) return { ok: false }; for (const field of NAMESPACE_IDENTITY_FIELDS) { diff --git a/test/context/memory-short-ref-legacy-key.test.ts b/test/context/memory-short-ref-legacy-key.test.ts index f30d10473..0f833bd36 100644 --- a/test/context/memory-short-ref-legacy-key.test.ts +++ b/test/context/memory-short-ref-legacy-key.test.ts @@ -101,3 +101,63 @@ describe('memory short refs — legacy NUL-separated namespace keys', () => { expect(resolveMemoryShortRef(ref)).toBeUndefined(); }); }); + +describe('memory short refs — namespace values that name nothing reachable', () => { + let tempDir: string; + let priorPath: string | undefined; + let priorLegacy: string | undefined; + + beforeEach(async () => { + priorPath = process.env.IMCODES_MEMORY_SHORT_REF_PATH; + priorLegacy = process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH; + delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; + process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = '/nonexistent/imcodes-test/legacy.json'; + tempDir = await createIsolatedSharedContextDb('memory-short-ref-identity'); + resetMemoryShortRefsForTests(); + }); + + afterEach(async () => { + resetMemoryShortRefsForTests(); + resetContextStoreClientForTests(); + if (priorPath === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_PATH = priorPath; + if (priorLegacy === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = priorLegacy; + await cleanupIsolatedSharedContextDb(tempDir); + }); + + // Each row below pairs the namespace with the key that namespaceStorageKey + // would produce for it, so the key-consistency check agrees and ONLY the + // scope/identity rules can reject the row. Earlier tests always used a valid + // key, which meant a mismatch masked whether identity was checked at all. + const keyFor = (ns: Record) => JSON.stringify([ + ns.scope, ns.userId ?? '', ns.projectId ?? '', ns.workspaceId ?? '', ns.enterpriseId ?? '', + ]); + + it('discards a personal-scope row missing the identity that scope requires', async () => { + const id = 'cccccccc-1111-2222-3333-444444444444'; + const ref = makeMemoryShortRef('projection', id); + const ns = { scope: 'personal', userId: 'u1' }; // no projectId + upsertMemoryShortRefs([{ + ref, kind: 'projection', id, + namespaceKey: keyFor(ns), namespaceJson: JSON.stringify(ns), lastSeenAt: 1, + }]); + + await expect(loadMemoryShortRefsFromStore()).resolves.toBe(0); + expect(resolveMemoryShortRef(ref, ns as never)).toBeUndefined(); + }); + + it('discards a row whose namespace column holds the JSON text null', async () => { + // Parses successfully, so a parse-failure guard alone lets it through; it + // still describes no namespace. + const id = 'dddddddd-1111-2222-3333-444444444444'; + const ref = makeMemoryShortRef('projection', id); + upsertMemoryShortRefs([{ + ref, kind: 'projection', id, + namespaceKey: '', namespaceJson: 'null', lastSeenAt: 1, + }]); + + await expect(loadMemoryShortRefsFromStore()).resolves.toBe(0); + expect(resolveMemoryShortRef(ref)).toBeUndefined(); + }); +}); diff --git a/web/src/ws-client.ts b/web/src/ws-client.ts index 3e276f41c..0ead91557 100644 --- a/web/src/ws-client.ts +++ b/web/src/ws-client.ts @@ -13,6 +13,7 @@ import { P2P_CONFIG_MSG } from '@shared/p2p-config-events.js'; import { P2P_WORKFLOW_MSG, isP2pWorkflowRequestId } from '@shared/p2p-workflow-messages.js'; import { TRANSPORT_EVENT } from '@shared/transport-events.js'; import type { DiskUsage } from '@shared/disk-usage.js'; +import type { MemoryShortRefHealth } from '@shared/memory-short-ref-health.js'; import { P2P_CAPABILITY_FRESHNESS_TTL_MS } from '@shared/p2p-workflow-constants.js'; import { TRANSPORT_MSG } from '@shared/transport-events.js'; import { DAEMON_COMMAND_TYPES } from '@shared/daemon-command-types.js'; @@ -187,7 +188,7 @@ export type ServerMessage = | { type: 'discussion.done'; discussionId: string; filePath: string; conclusion: string } | { type: 'discussion.error'; discussionId?: string; requestId?: string; error: string } | { type: 'discussion.list'; discussions: Array<{ id: string; requestId?: string; topic: string; state: string; currentRound: number; maxRounds: number; completedHops?: number; totalHops?: number; currentSpeaker?: string; conclusion?: string; filePath?: string }> } - | { type: 'daemon.stats'; daemonVersion?: string | null; cpu: number; memUsed: number; memTotal: number; load1: number; load5: number; load15: number; uptime: number; disks?: DiskUsage[] } + | { type: 'daemon.stats'; daemonVersion?: string | null; cpu: number; memUsed: number; memTotal: number; load1: number; load5: number; load15: number; uptime: number; disks?: DiskUsage[]; shortRefHealth?: MemoryShortRefHealth } | FsLsResponse | FsReadResponse | FsGitStatusResponse From a7abd97118d90a42d4eae28b32b170fb1e152604 Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sun, 26 Jul 2026 09:13:59 +0800 Subject: [PATCH 29/40] Show handle-persistence failures in the status bar The health record reached the browser but nothing rendered it, so nobody would ever find out. Surface it next to the other daemon stats as a warning marker, with the stage, failure count, time and error in the tooltip, and say plainly what it means: handles issued now will not resolve after a restart. Discarded rows now report through the same channel. A row dropped as unusable loses that memory's handle exactly as a failed write does, so leaving it in a process-local counter hid the same class of loss the off-box signal exists for. Namespace field typing also covers localTenant and canonicalRepoId, so an unexpected shape in either is rejected rather than carried along. They stay out of the storage key deliberately: the key mirrors namespaceKey(), which defines namespace equality, and adding fields there would both diverge from that equality and invalidate every stored key again. Co-Authored-By: Claude Opus 4.8 --- src/context/memory-short-ref.ts | 12 +++++++- test/context/memory-short-ref-health.test.ts | 11 +++++++ web/src/components/SubSessionBar.tsx | 30 ++++++++++++++++++++ web/src/i18n/locales/en.json | 4 ++- web/src/i18n/locales/es.json | 4 ++- web/src/i18n/locales/ja.json | 4 ++- web/src/i18n/locales/ko.json | 4 ++- web/src/i18n/locales/ru.json | 4 ++- web/src/i18n/locales/zh-CN.json | 4 ++- web/src/i18n/locales/zh-TW.json | 4 ++- 10 files changed, 73 insertions(+), 8 deletions(-) diff --git a/src/context/memory-short-ref.ts b/src/context/memory-short-ref.ts index 3b5a98c9f..b053cae54 100644 --- a/src/context/memory-short-ref.ts +++ b/src/context/memory-short-ref.ts @@ -187,7 +187,9 @@ function isMemoryShortRefKind(value: unknown): value is MemoryShortRefKind { } /** Namespace identity fields; every one must be a string when present. */ -const NAMESPACE_IDENTITY_FIELDS = ['projectId', 'userId', 'workspaceId', 'enterpriseId'] as const; +const NAMESPACE_IDENTITY_FIELDS = [ + 'projectId', 'userId', 'workspaceId', 'enterpriseId', 'localTenant', 'canonicalRepoId', +] as const; /** * Decode a stored namespace, refusing anything it cannot fully validate. @@ -325,6 +327,10 @@ function reportDiscardedShortRefRows(source: 'warm_load' | 'legacy_file' | 'json if (discarded <= 0) return; incrementCounter('mem.short_ref.discarded_row', { source }); warnOncePerHour(`mem.short_ref.discarded_row.${source}`, { discarded }); + // Discarded rows are handle loss too: those memories stop being reachable by + // handle after a restart. Route them to the same off-box signal as write and + // load failures instead of leaving them in machine-local counters. + recordShortRefFailure(`discarded_${source}`, `${discarded} unusable row(s) discarded`); } function reportShortRefFailure(stage: 'persist_store' | 'persist_file' | 'warm_load' | 'load_file', error: unknown, extra: Record = {}): void { @@ -335,6 +341,10 @@ function reportShortRefFailure(stage: 'persist_store' | 'persist_file' | 'warm_l // and the warning lands in the daemon log, on the same disk whose exhaustion // is usually what failed — with write errors swallowed. Hold the failure in // memory so the heartbeat can carry it off the box over the socket instead. + recordShortRefFailure(stage, message); +} + +function recordShortRefFailure(stage: string, message: string): void { shortRefHealth = { stage, failures: (shortRefHealth?.failures ?? 0) + 1, diff --git a/test/context/memory-short-ref-health.test.ts b/test/context/memory-short-ref-health.test.ts index d8c79d712..3cdba1c10 100644 --- a/test/context/memory-short-ref-health.test.ts +++ b/test/context/memory-short-ref-health.test.ts @@ -94,4 +94,15 @@ describe('memory short refs — persistence failure leaves the process', () => { await loadMemoryShortRefsFromStore(); expect(getMemoryShortRefHealth()).toMatchObject({ stage: 'warm_load' }); }); + it('reports discarded rows too, since those handles are lost the same way', async () => { + // A row dropped as unusable means that memory is no longer reachable by + // handle after a restart — the same loss as a failed write, so it belongs on + // the same off-box signal rather than only in machine-local counters. + runMock.mockImplementation(async (op: string) => (op === 'listMemoryShortRefs' + ? [{ ref: 'proj:aaaaaaaaaaaaa', kind: 'projection', id: 'x', namespaceKey: '', namespaceJson: '{bad', lastSeenAt: 1 }] + : 1)); + + await loadMemoryShortRefsFromStore(); + expect(getMemoryShortRefHealth()).toMatchObject({ stage: 'discarded_warm_load' }); + }); }); diff --git a/web/src/components/SubSessionBar.tsx b/web/src/components/SubSessionBar.tsx index 02d1609d0..598a8e96b 100644 --- a/web/src/components/SubSessionBar.tsx +++ b/web/src/components/SubSessionBar.tsx @@ -25,6 +25,7 @@ import { SharedStateIndicator } from './SharedStateIndicator.js'; import type { SharedStateSummary } from '../tab-sharing-ui.js'; import type { EmbeddingStatus } from '@shared/embedding-status.js'; import type { DiskUsage } from '@shared/disk-usage.js'; +import type { MemoryShortRefHealth } from '@shared/memory-short-ref-health.js'; import { formatDaemonVersionMobile, formatDaemonVersionShort } from '../util/format-version.js'; import { isAuthoritativeUsageContextWindowSource, type UsageContextWindowSource } from '@shared/usage-context-window.js'; import { resolveQuickAgentDelegationModel } from '../quick-agent-delegation-model.js'; @@ -50,6 +51,7 @@ interface DaemonStats { uptime: number; embedding?: EmbeddingStatus | null; disks?: DiskUsage[] | null; + shortRefHealth?: MemoryShortRefHealth | null; } type DiscussionSummary = P2pProgressDiscussion & { @@ -205,6 +207,26 @@ function diskUsageColor(usedPercent: number): string | undefined { * "+N" hint) and every partition is listed in the hover tooltip. Mobile clients * receive no disk data, so this renders nothing there. */ +/** + * Memory handles stopped persisting. The daemon can only report this over the + * socket — its counter is process-local and its log shares the disk that is + * usually what failed — so this marker is the one place an operator sees it. + */ +function renderShortRefAlert(health: MemoryShortRefHealth | null | undefined, t: (k: string, v?: Record) => string): JSX.Element | null { + if (!health) return null; + const when = new Date(health.lastFailureAt).toLocaleString(); + return ( + + ⚠️ + {t('memory.short_ref_failure_short')} + + ); +} + function renderDiskStats(disks: DiskUsage[] | null | undefined): JSX.Element | null { if (!disks || disks.length === 0) return null; const tooltip = disks @@ -855,6 +877,7 @@ export function SubSessionBar({ subSessions, openIds, maximizedIds, desktopLayou // Older daemons don't ship `disks`; null means "no data" so the // desktop strip simply hides the readout (and mobile never shows it). disks: msg.disks ?? null, + shortRefHealth: msg.shortRefHealth ?? null, }); } }); @@ -960,6 +983,12 @@ export function SubSessionBar({ subSessions, openIds, maximizedIds, desktopLayou {renderDiskStats(stats.disks)} )} + {stats.shortRefHealth && ( + <> + · + {renderShortRefAlert(stats.shortRefHealth, t)} + + )} · · @@ -998,6 +1027,7 @@ export function SubSessionBar({ subSessions, openIds, maximizedIds, desktopLayou {desktopLayoutCapable && stats.disks && stats.disks.length > 0 && ( <>{renderDiskStats(stats.disks)}{' '} )} + {stats.shortRefHealth && (<>{renderShortRefAlert(stats.shortRefHealth, t)}{' '})} {desktopLayoutCapable && ( <> diff --git a/web/src/i18n/locales/en.json b/web/src/i18n/locales/en.json index 3cecabdbc..7976ac338 100644 --- a/web/src/i18n/locales/en.json +++ b/web/src/i18n/locales/en.json @@ -2600,7 +2600,9 @@ "layerDiagnostics": "Skill layer diagnostics", "enforced": "Enforced", "additive": "Additive" - } + }, + "short_ref_failure_short": "memory handles not saved", + "short_ref_failure_detail": "Memory handles stopped being saved ({{stage}}, {{failures}} failure(s), last {{when}}): {{error}}. Handles issued now will not resolve after a daemon restart." }, "todos": { "title": "Tasks", diff --git a/web/src/i18n/locales/es.json b/web/src/i18n/locales/es.json index 59c1e1a2d..3aa59ebf9 100644 --- a/web/src/i18n/locales/es.json +++ b/web/src/i18n/locales/es.json @@ -2600,7 +2600,9 @@ "layerDiagnostics": "Diagnóstico de capas de habilidades", "enforced": "Obligatoria", "additive": "Aditiva" - } + }, + "short_ref_failure_short": "identificadores de memoria sin guardar", + "short_ref_failure_detail": "Los identificadores de memoria dejaron de guardarse ({{stage}}, {{failures}} fallo(s), último {{when}}): {{error}}. Los emitidos ahora no se resolverán tras reiniciar el daemon." }, "todos": { "title": "Tareas", diff --git a/web/src/i18n/locales/ja.json b/web/src/i18n/locales/ja.json index 8cb00cc19..484327fe0 100644 --- a/web/src/i18n/locales/ja.json +++ b/web/src/i18n/locales/ja.json @@ -2599,7 +2599,9 @@ "layerDiagnostics": "スキルレイヤー診断", "enforced": "強制", "additive": "追加" - } + }, + "short_ref_failure_short": "メモリハンドル未保存", + "short_ref_failure_detail": "メモリハンドルの保存が停止しました({{stage}}、{{failures}} 件、最終 {{when}}):{{error}}。現在発行されたハンドルは daemon 再起動後に解決できません。" }, "todos": { "title": "タスク", diff --git a/web/src/i18n/locales/ko.json b/web/src/i18n/locales/ko.json index 7427260f6..cfc40b6f3 100644 --- a/web/src/i18n/locales/ko.json +++ b/web/src/i18n/locales/ko.json @@ -2599,7 +2599,9 @@ "layerDiagnostics": "스킬 계층 진단", "enforced": "강제", "additive": "추가" - } + }, + "short_ref_failure_short": "메모리 핸들 저장 실패", + "short_ref_failure_detail": "메모리 핸들 저장이 중단되었습니다({{stage}}, {{failures}}회, 최근 {{when}}): {{error}}. 지금 발급된 핸들은 daemon 재시작 후 해석되지 않습니다." }, "todos": { "title": "작업", diff --git a/web/src/i18n/locales/ru.json b/web/src/i18n/locales/ru.json index 1a6fedc0e..be41787b7 100644 --- a/web/src/i18n/locales/ru.json +++ b/web/src/i18n/locales/ru.json @@ -2599,7 +2599,9 @@ "layerDiagnostics": "Диагностика слоев навыков", "enforced": "Обязательный", "additive": "Добавочный" - } + }, + "short_ref_failure_short": "дескрипторы памяти не сохраняются", + "short_ref_failure_detail": "Дескрипторы памяти перестали сохраняться ({{stage}}, ошибок: {{failures}}, последняя {{when}}): {{error}}. Выданные сейчас не разрешатся после перезапуска daemon." }, "todos": { "title": "Задачи", diff --git a/web/src/i18n/locales/zh-CN.json b/web/src/i18n/locales/zh-CN.json index 20c7404ea..00eef97e4 100644 --- a/web/src/i18n/locales/zh-CN.json +++ b/web/src/i18n/locales/zh-CN.json @@ -2600,7 +2600,9 @@ "layerDiagnostics": "技能层级诊断", "enforced": "强制", "additive": "附加" - } + }, + "short_ref_failure_short": "记忆句柄未保存", + "short_ref_failure_detail": "记忆句柄已停止保存({{stage}},{{failures}} 次失败,最近 {{when}}):{{error}}。现在签发的句柄在 daemon 重启后将无法解析。" }, "todos": { "title": "任务清单", diff --git a/web/src/i18n/locales/zh-TW.json b/web/src/i18n/locales/zh-TW.json index 688c50ef8..cd05b29c1 100644 --- a/web/src/i18n/locales/zh-TW.json +++ b/web/src/i18n/locales/zh-TW.json @@ -2600,7 +2600,9 @@ "layerDiagnostics": "技能層級診斷", "enforced": "強制", "additive": "附加" - } + }, + "short_ref_failure_short": "記憶句柄未儲存", + "short_ref_failure_detail": "記憶句柄已停止儲存({{stage}},{{failures}} 次失敗,最近 {{when}}):{{error}}。現在簽發的句柄在 daemon 重啟後將無法解析。" }, "todos": { "title": "任務清單", From 8bf59517a59eb6e94d978b8f462b8235d74f3301 Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sun, 26 Jul 2026 09:19:03 +0800 Subject: [PATCH 30/40] Cover the last hop of the short-ref failure signal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The health record already had tests for producing it and for relaying it through the bridge, but nothing asserted it becomes something a person sees. Removing either render site, or the field mapping off the daemon.stats frame, left the suite fully green — so the visible outlet was only as good as a typecheck. Covers both status-bar layouts, asserts a healthy daemon shows no marker (a permanently-lit warning is ignored precisely when it matters), and checks the tooltip names the stage, count and underlying error rather than just alarming. The locale check reads the real files: the t() mock renders any key handed to it, so it would stay green against a key that ships nowhere, which is how a warning turns into a raw i18n path in production. Co-Authored-By: Claude Opus 4.8 --- web/test/components/SubSessionBar.test.tsx | 109 +++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/web/test/components/SubSessionBar.test.tsx b/web/test/components/SubSessionBar.test.tsx index 8b1967c15..6b7569b5c 100644 --- a/web/test/components/SubSessionBar.test.tsx +++ b/web/test/components/SubSessionBar.test.tsx @@ -1,6 +1,8 @@ /** * @vitest-environment jsdom */ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { h } from 'preact'; import { act, cleanup, fireEvent, render, waitFor } from '@testing-library/preact'; @@ -13,6 +15,10 @@ vi.mock('react-i18next', () => ({ if (key === 'subsessionBar.p2p_discussions') return 'Team discussions'; if (key === 'repo.info_title') return 'Repository information'; if (key === 'subsessionBar.scheduled_tasks') return 'Scheduled Tasks'; + if (key === 'memory.short_ref_failure_short') return 'memory handles failing'; + if (key === 'memory.short_ref_failure_detail') { + return `stage=${vars?.stage} failures=${vars?.failures} at=${vars?.when} error=${vars?.error}`; + } return key; }, }), @@ -49,6 +55,9 @@ import { SubSessionBar } from '../../src/components/SubSessionBar.js'; import { reorderSubSessions } from '../../src/api.js'; import type { SubSession } from '../../src/hooks/useSubSessions.js'; import { SUBSESSION_ACCENT_COLORS } from '../../src/subsession-accent-colors.js'; +import { SUPPORTED_LOCALES } from '../../src/i18n/locales/index.js'; + +const LOCALE_DIR = join(process.cwd().endsWith('/web') ? process.cwd() : join(process.cwd(), 'web'), 'src/i18n/locales'); function makeSubSession(overrides: Partial = {}): SubSession { return { @@ -1056,4 +1065,104 @@ describe('SubSessionBar', () => { }); }); + // The health record already travelled daemon -> bridge -> browser; these cover the + // last hop, where it either becomes something an operator sees or dies silently. + const failingHealth = { + stage: 'persist_store', + failures: 3, + lastFailureAt: new Date(2026, 4, 12, 7, 8, 9).getTime(), + lastError: 'ENOSPC: no space left on device', + }; + + function renderBarWithStats(collapsed: boolean, statsWs: ReturnType) { + return render( + , + ); + } + + it.each([['collapsed', true], ['expanded', false]] as const)( + 'surfaces a short-ref persistence failure in the %s status bar', + async (_label, collapsed) => { + const statsWs = makeStatsWs(); + const view = renderBarWithStats(collapsed, statsWs); + await waitFor(() => expect(statsWs.ws.onMessage).toHaveBeenCalled()); + + // A healthy daemon must not show the marker, otherwise it is noise and + // gets ignored exactly when it matters. + act(() => { statsWs.emit(daemonStatsMessage); }); + expect(view.container.querySelector('.daemon-stat-shortref-alert')).toBeNull(); + + act(() => { statsWs.emit({ ...daemonStatsMessage, shortRefHealth: failingHealth }); }); + + const alert = view.container.querySelector('.daemon-stat-shortref-alert') as HTMLElement; + expect(alert).toBeTruthy(); + expect(alert.textContent).toContain('memory handles failing'); + // The tooltip has to name the stage, the count and the underlying error; + // "something is wrong" is not actionable at 3am. + expect(alert.title).toContain('persist_store'); + expect(alert.title).toContain('failures=3'); + expect(alert.title).toContain('ENOSPC'); + }, + ); + + it('keeps the marker visible on mobile, unlike the disk readout', async () => { + // Disk usage is desktop-only by request, but a failure that silently breaks + // memory recall should not depend on which client happens to be open. + const statsWs = makeStatsWs(); + const view = render( + , + ); + await waitFor(() => expect(statsWs.ws.onMessage).toHaveBeenCalled()); + act(() => { + statsWs.emit({ + ...daemonStatsMessage, + disks: [{ mount: '/', usedBytes: 5 * 1024 ** 4, totalBytes: 8 * 1024 ** 4, usedPercent: 62 }], + shortRefHealth: failingHealth, + }); + }); + + expect(view.container.querySelector('.daemon-stat-disk')).toBeNull(); + expect(view.container.querySelector('.daemon-stat-shortref-alert')).toBeTruthy(); + }); + + it('ships both alert strings in every locale, with the placeholders the tooltip passes', () => { + // The t() mock above would happily render a key that does not exist in + // production, so check the real locale files rather than trusting it. + for (const locale of SUPPORTED_LOCALES) { + const raw = JSON.parse(readFileSync(join(LOCALE_DIR, `${locale}.json`), 'utf8')) as Record>; + const short = raw.memory?.short_ref_failure_short; + const detail = raw.memory?.short_ref_failure_detail; + expect(short, `${locale}: memory.short_ref_failure_short`).toBeTruthy(); + expect(detail, `${locale}: memory.short_ref_failure_detail`).toBeTruthy(); + for (const placeholder of ['stage', 'failures', 'when', 'error']) { + expect(detail, `${locale}: {{${placeholder}}}`).toContain(`{{${placeholder}}}`); + } + } + }); }); From c740cee11eca1c4f87045f4e427e7db21c0a76d5 Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sun, 26 Jul 2026 09:21:02 +0800 Subject: [PATCH 31/40] Gate rework finalization on fresh audit --- src/daemon/supervision-automation.ts | 20 ++- src/daemon/supervision-prompts.ts | 14 +- test/daemon/supervision-automation.test.ts | 199 ++++++++++++++++++++- test/daemon/supervision-prompts.test.ts | 23 ++- 4 files changed, 240 insertions(+), 16 deletions(-) diff --git a/src/daemon/supervision-automation.ts b/src/daemon/supervision-automation.ts index 5451968ed..5a7ae05aa 100644 --- a/src/daemon/supervision-automation.ts +++ b/src/daemon/supervision-automation.ts @@ -94,6 +94,9 @@ interface ActiveTaskRunState { // as soon as either automation or the current session delegates the audit, // and stays false while waiting for the reply. requiresAudit: boolean; + // Sticky safety gate set by REWORK. A broker decision cannot clear it; only + // a later matching peer-audit PASS may authorize repository finalization. + freshAuditRequiredAfterRework: boolean; continueLoops: number; continueStreakCount: number; lastContinueBucket?: string; @@ -217,19 +220,19 @@ function classifyContinueBucket(decision: { nextAction?: string; gap?: string; r return text.slice(0, 120); } -const REPOSITORY_FINALIZATION_ACTION_RE = /(?:\b(?:git\s+(?:add|commit|push)|commit|push|stage|staging)\b|提交|推送|暂存)/iu; -const SUBSTANTIVE_PRE_AUDIT_ACTION_RE = /(?:\b(?:test|tests|testing|typecheck|lint|build|verify|verification|validate|validation|fix|repair|implement|edit|modify|update|write|refactor|deploy|release|restart)\b|测试|类型检查|构建|验证|修复|实现|修改|更新|编写|重构|部署|发布|重启)/iu; +const REPOSITORY_FINALIZATION_ACTION_RE = /(?:\b(?:git\s+(?:add|commit|push|merge)|commit|push|stage|staging|merge|release|deploy|publish)\b|提交|推送|暂存|合并|发布|部署|上线)/iu; +const SUBSTANTIVE_PRE_AUDIT_ACTION_RE = /(?:\b(?:test|tests|testing|typecheck|lint|build|verify|verification|validate|validation|fix|repair|implement|edit|modify|update|write|refactor|restart)\b|测试|类型检查|构建|验证|修复|实现|修改|更新|编写|重构|重启)/iu; const COMPLETED_PRE_AUDIT_WORK_RE = /(?:\b(?:implementation|fix(?:es)?|coding|changes?|tests?|testing|typecheck|lint|build|verification|validation)\b[\s\S]{0,80}\b(?:complete|completed|done|finished|pass(?:ed)?)\b|(?:修复|实现|代码|改动|测试|验证|检查|类型检查|构建)[\s\S]{0,60}(?:已完成|已经完成|均已完成|全部完成|完成并通过|已通过|验证通过|测试通过))/iu; const PENDING_PRE_AUDIT_WORK_RE = /(?:\b(?:still|yet|remaining|pending|missing|failed?|incomplete|need(?:s)?\s+to|must)\b[\s\S]{0,50}\b(?:implementation|fix(?:es)?|tests?|testing|typecheck|lint|build|verification|validation)\b|\b(?:implementation|fix(?:es)?|tests?|testing|typecheck|lint|build|verification|validation)\b[\s\S]{0,50}\b(?:remain(?:s|ing)?|pending|missing|fail(?:ed|ing)?|incomplete|not\s+(?:done|complete)|need(?:s)?|required)\b|(?:仍|还|尚|待|未|缺少|失败)[\s\S]{0,30}(?:测试|验证|修复|实现|构建|类型检查)|(?:测试|验证|修复|实现|构建|类型检查)[\s\S]{0,30}(?:未完成|仍需|还需|待处理|失败|缺失|未通过))/iu; -const POST_AUDIT_REPOSITORY_FINALIZATION_ACTION = 'Peer-audit has passed. Finalize only the already-audited repository changes: stage the intended task files, commit them, and push the current branch. Do not request or start another audit.'; +const POST_AUDIT_REPOSITORY_FINALIZATION_ACTION = 'Peer-audit has passed. Perform only the already-audited repository or delivery finalization requested for this task (stage/commit/push, merge, release, publish, or deploy as applicable). Do not perform additional implementation work. Do not request or start another audit.'; type RepositoryFinalizationClassification = 'none' | 'finalization_only' | 'completion_evidenced_mixed'; /** * `supervised_audit` must review the implementation before repository * finalization. Only hold an action whose imperative next step is purely - * stage/commit/push work. Any instruction that also asks for tests, fixes, - * implementation, build, deployment, or another substantive mutation stays + * stage/commit/push/merge/release/deploy work. Any instruction that also asks + * for tests, fixes, implementation, build, or another substantive mutation stays * in the normal pre-audit continue loop. Audit/review words are deliberately * not substantive here: "commit after peer-audit PASS" describes the gate * this function is deciding to start, rather than work the target session @@ -748,6 +751,7 @@ class SupervisionAutomation { userText: text, phase: 'execution', requiresAudit: snapshot.mode === SUPERVISION_MODE.SUPERVISED_AUDIT, + freshAuditRequiredAfterRework: false, continueLoops: 0, continueStreakCount: 0, evaluating: false, @@ -1139,7 +1143,7 @@ class SupervisionAutomation { const latest = this.activeRuns.get(run.sessionName); if (!latest || latest.generation !== run.generation || latest.phase !== evaluatedPhase) return; latest.evaluating = false; - latest.requiresAudit = decision.requiresAudit !== false; + latest.requiresAudit = latest.freshAuditRequiredAfterRework || decision.requiresAudit !== false; switch (decision.decision) { case 'complete': { @@ -1220,7 +1224,7 @@ class SupervisionAutomation { const guardedNextAction = latest.phase === 'execution' && latest.snapshot.mode === SUPERVISION_MODE.SUPERVISED_AUDIT && hasRepositoryFinalizationAction(decision) - ? 'Complete only the remaining substantive implementation or validation work described by the supervisor. Do not stage, commit, or push; repository finalization is deferred until peer-audit PASS.' + ? 'Complete only the remaining substantive implementation or validation work described by the supervisor. Do not stage, commit, or push; do not merge, release, publish, or deploy. Repository and delivery finalization are deferred until peer-audit PASS.' : decision.nextAction; await this.dispatchContinue(latest, { reason: decision.reason, @@ -1414,6 +1418,7 @@ class SupervisionAutomation { if (verdict === 'PASS') { this.emitOrchestratedAuditResult(current, 'pass', undefined, findings); current.auditAttemptId = undefined; + current.freshAuditRequiredAfterRework = false; if (current.deferredFinalization) { current.phase = 'finalizing'; current.ignoreIdleUntilPostAuditTurnActivity = options.settledWithoutIdle === true; @@ -1443,6 +1448,7 @@ class SupervisionAutomation { const reworkBrief = buildReworkBrief(current, findings); current.phase = 'execution'; current.requiresAudit = true; + current.freshAuditRequiredAfterRework = true; current.ignoreIdleUntilPostAuditTurnActivity = options.settledWithoutIdle === true; current.evaluating = false; current.sawAssistantOutput = false; diff --git a/src/daemon/supervision-prompts.ts b/src/daemon/supervision-prompts.ts index 095159d59..b16d53a8b 100644 --- a/src/daemon/supervision-prompts.ts +++ b/src/daemon/supervision-prompts.ts @@ -67,11 +67,12 @@ function buildAuditBeforeFinalizationRule(request: SupervisionBrokerRequest): st if (request.snapshot?.mode !== SUPERVISION_MODE.SUPERVISED_AUDIT) return ''; return [ 'Audit-order rule for supervised_audit:', - '- Peer audit MUST finish before repository commit/push finalization.', - '- If implementation and validation are complete and the ONLY remaining action is git add/commit/push, return continue with that exact finalization-only nextAction; the daemon will hold it until peer-audit PASS instead of sending it now.', - '- If fixes, tests, typecheck, lint, build, validation, documentation changes, deployment, or any other substantive work remains, return continue normally so that work happens before audit.', - '- Never combine substantive pre-audit work and post-audit commit/push in one nextAction.', - '- If both the assistant response and your reason say implementation/validation are already complete, NEVER invent generic "remaining implementation or validation" work. Return only the concrete git add/commit/push finalization nextAction.', + '- Peer audit MUST finish before repository or delivery finalization, including git add/commit/push, merge, release, publish, and deploy actions.', + '- A REWORK verdict means the previous audit did NOT pass and grants no repository-finalization authority. After applying the requested fixes and validations, require a fresh matching peer audit and a new PASS before any git add/commit/push, merge, release, publish, or deploy action.', + '- If implementation and validation are complete and the ONLY remaining action is finalization such as git add/commit/push, merge, release, publish, or deploy, return continue with that exact finalization-only nextAction; the daemon will hold it until peer-audit PASS instead of sending it now.', + '- If fixes, tests, typecheck, lint, build, validation, documentation changes, or any other non-finalization substantive work remains, return continue normally so that work happens before audit.', + '- Never combine substantive pre-audit work and post-audit finalization in one nextAction.', + '- If both the assistant response and your reason say implementation/validation are already complete, NEVER invent generic "remaining implementation or validation" work. Return only the concrete repository or delivery finalization nextAction (git add/commit/push, merge, release, publish, or deploy as applicable).', '- Do not ask the target session to arrange or resend the audit in a normal continue decision. The daemon emits a separate orchestration prompt containing the exact auditor session ID and reply-enabled send command, exactly once.', ].join('\n'); } @@ -394,5 +395,8 @@ export function buildReworkBriefPrompt( verdictText, '', 'Apply the required fixes and continue the same task.', + 'This REWORK verdict means the previous audit did not pass. It is not authorization to finalize the repository.', + 'After fixing and validating the change, stop before repository finalization and report that the implementation is ready for a fresh peer audit.', + 'Do not stage, commit, push, merge, release, publish, or deploy until a new matching peer audit returns PASS.', ].join('\n'); } diff --git a/test/daemon/supervision-automation.test.ts b/test/daemon/supervision-automation.test.ts index d47c31f95..b1547721b 100644 --- a/test/daemon/supervision-automation.test.ts +++ b/test/daemon/supervision-automation.test.ts @@ -109,6 +109,14 @@ async function waitForRunEnd(timeoutMs = 10_000) { } } +async function waitForTransportSendCount(expectedCount: number, timeoutMs = 10_000) { + const deadline = performance.now() + timeoutMs; + while (mockTransportRuntime.send.mock.calls.length < expectedCount) { + if (performance.now() >= deadline) return; + await new Promise((resolve) => setImmediate(resolve)); + } +} + let projectDir: string | null = null; beforeEach(() => { @@ -1194,14 +1202,201 @@ describe('SupervisionAutomation', () => { await sleep(25); expect(mockTransportRuntime.send).toHaveBeenCalledTimes(2); - expect(String(mockTransportRuntime.send.mock.calls[1]?.[0])).toContain('Audit verdict: REWORK'); - expect(String(mockTransportRuntime.send.mock.calls[1]?.[0])).not.toContain('Commit the completed changes and push to origin/dev.'); + const reworkPrompt = String(mockTransportRuntime.send.mock.calls[1]?.[0]); + expect(reworkPrompt).toContain('Audit verdict: REWORK'); + expect(reworkPrompt).toContain('Do not stage, commit, push, merge, release, publish, or deploy until a new matching peer audit returns PASS.'); + expect(reworkPrompt).not.toContain('Commit the completed changes and push to origin/dev.'); expect(supervisionAutomation.getActiveRun('deck_supervision_brain')).toMatchObject({ phase: 'execution', reworkDispatches: 1, }); }); + it('requires a fresh PASS after REWORK before releasing deferred commit and push', async () => { + const snapshot = await seedSession('supervised_audit', false, 1); + mockSupervisionDecide + .mockResolvedValueOnce({ + decision: 'continue', + reason: 'only repository finalization remains', + confidence: 0.9, + gap: 'changes are uncommitted', + nextAction: 'Commit the completed changes and push to origin/dev.', + }) + .mockResolvedValueOnce({ + decision: 'complete', + reason: 'the rework and validation are complete', + confidence: 0.9, + requiresAudit: false, + }); + + supervisionAutomation.init(); + supervisionAutomation.registerTaskIntent( + 'deck_supervision_brain', + 'cmd-rework-fresh-pass-before-push', + 'implement the feature', + snapshot, + ); + beginRun('cmd-rework-fresh-pass-before-push', 'implement the feature'); + completeTurn('Implementation and validation are complete.'); + await waitForRunPhase('auditing'); + + completeDelegatedAudit('REWORK', 'Add the missing regression coverage.'); + await waitForRunPhase('execution'); + expect(mockTransportRuntime.send).toHaveBeenCalledTimes(2); + + completeTurn('The requested rework and validation are complete; no repository finalization was performed.'); + await waitForRunPhase('auditing'); + expect(mockTransportRuntime.send).toHaveBeenCalledTimes(3); + expect(String(mockTransportRuntime.send.mock.calls[2]?.[0])).toContain('imcodes send --reply'); + expect(mockTransportRuntime.send.mock.calls.some((call) => + String(call[0]).includes('Commit the completed changes and push to origin/dev.'))).toBe(false); + expect(supervisionAutomation.getActiveRun('deck_supervision_brain')).toMatchObject({ + phase: 'auditing', + freshAuditRequiredAfterRework: true, + deferredFinalization: { + nextAction: 'Commit the completed changes and push to origin/dev.', + }, + }); + + completeDelegatedAudit('PASS', 'The corrected implementation and regression coverage pass.'); + await waitForRunPhase('finalizing'); + expect(mockTransportRuntime.send).toHaveBeenCalledTimes(4); + expect(String(mockTransportRuntime.send.mock.calls[3]?.[0])).toContain( + 'Commit the completed changes and push to origin/dev.', + ); + expect(supervisionAutomation.getActiveRun('deck_supervision_brain')).toMatchObject({ + phase: 'finalizing', + freshAuditRequiredAfterRework: false, + }); + }); + + it.each([ + ['merge', 'Merge the repaired branch into master.'], + ['release', 'Create the release for the repaired change.'], + ['publish', 'Publish the repaired package.'], + ['deploy', 'Deploy the repaired change to production.'], + ['Chinese merge', '将当前分支合并到 master。'], + ['Chinese release', '发布当前版本。'], + ['Chinese deploy', '部署当前版本到生产环境。'], + ['Chinese go-live', '将当前版本上线。'], + ])('holds %s finalization after REWORK until a fresh PASS', async (_kind, finalizationAction) => { + const snapshot = await seedSession('supervised_audit', false, 1); + mockSupervisionDecide + .mockResolvedValueOnce({ + decision: 'complete', + reason: 'the initial implementation is ready for audit', + confidence: 0.9, + requiresAudit: true, + }) + .mockResolvedValueOnce({ + decision: 'continue', + reason: 'the requested rework and validation are complete; only delivery finalization remains', + confidence: 0.9, + requiresAudit: false, + gap: 'the repaired change has not been finalized', + nextAction: finalizationAction, + }); + + supervisionAutomation.init(); + supervisionAutomation.registerTaskIntent( + 'deck_supervision_brain', + `cmd-rework-${_kind}-fresh-pass`, + 'implement and deliver the feature', + snapshot, + ); + beginRun(`cmd-rework-${_kind}-fresh-pass`, 'implement and deliver the feature'); + completeTurn('The initial implementation and validation are complete.'); + await waitForRunPhase('auditing'); + + completeDelegatedAudit('REWORK', 'Repair the audit finding before delivery.'); + await waitForRunPhase('execution'); + completeTurn('The audit finding is repaired and validation passes. No finalization was performed.'); + await waitForRunPhase('auditing'); + + expect(mockTransportRuntime.send).toHaveBeenCalledTimes(3); + expect(String(mockTransportRuntime.send.mock.calls[2]?.[0])).toContain('imcodes send --reply'); + expect(mockTransportRuntime.send.mock.calls.some((call) => + String(call[0]).includes(finalizationAction))).toBe(false); + expect(supervisionAutomation.getActiveRun('deck_supervision_brain')).toMatchObject({ + phase: 'auditing', + freshAuditRequiredAfterRework: true, + deferredFinalization: { nextAction: finalizationAction }, + }); + + completeDelegatedAudit('PASS', 'The repaired change passes the fresh audit.'); + await waitForRunPhase('finalizing'); + expect(mockTransportRuntime.send).toHaveBeenCalledTimes(4); + expect(String(mockTransportRuntime.send.mock.calls[3]?.[0])).toContain(finalizationAction); + expect(supervisionAutomation.getActiveRun('deck_supervision_brain')).toMatchObject({ + phase: 'finalizing', + freshAuditRequiredAfterRework: false, + }); + }); + + it('strips mixed pre-audit validation and publish/deploy finalization until fresh PASS', async () => { + const snapshot = await seedSession('supervised_audit', false, 1); + const mixedAction = 'Run the focused tests, then deploy and publish the repaired release.'; + const finalizationAction = 'Deploy and publish the repaired release.'; + mockSupervisionDecide + .mockResolvedValueOnce({ + decision: 'complete', + reason: 'the initial implementation is ready for audit', + confidence: 0.9, + requiresAudit: true, + }) + .mockResolvedValueOnce({ + decision: 'continue', + reason: 'focused validation remains before delivery finalization', + confidence: 0.9, + requiresAudit: false, + gap: 'the focused tests have not run', + nextAction: mixedAction, + }) + .mockResolvedValueOnce({ + decision: 'continue', + reason: 'the repaired implementation and focused tests are complete; only delivery finalization remains', + confidence: 0.9, + requiresAudit: false, + gap: 'the repaired release has not been delivered', + nextAction: finalizationAction, + }); + + supervisionAutomation.init(); + supervisionAutomation.registerTaskIntent( + 'deck_supervision_brain', + 'cmd-rework-mixed-publish-deploy', + 'implement and deliver the feature', + snapshot, + ); + beginRun('cmd-rework-mixed-publish-deploy', 'implement and deliver the feature'); + completeTurn('The initial implementation is complete.'); + await waitForRunPhase('auditing'); + + completeDelegatedAudit('REWORK', 'Repair the finding and run focused tests.'); + await waitForRunPhase('execution'); + completeTurn('The finding is repaired, but the focused tests still need to run.'); + await waitForTransportSendCount(3); + + expect(mockTransportRuntime.send).toHaveBeenCalledTimes(3); + const preAuditContinue = String(mockTransportRuntime.send.mock.calls[2]?.[0]); + expect(preAuditContinue).toContain('Complete only the remaining substantive implementation or validation work'); + expect(preAuditContinue).toContain('Do not stage, commit, or push; do not merge, release, publish, or deploy.'); + expect(preAuditContinue).not.toContain(mixedAction); + expect(preAuditContinue).not.toContain(finalizationAction); + + completeTurn('The repaired implementation and focused tests now pass; no finalization was performed.'); + await waitForRunPhase('auditing'); + expect(mockTransportRuntime.send).toHaveBeenCalledTimes(4); + expect(String(mockTransportRuntime.send.mock.calls[3]?.[0])).toContain('imcodes send --reply'); + expect(mockTransportRuntime.send.mock.calls.some((call) => + String(call[0]).includes(finalizationAction))).toBe(false); + + completeDelegatedAudit('PASS', 'The repaired implementation and focused tests pass audit.'); + await waitForRunPhase('finalizing'); + expect(mockTransportRuntime.send).toHaveBeenCalledTimes(5); + expect(String(mockTransportRuntime.send.mock.calls[4]?.[0])).toContain(finalizationAction); + }); + it('keeps ordinary supervised commit and push continuation immediate', async () => { const snapshot = await seedSession('supervised'); mockSupervisionDecide.mockResolvedValueOnce({ diff --git a/test/daemon/supervision-prompts.test.ts b/test/daemon/supervision-prompts.test.ts index 250197246..3b2d2e2e9 100644 --- a/test/daemon/supervision-prompts.test.ts +++ b/test/daemon/supervision-prompts.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest'; import { normalizeSessionSupervisionSnapshot, SUPERVISION_MODE } from '../../shared/supervision-config.js'; import { buildPeerAuditBriefV1, + buildReworkBriefPrompt, buildSupervisionContinuePrompt, buildSupervisionDecisionPrompt, buildSupervisionDecisionRepairPrompt, @@ -126,10 +127,14 @@ describe('supervision prompts', () => { assistantResponse: 'Implementation and tests are complete; changes are not committed.', }); - expect(prompt).toContain('Peer audit MUST finish before repository commit/push finalization.'); + expect(prompt).toContain('Peer audit MUST finish before repository or delivery finalization'); + expect(prompt).toContain('A REWORK verdict means the previous audit did NOT pass'); + expect(prompt).toContain('require a fresh matching peer audit and a new PASS before any git add/commit/push'); + expect(prompt).toContain('merge, release, publish, or deploy'); expect(prompt).toContain('the daemon will hold it until peer-audit PASS instead of sending it now'); - expect(prompt).toContain('Never combine substantive pre-audit work and post-audit commit/push in one nextAction.'); + expect(prompt).toContain('Never combine substantive pre-audit work and post-audit finalization in one nextAction.'); expect(prompt).toContain('NEVER invent generic "remaining implementation or validation" work'); + expect(prompt).toContain('Return only the concrete repository or delivery finalization nextAction (git add/commit/push, merge, release, publish, or deploy as applicable).'); expect(prompt).toContain('exact auditor session ID and reply-enabled send command, exactly once'); expect(prompt).toContain('"requiresAudit":true'); expect(prompt).toContain('Set false for ordinary read-only checks, status queries, lookups, explanations, simple verification, and read-only review/audit.'); @@ -139,6 +144,20 @@ describe('supervision prompts', () => { expect(prompt).toContain('A task that starts as a check but proceeds to modify/fix something requires audit unless its matching audit is already pending or passed.'); }); + it('forbids repository finalization after REWORK until a fresh peer audit passes', () => { + const prompt = buildReworkBriefPrompt( + 'deck_supervision_brain', + 'Implement and deliver the fix', + 'The first implementation is ready.', + 'The auditor found a missing regression test.', + ); + + expect(prompt).toContain('This REWORK verdict means the previous audit did not pass.'); + expect(prompt).toContain('stop before repository finalization'); + expect(prompt).toContain('ready for a fresh peer audit'); + expect(prompt).toContain('Do not stage, commit, push, merge, release, publish, or deploy until a new matching peer audit returns PASS.'); + }); + it('does NOT include IM.codes workflow background in the continue prompt', () => { // Regression guard. The continue prompt is sent to the TARGET session's // chat, not to the supervisor judge. Injecting the IM.codes capability From 485b2bb0d1799556de870d35587148972a1500c5 Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sun, 26 Jul 2026 10:32:21 +0800 Subject: [PATCH 32/40] Keep legacy memory handles readable on Node 24 --- src/context/memory-short-ref.ts | 28 +++++++++---------- .../memory-short-ref-legacy-key.test.ts | 26 +++++++++++++---- 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/src/context/memory-short-ref.ts b/src/context/memory-short-ref.ts index b053cae54..b04758328 100644 --- a/src/context/memory-short-ref.ts +++ b/src/context/memory-short-ref.ts @@ -65,13 +65,12 @@ function sameNamespace(a: ContextNamespace | undefined, b: ContextNamespace | un /** * Namespace key for the persisted row. * - * `namespaceKey()` separates fields with NUL, which is fine in memory but reads - * back from node:sqlite as just the leading scope: the driver stops at the first - * NUL when converting TEXT to a JS string. The bytes are stored intact and the - * primary key was never affected — a previous version of this comment claimed a - * collapsed primary key, which measurement disproved — but a key that cannot be - * read back is useless for verifying a row against its namespace. JSON keeps the - * same field order with no embedded NUL, so it survives the round trip. + * `namespaceKey()` separates fields with NUL, which is fine in memory. Older + * node:sqlite versions returned only the leading scope when reading it as TEXT; + * current versions return the complete string. The bytes are stored intact and + * the primary key was never affected, but the version-dependent read shape is a + * poor persisted identity. JSON keeps the same field order with no embedded NUL + * and round-trips consistently. */ function namespaceStorageKey(namespace: ContextNamespace | undefined): string { if (!namespace) return ''; @@ -439,15 +438,16 @@ export async function loadMemoryShortRefsFromStore(): Promise { // under the wrong namespace. // // Rows written before the key became a JSON tuple used NUL-separated - // fields, and node:sqlite truncates a NUL-containing TEXT value at the - // first NUL when it converts to a JS string — so those rows read back as - // just the scope. (The bytes are stored intact and the primary key was - // never affected; an earlier comment here claimed otherwise and was - // wrong.) Accept that legacy shape so an upgrade does not discard every - // handle written before it. + // fields. Older node:sqlite versions truncate that TEXT value at the + // first NUL when converting it to a JS string, while current versions + // return the complete string. Accept both read-back shapes so upgrading + // Node cannot discard every handle written before the key migration. const storedNamespaceKey = typeof row.namespaceKey === 'string' ? row.namespaceKey : ''; + const legacyNamespaceKey = namespaceKey(namespace); const legacyTruncatedKey = namespace ? namespace.scope : ''; - if (storedNamespaceKey !== namespaceStorageKey(namespace) && storedNamespaceKey !== legacyTruncatedKey) { + if (storedNamespaceKey !== namespaceStorageKey(namespace) + && storedNamespaceKey !== legacyNamespaceKey + && storedNamespaceKey !== legacyTruncatedKey) { discarded += 1; continue; } diff --git a/test/context/memory-short-ref-legacy-key.test.ts b/test/context/memory-short-ref-legacy-key.test.ts index 0f833bd36..fc5d93a56 100644 --- a/test/context/memory-short-ref-legacy-key.test.ts +++ b/test/context/memory-short-ref-legacy-key.test.ts @@ -15,10 +15,10 @@ import { cleanupIsolatedSharedContextDb, createIsolatedSharedContextDb } from '. * reproduce. * * Keys used to be NUL-separated. The bytes store fine and the primary key is - * unaffected, but node:sqlite stops at the first NUL when converting TEXT to a - * JS string, so such a row reads back as just its scope. Verifying that against - * the current JSON-tuple key would discard every handle written before the - * format changed — an upgrade that silently empties the cache. + * unaffected. Older node:sqlite versions returned only the leading scope, + * while current versions return the complete NUL-containing JS string. + * Verifying either legacy shape only against the current JSON-tuple key would + * discard every handle written before the format changed. */ describe('memory short refs — legacy NUL-separated namespace keys', () => { let tempDir: string; @@ -62,9 +62,23 @@ describe('memory short refs — legacy NUL-separated namespace keys', () => { expect(resolveMemoryShortRef(ref, namespace)).toMatchObject({ id }); }); + it('still accepts the truncated read-back shape from older node:sqlite versions', async () => { + const id = 'eeeeeeee-1111-2222-3333-444444444444'; + const ref = makeMemoryShortRef('projection', id); + upsertMemoryShortRefs([{ + ref, kind: 'projection', id, + namespaceKey: namespace.scope, + namespaceJson: JSON.stringify(namespace), + lastSeenAt: 1, + }]); + + await expect(loadMemoryShortRefsFromStore()).resolves.toBeGreaterThanOrEqual(1); + expect(resolveMemoryShortRef(ref, namespace)).toMatchObject({ id }); + }); + it('keeps two legacy rows that differ only by namespace apart', async () => { - // The legacy key reads back as the shared scope, so accepting it must not - // let one namespace's handle answer for another's. + // Accepting either legacy key shape must not let one namespace's handle + // answer for another's. const id = 'aaaaaaaa-1111-2222-3333-444444444444'; const ref = makeMemoryShortRef('projection', id); const other = { scope: 'personal' as const, userId: 'u2', projectId: 'p2' }; From 98be3acd3a4c9d8a3a6ff94955c42640aba14772 Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sun, 26 Jul 2026 10:44:56 +0800 Subject: [PATCH 33/40] Align memory MCP E2E refs with current handles --- test/e2e/memory-mcp-interface.test.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/e2e/memory-mcp-interface.test.ts b/test/e2e/memory-mcp-interface.test.ts index 9ee11d108..0687c55bf 100644 --- a/test/e2e/memory-mcp-interface.test.ts +++ b/test/e2e/memory-mcp-interface.test.ts @@ -13,6 +13,7 @@ import { import { ALIAS_MCP_TOOLS } from '../../shared/alias-types.js'; import { MEMORY_FEATURE_FLAGS_BY_NAME, memoryFeatureFlagEnvKey } from '../../shared/feature-flags.js'; import { MEMORY_MCP_ENV_KEYS, buildMemoryMcpServerEnv } from '../../shared/memory-mcp-env.js'; +import { makeMemoryShortRef } from '../../src/context/memory-short-ref.js'; import { createMemoryMcpToolHandlers } from '../../src/daemon/memory-mcp-tools.js'; import type { McpRuntimeCaller } from '../../src/daemon/memory-mcp-caller.js'; import { @@ -199,7 +200,7 @@ describe('memory MCP interface e2e', () => { })); expect(search).toMatchObject({ status: 'ok' }); const items = search.items as Array>; - const expectedRef = `obs:${observationId.replace(/[^a-f0-9]/gi, '').slice(0, 10)}`; + const expectedRef = makeMemoryShortRef('observation', observationId); expect(items[0]).toMatchObject({ observationId, ref: expectedRef, @@ -275,7 +276,7 @@ describe('memory MCP interface e2e', () => { const listedItems = listed.items as Array>; expect(listedItems.find((item) => item.projectionId === projection.id)).toMatchObject({ projectionId: projection.id, - ref: `proj:${projection.id.replace(/[^a-f0-9]/gi, '').slice(0, 10)}`, + ref: makeMemoryShortRef('projection', projection.id), recordKind: 'projection', projectionClass: 'recent_summary', sourceLookup: { @@ -305,7 +306,7 @@ describe('memory MCP interface e2e', () => { }, }); expect(['exact', 'semantic', 'trigram']).toContain(hit?.matchKind); - const expectedRef = `proj:${projection.id.replace(/[^a-f0-9]/gi, '').slice(0, 10)}`; + const expectedRef = makeMemoryShortRef('projection', projection.id); expect(hit?.ref).toBe(expectedRef); const sources = structured(await client.callTool({ From d2236b6d36305c242f5bd1047dcc8dc79d225fb2 Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sun, 26 Jul 2026 13:24:11 +0800 Subject: [PATCH 34/40] Self-heal blocked daemon npm upgrades --- server/src/ws/bridge.ts | 267 +++++++- server/src/ws/daemon-upgrade-coordinator.ts | 139 +++- server/test/bridge.test.ts | 625 +++++++++++++++++- .../test/daemon-upgrade-coordinator.test.ts | 113 ++++ shared/daemon-events.ts | 2 + shared/daemon-upgrade.ts | 18 + src/daemon/command-handler.ts | 89 ++- src/daemon/latency-tracer.ts | 3 + src/daemon/server-link.ts | 53 +- src/daemon/upgrade-blocked-outbox.ts | 197 ++++++ src/util/posix-upgrade-layout-recovery.ts | 175 +++++ test/daemon/server-link.test.ts | 85 +++ test/daemon/upgrade-blocked-outbox.test.ts | 90 +++ test/e2e/daemon-upgrade-gate.test.ts | 102 ++- test/shared/wire-protocol-contract.test.ts | 2 + .../posix-upgrade-layout-recovery.test.ts | 184 ++++++ web/src/app.tsx | 73 +- web/src/daemon-upgrade-blocked.ts | 4 + web/src/i18n/locales/en.json | 1 + web/src/i18n/locales/es.json | 1 + web/src/i18n/locales/ja.json | 1 + web/src/i18n/locales/ko.json | 1 + web/src/i18n/locales/ru.json | 1 + web/src/i18n/locales/zh-CN.json | 1 + web/src/i18n/locales/zh-TW.json | 1 + web/src/util/daemon-upgrade-status.ts | 8 +- web/src/ws-client.ts | 13 +- web/test/daemon-upgrade-blocked.test.ts | 2 + web/test/daemon-upgrade-status.test.ts | 18 +- 29 files changed, 2179 insertions(+), 90 deletions(-) create mode 100644 src/daemon/upgrade-blocked-outbox.ts create mode 100644 src/util/posix-upgrade-layout-recovery.ts create mode 100644 test/daemon/upgrade-blocked-outbox.test.ts create mode 100644 test/util/posix-upgrade-layout-recovery.test.ts diff --git a/server/src/ws/bridge.ts b/server/src/ws/bridge.ts index 2824de6f8..c026d755c 100644 --- a/server/src/ws/bridge.ts +++ b/server/src/ws/bridge.ts @@ -125,7 +125,11 @@ import { isKnownTestSessionLike } from '../../../shared/test-session-guard.js'; import { PUSH_TIMELINE_EVENT_MAX_AGE_MS, TIMELINE_SUPPRESS_PUSH_FIELD } from '../../../shared/push-notifications.js'; import { isServerLinkResyncStatePayload } from '../../../shared/session-activity-types.js'; import { + DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION, + DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL, + DAEMON_UPGRADE_BLOCK_REASON, DAEMON_UPGRADE_DELIVERY_STATUS, + type DaemonUpgradeBlockedAckDisposition, } from '../../../shared/daemon-upgrade.js'; import { P2P_WORKFLOW_MSG, @@ -231,6 +235,8 @@ const MAX_QUEUE_SIZE = 100; const DAEMON_UPGRADE_BLOCKED_RETRY_MS = 60_000; const DAEMON_UPGRADE_BLOCKED_MIN_RETRY_MS = 5_000; const DAEMON_UPGRADE_BLOCKED_MAX_RETRY_MS = 15 * 60 * 1000; +const DAEMON_UPGRADE_BLOCKED_FAILURE_DEDUP_TTL_MS = 7 * 24 * 60 * 60 * 1000; +const DAEMON_UPGRADE_BLOCKED_FAILURE_DEDUP_MAX = 1_000; const DAEMON_UPGRADE_TRANSIENT_BLOCK_REASONS = new Set([ 'p2p_active', 'auto_deliver_active', @@ -1137,6 +1143,11 @@ export class WsBridge { * auth check has settled. */ private authPromise: Promise | null = null; + /** Connection generation that advertised outbox-sync support during auth. */ + private upgradeBlockedSyncRequiredGeneration: number | null = null; + /** Connection generation whose persisted blocker replay has completed. */ + private upgradeBlockedSyncCompleteGeneration: number | null = null; + private seenUpgradeBlockedFailures = new Map(); private browserRateLimiter = new MemoryRateLimiter(); /** browser socket → session name → raw-enabled flag */ @@ -2415,6 +2426,7 @@ export class WsBridge { // generation (they resolve as indeterminate) so a reconnect never delivers a // stale result to a new waiter (10.6). this.daemonGeneration++; + const connectionGeneration = this.daemonGeneration; abandonPriorGenerations(this.serverId, this.daemonGeneration); // Invalidate every pending peer-audit response route: the prior daemon // is gone, and any reply that arrives for it must not be delivered to a @@ -2427,6 +2439,8 @@ export class WsBridge { // late-arriving messages don't await a stale (and possibly resolved // for a different `ws`) auth. this.authPromise = null; + this.upgradeBlockedSyncRequiredGeneration = null; + this.upgradeBlockedSyncCompleteGeneration = null; // Auth timeout this.authTimer = setTimeout(() => { @@ -2437,6 +2451,11 @@ export class WsBridge { }, AUTH_TIMEOUT_MS); ws.on('message', async (data, isBinary) => { + // A replaced socket can still emit already-buffered frames while ws is in + // CLOSING. Never let those frames borrow the replacement connection's + // authenticated/shared bridge state. + if (this.daemonWs !== ws || this.daemonGeneration !== connectionGeneration) return; + // Handle binary raw PTY frames if (isBinary) { // Binary PTY data is a FULL-daemon capability. Never route unauthenticated @@ -2464,10 +2483,11 @@ export class WsBridge { // `ws.close(4001, 'auth_required')` even though auth was about to // succeed milliseconds later. See `authPromise` field doc above. if (this.authPromise) { - try { await this.authPromise; } catch { /* ignore — closed below */ } + const pendingAuth = this.authPromise; + try { await pendingAuth; } catch { /* ignore — closed below */ } // The connection may have been closed while we awaited (auth // failed / timed out / replaced). Bail out before processing. - if (this.daemonWs !== ws) return; + if (this.daemonWs !== ws || this.daemonGeneration !== connectionGeneration) return; } if (!this.authenticated) { @@ -2486,7 +2506,19 @@ export class WsBridge { // Resolving (vs rejecting) avoids unhandled-rejection warnings // when no concurrent handler is currently awaiting. let resolveAuth!: () => void; - this.authPromise = new Promise((res) => { resolveAuth = res; }); + const localAuthPromise = new Promise((res) => { resolveAuth = res; }); + this.authPromise = localAuthPromise; + const isCurrentAuthConnection = (): boolean => + this.daemonWs === ws && this.daemonGeneration === connectionGeneration; + const finishLocalAuth = (): void => { + resolveAuth(); + // A replacement connection owns a different auth promise. A stale + // continuation may release only its own waiters; it must never clear + // the replacement's auth barrier. + if (this.authPromise === localAuthPromise) { + this.authPromise = null; + } + }; const tokenHash = sha256Hex(msg.token); let server: { token_hash: string; user_id?: string; node_role?: string | null; revoked_at?: number | null } | null = null; @@ -2496,16 +2528,23 @@ export class WsBridge { [this.serverId], ); } catch (err) { - resolveAuth(); - this.authPromise = null; + const stillCurrent = isCurrentAuthConnection(); + finishLocalAuth(); + if (!stillCurrent) return; throw err; } + // The DB lookup is the first asynchronous auth boundary. A replacement + // may have installed a new socket and auth promise while it was pending. + // Never let the stale continuation overwrite generation-bound state. + if (!isCurrentAuthConnection()) { + finishLocalAuth(); + return; + } if (!server || server.token_hash !== tokenHash) { logger.warn({ serverId: this.serverId }, 'Daemon auth failed'); ws.close(4001, 'auth_failed'); - resolveAuth(); - this.authPromise = null; + finishLocalAuth(); return; } @@ -2513,8 +2552,7 @@ export class WsBridge { if (server.revoked_at != null) { logger.warn({ serverId: this.serverId }, 'Daemon auth rejected: revoked'); ws.close(4003, 'revoked'); - resolveAuth(); - this.authPromise = null; + finishLocalAuth(); return; } @@ -2522,6 +2560,15 @@ export class WsBridge { // in the auth frame is IGNORED (10.2). A controlled node's WS is only a // presence/heartbeat + MACHINE_EXEC_RESULT surface. this.daemonNodeRole = server.node_role === NODE_ROLE.CONTROLLED ? NODE_ROLE.CONTROLLED : NODE_ROLE.FULL; + const supportsUpgradeBlockedSync = this.daemonNodeRole === NODE_ROLE.FULL + && msg[DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.AUTH_REVISION_FIELD] + === DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION; + this.upgradeBlockedSyncRequiredGeneration = supportsUpgradeBlockedSync + ? connectionGeneration + : null; + this.upgradeBlockedSyncCompleteGeneration = supportsUpgradeBlockedSync + ? null + : connectionGeneration; this.controlledFileTransferCapabilities = this.daemonNodeRole === NODE_ROLE.CONTROLLED ? new Set( (Array.isArray(msg.capabilities) ? msg.capabilities : []) @@ -2547,10 +2594,19 @@ export class WsBridge { // controlled node — it is a restricted presence/exec-result surface (10.2). if (this.daemonNodeRole !== NODE_ROLE.CONTROLLED && typeof server.user_id === 'string' && server.user_id.trim()) { try { - this.sendMemoryFeatureConfigApply(await this.readUserMemoryFeatureFlags(server.user_id)); + const memoryFeatureFlags = await this.readUserMemoryFeatureFlags(server.user_id); + if (!isCurrentAuthConnection()) { + finishLocalAuth(); + return; + } + this.sendMemoryFeatureConfigApply(memoryFeatureFlags); } catch (err) { logger.warn({ err, serverId: this.serverId }, 'failed to push global memory feature config on daemon auth'); } + if (!isCurrentAuthConnection()) { + finishLocalAuth(); + return; + } } this.daemonUpgradeCoordinator.clearIfTargetVersionMatches(this.daemonVersion); this.flushPendingDaemonUpgrade(ws); @@ -2655,7 +2711,11 @@ export class WsBridge { this.graceTimer = null; } this.daemonOfflineAnnounced = false; - await this.replayInflightToDaemon(); + await this.replayInflightToDaemon(ws, connectionGeneration); + if (!isCurrentAuthConnection()) { + finishLocalAuth(); + return; + } this.broadcastToBrowsers(JSON.stringify({ type: MSG_DAEMON_ONLINE })); this.startAckHousekeepingIfNeeded(); @@ -2664,8 +2724,20 @@ export class WsBridge { // (e.g. the `daemon.hello` that landed before auth finished) // will resume past their `await this.authPromise` and observe // `this.authenticated === true`. - resolveAuth(); - this.authPromise = null; + finishLocalAuth(); + // Do not arm an auto-upgrade timer until the auth turn has fully + // yielded. daemon.upgrade_blocked is sent immediately after auth by a + // reconnecting daemon, and its handler was waiting on authPromise. A + // next-turn flush lets that persisted terminal blocker run first, + // regardless of how long replayInflightToDaemon() made auth take. + setImmediate(() => { + if ( + this.daemonWs !== ws + || this.daemonGeneration !== connectionGeneration + || !this.authenticated + ) return; + this.flushPendingDaemonUpgrade(ws); + }); return; } @@ -2725,6 +2797,22 @@ export class WsBridge { return; } + if (msg.type === DAEMON_MSG.UPGRADE_BLOCKED_SYNC) { + if ( + this.daemonWs === ws + && this.daemonGeneration === connectionGeneration + && this.upgradeBlockedSyncRequiredGeneration === connectionGeneration + && msg.revision === DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION + ) { + this.upgradeBlockedSyncCompleteGeneration = connectionGeneration; + setImmediate(() => { + if (this.daemonWs !== ws || this.daemonGeneration !== connectionGeneration) return; + this.flushPendingDaemonUpgrade(ws); + }); + } + return; + } + if (msg.type === P2P_WORKFLOW_MSG.DAEMON_HELLO) { this.handleDaemonP2pWorkflowHello(msg); return; @@ -2749,7 +2837,7 @@ export class WsBridge { } if (msg.type === DAEMON_MSG.UPGRADE_BLOCKED) { - this.handleDaemonUpgradeBlocked(msg, ws); + if (!this.handleDaemonUpgradeBlocked(msg, ws)) return; } if (msg.type === 'heartbeat') { @@ -2826,6 +2914,8 @@ export class WsBridge { // closed, the awaiting handlers will fall through and observe // `this.daemonWs !== ws` and bail out. this.authPromise = null; + this.upgradeBlockedSyncRequiredGeneration = null; + this.upgradeBlockedSyncCompleteGeneration = null; this.recentTextBySession.clear(); this.clearPendingIdlePushes(); this.activeMainSessions.clear(); @@ -2865,6 +2955,9 @@ export class WsBridge { }); ws.on('error', (err) => { + // Errors from a replaced socket belong to its abandoned generation and + // must not reject requests owned by the current daemon connection. + if (this.daemonWs !== ws || this.daemonGeneration !== connectionGeneration) return; logger.error({ serverId: this.serverId, err }, 'Daemon WS error'); this.rejectAllPendingFileTransfers('daemon_error'); this.rejectAllPendingMemorySourcesRequests('daemon_error'); @@ -5708,20 +5801,46 @@ export class WsBridge { } /** Replay buffered + dispatched commands to the daemon after reconnect. */ - private async replayInflightToDaemon(): Promise { + private async replayInflightToDaemon(expectedWs: WebSocket, expectedGeneration: number): Promise { const ordered = [...this.inflightCommands.values()].sort((a, b) => a.sentAt - b.sentAt); for (const entry of ordered) { + if (this.daemonWs !== expectedWs || this.daemonGeneration !== expectedGeneration) return; if (entry.state === 'acked') continue; try { - await this.dispatchInflightToDaemon(entry, entry.dispatchAttempts > 0); + await this.dispatchInflightToDaemon( + entry, + entry.dispatchAttempts > 0, + { ws: expectedWs, generation: expectedGeneration }, + ); } catch (err) { logger.warn({ commandId: entry.commandId, err }, 'replayInflightToDaemon failed for entry'); } } } - private async dispatchInflightToDaemon(entry: InflightCommand, markBridgeRetry: boolean): Promise { + private async dispatchInflightToDaemon( + entry: InflightCommand, + markBridgeRetry: boolean, + expectedConnection?: { ws: WebSocket; generation: number }, + ): Promise { + if ( + expectedConnection + && ( + this.daemonWs !== expectedConnection.ws + || this.daemonGeneration !== expectedConnection.generation + ) + ) return; if (!await this.revalidateInflightShareCommand(entry)) return; + // Share revalidation can await DB/coverage work. If auth was replaced while + // it was pending, leave the entry for the new generation's replay rather + // than sending it through the replacement socket from a stale auth flow. + if ( + expectedConnection + && ( + this.daemonWs !== expectedConnection.ws + || this.daemonGeneration !== expectedConnection.generation + ) + ) return; const rawPayload = markBridgeRetry ? this.withBridgeRetryMarker(entry.rawPayload, entry.dispatchAttempts + 1) : entry.rawPayload; @@ -5946,19 +6065,80 @@ export class WsBridge { return this.seenCommandAcks.has(commandId); } - private handleDaemonUpgradeBlocked(msg: Record, ws: WebSocket): void { + private handleDaemonUpgradeBlocked(msg: Record, ws: WebSocket): boolean { const serverVersion = process.env.APP_VERSION; - if (!serverVersion || serverVersion === '0.0.0' || !this.daemonVersion || this.daemonVersion === serverVersion) return; + const failureId = typeof msg.failureId === 'string' && msg.failureId.length > 0 && msg.failureId.length <= 128 + ? msg.failureId + : null; + const failedTargetVersion = typeof msg.targetVersion === 'string' ? msg.targetVersion : null; + const failureUpgradeId = typeof msg.upgradeId === 'string' && msg.upgradeId.length > 0 && msg.upgradeId.length <= 128 + ? msg.upgradeId + : null; + if (!serverVersion || serverVersion === '0.0.0' || !this.daemonVersion || this.daemonVersion === serverVersion) { + if (failureId) { + this.sendDaemonUpgradeBlockedAck( + ws, + failureId, + DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.OBSOLETE, + ); + } + return false; + } + if ( + msg.reason === DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED + && failedTargetVersion + && failedTargetVersion !== serverVersion + ) { + this.sendDaemonUpgradeBlockedAck( + ws, + failureId, + DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.OBSOLETE, + ); + return false; + } const retryDelayMs = this.daemonUpgradeBlockedRetryDelayMs(msg); if (retryDelayMs == null) { + const replayBeforeSync = this.upgradeBlockedSyncRequiredGeneration === this.daemonGeneration + && this.upgradeBlockedSyncCompleteGeneration !== this.daemonGeneration; + const supersededByManual = msg.reason === DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED + && failedTargetVersion != null + && ( + this.daemonUpgradeCoordinator.isTerminalFailureSupersededByManual( + failedTargetVersion, + failureUpgradeId, + ) + || ( + failureUpgradeId == null + && replayBeforeSync + && this.daemonUpgradeCoordinator.hasManualLifecycleForTarget(failedTargetVersion) + ) + ); + if (supersededByManual) { + this.sendDaemonUpgradeBlockedAck( + ws, + failureId, + DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.SUPERSEDED, + ); + return false; + } + this.daemonUpgradeCoordinator.blockTargetAfterTerminalFailure(serverVersion); + const duplicate = failureId ? this.hasSeenUpgradeBlockedFailure(failureId) : false; + if (failureId) { + this.rememberUpgradeBlockedFailure(failureId); + this.sendDaemonUpgradeBlockedAck( + ws, + failureId, + DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.ACCEPTED, + ); + } logger.info({ serverId: this.serverId, daemonVersion: this.daemonVersion, serverVersion, reason: typeof msg.reason === 'string' ? msg.reason : 'unknown', }, 'daemon.upgrade blocked by non-retryable reason'); - return; + return !duplicate; } const result = this.daemonUpgradeCoordinator.retryAutoAfterBlocked({ @@ -5978,6 +6158,44 @@ export class WsBridge { upgradeId: result.upgradeId, }, 'daemon.upgrade blocked by transient daemon state — retry scheduled'); } + return true; + } + + private sendDaemonUpgradeBlockedAck( + ws: WebSocket, + failureId: string | null, + disposition: DaemonUpgradeBlockedAckDisposition, + ): void { + if (!failureId) return; + try { + ws.send(JSON.stringify({ + type: DAEMON_MSG.UPGRADE_BLOCKED_ACK, + failureId, + disposition, + })); + } catch { + // The daemon retains the blocker until it receives this ACK and will + // replay it after reconnect, rebuilding the coordinator gate. + } + } + + private hasSeenUpgradeBlockedFailure(failureId: string, now = Date.now()): boolean { + const seenAt = this.seenUpgradeBlockedFailures.get(failureId); + return seenAt !== undefined && now - seenAt <= DAEMON_UPGRADE_BLOCKED_FAILURE_DEDUP_TTL_MS; + } + + private rememberUpgradeBlockedFailure(failureId: string, now = Date.now()): void { + for (const [id, seenAt] of this.seenUpgradeBlockedFailures) { + if (now - seenAt > DAEMON_UPGRADE_BLOCKED_FAILURE_DEDUP_TTL_MS) { + this.seenUpgradeBlockedFailures.delete(id); + } + } + this.seenUpgradeBlockedFailures.set(failureId, now); + while (this.seenUpgradeBlockedFailures.size > DAEMON_UPGRADE_BLOCKED_FAILURE_DEDUP_MAX) { + const oldest = this.seenUpgradeBlockedFailures.keys().next().value as string | undefined; + if (!oldest) break; + this.seenUpgradeBlockedFailures.delete(oldest); + } } private daemonUpgradeBlockedRetryDelayMs(msg: Record): number | null { @@ -6011,7 +6229,7 @@ export class WsBridge { }); } - private flushPendingDaemonUpgrade(ws: WebSocket): void { + private flushPendingDaemonUpgrade(ws: WebSocket): RequestDaemonUpgradeResult | null { const result = this.daemonUpgradeCoordinator.flushPending({ skipPublicationGate: this.daemonNodeRole === NODE_ROLE.CONTROLLED, isDaemonReady: () => this.isDaemonReadyForUpgrade(), @@ -6031,10 +6249,13 @@ export class WsBridge { nextAttemptAt: result.nextAttemptAt, }, 'Pending daemon.upgrade is waiting for npm publication'); } + return result; } private isDaemonReadyForUpgrade(): boolean { - return Boolean(this.daemonWs && this.authenticated); + const syncReady = this.upgradeBlockedSyncRequiredGeneration !== this.daemonGeneration + || this.upgradeBlockedSyncCompleteGeneration === this.daemonGeneration; + return Boolean(this.daemonWs && this.authenticated && this.authPromise === null && syncReady); } private sendDirectToDaemon(message: Record): void { @@ -6053,6 +6274,8 @@ export class WsBridge { this.daemonWs = null; this.authenticated = false; this.authPromise = null; + this.upgradeBlockedSyncRequiredGeneration = null; + this.upgradeBlockedSyncCompleteGeneration = null; this.daemonP2pWorkflowCapabilities = null; this.controlledFileTransferCapabilities.clear(); } diff --git a/server/src/ws/daemon-upgrade-coordinator.ts b/server/src/ws/daemon-upgrade-coordinator.ts index 27d0ae1db..3f9eed994 100644 --- a/server/src/ws/daemon-upgrade-coordinator.ts +++ b/server/src/ws/daemon-upgrade-coordinator.ts @@ -17,7 +17,13 @@ const AUTO_UPGRADE_MAX_ATTEMPTS = 3; export type DaemonUpgradeSource = 'auto' | 'manual' | 'replay'; -type UpgradeLifecycleState = 'pending_offline' | 'pending_publication' | 'scheduled' | 'sent' | 'superseded'; +type UpgradeLifecycleState = + | 'pending_offline' + | 'pending_publication' + | 'scheduled' + | 'sent' + | 'terminal_blocked' + | 'superseded'; interface UpgradeState { upgradeId: string; @@ -76,15 +82,39 @@ export class DaemonUpgradeCoordinator { } const now = input.now ?? Date.now(); - const current = this.current?.targetVersion === targetVersion ? this.current : null; + let current = this.current?.targetVersion === targetVersion ? this.current : null; if (this.current && this.current.targetVersion !== targetVersion) { + const terminalTargetChanged = this.current.status === 'terminal_blocked'; this.supersedeCurrent(now); + if (terminalTargetChanged) this.lastAutoSentAt = null; } + if (current?.status === 'terminal_blocked') { + if (input.source !== 'manual') { + return { + ok: true, + upgradeId: current.upgradeId, + targetVersion, + deliveryStatus: DAEMON_UPGRADE_DELIVERY_STATUS.BACKOFF, + reason: 'terminal_install_failure', + }; + } + this.supersedeCurrent(now); + current = null; + } + + // A user-issued manual request is authoritative over reconnect-driven + // auto/replay traffic for the same target. In particular, an offline + // manual request remains pending while auth is incomplete; the auth + // version-mismatch probe must not silently demote it back to `auto`. + const effectiveInput = current?.source === 'manual' && input.source !== 'manual' + ? { ...input, source: 'manual' as const } + : input; + if (current?.status === 'pending_publication') { - current.source = input.source; + current.source = effectiveInput.source; current.updatedAt = now; - if (!input.isDaemonReady()) { + if (!effectiveInput.isDaemonReady()) { current.status = 'pending_offline'; return { ok: true, @@ -93,10 +123,14 @@ export class DaemonUpgradeCoordinator { deliveryStatus: DAEMON_UPGRADE_DELIVERY_STATUS.PENDING_OFFLINE, }; } - const publication = this.ensureTargetPublished(current, input, now); + const publication = this.ensureTargetPublished(current, effectiveInput, now); if (publication) return publication; - } else if (current && current.status !== 'superseded' && (current.status !== 'pending_offline' || !input.isDaemonReady())) { - if (input.source === 'auto') { + } else if ( + current + && current.status !== 'superseded' + && (current.status !== 'pending_offline' || !effectiveInput.isDaemonReady()) + ) { + if (effectiveInput.source === 'auto') { const nextAutoAt = this.nextAutoAttemptAt(now); if (nextAutoAt > now) { return { @@ -129,7 +163,7 @@ export class DaemonUpgradeCoordinator { const state = current ?? { upgradeId: randomUUID(), targetVersion, - source: input.source, + source: effectiveInput.source, status: 'pending_offline' as UpgradeLifecycleState, attempt: 0, createdAt: now, @@ -139,11 +173,11 @@ export class DaemonUpgradeCoordinator { publicationResumeInput: null, publicationCallbackRegistered: false, }; - state.source = input.source; + state.source = effectiveInput.source; state.updatedAt = now; this.current = state; - if (!input.isDaemonReady()) { + if (!effectiveInput.isDaemonReady()) { state.status = 'pending_offline'; return { ok: true, @@ -153,15 +187,15 @@ export class DaemonUpgradeCoordinator { }; } - if (input.source === 'auto') { - const publication = this.ensureTargetPublished(state, input, now); + if (effectiveInput.source === 'auto') { + const publication = this.ensureTargetPublished(state, effectiveInput, now); if (publication) return publication; - return this.scheduleAutoSend(state, input, now); + return this.scheduleAutoSend(state, effectiveInput, now); } - const publication = this.ensureTargetPublished(state, input, now); + const publication = this.ensureTargetPublished(state, effectiveInput, now); if (publication) return publication; - this.sendNow(state, input, now); + this.sendNow(state, effectiveInput, now); return { ok: true, upgradeId: state.upgradeId, @@ -221,6 +255,81 @@ export class DaemonUpgradeCoordinator { }; } + /** + * Cancel any pending send and keep the failed target terminally blocked. + * Auto/replay requests for the same target remain blocked; an explicit manual + * request or a different target version creates a fresh lifecycle. + */ + blockTargetAfterTerminalFailure(targetVersion: unknown, now = Date.now()): boolean { + let normalized: string; + try { + normalized = normalizeDaemonUpgradeTargetVersion(targetVersion); + } catch { + return false; + } + + if (this.current && this.current.targetVersion !== normalized) { + this.supersedeCurrent(now); + } + const state = this.current ?? { + upgradeId: randomUUID(), + targetVersion: normalized, + source: 'auto' as DaemonUpgradeSource, + status: 'terminal_blocked' as UpgradeLifecycleState, + attempt: 0, + createdAt: now, + updatedAt: now, + lastSentAt: null, + timer: null, + publicationResumeInput: null, + publicationCallbackRegistered: false, + }; + if (state.timer) clearTimeout(state.timer); + state.timer = null; + state.status = 'terminal_blocked'; + state.updatedAt = now; + state.publicationResumeInput = null; + state.publicationCallbackRegistered = false; + this.current = state; + return true; + } + + /** + * A terminal failure belongs to the upgrade command that produced it. + * If a newer manual lifecycle for the same target has a different id, the + * old persisted failure was explicitly superseded and must not re-block it. + */ + isTerminalFailureSupersededByManual(targetVersion: unknown, failureUpgradeId: unknown): boolean { + let normalized: string; + try { + normalized = normalizeDaemonUpgradeTargetVersion(targetVersion); + } catch { + return false; + } + return Boolean( + typeof failureUpgradeId === 'string' + && failureUpgradeId.length > 0 + && this.current?.targetVersion === normalized + && this.current.source === 'manual' + && this.current.status !== 'terminal_blocked' + && this.current.upgradeId !== failureUpgradeId, + ); + } + + hasManualLifecycleForTarget(targetVersion: unknown): boolean { + let normalized: string; + try { + normalized = normalizeDaemonUpgradeTargetVersion(targetVersion); + } catch { + return false; + } + return Boolean( + this.current?.targetVersion === normalized + && this.current.source === 'manual' + && this.current.status !== 'terminal_blocked', + ); + } + clearIfTargetVersionMatches(targetVersion: string | null | undefined): void { let normalized: string | null = null; try { diff --git a/server/test/bridge.test.ts b/server/test/bridge.test.ts index fa30bf5ba..dc84481ea 100644 --- a/server/test/bridge.test.ts +++ b/server/test/bridge.test.ts @@ -35,6 +35,12 @@ import { TIMELINE_PAYLOAD_BUDGET_BYTES } from '../../shared/timeline-payload-bud import { OPENSPEC_AUTO_DELIVER_MSG } from '../../shared/openspec-auto-deliver-constants.js'; import { EXECUTION_CLONE_KIND } from '../../shared/execution-clone.js'; import { DAEMON_MSG } from '../../shared/daemon-events.js'; +import { + DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION, + DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL, + DAEMON_UPGRADE_BLOCK_REASON, + DAEMON_UPGRADE_DELIVERY_STATUS, +} from '../../shared/daemon-upgrade.js'; import { DAEMON_COMMAND_TYPES } from '../../shared/daemon-command-types.js'; import { PEER_AUDIT_COMMAND_ERRORS, PEER_AUDIT_MESSAGES } from '../../shared/peer-audit.js'; import { @@ -297,6 +303,7 @@ function makeTimelineOwnershipDb(options: { vi.mock('../src/security/crypto.js', () => ({ sha256Hex: (_s: string) => 'valid-hash', + randomHex: (bytes: number) => 'a'.repeat(bytes * 2), })); vi.mock('../src/routes/push.js', () => ({ @@ -585,7 +592,10 @@ describe('WsBridge', () => { expect(ws.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(2); }); - it('does not retry auto daemon.upgrade after non-retryable daemon upgrade blockers', async () => { + it.each([ + 'toolchain_unavailable', + DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED, + ])('does not retry auto daemon.upgrade after non-retryable blocker %s', async (reason) => { vi.useFakeTimers(); process.env.APP_VERSION = '2026.4.905-dev.877'; @@ -600,7 +610,7 @@ describe('WsBridge', () => { expect(ws.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(1); - ws.emit('message', JSON.stringify({ type: DAEMON_MSG.UPGRADE_BLOCKED, reason: 'toolchain_unavailable' })); + ws.emit('message', JSON.stringify({ type: DAEMON_MSG.UPGRADE_BLOCKED, reason })); await flushAsync(); await vi.advanceTimersByTimeAsync(60_000); await flushAsync(); @@ -608,7 +618,334 @@ describe('WsBridge', () => { expect(ws.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(1); }); - it('does not send a stale scheduled daemon.upgrade after the daemon socket is replaced', async () => { + it('cancels the auth-scheduled auto upgrade when a persisted install failure replays', async () => { + vi.useFakeTimers(); + process.env.APP_VERSION = '2026.7.3192-dev.3593'; + markDaemonUpgradeTargetVersionPublishedForTest(process.env.APP_VERSION); + + const bridge = WsBridge.get(serverId); + const ws = new MockWs(); + bridge.handleDaemonConnection(ws as never, makeDb('valid-hash'), {} as never); + + ws.emit('message', JSON.stringify({ + type: 'auth', + serverId, + token: 'my-token', + daemonVersion: '2026.7.3157-dev.3556', + })); + ws.emit('message', JSON.stringify({ + type: DAEMON_MSG.UPGRADE_BLOCKED, + reason: DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED, + failureId: 'failure-after-link-down', + fromVersion: '2026.7.3157-dev.3556', + targetVersion: '2026.7.3192-dev.3593', + retryReason: 'stale-staging-dir', + exitCode: 217, + log: '/tmp/imcodes-upgrade-test/upgrade.log', + ts: Date.now(), + })); + await flushAsync(); + + await vi.advanceTimersByTimeAsync(5_000); + await flushAsync(); + + expect(ws.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(0); + expect(ws.sentStrings.some((raw) => { + const message = JSON.parse(raw) as Record; + return message.type === DAEMON_MSG.UPGRADE_BLOCKED_ACK + && message.failureId === 'failure-after-link-down' + && message.disposition === 'accepted'; + })).toBe(true); + + const autoRetry = bridge.requestDaemonUpgrade({ + targetVersion: process.env.APP_VERSION, + source: 'auto', + }); + expect(autoRetry).toMatchObject({ + deliveryStatus: DAEMON_UPGRADE_DELIVERY_STATUS.BACKOFF, + reason: 'terminal_install_failure', + }); + }); + + it('does not arm auto upgrade while slow auth replay keeps a persisted blocker waiting', async () => { + vi.useFakeTimers(); + process.env.APP_VERSION = '2026.7.3192-dev.3593'; + markDaemonUpgradeTargetVersionPublishedForTest(process.env.APP_VERSION); + + const bridge = WsBridge.get(serverId); + const daemonWs = new MockWs(); + bridge.handleDaemonConnection(daemonWs as never, makeDb('valid-hash'), {} as never); + + const target = { kind: 'main', serverId, sessionName: 'deck_slow_auth_brain' } as const; + const liveCoverage = { + target, + effectiveRole: 'participant', + historyCutoffAt: Date.now() - 1_000, + nextCoverageRecheckAt: null, + coveringShareIds: ['share-slow-auth'], + primaryShareId: 'share-slow-auth', + authorizedAt: Date.now(), + } as const; + bridge.setShareCoverageResolverForTests(async () => liveCoverage); + + const sharedBrowser = new MockWs(); + bridge.handleShareBrowserConnection(sharedBrowser as never, 'shared-user', makeDb('valid-hash'), { + ticketId: 'share-ticket-slow-auth', + target, + snapshot: liveCoverage, + }); + sharedBrowser.emit('message', JSON.stringify({ + type: 'session.send', + commandId: 'slow-auth-command', + sessionName: target.sessionName, + text: 'hold auth replay', + })); + await flushAsync(); + expect(bridge._getInflightCountForTest()).toBe(1); + + let resolveCoverage!: (value: typeof liveCoverage) => void; + const coveragePending = new Promise((resolve) => { + resolveCoverage = resolve; + }); + bridge.setShareCoverageResolverForTests(async () => coveragePending); + + daemonWs.emit('message', JSON.stringify({ + type: 'auth', + serverId, + token: 'my-token', + daemonVersion: '2026.7.3157-dev.3556', + })); + daemonWs.emit('message', JSON.stringify({ + type: DAEMON_MSG.UPGRADE_BLOCKED, + reason: DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED, + failureId: 'failure-during-slow-auth', + fromVersion: '2026.7.3157-dev.3556', + targetVersion: process.env.APP_VERSION, + })); + await flushAsync(); + + await vi.advanceTimersByTimeAsync(5_000); + await flushAsync(); + expect(daemonWs.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(0); + + resolveCoverage(liveCoverage); + await flushAsync(); + await vi.advanceTimersByTimeAsync(5_000); + await flushAsync(); + + expect(daemonWs.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(0); + expect(daemonWs.sentStrings.some((raw) => { + const message = JSON.parse(raw) as Record; + return message.type === DAEMON_MSG.UPGRADE_BLOCKED_ACK + && message.failureId === 'failure-during-slow-auth' + && message.disposition === 'accepted'; + })).toBe(true); + }); + + it('waits for daemon outbox sync even when auth finishes more than five seconds earlier', async () => { + vi.useFakeTimers(); + process.env.APP_VERSION = '2026.7.3192-dev.3593'; + markDaemonUpgradeTargetVersionPublishedForTest(process.env.APP_VERSION); + + const bridge = WsBridge.get(serverId); + const daemonWs = new MockWs(); + bridge.handleDaemonConnection(daemonWs as never, makeDb('valid-hash'), {} as never); + daemonWs.emit('message', JSON.stringify({ + type: 'auth', + serverId, + token: 'my-token', + daemonVersion: '2026.7.3157-dev.3556', + [DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.AUTH_REVISION_FIELD]: + DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION, + })); + await flushAsync(); + + await vi.advanceTimersByTimeAsync(10_000); + await flushAsync(); + expect(daemonWs.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(0); + + daemonWs.emit('message', JSON.stringify({ + type: DAEMON_MSG.UPGRADE_BLOCKED, + reason: DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED, + failureId: 'failure-after-delayed-outbox-read', + upgradeId: 'old-auto-upgrade', + fromVersion: '2026.7.3157-dev.3556', + targetVersion: process.env.APP_VERSION, + })); + daemonWs.emit('message', JSON.stringify({ + type: DAEMON_MSG.UPGRADE_BLOCKED_SYNC, + revision: DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION, + })); + await vi.advanceTimersByTimeAsync(0); + await flushAsync(); + await vi.advanceTimersByTimeAsync(5_000); + await flushAsync(); + + expect(daemonWs.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(0); + expect(daemonWs.sentStrings.map((raw) => JSON.parse(raw))).toContainEqual({ + type: DAEMON_MSG.UPGRADE_BLOCKED_ACK, + failureId: 'failure-after-delayed-outbox-read', + disposition: DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.ACCEPTED, + }); + }); + + it('starts a normal auto upgrade only after an empty outbox sync completes', async () => { + vi.useFakeTimers(); + process.env.APP_VERSION = '2026.7.3192-dev.3593'; + markDaemonUpgradeTargetVersionPublishedForTest(process.env.APP_VERSION); + + const bridge = WsBridge.get(serverId); + const daemonWs = new MockWs(); + bridge.handleDaemonConnection(daemonWs as never, makeDb('valid-hash'), {} as never); + daemonWs.emit('message', JSON.stringify({ + type: 'auth', + serverId, + token: 'my-token', + daemonVersion: '2026.7.3157-dev.3556', + [DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.AUTH_REVISION_FIELD]: + DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION, + })); + await flushAsync(); + await vi.advanceTimersByTimeAsync(5_000); + await flushAsync(); + expect(daemonWs.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(0); + + daemonWs.emit('message', JSON.stringify({ + type: DAEMON_MSG.UPGRADE_BLOCKED_SYNC, + revision: DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION, + })); + await vi.advanceTimersByTimeAsync(0); + await flushAsync(); + await vi.advanceTimersByTimeAsync(5_000); + await flushAsync(); + + expect(daemonWs.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(1); + }); + + it('keeps an offline manual override ahead of reconnect auto and supersedes the retained old blocker', async () => { + vi.useFakeTimers(); + process.env.APP_VERSION = '2026.7.3192-dev.3593'; + markDaemonUpgradeTargetVersionPublishedForTest(process.env.APP_VERSION); + + const bridge = WsBridge.get(serverId); + const firstWs = new MockWs(); + bridge.handleDaemonConnection(firstWs as never, makeDb('valid-hash'), {} as never); + firstWs.emit('message', JSON.stringify({ + type: 'auth', + serverId, + token: 'my-token', + daemonVersion: '2026.7.3157-dev.3556', + })); + await flushAsync(); + await vi.advanceTimersByTimeAsync(5_000); + await flushAsync(); + + const firstUpgrade = firstWs.sentStrings + .map((raw) => JSON.parse(raw) as Record) + .find((message) => message.type === DAEMON_COMMAND_TYPES.DAEMON_UPGRADE); + expect(firstUpgrade?.upgradeId).toEqual(expect.any(String)); + + firstWs.emit('message', JSON.stringify({ + type: DAEMON_MSG.UPGRADE_BLOCKED, + reason: DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED, + failureId: 'failure-before-manual-override', + upgradeId: firstUpgrade?.upgradeId, + fromVersion: '2026.7.3157-dev.3556', + targetVersion: process.env.APP_VERSION, + })); + await flushAsync(); + firstWs.emit('close'); + + const manual = bridge.requestDaemonUpgrade({ + targetVersion: process.env.APP_VERSION, + source: 'manual', + }); + expect(manual.deliveryStatus).toBe(DAEMON_UPGRADE_DELIVERY_STATUS.PENDING_OFFLINE); + expect(manual.upgradeId).not.toBe(firstUpgrade?.upgradeId); + + // Move beyond the auto retry interval so this regression cannot pass + // merely because the previous auto attempt is still rate-limited. The + // reconnect auto request must preserve manual authority on its own. + await vi.advanceTimersByTimeAsync(15 * 60 * 1000 + 1); + + const reconnectWs = new MockWs(); + bridge.handleDaemonConnection(reconnectWs as never, makeDb('valid-hash'), {} as never); + reconnectWs.emit('message', JSON.stringify({ + type: 'auth', + serverId, + token: 'my-token', + daemonVersion: '2026.7.3157-dev.3556', + [DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.AUTH_REVISION_FIELD]: + DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION, + })); + reconnectWs.emit('message', JSON.stringify({ + type: DAEMON_MSG.UPGRADE_BLOCKED, + reason: DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED, + failureId: 'failure-before-manual-override', + upgradeId: firstUpgrade?.upgradeId, + fromVersion: '2026.7.3157-dev.3556', + targetVersion: process.env.APP_VERSION, + })); + reconnectWs.emit('message', JSON.stringify({ + type: DAEMON_MSG.UPGRADE_BLOCKED_SYNC, + revision: DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION, + })); + await vi.advanceTimersByTimeAsync(0); + await flushAsync(); + + const reconnectMessages = reconnectWs.sentStrings.map((raw) => JSON.parse(raw) as Record); + expect(reconnectMessages.filter((message) => message.type === DAEMON_COMMAND_TYPES.DAEMON_UPGRADE)).toEqual([{ + type: DAEMON_COMMAND_TYPES.DAEMON_UPGRADE, + upgradeId: manual.upgradeId, + targetVersion: process.env.APP_VERSION, + }]); + expect(reconnectMessages).toContainEqual({ + type: DAEMON_MSG.UPGRADE_BLOCKED_ACK, + failureId: 'failure-before-manual-override', + disposition: DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.SUPERSEDED, + }); + + await vi.advanceTimersByTimeAsync(5_000); + await flushAsync(); + expect(reconnectWs.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(1); + }); + + it('acks an old-target install failure as obsolete without blocking the new target', async () => { + vi.useFakeTimers(); + process.env.APP_VERSION = '2026.7.3193-dev.3594'; + markDaemonUpgradeTargetVersionPublishedForTest(process.env.APP_VERSION); + + const bridge = WsBridge.get(serverId); + const ws = new MockWs(); + bridge.handleDaemonConnection(ws as never, makeDb('valid-hash'), {} as never); + + ws.emit('message', JSON.stringify({ + type: 'auth', + serverId, + token: 'my-token', + daemonVersion: '2026.7.3157-dev.3556', + })); + ws.emit('message', JSON.stringify({ + type: DAEMON_MSG.UPGRADE_BLOCKED, + reason: DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED, + failureId: 'failure-for-old-target', + fromVersion: '2026.7.3157-dev.3556', + targetVersion: '2026.7.3192-dev.3593', + })); + await flushAsync(); + await vi.advanceTimersByTimeAsync(5_000); + await flushAsync(); + + expect(ws.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(1); + expect(ws.sentStrings.some((raw) => { + const message = JSON.parse(raw) as Record; + return message.type === DAEMON_MSG.UPGRADE_BLOCKED_ACK + && message.failureId === 'failure-for-old-target' + && message.disposition === 'obsolete'; + })).toBe(true); + }); + + it('sends a deferred daemon.upgrade only to the replacement socket and ignores stale frames', async () => { vi.useFakeTimers(); process.env.APP_VERSION = '2026.4.905-dev.877'; @@ -626,7 +963,287 @@ describe('WsBridge', () => { await flushAsync(); expect(staleWs.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(0); + expect(replacementWs.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(1); + + const rawBrowser = new MockWs(); + bridge.handleBrowserConnection(rawBrowser as never, 'raw-user', makeDb('valid-hash')); + rawBrowser.emit('message', JSON.stringify({ + type: 'terminal.subscribe', + session: 'deck_stale_socket_binary', + raw: true, + })); + await flushAsync(); + rawBrowser.sent.length = 0; + + // ws can emit already-buffered frames while the replaced socket is still + // CLOSING. Neither binary data nor a terminal blocker from generation 1 + // may borrow generation 2's authenticated bridge state. + staleWs.emit('message', packFrame('deck_stale_socket_binary', Buffer.from('stale')), true); + staleWs.emit('message', JSON.stringify({ + type: DAEMON_MSG.UPGRADE_BLOCKED, + reason: DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED, + failureId: 'failure-from-replaced-socket', + fromVersion: '2026.4.904-dev.100', + targetVersion: process.env.APP_VERSION, + })); + await flushAsync(); + expect.soft(rawBrowser.sent.filter((item) => Buffer.isBuffer(item))).toHaveLength(0); + + await vi.advanceTimersByTimeAsync(15 * 60 * 1000 + 1); + const nextAuto = bridge.requestDaemonUpgrade({ + targetVersion: process.env.APP_VERSION, + source: 'auto', + }); + expect.soft(nextAuto.deliveryStatus).toBe(DAEMON_UPGRADE_DELIVERY_STATUS.SENT); + }); + + it('does not let an error from a replaced socket reject current-generation requests', async () => { + const bridge = WsBridge.get(serverId); + const staleWs = new MockWs(); + bridge.handleDaemonConnection(staleWs as never, makeDb('valid-hash'), {} as never); + staleWs.emit('message', JSON.stringify({ type: 'auth', serverId, token: 'my-token' })); + await flushAsync(); + + const replacementWs = new MockWs(); + bridge.handleDaemonConnection(replacementWs as never, makeDb('valid-hash'), {} as never); + replacementWs.emit('message', JSON.stringify({ type: 'auth', serverId, token: 'my-token' })); + await flushAsync(); + + const pending = bridge.requestTimelineHistory({ + sessionName: 'deck_replacement_request', + timeoutMs: 10_000, + }); + const outbound = replacementWs.sentStrings.find((raw) => raw.includes('"type":"timeline.history_request"')); + expect(outbound).toBeTruthy(); + const requestId = JSON.parse(outbound!).requestId as string; + let settled = false; + const observed = pending.then( + (value) => { + settled = true; + return { status: 'resolved' as const, value }; + }, + (error: unknown) => { + settled = true; + return { status: 'rejected' as const, error }; + }, + ); + + staleWs.emit('error', new Error('stale socket error')); + await flushAsync(); + expect(settled).toBe(false); + + replacementWs.emit('message', JSON.stringify({ + type: 'timeline.history', + sessionName: 'deck_replacement_request', + requestId, + events: [], + epoch: 1, + })); + await expect(observed).resolves.toMatchObject({ + status: 'resolved', + value: { + type: 'timeline.history', + requestId, + epoch: 1, + }, + }); + }); + + it('does not let a stale in-flight auth certify or dispatch through its replacement connection', async () => { + vi.useFakeTimers(); + process.env.APP_VERSION = '2026.7.3192-dev.3593'; + markDaemonUpgradeTargetVersionPublishedForTest(process.env.APP_VERSION); + + type AuthRow = { + token_hash: string; + node_role: 'full'; + revoked_at: null; + }; + const makeDeferredAuthDb = () => { + let resolveQuery!: (value: AuthRow) => void; + const queryPromise = new Promise((resolve) => { + resolveQuery = resolve; + }); + const db = { + queryOne: () => queryPromise, + query: async () => [], + execute: async () => ({ changes: 1 }), + exec: async () => {}, + transaction: async (fn: (tx: import('../src/db/client.js').Database) => Promise) => + fn(db as unknown as import('../src/db/client.js').Database), + close: () => {}, + } as unknown as import('../src/db/client.js').Database; + return { db, resolveQuery }; + }; + + const bridge = WsBridge.get(serverId); + const manual = bridge.requestDaemonUpgrade({ + targetVersion: process.env.APP_VERSION, + source: 'manual', + }); + expect(manual.deliveryStatus).toBe(DAEMON_UPGRADE_DELIVERY_STATUS.PENDING_OFFLINE); + + const firstAuth = makeDeferredAuthDb(); + const firstWs = new MockWs(); + bridge.handleDaemonConnection(firstWs as never, firstAuth.db, {} as never); + firstWs.emit('message', JSON.stringify({ + type: 'auth', + serverId, + token: 'my-token', + daemonVersion: '2026.7.3157-dev.3556', + [DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.AUTH_REVISION_FIELD]: + DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION, + })); + await flushAsync(); + + const replacementAuth = makeDeferredAuthDb(); + const replacementWs = new MockWs(); + bridge.handleDaemonConnection(replacementWs as never, replacementAuth.db, {} as never); + replacementWs.emit('message', JSON.stringify({ + type: 'auth', + serverId, + token: 'my-token', + daemonVersion: '2026.7.3157-dev.3556', + [DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.AUTH_REVISION_FIELD]: + DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION, + })); + await flushAsync(); + + // Finish generation 1 after generation 2 has installed its own socket + // and auth promise. The stale continuation must not authenticate gen 2, + // clear its promise, or rewrite its blocker-sync generation. + firstAuth.resolveQuery({ + token_hash: 'valid-hash', + node_role: 'full', + revoked_at: null, + }); + await flushAsync(); + expect(bridge.isAuthenticated).toBe(false); expect(replacementWs.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(0); + + // Generation 2 may authenticate, but remains unable to dispatch the + // pending manual upgrade until its own persisted-blocker sync arrives. + replacementAuth.resolveQuery({ + token_hash: 'valid-hash', + node_role: 'full', + revoked_at: null, + }); + await flushAsync(); + expect(bridge.isAuthenticated).toBe(true); + await vi.advanceTimersByTimeAsync(10_000); + await flushAsync(); + expect(replacementWs.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(0); + + replacementWs.emit('message', JSON.stringify({ + type: DAEMON_MSG.UPGRADE_BLOCKED_SYNC, + revision: DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION, + })); + await vi.advanceTimersByTimeAsync(0); + await flushAsync(); + + const upgrades = replacementWs.sentStrings + .map((raw) => JSON.parse(raw) as Record) + .filter((message) => message.type === DAEMON_COMMAND_TYPES.DAEMON_UPGRADE); + expect(upgrades).toEqual([{ + type: DAEMON_COMMAND_TYPES.DAEMON_UPGRADE, + upgradeId: manual.upgradeId, + targetVersion: process.env.APP_VERSION, + }]); + expect(firstWs.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(0); + }); + + it('does not replay an inflight command through a replacement while stale auth revalidation settles', async () => { + const bridge = WsBridge.get(serverId); + const firstWs = new MockWs(); + bridge.handleDaemonConnection(firstWs as never, makeDb('valid-hash'), {} as never); + + const target = { kind: 'main', serverId, sessionName: 'deck_stale_auth_replay_brain' } as const; + const liveCoverage = { + target, + effectiveRole: 'participant', + historyCutoffAt: Date.now() - 1_000, + nextCoverageRecheckAt: null, + coveringShareIds: ['share-stale-auth-replay'], + primaryShareId: 'share-stale-auth-replay', + authorizedAt: Date.now(), + } as const; + bridge.setShareCoverageResolverForTests(async () => liveCoverage); + const sharedBrowser = new MockWs(); + bridge.handleShareBrowserConnection(sharedBrowser as never, 'shared-user', makeDb('valid-hash'), { + ticketId: 'share-ticket-stale-auth-replay', + target, + snapshot: liveCoverage, + }); + sharedBrowser.emit('message', JSON.stringify({ + type: 'session.send', + commandId: 'stale-auth-replay-command', + sessionName: target.sessionName, + text: 'must reach only the authenticated replacement', + })); + await flushAsync(); + expect(bridge._getInflightCountForTest()).toBe(1); + + let resolveCoverage!: (value: typeof liveCoverage) => void; + const coveragePending = new Promise((resolve) => { + resolveCoverage = resolve; + }); + bridge.setShareCoverageResolverForTests(async () => coveragePending); + + firstWs.emit('message', JSON.stringify({ + type: 'auth', + serverId, + token: 'my-token', + })); + await flushAsync(); + + let resolveReplacementAuth!: (value: { + token_hash: string; + node_role: 'full'; + revoked_at: null; + }) => void; + const replacementQuery = new Promise<{ + token_hash: string; + node_role: 'full'; + revoked_at: null; + }>((resolve) => { + resolveReplacementAuth = resolve; + }); + const replacementDb = { + queryOne: () => replacementQuery, + query: async () => [], + execute: async () => ({ changes: 1 }), + exec: async () => {}, + transaction: async (fn: (tx: import('../src/db/client.js').Database) => Promise) => + fn(replacementDb as unknown as import('../src/db/client.js').Database), + close: () => {}, + } as unknown as import('../src/db/client.js').Database; + const replacementWs = new MockWs(); + bridge.handleDaemonConnection(replacementWs as never, replacementDb, {} as never); + replacementWs.emit('message', JSON.stringify({ + type: 'auth', + serverId, + token: 'my-token', + })); + await flushAsync(); + + // Let generation 1's share revalidation finish after generation 2 owns + // the bridge. Its auth flow must leave the entry buffered, not send it + // through generation 2 before that socket authenticates. + resolveCoverage(liveCoverage); + await flushAsync(); + expect(bridge.isAuthenticated).toBe(false); + expect(replacementWs.sentStrings.filter((raw) => raw.includes('"type":"session.send"'))).toHaveLength(0); + + resolveReplacementAuth({ + token_hash: 'valid-hash', + node_role: 'full', + revoked_at: null, + }); + await flushAsync(); + + expect(bridge.isAuthenticated).toBe(true); + expect(replacementWs.sentStrings.filter((raw) => raw.includes('"type":"session.send"'))).toHaveLength(1); + expect(firstWs.sentStrings.filter((raw) => raw.includes('"type":"session.send"'))).toHaveLength(0); }); }); @@ -1009,7 +1626,7 @@ describe('WsBridge', () => { const daemonWs = new MockWs(); bridge.handleDaemonConnection(daemonWs as never, makeDb('valid-hash'), {} as never); daemonWs.emit('message', JSON.stringify({ type: 'auth', serverId, token: 't' })); - await flushAsync(); + await flushOneBridgeDataPlaneTurn(); const upgradeMessages = daemonWs.sentStrings.filter((s) => s.includes('"type":"daemon.upgrade"')); expect(upgradeMessages).toHaveLength(1); diff --git a/server/test/daemon-upgrade-coordinator.test.ts b/server/test/daemon-upgrade-coordinator.test.ts index 075e3c69a..5b6023547 100644 --- a/server/test/daemon-upgrade-coordinator.test.ts +++ b/server/test/daemon-upgrade-coordinator.test.ts @@ -180,4 +180,117 @@ describe('DaemonUpgradeCoordinator npm publication gate', () => { await flushPromises(); expect(sent).toHaveLength(2); }); + + it('cancels a scheduled auto send and terminally blocks only that target until manual override', async () => { + vi.useFakeTimers(); + const coordinator = new DaemonUpgradeCoordinator(); + const sent: Record[] = []; + const targetVersion = '2026.7.3192-dev.3593'; + const request = { + targetVersion, + source: 'auto' as const, + skipPublicationGate: true, + isDaemonReady: () => true, + isStillCurrent: () => true, + send: (message: Record) => sent.push(message), + now: 0, + }; + + expect(coordinator.request(request).deliveryStatus).toBe(DAEMON_UPGRADE_DELIVERY_STATUS.SENT); + expect(coordinator.blockTargetAfterTerminalFailure(targetVersion, 1)).toBe(true); + + await vi.advanceTimersByTimeAsync(5_000); + await flushPromises(); + expect(sent).toEqual([]); + expect(coordinator.request({ ...request, now: 5_001 })).toMatchObject({ + deliveryStatus: DAEMON_UPGRADE_DELIVERY_STATUS.BACKOFF, + reason: 'terminal_install_failure', + }); + + expect(coordinator.request({ + ...request, + source: 'manual', + now: 5_002, + }).deliveryStatus).toBe(DAEMON_UPGRADE_DELIVERY_STATUS.SENT); + expect(sent).toHaveLength(1); + }); + + it('keeps an offline manual lifecycle authoritative over reconnect auto traffic', () => { + const coordinator = new DaemonUpgradeCoordinator(); + const sent: Record[] = []; + const targetVersion = '2026.7.3192-dev.3593'; + let ready = false; + const base = { + targetVersion, + skipPublicationGate: true, + isDaemonReady: () => ready, + isStillCurrent: () => true, + send: (message: Record) => sent.push(message), + }; + + const manual = coordinator.request({ ...base, source: 'manual' }); + expect(manual.deliveryStatus).toBe(DAEMON_UPGRADE_DELIVERY_STATUS.PENDING_OFFLINE); + + const authAuto = coordinator.request({ ...base, source: 'auto' }); + expect(authAuto).toMatchObject({ + upgradeId: manual.upgradeId, + deliveryStatus: DAEMON_UPGRADE_DELIVERY_STATUS.ALREADY_IN_PROGRESS, + }); + + ready = true; + const flushed = coordinator.flushPending({ + skipPublicationGate: true, + isDaemonReady: () => ready, + isStillCurrent: () => true, + send: (message) => sent.push(message), + }); + expect(flushed).toMatchObject({ + upgradeId: manual.upgradeId, + deliveryStatus: DAEMON_UPGRADE_DELIVERY_STATUS.SENT, + }); + expect(sent).toEqual([{ + type: DAEMON_COMMAND_TYPES.DAEMON_UPGRADE, + upgradeId: manual.upgradeId, + targetVersion, + }]); + }); + + it('identifies an older failure superseded by a newer manual lifecycle', () => { + const coordinator = new DaemonUpgradeCoordinator(); + const targetVersion = '2026.7.3192-dev.3593'; + coordinator.blockTargetAfterTerminalFailure(targetVersion, 1); + const manual = coordinator.request({ + targetVersion, + source: 'manual', + skipPublicationGate: true, + isDaemonReady: () => false, + send: () => {}, + now: 2, + }); + + expect(coordinator.isTerminalFailureSupersededByManual(targetVersion, 'older-upgrade')).toBe(true); + expect(coordinator.isTerminalFailureSupersededByManual(targetVersion, manual.upgradeId)).toBe(false); + expect(coordinator.hasManualLifecycleForTarget(targetVersion)).toBe(true); + }); + + it('allows a fresh auto lifecycle when the requested target version changes', async () => { + vi.useFakeTimers(); + const coordinator = new DaemonUpgradeCoordinator(); + const sent: Record[] = []; + coordinator.blockTargetAfterTerminalFailure('2026.7.3192-dev.3593', 0); + + expect(coordinator.request({ + targetVersion: '2026.7.3193-dev.3594', + source: 'auto', + skipPublicationGate: true, + isDaemonReady: () => true, + isStillCurrent: () => true, + send: (message) => sent.push(message), + now: 1, + }).deliveryStatus).toBe(DAEMON_UPGRADE_DELIVERY_STATUS.SENT); + + await vi.advanceTimersByTimeAsync(5_000); + await flushPromises(); + expect(sent).toHaveLength(1); + }); }); diff --git a/shared/daemon-events.ts b/shared/daemon-events.ts index 5a0dc1bce..a48d209b4 100644 --- a/shared/daemon-events.ts +++ b/shared/daemon-events.ts @@ -2,6 +2,8 @@ export const DAEMON_MSG = { RECONNECTED: 'daemon.reconnected', DISCONNECTED: 'daemon.disconnected', UPGRADE_BLOCKED: 'daemon.upgrade_blocked', + UPGRADE_BLOCKED_ACK: 'daemon.upgrade_blocked_ack', + UPGRADE_BLOCKED_SYNC: 'daemon.upgrade_blocked_sync', // Emitted by the daemon right after it spawns the (detached) upgrade script, // just before the running process is killed & restarted. The server relays it // to browsers so the UI can show an "upgrading…" state next to the daemon diff --git a/shared/daemon-upgrade.ts b/shared/daemon-upgrade.ts index a56ab8214..cc5ccdadc 100644 --- a/shared/daemon-upgrade.ts +++ b/shared/daemon-upgrade.ts @@ -1,5 +1,23 @@ export const DAEMON_UPGRADE_TARGET_LATEST = 'latest'; +export const DAEMON_UPGRADE_BLOCK_REASON = { + INSTALL_FAILED: 'install_failed', +} as const; + +export const DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL = { + AUTH_REVISION_FIELD: 'upgradeBlockedSyncRevision', + REVISION: 1, +} as const; + +export const DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION = { + ACCEPTED: 'accepted', + OBSOLETE: 'obsolete', + SUPERSEDED: 'superseded', +} as const; + +export type DaemonUpgradeBlockedAckDisposition = + (typeof DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION)[keyof typeof DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION]; + export const DAEMON_UPGRADE_DELIVERY_STATUS = { SENT: 'sent', PENDING_OFFLINE: 'pending_offline', diff --git a/src/daemon/command-handler.ts b/src/daemon/command-handler.ts index 2987bfdfc..19d87f723 100644 --- a/src/daemon/command-handler.ts +++ b/src/daemon/command-handler.ts @@ -129,6 +129,13 @@ import { resolveWindowsUpgradeRunnerPath, } from '../util/windows-upgrade-script.js'; import { buildBashSharpRepair } from '../util/sharp-repair-script.js'; +import { + buildPosixUpgradeLayoutRecoveryScript, + parsePosixUpgradeFailureStatus, + POSIX_UPGRADE_INSTALL_FAILURE_EXIT_CODE, +} from '../util/posix-upgrade-layout-recovery.js'; +import { warnOncePerHour } from '../util/rate-limited-warn.js'; +import { getDefaultUpgradeBlockedOutbox } from './upgrade-blocked-outbox.js'; import { encodeVbsAsUtf16, encodeCmdAsUtf8Bom } from '../util/windows-launch-artifacts.js'; import { registerTempFile, removeTrackedTempFile } from '../store/temp-file-store.js'; import { sanitizeProjectName } from '../../shared/sanitize-project-name.js'; @@ -156,7 +163,11 @@ import { getCodexRuntimeConfig } from '../agent/codex-runtime-config.js'; import { mergeCodexDisplayMetadata } from '../agent/codex-display.js'; import { P2P_TERMINAL_RUN_STATUSES } from '../../shared/p2p-status.js'; import { DAEMON_MSG } from '../../shared/daemon-events.js'; -import { DAEMON_UPGRADE_TARGET_LATEST, normalizeDaemonUpgradeTargetVersion } from '../../shared/daemon-upgrade.js'; +import { + DAEMON_UPGRADE_BLOCK_REASON, + DAEMON_UPGRADE_TARGET_LATEST, + normalizeDaemonUpgradeTargetVersion, +} from '../../shared/daemon-upgrade.js'; import { CC_PRESET_MSG, normalizeCcPresetName, type CcPreset } from '../../shared/cc-presets.js'; import { MEMORY_MCP_PROVIDER_IDS, @@ -1628,9 +1639,13 @@ function dispatchWebCommand(cmd: Record, serverLink: ServerLink case DAEMON_COMMAND_TYPES.DAEMON_UPGRADE: try { const normalizedTarget = normalizeDaemonUpgradeTargetVersion(cmd.targetVersion); + const upgradeId = typeof cmd.upgradeId === 'string' && cmd.upgradeId.length > 0 && cmd.upgradeId.length <= 128 + ? cmd.upgradeId + : undefined; void handleDaemonUpgrade( normalizedTarget === DAEMON_UPGRADE_TARGET_LATEST ? undefined : normalizedTarget, serverLink, + upgradeId, ); } catch { logger.warn({ targetVersion: cmd.targetVersion }, 'daemon.upgrade rejected invalid targetVersion'); @@ -6895,7 +6910,11 @@ async function resolveUpgradeRegistry(): Promise<{ base: string; explicit: boole return { base, explicit: base !== INSTALLER_OFFICIAL_NPM_REGISTRY }; } -async function handleDaemonUpgrade(targetVersion?: string, serverLink?: ServerLink): Promise { +async function handleDaemonUpgrade( + targetVersion?: string, + serverLink?: ServerLink, + upgradeId?: string, +): Promise { const UPGRADE_MEMORY_FREEZE_TTL_MS = 15 * 60 * 1000; // ── Opt-out: forcibly disable the daemon's self-upgrade ─────────────────── @@ -7198,6 +7217,7 @@ async function handleDaemonUpgrade(targetVersion?: string, serverLink?: ServerLi const scriptDir = mkdtempSync(join(tmpdir(), 'imcodes-upgrade-')); const logFile = join(scriptDir, 'upgrade.log'); + const statusFile = join(scriptDir, 'upgrade-status.json'); const scriptPath = join(scriptDir, 'upgrade.sh'); // Build the platform-specific restart command. // We always restart regardless of whether npm install succeeded, so the daemon @@ -7367,10 +7387,23 @@ launchctl load -w "${plist}"`; LOG="${logFile}" SCRIPT_DIR="${scriptDir}" +UPGRADE_STATUS_FILE="${statusFile}" CLEANUP_AFTER_SEC=${CLEANUP_AFTER_SEC} REGISTRY_ARG="${registryArg}" log() { echo "[$(date '+%Y-%m-%dT%H:%M:%S%z')] $*" >> "$LOG"; } +${buildPosixUpgradeLayoutRecoveryScript()} + +write_install_failure_status() { + local retry_reason="$1" + local attempts="$2" + local exit_code="$3" + local status_tmp="$UPGRADE_STATUS_FILE.tmp.$$" + printf '{"state":"blocked","reason":"${DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED}","retryReason":"%s","attempts":%s,"exitCode":%s}\\n' \ + "$retry_reason" "$attempts" "$exit_code" > "$status_tmp" 2>>"$LOG" || return 1 + mv "$status_tmp" "$UPGRADE_STATUS_FILE" 2>>"$LOG" +} + schedule_self_cleanup() { if [ -z "$SCRIPT_DIR" ] || [ ! -d "$SCRIPT_DIR" ]; then return 0 @@ -7558,6 +7591,12 @@ GLOBAL_ROOT=$(eval "$NPM_RUN root -g" 2>>"$LOG") log "[step 1] global root: $GLOBAL_ROOT" GLOBAL_PKG="$GLOBAL_ROOT/imcodes" +# npm's rename destination is created before the incoming package's +# preinstall hook can run, so the package-level cleanup cannot heal this +# failure. Remove interrupted reify leftovers here, before npm starts. +cleanup_stale_imcodes_staging_dirs "$GLOBAL_ROOT" "$UPGRADE_LOCK_STALE_AFTER_SEC" \ + || log "[step 1.5] stale npm staging cleanup was incomplete; install may retry targeted recovery" + # Remove existing npm link if any — it shadows install and prevents real upgrade. if [ -L "$GLOBAL_PKG" ]; then log "[step 1] removing pre-existing npm link: $GLOBAL_PKG -> $(readlink "$GLOBAL_PKG")" @@ -7609,6 +7648,7 @@ INSTALL_OUT="${scriptDir}/install-attempt.log" INSTALL_RC=1 ATTEMPT=0 MAX_ATTEMPTS=5 +RETRY_REASON="not-classified" # Indexed sequentially with $ATTEMPT (1-based), so element 0 is unused. # 15s / 30s / 60s / 120s keeps the common npm publish-CDN window quick # without stretching a bad target into a 10-minute local stall. @@ -7633,6 +7673,7 @@ while [ "$ATTEMPT" -lt "$MAX_ATTEMPTS" ]; do VIEW_RC=$? cat "$INSTALL_OUT" >> "$LOG" if [ "$VIEW_RC" -ne 0 ] && is_etarget_output "$INSTALL_OUT"; then + RETRY_REASON="target-not-visible" log "[step 2] ${pkgSpec} not visible in registry yet" if [ "$ATTEMPT" -ge "$MAX_ATTEMPTS" ]; then log "[step 2] target never became visible across $MAX_ATTEMPTS attempts — giving up before heavyweight install" @@ -7671,6 +7712,12 @@ while [ "$ATTEMPT" -lt "$MAX_ATTEMPTS" ]; do elif is_transient_npm_output "$INSTALL_OUT"; then IS_RETRYABLE=1 RETRY_REASON="transient-network" + elif is_recoverable_layout_output "$INSTALL_OUT" "$GLOBAL_ROOT"; then + IS_RETRYABLE=1 + RETRY_REASON="stale-staging-dir" + if ! recover_stale_layout_from_output "$INSTALL_OUT" "$GLOBAL_ROOT"; then + log "[step 2] targeted stale staging cleanup failed; retrying remains bounded" + fi fi if [ "$IS_RETRYABLE" -ne 1 ]; then log "[step 2] non-retryable npm failure — not retrying. Tail of npm output:" @@ -7687,9 +7734,11 @@ while [ "$ATTEMPT" -lt "$MAX_ATTEMPTS" ]; do done if [ "$INSTALL_RC" -ne 0 ]; then log "[step 2] install FAILED after $ATTEMPT attempts (final exit $INSTALL_RC) — keeping current daemon running" + write_install_failure_status "$RETRY_REASON" "$ATTEMPT" "$INSTALL_RC" \ + || log "[step 2] failed to write upgrade status marker: $UPGRADE_STATUS_FILE" log "=== upgrade aborted ===" schedule_self_cleanup - exit 0 + exit ${POSIX_UPGRADE_INSTALL_FAILURE_EXIT_CODE} fi log "[step 2] install succeeded after $ATTEMPT attempt(s)" @@ -7986,6 +8035,40 @@ schedule_self_cleanup detached: true, stdio: 'ignore', }); + child.on('exit', (code, signal) => { + if (code !== POSIX_UPGRADE_INSTALL_FAILURE_EXIT_CODE) return; + let status = null; + try { + status = parsePosixUpgradeFailureStatus(readFileSync(statusFile, 'utf8')); + } catch { + // The exit code is still authoritative even if the diagnostic marker + // could not be read (for example, the tmp directory was removed). + } + const diagnostic = { + targetVersion: targetVersion ?? DAEMON_UPGRADE_TARGET_LATEST, + log: logFile, + retryReason: status?.retryReason ?? 'unknown', + exitCode: status?.exitCode ?? code, + ...(signal ? { signal } : {}), + ...(status?.attempts !== undefined ? { attempts: status.attempts } : {}), + }; + warnOncePerHour('daemon.upgrade.install_failed', diagnostic); + const blockedMessage = { + type: DAEMON_MSG.UPGRADE_BLOCKED, + reason: DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED, + failureId: randomUUID(), + ...(upgradeId ? { upgradeId } : {}), + fromVersion: DAEMON_VERSION, + ...diagnostic, + ts: Date.now(), + } as const; + void getDefaultUpgradeBlockedOutbox() + .report(blockedMessage, (message) => serverLink?.trySend?.(message) ?? false) + .catch((err) => { + logger.warn({ err, failureId: blockedMessage.failureId }, 'daemon.upgrade: failed to persist install failure'); + }); + releaseUpgradeMemoryFreeze(); + }); child.unref(); logger.info({ log: logFile }, 'daemon.upgrade: upgrade script spawned, will restart in ~3 s'); diff --git a/src/daemon/latency-tracer.ts b/src/daemon/latency-tracer.ts index 6800e8c84..2fe83137a 100644 --- a/src/daemon/latency-tracer.ts +++ b/src/daemon/latency-tracer.ts @@ -4,6 +4,7 @@ import { homedir, loadavg } from 'node:os'; import { PerformanceObserver, monitorEventLoopDelay, performance } from 'node:perf_hooks'; import logger from '../util/logger.js'; import { MSG_COMMAND_ACK } from '../../shared/ack-protocol.js'; +import { DAEMON_MSG } from '../../shared/daemon-events.js'; import { TIMELINE_MESSAGES } from '../../shared/timeline-protocol.js'; import { TRANSPORT_EVENT, TRANSPORT_MSG } from '../../shared/transport-events.js'; @@ -643,6 +644,8 @@ export function classifyServerSendPlane(msgType: string | undefined): ServerSend || msgType === 'session.idle' || msgType === 'daemon.hello' || msgType === 'daemon.stats' + || msgType === DAEMON_MSG.UPGRADE_BLOCKED + || msgType === DAEMON_MSG.UPGRADE_BLOCKED_SYNC || msgType === 'heartbeat' // Live timeline events carry the chat stream/typewriter updates and // session.state transitions. They must bypass bulk history/data replay. diff --git a/src/daemon/server-link.ts b/src/daemon/server-link.ts index b51935236..f7174cd35 100644 --- a/src/daemon/server-link.ts +++ b/src/daemon/server-link.ts @@ -3,9 +3,15 @@ import { performance } from 'node:perf_hooks'; import type { TimelineEvent } from './timeline-event.js'; import logger from '../util/logger.js'; import { DAEMON_VERSION } from '../util/version.js'; +import { DAEMON_MSG } from '../../shared/daemon-events.js'; +import { + DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION, + DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL, +} from '../../shared/daemon-upgrade.js'; import { setTransportRelaySend } from './transport-relay.js'; import { setProviderRegistryServerLink } from '../agent/provider-registry.js'; import { getDefaultAckOutbox } from './ack-outbox.js'; +import { getDefaultUpgradeBlockedOutbox } from './upgrade-blocked-outbox.js'; import { getEmbeddingStatus } from '../context/embedding.js'; import type { EmbeddingStatus } from '../../shared/embedding-status.js'; import type { DiskUsage } from '../../shared/disk-usage.js'; @@ -386,7 +392,14 @@ export class ServerLink { }); // Send auth handshake immediately — server closes the socket if this is not // the first message or if credentials are invalid (5s timeout enforced server-side). - ws.send(JSON.stringify({ type: 'auth', serverId: this.serverId, token: this.token, daemonVersion: this.daemonVersion })); + ws.send(JSON.stringify({ + type: 'auth', + serverId: this.serverId, + token: this.token, + daemonVersion: this.daemonVersion, + [DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.AUTH_REVISION_FIELD]: + DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION, + })); this.sendDaemonHello(); // Wire transport relay so provider callbacks can send events to browsers via this socket. setTransportRelaySend((msg) => { @@ -410,6 +423,28 @@ export class ServerLink { outbox.flushOnReconnect(sender).catch((err) => { logger.warn({ err }, 'AckOutbox flush on reconnect failed'); }); + const sendUpgradeBlockedFrame = (message: unknown): boolean => { + if (this.ws !== ws) return false; + return this.trySend(message); + }; + getDefaultUpgradeBlockedOutbox() + .flushOnReconnect(sendUpgradeBlockedFrame, this.daemonVersion) + .then(() => { + // Connection-bound barrier: the server must not arm an automatic + // upgrade until this daemon has finished reading and replaying its + // persisted terminal blocker. Binding the sender to `ws` prevents a + // delayed flush from an old connection from certifying a replacement. + sendUpgradeBlockedFrame({ + type: DAEMON_MSG.UPGRADE_BLOCKED_SYNC, + revision: DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION, + }); + }) + .catch((err) => { + // Fail closed. Without a successful outbox read we cannot prove that + // the previous install did not terminally fail, so no sync-complete + // frame is sent and the server keeps automatic upgrade gated. + logger.warn({ err }, 'UpgradeBlockedOutbox flush on reconnect failed'); + }); // Resume the data-plane drain after reconnect. Anything that piled up // in `dataPlaneSendQueue` while the link was down is now safe to send @@ -481,6 +516,22 @@ export class ServerLink { } try { const msg = JSON.parse(event.data); + if ( + msg?.type === DAEMON_MSG.UPGRADE_BLOCKED_ACK + && typeof msg.failureId === 'string' + && ( + msg.disposition === DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.ACCEPTED + || msg.disposition === DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.OBSOLETE + || msg.disposition === DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.SUPERSEDED + ) + ) { + void getDefaultUpgradeBlockedOutbox() + .acknowledge(msg.failureId, msg.disposition) + .catch((err) => { + logger.warn({ err, failureId: msg.failureId }, 'UpgradeBlockedOutbox ack handling failed'); + }); + return; + } if (msg?.type === 'heartbeat_ack') { // Heartbeat acks are the CLI/status proof-of-life source. They must // not be throttled behind a just-written heartbeat-sent record, or diff --git a/src/daemon/upgrade-blocked-outbox.ts b/src/daemon/upgrade-blocked-outbox.ts new file mode 100644 index 000000000..dd59850a5 --- /dev/null +++ b/src/daemon/upgrade-blocked-outbox.ts @@ -0,0 +1,197 @@ +/** + * Crash-safe single-record outbox for detached daemon-upgrade failures. + * + * The detached npm process can finish while ServerLink is disconnected. + * A plain `serverLink.send()` drops that control-plane message forever, so + * persist the latest terminal install failure before attempting delivery and + * replay it after the next authenticated WebSocket connection. + */ + +import { mkdir, readFile, rename, unlink, writeFile } from 'node:fs/promises'; +import { homedir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { DAEMON_MSG } from '../../shared/daemon-events.js'; +import { + DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION, + DAEMON_UPGRADE_BLOCK_REASON, + type DaemonUpgradeBlockedAckDisposition, +} from '../../shared/daemon-upgrade.js'; +import logger from '../util/logger.js'; + +const UPGRADE_BLOCKED_OUTBOX_TTL_MS = 7 * 24 * 60 * 60 * 1000; +const DEFAULT_FILE = join(homedir(), '.imcodes', 'pending-upgrade-blocked.json'); + +export interface UpgradeInstallFailedMessage { + type: typeof DAEMON_MSG.UPGRADE_BLOCKED; + reason: typeof DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED; + failureId: string; + upgradeId?: string; + fromVersion: string; + targetVersion: string; + retryReason: string; + attempts?: number; + exitCode: number; + log: string; + signal?: string; + ts: number; + acknowledgedAt?: number; +} + +export type UpgradeBlockedSender = (message: UpgradeInstallFailedMessage) => boolean; + +function boundedString(value: unknown, max: number): string | null { + return typeof value === 'string' && value.length > 0 && value.length <= max ? value : null; +} + +export function parseUpgradeInstallFailedMessage(raw: string): UpgradeInstallFailedMessage | null { + try { + const value = JSON.parse(raw) as Record; + if (value.type !== DAEMON_MSG.UPGRADE_BLOCKED) return null; + if (value.reason !== DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED) return null; + const failureId = boundedString(value.failureId, 128); + const upgradeId = value.upgradeId === undefined ? null : boundedString(value.upgradeId, 128); + const fromVersion = boundedString(value.fromVersion, 128); + const targetVersion = boundedString(value.targetVersion, 128); + const retryReason = boundedString(value.retryReason, 64); + const log = boundedString(value.log, 4096); + if (!failureId || (value.upgradeId !== undefined && !upgradeId) || !fromVersion || !targetVersion || !retryReason || !log) return null; + if (!/^[a-z][a-z0-9-]{0,63}$/.test(retryReason)) return null; + if (!Number.isInteger(value.exitCode) || (value.exitCode as number) < 1 || (value.exitCode as number) > 255) return null; + if (!Number.isInteger(value.ts) || (value.ts as number) < 1) return null; + if (value.attempts !== undefined && (!Number.isInteger(value.attempts) || (value.attempts as number) < 1 || (value.attempts as number) > 100)) return null; + if (value.signal !== undefined && typeof value.signal !== 'string') return null; + if (value.acknowledgedAt !== undefined && (!Number.isInteger(value.acknowledgedAt) || (value.acknowledgedAt as number) < 1)) return null; + return { + type: DAEMON_MSG.UPGRADE_BLOCKED, + reason: DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED, + failureId, + ...(upgradeId ? { upgradeId } : {}), + fromVersion, + targetVersion, + retryReason, + ...(value.attempts === undefined ? {} : { attempts: value.attempts as number }), + exitCode: value.exitCode as number, + log, + ...(typeof value.signal === 'string' ? { signal: value.signal } : {}), + ts: value.ts as number, + ...(value.acknowledgedAt === undefined ? {} : { acknowledgedAt: value.acknowledgedAt as number }), + }; + } catch { + return null; + } +} + +export class UpgradeBlockedOutbox { + private operation: Promise = Promise.resolve(); + + constructor( + private readonly filePath = DEFAULT_FILE, + private readonly now: () => number = Date.now, + ) {} + + /** + * Persist before send. A false/throwing sender leaves the record on disk for + * reconnect replay. A successful WebSocket enqueue does not remove the + * terminal blocker: it must survive server restarts until this daemon moves + * off fromVersion or the server explicitly declares the target obsolete. + */ + report(message: UpgradeInstallFailedMessage, send: UpgradeBlockedSender): Promise { + return this.serialize(async () => { + await this.persist(message); + return this.tryDeliver(message, send); + }); + } + + /** Replay one still-relevant terminal failure after authenticated reconnect. */ + flushOnReconnect(send: UpgradeBlockedSender, currentDaemonVersion: string): Promise { + return this.serialize(async () => { + const message = await this.load(); + if (!message) return false; + if ( + message.fromVersion !== currentDaemonVersion + || this.now() - message.ts > UPGRADE_BLOCKED_OUTBOX_TTL_MS + ) { + await this.clear(); + return false; + } + return this.tryDeliver(message, send); + }); + } + + /** + * Record that the server applied the blocker. Keep accepted blockers on disk + * so a later server restart can rebuild its in-memory terminal target gate. + * A changed server target makes the old failure obsolete and safe to clear. + */ + acknowledge(failureId: string, disposition: DaemonUpgradeBlockedAckDisposition): Promise { + return this.serialize(async () => { + const message = await this.load(); + if (!message || message.failureId !== failureId) return false; + if ( + disposition === DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.OBSOLETE + || disposition === DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.SUPERSEDED + ) { + await this.clear(); + return true; + } + if (message.acknowledgedAt === undefined) { + await this.persist({ ...message, acknowledgedAt: this.now() }); + } + return true; + }); + } + + private serialize(task: () => Promise): Promise { + const result = this.operation.then(task, task); + this.operation = result.then(() => undefined, () => undefined); + return result; + } + + private async persist(message: UpgradeInstallFailedMessage): Promise { + await mkdir(dirname(this.filePath), { recursive: true }); + const tmp = `${this.filePath}.tmp`; + await writeFile(tmp, `${JSON.stringify(message)}\n`, { encoding: 'utf8', mode: 0o600 }); + await rename(tmp, this.filePath); + } + + private async load(): Promise { + try { + const message = parseUpgradeInstallFailedMessage(await readFile(this.filePath, 'utf8')); + if (message) return message; + logger.warn({ file: this.filePath }, 'UpgradeBlockedOutbox: discarding invalid record'); + await this.clear(); + return null; + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return null; + throw err; + } + } + + private async tryDeliver(message: UpgradeInstallFailedMessage, send: UpgradeBlockedSender): Promise { + try { + return send(message); + } catch (err) { + logger.warn({ err, failureId: message.failureId }, 'UpgradeBlockedOutbox: send failed; retaining for reconnect'); + return false; + } + } + + private async clear(): Promise { + try { + await unlink(this.filePath); + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err; + } + } +} + +let defaultOutbox: UpgradeBlockedOutbox | null = null; + +export function getDefaultUpgradeBlockedOutbox(): UpgradeBlockedOutbox { + if (!defaultOutbox) defaultOutbox = new UpgradeBlockedOutbox(); + return defaultOutbox; +} + +export function __resetDefaultUpgradeBlockedOutboxForTests(): void { + defaultOutbox = null; +} diff --git a/src/util/posix-upgrade-layout-recovery.ts b/src/util/posix-upgrade-layout-recovery.ts new file mode 100644 index 000000000..5288020b0 --- /dev/null +++ b/src/util/posix-upgrade-layout-recovery.ts @@ -0,0 +1,175 @@ +import { DAEMON_UPGRADE_BLOCK_REASON } from '../../shared/daemon-upgrade.js'; + +export const POSIX_UPGRADE_INSTALL_FAILURE_EXIT_CODE = 75; + +export interface PosixUpgradeFailureStatus { + state: 'blocked'; + reason: typeof DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED; + retryReason: string; + attempts: number; + exitCode: number; +} + +/** + * Parse the status marker written by the detached POSIX upgrade script. + * Keep this strict: the marker crosses a process boundary and is later + * forwarded to the server as diagnostics. + */ +export function parsePosixUpgradeFailureStatus(raw: string): PosixUpgradeFailureStatus | null { + try { + const value = JSON.parse(raw) as Record; + if (value.state !== 'blocked' || value.reason !== DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED) return null; + if (typeof value.retryReason !== 'string' || !/^[a-z][a-z0-9-]{0,63}$/.test(value.retryReason)) return null; + if (!Number.isInteger(value.attempts) || (value.attempts as number) < 1 || (value.attempts as number) > 100) return null; + if (!Number.isInteger(value.exitCode) || (value.exitCode as number) < 1 || (value.exitCode as number) > 255) return null; + return { + state: 'blocked', + reason: DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED, + retryReason: value.retryReason, + attempts: value.attempts as number, + exitCode: value.exitCode as number, + }; + } catch { + return null; + } +} + +/** + * Bash helpers embedded verbatim in the detached Linux/macOS upgrade script. + * + * npm atomically renames the installed global package to a sibling named + * `.imcodes-` during reify. If an earlier install was interrupted, + * that deterministic staging destination can remain non-empty and every later + * install fails before the new package's preinstall hook can run. + */ +export function buildPosixUpgradeLayoutRecoveryScript(): string { + return ` +npm_error_field() { + local output_file="$1" + local wanted="$2" + awk -v wanted="$wanted" ' + $1 == "npm" && ($2 == "error" || $2 == "ERR!") && $3 == wanted { + $1 = ""; $2 = ""; $3 = "" + sub(/^[[:space:]]+/, "") + value = $0 + } + END { + if (value != "") print value + } + ' "$output_file" 2>/dev/null +} + +is_safe_global_root() { + local global_root="$1" + case "$global_root" in + /*) ;; + *) return 1 ;; + esac + [ "$global_root" != "/" ] && [ -d "$global_root" ] +} + +is_imcodes_staging_path() { + local global_root="$1" + local candidate="$2" + local suffix + is_safe_global_root "$global_root" || return 1 + case "$candidate" in + "$global_root"/.imcodes-*) ;; + *) return 1 ;; + esac + suffix=\${candidate#"$global_root"/.imcodes-} + case "$suffix" in + ''|*/*) return 1 ;; + esac + return 0 +} + +cleanup_stale_imcodes_staging_dirs() { + local global_root="$1" + local min_age_seconds="$2" + local stale + local modified_at + local now + local age + local cleanup_failed=0 + if ! is_safe_global_root "$global_root"; then + log "[step 1.5] refusing stale staging cleanup for unsafe global root: \${global_root:-}" + return 1 + fi + case "$min_age_seconds" in + ''|*[!0-9]*) + log "[step 1.5] refusing stale staging cleanup with invalid age threshold: \${min_age_seconds:-}" + return 1 + ;; + esac + now=$(date +%s) + for stale in "$global_root"/.imcodes-*; do + if [ ! -e "$stale" ] && [ ! -L "$stale" ]; then + continue + fi + if ! is_imcodes_staging_path "$global_root" "$stale"; then + log "[step 1.5] skipped unexpected staging path: $stale" + cleanup_failed=1 + continue + fi + # The daemon upgrade lock cannot coordinate with a user running npm + # manually. A freshly-created .imcodes-* directory may therefore belong + # to an active external npm reify and must not be removed by this broad + # sweep. Recent collisions are handled later from npm's exact rename + # error, where the reported destination proves which path blocked us. + modified_at=$(stat -c %Y "$stale" 2>/dev/null || stat -f %m "$stale" 2>/dev/null || true) + case "$modified_at" in + ''|*[!0-9]*) + log "[step 1.5] preserving staging directory with unknown age: $stale" + continue + ;; + esac + age=$((now - modified_at)) + if [ "$age" -lt 0 ] || [ "$age" -lt "$min_age_seconds" ]; then + log "[step 1.5] preserving recent npm staging directory (age \${age}s): $stale" + continue + fi + log "[step 1.5] removing stale npm staging directory: $stale" + if ! rm -rf -- "$stale"; then + log "[step 1.5] failed to remove stale npm staging directory: $stale" + cleanup_failed=1 + fi + done + return "$cleanup_failed" +} + +is_recoverable_layout_output() { + local output_file="$1" + local global_root="$2" + local error_code + local error_syscall + local error_path + local error_dest + error_code=$(npm_error_field "$output_file" code) + error_syscall=$(npm_error_field "$output_file" syscall) + error_path=$(npm_error_field "$output_file" path) + error_dest=$(npm_error_field "$output_file" dest) + case "$error_code" in + ENOTEMPTY|EEXIST) ;; + *) return 1 ;; + esac + [ "$error_syscall" = "rename" ] || return 1 + [ "$error_path" = "$global_root/imcodes" ] || return 1 + is_imcodes_staging_path "$global_root" "$error_dest" +} + +recover_stale_layout_from_output() { + local output_file="$1" + local global_root="$2" + local error_dest + is_recoverable_layout_output "$output_file" "$global_root" || return 1 + error_dest=$(npm_error_field "$output_file" dest) + if [ ! -e "$error_dest" ] && [ ! -L "$error_dest" ]; then + log "[step 2] stale npm staging destination already absent: $error_dest" + return 0 + fi + log "[step 2] removing npm staging destination from rename failure: $error_dest" + rm -rf -- "$error_dest" +} +`; +} diff --git a/test/daemon/server-link.test.ts b/test/daemon/server-link.test.ts index bfa3c0f19..7c86fcd62 100644 --- a/test/daemon/server-link.test.ts +++ b/test/daemon/server-link.test.ts @@ -1,5 +1,14 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +const upgradeBlockedOutboxMock = vi.hoisted(() => ({ + flushOnReconnect: vi.fn(async () => false), + acknowledge: vi.fn(async () => true), +})); + +vi.mock('../../src/daemon/upgrade-blocked-outbox.js', () => ({ + getDefaultUpgradeBlockedOutbox: () => upgradeBlockedOutboxMock, +})); + // Mock WebSocket before importing ServerLink const mockWsInstance = { send: vi.fn(), @@ -22,6 +31,11 @@ import { recordDaemonServerLinkStatus } from '../../src/util/daemon-status.js'; import { TIMELINE_MESSAGES, TIMELINE_PROTOCOL_CAPABILITY } from '../../shared/timeline-protocol.js'; import { TRANSPORT_EVENT } from '../../shared/transport-events.js'; import { FILE_TRANSFER_UPLOAD_FETCH_CAPABILITY } from '../../shared/transport/file-transfer.js'; +import { DAEMON_MSG } from '../../shared/daemon-events.js'; +import { + DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION, + DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL, +} from '../../shared/daemon-upgrade.js'; const recordDaemonServerLinkStatusMock = vi.mocked(recordDaemonServerLinkStatus); @@ -30,6 +44,7 @@ describe('ServerLink', () => { beforeEach(() => { vi.clearAllMocks(); + upgradeBlockedOutboxMock.flushOnReconnect.mockResolvedValue(false); link = new ServerLink({ workerUrl: 'wss://test.workers.dev', serverId: 'srv-123', @@ -56,6 +71,76 @@ describe('ServerLink', () => { ); }); + it('replays one persisted terminal upgrade failure after the link opens', async () => { + link.connect(); + const openHandler = mockWsInstance.addEventListener.mock.calls.find(([type]) => type === 'open')?.[1] as + | (() => void) + | undefined; + expect(openHandler).toBeDefined(); + + openHandler?.(); + + expect(upgradeBlockedOutboxMock.flushOnReconnect).toHaveBeenCalledTimes(1); + expect(upgradeBlockedOutboxMock.flushOnReconnect).toHaveBeenCalledWith( + expect.any(Function), + link.daemonVersion, + ); + }); + + it('advertises blocker-sync support and certifies the connection only after outbox replay settles', async () => { + let resolveFlush!: (value: boolean) => void; + upgradeBlockedOutboxMock.flushOnReconnect.mockImplementationOnce( + () => new Promise((resolve) => { resolveFlush = resolve; }), + ); + + link.connect(); + const openHandler = mockWsInstance.addEventListener.mock.calls.find(([type]) => type === 'open')?.[1] as + | (() => void) + | undefined; + openHandler?.(); + + const beforeFlush = mockWsInstance.send.mock.calls.map(([raw]) => JSON.parse(String(raw)) as Record); + expect(beforeFlush[0]).toMatchObject({ + type: 'auth', + [DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.AUTH_REVISION_FIELD]: + DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION, + }); + expect(beforeFlush.some((message) => message.type === DAEMON_MSG.UPGRADE_BLOCKED_SYNC)).toBe(false); + + resolveFlush(false); + await Promise.resolve(); + await Promise.resolve(); + + expect(mockWsInstance.send.mock.calls.map(([raw]) => JSON.parse(String(raw)))).toContainEqual(expect.objectContaining({ + type: DAEMON_MSG.UPGRADE_BLOCKED_SYNC, + revision: DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION, + })); + }); + + it('records the server acknowledgement without exposing it to ordinary message handlers', async () => { + link.connect(); + const messageHandler = mockWsInstance.addEventListener.mock.calls.find(([type]) => type === 'message')?.[1] as + | ((event: MessageEvent) => void) + | undefined; + const ordinaryHandler = vi.fn(); + link.onMessage(ordinaryHandler); + + messageHandler?.({ + data: JSON.stringify({ + type: DAEMON_MSG.UPGRADE_BLOCKED_ACK, + failureId: 'failure-1', + disposition: DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.ACCEPTED, + }), + } as MessageEvent); + await Promise.resolve(); + + expect(upgradeBlockedOutboxMock.acknowledge).toHaveBeenCalledWith( + 'failure-1', + DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.ACCEPTED, + ); + expect(ordinaryHandler).not.toHaveBeenCalled(); + }); + it('send() silently drops messages when not connected (fire-and-forget safe)', () => { // The daemon must never die from transient disconnects — ServerLink.send() // is best-effort and must not throw. Callers that need delivery diff --git a/test/daemon/upgrade-blocked-outbox.test.ts b/test/daemon/upgrade-blocked-outbox.test.ts new file mode 100644 index 000000000..f3fd17be4 --- /dev/null +++ b/test/daemon/upgrade-blocked-outbox.test.ts @@ -0,0 +1,90 @@ +import { existsSync, mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { DAEMON_MSG } from '../../shared/daemon-events.js'; +import { DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION } from '../../shared/daemon-upgrade.js'; +import { UpgradeBlockedOutbox, type UpgradeInstallFailedMessage } from '../../src/daemon/upgrade-blocked-outbox.js'; + +const tempDirs: string[] = []; + +function makeFixture(now = 10_000) { + const dir = mkdtempSync(join(tmpdir(), 'imcodes-upgrade-blocked-outbox-')); + tempDirs.push(dir); + const file = join(dir, 'pending.json'); + const message: UpgradeInstallFailedMessage = { + type: DAEMON_MSG.UPGRADE_BLOCKED, + reason: 'install_failed', + failureId: 'failure-1', + upgradeId: 'upgrade-1', + fromVersion: '2026.7.3157-dev.3556', + targetVersion: '2026.7.3192-dev.3593', + retryReason: 'stale-staging-dir', + attempts: 5, + exitCode: 217, + log: '/tmp/imcodes-upgrade-test/upgrade.log', + ts: now, + }; + return { file, message, now }; +} + +afterEach(() => { + vi.restoreAllMocks(); + while (tempDirs.length > 0) rmSync(tempDirs.pop()!, { recursive: true, force: true }); +}); + +describe('UpgradeBlockedOutbox', () => { + it('retains an accepted terminal blocker so a later server restart can rebuild the target gate', async () => { + const { file, message, now } = makeFixture(); + const first = new UpgradeBlockedOutbox(file, () => now); + + expect(await first.report(message, () => false)).toBe(false); + expect(existsSync(file)).toBe(true); + + const afterRestart = new UpgradeBlockedOutbox(file, () => now + 1); + const send = vi.fn(() => true); + expect(await afterRestart.flushOnReconnect(send, message.fromVersion)).toBe(true); + expect(send).toHaveBeenCalledTimes(1); + expect(send).toHaveBeenCalledWith(message); + expect(existsSync(file)).toBe(true); + + expect(await afterRestart.acknowledge( + message.failureId, + DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.ACCEPTED, + )).toBe(true); + expect(existsSync(file)).toBe(true); + + const afterServerRestart = new UpgradeBlockedOutbox(file, () => now + 2); + expect(await afterServerRestart.flushOnReconnect(send, message.fromVersion)).toBe(true); + expect(send).toHaveBeenCalledTimes(2); + + expect(await afterServerRestart.acknowledge( + message.failureId, + DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.OBSOLETE, + )).toBe(true); + expect(existsSync(file)).toBe(false); + }); + + it('clears a persisted failure superseded by a newer explicit manual upgrade', async () => { + const { file, message, now } = makeFixture(); + const outbox = new UpgradeBlockedOutbox(file, () => now); + await outbox.report(message, () => false); + + expect(await outbox.acknowledge( + message.failureId, + DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.SUPERSEDED, + )).toBe(true); + expect(existsSync(file)).toBe(false); + }); + + it('clears an obsolete failure after a later daemon version starts', async () => { + const { file, message, now } = makeFixture(); + const outbox = new UpgradeBlockedOutbox(file, () => now); + await outbox.report(message, () => false); + + const send = vi.fn(() => true); + expect(await outbox.flushOnReconnect(send, '2026.7.3192-dev.3593')).toBe(false); + expect(send).not.toHaveBeenCalled(); + expect(existsSync(file)).toBe(false); + }); +}); diff --git a/test/e2e/daemon-upgrade-gate.test.ts b/test/e2e/daemon-upgrade-gate.test.ts index c4b8e292b..1a44846d0 100644 --- a/test/e2e/daemon-upgrade-gate.test.ts +++ b/test/e2e/daemon-upgrade-gate.test.ts @@ -30,18 +30,26 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { DAEMON_MSG } from '../../shared/daemon-events.js'; +import { DAEMON_UPGRADE_BLOCK_REASON } from '../../shared/daemon-upgrade.js'; const mocks = vi.hoisted(() => { const store = new Map>(); const runtimes = new Map string; sending: boolean; pendingCount: number }>(); const spawnCalls: Array<{ command: string; args: readonly string[] }> = []; + const spawnedChildren: Array<{ + command: string; + emit: (event: string, ...args: unknown[]) => void; + }> = []; const writeCalls: Array<{ path: string; data: string }> = []; + const upgradeBlockedReport = vi.fn(async () => false); let compressionState = { active: false, activeCount: 0, queued: 0, idle: true }; return { store, runtimes, spawnCalls, + spawnedChildren, writeCalls, + upgradeBlockedReport, getCompressionState: () => compressionState, setCompressionState: (next: typeof compressionState) => { compressionState = next; }, }; @@ -62,6 +70,12 @@ vi.mock('../../src/store/session-store.js', () => ({ updateSessionState: vi.fn(), })); +vi.mock('../../src/daemon/upgrade-blocked-outbox.js', () => ({ + getDefaultUpgradeBlockedOutbox: () => ({ + report: mocks.upgradeBlockedReport, + }), +})); + vi.mock('../../src/agent/session-manager.js', async (importOriginal) => { const actual = await importOriginal(); return { @@ -78,12 +92,25 @@ vi.mock('../../src/agent/session-manager.js', async (importOriginal) => { // caring about platform. function captureSpawn(command: string, args: readonly string[]) { mocks.spawnCalls.push({ command, args }); - return { + const handlers = new Map void>>(); + const child = { unref: vi.fn(), - on: vi.fn(), + on: vi.fn((event: string, handler: (...args: unknown[]) => void) => { + const existing = handlers.get(event) ?? []; + existing.push(handler); + handlers.set(event, existing); + return child; + }), stdout: { on: vi.fn() }, stderr: { on: vi.fn() }, } as any; + mocks.spawnedChildren.push({ + command, + emit: (event, ...eventArgs) => { + for (const handler of handlers.get(event) ?? []) handler(...eventArgs); + }, + }); + return child; } vi.mock('child_process', async (importOriginal) => { @@ -303,6 +330,8 @@ describe('daemon.upgrade gate (e2e regression for 3389fab2)', () => { mocks.store.clear(); mocks.runtimes.clear(); mocks.spawnCalls.length = 0; + mocks.spawnedChildren.length = 0; + mocks.upgradeBlockedReport.mockClear(); mocks.setCompressionState({ active: false, activeCount: 0, queued: 0, idle: true }); }); @@ -573,6 +602,7 @@ skipOnWindows('daemon.upgrade — Linux/macOS upgrade.sh contract', () => { mocks.store.clear(); mocks.runtimes.clear(); mocks.spawnCalls.length = 0; + mocks.spawnedChildren.length = 0; mocks.writeCalls.length = 0; }); @@ -580,16 +610,27 @@ skipOnWindows('daemon.upgrade — Linux/macOS upgrade.sh contract', () => { vi.clearAllMocks(); }); - async function captureUpgradeScript(): Promise { + async function captureUpgradeAttempt(): Promise<{ + script: string; + serverLink: { send: ReturnType }; + }> { const serverLink = { send: vi.fn() } as { send: ReturnType }; - handleWebCommand({ type: 'daemon.upgrade', targetVersion: '99.99.99-test' } as any, serverLink as any); + handleWebCommand({ + type: 'daemon.upgrade', + targetVersion: '99.99.99-test', + upgradeId: 'upgrade-attempt-test', + } as any, serverLink as any); await waitForCondition( () => mocks.writeCalls.some((c) => c.path.endsWith('upgrade.sh')), 5000, ); const script = mocks.writeCalls.find((c) => c.path.endsWith('upgrade.sh')); if (!script) throw new Error('upgrade.sh was never written'); - return script.data; + return { script: script.data, serverLink }; + } + + async function captureUpgradeScript(): Promise { + return (await captureUpgradeAttempt()).script; } it('captures the real install exit code (regression: pre-fix logged "(exit 0)" on every failure)', async () => { @@ -626,6 +667,57 @@ skipOnWindows('daemon.upgrade — Linux/macOS upgrade.sh contract', () => { expect(sh).toMatch(/tail -20 "\$INSTALL_OUT"/); }); + it('cleans stale npm staging before install and retries only matching rename collisions', async () => { + const sh = await captureUpgradeScript(); + + const cleanupIdx = sh.indexOf('cleanup_stale_imcodes_staging_dirs "$GLOBAL_ROOT"'); + const installIdx = sh.indexOf('install -g --ignore-scripts --prefer-online'); + expect(cleanupIdx).toBeGreaterThan(-1); + expect(installIdx).toBeGreaterThan(-1); + expect(cleanupIdx).toBeLessThan(installIdx); + expect(sh).toContain('is_recoverable_layout_output "$INSTALL_OUT" "$GLOBAL_ROOT"'); + expect(sh).toContain('recover_stale_layout_from_output "$INSTALL_OUT" "$GLOBAL_ROOT"'); + expect(sh).toContain('RETRY_REASON="stale-staging-dir"'); + expect(sh).toMatch(/ENOTEMPTY\|EEXIST/); + expect(sh).toContain('[ "$error_syscall" = "rename" ]'); + expect(sh).toContain('[ "$error_path" = "$global_root/imcodes" ]'); + }); + + it('reports a final detached npm install failure to the server with bounded diagnostics', async () => { + const realFs = await vi.importActual('node:fs'); + const statusDir = '/tmp/imcodes-upgrade-gate-test'; + const statusPath = `${statusDir}/upgrade-status.json`; + realFs.mkdirSync(statusDir, { recursive: true }); + try { + const { serverLink } = await captureUpgradeAttempt(); + const bashChild = mocks.spawnedChildren.find((entry) => entry.command === '/bin/bash'); + expect(bashChild).toBeDefined(); + realFs.writeFileSync(statusPath, JSON.stringify({ + state: 'blocked', + reason: DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED, + retryReason: 'stale-staging-dir', + attempts: 5, + exitCode: 217, + })); + + bashChild!.emit('exit', 75, null); + + await vi.waitFor(() => expect(mocks.upgradeBlockedReport).toHaveBeenCalledWith(expect.objectContaining({ + type: DAEMON_MSG.UPGRADE_BLOCKED, + reason: DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED, + upgradeId: 'upgrade-attempt-test', + retryReason: 'stale-staging-dir', + attempts: 5, + exitCode: 217, + }), expect.any(Function))); + expect(serverLink.send).not.toHaveBeenCalledWith(expect.objectContaining({ + reason: DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED, + })); + } finally { + realFs.rmSync(statusPath, { force: true }); + } + }); + it('uses an atomic single-flight lock before touching the global npm install', async () => { const sh = await captureUpgradeScript(); diff --git a/test/shared/wire-protocol-contract.test.ts b/test/shared/wire-protocol-contract.test.ts index f442ff3e4..9f72d7212 100644 --- a/test/shared/wire-protocol-contract.test.ts +++ b/test/shared/wire-protocol-contract.test.ts @@ -68,6 +68,8 @@ describe('shared daemon/server/web wire protocol contracts', () => { RECONNECTED: 'daemon.reconnected', DISCONNECTED: 'daemon.disconnected', UPGRADE_BLOCKED: 'daemon.upgrade_blocked', + UPGRADE_BLOCKED_ACK: 'daemon.upgrade_blocked_ack', + UPGRADE_BLOCKED_SYNC: 'daemon.upgrade_blocked_sync', UPGRADING: 'daemon.upgrading', MACHINE_EXEC_CHUNK: 'machine.exec_chunk', MACHINE_EXEC_RESULT: 'machine.exec_result', diff --git a/test/util/posix-upgrade-layout-recovery.test.ts b/test/util/posix-upgrade-layout-recovery.test.ts new file mode 100644 index 000000000..65416d317 --- /dev/null +++ b/test/util/posix-upgrade-layout-recovery.test.ts @@ -0,0 +1,184 @@ +import { mkdtempSync, mkdirSync, rmSync, utimesSync, writeFileSync, existsSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { afterEach, describe, expect, it } from 'vitest'; +import { + buildPosixUpgradeLayoutRecoveryScript, + parsePosixUpgradeFailureStatus, +} from '../../src/util/posix-upgrade-layout-recovery.js'; + +const describePosix = process.platform === 'win32' ? describe.skip : describe; +const tempDirs: string[] = []; + +function makeTempDir(): string { + const dir = mkdtempSync(join(tmpdir(), 'imcodes-upgrade-layout-')); + tempDirs.push(dir); + return dir; +} + +function runRecoveryHarness(body: string, env: Record) { + return spawnSync('bash', ['-c', ` +set -u +log() { :; } +${buildPosixUpgradeLayoutRecoveryScript()} +${body} +`], { + encoding: 'utf8', + env: { ...process.env, ...env }, + }); +} + +afterEach(() => { + while (tempDirs.length > 0) { + rmSync(tempDirs.pop()!, { recursive: true, force: true }); + } +}); + +describePosix('POSIX daemon-upgrade npm layout recovery', () => { + it('removes only old interrupted .imcodes-* siblings before npm install', () => { + const root = makeTempDir(); + const current = join(root, 'imcodes'); + const staleA = join(root, '.imcodes-Vuo7WXWs'); + const staleB = join(root, '.imcodes-other'); + const recent = join(root, '.imcodes-active-install'); + mkdirSync(current); + mkdirSync(join(staleA, 'nested'), { recursive: true }); + mkdirSync(staleB); + mkdirSync(recent); + writeFileSync(join(staleA, 'nested', 'leftover'), 'stale'); + const old = new Date(Date.now() - 2 * 60 * 60_000); + utimesSync(staleA, old, old); + utimesSync(staleB, old, old); + + const result = runRecoveryHarness('cleanup_stale_imcodes_staging_dirs "$ROOT" 1800', { ROOT: root }); + + expect(result.status, result.stderr).toBe(0); + expect(existsSync(current)).toBe(true); + expect(existsSync(staleA)).toBe(false); + expect(existsSync(staleB)).toBe(false); + expect(existsSync(recent)).toBe(true); + }); + + it('preserves a recent staging directory that may belong to an active external npm install', () => { + const root = makeTempDir(); + const active = join(root, '.imcodes-active-install'); + mkdirSync(active); + writeFileSync(join(active, 'in-progress'), 'active'); + + const result = runRecoveryHarness('cleanup_stale_imcodes_staging_dirs "$ROOT" 1800', { ROOT: root }); + + expect(result.status, result.stderr).toBe(0); + expect(existsSync(active)).toBe(true); + }); + + it.each(['ENOTEMPTY', 'EEXIST'])('classifies %s rename collisions and removes the exact stale destination', (code) => { + const root = makeTempDir(); + const current = join(root, 'imcodes'); + const stale = join(root, '.imcodes-Vuo7WXWs'); + const output = join(root, 'npm-output.log'); + mkdirSync(current); + mkdirSync(stale); + writeFileSync(join(stale, 'leftover'), 'stale'); + writeFileSync(output, [ + `npm error code ${code}`, + 'npm error syscall rename', + `npm error path ${current}`, + `npm error dest ${stale}`, + `npm error ${code}: directory not empty`, + '', + ].join('\n')); + + const result = runRecoveryHarness( + 'is_recoverable_layout_output "$OUTPUT" "$ROOT" && recover_stale_layout_from_output "$OUTPUT" "$ROOT"', + { ROOT: root, OUTPUT: output }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(existsSync(current)).toBe(true); + expect(existsSync(stale)).toBe(false); + }); + + it.each([ + ['EACCES', 'rename'], + ['E401', 'rename'], + ['ENOTEMPTY', 'unlink'], + ])('does not retry or clean a genuine non-layout failure (%s/%s)', (code, syscall) => { + const root = makeTempDir(); + const current = join(root, 'imcodes'); + const stale = join(root, '.imcodes-Vuo7WXWs'); + const output = join(root, 'npm-output.log'); + mkdirSync(current); + mkdirSync(stale); + writeFileSync(join(stale, 'must-stay'), 'data'); + writeFileSync(output, [ + `npm error code ${code}`, + `npm error syscall ${syscall}`, + `npm error path ${current}`, + `npm error dest ${stale}`, + '', + ].join('\n')); + + const result = runRecoveryHarness( + 'if is_recoverable_layout_output "$OUTPUT" "$ROOT"; then exit 42; fi', + { ROOT: root, OUTPUT: output }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(existsSync(stale)).toBe(true); + }); + + it('rejects a rename destination outside the global package root', () => { + const root = makeTempDir(); + const outsideRoot = makeTempDir(); + const current = join(root, 'imcodes'); + const outside = join(outsideRoot, '.imcodes-Vuo7WXWs'); + const output = join(root, 'npm-output.log'); + mkdirSync(current); + mkdirSync(outside); + writeFileSync(output, [ + 'npm error code ENOTEMPTY', + 'npm error syscall rename', + `npm error path ${current}`, + `npm error dest ${outside}`, + '', + ].join('\n')); + + const result = runRecoveryHarness( + 'if is_recoverable_layout_output "$OUTPUT" "$ROOT"; then exit 42; fi', + { ROOT: root, OUTPUT: output }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(existsSync(outside)).toBe(true); + }); +}); + +describe('POSIX daemon-upgrade failure marker', () => { + it('accepts the bounded install-failure status emitted by the detached script', () => { + expect(parsePosixUpgradeFailureStatus(JSON.stringify({ + state: 'blocked', + reason: 'install_failed', + retryReason: 'stale-staging-dir', + attempts: 3, + exitCode: 217, + }))).toEqual({ + state: 'blocked', + reason: 'install_failed', + retryReason: 'stale-staging-dir', + attempts: 3, + exitCode: 217, + }); + }); + + it('rejects malformed or unbounded failure markers', () => { + expect(parsePosixUpgradeFailureStatus('not-json')).toBeNull(); + expect(parsePosixUpgradeFailureStatus(JSON.stringify({ + state: 'blocked', + reason: 'install_failed', + retryReason: '../../escape', + attempts: 0, + exitCode: 999, + }))).toBeNull(); + }); +}); diff --git a/web/src/app.tsx b/web/src/app.tsx index ce453278d..7aa0fa657 100644 --- a/web/src/app.tsx +++ b/web/src/app.tsx @@ -3486,42 +3486,6 @@ export function App() { } } } - if (msg.type === DAEMON_MSG.UPGRADE_BLOCKED) { - const now = Date.now(); - if (!shouldShowDaemonUpgradeBlockedToast(lastUpgradeBlockedToastRef.current, msg.reason, now)) return; - lastUpgradeBlockedToastRef.current = { reason: msg.reason, shownAt: now }; - const message = trans(daemonUpgradeBlockedToastKey(msg.reason)); - const id = Date.now() + Math.random(); - setToasts((prev) => [...prev, { - id, - sessionName: '', - project: '', - kind: 'notification', - title: trans('toast.upgrade_blocked_title'), - message, - }]); - setTimeout(() => setToasts((prev) => prev.filter((x) => x.id !== id)), 8000); - } - if (msg.type === DAEMON_MSG.DISCONNECTED) { - // Mark projection stale immediately — that's just a data-freshness - // hint, not the user-facing status badge. But do NOT flip the - // "Daemon Offline" badge yet: the server side still has a - // RECONNECT_GRACE_MS window during which the daemon can reconnect - // and inflight commands are replayed without surfacing any failure. - // Matching that grace period here prevents the badge from flashing - // on every pod restart / brief network blip while the user's turn - // is actually landing fine. If the daemon does stay gone, the - // server will broadcast MSG_DAEMON_OFFLINE (no reconnect event) and - // this timer fires, putting the badge into the Daemon-Offline - // state. RECONNECTED / session_list clear the timer below. - watchProjectionStore.setSnapshotStatus('stale'); - if (daemonOfflineGraceTimerRef.current) clearTimeout(daemonOfflineGraceTimerRef.current); - daemonOfflineGraceTimerRef.current = setTimeout(() => { - daemonOfflineGraceTimerRef.current = null; - setDaemonOnline(false); - setServers((prev) => markServerOffline(prev, selectedServerId)); - }, RECONNECT_GRACE_MS); - } { // Daemon-upgrade badge lifecycle (pure reducer): UPGRADING begins it // (so the version shows "upgrading…" through the restart instead of a @@ -3547,6 +3511,43 @@ export function App() { } } } + if (msg.type === DAEMON_MSG.UPGRADE_BLOCKED) { + const now = Date.now(); + if (shouldShowDaemonUpgradeBlockedToast(lastUpgradeBlockedToastRef.current, msg.reason, now)) { + lastUpgradeBlockedToastRef.current = { reason: msg.reason, shownAt: now }; + const message = trans(daemonUpgradeBlockedToastKey(msg.reason)); + const id = Date.now() + Math.random(); + setToasts((prev) => [...prev, { + id, + sessionName: '', + project: '', + kind: 'notification', + title: trans('toast.upgrade_blocked_title'), + message, + }]); + setTimeout(() => setToasts((prev) => prev.filter((x) => x.id !== id)), 8000); + } + } + if (msg.type === DAEMON_MSG.DISCONNECTED) { + // Mark projection stale immediately — that's just a data-freshness + // hint, not the user-facing status badge. But do NOT flip the + // "Daemon Offline" badge yet: the server side still has a + // RECONNECT_GRACE_MS window during which the daemon can reconnect + // and inflight commands are replayed without surfacing any failure. + // Matching that grace period here prevents the badge from flashing + // on every pod restart / brief network blip while the user's turn + // is actually landing fine. If the daemon does stay gone, the + // server will broadcast MSG_DAEMON_OFFLINE (no reconnect event) and + // this timer fires, putting the badge into the Daemon-Offline + // state. RECONNECTED / session_list clear the timer below. + watchProjectionStore.setSnapshotStatus('stale'); + if (daemonOfflineGraceTimerRef.current) clearTimeout(daemonOfflineGraceTimerRef.current); + daemonOfflineGraceTimerRef.current = setTimeout(() => { + daemonOfflineGraceTimerRef.current = null; + setDaemonOnline(false); + setServers((prev) => markServerOffline(prev, selectedServerId)); + }, RECONNECT_GRACE_MS); + } if (msg.type === MSG_DAEMON_ONLINE || msg.type === DAEMON_MSG.RECONNECTED) { void checkForAppUpdate(); if (daemonOfflineGraceTimerRef.current) { diff --git a/web/src/daemon-upgrade-blocked.ts b/web/src/daemon-upgrade-blocked.ts index d4485a0b2..72bef141e 100644 --- a/web/src/daemon-upgrade-blocked.ts +++ b/web/src/daemon-upgrade-blocked.ts @@ -1,3 +1,5 @@ +import { DAEMON_UPGRADE_BLOCK_REASON } from '@shared/daemon-upgrade.js'; + export const DAEMON_UPGRADE_BLOCKED_TOAST_THROTTLE_MS = 15 * 60_000; export type DaemonUpgradeBlockedToastState = { @@ -14,6 +16,7 @@ export type DaemonUpgradeBlockedToastKey = | 'toast.upgrade_blocked_session_busy' | 'toast.upgrade_blocked_cooldown_active' | 'toast.upgrade_blocked_toolchain_unavailable' + | 'toast.upgrade_blocked_install_failed' | 'toast.upgrade_blocked_unknown'; /** @@ -31,6 +34,7 @@ export function daemonUpgradeBlockedToastKey(reason: string): DaemonUpgradeBlock case 'session_busy': return 'toast.upgrade_blocked_session_busy'; case 'cooldown_active': return 'toast.upgrade_blocked_cooldown_active'; case 'toolchain_unavailable': return 'toast.upgrade_blocked_toolchain_unavailable'; + case DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED: return 'toast.upgrade_blocked_install_failed'; default: return 'toast.upgrade_blocked_unknown'; } } diff --git a/web/src/i18n/locales/en.json b/web/src/i18n/locales/en.json index 7976ac338..f549c0352 100644 --- a/web/src/i18n/locales/en.json +++ b/web/src/i18n/locales/en.json @@ -860,6 +860,7 @@ "upgrade_blocked_session_busy": "A session still has active work. Wait until it goes idle before upgrading the daemon.", "upgrade_blocked_cooldown_active": "The daemon was upgraded recently. Automatic upgrade will retry after the cooldown.", "upgrade_blocked_toolchain_unavailable": "Node is missing on the daemon host (toolchain removed). Reinstall Node — auto-upgrade is disabled until then.", + "upgrade_blocked_install_failed": "Automatic upgrade failed during npm install. Check the daemon host's upgrade log; it will remain on the current version.", "upgrade_blocked_unknown": "The daemon is temporarily busy. Automatic upgrade will retry." }, "discussion": { diff --git a/web/src/i18n/locales/es.json b/web/src/i18n/locales/es.json index 3aa59ebf9..2aa159b4d 100644 --- a/web/src/i18n/locales/es.json +++ b/web/src/i18n/locales/es.json @@ -860,6 +860,7 @@ "upgrade_blocked_session_busy": "Una sesión todavía tiene trabajo activo. Espera a que quede inactiva antes de actualizar el daemon.", "upgrade_blocked_cooldown_active": "El daemon se actualizó recientemente. La actualización automática volverá a intentarlo después del tiempo de espera.", "upgrade_blocked_toolchain_unavailable": "Falta Node en el host del daemon (toolchain eliminado). Reinstala Node; la actualización automática está deshabilitada hasta entonces.", + "upgrade_blocked_install_failed": "La actualización automática falló durante la instalación de npm. Revisa el registro de actualización del host; el daemon seguirá usando la versión actual.", "upgrade_blocked_unknown": "El daemon está ocupado temporalmente. La actualización automática volverá a intentarlo." }, "discussion": { diff --git a/web/src/i18n/locales/ja.json b/web/src/i18n/locales/ja.json index 484327fe0..a31384614 100644 --- a/web/src/i18n/locales/ja.json +++ b/web/src/i18n/locales/ja.json @@ -860,6 +860,7 @@ "upgrade_blocked_session_busy": "セッションに実行中の処理があります。idle になってから daemon をアップグレードしてください。", "upgrade_blocked_cooldown_active": "daemon は最近アップグレードされました。クールダウン後に自動アップグレードを再試行します。", "upgrade_blocked_toolchain_unavailable": "daemon ホストの Node が見つかりません（ツールチェーンが削除されました）。Node を再インストールしてください。それまで自動アップグレードは無効です。", + "upgrade_blocked_install_failed": "npm のインストール中に自動アップグレードが失敗しました。daemon ホストのアップグレードログを確認してください。現在のバージョンで実行を続けます。", "upgrade_blocked_unknown": "daemon は一時的にビジーです。自動アップグレードを再試行します。" }, "discussion": { diff --git a/web/src/i18n/locales/ko.json b/web/src/i18n/locales/ko.json index cfc40b6f3..82a97f9fe 100644 --- a/web/src/i18n/locales/ko.json +++ b/web/src/i18n/locales/ko.json @@ -860,6 +860,7 @@ "upgrade_blocked_session_busy": "세션에 아직 활성 작업이 있습니다. idle 상태가 된 뒤 daemon을 업그레이드하세요.", "upgrade_blocked_cooldown_active": "daemon이 최근 업그레이드되었습니다. 대기 시간이 끝나면 자동 업그레이드를 다시 시도합니다.", "upgrade_blocked_toolchain_unavailable": "daemon 호스트에 Node가 없습니다(툴체인 제거됨). Node를 다시 설치하세요. 그 전까지 자동 업그레이드가 비활성화됩니다.", + "upgrade_blocked_install_failed": "npm 설치 중 자동 업그레이드가 실패했습니다. daemon 호스트의 업그레이드 로그를 확인하세요. 현재 버전으로 계속 실행됩니다.", "upgrade_blocked_unknown": "daemon이 일시적으로 사용 중입니다. 자동 업그레이드를 다시 시도합니다." }, "discussion": { diff --git a/web/src/i18n/locales/ru.json b/web/src/i18n/locales/ru.json index be41787b7..911c6f226 100644 --- a/web/src/i18n/locales/ru.json +++ b/web/src/i18n/locales/ru.json @@ -860,6 +860,7 @@ "upgrade_blocked_session_busy": "В сессии ещё выполняется работа. Дождитесь состояния idle перед обновлением daemon.", "upgrade_blocked_cooldown_active": "daemon недавно обновлялся. Автообновление повторится после периода ожидания.", "upgrade_blocked_toolchain_unavailable": "На хосте daemon отсутствует Node (toolchain удалён). Переустановите Node — автообновление отключено до этого.", + "upgrade_blocked_install_failed": "Автообновление завершилось с ошибкой во время установки npm. Проверьте журнал обновления на хосте daemon; работа продолжится на текущей версии.", "upgrade_blocked_unknown": "daemon временно занят. Автообновление повторит попытку." }, "discussion": { diff --git a/web/src/i18n/locales/zh-CN.json b/web/src/i18n/locales/zh-CN.json index 00eef97e4..a35a7f332 100644 --- a/web/src/i18n/locales/zh-CN.json +++ b/web/src/i18n/locales/zh-CN.json @@ -860,6 +860,7 @@ "upgrade_blocked_session_busy": "仍有会话正在执行任务。等它回到 idle 再升级 daemon。", "upgrade_blocked_cooldown_active": "daemon 最近刚完成升级，冷却结束后会自动重试。", "upgrade_blocked_toolchain_unavailable": "daemon 主机上的 Node 已丢失（工具链被删除）。请重新安装 Node——在此之前自动升级已禁用。", + "upgrade_blocked_install_failed": "npm 安装期间自动升级失败。请检查 daemon 主机上的升级日志；daemon 将继续运行当前版本。", "upgrade_blocked_unknown": "daemon 暂时繁忙，稍后会自动重试升级。" }, "discussion": { diff --git a/web/src/i18n/locales/zh-TW.json b/web/src/i18n/locales/zh-TW.json index cd05b29c1..e048730e6 100644 --- a/web/src/i18n/locales/zh-TW.json +++ b/web/src/i18n/locales/zh-TW.json @@ -860,6 +860,7 @@ "upgrade_blocked_session_busy": "仍有工作階段正在執行任務。等它回到 idle 再升級 daemon。", "upgrade_blocked_cooldown_active": "daemon 最近剛完成升級，冷卻結束後會自動重試。", "upgrade_blocked_toolchain_unavailable": "daemon 主機上的 Node 已遺失（工具鏈被移除）。請重新安裝 Node——在此之前自動升級已停用。", + "upgrade_blocked_install_failed": "npm 安裝期間自動升級失敗。請檢查 daemon 主機上的升級日誌；daemon 將繼續執行目前版本。", "upgrade_blocked_unknown": "daemon 暫時忙碌，稍後會自動重試升級。" }, "discussion": { diff --git a/web/src/util/daemon-upgrade-status.ts b/web/src/util/daemon-upgrade-status.ts index 25c1a6a4b..48decd5eb 100644 --- a/web/src/util/daemon-upgrade-status.ts +++ b/web/src/util/daemon-upgrade-status.ts @@ -11,6 +11,8 @@ export type DaemonUpgradingState = { targetVersion: string } | null; * version (empty string when the daemon didn't supply a usable one). * - `MSG_DAEMON_ONLINE` / `DAEMON_MSG.RECONNECTED` → the (possibly upgraded) * daemon is back, so clear the badge (`null`). + * - `DAEMON_MSG.UPGRADE_BLOCKED` → the detached upgrade stopped and the + * current daemon remains active, so clear the badge immediately. * - anything else → `undefined`, meaning "not relevant, keep current state". * * Returning a discriminated `undefined` (vs `null`) lets the caller distinguish @@ -23,7 +25,11 @@ export function nextDaemonUpgradingState( if (msgType === DAEMON_MSG.UPGRADING) { return { targetVersion: typeof rawTargetVersion === 'string' ? rawTargetVersion : '' }; } - if (msgType === MSG_DAEMON_ONLINE || msgType === DAEMON_MSG.RECONNECTED) { + if ( + msgType === MSG_DAEMON_ONLINE + || msgType === DAEMON_MSG.RECONNECTED + || msgType === DAEMON_MSG.UPGRADE_BLOCKED + ) { return null; } return undefined; diff --git a/web/src/ws-client.ts b/web/src/ws-client.ts index 0ead91557..e2278cc0e 100644 --- a/web/src/ws-client.ts +++ b/web/src/ws-client.ts @@ -8,6 +8,7 @@ import { apiFetch, ApiError } from './api.js'; import type { TimelineEvent } from '../../src/shared/timeline/types.js'; import { REPO_MSG } from '@shared/repo-types.js'; import { DAEMON_MSG } from '@shared/daemon-events.js'; +import { DAEMON_UPGRADE_BLOCK_REASON } from '@shared/daemon-upgrade.js'; import type { ResourceChangedMessage } from '@shared/resource-events.js'; import { P2P_CONFIG_MSG } from '@shared/p2p-config-events.js'; import { P2P_WORKFLOW_MSG, isP2pWorkflowRequestId } from '@shared/p2p-workflow-messages.js'; @@ -153,7 +154,8 @@ export type ServerMessage = | 'transport_busy' | 'session_busy' | 'cooldown_active' - | 'toolchain_unavailable'; + | 'toolchain_unavailable' + | typeof DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED; activeRunIds?: string[]; activeSessionNames?: string[]; blockedSessions?: TransportUpgradeBlockedSession[]; @@ -163,6 +165,15 @@ export type ServerMessage = lastUpgradeAt?: number | null; nodeBinPresent?: boolean; npmAvailable?: boolean; + failureId?: string; + fromVersion?: string; + targetVersion?: string; + retryReason?: string; + attempts?: number; + exitCode?: number; + log?: string; + signal?: string; + ts?: number; } | { type: 'daemon.error'; kind: 'uncaughtException' | 'unhandledRejection' | 'warning'; message: string; stack?: string; ts: number } | { type: 'session_list'; daemonVersion?: string | null; sessions: Array<{ name: string; sessionInstanceId?: string; runtimeEpoch?: string; project: string; role: string; agentType: string; providerId?: string; agentVersion?: string; state: string; error?: string | null; projectDir?: string; runtimeType?: 'process' | 'transport'; label?: string; description?: string; userCreated?: boolean; ccPreset?: string | null; qwenModel?: string; requestedModel?: string; activeModel?: string; qwenAuthType?: string; qwenAuthLimit?: string; qwenAvailableModels?: string[]; copilotAvailableModels?: string[]; cursorAvailableModels?: string[]; codexAvailableModels?: string[]; modelDisplay?: string; planLabel?: string; permissionLabel?: string; quotaLabel?: string; quotaUsageLabel?: string; quotaMeta?: import('../../shared/provider-quota.js').ProviderQuotaMeta | null; effort?: import('../../shared/effort-levels.js').TransportEffortLevel; contextNamespace?: import('../../shared/session-context-bootstrap.js').SessionContextBootstrapState['contextNamespace']; contextNamespaceDiagnostics?: string[]; contextRemoteProcessedFreshness?: import('../../shared/context-types.js').ContextFreshness; contextLocalProcessedFreshness?: import('../../shared/context-types.js').ContextFreshness; contextRetryExhausted?: boolean; contextSharedPolicyOverride?: import('../../shared/context-types.js').SharedScopePolicyOverride; transportConfig?: Record | null; transportPendingMessages?: string[]; transportPendingMessageEntries?: TransportPendingMessageEntry[]; queueEpoch?: string; queueAuthorityId?: string; failedMessageEntries?: TransportPendingMessageEntry[]; pendingMessageVersion?: number; transportPendingMessageVersion?: number }> } diff --git a/web/test/daemon-upgrade-blocked.test.ts b/web/test/daemon-upgrade-blocked.test.ts index fca8f7322..e044c8061 100644 --- a/web/test/daemon-upgrade-blocked.test.ts +++ b/web/test/daemon-upgrade-blocked.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from 'vitest'; +import { DAEMON_UPGRADE_BLOCK_REASON } from '../../shared/daemon-upgrade.js'; import { DAEMON_UPGRADE_BLOCKED_TOAST_THROTTLE_MS, daemonUpgradeBlockedToastKey, @@ -15,6 +16,7 @@ describe('daemon upgrade blocked toast', () => { expect(daemonUpgradeBlockedToastKey('session_busy')).toBe('toast.upgrade_blocked_session_busy'); expect(daemonUpgradeBlockedToastKey('cooldown_active')).toBe('toast.upgrade_blocked_cooldown_active'); expect(daemonUpgradeBlockedToastKey('toolchain_unavailable')).toBe('toast.upgrade_blocked_toolchain_unavailable'); + expect(daemonUpgradeBlockedToastKey(DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED)).toBe('toast.upgrade_blocked_install_failed'); expect(daemonUpgradeBlockedToastKey('future_reason')).toBe('toast.upgrade_blocked_unknown'); }); diff --git a/web/test/daemon-upgrade-status.test.ts b/web/test/daemon-upgrade-status.test.ts index 9a0c1ab76..2af226696 100644 --- a/web/test/daemon-upgrade-status.test.ts +++ b/web/test/daemon-upgrade-status.test.ts @@ -1,4 +1,6 @@ import { describe, it, expect, vi } from 'vitest'; +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; import { DAEMON_MSG } from '@shared/daemon-events.js'; import { MSG_DAEMON_ONLINE, MSG_DAEMON_OFFLINE } from '@shared/ack-protocol.js'; import { @@ -21,15 +23,15 @@ describe('nextDaemonUpgradingState', () => { expect(nextDaemonUpgradingState(DAEMON_MSG.UPGRADING, null as unknown)).toEqual({ targetVersion: '' }); }); - it('clears the badge when the daemon comes back (online / reconnected)', () => { + it('clears the badge when the daemon comes back or reports the upgrade blocked', () => { expect(nextDaemonUpgradingState(MSG_DAEMON_ONLINE)).toBeNull(); expect(nextDaemonUpgradingState(DAEMON_MSG.RECONNECTED)).toBeNull(); + expect(nextDaemonUpgradingState(DAEMON_MSG.UPGRADE_BLOCKED)).toBeNull(); }); it('leaves the badge untouched (undefined) for unrelated messages', () => { // undefined is the "keep current state" sentinel — distinct from null (clear). expect(nextDaemonUpgradingState(DAEMON_MSG.DISCONNECTED)).toBeUndefined(); - expect(nextDaemonUpgradingState(DAEMON_MSG.UPGRADE_BLOCKED)).toBeUndefined(); expect(nextDaemonUpgradingState(MSG_DAEMON_OFFLINE)).toBeUndefined(); expect(nextDaemonUpgradingState('daemon.stats')).toBeUndefined(); expect(nextDaemonUpgradingState('session_list')).toBeUndefined(); @@ -40,6 +42,18 @@ describe('nextDaemonUpgradingState', () => { // stays up until the new version reconnects. So DISCONNECTED is a no-op. expect(nextDaemonUpgradingState(DAEMON_MSG.DISCONNECTED)).toBeUndefined(); }); + + it('applies the badge reducer before toast throttling can skip blocker presentation', () => { + const appSource = readFileSync(resolve(process.cwd(), 'src/app.tsx'), 'utf8'); + const reducerIndex = appSource.indexOf('const upgradingNext = nextDaemonUpgradingState('); + const blockerToastIndex = appSource.indexOf('if (msg.type === DAEMON_MSG.UPGRADE_BLOCKED)'); + + expect(reducerIndex).toBeGreaterThan(-1); + expect(blockerToastIndex).toBeGreaterThan(-1); + expect(reducerIndex).toBeLessThan(blockerToastIndex); + expect(appSource.slice(blockerToastIndex, blockerToastIndex + 1200)) + .not.toContain('if (!shouldShowDaemonUpgradeBlockedToast'); + }); }); describe('daemonUpgradingLabel', () => { From ca293342963b2cbdbec77a9a853d74da88bd4304 Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sun, 26 Jul 2026 15:54:23 +0800 Subject: [PATCH 35/40] Fix Web upgrade test root invocation --- web/test/daemon-upgrade-status.test.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/web/test/daemon-upgrade-status.test.ts b/web/test/daemon-upgrade-status.test.ts index 2af226696..8b2d42e8e 100644 --- a/web/test/daemon-upgrade-status.test.ts +++ b/web/test/daemon-upgrade-status.test.ts @@ -1,8 +1,7 @@ import { describe, it, expect, vi } from 'vitest'; -import { readFileSync } from 'node:fs'; -import { resolve } from 'node:path'; import { DAEMON_MSG } from '@shared/daemon-events.js'; import { MSG_DAEMON_ONLINE, MSG_DAEMON_OFFLINE } from '@shared/ack-protocol.js'; +import appSource from '../src/app.tsx?raw'; import { nextDaemonUpgradingState, daemonUpgradingLabel, @@ -44,7 +43,6 @@ describe('nextDaemonUpgradingState', () => { }); it('applies the badge reducer before toast throttling can skip blocker presentation', () => { - const appSource = readFileSync(resolve(process.cwd(), 'src/app.tsx'), 'utf8'); const reducerIndex = appSource.indexOf('const upgradingNext = nextDaemonUpgradingState('); const blockerToastIndex = appSource.indexOf('if (msg.type === DAEMON_MSG.UPGRADE_BLOCKED)'); From edbe525204f8748ecc135c9e420555de45400960 Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sun, 26 Jul 2026 16:29:07 +0800 Subject: [PATCH 36/40] Stop discarding memory handles that have no explicit owner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reported as ⚠️ "记忆句柄未保存". The handles were in fact written: 302 rows sat in memory_short_refs. 290 of them (96%) were unusable, and the store says why — namespace_key `["personal","","","",""]` for every injected handle against `["personal","daemon-local","","",""]` for the ones that worked. One field apart. Probed live before touching anything: four refs injected into a session all redeemed to `sources: []` while search-provided refs from the same session redeemed fine. Recall items carry `scope` and `projectId` but never `userId`, so registration derived an owner-less namespace while the MCP server resolves personal / user_private memory as `daemon-local`, and resolveMemoryShortRef refuses a cross-namespace match on purpose. It was worse than unresolvable. `personal` declares `requiredIdentityFields: ['user_id', 'project_id']`, so those rows FAILED validation in decodeNamespace and the warm loader discarded them — every daemon start destroyed them rather than merely missing them. My first two attempts at this got that wrong in both directions, so both halves are now pinned separately: 1. Registration makes the owner explicit, so newly minted rows are policy-valid. 2. The warm loader backfills the owner on already-stored owner-less rows instead of dropping them, and accepts their pre-backfill namespace_key shape — which is what rescues the existing 290 instead of only fixing future ones. Root cause of the root cause: `daemon-local` was copy-pasted into four modules while shared/memory-namespace.ts already exported LEGACY_DAEMON_LOCAL_USER_ID. Because nobody imported the shared one, the rule that fills the owner in existed in exactly one of the two places that needed it. That is the duplication CLAUDE.md forbids, so the helper now lives beside the sentinel and both sides derive from it. Deliberately NOT done: dropping userId from the local identity instead. It is tempting — this store is per-device and a device has one owner, so the field carries no information locally (its only real values here were `''` 254× and the sentinel 43×). But `namespaceKey` is also what legacy NUL-separated rows are matched by, and rewriting it broke the legacy-key migration outright; and the scope policy requires an owner for owner-private memory regardless. Making the single owner explicit satisfies both. Boundary kept: SHARED scopes still isolate by owner, because sharing does not transfer ownership. The sentinel stands for "this device's owner", never for "anybody" — a test asserts a real user id and the sentinel stay distinct. Counterfactuals, each isolating one half: revert registration and the two redemption tests fail (`expected undefined to be 'proj-id-1'`); revert the loader rescue and the legacy row loads as `expected +0 to be 1` — the discard, reproduced. Co-Authored-By: Claude Opus 5 --- shared/memory-namespace.ts | 29 +++ src/agent/providers/getDefaultMcpServers.ts | 21 +- src/context/memory-recall-refs.ts | 25 ++- src/context/memory-short-ref.ts | 21 +- .../memory-recall-refs-namespace.test.ts | 210 ++++++++++++++++++ 5 files changed, 294 insertions(+), 12 deletions(-) create mode 100644 test/context/memory-recall-refs-namespace.test.ts diff --git a/shared/memory-namespace.ts b/shared/memory-namespace.ts index b8dd30892..5e98b6593 100644 --- a/shared/memory-namespace.ts +++ b/shared/memory-namespace.ts @@ -14,6 +14,35 @@ import type { ContextNamespace as LegacyContextNamespace } from './context-types */ export const LEGACY_DAEMON_LOCAL_USER_ID = 'daemon-local'; +/** + * Fill in the implicit daemon-local owner so a memory handle is REGISTERED under + * the same namespace it will be RESOLVED under. + * + * A handle (`proj:…` / `obs:…`) only redeems when those two namespaces are + * byte-identical — the resolver compares the whole namespace key and refuses a + * cross-namespace match deliberately. This rule previously existed in exactly one + * of the two places that need it: the MCP server was configured with `userId` + * filled in for personal/user_private scopes, while handle registration derived + * its namespace straight from recall-item fields, and recall items carry no + * `userId`. Every handle injected into an agent's context was therefore stored + * under `userId: ''` and could never match a resolver asking for + * `userId: 'daemon-local'`. The row persisted correctly and still resolved to + * nothing — the failure surfaced as "handle not saved" while it sat in the table. + * + * Both sides derive it from here now, so they cannot drift apart again. + * + * Scopes that are not user-owned are returned untouched: inventing an owner for + * them would mint a namespace no resolver ever asks for, which is the identical + * dead-handle bug pointing the other way. + */ +export function normalizeDaemonLocalMemoryNamespace(namespace: T): T { + if (namespace.userId?.trim()) return namespace; + if (namespace.scope === 'personal' || namespace.scope === 'user_private') { + return { ...namespace, userId: LEGACY_DAEMON_LOCAL_USER_ID }; + } + return namespace; +} + export type MemoryNamespaceVisibility = 'owner_private' | 'shared_authorized'; export interface MemoryNamespaceInput { diff --git a/src/agent/providers/getDefaultMcpServers.ts b/src/agent/providers/getDefaultMcpServers.ts index 08ccf0d76..175605a60 100644 --- a/src/agent/providers/getDefaultMcpServers.ts +++ b/src/agent/providers/getDefaultMcpServers.ts @@ -10,10 +10,17 @@ import { IMCODES_DAEMON_USER_ID_ENV, } from '../../../shared/memory-mcp-env.js'; import { IMCODES_MEMORY_MCP_SERVER_NAME } from '../../../shared/memory-mcp-server-name.js'; +import { + LEGACY_DAEMON_LOCAL_USER_ID, + normalizeDaemonLocalMemoryNamespace, +} from '../../../shared/memory-namespace.js'; export const IMCODES_MEMORY_MCP_COMMAND = 'imcodes'; export const IMCODES_MEMORY_MCP_ARGS = ['memory', 'mcp'] as const; -const DAEMON_LOCAL_MEMORY_USER_ID = 'daemon-local'; +// Was a local copy of the sentinel that shared/memory-namespace.ts already +// exports. Four such copies existed, which is how the two halves of the +// register/resolve invariant came to disagree in the first place. +const DAEMON_LOCAL_MEMORY_USER_ID = LEGACY_DAEMON_LOCAL_USER_ID; export interface DefaultMcpServerConfig { type: 'stdio'; @@ -42,14 +49,16 @@ function projectNameFromSessionName(sessionName: string | undefined): string | u return rest.slice(0, idx) || undefined; } +/** + * The namespace this MCP server RESOLVES memory handles under. Shares one helper + * with handle registration (see normalizeDaemonLocalMemoryNamespace): when only + * this side filled in the daemon-local owner, every injected handle was registered + * under a different namespace and redeemed to nothing. + */ function namespaceForMcp(config: SessionConfig): SessionConfig['contextNamespace'] { const namespace = config.contextNamespace ?? undefined; if (!namespace) return undefined; - if (namespace.userId?.trim()) return namespace; - if (namespace.scope === 'personal' || namespace.scope === 'user_private') { - return { ...namespace, userId: DAEMON_LOCAL_MEMORY_USER_ID }; - } - return namespace; + return normalizeDaemonLocalMemoryNamespace(namespace); } function buildIdentityEnv(config: SessionConfig): Record { diff --git a/src/context/memory-recall-refs.ts b/src/context/memory-recall-refs.ts index 5a8c588fd..e00405996 100644 --- a/src/context/memory-recall-refs.ts +++ b/src/context/memory-recall-refs.ts @@ -1,4 +1,5 @@ import type { ContextNamespace, ContextScope } from '../../shared/context-types.js'; +import { normalizeDaemonLocalMemoryNamespace } from '../../shared/memory-namespace.js'; import { registerMemoryShortRefs, type MemoryShortRefEntry, type MemoryShortRefKind } from './memory-short-ref.js'; /** @@ -33,17 +34,33 @@ function shortRefKind(type: MemoryShortRefSource['type']): MemoryShortRefKind | return 'projection'; } -/** The namespace the record is STORED in, so the cached entry lines up with what - * the record fetch authorizes against. */ +/** + * The namespace this handle must be registered under. + * + * Recall items carry `scope` and `projectId` but never `userId`, so deriving the + * namespace from item fields alone stored every injected handle with an empty + * owner. Two things then went wrong at once: the MCP resolver asks with + * `userId: 'daemon-local'` and `resolveMemoryShortRef` refuses a cross-namespace + * match, so the handle redeemed to zero sources; and `personal` declares + * `requiredIdentityFields: ['user_id', 'project_id']`, so the warm loader + * DISCARDED the row outright on the next daemon start. 290 of 302 stored handles + * (96%) were in that state — the failure the user saw as "记忆句柄未保存" while the + * rows sat in the table. + * + * The owner is not optional for owner-private memory; on a single-user device it is + * simply implicit. Making it explicit here — with the same helper the MCP server + * config uses — is what keeps registration, resolution and the scope policy in + * agreement instead of two of the three. + */ function namespaceForItem(item: MemoryShortRefSource): ContextNamespace | undefined { if (!item.scope) return undefined; - return { + return normalizeDaemonLocalMemoryNamespace({ scope: item.scope as ContextScope, ...(item.projectId ? { projectId: item.projectId } : {}), ...(item.userId ? { userId: item.userId } : {}), ...(item.workspaceId ? { workspaceId: item.workspaceId } : {}), ...(item.enterpriseId ? { enterpriseId: item.enterpriseId } : {}), - }; + }); } /** diff --git a/src/context/memory-short-ref.ts b/src/context/memory-short-ref.ts index b04758328..76ea18506 100644 --- a/src/context/memory-short-ref.ts +++ b/src/context/memory-short-ref.ts @@ -1,5 +1,6 @@ import type { ContextNamespace } from '../../shared/context-types.js'; import { isMemoryScope, validateMemoryScopeIdentity } from '../../shared/memory-scope.js'; +import { normalizeDaemonLocalMemoryNamespace } from '../../shared/memory-namespace.js'; import { MEMORY_SHORT_REF_HEALTH_ERROR_MAX_CHARS, type MemoryShortRefHealth } from '../../shared/memory-short-ref-health.js'; import { createHash } from 'node:crypto'; import { encodeBase32 } from '../util/base32.js'; @@ -215,7 +216,15 @@ function decodeNamespace(raw: unknown): { ok: true; namespace: ContextNamespace // A scope also dictates which identity fields must be present and which are // forbidden. `{ scope: 'personal' }` with no userId/projectId is structurally // fine but names no actual namespace, so it would resolve for nobody. - const namespace = record as unknown as ContextNamespace; + // + // Owner-private rows written before registration filled the owner in have no + // `userId`, and `personal` requires one — so they failed validation and were + // DISCARDED at every daemon start. That silently destroyed 290 of 302 stored + // handles on a real machine. This store is per-device and a device has exactly + // one owner, so an owner-less owner-private row is not ambiguous: it belongs to + // this daemon. Backfill the sentinel and keep the row instead of dropping the + // user's memory handles on a technicality. + const namespace = normalizeDaemonLocalMemoryNamespace(record as unknown as ContextNamespace); const identity = { user_id: namespace.userId, project_id: namespace.projectId, @@ -445,9 +454,17 @@ export async function loadMemoryShortRefsFromStore(): Promise { const storedNamespaceKey = typeof row.namespaceKey === 'string' ? row.namespaceKey : ''; const legacyNamespaceKey = namespaceKey(namespace); const legacyTruncatedKey = namespace ? namespace.scope : ''; + // Rows written before the owner was made explicit recorded their key WITHOUT + // it, so the keys recomputed from the backfilled namespace no longer match + // and the row would be rejected here as corrupt — turning the rescue in + // decodeNamespace into a no-op. Accept the pre-backfill shape too; it is the + // same identity, one field less spelled out. + const ownerlessNamespace = namespace ? { ...namespace, userId: '' } : undefined; if (storedNamespaceKey !== namespaceStorageKey(namespace) && storedNamespaceKey !== legacyNamespaceKey - && storedNamespaceKey !== legacyTruncatedKey) { + && storedNamespaceKey !== legacyTruncatedKey + && storedNamespaceKey !== namespaceStorageKey(ownerlessNamespace) + && storedNamespaceKey !== namespaceKey(ownerlessNamespace)) { discarded += 1; continue; } diff --git a/test/context/memory-recall-refs-namespace.test.ts b/test/context/memory-recall-refs-namespace.test.ts new file mode 100644 index 000000000..3c906ba0f --- /dev/null +++ b/test/context/memory-recall-refs-namespace.test.ts @@ -0,0 +1,210 @@ +/** + * A handle injected into an agent's context must be redeemable by that agent. + * + * Live failure this pins: the ⚠️ "记忆句柄未保存" warning with 302 handle rows + * sitting in the database. The handles WERE persisted — under the wrong + * namespace. Registration derived it from recall-item fields, and recall items + * carry `scope` + `projectId` but no `userId`, so injected handles landed under + * `userId: ''`. The MCP server resolves personal / user_private memory as + * `userId: 'daemon-local'`, and `resolveMemoryShortRef` refuses a cross-namespace + * match on purpose, so `get_memory_sources` answered `sources: []` for every + * injected ref while the very same session's search-provided refs redeemed fine. + * + * Verified against the real store before the fix: the four injected refs I probed + * were all present with namespace_key `["personal","","","",""]`, while + * the working search-path refs read `["personal","daemon-local","","",""]` + * — one field apart. 290 of 302 rows (96%) carried the empty spelling, so nearly + * every handle ever minted on that machine was unreachable. + * + * Worse than unresolvable: `personal` declares + * `requiredIdentityFields: ['user_id', 'project_id']`, so those rows FAILED + * validation and the warm loader discarded them on every daemon start. The handles + * were being destroyed, not just missed. + * + * The owner is therefore not optional for owner-private memory — on a single-user + * device it is merely implicit. So the fix has two halves, and either one alone + * leaves the bug half-alive: + * 1. registration makes the owner explicit, so new rows are policy-valid; + * 2. the warm loader backfills the owner on already-stored owner-less rows + * instead of dropping them, which is what rescues the existing 290. + * + * Sharing does not transfer ownership, so SHARED scopes keep a real owner and stay + * isolated — the sentinel stands for "this device's owner", never for "anybody". + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +const { runMock } = vi.hoisted(() => ({ runMock: vi.fn() })); + +vi.mock('../../src/store/context-store-worker-client.js', () => ({ + getContextStoreClient: () => ({ run: runMock }), +})); + +import { attachMemoryShortRefs } from '../../src/context/memory-recall-refs.js'; +import { + loadMemoryShortRefsFromStore, + resetMemoryShortRefsForTests, + resolveMemoryShortRef, +} from '../../src/context/memory-short-ref.js'; +import { LEGACY_DAEMON_LOCAL_USER_ID } from '../../shared/memory-namespace.js'; +import type { ContextNamespace } from '../../shared/context-types.js'; + +const PROJECT = 'github-im4codes/im4codes/imcodes'; + +/** Exactly what getDefaultMcpServers hands the MCP server for a personal scope. */ +const RESOLVER_NAMESPACE: ContextNamespace = { + scope: 'personal', + userId: LEGACY_DAEMON_LOCAL_USER_ID, + projectId: PROJECT, +}; + +describe('injected memory handles resolve for the agent they were injected into', () => { + let dir: string; + let priorPath: string | undefined; + let priorLegacy: string | undefined; + + beforeEach(() => { + priorPath = process.env.IMCODES_MEMORY_SHORT_REF_PATH; + priorLegacy = process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH; + dir = mkdtempSync(join(tmpdir(), 'imc-recall-refs-')); + // Keep the store path unset so persistence goes through the mocked client, + // and point the legacy file at a temp dir so the developer's real cache is + // never read. + delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; + process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = join(dir, 'legacy.json'); + runMock.mockReset(); + runMock.mockResolvedValue(undefined); + resetMemoryShortRefsForTests(); + }); + + afterEach(() => { + if (priorPath === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_PATH = priorPath; + if (priorLegacy === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = priorLegacy; + rmSync(dir, { recursive: true, force: true }); + }); + + it('redeems a personal projection handle even though the recall item carries no userId', () => { + // The injected shape, verbatim: scope + projectId, no userId. + const [item] = attachMemoryShortRefs([ + { id: 'proj-id-1', type: 'processed', scope: 'personal', projectId: PROJECT }, + ]); + + expect(item.ref).toBeTruthy(); + const resolved = resolveMemoryShortRef(item.ref!, RESOLVER_NAMESPACE); + expect(resolved?.id).toBe('proj-id-1'); + expect(resolved?.kind).toBe('projection'); + }); + + it('redeems an observation handle from the same injected shape', () => { + const [item] = attachMemoryShortRefs([ + { id: 'obs-id-1', type: 'observation', scope: 'user_private' }, + ]); + + expect(item.ref).toBeTruthy(); + const resolved = resolveMemoryShortRef(item.ref!, { + scope: 'user_private', + userId: LEGACY_DAEMON_LOCAL_USER_ID, + }); + expect(resolved?.id).toBe('obs-id-1'); + expect(resolved?.kind).toBe('observation'); + }); + + it('still redeems after a daemon restart, via the row that actually reached the store', async () => { + // The in-memory index would mask a namespace mismatch for the life of the + // process, so assert the round trip: persist, drop the index, warm-load from + // exactly what was written, resolve again. + const [item] = attachMemoryShortRefs([ + { id: 'proj-id-2', type: 'processed', scope: 'personal', projectId: PROJECT }, + ]); + const upsert = runMock.mock.calls.find(([name]) => name === 'upsertMemoryShortRefs'); + expect(upsert).toBeTruthy(); + const written = (upsert![1] as [Array>])[0]; + expect(written).toHaveLength(1); + + resetMemoryShortRefsForTests(); + runMock.mockReset(); + // `listMemoryShortRefs` in the store already maps snake_case columns to + // camelCase, so the warm loader receives exactly the rows that were written. + // (Re-converting them to snake_case here made this test fail against correct + // product code — the mock, not the loader, was wrong.) + runMock.mockImplementation(async (name: string) => ( + name === 'listMemoryShortRefs' ? written : undefined + )); + await loadMemoryShortRefsFromStore(); + + expect(resolveMemoryShortRef(item.ref!, RESOLVER_NAMESPACE)?.id).toBe('proj-id-2'); + }); + + it('rescues an already-stored owner-less row instead of discarding it at startup', async () => { + // The 290 rows written before registration filled the owner in. `personal` + // declares requiredIdentityFields ['user_id','project_id'], so validation + // rejected them and the warm loader dropped every one on each daemon start — + // the handles were not merely unresolvable, they were destroyed. A device has + // one owner, so an owner-less owner-private row is unambiguous. + resetMemoryShortRefsForTests(); + runMock.mockReset(); + runMock.mockImplementation(async (name: string) => ( + name === 'listMemoryShortRefs' + ? [{ + ref: 'proj:legacyownerless', + kind: 'projection', + id: 'legacy-id-1', + namespaceKey: JSON.stringify(['personal', '', PROJECT, '', '']), + namespaceJson: JSON.stringify({ scope: 'personal', projectId: PROJECT }), + lastSeenAt: Date.now(), + }] + : undefined + )); + + const loaded = await loadMemoryShortRefsFromStore(); + expect(loaded).toBe(1); + expect(resolveMemoryShortRef('proj:legacyownerless', RESOLVER_NAMESPACE)?.id).toBe('legacy-id-1'); + }); + + it('keeps SHARED scopes isolated by owner — sharing does not transfer ownership', () => { + // The narrow boundary: the owner is only implied for owner-private scopes. A + // shared record can genuinely belong to somebody else, so collapsing owners + // there would let one owner's handle resolve as another's. + const [item] = attachMemoryShortRefs([ + { id: 'proj-id-3', type: 'processed', scope: 'project_shared', projectId: PROJECT, userId: 'user-42' }, + ]); + + expect(item.ref).toBeTruthy(); + expect(resolveMemoryShortRef(item.ref!, { + scope: 'project_shared', projectId: PROJECT, userId: 'user-42', + })?.id).toBe('proj-id-3'); + // A different owner must NOT redeem it. + expect(resolveMemoryShortRef(item.ref!, { + scope: 'project_shared', projectId: PROJECT, userId: 'someone-else', + })).toBeUndefined(); + }); + + it('keeps an explicit real owner distinct from the daemon-local one', () => { + // Filling the owner in must not flatten real user ids together: the sentinel + // stands in for "this device's owner", not for "anybody". + const [item] = attachMemoryShortRefs([ + { id: 'proj-id-4', type: 'processed', scope: 'personal', projectId: PROJECT, userId: 'user-42' }, + ]); + + expect(resolveMemoryShortRef(item.ref!, { + scope: 'personal', projectId: PROJECT, userId: 'user-42', + })?.id).toBe('proj-id-4'); + expect(resolveMemoryShortRef(item.ref!, RESOLVER_NAMESPACE)).toBeUndefined(); + }); + + it('still isolates the dimensions that DO carry information', () => { + // Dropping the owner must not turn into "namespace no longer matters". + const [item] = attachMemoryShortRefs([ + { id: 'proj-id-5', type: 'processed', scope: 'personal', projectId: PROJECT }, + ]); + + expect(resolveMemoryShortRef(item.ref!, { scope: 'personal', projectId: 'other/project' })) + .toBeUndefined(); + expect(resolveMemoryShortRef(item.ref!, { scope: 'project_shared', projectId: PROJECT })) + .toBeUndefined(); + }); +}); From 96cd604fdfcd454c166ab3485e411e0ff8b9c7b5 Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sun, 26 Jul 2026 16:39:50 +0800 Subject: [PATCH 37/40] Narrow the owner-less handle-key rescue to the rows that need it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to edbe52520, from the independent audit of it. That commit taught the warm loader to accept a row whose persisted namespace_key was written before the owner became explicit — otherwise the key-consistency check rejects the very rows the owner backfill exists to rescue. But it offered that alternative for EVERY decoded namespace, so it also excused a row whose namespace_json names an explicit owner (or a shared scope) while its key is the owner-less tuple. That is a genuine key/JSON disagreement, which is exactly the corruption the check is there to catch — the guard had become broader than its own comment claimed. Those rows could never cross-resolve (they are filed under the validated JSON namespace, and every namespaced lookup compares the full tuple), so this was not an isolation hole. It was an integrity check quietly not doing its job. `decodeNamespace` now reports whether it actually backfilled an owner, and only then is the pre-backfill key shape accepted. Pinned by a test that feeds a row with an owner-less key but an explicit `user-42` in the JSON: it must be discarded. Counterfactual: with the narrowing reverted that row loads (`expected 1 to be +0`). Also finishes the de-duplication that caused the original bug. `daemon-local` was copy-pasted into four modules while shared/memory-namespace.ts already exported LEGACY_DAEMON_LOCAL_USER_ID; edbe52520 only fixed one of them. memory-read-tools.ts was the starkest case — it imported the shared constant and then shadowed it with its own literal of the same value. Both now alias the shared sentinel, so no memory module defines that string again. command-handler.ts keeps its own DAEMON_LOCAL_PREFERENCE_USER_ID: same text, different domain (preferences, not memory), and merging domains that may legitimately diverge would be a worse bug than the duplication. Co-Authored-By: Claude Opus 5 --- src/context/memory-read-tools.ts | 6 ++++- src/context/memory-short-ref.ts | 26 +++++++++++++----- src/daemon/memory-mcp-caller.ts | 4 ++- .../memory-recall-refs-namespace.test.ts | 27 +++++++++++++++++++ 4 files changed, 55 insertions(+), 8 deletions(-) diff --git a/src/context/memory-read-tools.ts b/src/context/memory-read-tools.ts index 8bb5f1421..9ddd81274 100644 --- a/src/context/memory-read-tools.ts +++ b/src/context/memory-read-tools.ts @@ -16,7 +16,11 @@ import { getContextStoreClient } from '../store/context-store-worker-client.js'; const MEMORY_TOOL_CALLER_BRAND: unique symbol = Symbol('MemoryToolCaller'); const INTERNAL_MEMORY_TOOL_CALLER_BRAND: unique symbol = Symbol('InternalMemoryToolCaller'); -const DAEMON_LOCAL_MEMORY_USER_ID = 'daemon-local'; +// Alias of the shared sentinel, not a second definition: this module already +// imports LEGACY_DAEMON_LOCAL_USER_ID. Two literals of the same value in one +// codebase are how the register/resolve halves of the handle namespace drifted +// apart in the first place. +const DAEMON_LOCAL_MEMORY_USER_ID = LEGACY_DAEMON_LOCAL_USER_ID; /** * Caller identity passed to public read-tool handlers. diff --git a/src/context/memory-short-ref.ts b/src/context/memory-short-ref.ts index 76ea18506..c71752ca6 100644 --- a/src/context/memory-short-ref.ts +++ b/src/context/memory-short-ref.ts @@ -200,7 +200,9 @@ const NAMESPACE_IDENTITY_FIELDS = [ * entry that looks healthy but can never resolve for a namespaced caller, and * counts it as loaded, so the handle disappears with no signal at all. */ -function decodeNamespace(raw: unknown): { ok: true; namespace: ContextNamespace | undefined } | { ok: false } { +function decodeNamespace( + raw: unknown, +): { ok: true; namespace: ContextNamespace | undefined; ownerBackfilled?: boolean } | { ok: false } { // Only a value that is not there at all means "no namespace". An explicit // null — including a column holding the JSON text `null` — is a value that // failed to describe a namespace, and degrading it to namespace-less loads a @@ -224,7 +226,9 @@ function decodeNamespace(raw: unknown): { ok: true; namespace: ContextNamespace // one owner, so an owner-less owner-private row is not ambiguous: it belongs to // this daemon. Backfill the sentinel and keep the row instead of dropping the // user's memory handles on a technicality. - const namespace = normalizeDaemonLocalMemoryNamespace(record as unknown as ContextNamespace); + const rawNamespace = record as unknown as ContextNamespace; + const namespace = normalizeDaemonLocalMemoryNamespace(rawNamespace); + const ownerBackfilled = namespace !== rawNamespace; const identity = { user_id: namespace.userId, project_id: namespace.projectId, @@ -233,7 +237,7 @@ function decodeNamespace(raw: unknown): { ok: true; namespace: ContextNamespace tenant_id: namespace.localTenant, }; if (!validateMemoryScopeIdentity(record.scope, identity).ok) return { ok: false }; - return { ok: true, namespace }; + return { ok: true, namespace, ...(ownerBackfilled ? { ownerBackfilled } : {}) }; } function isContextNamespace(value: unknown): value is ContextNamespace { @@ -459,12 +463,22 @@ export async function loadMemoryShortRefsFromStore(): Promise { // and the row would be rejected here as corrupt — turning the rescue in // decodeNamespace into a no-op. Accept the pre-backfill shape too; it is the // same identity, one field less spelled out. - const ownerlessNamespace = namespace ? { ...namespace, userId: '' } : undefined; + // + // ONLY for rows whose owner was actually backfilled. Offering this + // alternative unconditionally also excused a row carrying an explicit owner + // (or a shared scope) whose key was the owner-less tuple — a genuine + // key/JSON disagreement that this check exists to catch. Those rows could + // never cross-resolve (they are filed under the validated JSON namespace), + // but an integrity check should not be broader than its own comment. + const ownerlessNamespace = decoded.ownerBackfilled && namespace + ? { ...namespace, userId: '' } + : undefined; if (storedNamespaceKey !== namespaceStorageKey(namespace) && storedNamespaceKey !== legacyNamespaceKey && storedNamespaceKey !== legacyTruncatedKey - && storedNamespaceKey !== namespaceStorageKey(ownerlessNamespace) - && storedNamespaceKey !== namespaceKey(ownerlessNamespace)) { + && !(ownerlessNamespace + && (storedNamespaceKey === namespaceStorageKey(ownerlessNamespace) + || storedNamespaceKey === namespaceKey(ownerlessNamespace)))) { discarded += 1; continue; } diff --git a/src/daemon/memory-mcp-caller.ts b/src/daemon/memory-mcp-caller.ts index 4757f4b95..c71116fb3 100644 --- a/src/daemon/memory-mcp-caller.ts +++ b/src/daemon/memory-mcp-caller.ts @@ -1,4 +1,5 @@ import type { ContextNamespace } from '../../shared/context-types.js'; +import { LEGACY_DAEMON_LOCAL_USER_ID } from '../../shared/memory-namespace.js'; import { MEMORY_MCP_ENV_KEYS, type MemoryMcpEnvSource } from '../../shared/memory-mcp-env.js'; import { isMemoryScope, validateMemoryScopeIdentity } from '../../shared/memory-scope.js'; import { isValidImcodesSessionName } from '../../shared/session-scope.js'; @@ -23,7 +24,8 @@ export class MemoryMcpCallerEnvError extends Error { } } -const DAEMON_LOCAL_MEMORY_USER_ID = 'daemon-local'; +// Alias of the shared sentinel rather than a fourth copy of the literal. +const DAEMON_LOCAL_MEMORY_USER_ID = LEGACY_DAEMON_LOCAL_USER_ID; function optionalString(value: string | undefined): string | null { const trimmed = value?.trim(); diff --git a/test/context/memory-recall-refs-namespace.test.ts b/test/context/memory-recall-refs-namespace.test.ts index 3c906ba0f..c5166e5c2 100644 --- a/test/context/memory-recall-refs-namespace.test.ts +++ b/test/context/memory-recall-refs-namespace.test.ts @@ -165,6 +165,33 @@ describe('injected memory handles resolve for the agent they were injected into' expect(resolveMemoryShortRef('proj:legacyownerless', RESOLVER_NAMESPACE)?.id).toBe('legacy-id-1'); }); + it('does not excuse an owner-less KEY on a row that carries an explicit owner', async () => { + // The rescue must only forgive rows that actually needed it. Offering the + // owner-less key shape to every row also excused a genuine key/JSON + // disagreement — the exact corruption this integrity check exists to catch. + // (Found by the independent audit of the first version of this fix.) + resetMemoryShortRefsForTests(); + runMock.mockReset(); + runMock.mockImplementation(async (name: string) => ( + name === 'listMemoryShortRefs' + ? [{ + ref: 'proj:mismatchedowner', + kind: 'projection', + id: 'mismatch-id-1', + // Key says nobody owns it; JSON says user-42 does. Inconsistent. + namespaceKey: JSON.stringify(['personal', '', PROJECT, '', '']), + namespaceJson: JSON.stringify({ scope: 'personal', projectId: PROJECT, userId: 'user-42' }), + lastSeenAt: Date.now(), + }] + : undefined + )); + + expect(await loadMemoryShortRefsFromStore()).toBe(0); + expect(resolveMemoryShortRef('proj:mismatchedowner', { + scope: 'personal', projectId: PROJECT, userId: 'user-42', + })).toBeUndefined(); + }); + it('keeps SHARED scopes isolated by owner — sharing does not transfer ownership', () => { // The narrow boundary: the owner is only implied for owner-private scopes. A // shared record can genuinely belong to somebody else, so collapsing owners From df63fece4593ab3b0a773d6702d51c1d81827d9b Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sun, 26 Jul 2026 18:12:19 +0800 Subject: [PATCH 38/40] Carry the pre-backfill namespace instead of guessing what it was MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to 96cd604fd, from the second independent audit — which found this with a real-SQLite probe rather than a mock, and was right. The loader accepts a row whose persisted namespace_key predates the owner being made explicit, but it rebuilt that key by ASSUMING the missing owner had been the empty string. `normalizeDaemonLocalMemoryNamespace` also treats a whitespace-only owner as missing, so such a row IS backfilled — and then the reconstruction produced `''` where the row had stored `" "`, the keys disagreed, and the row was discarded as corrupt. Exactly the destruction this whole line of work exists to stop, in a shape I had not considered. `decodeNamespace` now hands back the pre-backfill namespace itself rather than a boolean, so the loader compares against the bytes the row was actually written with. That removes the assumption instead of extending it: any owner spelling the normalizer accepts as "missing" is now automatically handled, including ones nobody has thought of yet. Counterfactual: with the reconstruction restored, the new whitespace-owner test fails `expected +0 to be 1` — the row discarded, reproducing the auditor's real-SQLite finding. Also worth recording why the audit caught this and my own tests did not: mine mock the context-store client, so they only ever exercised the shapes I imagined. The auditor wrote adversarial rows into a real SQLite database and probed five shapes (owner-less JSON tuple, full NUL key, truncated scope-only key, explicit-owner mismatch, project mismatch). Three loaded, two were correctly refused, and the whitespace case fell outside all five of my mocked scenarios. Real-store coverage for this rescue remains a gap worth closing. Co-Authored-By: Claude Opus 5 --- src/context/memory-short-ref.ts | 22 +++++++++------ .../memory-recall-refs-namespace.test.ts | 27 +++++++++++++++++++ 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/src/context/memory-short-ref.ts b/src/context/memory-short-ref.ts index c71752ca6..498dd880b 100644 --- a/src/context/memory-short-ref.ts +++ b/src/context/memory-short-ref.ts @@ -202,7 +202,7 @@ const NAMESPACE_IDENTITY_FIELDS = [ */ function decodeNamespace( raw: unknown, -): { ok: true; namespace: ContextNamespace | undefined; ownerBackfilled?: boolean } | { ok: false } { +): { ok: true; namespace: ContextNamespace | undefined; preBackfillNamespace?: ContextNamespace } | { ok: false } { // Only a value that is not there at all means "no namespace". An explicit // null — including a column holding the JSON text `null` — is a value that // failed to describe a namespace, and degrading it to namespace-less loads a @@ -228,6 +228,12 @@ function decodeNamespace( // user's memory handles on a technicality. const rawNamespace = record as unknown as ContextNamespace; const namespace = normalizeDaemonLocalMemoryNamespace(rawNamespace); + // Hand back the PRE-backfill namespace rather than a boolean, so the loader can + // rebuild the key this row was actually written with instead of assuming the + // absent owner was the empty string. The helper treats a whitespace-only owner + // as missing too, and a row that stored `" "` in both its JSON and its key was + // being discarded because the reconstruction guessed `''`. Carrying the real + // value removes the guess. const ownerBackfilled = namespace !== rawNamespace; const identity = { user_id: namespace.userId, @@ -237,7 +243,7 @@ function decodeNamespace( tenant_id: namespace.localTenant, }; if (!validateMemoryScopeIdentity(record.scope, identity).ok) return { ok: false }; - return { ok: true, namespace, ...(ownerBackfilled ? { ownerBackfilled } : {}) }; + return { ok: true, namespace, ...(ownerBackfilled ? { preBackfillNamespace: rawNamespace } : {}) }; } function isContextNamespace(value: unknown): value is ContextNamespace { @@ -470,15 +476,15 @@ export async function loadMemoryShortRefsFromStore(): Promise { // key/JSON disagreement that this check exists to catch. Those rows could // never cross-resolve (they are filed under the validated JSON namespace), // but an integrity check should not be broader than its own comment. - const ownerlessNamespace = decoded.ownerBackfilled && namespace - ? { ...namespace, userId: '' } - : undefined; + // The exact namespace this row was written with, when the owner was + // backfilled — not a reconstruction of it. + const preBackfillNamespace = decoded.preBackfillNamespace; if (storedNamespaceKey !== namespaceStorageKey(namespace) && storedNamespaceKey !== legacyNamespaceKey && storedNamespaceKey !== legacyTruncatedKey - && !(ownerlessNamespace - && (storedNamespaceKey === namespaceStorageKey(ownerlessNamespace) - || storedNamespaceKey === namespaceKey(ownerlessNamespace)))) { + && !(preBackfillNamespace + && (storedNamespaceKey === namespaceStorageKey(preBackfillNamespace) + || storedNamespaceKey === namespaceKey(preBackfillNamespace)))) { discarded += 1; continue; } diff --git a/test/context/memory-recall-refs-namespace.test.ts b/test/context/memory-recall-refs-namespace.test.ts index c5166e5c2..30a92402b 100644 --- a/test/context/memory-recall-refs-namespace.test.ts +++ b/test/context/memory-recall-refs-namespace.test.ts @@ -165,6 +165,33 @@ describe('injected memory handles resolve for the agent they were injected into' expect(resolveMemoryShortRef('proj:legacyownerless', RESOLVER_NAMESPACE)?.id).toBe('legacy-id-1'); }); + it('rescues a row whose owner was whitespace, not just absent or empty', async () => { + // Found by the second independent audit, with a real-SQLite probe: the + // normalizer treats a whitespace-only owner as missing, so such a row IS + // backfilled — but the loader used to rebuild the pre-backfill key by ASSUMING + // the owner had been `''`, so a row that stored `" "` in both its JSON and its + // key was discarded anyway. The pre-backfill namespace is now carried through + // verbatim instead of reconstructed, which removes the assumption. + resetMemoryShortRefsForTests(); + runMock.mockReset(); + runMock.mockImplementation(async (name: string) => ( + name === 'listMemoryShortRefs' + ? [{ + ref: 'proj:whitespaceowner', + kind: 'projection', + id: 'whitespace-id-1', + namespaceKey: JSON.stringify(['personal', ' ', PROJECT, '', '']), + namespaceJson: JSON.stringify({ scope: 'personal', projectId: PROJECT, userId: ' ' }), + lastSeenAt: Date.now(), + }] + : undefined + )); + + expect(await loadMemoryShortRefsFromStore()).toBe(1); + expect(resolveMemoryShortRef('proj:whitespaceowner', RESOLVER_NAMESPACE)?.id) + .toBe('whitespace-id-1'); + }); + it('does not excuse an owner-less KEY on a row that carries an explicit owner', async () => { // The rescue must only forgive rows that actually needed it. Offering the // owner-less key shape to every row also excused a genuine key/JSON From 15ba3bad74bea93a8e21d3edc2a97fb01ec1b767 Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sun, 26 Jul 2026 19:25:44 +0800 Subject: [PATCH 39/40] Hydrate memory short refs from SQLite --- shared/context-store-rpc.ts | 1 + shared/memory-short-ref-health.ts | 2 +- src/context/memory-short-ref.ts | 241 ++++++++++++++------ src/daemon/memory-mcp-tools.ts | 25 +- src/store/context-store.ts | 19 ++ test/context/memory-short-ref-store.test.ts | 109 ++++++++- test/e2e/memory-mcp-interface.test.ts | 112 +++++++++ 7 files changed, 423 insertions(+), 86 deletions(-) diff --git a/shared/context-store-rpc.ts b/shared/context-store-rpc.ts index fea033cb5..e4cbb05b9 100644 --- a/shared/context-store-rpc.ts +++ b/shared/context-store-rpc.ts @@ -41,6 +41,7 @@ export type ContextStoreRpcPriority = export const CONTEXT_STORE_L1_OPS = [ // memory short-ref handle map (ref → id), durable across daemon restarts 'listMemoryShortRefs', + 'listMemoryShortRefsByRef', 'upsertMemoryShortRefs', // reads 'getProcessedProjectionById', diff --git a/shared/memory-short-ref-health.ts b/shared/memory-short-ref-health.ts index 30749ab74..55ac6824f 100644 --- a/shared/memory-short-ref-health.ts +++ b/shared/memory-short-ref-health.ts @@ -13,7 +13,7 @@ * so it is still visible to whoever reconnects after the fact. */ export interface MemoryShortRefHealth { - /** Where the last failure happened (persist_store, persist_file, warm_load, load_file). */ + /** Where the last failure happened (persist_store, persist_file, warm_load, hydrate_ref, load_file). */ stage: string; /** Failures since the process started; keeps a single blip distinguishable from a stuck disk. */ failures: number; diff --git a/src/context/memory-short-ref.ts b/src/context/memory-short-ref.ts index 498dd880b..c10d4430c 100644 --- a/src/context/memory-short-ref.ts +++ b/src/context/memory-short-ref.ts @@ -37,6 +37,7 @@ const MEMORY_SHORT_REF_LENGTH = 13; * algorithm are dropped instead of resolving to a stale/wrong record. */ const SHORT_REF_SCHEMA_VERSION = 2; const entriesByRef = new Map(); +const persistedRefHydrations = new Map>(); let persistedLoaded = false; let shortRefHealth: MemoryShortRefHealth | undefined; @@ -341,7 +342,10 @@ function persistShortRefsToFile(): void { */ /** Rows dropped while loading are a silent form of handle loss, so count them * under a fixed-cardinality source and warn once per hour. */ -function reportDiscardedShortRefRows(source: 'warm_load' | 'legacy_file' | 'json_file', discarded: number): void { +function reportDiscardedShortRefRows( + source: 'warm_load' | 'hydrate_ref' | 'legacy_file' | 'json_file', + discarded: number, +): void { if (discarded <= 0) return; incrementCounter('mem.short_ref.discarded_row', { source }); warnOncePerHour(`mem.short_ref.discarded_row.${source}`, { discarded }); @@ -351,7 +355,11 @@ function reportDiscardedShortRefRows(source: 'warm_load' | 'legacy_file' | 'json recordShortRefFailure(`discarded_${source}`, `${discarded} unusable row(s) discarded`); } -function reportShortRefFailure(stage: 'persist_store' | 'persist_file' | 'warm_load' | 'load_file', error: unknown, extra: Record = {}): void { +function reportShortRefFailure( + stage: 'persist_store' | 'persist_file' | 'warm_load' | 'hydrate_ref' | 'load_file', + error: unknown, + extra: Record = {}, +): void { const message = error instanceof Error ? error.message : String(error); incrementCounter('mem.short_ref.persist_failure', { stage }); warnOncePerHour(`mem.short_ref.persist_failure.${stage}`, { ...extra, error: message }); @@ -416,12 +424,83 @@ function persistShortRefs(touched: ReadonlyArray<{ ref: string; entry: MemorySho }); } +function normalizePersistedShortRefRow( + row: Record, +): { ref: string; entry: MemoryShortRefEntry } | undefined { + // A row that stored a namespace but can no longer produce one must be + // dropped, not loaded namespace-less: the entry would look healthy while + // being unresolvable for every namespaced caller — silent handle loss + // wearing the shape of a successful load. + const namespaceColumnPresent = typeof row.namespaceJson === 'string' && row.namespaceJson.trim().length > 0; + const parsed = namespaceColumnPresent ? safeParseNamespace(row.namespaceJson as string) : undefined; + if (namespaceColumnPresent && parsed === undefined) return undefined; + const decoded = decodeNamespace(parsed); + if (!decoded.ok) return undefined; + const namespace = decoded.namespace; + // namespace_key records what the row was written under; a disagreement + // means the row's identity is corrupt and loading it would file the handle + // under the wrong namespace. + // + // Rows written before the key became a JSON tuple used NUL-separated + // fields. Older node:sqlite versions truncate that TEXT value at the + // first NUL when converting it to a JS string, while current versions + // return the complete string. Accept both read-back shapes so upgrading + // Node cannot discard every handle written before the key migration. + const storedNamespaceKey = typeof row.namespaceKey === 'string' ? row.namespaceKey : ''; + const legacyNamespaceKey = namespaceKey(namespace); + const legacyTruncatedKey = namespace ? namespace.scope : ''; + // Rows written before the owner was made explicit recorded their key WITHOUT + // it, so the keys recomputed from the backfilled namespace no longer match. + // Accept that shape ONLY when decodeNamespace actually backfilled the owner. + const preBackfillNamespace = decoded.preBackfillNamespace; + if (storedNamespaceKey !== namespaceStorageKey(namespace) + && storedNamespaceKey !== legacyNamespaceKey + && storedNamespaceKey !== legacyTruncatedKey + && !(preBackfillNamespace + && (storedNamespaceKey === namespaceStorageKey(preBackfillNamespace) + || storedNamespaceKey === namespaceKey(preBackfillNamespace)))) { + return undefined; + } + return normalizeEntry({ + ref: row.ref, + kind: row.kind, + id: row.id, + lastSeenAt: row.lastSeenAt, + namespace, + }); +} + +function mergePersistedShortRefRows( + rows: readonly Record[], + options: { touchLastSeenAt?: boolean } = {}, +): { loaded: number; discarded: number } { + let loaded = 0; + let discarded = 0; + const touchedAt = options.touchLastSeenAt ? Date.now() : undefined; + for (const row of rows) { + const normalized = normalizePersistedShortRefRow(row); + if (!normalized) { + discarded += 1; + continue; + } + if (touchedAt !== undefined) normalized.entry.lastSeenAt = touchedAt; + const bucket = entriesByRef.get(normalized.ref) ?? []; + if (bucket.some((entry) => entry.kind === normalized.entry.kind + && entry.id === normalized.entry.id + && sameNamespace(entry.namespace, normalized.entry.namespace))) continue; + bucket.push(normalized.entry); + entriesByRef.set(normalized.ref, bucket); + loaded += 1; + } + return { loaded, discarded }; +} + /** * Warm the in-memory index from the context store once at daemon startup. * - * Resolution stays synchronous (it is called from synchronous render paths), so - * the store is read once here instead of per lookup. Handles registered before - * this resolves are unaffected — the in-memory index is authoritative in-process. + * Synchronous render paths continue to use this warm index. Independent MCP + * processes additionally perform an indexed SQLite lookup on a local miss, + * because another process can persist a handle after this warm-load finishes. */ export async function loadMemoryShortRefsFromStore(): Promise { if (shortRefStorePath()) return 0; @@ -435,79 +514,12 @@ export async function loadMemoryShortRefsFromStore(): Promise { reportShortRefFailure('warm_load', new Error('listMemoryShortRefs returned a non-array response')); return 0; } - let loaded = 0; - let discarded = 0; - for (const row of rows) { - // A row that stored a namespace but can no longer produce one must be - // dropped, not loaded namespace-less: the entry would look healthy while - // being unresolvable for every namespaced caller — silent handle loss - // wearing the shape of a successful load. - // Absent means the column itself carried nothing. A column that IS present - // but cannot be parsed is corruption, not "no namespace" — degrading it to - // namespace-less loads an entry that looks healthy while being - // unresolvable for every namespaced caller. - const namespaceColumnPresent = typeof row.namespaceJson === 'string' && row.namespaceJson.trim().length > 0; - const parsed = namespaceColumnPresent ? safeParseNamespace(row.namespaceJson as string) : undefined; - if (namespaceColumnPresent && parsed === undefined) { discarded += 1; continue; } - const decoded = decodeNamespace(parsed); - if (!decoded.ok) { discarded += 1; continue; } - const namespace = decoded.namespace; - // namespace_key records what the row was written under; a disagreement - // means the row's identity is corrupt and loading it would file the handle - // under the wrong namespace. - // - // Rows written before the key became a JSON tuple used NUL-separated - // fields. Older node:sqlite versions truncate that TEXT value at the - // first NUL when converting it to a JS string, while current versions - // return the complete string. Accept both read-back shapes so upgrading - // Node cannot discard every handle written before the key migration. - const storedNamespaceKey = typeof row.namespaceKey === 'string' ? row.namespaceKey : ''; - const legacyNamespaceKey = namespaceKey(namespace); - const legacyTruncatedKey = namespace ? namespace.scope : ''; - // Rows written before the owner was made explicit recorded their key WITHOUT - // it, so the keys recomputed from the backfilled namespace no longer match - // and the row would be rejected here as corrupt — turning the rescue in - // decodeNamespace into a no-op. Accept the pre-backfill shape too; it is the - // same identity, one field less spelled out. - // - // ONLY for rows whose owner was actually backfilled. Offering this - // alternative unconditionally also excused a row carrying an explicit owner - // (or a shared scope) whose key was the owner-less tuple — a genuine - // key/JSON disagreement that this check exists to catch. Those rows could - // never cross-resolve (they are filed under the validated JSON namespace), - // but an integrity check should not be broader than its own comment. - // The exact namespace this row was written with, when the owner was - // backfilled — not a reconstruction of it. - const preBackfillNamespace = decoded.preBackfillNamespace; - if (storedNamespaceKey !== namespaceStorageKey(namespace) - && storedNamespaceKey !== legacyNamespaceKey - && storedNamespaceKey !== legacyTruncatedKey - && !(preBackfillNamespace - && (storedNamespaceKey === namespaceStorageKey(preBackfillNamespace) - || storedNamespaceKey === namespaceKey(preBackfillNamespace)))) { - discarded += 1; - continue; - } - const normalized = normalizeEntry({ - ref: row.ref, - kind: row.kind, - id: row.id, - lastSeenAt: row.lastSeenAt, - namespace, - }); - if (!normalized) { discarded += 1; continue; } - const bucket = entriesByRef.get(normalized.ref) ?? []; - if (bucket.some((entry) => entry.kind === normalized.entry.kind - && entry.id === normalized.entry.id - && sameNamespace(entry.namespace, normalized.entry.namespace))) continue; - bucket.push(normalized.entry); - entriesByRef.set(normalized.ref, bucket); - loaded += 1; - } + const merged = mergePersistedShortRefRows(rows); + let loaded = merged.loaded; // A well-formed array of malformed rows loads nothing and leaves every // earlier handle unresolvable — indistinguishable from an empty store // unless the discards are reported. - reportDiscardedShortRefRows('warm_load', discarded); + reportDiscardedShortRefRows('warm_load', merged.discarded); // Carry over anything still only in the retired JSON cache, and write it to // the store so the next start no longer depends on that file. const legacy = importLegacyShortRefFile(); @@ -639,6 +651,83 @@ export function resolveMemoryShortRef(ref: string, namespace?: ContextNamespace) return bucket.length === 1 ? bucket[0] : undefined; } +/** + * Hydrate one handle from SQLite after an in-process miss. + * + * The memory MCP server is a separate, long-lived process. Startup/recall can + * persist new handles from the daemon after the MCP process has populated its + * local Map, so a one-time warm-load cannot keep the two processes coherent. + * Querying by exact ref uses the memory_short_refs primary-key prefix and avoids + * rescanning the bounded warm-load window on every miss. + */ +async function hydrateMemoryShortRefFromStore(ref: string): Promise { + if (shortRefStorePath()) return 0; + const normalizedRef = normalizeRef(ref); + if (!normalizedRef) return 0; + const existing = persistedRefHydrations.get(normalizedRef); + if (existing) return existing; + const hydration = (async () => { + try { + const rows = await getContextStoreClient() + .run>>('listMemoryShortRefsByRef', [normalizedRef]); + if (!Array.isArray(rows)) { + reportShortRefFailure('hydrate_ref', new Error('listMemoryShortRefsByRef returned a non-array response'), { + ref: normalizedRef, + }); + return 0; + } + // This ref is being actively redeemed. Promote hydrated rows in the + // process-local LRU before pruning so an old persisted handle cannot be + // inserted and immediately evicted while the index is at its 10k cap. + const merged = mergePersistedShortRefRows(rows, { touchLastSeenAt: true }); + reportDiscardedShortRefRows('hydrate_ref', merged.discarded); + pruneShortRefs(); + return merged.loaded; + } catch (error) { + reportShortRefFailure('hydrate_ref', error, { ref: normalizedRef }); + return 0; + } + })(); + persistedRefHydrations.set(normalizedRef, hydration); + try { + return await hydration; + } finally { + if (persistedRefHydrations.get(normalizedRef) === hydration) { + persistedRefHydrations.delete(normalizedRef); + } + } +} + +/** + * Async resolver for MCP reads. It preserves the synchronous resolver's exact + * namespace and collision semantics, adding only an indexed SQLite retry when + * this process has no candidate in the caller namespace. + */ +export async function resolveMemoryShortRefCandidatesWithStore( + ref: string, + namespace: ContextNamespace, +): Promise { + const candidates = resolveMemoryShortRefCandidates(ref, namespace); + if (candidates.length > 0) return candidates; + await hydrateMemoryShortRefFromStore(ref); + return resolveMemoryShortRefCandidates(ref, namespace); +} + +/** + * Strict async resolver for MCP mutations. Ambiguous handles and handles from a + * different namespace remain unresolved; the store lookup never authorizes a + * cross-namespace fallback. + */ +export async function resolveMemoryShortRefWithStore( + ref: string, + namespace?: ContextNamespace, +): Promise { + const resolved = resolveMemoryShortRef(ref, namespace); + if (resolved) return resolved; + await hydrateMemoryShortRefFromStore(ref); + return resolveMemoryShortRef(ref, namespace); +} + /** * Force several records onto one handle. A 65-bit digest collision cannot be * produced by registering real ids, so the ambiguous-handle paths would @@ -651,12 +740,14 @@ export function seedMemoryShortRefCollisionForTests(ref: string, entries: readon export function resetMemoryShortRefsForTests(): void { entriesByRef.clear(); + persistedRefHydrations.clear(); shortRefHealth = undefined; persistedLoaded = true; } export function reloadMemoryShortRefsForTests(): void { entriesByRef.clear(); + persistedRefHydrations.clear(); persistedLoaded = false; ensurePersistedLoaded(); } diff --git a/src/daemon/memory-mcp-tools.ts b/src/daemon/memory-mcp-tools.ts index 1533286d9..fc92e83ad 100644 --- a/src/daemon/memory-mcp-tools.ts +++ b/src/daemon/memory-mcp-tools.ts @@ -94,7 +94,11 @@ import { getContextStoreClient } from '../store/context-store-worker-client.js'; import { listSessions as listStoredSessions, loadStore, type SessionRecord } from '../store/session-store.js'; import { dispatchDestroyExecutionClone, dispatchSendMessage, dispatchSendStop, listSendTargets, type SendMessageCloneRequest, type SendToolDeps } from './send-tool.js'; import { cronMcpCreate, cronMcpCreateSelf, cronMcpDelete, cronMcpList, cronMcpUpdate, cronMcpUpdateSelf, type CronMcpClientOptions } from './cron-mcp-client.js'; -import { registerMemoryShortRef, resolveMemoryShortRef, resolveMemoryShortRefCandidates } from '../context/memory-short-ref.js'; +import { + registerMemoryShortRef, + resolveMemoryShortRefCandidatesWithStore, + resolveMemoryShortRefWithStore, +} from '../context/memory-short-ref.js'; /** Upper bound on records expanded for one colliding handle. */ const AMBIGUOUS_REF_CANDIDATE_CAP = 4; @@ -762,13 +766,16 @@ function canManageProjectionNamespace(projectionNamespace: ContextNamespace, cal return !projectionUserId || projectionUserId === LEGACY_DAEMON_LOCAL_USER_ID || projectionUserId === callerUserId; } -function resolveProjectionRefArg(args: Record, namespace: ContextNamespace): string | ToolResult { +async function resolveProjectionRefArg( + args: Record, + namespace: ContextNamespace, +): Promise { const projectionId = stringArg(args, 'projectionId'); const ref = stringArg(args, 'ref'); if (projectionId && ref) return error(MCP_ERROR_REASONS.VALIDATION_FAILED, 'ref cannot be combined with projectionId'); if (projectionId) return projectionId; if (!ref) return error(MCP_ERROR_REASONS.VALIDATION_FAILED, 'projectionId or ref is required'); - const resolved = resolveMemoryShortRef(ref, namespace); + const resolved = await resolveMemoryShortRefWithStore(ref, namespace); if (!resolved || resolved.kind !== 'projection') { return error(MCP_ERROR_REASONS.PROJECTION_UNAVAILABLE, 'projection is not available in the caller namespace'); } @@ -955,7 +962,7 @@ export function createMemoryMcpToolHandlers(caller: McpRuntimeCaller, deps: Memo } if (!projectId) return emptySources(); if (ref) { - const candidates = resolveMemoryShortRefCandidates(ref, scopedCaller.namespace); + const candidates = await resolveMemoryShortRefCandidatesWithStore(ref, scopedCaller.namespace); if (candidates.length === 0) return { status: 'ok', ref, sourceEventCount: 0, sources: [] }; // A digest collision (two records deriving one handle) should never // happen at 65 bits. If it does, answering with one of them would be a @@ -1041,7 +1048,7 @@ export function createMemoryMcpToolHandlers(caller: McpRuntimeCaller, deps: Memo if (gate) return gate; const args = pickAllowedMcpArgs(input, ['projectionId', 'ref']); const scopedCaller = scopedCallerForDeps(caller, deps); - const projectionId = resolveProjectionRefArg(args, scopedCaller.namespace); + const projectionId = await resolveProjectionRefArg(args, scopedCaller.namespace); if (typeof projectionId !== 'string') return projectionId; const projection = await loadManageableProjection(projectionId, scopedCaller, getProcessedProjectionById); if (isToolResultValue(projection)) return projection; @@ -1054,7 +1061,7 @@ export function createMemoryMcpToolHandlers(caller: McpRuntimeCaller, deps: Memo if (gate) return gate; const args = pickAllowedMcpArgs(input, ['projectionId', 'ref']); const scopedCaller = scopedCallerForDeps(caller, deps); - const projectionId = resolveProjectionRefArg(args, scopedCaller.namespace); + const projectionId = await resolveProjectionRefArg(args, scopedCaller.namespace); if (typeof projectionId !== 'string') return projectionId; const projection = await loadManageableProjection(projectionId, scopedCaller, getProcessedProjectionById); if (isToolResultValue(projection)) return projection; @@ -1067,7 +1074,7 @@ export function createMemoryMcpToolHandlers(caller: McpRuntimeCaller, deps: Memo if (gate) return gate; const args = pickAllowedMcpArgs(input, ['projectionId', 'ref']); const scopedCaller = scopedCallerForDeps(caller, deps); - const projectionId = resolveProjectionRefArg(args, scopedCaller.namespace); + const projectionId = await resolveProjectionRefArg(args, scopedCaller.namespace); if (typeof projectionId !== 'string') return projectionId; const projection = await loadManageableProjection(projectionId, scopedCaller, getProcessedProjectionById); if (isToolResultValue(projection)) return projection; @@ -1082,7 +1089,7 @@ export function createMemoryMcpToolHandlers(caller: McpRuntimeCaller, deps: Memo const text = stringArg(args, 'text'); if (!text) return error(MCP_ERROR_REASONS.VALIDATION_FAILED, 'text is required'); const scopedCaller = scopedCallerForDeps(caller, deps); - const projectionId = resolveProjectionRefArg(args, scopedCaller.namespace); + const projectionId = await resolveProjectionRefArg(args, scopedCaller.namespace); if (typeof projectionId !== 'string') return projectionId; const projection = await loadManageableProjection(projectionId, scopedCaller, getProcessedProjectionById); if (isToolResultValue(projection)) return projection; @@ -1105,7 +1112,7 @@ export function createMemoryMcpToolHandlers(caller: McpRuntimeCaller, deps: Memo return error(MCP_ERROR_REASONS.VALIDATION_FAILED, 'feedback must be not_relevant or relevant'); } const scopedCaller = scopedCallerForDeps(caller, deps); - const projectionId = resolveProjectionRefArg(args, scopedCaller.namespace); + const projectionId = await resolveProjectionRefArg(args, scopedCaller.namespace); if (typeof projectionId !== 'string') return projectionId; const projection = await loadManageableProjection(projectionId, scopedCaller, getProcessedProjectionById); if (isToolResultValue(projection)) return projection; diff --git a/src/store/context-store.ts b/src/store/context-store.ts index 93c6a2964..f7b39148c 100644 --- a/src/store/context-store.ts +++ b/src/store/context-store.ts @@ -2799,6 +2799,25 @@ export function listMemoryShortRefs(limit?: number): MemoryShortRefRow[] { + (typeof limit === 'number' && limit > 0 ? ' LIMIT ?' : ''); const statement = database.prepare(sql); const rows = (typeof limit === 'number' && limit > 0 ? statement.all(limit) : statement.all()) as Array>; + return mapMemoryShortRefRows(rows); +} + +/** + * Exact lookup used when another process persisted a handle after this + * process's in-memory short-ref index was populated. The primary key starts + * with `ref`, so this is an indexed point lookup rather than another warm-load. + */ +export function listMemoryShortRefsByRef(ref: string): MemoryShortRefRow[] { + const database = ensureDb(); + const rows = database.prepare(` + SELECT * FROM memory_short_refs + WHERE ref = ? + ORDER BY last_seen_at DESC + `).all(ref) as Array>; + return mapMemoryShortRefRows(rows); +} + +function mapMemoryShortRefRows(rows: Array>): MemoryShortRefRow[] { return rows.map((row) => ({ ref: String(row.ref), kind: String(row.kind), diff --git a/test/context/memory-short-ref-store.test.ts b/test/context/memory-short-ref-store.test.ts index 85e081206..9a874aeb7 100644 --- a/test/context/memory-short-ref-store.test.ts +++ b/test/context/memory-short-ref-store.test.ts @@ -1,6 +1,10 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { resetContextStoreClientForTests } from '../../src/store/context-store-worker-client.js'; -import { listMemoryShortRefs } from '../../src/store/context-store.js'; +import { + listMemoryShortRefs, + listMemoryShortRefsByRef, + upsertMemoryShortRefs, +} from '../../src/store/context-store.js'; import { readFileSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { @@ -9,6 +13,9 @@ import { registerMemoryShortRefs, resetMemoryShortRefsForTests, resolveMemoryShortRef, + resolveMemoryShortRefCandidatesWithStore, + resolveMemoryShortRefWithStore, + seedMemoryShortRefCollisionForTests, } from '../../src/context/memory-short-ref.js'; import { cleanupIsolatedSharedContextDb, createIsolatedSharedContextDb } from '../util/shared-context-db.js'; @@ -121,4 +128,104 @@ describe('memory short refs — durable store persistence', () => { expect(resolveMemoryShortRef(ref!, { ...namespace, projectId: 'other-repo' })).toBeUndefined(); expect(resolveMemoryShortRef(ref!, namespace)).toMatchObject({ id: 'cccccccccc-1111-2222-3333-444444444444' }); }); + + it('hydrates an exact ref persisted after this process cache was populated', async () => { + const localId = 'local-cache-entry'; + const remoteId = 'persisted-by-another-process'; + const remoteRef = makeMemoryShortRef('projection', remoteId); + + // Populate this process's Map first, then write a different ref straight to + // SQLite as another daemon/MCP process would. A one-time warm-load cannot + // observe this later write. + seedMemoryShortRefCollisionForTests(makeMemoryShortRef('projection', localId), [{ + kind: 'projection', + id: localId, + namespace, + }]); + upsertMemoryShortRefs([{ + ref: remoteRef, + kind: 'projection', + id: remoteId, + namespaceKey: JSON.stringify([ + namespace.scope, + namespace.userId, + namespace.projectId, + '', + '', + ]), + namespaceJson: JSON.stringify(namespace), + lastSeenAt: Date.now(), + }]); + + expect(resolveMemoryShortRef(remoteRef, namespace)).toBeUndefined(); + expect(listMemoryShortRefsByRef(remoteRef)).toHaveLength(1); + + await expect(resolveMemoryShortRefCandidatesWithStore(remoteRef, namespace)) + .resolves.toEqual([ + expect.objectContaining({ kind: 'projection', id: remoteId }), + ]); + await expect(resolveMemoryShortRefWithStore(remoteRef, namespace)) + .resolves.toMatchObject({ kind: 'projection', id: remoteId }); + }); + + it('does not cross namespaces while hydrating an exact persisted ref', async () => { + const id = 'persisted-private-entry'; + const ref = makeMemoryShortRef('projection', id); + upsertMemoryShortRefs([{ + ref, + kind: 'projection', + id, + namespaceKey: JSON.stringify([ + namespace.scope, + namespace.userId, + namespace.projectId, + '', + '', + ]), + namespaceJson: JSON.stringify(namespace), + lastSeenAt: Date.now(), + }]); + + await expect(resolveMemoryShortRefCandidatesWithStore(ref, { + ...namespace, + userId: 'other-user', + })).resolves.toEqual([]); + await expect(resolveMemoryShortRefWithStore(ref, { + ...namespace, + projectId: 'other-repo', + })).resolves.toBeUndefined(); + await expect(resolveMemoryShortRefWithStore(ref, namespace)) + .resolves.toMatchObject({ id }); + }); + + it('keeps an old hydrated ref long enough to resolve when the local LRU is full', async () => { + for (let index = 0; index < 10_000; index += 1) { + seedMemoryShortRefCollisionForTests(`proj:cached${String(index).padStart(7, '0')}`, [{ + kind: 'projection', + id: `cached-${index}`, + namespace, + lastSeenAt: 1_000 + index, + }]); + } + const id = 'old-persisted-ref-at-lru-cap'; + const ref = makeMemoryShortRef('projection', id); + upsertMemoryShortRefs([{ + ref, + kind: 'projection', + id, + namespaceKey: JSON.stringify([ + namespace.scope, + namespace.userId, + namespace.projectId, + '', + '', + ]), + namespaceJson: JSON.stringify(namespace), + lastSeenAt: 1, + }]); + + expect(resolveMemoryShortRef(ref, namespace)).toBeUndefined(); + await expect(resolveMemoryShortRefWithStore(ref, namespace)) + .resolves.toMatchObject({ id }); + }); }); diff --git a/test/e2e/memory-mcp-interface.test.ts b/test/e2e/memory-mcp-interface.test.ts index 0687c55bf..49ea4d94d 100644 --- a/test/e2e/memory-mcp-interface.test.ts +++ b/test/e2e/memory-mcp-interface.test.ts @@ -18,9 +18,12 @@ import { createMemoryMcpToolHandlers } from '../../src/daemon/memory-mcp-tools.j import type { McpRuntimeCaller } from '../../src/daemon/memory-mcp-caller.js'; import { archiveEventsForMaterialization, + ensureContextNamespace, listContextObservations, recordContextEvent, resetContextStoreForTests, + upsertMemoryShortRefs, + writeContextObservation, writeProcessedProjection, } from '../../src/store/context-store.js'; import { cleanupIsolatedSharedContextDb, createIsolatedSharedContextDb } from '../util/shared-context-db.js'; @@ -253,6 +256,115 @@ describe('memory MCP interface e2e', () => { }); }); + it('redeems and mutates refs persisted after the independent MCP process starts', async () => { + // The MCP project scoping layer intentionally maps user_private to the + // project-local personal namespace before resolving refs. + const lateRefNamespace = { + scope: 'personal' as const, + userId: USER_ID, + projectId: PROJECT_NAME, + }; + const readable = writeProcessedProjection({ + namespace: lateRefNamespace, + class: 'recent_summary', + sourceEventIds: ['evt-late-ref-readable'], + summary: 'projection persisted after MCP startup remains directly redeemable', + content: { ownerUserId: USER_ID, eventCount: 1 }, + origin: 'chat_compacted', + createdAt: 1_100, + updatedAt: 1_100, + }); + const manageable = writeProcessedProjection({ + namespace: lateRefNamespace, + class: 'recent_summary', + sourceEventIds: ['evt-late-ref-manageable'], + summary: 'projection persisted after MCP startup remains manageable by ref', + content: { ownerUserId: USER_ID, eventCount: 1 }, + origin: 'chat_compacted', + createdAt: 1_200, + updatedAt: 1_200, + }); + const observationNamespace = ensureContextNamespace(lateRefNamespace, 1_250); + const observation = writeContextObservation({ + namespaceId: observationNamespace.id, + scope: lateRefNamespace.scope, + class: 'note', + origin: 'agent_learned', + fingerprint: 'late-cross-process-observation-ref', + content: { text: 'observation persisted after MCP startup remains directly redeemable' }, + text: 'observation persisted after MCP startup remains directly redeemable', + sourceEventIds: ['evt-late-ref-observation'], + state: 'active', + now: 1_250, + }); + const readableRef = makeMemoryShortRef('projection', readable.id); + const manageableRef = makeMemoryShortRef('projection', manageable.id); + const observationRef = makeMemoryShortRef('observation', observation.id); + + await withStdioClient(childEnv(), async (client) => { + // Ensure the child is fully started before another process writes these + // rows. No search/list call is allowed to register them in the child Map. + await client.listTools(); + upsertMemoryShortRefs([ + { ref: readableRef, kind: 'projection', id: readable.id }, + { ref: manageableRef, kind: 'projection', id: manageable.id }, + { ref: observationRef, kind: 'observation', id: observation.id }, + ].map((entry, index) => ({ + ...entry, + namespaceKey: JSON.stringify([ + lateRefNamespace.scope, + lateRefNamespace.userId, + lateRefNamespace.projectId, + '', + '', + ]), + namespaceJson: JSON.stringify(lateRefNamespace), + lastSeenAt: Date.now() + index, + }))); + + const sources = structured(await client.callTool({ + name: MEMORY_MCP_TOOL_NAMES.GET_MEMORY_SOURCES, + arguments: { ref: readableRef, kind: 'projection' }, + })); + expect(sources).toMatchObject({ + status: 'ok', + projectionId: readable.id, + sourceEventCount: 1, + projectionSource: expect.objectContaining({ + eventId: 'evt-late-ref-readable', + content: readable.summary, + }), + }); + + const observationSources = structured(await client.callTool({ + name: MEMORY_MCP_TOOL_NAMES.GET_MEMORY_SOURCES, + arguments: { ref: observationRef, kind: 'observation' }, + })); + expect(observationSources).toMatchObject({ + status: 'ok', + observationId: observation.id, + sourceEventCount: 1, + sources: [ + expect.objectContaining({ + eventId: 'evt-late-ref-observation', + status: 'observation', + content: 'observation persisted after MCP startup remains directly redeemable', + }), + ], + }); + + const archived = structured(await client.callTool({ + name: MEMORY_MCP_TOOL_NAMES.ARCHIVE_MEMORY, + arguments: { ref: manageableRef }, + })); + expect(archived).toMatchObject({ + status: 'ok', + projectionId: manageable.id, + changed: true, + }); + }); + }); + it('finds a projection by MCP search and expands summary fallback by projectionId or short ref', async () => { const projection = writeProcessedProjection({ namespace, From 3588a4dbe48c16e427f4a9c3a34d81b54eae8a66 Mon Sep 17 00:00:00 2001 From: "IM.codes" Date: Sun, 26 Jul 2026 21:05:32 +0800 Subject: [PATCH 40/40] Show redeemable refs in memory recall cards --- shared/context-types.ts | 2 ++ src/daemon/command-handler.ts | 19 +++++++++++++- src/daemon/memory-context-timeline.ts | 10 ++++++-- src/shared/timeline/types.ts | 2 ++ .../memory-recall-refs-namespace.test.ts | 16 ++++++++++++ .../command-handler-memory-context.test.ts | 3 +++ web/src/components/ChatView.tsx | 3 +++ web/src/memory-summary-sync.ts | 25 ++++++++++++------- web/src/styles.css | 1 + web/test/components/ChatView.test.tsx | 3 +++ web/test/memory-summary-sync.test.ts | 20 +++++++++------ 11 files changed, 85 insertions(+), 19 deletions(-) diff --git a/shared/context-types.ts b/shared/context-types.ts index 48c571bce..9751b9494 100644 --- a/shared/context-types.ts +++ b/shared/context-types.ts @@ -305,6 +305,8 @@ export interface ContextMemoryProjectView { export interface ContextMemoryRecordView { id: string; + /** Daemon-issued, persisted projection handle when this record is addressable. */ + ref?: string; scope: MemoryScope; projectId: string; ownerUserId?: string; diff --git a/src/daemon/command-handler.ts b/src/daemon/command-handler.ts index 19d87f723..1c2648747 100644 --- a/src/daemon/command-handler.ts +++ b/src/daemon/command-handler.ts @@ -10686,11 +10686,28 @@ async function handlePersonalMemoryQuery(cmd: Record, serverLin includeArchived, }; const projects = await getContextStoreClient().run('listMemoryProjectSummaries', [summaryArgs]); + // The browser's manual "sync recent summaries" path must consume handles + // issued by the daemon. It previously derived a legacy 10-character value in + // Web code without registering it, so the displayed `proj:` token was dead. + // Keep ordinary management-list reads side-effect free; only the explicit + // sync caller asks us to register and persist handles. + const referencedRecords = cmd.includeShortRefs === true + ? attachMemoryShortRefs(records.map((record) => ({ + ...record, + type: 'processed' as const, + }))) + : []; + const responseRecords = referencedRecords.length > 0 + ? records.map((record, index) => ({ + ...record, + ...(referencedRecords[index]?.ref ? { ref: referencedRecords[index].ref } : {}), + })) + : records; serverLink.send({ type: MEMORY_WS.PERSONAL_RESPONSE, requestId, stats, - records, + records: responseRecords, pendingRecords, projects, }); diff --git a/src/daemon/memory-context-timeline.ts b/src/daemon/memory-context-timeline.ts index b83b21b08..dc5407ac8 100644 --- a/src/daemon/memory-context-timeline.ts +++ b/src/daemon/memory-context-timeline.ts @@ -32,8 +32,14 @@ export function buildMemoryContextTimelinePayload( .map((item) => ({ ...item, text: item.text.trim() })) .filter((item) => item.text); if (items.length === 0 && preferenceItems.length === 0) return null; - const timelineItems: MemoryContextTimelineItem[] = items.map((item) => ({ + // Register once, then reuse the exact same handles in both the agent-facing + // injected text and the user-facing timeline payload. Mapping first used to + // drop `type`, which made observation evidence look like a projection in the + // reconstructed injectedText and left the web card with no ref at all. + const referencedItems = attachMemoryShortRefs(items); + const timelineItems: MemoryContextTimelineItem[] = referencedItems.map((item) => ({ id: item.id, + ...(item.ref ? { ref: item.ref } : {}), projectId: item.projectId, ...('scope' in item && item.scope ? { scope: item.scope } : {}), ...('enterpriseId' in item && item.enterpriseId ? { enterpriseId: item.enterpriseId } : {}), @@ -47,7 +53,7 @@ export function buildMemoryContextTimelinePayload( relevanceScore: item.relevanceScore, })); const injectedText = options?.injectedText - ?? (timelineItems.length > 0 ? buildRelatedPastWorkText(attachMemoryShortRefs(timelineItems)) : undefined); + ?? (referencedItems.length > 0 ? buildRelatedPastWorkText(referencedItems) : undefined); return { ...(query ? { query } : {}), ...(injectedText ? { injectedText } : {}), diff --git a/src/shared/timeline/types.ts b/src/shared/timeline/types.ts index cd817ad96..2f85e7856 100644 --- a/src/shared/timeline/types.ts +++ b/src/shared/timeline/types.ts @@ -164,6 +164,8 @@ export interface PeerAuditStatusTimelinePayload { export interface MemoryContextTimelineItem { id: string; + /** Redeemable projection/observation handle shown to users and agents. */ + ref?: string; projectId: string; scope?: string; enterpriseId?: string; diff --git a/test/context/memory-recall-refs-namespace.test.ts b/test/context/memory-recall-refs-namespace.test.ts index 30a92402b..e4fabe76a 100644 --- a/test/context/memory-recall-refs-namespace.test.ts +++ b/test/context/memory-recall-refs-namespace.test.ts @@ -43,6 +43,7 @@ vi.mock('../../src/store/context-store-worker-client.js', () => ({ })); import { attachMemoryShortRefs } from '../../src/context/memory-recall-refs.js'; +import { buildMemoryContextTimelinePayload } from '../../src/daemon/memory-context-timeline.js'; import { loadMemoryShortRefsFromStore, resetMemoryShortRefsForTests, @@ -113,6 +114,21 @@ describe('injected memory handles resolve for the agent they were injected into' expect(resolved?.kind).toBe('observation'); }); + it('preserves the same observation handle in injected text and the web timeline payload', () => { + const payload = buildMemoryContextTimelinePayload('remember this observation', [{ + id: 'obs-timeline-1', + type: 'observation', + scope: 'user_private', + projectId: PROJECT, + summary: 'The user prefers compact audit reports', + }]); + + const item = payload?.items[0]; + expect(item?.ref).toMatch(/^obs:[a-z2-7]{13}$/); + expect(payload?.injectedText).toContain(`(${item?.ref})`); + expect(payload?.injectedText).not.toContain('(proj:'); + }); + it('still redeems after a daemon restart, via the row that actually reached the store', async () => { // The in-memory index would mask a namespace mismatch for the life of the // process, so assert the round trip: persist, drop the index, warm-load from diff --git a/test/daemon/command-handler-memory-context.test.ts b/test/daemon/command-handler-memory-context.test.ts index 41967ccb5..bfe5f1b37 100644 --- a/test/daemon/command-handler-memory-context.test.ts +++ b/test/daemon/command-handler-memory-context.test.ts @@ -605,6 +605,7 @@ describe('handleWebCommand memory context timeline', () => { type: MEMORY_WS.PERSONAL_QUERY, requestId: 'personal-list', projectId: 'github.com/acme/repo', + includeShortRefs: true, [MEMORY_MANAGEMENT_CONTEXT_FIELD]: { actorId: 'user-bob', userId: 'user-bob', @@ -645,6 +646,7 @@ describe('handleWebCommand memory context timeline', () => { requestId: 'personal-list', records: [expect.objectContaining({ id: 'bob-proj', + ref: makeMemoryShortRef('projection', 'bob-proj'), summary: 'Bob private project memory', ownerUserId: 'user-bob', createdByUserId: 'user-bob', @@ -1406,6 +1408,7 @@ describe('handleWebCommand memory context timeline', () => { items: [ expect.objectContaining({ id: 'mem-1', + ref: makeMemoryShortRef('projection', 'mem-1'), projectId: 'codedeck', relevanceScore: 0.812, hitCount: 4, diff --git a/web/src/components/ChatView.tsx b/web/src/components/ChatView.tsx index 37bb966d9..552f32eef 100644 --- a/web/src/components/ChatView.tsx +++ b/web/src/components/ChatView.tsx @@ -3843,6 +3843,9 @@ const MemoryContextEvent = memo(function MemoryContextEvent({ event }: { event:
{item.summary}
+ {item.ref && ( + {item.ref} + )} {item.projectId} {score && {t('chat.memory_context_score', { score })}} {typeof item.hitCount === 'number' && item.hitCount > 0 ? ( diff --git a/web/src/memory-summary-sync.ts b/web/src/memory-summary-sync.ts index cc21838de..6c9c58e0a 100644 --- a/web/src/memory-summary-sync.ts +++ b/web/src/memory-summary-sync.ts @@ -47,15 +47,25 @@ function newestRecords(records: ContextMemoryRecordView[], projectId: string, li function mergeMemoryRecordsByPriority(recordGroups: ContextMemoryRecordView[][]): ContextMemoryRecordView[] { const merged: ContextMemoryRecordView[] = []; - const seen = new Set(); + const indexByKey = new Map(); for (const records of recordGroups) { for (const record of records) { const summary = record.summary.trim(); const key = record.id ? `id:${record.id}` : `summary:${record.projectId}:${record.projectionClass}:${summary}`; - if (seen.has(key)) continue; - seen.add(key); + const existingIndex = indexByKey.get(key); + if (existingIndex !== undefined) { + // Cloud stays authoritative for summary content/order, while a matching + // local daemon result can enrich it with the persisted short ref that + // only the daemon is able to issue. + const existing = merged[existingIndex]; + if (existing && !existing.ref && record.ref) { + merged[existingIndex] = { ...existing, ref: record.ref }; + } + continue; + } + indexByKey.set(key, merged.length); merged.push(record); } } @@ -109,6 +119,7 @@ export function localPersonalMemorySummarySource( projectId: input.projectId, projectionClass: input.projectionClass, limit: input.limit, + includeShortRefs: true, }); } catch (error) { finish(() => reject(error)); @@ -116,11 +127,6 @@ export function localPersonalMemorySummarySource( }); } -function projectionRef(projectionId: string): string { - const compact = projectionId.replace(/[^a-f0-9]/gi, '').slice(0, 10) || projectionId.slice(0, 10); - return `proj:${compact.toLowerCase()}`; -} - function sourceLookupLine(projectionId: string): string { return `sourceLookup: ${JSON.stringify({ tool: 'get_memory_sources', @@ -185,7 +191,8 @@ export async function buildMemorySummarySyncMessage( remainingSummaryChars -= trimmed.text.length; truncated = truncated || trimmed.truncated; const project = record.projectId ? `[${record.projectId}] ` : ''; - lines.push(`${lines.length + 1}. [ref: ${projectionRef(record.id)}] ${project}${trimmed.text}\n ${sourceLookupLine(record.id)}`); + const ref = record.ref ? `[ref: ${record.ref}] ` : ''; + lines.push(`${lines.length + 1}. ${ref}${project}${trimmed.text}\n ${sourceLookupLine(record.id)}`); } if (lines.length === 0) return null; diff --git a/web/src/styles.css b/web/src/styles.css index 519c9e195..d12f5003a 100644 --- a/web/src/styles.css +++ b/web/src/styles.css @@ -3121,6 +3121,7 @@ html.native-ios-mac .chat-dl-btn { .chat-memory-context-item-summary { font-size: 12px; line-height: 1.5; color: #e2e8f0; white-space: pre-wrap; word-break: break-word; } .chat-memory-context-item-meta { display: flex; flex-wrap: wrap; gap: 6px; } .chat-memory-context-chip { display: inline-flex; align-items: center; padding: 2px 7px; border-radius: 999px; background: rgba(30,41,59,0.9); border: 1px solid #334155; color: #94a3b8; font-size: 10px; } +.chat-memory-context-ref { color: #c4b5fd; font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; user-select: all; } .chat-memory-context-chip-muted { color: #64748b; } .chat-memory-context-collapse-bottom { align-self: flex-start; background: none; border: 1px solid #334155; color: #94a3b8; border-radius: 6px; padding: 6px 10px; font-size: 11px; cursor: pointer; transition: color 0.15s, border-color 0.15s, background 0.15s; } .chat-memory-context-collapse-bottom:hover { color: #e2e8f0; border-color: #6366f1; background: rgba(99,102,241,0.08); } diff --git a/web/test/components/ChatView.test.tsx b/web/test/components/ChatView.test.tsx index 67c82c555..c92e92ab5 100644 --- a/web/test/components/ChatView.test.tsx +++ b/web/test/components/ChatView.test.tsx @@ -1712,6 +1712,7 @@ describe('ChatView', () => { items: [ { id: 'mem-transport-1', + ref: 'proj:transportrefa', projectId: 'repo-1', summary: 'Fixed transport recall visibility', relevanceScore: 0.91, @@ -1732,6 +1733,8 @@ describe('ChatView', () => { await waitFor(() => { expect(container.textContent).toContain('Fixed transport recall visibility'); + expect(container.textContent).toContain('proj:transportrefa'); + expect(container.querySelector('.chat-memory-context-ref')?.textContent).toBe('proj:transportrefa'); expect(container.querySelector('.chat-memory-context')?.getAttribute('data-related-to')).toBe('evt-user-transport'); }); }); diff --git a/web/test/memory-summary-sync.test.ts b/web/test/memory-summary-sync.test.ts index 6628d716e..57496f24a 100644 --- a/web/test/memory-summary-sync.test.ts +++ b/web/test/memory-summary-sync.test.ts @@ -38,8 +38,9 @@ describe('memory summary sync message', () => { }); expect(message).toContain('SYNC ONLY'); expect(message).toContain('Recent summaries (2/5, max 7200 chars):'); - expect(message).toContain('1. [ref: proj:aaaaaaaaaa] [repo-1] Newest summary'); - expect(message).toContain('2. [ref: proj:1111111111] [repo-1] Older summary'); + expect(message).toContain('1. [repo-1] Newest summary'); + expect(message).toContain('2. [repo-1] Older summary'); + expect(message).not.toContain('[ref:'); expect(message).toContain('"tool":"get_memory_sources"'); expect(message).toContain('"kind":"projection"'); expect(message).toContain('"projectionId":"aaaaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"'); @@ -64,8 +65,8 @@ describe('memory summary sync message', () => { it('merges cloud and local summaries while keeping cloud records first for duplicates', async () => { const localSource = vi.fn().mockResolvedValue({ records: [ - { id: 'duplicate-summary', projectId: 'repo-1', projectionClass: 'recent_summary', summary: 'Local duplicate summary', updatedAt: 700 }, - { id: 'local-summary', projectId: 'repo-1', projectionClass: 'recent_summary', summary: 'Local only summary', updatedAt: 600 }, + { id: 'duplicate-summary', ref: 'proj:duplicateaaaa', projectId: 'repo-1', projectionClass: 'recent_summary', summary: 'Local duplicate summary', updatedAt: 700 }, + { id: 'local-summary', ref: 'proj:localsummaryaa', projectId: 'repo-1', projectionClass: 'recent_summary', summary: 'Local only summary', updatedAt: 600 }, ], }); getPersonalCloudMemory.mockResolvedValueOnce({ @@ -92,6 +93,8 @@ describe('memory summary sync message', () => { expect(message).toContain('Local only summary'); expect(message).toContain('Cloud duplicate summary'); expect(message).toContain('Cloud only summary'); + expect(message).toContain('[ref: proj:duplicateaaaa] [repo-1] Cloud duplicate summary'); + expect(message).toContain('[ref: proj:localsummaryaa] [repo-1] Local only summary'); expect(message).not.toContain('Local duplicate summary'); }); @@ -144,7 +147,7 @@ describe('memory summary sync message', () => { requestId: message.requestId, stats: {}, records: [ - { id: 'local-ws-summary', projectId: 'repo-1', projectionClass: 'recent_summary', summary: 'Local ws summary', updatedAt: 700 }, + { id: 'local-ws-summary', ref: 'proj:localwssumaaa', projectId: 'repo-1', projectionClass: 'recent_summary', summary: 'Local ws summary', updatedAt: 700 }, ], }); } @@ -160,6 +163,7 @@ describe('memory summary sync message', () => { projectId: 'repo-1', projectionClass: 'recent_summary', limit: 4, + includeShortRefs: true, }); expect(ws.send).toHaveBeenCalledWith(expect.objectContaining({ @@ -170,6 +174,7 @@ describe('memory summary sync message', () => { limit: 4, })); expect(view?.records[0].summary).toBe('Local ws summary'); + expect(view?.records[0].ref).toBe('proj:localwssumaaa'); expect(handlers.size).toBe(0); }); @@ -198,8 +203,9 @@ describe('memory summary sync message', () => { limit: 10, }); expect(message).toContain('Recent summaries (10/10, max 7200 chars):'); - expect(message).toContain('1. [ref: proj:a12] [repo-1] Summary 12'); - expect(message).toContain('10. [ref: proj:a3] [repo-1] Summary 3'); + expect(message).toContain('1. [repo-1] Summary 12'); + expect(message).toContain('10. [repo-1] Summary 3'); + expect(message).not.toContain('[ref:'); expect(message).not.toContain('Summary 2'); });