diff --git a/server/src/routes/file-transfer.ts b/server/src/routes/file-transfer.ts index cb19029bf..9c3a032dd 100644 --- a/server/src/routes/file-transfer.ts +++ b/server/src/routes/file-transfer.ts @@ -9,6 +9,7 @@ import { WsBridge } from '../ws/bridge.js'; import { randomHex } from '../security/crypto.js'; import { FILE_TRANSFER_LIMITS, + FILE_TRANSFER_UPLOAD_ERROR_CODE, FILE_TRANSFER_UPLOAD_FETCH_CAPABILITY, FILE_TRANSFER_DOWNLOAD_STREAM_CAPABILITY, FILE_TRANSFER_PATH_HANDLE_CAPABILITY, @@ -685,8 +686,13 @@ fileTransferRoutes.post('/:id/upload', async (c) => { ); if (result.type === 'file.upload_error') { - logger.warn({ serverId, uploadId, error: result.message }, 'Daemon upload error'); - return c.json({ error: 'upload_failed', message: result.message }, 500); + const code = typeof (result as { code?: unknown }).code === 'string' + ? (result as { code: string }).code + : 'upload_failed'; + logger.warn({ serverId, uploadId, error: result.message, code }, 'Daemon upload error'); + // 507 Insufficient Storage when the daemon's disk is full; 500 otherwise. + const status = code === FILE_TRANSFER_UPLOAD_ERROR_CODE.INSUFFICIENT_CAPACITY ? 507 : 500; + return c.json({ error: code, message: result.message }, status); } const attachment = result.attachment as AttachmentRef; diff --git a/server/src/util/metrics.ts b/server/src/util/metrics.ts index db953680b..d56ab3a27 100644 --- a/server/src/util/metrics.ts +++ b/server/src/util/metrics.ts @@ -1,41 +1,17 @@ -export type MetricLabels = Record; - -const counters = new Map(); -const MAX_COUNTERS = 1000; - -function labelsKey(labels?: MetricLabels): string { - if (!labels) return ''; - const entries = Object.entries(labels) - .filter(([, value]) => typeof value === 'string') - .sort(([a], [b]) => a.localeCompare(b)); - return entries.map(([key, value]) => `${key}=${value}`).join(','); -} - -function counterKey(name: string, labels?: MetricLabels): string { - const suffix = labelsKey(labels); - return suffix ? `${name}{${suffix}}` : name; -} - -export function incrementCounter(name: string, labels?: MetricLabels): void { - addCounter(name, 1, labels); -} - -export function addCounter(name: string, amount: number, labels?: MetricLabels): void { - if (!name) return; - if (!Number.isFinite(amount) || amount <= 0) return; - const key = counterKey(name, labels); - if (!counters.has(key) && counters.size >= MAX_COUNTERS) return; - counters.set(key, (counters.get(key) ?? 0) + amount); -} - -export function getCounter(name: string, labels?: MetricLabels): number { - return counters.get(counterKey(name, labels)) ?? 0; -} - -export function snapshotCounters(): Record { - return Object.fromEntries(counters.entries()); -} - -export function resetMetricsForTests(): void { - counters.clear(); -} +/** + * Server-facing entry point for in-process counters. + * + * The implementation lives in `shared/metrics.ts` so the daemon and the server + * cannot drift apart again. Note the scope: counters are per-process, and the + * server runs multiple replicas, so a number here reflects ONE pod's traffic and + * resets on every deploy. + */ +export { + addCounter, + counterKeyCount, + getCounter, + incrementCounter, + resetMetricsForTests, + snapshotCounters, +} from '../../../shared/metrics.js'; +export type { MetricLabels } from '../../../shared/metrics.js'; diff --git a/server/src/ws/bridge.ts b/server/src/ws/bridge.ts index 22753dee1..c026d755c 100644 --- a/server/src/ws/bridge.ts +++ b/server/src/ws/bridge.ts @@ -115,12 +115,21 @@ import { updateServerHeartbeat, updateServerStatus, upsertDiscussion, insertDisc import { toDiscussionCommentView } from '../share/discussion-comment-view.js'; import { resolveCoveredSessionNames } from '../share/covered-sessions.js'; import logger from '../util/logger.js'; +import { + TIMELINE_DELIVERY_METRICS, + countableTimelineEventType, +} from '../../../shared/timeline-delivery-telemetry.js'; import { incrementCounter } from '../util/metrics.js'; import { pickReadableSessionDisplay } from '../../../shared/session-display.js'; import { isKnownTestSessionLike } from '../../../shared/test-session-guard.js'; import { PUSH_TIMELINE_EVENT_MAX_AGE_MS, TIMELINE_SUPPRESS_PUSH_FIELD } from '../../../shared/push-notifications.js'; +import { isServerLinkResyncStatePayload } from '../../../shared/session-activity-types.js'; import { + DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION, + DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL, + DAEMON_UPGRADE_BLOCK_REASON, DAEMON_UPGRADE_DELIVERY_STATUS, + type DaemonUpgradeBlockedAckDisposition, } from '../../../shared/daemon-upgrade.js'; import { P2P_WORKFLOW_MSG, @@ -226,6 +235,8 @@ const MAX_QUEUE_SIZE = 100; const DAEMON_UPGRADE_BLOCKED_RETRY_MS = 60_000; const DAEMON_UPGRADE_BLOCKED_MIN_RETRY_MS = 5_000; const DAEMON_UPGRADE_BLOCKED_MAX_RETRY_MS = 15 * 60 * 1000; +const DAEMON_UPGRADE_BLOCKED_FAILURE_DEDUP_TTL_MS = 7 * 24 * 60 * 60 * 1000; +const DAEMON_UPGRADE_BLOCKED_FAILURE_DEDUP_MAX = 1_000; const DAEMON_UPGRADE_TRANSIENT_BLOCK_REASONS = new Set([ 'p2p_active', 'auto_deliver_active', @@ -590,6 +601,9 @@ type TimelineDataPlaneJob = { const WATCH_RECENT_TEXT_CAP = 5; const WATCH_RECENT_TEXT_MAX_CHARS = 160; let idlePushSettleMs = process.env.NODE_ENV === 'test' ? 0 : 300; +// Human-facing heartbeat for no-subscriber drops. The counter records each one; +// this keeps the log readable when a whole app-background window is discarded. +const TIMELINE_NO_SUBSCRIBER_LOG_THROTTLE_MS = 30_000; const HTTP_TIMELINE_TIMEOUT_MS = 15_000; const TIMELINE_PENDING_UNICAST_TIMEOUT_MS = 30_000; // Bumped from 128 → 4096 and 15s → 60s as part of the commit-42dfabec @@ -1129,6 +1143,11 @@ export class WsBridge { * auth check has settled. */ private authPromise: Promise | null = null; + /** Connection generation that advertised outbox-sync support during auth. */ + private upgradeBlockedSyncRequiredGeneration: number | null = null; + /** Connection generation whose persisted blocker replay has completed. */ + private upgradeBlockedSyncCompleteGeneration: number | null = null; + private seenUpgradeBlockedFailures = new Map(); private browserRateLimiter = new MemoryRateLimiter(); /** browser socket → session name → raw-enabled flag */ @@ -1216,6 +1235,9 @@ export class WsBridge { /** Lightweight per-session hot cache for Watch first-paint text. */ private recentTextBySession = new Map(); + /** Content-bearing timeline events discarded because nobody was subscribed. */ + private timelineNoSubscriberDrops = 0; + private lastTimelineNoSubscriberLogAt = 0; private pendingIdlePushes = new Map | null; db: Database; @@ -2404,6 +2426,7 @@ export class WsBridge { // generation (they resolve as indeterminate) so a reconnect never delivers a // stale result to a new waiter (10.6). this.daemonGeneration++; + const connectionGeneration = this.daemonGeneration; abandonPriorGenerations(this.serverId, this.daemonGeneration); // Invalidate every pending peer-audit response route: the prior daemon // is gone, and any reply that arrives for it must not be delivered to a @@ -2416,6 +2439,8 @@ export class WsBridge { // late-arriving messages don't await a stale (and possibly resolved // for a different `ws`) auth. this.authPromise = null; + this.upgradeBlockedSyncRequiredGeneration = null; + this.upgradeBlockedSyncCompleteGeneration = null; // Auth timeout this.authTimer = setTimeout(() => { @@ -2426,6 +2451,11 @@ export class WsBridge { }, AUTH_TIMEOUT_MS); ws.on('message', async (data, isBinary) => { + // A replaced socket can still emit already-buffered frames while ws is in + // CLOSING. Never let those frames borrow the replacement connection's + // authenticated/shared bridge state. + if (this.daemonWs !== ws || this.daemonGeneration !== connectionGeneration) return; + // Handle binary raw PTY frames if (isBinary) { // Binary PTY data is a FULL-daemon capability. Never route unauthenticated @@ -2453,10 +2483,11 @@ export class WsBridge { // `ws.close(4001, 'auth_required')` even though auth was about to // succeed milliseconds later. See `authPromise` field doc above. if (this.authPromise) { - try { await this.authPromise; } catch { /* ignore — closed below */ } + const pendingAuth = this.authPromise; + try { await pendingAuth; } catch { /* ignore — closed below */ } // The connection may have been closed while we awaited (auth // failed / timed out / replaced). Bail out before processing. - if (this.daemonWs !== ws) return; + if (this.daemonWs !== ws || this.daemonGeneration !== connectionGeneration) return; } if (!this.authenticated) { @@ -2475,7 +2506,19 @@ export class WsBridge { // Resolving (vs rejecting) avoids unhandled-rejection warnings // when no concurrent handler is currently awaiting. let resolveAuth!: () => void; - this.authPromise = new Promise((res) => { resolveAuth = res; }); + const localAuthPromise = new Promise((res) => { resolveAuth = res; }); + this.authPromise = localAuthPromise; + const isCurrentAuthConnection = (): boolean => + this.daemonWs === ws && this.daemonGeneration === connectionGeneration; + const finishLocalAuth = (): void => { + resolveAuth(); + // A replacement connection owns a different auth promise. A stale + // continuation may release only its own waiters; it must never clear + // the replacement's auth barrier. + if (this.authPromise === localAuthPromise) { + this.authPromise = null; + } + }; const tokenHash = sha256Hex(msg.token); let server: { token_hash: string; user_id?: string; node_role?: string | null; revoked_at?: number | null } | null = null; @@ -2485,16 +2528,23 @@ export class WsBridge { [this.serverId], ); } catch (err) { - resolveAuth(); - this.authPromise = null; + const stillCurrent = isCurrentAuthConnection(); + finishLocalAuth(); + if (!stillCurrent) return; throw err; } + // The DB lookup is the first asynchronous auth boundary. A replacement + // may have installed a new socket and auth promise while it was pending. + // Never let the stale continuation overwrite generation-bound state. + if (!isCurrentAuthConnection()) { + finishLocalAuth(); + return; + } if (!server || server.token_hash !== tokenHash) { logger.warn({ serverId: this.serverId }, 'Daemon auth failed'); ws.close(4001, 'auth_failed'); - resolveAuth(); - this.authPromise = null; + finishLocalAuth(); return; } @@ -2502,8 +2552,7 @@ export class WsBridge { if (server.revoked_at != null) { logger.warn({ serverId: this.serverId }, 'Daemon auth rejected: revoked'); ws.close(4003, 'revoked'); - resolveAuth(); - this.authPromise = null; + finishLocalAuth(); return; } @@ -2511,6 +2560,15 @@ export class WsBridge { // in the auth frame is IGNORED (10.2). A controlled node's WS is only a // presence/heartbeat + MACHINE_EXEC_RESULT surface. this.daemonNodeRole = server.node_role === NODE_ROLE.CONTROLLED ? NODE_ROLE.CONTROLLED : NODE_ROLE.FULL; + const supportsUpgradeBlockedSync = this.daemonNodeRole === NODE_ROLE.FULL + && msg[DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.AUTH_REVISION_FIELD] + === DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION; + this.upgradeBlockedSyncRequiredGeneration = supportsUpgradeBlockedSync + ? connectionGeneration + : null; + this.upgradeBlockedSyncCompleteGeneration = supportsUpgradeBlockedSync + ? null + : connectionGeneration; this.controlledFileTransferCapabilities = this.daemonNodeRole === NODE_ROLE.CONTROLLED ? new Set( (Array.isArray(msg.capabilities) ? msg.capabilities : []) @@ -2536,10 +2594,19 @@ export class WsBridge { // controlled node — it is a restricted presence/exec-result surface (10.2). if (this.daemonNodeRole !== NODE_ROLE.CONTROLLED && typeof server.user_id === 'string' && server.user_id.trim()) { try { - this.sendMemoryFeatureConfigApply(await this.readUserMemoryFeatureFlags(server.user_id)); + const memoryFeatureFlags = await this.readUserMemoryFeatureFlags(server.user_id); + if (!isCurrentAuthConnection()) { + finishLocalAuth(); + return; + } + this.sendMemoryFeatureConfigApply(memoryFeatureFlags); } catch (err) { logger.warn({ err, serverId: this.serverId }, 'failed to push global memory feature config on daemon auth'); } + if (!isCurrentAuthConnection()) { + finishLocalAuth(); + return; + } } this.daemonUpgradeCoordinator.clearIfTargetVersionMatches(this.daemonVersion); this.flushPendingDaemonUpgrade(ws); @@ -2644,7 +2711,11 @@ export class WsBridge { this.graceTimer = null; } this.daemonOfflineAnnounced = false; - await this.replayInflightToDaemon(); + await this.replayInflightToDaemon(ws, connectionGeneration); + if (!isCurrentAuthConnection()) { + finishLocalAuth(); + return; + } this.broadcastToBrowsers(JSON.stringify({ type: MSG_DAEMON_ONLINE })); this.startAckHousekeepingIfNeeded(); @@ -2653,8 +2724,20 @@ export class WsBridge { // (e.g. the `daemon.hello` that landed before auth finished) // will resume past their `await this.authPromise` and observe // `this.authenticated === true`. - resolveAuth(); - this.authPromise = null; + finishLocalAuth(); + // Do not arm an auto-upgrade timer until the auth turn has fully + // yielded. daemon.upgrade_blocked is sent immediately after auth by a + // reconnecting daemon, and its handler was waiting on authPromise. A + // next-turn flush lets that persisted terminal blocker run first, + // regardless of how long replayInflightToDaemon() made auth take. + setImmediate(() => { + if ( + this.daemonWs !== ws + || this.daemonGeneration !== connectionGeneration + || !this.authenticated + ) return; + this.flushPendingDaemonUpgrade(ws); + }); return; } @@ -2714,6 +2797,22 @@ export class WsBridge { return; } + if (msg.type === DAEMON_MSG.UPGRADE_BLOCKED_SYNC) { + if ( + this.daemonWs === ws + && this.daemonGeneration === connectionGeneration + && this.upgradeBlockedSyncRequiredGeneration === connectionGeneration + && msg.revision === DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION + ) { + this.upgradeBlockedSyncCompleteGeneration = connectionGeneration; + setImmediate(() => { + if (this.daemonWs !== ws || this.daemonGeneration !== connectionGeneration) return; + this.flushPendingDaemonUpgrade(ws); + }); + } + return; + } + if (msg.type === P2P_WORKFLOW_MSG.DAEMON_HELLO) { this.handleDaemonP2pWorkflowHello(msg); return; @@ -2738,7 +2837,7 @@ export class WsBridge { } if (msg.type === DAEMON_MSG.UPGRADE_BLOCKED) { - this.handleDaemonUpgradeBlocked(msg, ws); + if (!this.handleDaemonUpgradeBlocked(msg, ws)) return; } if (msg.type === 'heartbeat') { @@ -2780,6 +2879,20 @@ export class WsBridge { const payload = event.payload as Record; const eventTs = typeof event.ts === 'number' ? event.ts : undefined; if (payload[TIMELINE_SUPPRESS_PUSH_FIELD] === true) return; + // A daemon link-restore resync re-states every session's CURRENT + // state so browsers stop rendering a stale "working" spinner. It is + // a re-announcement, never a fresh completion, so it must not push: + // one link restore would otherwise fan one "Task complete" per idle + // session (seen on the 43 deployment as ~90 dispatches in 500ms right + // after four daemons re-authenticated). The age guard below cannot + // catch it — resync events are newly minted even though the state + // they describe is old. + // + // Checked HERE as well as suppressed at the daemon, on purpose: + // daemons are user-installed and upgrade on their own schedule, so a + // daemon-only fix leaves every not-yet-updated daemon still storming + // until it happens to update. This makes a server deploy sufficient. + if (isServerLinkResyncStatePayload(payload)) return; if (eventTs && Date.now() - eventTs > PUSH_TIMELINE_EVENT_MAX_AGE_MS) return; this.scheduleIdleEventPush(db, env, { type: 'session.idle', @@ -2801,6 +2914,8 @@ export class WsBridge { // closed, the awaiting handlers will fall through and observe // `this.daemonWs !== ws` and bail out. this.authPromise = null; + this.upgradeBlockedSyncRequiredGeneration = null; + this.upgradeBlockedSyncCompleteGeneration = null; this.recentTextBySession.clear(); this.clearPendingIdlePushes(); this.activeMainSessions.clear(); @@ -2840,6 +2955,9 @@ export class WsBridge { }); ws.on('error', (err) => { + // Errors from a replaced socket belong to its abandoned generation and + // must not reject requests owned by the current daemon connection. + if (this.daemonWs !== ws || this.daemonGeneration !== connectionGeneration) return; logger.error({ serverId: this.serverId, err }, 'Daemon WS error'); this.rejectAllPendingFileTransfers('daemon_error'); this.rejectAllPendingMemorySourcesRequests('daemon_error'); @@ -4522,7 +4640,8 @@ export class WsBridge { // Bypass TerminalForwardQueue: timeline events are control-plane and // must never queue behind PTY data. Critical for cancel/stop UX — // session.state(idle) used to arrive seconds after the push notification. - this.sendJsonToSessionSubscribers(sessionId, JSON.stringify(msg)); + const timelineRecipients = this.sendJsonToSessionSubscribers(sessionId, JSON.stringify(msg)); + this.recordTimelineFanout(sessionId, rawEvent, timelineRecipients); return; } @@ -4906,6 +5025,11 @@ export class WsBridge { daemonVersion: typeof msg.daemonVersion === 'string' ? msg.daemonVersion : this.daemonVersion, cpu: msg.cpu, memUsed: msg.memUsed, memTotal: msg.memTotal, load1: msg.load1, load5: msg.load5, load15: msg.load15, uptime: msg.uptime, + // Memory-handle persistence failures ride this frame because the + // daemon's own counter and log cannot leave a machine whose disk is + // full. Rebuilding the payload without them put the signal in a hole: + // it reached this pod and went no further. + ...(isPlainRecord(msg.shortRefHealth) ? { shortRefHealth: msg.shortRefHealth } : {}), })); return; } @@ -5165,7 +5289,7 @@ export class WsBridge { * subscriptions to); we dedup per-WS so the same JSON is never sent * twice. */ - private sendJsonToSessionSubscribers(sessionName: string, json: string): void { + private sendJsonToSessionSubscribers(sessionName: string, json: string): number { const sent = new Set(); for (const [ws, sessions] of this.browserSubscriptions) { if (!sessions.has(sessionName)) continue; @@ -5185,6 +5309,47 @@ export class WsBridge { sent.add(ws); safeSend(ws, outgoing); } + return sent.size; + } + + /** + * Observe the fan-out result for a live timeline event. + * + * With zero subscribers both loops in `sendJsonToSessionSubscribers` simply never + * execute a body: the event is discarded with no queue, no replay and — until now + * — no trace. That is the normal state of a BACKGROUNDED app, which is exactly why + * "the push notification arrived but the chat was empty" was so reproducible. The + * browser now heals this with a no-lower-bound backfill on activation, so without + * a counter a rising drop rate would be permanently masked by recovery. + * + * Takes the already-parsed event and the recipient count the fan-out returned: + * this runs for every event of every session, so it must not re-parse the + * serialized copy. Only content-bearing events are counted; status/usage chatter + * fires about once a second during a turn and would drown the signal. + */ + private recordTimelineFanout(sessionName: string, event: unknown, recipientCount: number): void { + const eventType = countableTimelineEventType(event); + if (!eventType) return; + if (recipientCount > 0) { + incrementCounter(TIMELINE_DELIVERY_METRICS.SERVER_DELIVERED, { eventType }); + return; + } + incrementCounter(TIMELINE_DELIVERY_METRICS.SERVER_NO_SUBSCRIBER_DROPPED, { eventType }); + this.timelineNoSubscriberDrops += 1; + const now = Date.now(); + if (now - this.lastTimelineNoSubscriberLogAt < TIMELINE_NO_SUBSCRIBER_LOG_THROTTLE_MS) return; + this.lastTimelineNoSubscriberLogAt = now; + logger.warn({ + serverId: this.serverId, + sessionName, + eventType, + droppedTotal: this.timelineNoSubscriberDrops, + }, 'Timeline event discarded: no subscribed viewer (client must heal via backfill)'); + } + + /** Total content-bearing timeline events discarded for lack of a subscriber. */ + get timelineNoSubscriberDropCount(): number { + return this.timelineNoSubscriberDrops; } private sendToRawSessionSubscribers(sessionName: string, data: string | Buffer): void { @@ -5636,20 +5801,46 @@ export class WsBridge { } /** Replay buffered + dispatched commands to the daemon after reconnect. */ - private async replayInflightToDaemon(): Promise { + private async replayInflightToDaemon(expectedWs: WebSocket, expectedGeneration: number): Promise { const ordered = [...this.inflightCommands.values()].sort((a, b) => a.sentAt - b.sentAt); for (const entry of ordered) { + if (this.daemonWs !== expectedWs || this.daemonGeneration !== expectedGeneration) return; if (entry.state === 'acked') continue; try { - await this.dispatchInflightToDaemon(entry, entry.dispatchAttempts > 0); + await this.dispatchInflightToDaemon( + entry, + entry.dispatchAttempts > 0, + { ws: expectedWs, generation: expectedGeneration }, + ); } catch (err) { logger.warn({ commandId: entry.commandId, err }, 'replayInflightToDaemon failed for entry'); } } } - private async dispatchInflightToDaemon(entry: InflightCommand, markBridgeRetry: boolean): Promise { + private async dispatchInflightToDaemon( + entry: InflightCommand, + markBridgeRetry: boolean, + expectedConnection?: { ws: WebSocket; generation: number }, + ): Promise { + if ( + expectedConnection + && ( + this.daemonWs !== expectedConnection.ws + || this.daemonGeneration !== expectedConnection.generation + ) + ) return; if (!await this.revalidateInflightShareCommand(entry)) return; + // Share revalidation can await DB/coverage work. If auth was replaced while + // it was pending, leave the entry for the new generation's replay rather + // than sending it through the replacement socket from a stale auth flow. + if ( + expectedConnection + && ( + this.daemonWs !== expectedConnection.ws + || this.daemonGeneration !== expectedConnection.generation + ) + ) return; const rawPayload = markBridgeRetry ? this.withBridgeRetryMarker(entry.rawPayload, entry.dispatchAttempts + 1) : entry.rawPayload; @@ -5874,19 +6065,80 @@ export class WsBridge { return this.seenCommandAcks.has(commandId); } - private handleDaemonUpgradeBlocked(msg: Record, ws: WebSocket): void { + private handleDaemonUpgradeBlocked(msg: Record, ws: WebSocket): boolean { const serverVersion = process.env.APP_VERSION; - if (!serverVersion || serverVersion === '0.0.0' || !this.daemonVersion || this.daemonVersion === serverVersion) return; + const failureId = typeof msg.failureId === 'string' && msg.failureId.length > 0 && msg.failureId.length <= 128 + ? msg.failureId + : null; + const failedTargetVersion = typeof msg.targetVersion === 'string' ? msg.targetVersion : null; + const failureUpgradeId = typeof msg.upgradeId === 'string' && msg.upgradeId.length > 0 && msg.upgradeId.length <= 128 + ? msg.upgradeId + : null; + if (!serverVersion || serverVersion === '0.0.0' || !this.daemonVersion || this.daemonVersion === serverVersion) { + if (failureId) { + this.sendDaemonUpgradeBlockedAck( + ws, + failureId, + DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.OBSOLETE, + ); + } + return false; + } + if ( + msg.reason === DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED + && failedTargetVersion + && failedTargetVersion !== serverVersion + ) { + this.sendDaemonUpgradeBlockedAck( + ws, + failureId, + DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.OBSOLETE, + ); + return false; + } const retryDelayMs = this.daemonUpgradeBlockedRetryDelayMs(msg); if (retryDelayMs == null) { + const replayBeforeSync = this.upgradeBlockedSyncRequiredGeneration === this.daemonGeneration + && this.upgradeBlockedSyncCompleteGeneration !== this.daemonGeneration; + const supersededByManual = msg.reason === DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED + && failedTargetVersion != null + && ( + this.daemonUpgradeCoordinator.isTerminalFailureSupersededByManual( + failedTargetVersion, + failureUpgradeId, + ) + || ( + failureUpgradeId == null + && replayBeforeSync + && this.daemonUpgradeCoordinator.hasManualLifecycleForTarget(failedTargetVersion) + ) + ); + if (supersededByManual) { + this.sendDaemonUpgradeBlockedAck( + ws, + failureId, + DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.SUPERSEDED, + ); + return false; + } + this.daemonUpgradeCoordinator.blockTargetAfterTerminalFailure(serverVersion); + const duplicate = failureId ? this.hasSeenUpgradeBlockedFailure(failureId) : false; + if (failureId) { + this.rememberUpgradeBlockedFailure(failureId); + this.sendDaemonUpgradeBlockedAck( + ws, + failureId, + DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.ACCEPTED, + ); + } logger.info({ serverId: this.serverId, daemonVersion: this.daemonVersion, serverVersion, reason: typeof msg.reason === 'string' ? msg.reason : 'unknown', }, 'daemon.upgrade blocked by non-retryable reason'); - return; + return !duplicate; } const result = this.daemonUpgradeCoordinator.retryAutoAfterBlocked({ @@ -5906,6 +6158,44 @@ export class WsBridge { upgradeId: result.upgradeId, }, 'daemon.upgrade blocked by transient daemon state — retry scheduled'); } + return true; + } + + private sendDaemonUpgradeBlockedAck( + ws: WebSocket, + failureId: string | null, + disposition: DaemonUpgradeBlockedAckDisposition, + ): void { + if (!failureId) return; + try { + ws.send(JSON.stringify({ + type: DAEMON_MSG.UPGRADE_BLOCKED_ACK, + failureId, + disposition, + })); + } catch { + // The daemon retains the blocker until it receives this ACK and will + // replay it after reconnect, rebuilding the coordinator gate. + } + } + + private hasSeenUpgradeBlockedFailure(failureId: string, now = Date.now()): boolean { + const seenAt = this.seenUpgradeBlockedFailures.get(failureId); + return seenAt !== undefined && now - seenAt <= DAEMON_UPGRADE_BLOCKED_FAILURE_DEDUP_TTL_MS; + } + + private rememberUpgradeBlockedFailure(failureId: string, now = Date.now()): void { + for (const [id, seenAt] of this.seenUpgradeBlockedFailures) { + if (now - seenAt > DAEMON_UPGRADE_BLOCKED_FAILURE_DEDUP_TTL_MS) { + this.seenUpgradeBlockedFailures.delete(id); + } + } + this.seenUpgradeBlockedFailures.set(failureId, now); + while (this.seenUpgradeBlockedFailures.size > DAEMON_UPGRADE_BLOCKED_FAILURE_DEDUP_MAX) { + const oldest = this.seenUpgradeBlockedFailures.keys().next().value as string | undefined; + if (!oldest) break; + this.seenUpgradeBlockedFailures.delete(oldest); + } } private daemonUpgradeBlockedRetryDelayMs(msg: Record): number | null { @@ -5939,7 +6229,7 @@ export class WsBridge { }); } - private flushPendingDaemonUpgrade(ws: WebSocket): void { + private flushPendingDaemonUpgrade(ws: WebSocket): RequestDaemonUpgradeResult | null { const result = this.daemonUpgradeCoordinator.flushPending({ skipPublicationGate: this.daemonNodeRole === NODE_ROLE.CONTROLLED, isDaemonReady: () => this.isDaemonReadyForUpgrade(), @@ -5959,10 +6249,13 @@ export class WsBridge { nextAttemptAt: result.nextAttemptAt, }, 'Pending daemon.upgrade is waiting for npm publication'); } + return result; } private isDaemonReadyForUpgrade(): boolean { - return Boolean(this.daemonWs && this.authenticated); + const syncReady = this.upgradeBlockedSyncRequiredGeneration !== this.daemonGeneration + || this.upgradeBlockedSyncCompleteGeneration === this.daemonGeneration; + return Boolean(this.daemonWs && this.authenticated && this.authPromise === null && syncReady); } private sendDirectToDaemon(message: Record): void { @@ -5981,6 +6274,8 @@ export class WsBridge { this.daemonWs = null; this.authenticated = false; this.authPromise = null; + this.upgradeBlockedSyncRequiredGeneration = null; + this.upgradeBlockedSyncCompleteGeneration = null; this.daemonP2pWorkflowCapabilities = null; this.controlledFileTransferCapabilities.clear(); } diff --git a/server/src/ws/daemon-upgrade-coordinator.ts b/server/src/ws/daemon-upgrade-coordinator.ts index 27d0ae1db..3f9eed994 100644 --- a/server/src/ws/daemon-upgrade-coordinator.ts +++ b/server/src/ws/daemon-upgrade-coordinator.ts @@ -17,7 +17,13 @@ const AUTO_UPGRADE_MAX_ATTEMPTS = 3; export type DaemonUpgradeSource = 'auto' | 'manual' | 'replay'; -type UpgradeLifecycleState = 'pending_offline' | 'pending_publication' | 'scheduled' | 'sent' | 'superseded'; +type UpgradeLifecycleState = + | 'pending_offline' + | 'pending_publication' + | 'scheduled' + | 'sent' + | 'terminal_blocked' + | 'superseded'; interface UpgradeState { upgradeId: string; @@ -76,15 +82,39 @@ export class DaemonUpgradeCoordinator { } const now = input.now ?? Date.now(); - const current = this.current?.targetVersion === targetVersion ? this.current : null; + let current = this.current?.targetVersion === targetVersion ? this.current : null; if (this.current && this.current.targetVersion !== targetVersion) { + const terminalTargetChanged = this.current.status === 'terminal_blocked'; this.supersedeCurrent(now); + if (terminalTargetChanged) this.lastAutoSentAt = null; } + if (current?.status === 'terminal_blocked') { + if (input.source !== 'manual') { + return { + ok: true, + upgradeId: current.upgradeId, + targetVersion, + deliveryStatus: DAEMON_UPGRADE_DELIVERY_STATUS.BACKOFF, + reason: 'terminal_install_failure', + }; + } + this.supersedeCurrent(now); + current = null; + } + + // A user-issued manual request is authoritative over reconnect-driven + // auto/replay traffic for the same target. In particular, an offline + // manual request remains pending while auth is incomplete; the auth + // version-mismatch probe must not silently demote it back to `auto`. + const effectiveInput = current?.source === 'manual' && input.source !== 'manual' + ? { ...input, source: 'manual' as const } + : input; + if (current?.status === 'pending_publication') { - current.source = input.source; + current.source = effectiveInput.source; current.updatedAt = now; - if (!input.isDaemonReady()) { + if (!effectiveInput.isDaemonReady()) { current.status = 'pending_offline'; return { ok: true, @@ -93,10 +123,14 @@ export class DaemonUpgradeCoordinator { deliveryStatus: DAEMON_UPGRADE_DELIVERY_STATUS.PENDING_OFFLINE, }; } - const publication = this.ensureTargetPublished(current, input, now); + const publication = this.ensureTargetPublished(current, effectiveInput, now); if (publication) return publication; - } else if (current && current.status !== 'superseded' && (current.status !== 'pending_offline' || !input.isDaemonReady())) { - if (input.source === 'auto') { + } else if ( + current + && current.status !== 'superseded' + && (current.status !== 'pending_offline' || !effectiveInput.isDaemonReady()) + ) { + if (effectiveInput.source === 'auto') { const nextAutoAt = this.nextAutoAttemptAt(now); if (nextAutoAt > now) { return { @@ -129,7 +163,7 @@ export class DaemonUpgradeCoordinator { const state = current ?? { upgradeId: randomUUID(), targetVersion, - source: input.source, + source: effectiveInput.source, status: 'pending_offline' as UpgradeLifecycleState, attempt: 0, createdAt: now, @@ -139,11 +173,11 @@ export class DaemonUpgradeCoordinator { publicationResumeInput: null, publicationCallbackRegistered: false, }; - state.source = input.source; + state.source = effectiveInput.source; state.updatedAt = now; this.current = state; - if (!input.isDaemonReady()) { + if (!effectiveInput.isDaemonReady()) { state.status = 'pending_offline'; return { ok: true, @@ -153,15 +187,15 @@ export class DaemonUpgradeCoordinator { }; } - if (input.source === 'auto') { - const publication = this.ensureTargetPublished(state, input, now); + if (effectiveInput.source === 'auto') { + const publication = this.ensureTargetPublished(state, effectiveInput, now); if (publication) return publication; - return this.scheduleAutoSend(state, input, now); + return this.scheduleAutoSend(state, effectiveInput, now); } - const publication = this.ensureTargetPublished(state, input, now); + const publication = this.ensureTargetPublished(state, effectiveInput, now); if (publication) return publication; - this.sendNow(state, input, now); + this.sendNow(state, effectiveInput, now); return { ok: true, upgradeId: state.upgradeId, @@ -221,6 +255,81 @@ export class DaemonUpgradeCoordinator { }; } + /** + * Cancel any pending send and keep the failed target terminally blocked. + * Auto/replay requests for the same target remain blocked; an explicit manual + * request or a different target version creates a fresh lifecycle. + */ + blockTargetAfterTerminalFailure(targetVersion: unknown, now = Date.now()): boolean { + let normalized: string; + try { + normalized = normalizeDaemonUpgradeTargetVersion(targetVersion); + } catch { + return false; + } + + if (this.current && this.current.targetVersion !== normalized) { + this.supersedeCurrent(now); + } + const state = this.current ?? { + upgradeId: randomUUID(), + targetVersion: normalized, + source: 'auto' as DaemonUpgradeSource, + status: 'terminal_blocked' as UpgradeLifecycleState, + attempt: 0, + createdAt: now, + updatedAt: now, + lastSentAt: null, + timer: null, + publicationResumeInput: null, + publicationCallbackRegistered: false, + }; + if (state.timer) clearTimeout(state.timer); + state.timer = null; + state.status = 'terminal_blocked'; + state.updatedAt = now; + state.publicationResumeInput = null; + state.publicationCallbackRegistered = false; + this.current = state; + return true; + } + + /** + * A terminal failure belongs to the upgrade command that produced it. + * If a newer manual lifecycle for the same target has a different id, the + * old persisted failure was explicitly superseded and must not re-block it. + */ + isTerminalFailureSupersededByManual(targetVersion: unknown, failureUpgradeId: unknown): boolean { + let normalized: string; + try { + normalized = normalizeDaemonUpgradeTargetVersion(targetVersion); + } catch { + return false; + } + return Boolean( + typeof failureUpgradeId === 'string' + && failureUpgradeId.length > 0 + && this.current?.targetVersion === normalized + && this.current.source === 'manual' + && this.current.status !== 'terminal_blocked' + && this.current.upgradeId !== failureUpgradeId, + ); + } + + hasManualLifecycleForTarget(targetVersion: unknown): boolean { + let normalized: string; + try { + normalized = normalizeDaemonUpgradeTargetVersion(targetVersion); + } catch { + return false; + } + return Boolean( + this.current?.targetVersion === normalized + && this.current.source === 'manual' + && this.current.status !== 'terminal_blocked', + ); + } + clearIfTargetVersionMatches(targetVersion: string | null | undefined): void { let normalized: string | null = null; try { diff --git a/server/test/bridge-resync-push-suppression.test.ts b/server/test/bridge-resync-push-suppression.test.ts new file mode 100644 index 000000000..52261af79 --- /dev/null +++ b/server/test/bridge-resync-push-suppression.test.ts @@ -0,0 +1,154 @@ +/** + * A daemon link-restore resync must never raise push notifications. + * + * Observed on the 43 deployment: a gateway blip dropped four daemons' sockets; + * all four re-authenticated within two seconds, and 276ms later ~90 push + * dispatches went out inside a 500ms window — one "Task complete — ready for + * input" per idle session, several quoting work that had finished long before. + * + * `resyncTransportSessionStatesAfterLinkRestore()` re-states every session's + * CURRENT state so browsers stop rendering a stale "working" spinner. Those + * events are re-announcements, not completions. The existing + * `PUSH_TIMELINE_EVENT_MAX_AGE_MS` guard cannot filter them, because a resync + * event is newly minted (ts = now) even though the state it describes is old. + * + * The daemon now stamps `suppressPush`, but this server-side check is what makes + * a server deploy sufficient: daemons are user-installed and upgrade on their own + * schedule, so recognising the resync marker here stops the storm for every + * daemon version already in the field. + * + * @vitest-environment node + */ + +import { EventEmitter } from 'node:events'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { WsBridge } from '../src/ws/bridge.js'; +import { TIMELINE_MESSAGES } from '../../shared/timeline-protocol.js'; +import { TIMELINE_SUPPRESS_PUSH_FIELD } from '../../shared/push-notifications.js'; +import { SESSION_STATE_DECISION_REASON_SERVER_LINK_RESYNC } from '../../shared/session-activity-types.js'; + +vi.mock('../src/security/crypto.js', () => ({ sha256Hex: (_s: string) => 'valid-hash' })); +vi.mock('../src/routes/push.js', () => ({ dispatchPush: vi.fn() })); + +class MockWs extends EventEmitter { + sent: Array = []; + readyState = 1; + send(data: string | Buffer, _opts?: unknown, cb?: (err?: Error) => void): void { + this.sent.push(data); + cb?.(); + } + close(): void { this.readyState = 3; this.emit('close'); } +} + +function makeDb() { + return { + queryOne: async () => ({ token_hash: 'valid-hash', node_role: 'full', revoked_at: null }), + query: async () => [], + execute: async () => ({ changes: 1 }), + exec: async () => {}, + transaction: async (fn: (tx: unknown) => Promise) => fn({}), + close: () => {}, + } as unknown as import('../src/db/client.js').Database; +} + +const SESSION = 'deck_resyncpush_brain'; + +function idleStateEvent(payload: Record): string { + return JSON.stringify({ + type: TIMELINE_MESSAGES.EVENT, + event: { + eventId: `evt-${Math.random().toString(36).slice(2)}`, + sessionId: SESSION, + ts: Date.now(), + seq: 1, + epoch: 1, + type: 'session.state', + payload: { state: 'idle', ...payload }, + }, + }); +} + +async function flush(): Promise { + await new Promise((resolve) => { setImmediate(resolve); }); +} + +/** + * Whether a push is queued for this session. Read synchronously right after the + * daemon message: with the test settle window at 0ms the flush is a microtask, + * so waiting first would drain the map and hide the difference. + */ +function pendingIdlePushCount(bridge: WsBridge): number { + const pending = (bridge as unknown as { pendingIdlePushes: Map }).pendingIdlePushes; + return pending.size; +} + +describe('WsBridge — link-restore resync must not push', () => { + let serverId: string; + + beforeEach(() => { serverId = `resync-${Math.random().toString(36).slice(2)}`; }); + afterEach(() => { WsBridge.getAll().clear(); vi.clearAllMocks(); }); + + async function authedDaemon() { + const bridge = WsBridge.get(serverId); + const daemon = new MockWs(); + // env must be truthy for the push branch to be reachable at all. + bridge.handleDaemonConnection(daemon as never, makeDb() as never, {} as never); + daemon.emit('message', JSON.stringify({ type: 'auth', serverId, token: 'tok' })); + await flush(); + expect(bridge.isAuthenticated).toBe(true); + return { bridge, daemon }; + } + + it('schedules no push for a resync idle even without the daemon-side suppressPush flag', async () => { + // The already-deployed-daemon case: only `decisionReason` identifies it. + const { bridge, daemon } = await authedDaemon(); + + daemon.emit('message', idleStateEvent({ + decisionReason: SESSION_STATE_DECISION_REASON_SERVER_LINK_RESYNC, + })); + + expect(pendingIdlePushCount(bridge)).toBe(0); + }); + + it('schedules no push when the daemon marks the resync with suppressPush', async () => { + const { bridge, daemon } = await authedDaemon(); + + daemon.emit('message', idleStateEvent({ + decisionReason: SESSION_STATE_DECISION_REASON_SERVER_LINK_RESYNC, + [TIMELINE_SUPPRESS_PUSH_FIELD]: true, + })); + + expect(pendingIdlePushCount(bridge)).toBe(0); + }); + + it('STILL pushes for a genuine live idle — the guard must not silence real completions', async () => { + // Without this the fix could "work" by breaking notifications entirely. + const { bridge, daemon } = await authedDaemon(); + + daemon.emit('message', idleStateEvent({ decisionReason: 'activity_reconciler_clear' })); + + expect(pendingIdlePushCount(bridge)).toBe(1); + }); + + it('does not fan one push per session when many sessions resync at once', async () => { + // The storm's actual shape: every session re-stated in one burst. + const { bridge, daemon } = await authedDaemon(); + + for (let i = 0; i < 25; i++) { + daemon.emit('message', JSON.stringify({ + type: TIMELINE_MESSAGES.EVENT, + event: { + eventId: `evt-${i}`, + sessionId: `deck_resyncpush_w${i}`, + ts: Date.now(), + seq: i, + epoch: 1, + type: 'session.state', + payload: { state: 'idle', decisionReason: SESSION_STATE_DECISION_REASON_SERVER_LINK_RESYNC }, + }, + })); + } + + expect(pendingIdlePushCount(bridge)).toBe(0); + }); +}); diff --git a/server/test/bridge-short-ref-health.test.ts b/server/test/bridge-short-ref-health.test.ts new file mode 100644 index 000000000..74d941984 --- /dev/null +++ b/server/test/bridge-short-ref-health.test.ts @@ -0,0 +1,126 @@ +/** + * Memory-handle persistence failures have to reach a person. + * + * The daemon's own two signals cannot: the counter is a process-local map that a + * restart clears, and the throttled warning goes to the daemon log — on the same + * disk whose exhaustion is usually the failure being reported. So the daemon + * puts a sticky health record on its heartbeat instead. + * + * That only helps if the bridge forwards it. It previously rebuilt daemon.stats + * from a fixed field list, so the record arrived at the pod and went no further: + * no UI, no alert, nothing. A daemon-side test cannot catch that — it has to be + * asserted here, where the frame crosses to the browser. + * + * @vitest-environment node + */ + +import { EventEmitter } from 'node:events'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { WsBridge } from '../src/ws/bridge.js'; + +vi.mock('../src/security/crypto.js', () => ({ sha256Hex: (_s: string) => 'valid-hash' })); +vi.mock('../src/routes/push.js', () => ({ dispatchPush: vi.fn() })); + +class MockWs extends EventEmitter { + sent: Array = []; + closed = false; + readyState = 1; + send(data: string | Buffer, _opts?: unknown, callback?: (err?: Error) => void): void { + this.sent.push(data); + callback?.(); + } + close(): void { this.closed = true; this.readyState = 3; this.emit('close'); } + get sentStrings(): string[] { return this.sent.filter((s): s is string => typeof s === 'string'); } +} + +function makeDb() { + return { + queryOne: async () => ({ token_hash: 'valid-hash', node_role: 'full', revoked_at: null }), + query: async () => [], + execute: async () => ({ changes: 1 }), + exec: async () => {}, + transaction: async (fn: (tx: unknown) => Promise) => fn({}), + close: () => {}, + } as unknown as import('../src/db/client.js').Database; +} + +const flushAsync = () => new Promise((resolve) => { setImmediate(resolve); }); + +describe('WsBridge forwards memory-handle persistence health to browsers', () => { + let serverId: string; + + beforeEach(() => { serverId = `test-${Math.random().toString(36).slice(2)}`; }); + afterEach(() => { WsBridge.getAll().clear(); vi.clearAllMocks(); }); + + const shortRefHealth = { + stage: 'persist_store', + failures: 3, + lastFailureAt: 1_700_000_000_000, + lastError: 'ENOSPC: no space left on device', + }; + + const baseStats = { + daemonVersion: '1.2.3', + cpu: 12, memUsed: 1, memTotal: 2, + load1: 0.1, load5: 0.2, load15: 0.3, uptime: 100, + }; + + async function connect() { + const bridge = WsBridge.get(serverId); + const daemon = new MockWs(); + const browser = new MockWs(); + bridge.handleDaemonConnection(daemon as never, makeDb() as never, {} as never); + bridge.handleBrowserConnection(browser as never, { id: 'user-a' } as never); + daemon.emit('message', JSON.stringify({ type: 'auth', serverId, token: 'my-token' })); + await flushAsync(); + return { daemon, browser }; + } + + function statsSeenByBrowser(browser: MockWs): Record | undefined { + for (const raw of browser.sentStrings) { + const parsed = JSON.parse(raw) as Record; + if (parsed.type === 'daemon.stats') return parsed; + } + return undefined; + } + + it('delivers the health record from a daemon.stats frame to the browser', async () => { + const { daemon, browser } = await connect(); + + daemon.emit('message', JSON.stringify({ type: 'daemon.stats', ...baseStats, shortRefHealth })); + await flushAsync(); + + expect(statsSeenByBrowser(browser)).toMatchObject({ type: 'daemon.stats', shortRefHealth }); + }); + + it('delivers it from a heartbeat frame as well, so a reconnect still surfaces it', async () => { + // The daemon repeats the sticky record on every beat; an operator who + // connects after the incident learns about it from the next one. + const { daemon, browser } = await connect(); + + daemon.emit('message', JSON.stringify({ type: 'heartbeat', ...baseStats, shortRefHealth })); + await flushAsync(); + + expect(statsSeenByBrowser(browser)).toMatchObject({ shortRefHealth }); + }); + + it('omits the field entirely while persistence is healthy', async () => { + const { daemon, browser } = await connect(); + + daemon.emit('message', JSON.stringify({ type: 'daemon.stats', ...baseStats })); + await flushAsync(); + + const stats = statsSeenByBrowser(browser); + expect(stats).toBeDefined(); + expect(stats).not.toHaveProperty('shortRefHealth'); + }); + + it('drops a non-object health value rather than forwarding junk', async () => { + const { daemon, browser } = await connect(); + + daemon.emit('message', JSON.stringify({ type: 'daemon.stats', ...baseStats, shortRefHealth: 'disk full' })); + await flushAsync(); + + expect(statsSeenByBrowser(browser)).not.toHaveProperty('shortRefHealth'); + }); +}); diff --git a/server/test/bridge-timeline-drop-telemetry.test.ts b/server/test/bridge-timeline-drop-telemetry.test.ts new file mode 100644 index 000000000..58278843d --- /dev/null +++ b/server/test/bridge-timeline-drop-telemetry.test.ts @@ -0,0 +1,146 @@ +/** + * Bridge-level observability for timeline events that never reach a browser. + * + * `sendJsonToSessionSubscribers` writes only to sockets currently subscribed to + * that session. With no subscriber both loops execute zero bodies, so the event + * is discarded with no queue and no replay. That is the NORMAL state of a + * backgrounded app — which is why "the push notification arrived but the chat was + * empty until I hit force-refresh" reproduced so reliably. + * + * The client now heals that automatically (activation/reconnect request the full + * newest window with no lower bound), so the drop itself has to be counted or a + * rising drop rate would be permanently masked by the recovery. + * + * @vitest-environment node + */ + +import { EventEmitter } from 'node:events'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { WsBridge } from '../src/ws/bridge.js'; +import { TIMELINE_MESSAGES } from '../../shared/timeline-protocol.js'; +import { TIMELINE_DELIVERY_METRICS } from '../../shared/timeline-delivery-telemetry.js'; +import { getCounter, resetMetricsForTests } from '../src/util/metrics.js'; + +vi.mock('../src/security/crypto.js', () => ({ + sha256Hex: (_s: string) => 'valid-hash', +})); + +vi.mock('../src/routes/push.js', () => ({ + dispatchPush: vi.fn(), +})); + +class MockWs extends EventEmitter { + sent: Array = []; + closed = false; + readyState = 1; + + send(data: string | Buffer, _opts?: unknown, callback?: (err?: Error) => void): void { + if (this.closed) { callback?.(new Error('socket closed')); return; } + this.sent.push(data); + callback?.(); + } + + close(): void { + this.closed = true; + this.readyState = 3; + this.emit('close'); + } + + get sentStrings(): string[] { + return this.sent.filter((s): s is string => typeof s === 'string'); + } +} + +function makeDb() { + return { + queryOne: async () => ({ token_hash: 'valid-hash', node_role: 'full', revoked_at: null }), + query: async () => [], + execute: async () => ({ changes: 1 }), + exec: async () => {}, + transaction: async (fn: (tx: unknown) => Promise) => fn({}), + close: () => {}, + } as unknown as import('../src/db/client.js').Database; +} + +async function flushAsync(): Promise { + await new Promise((resolve) => { setImmediate(resolve); }); +} + +const SESSION = 'deck_droptelemetry_brain'; + +function timelineEvent(type: string, text = 'hello'): string { + return JSON.stringify({ + type: TIMELINE_MESSAGES.EVENT, + event: { + eventId: `evt-${Math.random().toString(36).slice(2)}`, + sessionId: SESSION, + ts: Date.now(), + seq: 1, + epoch: 1, + type, + payload: { text }, + }, + }); +} + +describe('WsBridge timeline drop telemetry', () => { + let serverId: string; + + beforeEach(() => { + serverId = `drop-${Math.random().toString(36).slice(2)}`; + resetMetricsForTests(); + }); + + afterEach(() => { + WsBridge.getAll().clear(); + vi.clearAllMocks(); + }); + + async function setupAuthedDaemon() { + const bridge = WsBridge.get(serverId); + const daemon = new MockWs(); + bridge.handleDaemonConnection(daemon as never, makeDb() as never, {} as never); + daemon.emit('message', JSON.stringify({ type: 'auth', serverId, token: 'tok' })); + await flushAsync(); + expect(bridge.isAuthenticated).toBe(true); + return { bridge, daemon }; + } + + it('counts a content-bearing timeline event discarded because nobody is subscribed', async () => { + // Exactly the backgrounded-app case: daemon relays, zero viewers subscribed. + const { daemon } = await setupAuthedDaemon(); + + daemon.emit('message', timelineEvent('assistant.text', 'answer nobody saw')); + await flushAsync(); + + expect(getCounter(TIMELINE_DELIVERY_METRICS.SERVER_NO_SUBSCRIBER_DROPPED, { eventType: 'assistant.text' })) + .toBe(1); + expect(getCounter(TIMELINE_DELIVERY_METRICS.SERVER_DELIVERED, { eventType: 'assistant.text' })) + .toBe(0); + }); + + it('does not count high-frequency status chatter, only content', async () => { + // agent.status fires ~1/s during a turn; counting it would drown the signal. + const { daemon } = await setupAuthedDaemon(); + + daemon.emit('message', timelineEvent('agent.status')); + daemon.emit('message', timelineEvent('usage.update')); + await flushAsync(); + + expect(getCounter(TIMELINE_DELIVERY_METRICS.SERVER_NO_SUBSCRIBER_DROPPED, { eventType: 'agent.status' })) + .toBe(0); + expect(getCounter(TIMELINE_DELIVERY_METRICS.SERVER_NO_SUBSCRIBER_DROPPED, { eventType: 'usage.update' })) + .toBe(0); + }); + + it('exposes a running total so a whole backgrounded window is visible as one number', async () => { + const { bridge, daemon } = await setupAuthedDaemon(); + + daemon.emit('message', timelineEvent('assistant.text', 'one')); + daemon.emit('message', timelineEvent('tool.call', 'two')); + daemon.emit('message', timelineEvent('user.message', 'three')); + await flushAsync(); + + expect(bridge.timelineNoSubscriberDropCount).toBe(3); + }); +}); diff --git a/server/test/bridge.test.ts b/server/test/bridge.test.ts index fa30bf5ba..dc84481ea 100644 --- a/server/test/bridge.test.ts +++ b/server/test/bridge.test.ts @@ -35,6 +35,12 @@ import { TIMELINE_PAYLOAD_BUDGET_BYTES } from '../../shared/timeline-payload-bud import { OPENSPEC_AUTO_DELIVER_MSG } from '../../shared/openspec-auto-deliver-constants.js'; import { EXECUTION_CLONE_KIND } from '../../shared/execution-clone.js'; import { DAEMON_MSG } from '../../shared/daemon-events.js'; +import { + DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION, + DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL, + DAEMON_UPGRADE_BLOCK_REASON, + DAEMON_UPGRADE_DELIVERY_STATUS, +} from '../../shared/daemon-upgrade.js'; import { DAEMON_COMMAND_TYPES } from '../../shared/daemon-command-types.js'; import { PEER_AUDIT_COMMAND_ERRORS, PEER_AUDIT_MESSAGES } from '../../shared/peer-audit.js'; import { @@ -297,6 +303,7 @@ function makeTimelineOwnershipDb(options: { vi.mock('../src/security/crypto.js', () => ({ sha256Hex: (_s: string) => 'valid-hash', + randomHex: (bytes: number) => 'a'.repeat(bytes * 2), })); vi.mock('../src/routes/push.js', () => ({ @@ -585,7 +592,10 @@ describe('WsBridge', () => { expect(ws.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(2); }); - it('does not retry auto daemon.upgrade after non-retryable daemon upgrade blockers', async () => { + it.each([ + 'toolchain_unavailable', + DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED, + ])('does not retry auto daemon.upgrade after non-retryable blocker %s', async (reason) => { vi.useFakeTimers(); process.env.APP_VERSION = '2026.4.905-dev.877'; @@ -600,7 +610,7 @@ describe('WsBridge', () => { expect(ws.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(1); - ws.emit('message', JSON.stringify({ type: DAEMON_MSG.UPGRADE_BLOCKED, reason: 'toolchain_unavailable' })); + ws.emit('message', JSON.stringify({ type: DAEMON_MSG.UPGRADE_BLOCKED, reason })); await flushAsync(); await vi.advanceTimersByTimeAsync(60_000); await flushAsync(); @@ -608,7 +618,334 @@ describe('WsBridge', () => { expect(ws.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(1); }); - it('does not send a stale scheduled daemon.upgrade after the daemon socket is replaced', async () => { + it('cancels the auth-scheduled auto upgrade when a persisted install failure replays', async () => { + vi.useFakeTimers(); + process.env.APP_VERSION = '2026.7.3192-dev.3593'; + markDaemonUpgradeTargetVersionPublishedForTest(process.env.APP_VERSION); + + const bridge = WsBridge.get(serverId); + const ws = new MockWs(); + bridge.handleDaemonConnection(ws as never, makeDb('valid-hash'), {} as never); + + ws.emit('message', JSON.stringify({ + type: 'auth', + serverId, + token: 'my-token', + daemonVersion: '2026.7.3157-dev.3556', + })); + ws.emit('message', JSON.stringify({ + type: DAEMON_MSG.UPGRADE_BLOCKED, + reason: DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED, + failureId: 'failure-after-link-down', + fromVersion: '2026.7.3157-dev.3556', + targetVersion: '2026.7.3192-dev.3593', + retryReason: 'stale-staging-dir', + exitCode: 217, + log: '/tmp/imcodes-upgrade-test/upgrade.log', + ts: Date.now(), + })); + await flushAsync(); + + await vi.advanceTimersByTimeAsync(5_000); + await flushAsync(); + + expect(ws.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(0); + expect(ws.sentStrings.some((raw) => { + const message = JSON.parse(raw) as Record; + return message.type === DAEMON_MSG.UPGRADE_BLOCKED_ACK + && message.failureId === 'failure-after-link-down' + && message.disposition === 'accepted'; + })).toBe(true); + + const autoRetry = bridge.requestDaemonUpgrade({ + targetVersion: process.env.APP_VERSION, + source: 'auto', + }); + expect(autoRetry).toMatchObject({ + deliveryStatus: DAEMON_UPGRADE_DELIVERY_STATUS.BACKOFF, + reason: 'terminal_install_failure', + }); + }); + + it('does not arm auto upgrade while slow auth replay keeps a persisted blocker waiting', async () => { + vi.useFakeTimers(); + process.env.APP_VERSION = '2026.7.3192-dev.3593'; + markDaemonUpgradeTargetVersionPublishedForTest(process.env.APP_VERSION); + + const bridge = WsBridge.get(serverId); + const daemonWs = new MockWs(); + bridge.handleDaemonConnection(daemonWs as never, makeDb('valid-hash'), {} as never); + + const target = { kind: 'main', serverId, sessionName: 'deck_slow_auth_brain' } as const; + const liveCoverage = { + target, + effectiveRole: 'participant', + historyCutoffAt: Date.now() - 1_000, + nextCoverageRecheckAt: null, + coveringShareIds: ['share-slow-auth'], + primaryShareId: 'share-slow-auth', + authorizedAt: Date.now(), + } as const; + bridge.setShareCoverageResolverForTests(async () => liveCoverage); + + const sharedBrowser = new MockWs(); + bridge.handleShareBrowserConnection(sharedBrowser as never, 'shared-user', makeDb('valid-hash'), { + ticketId: 'share-ticket-slow-auth', + target, + snapshot: liveCoverage, + }); + sharedBrowser.emit('message', JSON.stringify({ + type: 'session.send', + commandId: 'slow-auth-command', + sessionName: target.sessionName, + text: 'hold auth replay', + })); + await flushAsync(); + expect(bridge._getInflightCountForTest()).toBe(1); + + let resolveCoverage!: (value: typeof liveCoverage) => void; + const coveragePending = new Promise((resolve) => { + resolveCoverage = resolve; + }); + bridge.setShareCoverageResolverForTests(async () => coveragePending); + + daemonWs.emit('message', JSON.stringify({ + type: 'auth', + serverId, + token: 'my-token', + daemonVersion: '2026.7.3157-dev.3556', + })); + daemonWs.emit('message', JSON.stringify({ + type: DAEMON_MSG.UPGRADE_BLOCKED, + reason: DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED, + failureId: 'failure-during-slow-auth', + fromVersion: '2026.7.3157-dev.3556', + targetVersion: process.env.APP_VERSION, + })); + await flushAsync(); + + await vi.advanceTimersByTimeAsync(5_000); + await flushAsync(); + expect(daemonWs.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(0); + + resolveCoverage(liveCoverage); + await flushAsync(); + await vi.advanceTimersByTimeAsync(5_000); + await flushAsync(); + + expect(daemonWs.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(0); + expect(daemonWs.sentStrings.some((raw) => { + const message = JSON.parse(raw) as Record; + return message.type === DAEMON_MSG.UPGRADE_BLOCKED_ACK + && message.failureId === 'failure-during-slow-auth' + && message.disposition === 'accepted'; + })).toBe(true); + }); + + it('waits for daemon outbox sync even when auth finishes more than five seconds earlier', async () => { + vi.useFakeTimers(); + process.env.APP_VERSION = '2026.7.3192-dev.3593'; + markDaemonUpgradeTargetVersionPublishedForTest(process.env.APP_VERSION); + + const bridge = WsBridge.get(serverId); + const daemonWs = new MockWs(); + bridge.handleDaemonConnection(daemonWs as never, makeDb('valid-hash'), {} as never); + daemonWs.emit('message', JSON.stringify({ + type: 'auth', + serverId, + token: 'my-token', + daemonVersion: '2026.7.3157-dev.3556', + [DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.AUTH_REVISION_FIELD]: + DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION, + })); + await flushAsync(); + + await vi.advanceTimersByTimeAsync(10_000); + await flushAsync(); + expect(daemonWs.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(0); + + daemonWs.emit('message', JSON.stringify({ + type: DAEMON_MSG.UPGRADE_BLOCKED, + reason: DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED, + failureId: 'failure-after-delayed-outbox-read', + upgradeId: 'old-auto-upgrade', + fromVersion: '2026.7.3157-dev.3556', + targetVersion: process.env.APP_VERSION, + })); + daemonWs.emit('message', JSON.stringify({ + type: DAEMON_MSG.UPGRADE_BLOCKED_SYNC, + revision: DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION, + })); + await vi.advanceTimersByTimeAsync(0); + await flushAsync(); + await vi.advanceTimersByTimeAsync(5_000); + await flushAsync(); + + expect(daemonWs.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(0); + expect(daemonWs.sentStrings.map((raw) => JSON.parse(raw))).toContainEqual({ + type: DAEMON_MSG.UPGRADE_BLOCKED_ACK, + failureId: 'failure-after-delayed-outbox-read', + disposition: DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.ACCEPTED, + }); + }); + + it('starts a normal auto upgrade only after an empty outbox sync completes', async () => { + vi.useFakeTimers(); + process.env.APP_VERSION = '2026.7.3192-dev.3593'; + markDaemonUpgradeTargetVersionPublishedForTest(process.env.APP_VERSION); + + const bridge = WsBridge.get(serverId); + const daemonWs = new MockWs(); + bridge.handleDaemonConnection(daemonWs as never, makeDb('valid-hash'), {} as never); + daemonWs.emit('message', JSON.stringify({ + type: 'auth', + serverId, + token: 'my-token', + daemonVersion: '2026.7.3157-dev.3556', + [DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.AUTH_REVISION_FIELD]: + DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION, + })); + await flushAsync(); + await vi.advanceTimersByTimeAsync(5_000); + await flushAsync(); + expect(daemonWs.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(0); + + daemonWs.emit('message', JSON.stringify({ + type: DAEMON_MSG.UPGRADE_BLOCKED_SYNC, + revision: DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION, + })); + await vi.advanceTimersByTimeAsync(0); + await flushAsync(); + await vi.advanceTimersByTimeAsync(5_000); + await flushAsync(); + + expect(daemonWs.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(1); + }); + + it('keeps an offline manual override ahead of reconnect auto and supersedes the retained old blocker', async () => { + vi.useFakeTimers(); + process.env.APP_VERSION = '2026.7.3192-dev.3593'; + markDaemonUpgradeTargetVersionPublishedForTest(process.env.APP_VERSION); + + const bridge = WsBridge.get(serverId); + const firstWs = new MockWs(); + bridge.handleDaemonConnection(firstWs as never, makeDb('valid-hash'), {} as never); + firstWs.emit('message', JSON.stringify({ + type: 'auth', + serverId, + token: 'my-token', + daemonVersion: '2026.7.3157-dev.3556', + })); + await flushAsync(); + await vi.advanceTimersByTimeAsync(5_000); + await flushAsync(); + + const firstUpgrade = firstWs.sentStrings + .map((raw) => JSON.parse(raw) as Record) + .find((message) => message.type === DAEMON_COMMAND_TYPES.DAEMON_UPGRADE); + expect(firstUpgrade?.upgradeId).toEqual(expect.any(String)); + + firstWs.emit('message', JSON.stringify({ + type: DAEMON_MSG.UPGRADE_BLOCKED, + reason: DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED, + failureId: 'failure-before-manual-override', + upgradeId: firstUpgrade?.upgradeId, + fromVersion: '2026.7.3157-dev.3556', + targetVersion: process.env.APP_VERSION, + })); + await flushAsync(); + firstWs.emit('close'); + + const manual = bridge.requestDaemonUpgrade({ + targetVersion: process.env.APP_VERSION, + source: 'manual', + }); + expect(manual.deliveryStatus).toBe(DAEMON_UPGRADE_DELIVERY_STATUS.PENDING_OFFLINE); + expect(manual.upgradeId).not.toBe(firstUpgrade?.upgradeId); + + // Move beyond the auto retry interval so this regression cannot pass + // merely because the previous auto attempt is still rate-limited. The + // reconnect auto request must preserve manual authority on its own. + await vi.advanceTimersByTimeAsync(15 * 60 * 1000 + 1); + + const reconnectWs = new MockWs(); + bridge.handleDaemonConnection(reconnectWs as never, makeDb('valid-hash'), {} as never); + reconnectWs.emit('message', JSON.stringify({ + type: 'auth', + serverId, + token: 'my-token', + daemonVersion: '2026.7.3157-dev.3556', + [DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.AUTH_REVISION_FIELD]: + DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION, + })); + reconnectWs.emit('message', JSON.stringify({ + type: DAEMON_MSG.UPGRADE_BLOCKED, + reason: DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED, + failureId: 'failure-before-manual-override', + upgradeId: firstUpgrade?.upgradeId, + fromVersion: '2026.7.3157-dev.3556', + targetVersion: process.env.APP_VERSION, + })); + reconnectWs.emit('message', JSON.stringify({ + type: DAEMON_MSG.UPGRADE_BLOCKED_SYNC, + revision: DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION, + })); + await vi.advanceTimersByTimeAsync(0); + await flushAsync(); + + const reconnectMessages = reconnectWs.sentStrings.map((raw) => JSON.parse(raw) as Record); + expect(reconnectMessages.filter((message) => message.type === DAEMON_COMMAND_TYPES.DAEMON_UPGRADE)).toEqual([{ + type: DAEMON_COMMAND_TYPES.DAEMON_UPGRADE, + upgradeId: manual.upgradeId, + targetVersion: process.env.APP_VERSION, + }]); + expect(reconnectMessages).toContainEqual({ + type: DAEMON_MSG.UPGRADE_BLOCKED_ACK, + failureId: 'failure-before-manual-override', + disposition: DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.SUPERSEDED, + }); + + await vi.advanceTimersByTimeAsync(5_000); + await flushAsync(); + expect(reconnectWs.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(1); + }); + + it('acks an old-target install failure as obsolete without blocking the new target', async () => { + vi.useFakeTimers(); + process.env.APP_VERSION = '2026.7.3193-dev.3594'; + markDaemonUpgradeTargetVersionPublishedForTest(process.env.APP_VERSION); + + const bridge = WsBridge.get(serverId); + const ws = new MockWs(); + bridge.handleDaemonConnection(ws as never, makeDb('valid-hash'), {} as never); + + ws.emit('message', JSON.stringify({ + type: 'auth', + serverId, + token: 'my-token', + daemonVersion: '2026.7.3157-dev.3556', + })); + ws.emit('message', JSON.stringify({ + type: DAEMON_MSG.UPGRADE_BLOCKED, + reason: DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED, + failureId: 'failure-for-old-target', + fromVersion: '2026.7.3157-dev.3556', + targetVersion: '2026.7.3192-dev.3593', + })); + await flushAsync(); + await vi.advanceTimersByTimeAsync(5_000); + await flushAsync(); + + expect(ws.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(1); + expect(ws.sentStrings.some((raw) => { + const message = JSON.parse(raw) as Record; + return message.type === DAEMON_MSG.UPGRADE_BLOCKED_ACK + && message.failureId === 'failure-for-old-target' + && message.disposition === 'obsolete'; + })).toBe(true); + }); + + it('sends a deferred daemon.upgrade only to the replacement socket and ignores stale frames', async () => { vi.useFakeTimers(); process.env.APP_VERSION = '2026.4.905-dev.877'; @@ -626,7 +963,287 @@ describe('WsBridge', () => { await flushAsync(); expect(staleWs.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(0); + expect(replacementWs.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(1); + + const rawBrowser = new MockWs(); + bridge.handleBrowserConnection(rawBrowser as never, 'raw-user', makeDb('valid-hash')); + rawBrowser.emit('message', JSON.stringify({ + type: 'terminal.subscribe', + session: 'deck_stale_socket_binary', + raw: true, + })); + await flushAsync(); + rawBrowser.sent.length = 0; + + // ws can emit already-buffered frames while the replaced socket is still + // CLOSING. Neither binary data nor a terminal blocker from generation 1 + // may borrow generation 2's authenticated bridge state. + staleWs.emit('message', packFrame('deck_stale_socket_binary', Buffer.from('stale')), true); + staleWs.emit('message', JSON.stringify({ + type: DAEMON_MSG.UPGRADE_BLOCKED, + reason: DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED, + failureId: 'failure-from-replaced-socket', + fromVersion: '2026.4.904-dev.100', + targetVersion: process.env.APP_VERSION, + })); + await flushAsync(); + expect.soft(rawBrowser.sent.filter((item) => Buffer.isBuffer(item))).toHaveLength(0); + + await vi.advanceTimersByTimeAsync(15 * 60 * 1000 + 1); + const nextAuto = bridge.requestDaemonUpgrade({ + targetVersion: process.env.APP_VERSION, + source: 'auto', + }); + expect.soft(nextAuto.deliveryStatus).toBe(DAEMON_UPGRADE_DELIVERY_STATUS.SENT); + }); + + it('does not let an error from a replaced socket reject current-generation requests', async () => { + const bridge = WsBridge.get(serverId); + const staleWs = new MockWs(); + bridge.handleDaemonConnection(staleWs as never, makeDb('valid-hash'), {} as never); + staleWs.emit('message', JSON.stringify({ type: 'auth', serverId, token: 'my-token' })); + await flushAsync(); + + const replacementWs = new MockWs(); + bridge.handleDaemonConnection(replacementWs as never, makeDb('valid-hash'), {} as never); + replacementWs.emit('message', JSON.stringify({ type: 'auth', serverId, token: 'my-token' })); + await flushAsync(); + + const pending = bridge.requestTimelineHistory({ + sessionName: 'deck_replacement_request', + timeoutMs: 10_000, + }); + const outbound = replacementWs.sentStrings.find((raw) => raw.includes('"type":"timeline.history_request"')); + expect(outbound).toBeTruthy(); + const requestId = JSON.parse(outbound!).requestId as string; + let settled = false; + const observed = pending.then( + (value) => { + settled = true; + return { status: 'resolved' as const, value }; + }, + (error: unknown) => { + settled = true; + return { status: 'rejected' as const, error }; + }, + ); + + staleWs.emit('error', new Error('stale socket error')); + await flushAsync(); + expect(settled).toBe(false); + + replacementWs.emit('message', JSON.stringify({ + type: 'timeline.history', + sessionName: 'deck_replacement_request', + requestId, + events: [], + epoch: 1, + })); + await expect(observed).resolves.toMatchObject({ + status: 'resolved', + value: { + type: 'timeline.history', + requestId, + epoch: 1, + }, + }); + }); + + it('does not let a stale in-flight auth certify or dispatch through its replacement connection', async () => { + vi.useFakeTimers(); + process.env.APP_VERSION = '2026.7.3192-dev.3593'; + markDaemonUpgradeTargetVersionPublishedForTest(process.env.APP_VERSION); + + type AuthRow = { + token_hash: string; + node_role: 'full'; + revoked_at: null; + }; + const makeDeferredAuthDb = () => { + let resolveQuery!: (value: AuthRow) => void; + const queryPromise = new Promise((resolve) => { + resolveQuery = resolve; + }); + const db = { + queryOne: () => queryPromise, + query: async () => [], + execute: async () => ({ changes: 1 }), + exec: async () => {}, + transaction: async (fn: (tx: import('../src/db/client.js').Database) => Promise) => + fn(db as unknown as import('../src/db/client.js').Database), + close: () => {}, + } as unknown as import('../src/db/client.js').Database; + return { db, resolveQuery }; + }; + + const bridge = WsBridge.get(serverId); + const manual = bridge.requestDaemonUpgrade({ + targetVersion: process.env.APP_VERSION, + source: 'manual', + }); + expect(manual.deliveryStatus).toBe(DAEMON_UPGRADE_DELIVERY_STATUS.PENDING_OFFLINE); + + const firstAuth = makeDeferredAuthDb(); + const firstWs = new MockWs(); + bridge.handleDaemonConnection(firstWs as never, firstAuth.db, {} as never); + firstWs.emit('message', JSON.stringify({ + type: 'auth', + serverId, + token: 'my-token', + daemonVersion: '2026.7.3157-dev.3556', + [DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.AUTH_REVISION_FIELD]: + DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION, + })); + await flushAsync(); + + const replacementAuth = makeDeferredAuthDb(); + const replacementWs = new MockWs(); + bridge.handleDaemonConnection(replacementWs as never, replacementAuth.db, {} as never); + replacementWs.emit('message', JSON.stringify({ + type: 'auth', + serverId, + token: 'my-token', + daemonVersion: '2026.7.3157-dev.3556', + [DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.AUTH_REVISION_FIELD]: + DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION, + })); + await flushAsync(); + + // Finish generation 1 after generation 2 has installed its own socket + // and auth promise. The stale continuation must not authenticate gen 2, + // clear its promise, or rewrite its blocker-sync generation. + firstAuth.resolveQuery({ + token_hash: 'valid-hash', + node_role: 'full', + revoked_at: null, + }); + await flushAsync(); + expect(bridge.isAuthenticated).toBe(false); expect(replacementWs.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(0); + + // Generation 2 may authenticate, but remains unable to dispatch the + // pending manual upgrade until its own persisted-blocker sync arrives. + replacementAuth.resolveQuery({ + token_hash: 'valid-hash', + node_role: 'full', + revoked_at: null, + }); + await flushAsync(); + expect(bridge.isAuthenticated).toBe(true); + await vi.advanceTimersByTimeAsync(10_000); + await flushAsync(); + expect(replacementWs.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(0); + + replacementWs.emit('message', JSON.stringify({ + type: DAEMON_MSG.UPGRADE_BLOCKED_SYNC, + revision: DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION, + })); + await vi.advanceTimersByTimeAsync(0); + await flushAsync(); + + const upgrades = replacementWs.sentStrings + .map((raw) => JSON.parse(raw) as Record) + .filter((message) => message.type === DAEMON_COMMAND_TYPES.DAEMON_UPGRADE); + expect(upgrades).toEqual([{ + type: DAEMON_COMMAND_TYPES.DAEMON_UPGRADE, + upgradeId: manual.upgradeId, + targetVersion: process.env.APP_VERSION, + }]); + expect(firstWs.sentStrings.filter((msg) => msg.includes('"type":"daemon.upgrade"'))).toHaveLength(0); + }); + + it('does not replay an inflight command through a replacement while stale auth revalidation settles', async () => { + const bridge = WsBridge.get(serverId); + const firstWs = new MockWs(); + bridge.handleDaemonConnection(firstWs as never, makeDb('valid-hash'), {} as never); + + const target = { kind: 'main', serverId, sessionName: 'deck_stale_auth_replay_brain' } as const; + const liveCoverage = { + target, + effectiveRole: 'participant', + historyCutoffAt: Date.now() - 1_000, + nextCoverageRecheckAt: null, + coveringShareIds: ['share-stale-auth-replay'], + primaryShareId: 'share-stale-auth-replay', + authorizedAt: Date.now(), + } as const; + bridge.setShareCoverageResolverForTests(async () => liveCoverage); + const sharedBrowser = new MockWs(); + bridge.handleShareBrowserConnection(sharedBrowser as never, 'shared-user', makeDb('valid-hash'), { + ticketId: 'share-ticket-stale-auth-replay', + target, + snapshot: liveCoverage, + }); + sharedBrowser.emit('message', JSON.stringify({ + type: 'session.send', + commandId: 'stale-auth-replay-command', + sessionName: target.sessionName, + text: 'must reach only the authenticated replacement', + })); + await flushAsync(); + expect(bridge._getInflightCountForTest()).toBe(1); + + let resolveCoverage!: (value: typeof liveCoverage) => void; + const coveragePending = new Promise((resolve) => { + resolveCoverage = resolve; + }); + bridge.setShareCoverageResolverForTests(async () => coveragePending); + + firstWs.emit('message', JSON.stringify({ + type: 'auth', + serverId, + token: 'my-token', + })); + await flushAsync(); + + let resolveReplacementAuth!: (value: { + token_hash: string; + node_role: 'full'; + revoked_at: null; + }) => void; + const replacementQuery = new Promise<{ + token_hash: string; + node_role: 'full'; + revoked_at: null; + }>((resolve) => { + resolveReplacementAuth = resolve; + }); + const replacementDb = { + queryOne: () => replacementQuery, + query: async () => [], + execute: async () => ({ changes: 1 }), + exec: async () => {}, + transaction: async (fn: (tx: import('../src/db/client.js').Database) => Promise) => + fn(replacementDb as unknown as import('../src/db/client.js').Database), + close: () => {}, + } as unknown as import('../src/db/client.js').Database; + const replacementWs = new MockWs(); + bridge.handleDaemonConnection(replacementWs as never, replacementDb, {} as never); + replacementWs.emit('message', JSON.stringify({ + type: 'auth', + serverId, + token: 'my-token', + })); + await flushAsync(); + + // Let generation 1's share revalidation finish after generation 2 owns + // the bridge. Its auth flow must leave the entry buffered, not send it + // through generation 2 before that socket authenticates. + resolveCoverage(liveCoverage); + await flushAsync(); + expect(bridge.isAuthenticated).toBe(false); + expect(replacementWs.sentStrings.filter((raw) => raw.includes('"type":"session.send"'))).toHaveLength(0); + + resolveReplacementAuth({ + token_hash: 'valid-hash', + node_role: 'full', + revoked_at: null, + }); + await flushAsync(); + + expect(bridge.isAuthenticated).toBe(true); + expect(replacementWs.sentStrings.filter((raw) => raw.includes('"type":"session.send"'))).toHaveLength(1); + expect(firstWs.sentStrings.filter((raw) => raw.includes('"type":"session.send"'))).toHaveLength(0); }); }); @@ -1009,7 +1626,7 @@ describe('WsBridge', () => { const daemonWs = new MockWs(); bridge.handleDaemonConnection(daemonWs as never, makeDb('valid-hash'), {} as never); daemonWs.emit('message', JSON.stringify({ type: 'auth', serverId, token: 't' })); - await flushAsync(); + await flushOneBridgeDataPlaneTurn(); const upgradeMessages = daemonWs.sentStrings.filter((s) => s.includes('"type":"daemon.upgrade"')); expect(upgradeMessages).toHaveLength(1); diff --git a/server/test/daemon-upgrade-coordinator.test.ts b/server/test/daemon-upgrade-coordinator.test.ts index 075e3c69a..5b6023547 100644 --- a/server/test/daemon-upgrade-coordinator.test.ts +++ b/server/test/daemon-upgrade-coordinator.test.ts @@ -180,4 +180,117 @@ describe('DaemonUpgradeCoordinator npm publication gate', () => { await flushPromises(); expect(sent).toHaveLength(2); }); + + it('cancels a scheduled auto send and terminally blocks only that target until manual override', async () => { + vi.useFakeTimers(); + const coordinator = new DaemonUpgradeCoordinator(); + const sent: Record[] = []; + const targetVersion = '2026.7.3192-dev.3593'; + const request = { + targetVersion, + source: 'auto' as const, + skipPublicationGate: true, + isDaemonReady: () => true, + isStillCurrent: () => true, + send: (message: Record) => sent.push(message), + now: 0, + }; + + expect(coordinator.request(request).deliveryStatus).toBe(DAEMON_UPGRADE_DELIVERY_STATUS.SENT); + expect(coordinator.blockTargetAfterTerminalFailure(targetVersion, 1)).toBe(true); + + await vi.advanceTimersByTimeAsync(5_000); + await flushPromises(); + expect(sent).toEqual([]); + expect(coordinator.request({ ...request, now: 5_001 })).toMatchObject({ + deliveryStatus: DAEMON_UPGRADE_DELIVERY_STATUS.BACKOFF, + reason: 'terminal_install_failure', + }); + + expect(coordinator.request({ + ...request, + source: 'manual', + now: 5_002, + }).deliveryStatus).toBe(DAEMON_UPGRADE_DELIVERY_STATUS.SENT); + expect(sent).toHaveLength(1); + }); + + it('keeps an offline manual lifecycle authoritative over reconnect auto traffic', () => { + const coordinator = new DaemonUpgradeCoordinator(); + const sent: Record[] = []; + const targetVersion = '2026.7.3192-dev.3593'; + let ready = false; + const base = { + targetVersion, + skipPublicationGate: true, + isDaemonReady: () => ready, + isStillCurrent: () => true, + send: (message: Record) => sent.push(message), + }; + + const manual = coordinator.request({ ...base, source: 'manual' }); + expect(manual.deliveryStatus).toBe(DAEMON_UPGRADE_DELIVERY_STATUS.PENDING_OFFLINE); + + const authAuto = coordinator.request({ ...base, source: 'auto' }); + expect(authAuto).toMatchObject({ + upgradeId: manual.upgradeId, + deliveryStatus: DAEMON_UPGRADE_DELIVERY_STATUS.ALREADY_IN_PROGRESS, + }); + + ready = true; + const flushed = coordinator.flushPending({ + skipPublicationGate: true, + isDaemonReady: () => ready, + isStillCurrent: () => true, + send: (message) => sent.push(message), + }); + expect(flushed).toMatchObject({ + upgradeId: manual.upgradeId, + deliveryStatus: DAEMON_UPGRADE_DELIVERY_STATUS.SENT, + }); + expect(sent).toEqual([{ + type: DAEMON_COMMAND_TYPES.DAEMON_UPGRADE, + upgradeId: manual.upgradeId, + targetVersion, + }]); + }); + + it('identifies an older failure superseded by a newer manual lifecycle', () => { + const coordinator = new DaemonUpgradeCoordinator(); + const targetVersion = '2026.7.3192-dev.3593'; + coordinator.blockTargetAfterTerminalFailure(targetVersion, 1); + const manual = coordinator.request({ + targetVersion, + source: 'manual', + skipPublicationGate: true, + isDaemonReady: () => false, + send: () => {}, + now: 2, + }); + + expect(coordinator.isTerminalFailureSupersededByManual(targetVersion, 'older-upgrade')).toBe(true); + expect(coordinator.isTerminalFailureSupersededByManual(targetVersion, manual.upgradeId)).toBe(false); + expect(coordinator.hasManualLifecycleForTarget(targetVersion)).toBe(true); + }); + + it('allows a fresh auto lifecycle when the requested target version changes', async () => { + vi.useFakeTimers(); + const coordinator = new DaemonUpgradeCoordinator(); + const sent: Record[] = []; + coordinator.blockTargetAfterTerminalFailure('2026.7.3192-dev.3593', 0); + + expect(coordinator.request({ + targetVersion: '2026.7.3193-dev.3594', + source: 'auto', + skipPublicationGate: true, + isDaemonReady: () => true, + isStillCurrent: () => true, + send: (message) => sent.push(message), + now: 1, + }).deliveryStatus).toBe(DAEMON_UPGRADE_DELIVERY_STATUS.SENT); + + await vi.advanceTimersByTimeAsync(5_000); + await flushPromises(); + expect(sent).toHaveLength(1); + }); }); diff --git a/shared/context-store-rpc.ts b/shared/context-store-rpc.ts index f7a379d25..e4cbb05b9 100644 --- a/shared/context-store-rpc.ts +++ b/shared/context-store-rpc.ts @@ -39,6 +39,10 @@ export type ContextStoreRpcPriority = // ── L1: allowlisted direct store wrappers (each maps 1:1 to a context-store.ts export) ── export const CONTEXT_STORE_L1_OPS = [ + // memory short-ref handle map (ref → id), durable across daemon restarts + 'listMemoryShortRefs', + 'listMemoryShortRefsByRef', + 'upsertMemoryShortRefs', // reads 'getProcessedProjectionById', 'listProcessedProjections', diff --git a/shared/context-types.ts b/shared/context-types.ts index 48c571bce..9751b9494 100644 --- a/shared/context-types.ts +++ b/shared/context-types.ts @@ -305,6 +305,8 @@ export interface ContextMemoryProjectView { export interface ContextMemoryRecordView { id: string; + /** Daemon-issued, persisted projection handle when this record is addressable. */ + ref?: string; scope: MemoryScope; projectId: string; ownerUserId?: string; diff --git a/shared/daemon-events.ts b/shared/daemon-events.ts index 5a0dc1bce..a48d209b4 100644 --- a/shared/daemon-events.ts +++ b/shared/daemon-events.ts @@ -2,6 +2,8 @@ export const DAEMON_MSG = { RECONNECTED: 'daemon.reconnected', DISCONNECTED: 'daemon.disconnected', UPGRADE_BLOCKED: 'daemon.upgrade_blocked', + UPGRADE_BLOCKED_ACK: 'daemon.upgrade_blocked_ack', + UPGRADE_BLOCKED_SYNC: 'daemon.upgrade_blocked_sync', // Emitted by the daemon right after it spawns the (detached) upgrade script, // just before the running process is killed & restarted. The server relays it // to browsers so the UI can show an "upgrading…" state next to the daemon diff --git a/shared/daemon-upgrade.ts b/shared/daemon-upgrade.ts index a56ab8214..cc5ccdadc 100644 --- a/shared/daemon-upgrade.ts +++ b/shared/daemon-upgrade.ts @@ -1,5 +1,23 @@ export const DAEMON_UPGRADE_TARGET_LATEST = 'latest'; +export const DAEMON_UPGRADE_BLOCK_REASON = { + INSTALL_FAILED: 'install_failed', +} as const; + +export const DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL = { + AUTH_REVISION_FIELD: 'upgradeBlockedSyncRevision', + REVISION: 1, +} as const; + +export const DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION = { + ACCEPTED: 'accepted', + OBSOLETE: 'obsolete', + SUPERSEDED: 'superseded', +} as const; + +export type DaemonUpgradeBlockedAckDisposition = + (typeof DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION)[keyof typeof DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION]; + export const DAEMON_UPGRADE_DELIVERY_STATUS = { SENT: 'sent', PENDING_OFFLINE: 'pending_offline', diff --git a/shared/disk-usage.ts b/shared/disk-usage.ts new file mode 100644 index 000000000..0a3874f90 --- /dev/null +++ b/shared/disk-usage.ts @@ -0,0 +1,16 @@ +/** + * A mounted filesystem's capacity, reported by the daemon inside `daemon.stats` + * for the desktop status bar. Shared between the daemon (producer) and web + * (consumer). Mobile clients ignore it; the desktop UI renders up to two inline + * and spills the rest into a tooltip. + */ +export interface DiskUsage { + /** Mount point on POSIX (e.g. "/", "/data") or drive root on Windows ("C:\\"). */ + mount: string; + /** Total filesystem size in bytes. */ + totalBytes: number; + /** Used bytes (total − free, includes reserved blocks). */ + usedBytes: number; + /** used/total as a whole-number percentage 0–100. */ + usedPercent: number; +} diff --git a/shared/memory-counters.ts b/shared/memory-counters.ts index 7b792825b..8391f149a 100644 --- a/shared/memory-counters.ts +++ b/shared/memory-counters.ts @@ -62,6 +62,13 @@ export const MEMORY_COUNTERS = [ 'mem.compression.admission_closed', 'mem.pinned_notes_overflow', 'mem.telemetry.buffer_overflow', + // Compact memory handles (ref → id). A handle that fails to persist, or a row + // discarded while loading, still resolves in-process but dies on the next + // restart — these have to be collectable, not just counted in memory. + 'mem.short_ref.persist_failure', + 'mem.short_ref.collision', + 'mem.short_ref.legacy_import', + 'mem.short_ref.discarded_row', ] as const; export type MemoryCounter = (typeof MEMORY_COUNTERS)[number]; diff --git a/shared/memory-mcp-contracts.ts b/shared/memory-mcp-contracts.ts index 181ae6112..9a434b01a 100644 --- a/shared/memory-mcp-contracts.ts +++ b/shared/memory-mcp-contracts.ts @@ -272,7 +272,7 @@ export const MEMORY_MCP_TOOL_CONTRACTS: Readonly(namespace: T): T { + if (namespace.userId?.trim()) return namespace; + if (namespace.scope === 'personal' || namespace.scope === 'user_private') { + return { ...namespace, userId: LEGACY_DAEMON_LOCAL_USER_ID }; + } + return namespace; +} + export type MemoryNamespaceVisibility = 'owner_private' | 'shared_authorized'; export interface MemoryNamespaceInput { diff --git a/shared/memory-recall-format.ts b/shared/memory-recall-format.ts index f8df802a3..9f5000810 100644 --- a/shared/memory-recall-format.ts +++ b/shared/memory-recall-format.ts @@ -3,6 +3,10 @@ import type { ProcessedContextClass, ProcessedContextProjectionStatus } from './ export interface RelatedPastWorkRenderableItem { projectId: string; summary: string; + /** Compact handle (e.g. `proj:k3m7q2xw9pz4a`) the agent can redeem via + * get_memory_sources when a summary alone isn't enough. Omitted for records + * that aren't addressable (raw events). */ + ref?: string; projectionClass?: ProcessedContextClass; hitCount?: number; lastUsedAt?: number; @@ -14,22 +18,49 @@ export const RELATED_PAST_WORK_HEADER = '[Related past work]'; export const STARTUP_PROJECT_MEMORY_HEADER = '# Recent project memory (reference only)'; export const STARTUP_SKILL_INDEX_HEADER = '# Available skills (read on demand)'; +const MARKDOWN_HEADING = /^\s{0,3}#{1,6}\s+/; + +/** + * Condense a stored summary into one injected line. + * + * Taking `split('\n')[0]` verbatim produced lines that were nothing but a + * markdown heading — a whole recall block reading `- [recent] ## Problem` tells + * the agent nothing about whether the memory is relevant. Keep the heading as a + * label and pull the first real content line up next to it, so the line reads + * `Problem: uploads fail when the disk is full`. + */ export function formatRelatedPastWorkSummary(summary: string, maxLength = 200): string { - return summary.split('\n')[0]?.slice(0, maxLength) ?? ''; + const lines = summary.split('\n').map((line) => line.trim()).filter(Boolean); + const firstLine = lines[0]; + if (!firstLine) return ''; + let label = ''; + let rest = lines; + if (MARKDOWN_HEADING.test(firstLine)) { + label = firstLine.replace(MARKDOWN_HEADING, '').replace(/\s*:\s*$/, ''); + rest = lines.slice(1); + } + const body = rest.find((line) => !MARKDOWN_HEADING.test(line)) ?? ''; + const text = label && body ? `${label}: ${body}` : (body || label || firstLine); + return text.slice(0, maxLength); +} + +/** `(proj:abc…) ` prefix, or '' when the record has no redeemable handle. */ +function formatRefPrefix(ref: string | undefined): string { + return ref ? `(${ref}) ` : ''; } -export function formatRelatedPastWorkLine(item: Pick): string { - return `- [${item.projectId}] ${formatRelatedPastWorkSummary(item.summary)}`; +export function formatRelatedPastWorkLine(item: Pick): string { + return `- [${item.projectId}] ${formatRefPrefix(item.ref)}${formatRelatedPastWorkSummary(item.summary)}`; } -export function buildRelatedPastWorkText(items: ReadonlyArray>): string { +export function buildRelatedPastWorkText(items: ReadonlyArray>): string { return `${RELATED_PAST_WORK_HEADER}\n\n${items.map((item) => formatRelatedPastWorkLine(item)).join('\n')}\n`; } -export function buildStartupProjectMemoryText(items: ReadonlyArray>): string { +export function buildStartupProjectMemoryText(items: ReadonlyArray>): string { const lines = items.map((item) => { const label = item.projectionClass === 'durable_memory_candidate' ? 'important' : 'recent'; - return `- [${label}] ${formatRelatedPastWorkSummary(item.summary, 300)}`; + return `- [${label}] ${formatRefPrefix(item.ref)}${formatRelatedPastWorkSummary(item.summary, 300)}`; }); return `${STARTUP_PROJECT_MEMORY_HEADER}\n\n${lines.join('\n')}\n`; } diff --git a/shared/memory-short-ref-health.ts b/shared/memory-short-ref-health.ts new file mode 100644 index 000000000..55ac6824f --- /dev/null +++ b/shared/memory-short-ref-health.ts @@ -0,0 +1,26 @@ +/** + * Persistence health for the compact memory handles, reported on the daemon's + * control-plane heartbeat. + * + * A handle that fails to persist still resolves in the running process but dies + * on the next restart. The counters and the throttled warning that report it + * both stay inside the daemon: counters live in a process-local map, and the + * warning ends up in the daemon log — on the same disk whose exhaustion is the + * failure being reported, with write errors swallowed. In the original + * disk-full incident neither signal could leave the machine. + * + * This travels the WebSocket instead, and is sticky rather than edge-triggered, + * so it is still visible to whoever reconnects after the fact. + */ +export interface MemoryShortRefHealth { + /** Where the last failure happened (persist_store, persist_file, warm_load, hydrate_ref, load_file). */ + stage: string; + /** Failures since the process started; keeps a single blip distinguishable from a stuck disk. */ + failures: number; + /** Epoch ms of the most recent failure. */ + lastFailureAt: number; + /** Message of the most recent failure, truncated for the wire. */ + lastError: string; +} + +export const MEMORY_SHORT_REF_HEALTH_ERROR_MAX_CHARS = 200; diff --git a/shared/metrics.ts b/shared/metrics.ts new file mode 100644 index 000000000..adfd7fd63 --- /dev/null +++ b/shared/metrics.ts @@ -0,0 +1,71 @@ +/** + * In-process counters. THE single implementation — daemon and server both + * re-export this module from their own `util/metrics.ts`. + * + * These are NOT persisted. The store is a plain in-memory Map, so every counter + * resets when the process restarts, and on the server each replica counts only + * the traffic that hit that pod. Anything that needs a durable or cluster-wide + * number has to be written to the database explicitly — do not assume these + * survive a deploy. + * + * Previously this existed as two near-identical copies (`src/util/metrics.ts` and + * `server/src/util/metrics.ts`) which had already drifted: only the server copy + * had `addCounter`. That is exactly the duplication the repo rules forbid, so the + * two files are now thin re-exports of this one. + */ + +export type MetricLabels = Record; + +const counters = new Map(); + +/** + * Cardinality guard, not a capacity target. + * + * Each distinct name+labels combination is one entry, so a caller that labels by + * something unbounded (a session id, a file path) would otherwise grow the Map + * forever. Past the cap, NEW keys are dropped while existing ones keep counting — + * so a hit means later metrics silently go missing, which is worth noticing. + */ +const MAX_COUNTERS = 10_000; + +function labelsKey(labels?: MetricLabels): string { + if (!labels) return ''; + const entries = Object.entries(labels) + .filter(([, value]) => typeof value === 'string') + .sort(([a], [b]) => a.localeCompare(b)); + return entries.map(([key, value]) => `${key}=${value}`).join(','); +} + +function counterKey(name: string, labels?: MetricLabels): string { + const suffix = labelsKey(labels); + return suffix ? `${name}{${suffix}}` : name; +} + +export function incrementCounter(name: string, labels?: MetricLabels): void { + addCounter(name, 1, labels); +} + +export function addCounter(name: string, amount: number, labels?: MetricLabels): void { + if (!name) return; + if (!Number.isFinite(amount) || amount <= 0) return; + const key = counterKey(name, labels); + if (!counters.has(key) && counters.size >= MAX_COUNTERS) return; + counters.set(key, (counters.get(key) ?? 0) + amount); +} + +export function getCounter(name: string, labels?: MetricLabels): number { + return counters.get(counterKey(name, labels)) ?? 0; +} + +export function snapshotCounters(): Record { + return Object.fromEntries(counters.entries()); +} + +/** Number of distinct counter keys currently held (for cap/leak assertions). */ +export function counterKeyCount(): number { + return counters.size; +} + +export function resetMetricsForTests(): void { + counters.clear(); +} diff --git a/shared/session-activity-types.ts b/shared/session-activity-types.ts index 5aaba3742..f700bca21 100644 --- a/shared/session-activity-types.ts +++ b/shared/session-activity-types.ts @@ -1,3 +1,6 @@ +import { TIMELINE_EVENT_FILE_CHANGE } from './file-change.js'; +import { EXECUTION_CLONE_TIMELINE } from './execution-clone.js'; + export type SessionActivityBusyReason = | 'runtime_dispatch' | 'active_dispatch_entry' @@ -186,6 +189,61 @@ export function isWorkingSessionState(value: unknown): boolean { return typeof value === 'string' && WORKING_SESSION_STATES.has(value); } +/** + * `decisionReason` stamped on a `session.state` that is a RE-BROADCAST of state + * the daemon already knew, emitted after the server link is restored so the + * server and browsers can resync. It carries no new information. + * + * Daemon-local listeners MUST ignore it. Treating it as a genuine idle + * transition made every reconnect replay a full "turn finished" edge for every + * transport session, which re-ran `drainQueue()` across all of them: queued + * messages that had been sitting for days were delivered at once, every agent + * answered, and each answer fired an idle hook — hundreds of push notifications + * quoting days-old messages. The resync must inform remote consumers WITHOUT + * re-triggering local side effects. + */ +export const SESSION_STATE_DECISION_REASON_SERVER_LINK_RESYNC = 'server_link_resync' as const; + +/** True when a `session.state` payload is a link-restore re-broadcast. */ +export function isServerLinkResyncStatePayload(payload: unknown): boolean { + return Boolean(payload) + && typeof payload === 'object' + && !Array.isArray(payload) + && (payload as { decisionReason?: unknown }).decisionReason + === SESSION_STATE_DECISION_REASON_SERVER_LINK_RESYNC; +} + +/** + * Timeline events that are pure metadata/telemetry: they say nothing about + * whether a turn is still running, so any tail scan looking for activity + * evidence must WALK THROUGH them rather than stop. + * + * This is not cosmetic. `memory.context` is emitted immediately after the + * `user.message` of every context-injecting send, i.e. it is the tail event for + * the whole "model is thinking before its first tool call" window. While it + * terminated the scan, `hasActiveTimelineTurn()` returned false for that entire + * window, and `resolveTimelineBackedSessionState()` then overrode the daemon's + * `session.state: running` with the (possibly lagging) store snapshot — the + * session rendered IDLE while the agent was demonstrably working. Observed live + * on deck_sub_704c1g42: `running` at 08:53:32 followed by 34 s of thinking with + * a `memory.context` tail, footer asleep the whole time. + * + * Shared so the tail-activity scan and the thinking scan cannot drift apart. + */ +export const TIMELINE_METADATA_EVENT_TYPES: readonly string[] = [ + 'memory.context', + 'memory.compression', + TIMELINE_EVENT_FILE_CHANGE, + 'peer_audit.status', + 'peer_audit.result', + 'transport.queue.snapshot', + 'transport.queue.delivery', + 'transport.queue.receipt', + 'transport.queue.failure', + 'transport.queue.reset', + EXECUTION_CLONE_TIMELINE.TERMINAL, +]; + export function normalizeActivityGeneration(value: ActivityGenerationLike): string | null { if (value == null) return null; if (typeof value === 'number') return Number.isFinite(value) ? `session:${value}` : null; diff --git a/shared/timeline-delivery-telemetry.ts b/shared/timeline-delivery-telemetry.ts new file mode 100644 index 000000000..30b6aa387 --- /dev/null +++ b/shared/timeline-delivery-telemetry.ts @@ -0,0 +1,86 @@ +/** + * Observability for timeline events that are DROPPED before reaching a browser. + * + * Why this exists: the timeline has two silent-discard points, and both were + * invisible. + * + * 1. Daemon → server. Live timeline events are control-plane, so + * `ServerLink.trySend` returns false and drops them whenever the socket is + * not OPEN. There is no queue and no replay (only the data plane has + * `dataPlaneSendQueue`). + * 2. Server → browser. `sendJsonToSessionSubscribers` only writes to sockets + * that are currently subscribed to that session. With no subscriber the loop + * body never runs at all — a backgrounded app therefore has its events + * discarded outright. + * + * The healing fix (activation / reconnect requesting the full newest window with + * no lower bound) makes the user-visible symptom go away, which is exactly why + * these counters matter: without them a rising drop rate would now be masked by + * recovery instead of reported. Counter names are shared so daemon and server + * emit the same series and a regression shows up as a number, not a bug report. + */ + +/** Counter names. Kept in one place so daemon/server/tests cannot drift. */ +export const TIMELINE_DELIVERY_METRICS = { + /** Daemon-side: a control-plane timeline event hit a non-OPEN socket. */ + DAEMON_LINK_DOWN_DROPPED: 'timeline.delivery.daemon_link_down_dropped', + /** Server-side: relayed to a session with zero subscribed viewers. */ + SERVER_NO_SUBSCRIBER_DROPPED: 'timeline.delivery.server_no_subscriber_dropped', + /** Server-side: relayed and delivered to at least one viewer. */ + SERVER_DELIVERED: 'timeline.delivery.server_delivered', +} as const; + +export type TimelineDeliveryMetric = + typeof TIMELINE_DELIVERY_METRICS[keyof typeof TIMELINE_DELIVERY_METRICS]; + +/** + * Message types whose loss leaves a user-visible hole in the chat, as opposed to + * status/telemetry chatter that the next update supersedes anyway. Only these are + * worth counting: `agent.status` fires about once a second during a turn, so + * counting it would bury the signal that actually matters. + */ +const CONTENT_BEARING_TIMELINE_EVENT_TYPES: ReadonlySet = new Set([ + 'user.message', + 'assistant.text', + 'tool.call', + 'tool.result', + 'file.change', + 'ask.question', + 'peer_audit.result', +]); + +/** + * True when a dropped `timeline.event` would leave a visible gap. + * + * `event` is the inner timeline event of a `timeline.event` envelope. Anything + * unrecognised counts as NOT content-bearing so a future noisy event type cannot + * silently inflate the drop series. + */ +export function isContentBearingTimelineEvent(event: unknown): boolean { + if (!event || typeof event !== 'object' || Array.isArray(event)) return false; + const type = (event as { type?: unknown }).type; + return typeof type === 'string' && CONTENT_BEARING_TIMELINE_EVENT_TYPES.has(type); +} + +/** + * The event type when this event is worth counting, otherwise undefined. + * + * Both drop sites need exactly this pair of answers (is it countable, and under + * which label), so they share one call instead of asking twice. Callers that + * already hold the parsed event — the server relay does — must never re-parse the + * serialized copy just to build a counter label; that path runs for every event + * of every session. + */ +export function countableTimelineEventType(event: unknown): string | undefined { + if (!isContentBearingTimelineEvent(event)) return undefined; + return (event as { type: string }).type; +} + +/** Extract the timeline event type from a `timeline.event` envelope, if any. */ +export function timelineEventTypeOf(msg: unknown): string | undefined { + if (!msg || typeof msg !== 'object' || Array.isArray(msg)) return undefined; + const event = (msg as { event?: unknown }).event; + if (!event || typeof event !== 'object' || Array.isArray(event)) return undefined; + const type = (event as { type?: unknown }).type; + return typeof type === 'string' ? type : undefined; +} diff --git a/shared/transport/file-transfer.ts b/shared/transport/file-transfer.ts index 3cd93f160..9a004a223 100644 --- a/shared/transport/file-transfer.ts +++ b/shared/transport/file-transfer.ts @@ -76,6 +76,13 @@ export const FILE_TRANSFER_PATH_HANDLE_CAPABILITY = 'file.transfer.path_handle.v export const FILE_TRANSFER_PATH_MAX_BYTES = 4 * 1024; export const FILE_TRANSFER_ERROR_MAX_BYTES = 256; +/** Machine-readable upload-error codes shared by the daemon (producer), server + * (relay) and web (localized display). */ +export const FILE_TRANSFER_UPLOAD_ERROR_CODE = { + /** The daemon ran out of disk space writing the upload (ENOSPC). */ + INSUFFICIENT_CAPACITY: 'insufficient_capacity', +} as const; + // ── Server → Daemon messages ────────────────────────────────────────────────── export const FILE_TRANSFER_MSG = { @@ -156,6 +163,8 @@ export interface FileUploadError { type: 'file.upload_error'; uploadId: string; message: string; + /** Machine-readable code so the UI can localize (e.g. ENOSPC → "insufficient capacity"). */ + code?: string; } export interface FileUploadProgress { diff --git a/src/agent/providers/getDefaultMcpServers.ts b/src/agent/providers/getDefaultMcpServers.ts index 08ccf0d76..175605a60 100644 --- a/src/agent/providers/getDefaultMcpServers.ts +++ b/src/agent/providers/getDefaultMcpServers.ts @@ -10,10 +10,17 @@ import { IMCODES_DAEMON_USER_ID_ENV, } from '../../../shared/memory-mcp-env.js'; import { IMCODES_MEMORY_MCP_SERVER_NAME } from '../../../shared/memory-mcp-server-name.js'; +import { + LEGACY_DAEMON_LOCAL_USER_ID, + normalizeDaemonLocalMemoryNamespace, +} from '../../../shared/memory-namespace.js'; export const IMCODES_MEMORY_MCP_COMMAND = 'imcodes'; export const IMCODES_MEMORY_MCP_ARGS = ['memory', 'mcp'] as const; -const DAEMON_LOCAL_MEMORY_USER_ID = 'daemon-local'; +// Was a local copy of the sentinel that shared/memory-namespace.ts already +// exports. Four such copies existed, which is how the two halves of the +// register/resolve invariant came to disagree in the first place. +const DAEMON_LOCAL_MEMORY_USER_ID = LEGACY_DAEMON_LOCAL_USER_ID; export interface DefaultMcpServerConfig { type: 'stdio'; @@ -42,14 +49,16 @@ function projectNameFromSessionName(sessionName: string | undefined): string | u return rest.slice(0, idx) || undefined; } +/** + * The namespace this MCP server RESOLVES memory handles under. Shares one helper + * with handle registration (see normalizeDaemonLocalMemoryNamespace): when only + * this side filled in the daemon-local owner, every injected handle was registered + * under a different namespace and redeemed to nothing. + */ function namespaceForMcp(config: SessionConfig): SessionConfig['contextNamespace'] { const namespace = config.contextNamespace ?? undefined; if (!namespace) return undefined; - if (namespace.userId?.trim()) return namespace; - if (namespace.scope === 'personal' || namespace.scope === 'user_private') { - return { ...namespace, userId: DAEMON_LOCAL_MEMORY_USER_ID }; - } - return namespace; + return normalizeDaemonLocalMemoryNamespace(namespace); } function buildIdentityEnv(config: SessionConfig): Record { diff --git a/src/agent/runtime-context-bootstrap.ts b/src/agent/runtime-context-bootstrap.ts index d5d60775c..f979f5f96 100644 --- a/src/agent/runtime-context-bootstrap.ts +++ b/src/agent/runtime-context-bootstrap.ts @@ -29,6 +29,7 @@ import { } from '../../shared/memory-recall-format.js'; import { isMemoryScope } from '../../shared/memory-scope.js'; import { registerMemoryShortRef } from '../context/memory-short-ref.js'; +import { attachMemoryShortRefs } from '../context/memory-recall-refs.js'; export interface TransportContextBootstrapInput { projectDir?: string; @@ -298,7 +299,7 @@ function renderStartupMemoryText( .filter((item) => item.type === 'observation'); const sections: string[] = []; if (memoryItems.length > 0) { - sections.push(buildStartupProjectMemoryText(memoryItems)); + sections.push(buildStartupProjectMemoryText(attachMemoryShortRefs(memoryItems))); } if (observationItems.length > 0) { sections.push(renderStartupObservationIndexText(observationItems)); diff --git a/src/agent/session-manager.ts b/src/agent/session-manager.ts index 6993cafbb..2650c756a 100644 --- a/src/agent/session-manager.ts +++ b/src/agent/session-manager.ts @@ -52,7 +52,7 @@ import { resolveTransportContextBootstrap } from './runtime-context-bootstrap.js import { QWEN_AUTH_TYPES } from '../../shared/qwen-auth.js'; import { TIMELINE_SUPPRESS_PUSH_FIELD } from '../../shared/push-notifications.js'; import { IMCODES_SESSION_ENV, IMCODES_SESSION_LABEL_ENV } from '../../shared/imcodes-send.js'; -import { buildCodexLifecycleTerminalMetadata, isWorkingSessionState, type ActivityGenerationLike } from '../../shared/session-activity-types.js'; +import { SESSION_STATE_DECISION_REASON_SERVER_LINK_RESYNC, buildCodexLifecycleTerminalMetadata, isWorkingSessionState, type ActivityGenerationLike } from '../../shared/session-activity-types.js'; import { SDK_SUBAGENT_DETAIL_KIND, SDK_SUBAGENT_DIAGNOSTIC, @@ -1477,6 +1477,142 @@ async function drainTransportResendQueueIntoRuntime( } } +interface TransportSessionStatePayloadBuild { + /** Raw status mapped to the session-state vocabulary ('running' | 'idle' | 'error' | …). */ + mapped: string; + /** `mapped` after the blocking-work override (idle with blocking work → running). */ + effectiveState: string; + payload: Record; + providerErrorMessage?: string; +} + +/** + * Build the `session.state` timeline payload for a transport session's CURRENT + * runtime status. Shared between the live `onStatusChange` edge emitter and the + * server-link reconnect resync so both produce byte-identical authoritative + * payloads (`isAuthoritativeIdlePayloadShape` requires the exact idle field + * contract — see the pendingCount/pendingVersion alias note below). + */ +function buildTransportSessionStatePayload( + sessionName: string, + runtime: TransportSessionRuntime, + status: string, + decision: { decisionReason: string; clearSource: string; queueReason: string }, +): TransportSessionStatePayloadBuild { + const mapped = isWorkingSessionState(status) ? 'running' : status; + // Include pending info only on idle — the authoritative "turn done, queue empty" signal. + // During running/streaming, command-handler's 'queued' event is the sole queue-update + // authority. This keeps queued messages visible in the UI until the drained turn completes. + const activity = runtime.getDiagnosticSnapshot(); + const effectiveState = mapped === 'idle' && activity.blockingWorkCount > 0 ? 'running' : mapped; + const providerError = runtime.lastProviderError; + const payload: Record = { state: effectiveState }; + if (effectiveState === 'running') { + payload.activityGeneration = activity.activityGeneration; + payload.blockingWorkCount = activity.blockingWorkCount; + payload.activeWorkCount = activity.blockingWorkCount; + payload.activeToolCount = activity.activeToolCount; + payload.busyReasons = activity.busyReasons; + } + if (effectiveState === 'idle') { + payload.authoritative = true; + payload.activityGeneration = activity.activityGeneration; + payload.blockingWorkCount = 0; + payload.activeWorkCount = 0; + payload.activeToolCount = 0; + payload.busyReasons = []; + payload.decisionReason = decision.decisionReason; + payload.clearInputs = [ + { source: decision.clearSource, reason: 'clear', count: 0 }, + ]; + if (activity.completedTurn) { + payload[PEER_AUDIT_COMPLETED_TURN_PAYLOAD_FIELD] = activity.completedTurn; + } + const queuePayload = buildTransportQueueSnapshotPayload(sessionName, decision.queueReason); + Object.assign(payload, queuePayload); + // Canonical authoritative-idle contract fields. The shared validator + // (`isAuthoritativeIdlePayloadShape`) requires numeric `pendingCount` and + // `pendingVersion`; the queue snapshot only carries + // `pendingMessageVersion`/`pendingMessageEntries`, so without these two + // aliases the web demotes every daemon idle to a WEAK idle — and any + // unmatched tool.call in the timeline then keeps the session rendered + // "working" forever even though the reconciler proved clean idle + // (observed live on deck_sub_3l6z4l39). + payload.pendingCount = queuePayload.pendingMessageEntries.length; + payload.pendingVersion = queuePayload.pendingMessageVersion; + } else if (mapped === 'error' && providerError?.message) { + payload.error = providerError.message; + } + return { + mapped, + effectiveState, + payload, + ...(providerError?.message ? { providerErrorMessage: providerError.message } : {}), + }; +} + +/** Monotonic epoch so each reconnect resync gets its own stable eventId set. */ +let transportStateResyncEpoch = 0; + +/** + * Re-broadcast every live transport session's CURRENT authoritative + * `session.state` after the server link is restored. + * + * Why: live `timeline.event` messages are control-plane — `ServerLink.trySend` + * silently drops them while the socket is not OPEN, and there is no replay. + * A turn that settles during a link outage therefore emits its authoritative + * idle into a dead socket: the server (and every browser) keeps rendering the + * session as "working" forever, queued composer sends stay held, and no later + * event ever corrects it (observed live: deck_sub_26624c1t stuck "working" + * 23:46→01:43 while the daemon store/timeline were idle the whole time). + * + * The resync also re-pushes the session record through the persist callback so + * a store PUT lost in the same outage heals too. Payloads reuse the exact + * builder the live path uses, so authoritative-idle shape guarantees hold. + * + * `entries` is injectable for tests; production callers pass nothing. + */ +export function resyncTransportSessionStatesAfterLinkRestore( + entries?: ReadonlyArray, +): number { + const list = entries ?? [...transportRuntimes.entries()]; + if (list.length === 0) return 0; + transportStateResyncEpoch += 1; + const epoch = transportStateResyncEpoch; + let emitted = 0; + for (const [sessionName, runtime] of list) { + try { + const built = buildTransportSessionStatePayload(sessionName, runtime, runtime.getStatus(), { + decisionReason: SESSION_STATE_DECISION_REASON_SERVER_LINK_RESYNC, + clearSource: 'server-link-resync', + queueReason: 'server_link_resync', + }); + // A resync RE-STATES what the daemon already knew; it is never a fresh + // completion, so it must never raise a push. Without this every link + // restore fanned one "Task complete" notification per idle session: + // observed live on the 43 deployment as ~90 dispatches inside 500ms, + // 276ms after four daemons re-authenticated following a gateway blip. The + // server's age guard cannot catch these because the events are newly + // minted (ts = now) even though the state they describe is old. + timelineEmitter.emit(sessionName, 'session.state', { + ...built.payload, + [TIMELINE_SUPPRESS_PUSH_FIELD]: true, + }, { + source: 'daemon', + confidence: 'high', + eventId: `transport-state-resync:${sessionName}:${epoch}`, + }); + const record = getSession(sessionName); + if (record) emitSessionPersist(record, sessionName); + emitted += 1; + } catch (err) { + logger.warn({ err, session: sessionName }, 'transport session state resync failed'); + } + } + logger.info({ sessions: emitted, epoch }, 'ServerLink resync: re-broadcast transport session states'); + return emitted; +} + function wireTransportCallbacks(runtime: TransportSessionRuntime, sessionName: string): void { const transportUserEventId = (clientMessageId: string) => `transport-user:${clientMessageId}`; const persistTransportState = (state: unknown, error?: string): void => { @@ -1501,52 +1637,13 @@ function wireTransportCallbacks(runtime: TransportSessionRuntime, sessionName: s if (status === 'thinking') { timelineEmitter.emit(sessionName, 'assistant.thinking', { text: '' }, { source: 'daemon', confidence: 'high' }); } - const mapped = isWorkingSessionState(status) ? 'running' : status; - // Include pending info only on idle — the authoritative "turn done, queue empty" signal. - // During running/streaming, command-handler's 'queued' event is the sole queue-update - // authority. This keeps queued messages visible in the UI until the drained turn completes. - const activity = runtime.getDiagnosticSnapshot(); - const effectiveMapped = mapped === 'idle' && activity.blockingWorkCount > 0 ? 'running' : mapped; - const providerError = runtime.lastProviderError; - persistTransportState(effectiveMapped, mapped === 'error' ? providerError?.message : undefined); - const payload: Record = { state: effectiveMapped }; - if (effectiveMapped === 'running') { - payload.activityGeneration = activity.activityGeneration; - payload.blockingWorkCount = activity.blockingWorkCount; - payload.activeWorkCount = activity.blockingWorkCount; - payload.activeToolCount = activity.activeToolCount; - payload.busyReasons = activity.busyReasons; - } - if (effectiveMapped === 'idle') { - payload.authoritative = true; - payload.activityGeneration = activity.activityGeneration; - payload.blockingWorkCount = 0; - payload.activeWorkCount = 0; - payload.activeToolCount = 0; - payload.busyReasons = []; - payload.decisionReason = 'activity_reconciler_clear'; - payload.clearInputs = [ - { source: 'transport-runtime', reason: 'clear', count: 0 }, - ]; - if (activity.completedTurn) { - payload[PEER_AUDIT_COMPLETED_TURN_PAYLOAD_FIELD] = activity.completedTurn; - } - const queuePayload = buildTransportQueueSnapshotPayload(sessionName, 'transport_status_idle'); - Object.assign(payload, queuePayload); - // Canonical authoritative-idle contract fields. The shared validator - // (`isAuthoritativeIdlePayloadShape`) requires numeric `pendingCount` and - // `pendingVersion`; the queue snapshot only carries - // `pendingMessageVersion`/`pendingMessageEntries`, so without these two - // aliases the web demotes every daemon idle to a WEAK idle — and any - // unmatched tool.call in the timeline then keeps the session rendered - // "working" forever even though the reconciler proved clean idle - // (observed live on deck_sub_3l6z4l39). - payload.pendingCount = queuePayload.pendingMessageEntries.length; - payload.pendingVersion = queuePayload.pendingMessageVersion; - } else if (mapped === 'error' && providerError?.message) { - payload.error = providerError.message; - } - timelineEmitter.emit(sessionName, 'session.state', payload, { source: 'daemon', confidence: 'high' }); + const built = buildTransportSessionStatePayload(sessionName, runtime, status, { + decisionReason: 'activity_reconciler_clear', + clearSource: 'transport-runtime', + queueReason: 'transport_status_idle', + }); + persistTransportState(built.effectiveState, built.mapped === 'error' ? built.providerErrorMessage : undefined); + timelineEmitter.emit(sessionName, 'session.state', built.payload, { source: 'daemon', confidence: 'high' }); if (status === 'error') { void recoverTransportRuntimeAfterError(sessionName, runtime); } diff --git a/src/agent/transport-runtime-assembly.ts b/src/agent/transport-runtime-assembly.ts index 05bb1d066..f26447d9e 100644 --- a/src/agent/transport-runtime-assembly.ts +++ b/src/agent/transport-runtime-assembly.ts @@ -21,6 +21,7 @@ import type { TransportMemoryRecallItem, } from '../../shared/context-types.js'; import { buildStartupProjectMemoryText } from '../../shared/memory-recall-format.js'; +import { attachMemoryShortRefs } from '../context/memory-recall-refs.js'; import { buildFilePathReportingPrompt, buildTransportImcodesIdentityPrompt } from '../../shared/transport-runtime-prompts.js'; export interface TransportRuntimeAssemblyInput { @@ -265,7 +266,7 @@ function filterStartupMemoryForAuthority( authoritySource: 'processed_remote', sourceKind: resolveRecallSourceKind(remoteItems), items: remoteItems, - injectedText: buildStartupProjectMemoryText(remoteItems), + injectedText: buildStartupProjectMemoryText(attachMemoryShortRefs(remoteItems)), }; } diff --git a/src/agent/transport-session-runtime.ts b/src/agent/transport-session-runtime.ts index c6b1784fc..fbe747367 100644 --- a/src/agent/transport-session-runtime.ts +++ b/src/agent/transport-session-runtime.ts @@ -80,6 +80,7 @@ import { resolveSummarySyncSourceKind, } from '../context/summary-sync.js'; import { buildRelatedPastWorkText, buildStartupProjectMemoryText } from '../../shared/memory-recall-format.js'; +import { attachMemoryShortRefs } from '../context/memory-recall-refs.js'; import { getContextModelConfig } from '../context/context-model-config.js'; import { PREFERENCE_CONTEXT_END, PREFERENCE_CONTEXT_START } from '../../shared/preference-ingest.js'; import { clampUserSessionText } from '../../shared/user-session-text-caps.js'; @@ -3298,8 +3299,8 @@ export class TransportSessionRuntime implements SessionRuntime { : 'degraded-message-side'; const combinedItems = [...summaryItems, ...items]; const sections: string[] = []; - if (summaryItems.length > 0) sections.push(buildStartupProjectMemoryText(summaryItems)); - if (items.length > 0) sections.push(buildRelatedPastWorkText(items)); + if (summaryItems.length > 0) sections.push(buildStartupProjectMemoryText(attachMemoryShortRefs(summaryItems))); + if (items.length > 0) sections.push(buildRelatedPastWorkText(attachMemoryShortRefs(items))); const injectedText = sections.join('\n\n'); const sourceKind = resolveSummarySyncSourceKind(combinedItems); const payload = buildMemoryContextTimelinePayload(query, combinedItems, 'message', { diff --git a/src/context/memory-read-tools.ts b/src/context/memory-read-tools.ts index 8bb5f1421..9ddd81274 100644 --- a/src/context/memory-read-tools.ts +++ b/src/context/memory-read-tools.ts @@ -16,7 +16,11 @@ import { getContextStoreClient } from '../store/context-store-worker-client.js'; const MEMORY_TOOL_CALLER_BRAND: unique symbol = Symbol('MemoryToolCaller'); const INTERNAL_MEMORY_TOOL_CALLER_BRAND: unique symbol = Symbol('InternalMemoryToolCaller'); -const DAEMON_LOCAL_MEMORY_USER_ID = 'daemon-local'; +// Alias of the shared sentinel, not a second definition: this module already +// imports LEGACY_DAEMON_LOCAL_USER_ID. Two literals of the same value in one +// codebase are how the register/resolve halves of the handle namespace drifted +// apart in the first place. +const DAEMON_LOCAL_MEMORY_USER_ID = LEGACY_DAEMON_LOCAL_USER_ID; /** * Caller identity passed to public read-tool handlers. diff --git a/src/context/memory-recall-refs.ts b/src/context/memory-recall-refs.ts new file mode 100644 index 000000000..e00405996 --- /dev/null +++ b/src/context/memory-recall-refs.ts @@ -0,0 +1,98 @@ +import type { ContextNamespace, ContextScope } from '../../shared/context-types.js'; +import { normalizeDaemonLocalMemoryNamespace } from '../../shared/memory-namespace.js'; +import { registerMemoryShortRefs, type MemoryShortRefEntry, type MemoryShortRefKind } from './memory-short-ref.js'; + +/** + * The minimum an injected memory item must expose to get a redeemable handle. + * Deliberately structural rather than tied to one concrete item type: the recall + * surfaces feed this from `MemorySearchResultItem`, `TransportMemoryRecallItem` + * and `MemoryContextTimelineItem`, which agree on these fields but not much else. + */ +export interface MemoryShortRefSource { + id?: string; + type?: 'raw' | 'processed' | 'observation'; + scope?: string; + projectId?: string; + userId?: string; + workspaceId?: string; + enterpriseId?: string; +} + +/** + * `get_memory_sources` addresses projections and observations only, so raw staged + * events get no handle rather than a dead one. + * + * Some recall surfaces carry no `type` discriminator at all; those come from the + * processed-memory path, so they are treated as projections. That default is + * fail-safe rather than a guess: if such a record were actually an observation, + * the projection lookup simply misses and the fetch returns zero sources — the + * same outcome as shipping no handle, never another record's content. + */ +function shortRefKind(type: MemoryShortRefSource['type']): MemoryShortRefKind | undefined { + if (type === 'observation') return 'observation'; + if (type === 'raw') return undefined; + return 'projection'; +} + +/** + * The namespace this handle must be registered under. + * + * Recall items carry `scope` and `projectId` but never `userId`, so deriving the + * namespace from item fields alone stored every injected handle with an empty + * owner. Two things then went wrong at once: the MCP resolver asks with + * `userId: 'daemon-local'` and `resolveMemoryShortRef` refuses a cross-namespace + * match, so the handle redeemed to zero sources; and `personal` declares + * `requiredIdentityFields: ['user_id', 'project_id']`, so the warm loader + * DISCARDED the row outright on the next daemon start. 290 of 302 stored handles + * (96%) were in that state — the failure the user saw as "记忆句柄未保存" while the + * rows sat in the table. + * + * The owner is not optional for owner-private memory; on a single-user device it is + * simply implicit. Making it explicit here — with the same helper the MCP server + * config uses — is what keeps registration, resolution and the scope policy in + * agreement instead of two of the three. + */ +function namespaceForItem(item: MemoryShortRefSource): ContextNamespace | undefined { + if (!item.scope) return undefined; + return normalizeDaemonLocalMemoryNamespace({ + scope: item.scope as ContextScope, + ...(item.projectId ? { projectId: item.projectId } : {}), + ...(item.userId ? { userId: item.userId } : {}), + ...(item.workspaceId ? { workspaceId: item.workspaceId } : {}), + ...(item.enterpriseId ? { enterpriseId: item.enterpriseId } : {}), + }); +} + +/** + * Attach a redeemable handle to every injected memory item. + * + * Injected memory used to carry only a summary, while the guidance shipped + * alongside it told the agent to "call get_memory_sources with that ref" — there + * was no ref to call with, so a summary that looked relevant was a dead end. + * Registering here (batched: one cache write, not one per item) is what makes the + * emitted handle resolvable. + */ +export function attachMemoryShortRefs( + items: readonly T[], +): Array { + const registrable: Array<{ index: number; entry: MemoryShortRefEntry }> = []; + items.forEach((item, index) => { + const kind = shortRefKind(item.type); + if (!kind || !item.id) return; + const namespace = namespaceForItem(item); + registrable.push({ + index, + entry: { kind, id: item.id, ...(namespace ? { namespace } : {}) }, + }); + }); + const refs = registerMemoryShortRefs(registrable.map((candidate) => candidate.entry)); + const refByIndex = new Map(); + registrable.forEach((candidate, position) => { + const ref = refs[position]; + if (ref) refByIndex.set(candidate.index, ref); + }); + return items.map((item, index) => { + const ref = refByIndex.get(index); + return ref ? { ...item, ref } : item; + }); +} diff --git a/src/context/memory-short-ref.ts b/src/context/memory-short-ref.ts index 83f6563fc..c10d4430c 100644 --- a/src/context/memory-short-ref.ts +++ b/src/context/memory-short-ref.ts @@ -1,4 +1,13 @@ import type { ContextNamespace } from '../../shared/context-types.js'; +import { isMemoryScope, validateMemoryScopeIdentity } from '../../shared/memory-scope.js'; +import { normalizeDaemonLocalMemoryNamespace } from '../../shared/memory-namespace.js'; +import { MEMORY_SHORT_REF_HEALTH_ERROR_MAX_CHARS, type MemoryShortRefHealth } from '../../shared/memory-short-ref-health.js'; +import { createHash } from 'node:crypto'; +import { encodeBase32 } from '../util/base32.js'; +import { getContextStoreClient } from '../store/context-store-worker-client.js'; +import { warnOncePerHour } from '../util/rate-limited-warn.js'; +import { incrementCounter } from '../util/metrics.js'; +import logger from '../util/logger.js'; import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { dirname, join } from 'node:path'; @@ -12,9 +21,25 @@ export interface MemoryShortRefEntry { lastSeenAt?: number; } -const MAX_SHORT_REF_ENTRIES = 4096; +const MAX_SHORT_REF_ENTRIES = 10_000; + +/** + * base32 characters kept from the md5 digest. 5 bits per character, so 13 chars + * ≈ 65 bits: a birthday collision needs ~6e9 distinct ids in one namespace, + * versus the previous 40-bit handle which collided at ~0.5% by 100k ids. + * Collisions here are not benign — a handle that maps to several records + * resolves to an arbitrary one, i.e. the agent silently fetches the WRONG + * memory. base32 carries the same 65 bits in 13 chars that hex needs 16 for. + */ +const MEMORY_SHORT_REF_LENGTH = 13; + +/** Bumped whenever the ref derivation changes, so cached refs from an older + * algorithm are dropped instead of resolving to a stale/wrong record. */ +const SHORT_REF_SCHEMA_VERSION = 2; const entriesByRef = new Map(); +const persistedRefHydrations = new Map>(); let persistedLoaded = false; +let shortRefHealth: MemoryShortRefHealth | undefined; function refPrefix(kind: MemoryShortRefKind): 'proj' | 'obs' { return kind === 'projection' ? 'proj' : 'obs'; @@ -39,8 +64,25 @@ function sameNamespace(a: ContextNamespace | undefined, b: ContextNamespace | un return namespaceKey(a) === namespaceKey(b); } -function newestEntry(entries: MemoryShortRefEntry[]): MemoryShortRefEntry | undefined { - return [...entries].sort((a, b) => (b.lastSeenAt ?? 0) - (a.lastSeenAt ?? 0))[0]; +/** + * Namespace key for the persisted row. + * + * `namespaceKey()` separates fields with NUL, which is fine in memory. Older + * node:sqlite versions returned only the leading scope when reading it as TEXT; + * current versions return the complete string. The bytes are stored intact and + * the primary key was never affected, but the version-dependent read shape is a + * poor persisted identity. JSON keeps the same field order with no embedded NUL + * and round-trips consistently. + */ +function namespaceStorageKey(namespace: ContextNamespace | undefined): string { + if (!namespace) return ''; + return JSON.stringify([ + namespace.scope, + namespace.userId ?? '', + namespace.projectId ?? '', + namespace.workspaceId ?? '', + namespace.enterpriseId ?? '', + ]); } function pruneShortRefs(): void { @@ -61,21 +103,153 @@ function pruneShortRefs(): void { } } +/** + * JSON file target, or undefined to use the context store. + * + * ONLY an explicit `IMCODES_MEMORY_SHORT_REF_PATH` selects the file. The default + * — including production — is the store. + * + * This used to fall back to `~/.imcodes/memory-short-refs.json` whenever the + * process wasn't a test, which inverted the intended routing: tests took the + * store path while real daemons kept writing the JSON file, so the migration off + * that file (and the failure reporting added alongside it) never executed + * anywhere it mattered. Handles written to that file while it was still the + * write target are carried over once by importLegacyShortRefFile(). + */ function shortRefStorePath(): string | undefined { const configured = process.env.IMCODES_MEMORY_SHORT_REF_PATH?.trim(); - if (configured) return configured; - if (process.env.VITEST === 'true' || process.env.NODE_ENV === 'test') return undefined; - return join(homedir(), '.imcodes', 'memory-short-refs.json'); + return configured ? configured : undefined; +} + +/** + * Where the retired JSON cache lives. Read-only: handles are only ever written + * to the store now. Overridable so tests never read the developer's real file. + */ +function legacyShortRefFilePath(): string { + const configured = process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH?.trim(); + return configured ? configured : join(homedir(), '.imcodes', 'memory-short-refs.json'); +} + +/** + * One-way bridge off the retired JSON cache. + * + * Handles written while persistence still went to a file are valid, so import + * them into the store on the first warm-load rather than stranding them until + * their memory happens to be injected again. Strictly read-only, and the file is + * left in place as a manual recovery point — it is simply never written again. + * The counter shows when this stops finding anything, which is when the file can + * be dropped for good. + */ +function importLegacyShortRefFile(): Array<{ ref: string; entry: MemoryShortRefEntry }> { + const path = legacyShortRefFilePath(); + const imported: Array<{ ref: string; entry: MemoryShortRefEntry }> = []; + try { + const parsed = JSON.parse(readFileSync(path, 'utf8')) as { schemaVersion?: unknown; entries?: unknown[] }; + // Older derivations produced handles that denote different records now, so + // rejecting that whole file is expected and stays quiet. + if (parsed.schemaVersion !== SHORT_REF_SCHEMA_VERSION) return imported; + // A current-schema file whose entries are not an array is corrupt, and every + // handle in it vanishes — that is a failure, not a steady state. + if (!Array.isArray(parsed.entries)) { + reportShortRefFailure('load_file', new Error('legacy cache entries is not an array'), { path, legacy: true }); + return imported; + } + let discarded = 0; + for (const raw of parsed.entries) { + const normalized = normalizeEntry(raw); + if (!normalized) { discarded += 1; continue; } + const bucket = entriesByRef.get(normalized.ref) ?? []; + if (bucket.some((entry) => entry.kind === normalized.entry.kind + && entry.id === normalized.entry.id + && sameNamespace(entry.namespace, normalized.entry.namespace))) continue; + bucket.push(normalized.entry); + entriesByRef.set(normalized.ref, bucket); + imported.push(normalized); + } + reportDiscardedShortRefRows('legacy_file', discarded); + if (imported.length > 0) { + incrementCounter('mem.short_ref.legacy_import', { source: 'json_file' }); + // The counter lives in an in-process map that a restart clears, so the + // "has this stopped finding anything yet" question needs a durable line + // in the log too — that is the signal for retiring the fallback. + logger.info({ imported: imported.length, path }, 'memory short-ref: imported handles from the retired JSON cache'); + } + } catch (error) { + // A missing file is the expected steady state once everyone has migrated. + if ((error as NodeJS.ErrnoException | null)?.code !== 'ENOENT') { + reportShortRefFailure('load_file', error, { path, legacy: true }); + } + } + return imported; } function isMemoryShortRefKind(value: unknown): value is MemoryShortRefKind { return value === 'projection' || value === 'observation'; } +/** Namespace identity fields; every one must be a string when present. */ +const NAMESPACE_IDENTITY_FIELDS = [ + 'projectId', 'userId', 'workspaceId', 'enterpriseId', 'localTenant', 'canonicalRepoId', +] as const; + +/** + * Decode a stored namespace, refusing anything it cannot fully validate. + * + * `{ ok: false }` means the row carried a namespace that is not usable, and the + * caller must discard the whole row. Degrading such a row to namespace-less + * instead — which an earlier `scope is a non-empty string` check did — loads an + * entry that looks healthy but can never resolve for a namespaced caller, and + * counts it as loaded, so the handle disappears with no signal at all. + */ +function decodeNamespace( + raw: unknown, +): { ok: true; namespace: ContextNamespace | undefined; preBackfillNamespace?: ContextNamespace } | { ok: false } { + // Only a value that is not there at all means "no namespace". An explicit + // null — including a column holding the JSON text `null` — is a value that + // failed to describe a namespace, and degrading it to namespace-less loads a + // handle no namespaced caller can ever reach. + if (raw === undefined) return { ok: true, namespace: undefined }; + if (raw === null || typeof raw !== 'object') return { ok: false }; + const record = raw as Record; + if (!isMemoryScope(record.scope)) return { ok: false }; + for (const field of NAMESPACE_IDENTITY_FIELDS) { + const value = record[field]; + if (value !== undefined && typeof value !== 'string') return { ok: false }; + } + // A scope also dictates which identity fields must be present and which are + // forbidden. `{ scope: 'personal' }` with no userId/projectId is structurally + // fine but names no actual namespace, so it would resolve for nobody. + // + // Owner-private rows written before registration filled the owner in have no + // `userId`, and `personal` requires one — so they failed validation and were + // DISCARDED at every daemon start. That silently destroyed 290 of 302 stored + // handles on a real machine. This store is per-device and a device has exactly + // one owner, so an owner-less owner-private row is not ambiguous: it belongs to + // this daemon. Backfill the sentinel and keep the row instead of dropping the + // user's memory handles on a technicality. + const rawNamespace = record as unknown as ContextNamespace; + const namespace = normalizeDaemonLocalMemoryNamespace(rawNamespace); + // Hand back the PRE-backfill namespace rather than a boolean, so the loader can + // rebuild the key this row was actually written with instead of assuming the + // absent owner was the empty string. The helper treats a whitespace-only owner + // as missing too, and a row that stored `" "` in both its JSON and its key was + // being discarded because the reconstruction guessed `''`. Carrying the real + // value removes the guess. + const ownerBackfilled = namespace !== rawNamespace; + const identity = { + user_id: namespace.userId, + project_id: namespace.projectId, + workspace_id: namespace.workspaceId, + org_id: namespace.enterpriseId, + tenant_id: namespace.localTenant, + }; + if (!validateMemoryScopeIdentity(record.scope, identity).ok) return { ok: false }; + return { ok: true, namespace, ...(ownerBackfilled ? { preBackfillNamespace: rawNamespace } : {}) }; +} + function isContextNamespace(value: unknown): value is ContextNamespace { - if (!value || typeof value !== 'object') return false; - const scope = (value as { scope?: unknown }).scope; - return typeof scope === 'string' && scope.length > 0; + const decoded = decodeNamespace(value); + return decoded.ok && decoded.namespace !== undefined; } function normalizeEntry(raw: unknown): { ref: string; entry: MemoryShortRefEntry } | undefined { @@ -85,7 +259,11 @@ function normalizeEntry(raw: unknown): { ref: string; entry: MemoryShortRefEntry const kind = record.kind; const id = typeof record.id === 'string' && record.id.trim() ? record.id.trim() : undefined; if (!ref || !isMemoryShortRefKind(kind) || !id) return undefined; - const namespace = isContextNamespace(record.namespace) ? record.namespace : undefined; + // A row that carries a namespace it cannot validate is discarded, never + // silently loaded namespace-less. + const decodedNamespace = decodeNamespace(record.namespace); + if (!decodedNamespace.ok) return undefined; + const namespace = decodedNamespace.namespace; const lastSeenAt = typeof record.lastSeenAt === 'number' && Number.isFinite(record.lastSeenAt) ? record.lastSeenAt : undefined; @@ -101,11 +279,21 @@ function ensurePersistedLoaded(): void { const path = shortRefStorePath(); if (!path) return; try { - const parsed = JSON.parse(readFileSync(path, 'utf8')) as { entries?: unknown[] }; - if (!Array.isArray(parsed.entries)) return; + const parsed = JSON.parse(readFileSync(path, 'utf8')) as { schemaVersion?: unknown; entries?: unknown[] }; + // Refs cached by an older derivation must NOT be trusted: the same ref + // string can denote a different record under the new algorithm, which would + // resolve to the wrong memory. Drop the whole file and re-register lazily. + if (parsed.schemaVersion !== SHORT_REF_SCHEMA_VERSION) return; + // Current schema with non-array entries is corruption, not an empty cache: + // every handle in the file disappears, so it has to be reported. + if (!Array.isArray(parsed.entries)) { + reportShortRefFailure('load_file', new Error('cache entries is not an array'), { path }); + return; + } + let discarded = 0; for (const raw of parsed.entries) { const normalized = normalizeEntry(raw); - if (!normalized) continue; + if (!normalized) { discarded += 1; continue; } const bucket = entriesByRef.get(normalized.ref) ?? []; if (!bucket.some((entry) => entry.kind === normalized.entry.kind && entry.id === normalized.entry.id @@ -114,13 +302,18 @@ function ensurePersistedLoaded(): void { entriesByRef.set(normalized.ref, bucket); } } + reportDiscardedShortRefRows('json_file', discarded); pruneShortRefs(); - } catch { - // Missing or corrupt cache is non-fatal: sourceLookup full ids remain canonical. + } catch (error) { + // A missing file is the normal first run. Anything else — unreadable, or + // corrupt JSON — silently strands every handle written before this start, + // which is the same invisible failure this module exists to avoid. + if ((error as NodeJS.ErrnoException | null)?.code === 'ENOENT') return; + reportShortRefFailure('load_file', error, { path }); } } -function persistShortRefs(): void { +function persistShortRefsToFile(): void { const path = shortRefStorePath(); if (!path) return; const entries: Array<{ ref: string } & MemoryShortRefEntry> = []; @@ -131,21 +324,252 @@ function persistShortRefs(): void { try { mkdirSync(dirname(path), { recursive: true }); writeFileSync(path, JSON.stringify({ - schemaVersion: 1, + schemaVersion: SHORT_REF_SCHEMA_VERSION, entries: entries.slice(0, MAX_SHORT_REF_ENTRIES), }), 'utf8'); + } catch (error) { + // Non-fatal, but never silent: an unwritable file is exactly how handles + // used to disappear without a trace. + reportShortRefFailure('persist_file', error, { path }); + } +} + +/** + * A handle that fails to persist still resolves in this process but dies on the + * next restart, so every such failure gets a fixed-cardinality counter and an + * hourly-throttled warning. `stage` is a constant per call site — error text and + * volatile fields stay in the warning payload, never in metric labels. + */ +/** Rows dropped while loading are a silent form of handle loss, so count them + * under a fixed-cardinality source and warn once per hour. */ +function reportDiscardedShortRefRows( + source: 'warm_load' | 'hydrate_ref' | 'legacy_file' | 'json_file', + discarded: number, +): void { + if (discarded <= 0) return; + incrementCounter('mem.short_ref.discarded_row', { source }); + warnOncePerHour(`mem.short_ref.discarded_row.${source}`, { discarded }); + // Discarded rows are handle loss too: those memories stop being reachable by + // handle after a restart. Route them to the same off-box signal as write and + // load failures instead of leaving them in machine-local counters. + recordShortRefFailure(`discarded_${source}`, `${discarded} unusable row(s) discarded`); +} + +function reportShortRefFailure( + stage: 'persist_store' | 'persist_file' | 'warm_load' | 'hydrate_ref' | 'load_file', + error: unknown, + extra: Record = {}, +): void { + const message = error instanceof Error ? error.message : String(error); + incrementCounter('mem.short_ref.persist_failure', { stage }); + warnOncePerHour(`mem.short_ref.persist_failure.${stage}`, { ...extra, error: message }); + // Both signals above stay on this machine: the counter is a process-local map + // and the warning lands in the daemon log, on the same disk whose exhaustion + // is usually what failed — with write errors swallowed. Hold the failure in + // memory so the heartbeat can carry it off the box over the socket instead. + recordShortRefFailure(stage, message); +} + +function recordShortRefFailure(stage: string, message: string): void { + shortRefHealth = { + stage, + failures: (shortRefHealth?.failures ?? 0) + 1, + lastFailureAt: Date.now(), + lastError: message.slice(0, MEMORY_SHORT_REF_HEALTH_ERROR_MAX_CHARS), + }; +} + +/** + * Latest persistence failure, or undefined while healthy. Sticky rather than + * edge-triggered, so a reader that connects after the failure still sees it. + */ +export function getMemoryShortRefHealth(): MemoryShortRefHealth | undefined { + return shortRefHealth; +} + +/** + * Persist the handles just registered. + * + * Default path is the context store (SQLite): an incremental upsert of only the + * touched rows, rather than rewriting a whole JSON file on every registration — + * that rewrite silently stopped persisting anything once the disk filled up, + * because the write error was swallowed. Fire-and-forget so the synchronous + * registration path (called from render functions) never blocks on I/O. + * + * `IMCODES_MEMORY_SHORT_REF_PATH` still selects the JSON file, which keeps tests + * hermetic and gives a debuggable plain-text dump. + */ +function persistShortRefs(touched: ReadonlyArray<{ ref: string; entry: MemoryShortRefEntry }>): void { + if (shortRefStorePath()) { + persistShortRefsToFile(); + return; + } + if (touched.length === 0) return; + const rows = touched.map(({ ref, entry }) => ({ + ref, + kind: entry.kind, + id: entry.id, + namespaceKey: namespaceStorageKey(entry.namespace), + namespaceJson: entry.namespace ? JSON.stringify(entry.namespace) : null, + lastSeenAt: entry.lastSeenAt ?? Date.now(), + })); + void getContextStoreClient() + .run('upsertMemoryShortRefs', [rows]) + .catch((error: unknown) => { + // Non-fatal for this process — the in-memory index still resolves, handles + // are a pure function of the id, and sourceLookup full ids stay canonical. + // But it IS the failure this change exists to make visible: handles that + // never land stop resolving after a restart. + reportShortRefFailure('persist_store', error, { rows: rows.length }); + }); +} + +function normalizePersistedShortRefRow( + row: Record, +): { ref: string; entry: MemoryShortRefEntry } | undefined { + // A row that stored a namespace but can no longer produce one must be + // dropped, not loaded namespace-less: the entry would look healthy while + // being unresolvable for every namespaced caller — silent handle loss + // wearing the shape of a successful load. + const namespaceColumnPresent = typeof row.namespaceJson === 'string' && row.namespaceJson.trim().length > 0; + const parsed = namespaceColumnPresent ? safeParseNamespace(row.namespaceJson as string) : undefined; + if (namespaceColumnPresent && parsed === undefined) return undefined; + const decoded = decodeNamespace(parsed); + if (!decoded.ok) return undefined; + const namespace = decoded.namespace; + // namespace_key records what the row was written under; a disagreement + // means the row's identity is corrupt and loading it would file the handle + // under the wrong namespace. + // + // Rows written before the key became a JSON tuple used NUL-separated + // fields. Older node:sqlite versions truncate that TEXT value at the + // first NUL when converting it to a JS string, while current versions + // return the complete string. Accept both read-back shapes so upgrading + // Node cannot discard every handle written before the key migration. + const storedNamespaceKey = typeof row.namespaceKey === 'string' ? row.namespaceKey : ''; + const legacyNamespaceKey = namespaceKey(namespace); + const legacyTruncatedKey = namespace ? namespace.scope : ''; + // Rows written before the owner was made explicit recorded their key WITHOUT + // it, so the keys recomputed from the backfilled namespace no longer match. + // Accept that shape ONLY when decodeNamespace actually backfilled the owner. + const preBackfillNamespace = decoded.preBackfillNamespace; + if (storedNamespaceKey !== namespaceStorageKey(namespace) + && storedNamespaceKey !== legacyNamespaceKey + && storedNamespaceKey !== legacyTruncatedKey + && !(preBackfillNamespace + && (storedNamespaceKey === namespaceStorageKey(preBackfillNamespace) + || storedNamespaceKey === namespaceKey(preBackfillNamespace)))) { + return undefined; + } + return normalizeEntry({ + ref: row.ref, + kind: row.kind, + id: row.id, + lastSeenAt: row.lastSeenAt, + namespace, + }); +} + +function mergePersistedShortRefRows( + rows: readonly Record[], + options: { touchLastSeenAt?: boolean } = {}, +): { loaded: number; discarded: number } { + let loaded = 0; + let discarded = 0; + const touchedAt = options.touchLastSeenAt ? Date.now() : undefined; + for (const row of rows) { + const normalized = normalizePersistedShortRefRow(row); + if (!normalized) { + discarded += 1; + continue; + } + if (touchedAt !== undefined) normalized.entry.lastSeenAt = touchedAt; + const bucket = entriesByRef.get(normalized.ref) ?? []; + if (bucket.some((entry) => entry.kind === normalized.entry.kind + && entry.id === normalized.entry.id + && sameNamespace(entry.namespace, normalized.entry.namespace))) continue; + bucket.push(normalized.entry); + entriesByRef.set(normalized.ref, bucket); + loaded += 1; + } + return { loaded, discarded }; +} + +/** + * Warm the in-memory index from the context store once at daemon startup. + * + * Synchronous render paths continue to use this warm index. Independent MCP + * processes additionally perform an indexed SQLite lookup on a local miss, + * because another process can persist a handle after this warm-load finishes. + */ +export async function loadMemoryShortRefsFromStore(): Promise { + if (shortRefStorePath()) return 0; + try { + const rows = await getContextStoreClient() + .run>>('listMemoryShortRefs', [MAX_SHORT_REF_ENTRIES]); + if (!Array.isArray(rows)) { + // Contract violation rather than an ordinary failure, but the effect is + // the same as a failed load — report instead of returning a 0 that reads + // as "nothing stored". + reportShortRefFailure('warm_load', new Error('listMemoryShortRefs returned a non-array response')); + return 0; + } + const merged = mergePersistedShortRefRows(rows); + let loaded = merged.loaded; + // A well-formed array of malformed rows loads nothing and leaves every + // earlier handle unresolvable — indistinguishable from an empty store + // unless the discards are reported. + reportDiscardedShortRefRows('warm_load', merged.discarded); + // Carry over anything still only in the retired JSON cache, and write it to + // the store so the next start no longer depends on that file. + const legacy = importLegacyShortRefFile(); + if (legacy.length > 0) persistShortRefs(legacy); + loaded += legacy.length; + pruneShortRefs(); + persistedLoaded = true; + return loaded; + } catch (error) { + // A failed warm-load means every handle issued before this restart is + // unresolvable for the life of the process — report rather than return a + // zero that is indistinguishable from "nothing stored". + reportShortRefFailure('warm_load', error); + return 0; + } +} + +function safeParseNamespace(raw: string): unknown { + try { + return JSON.parse(raw); } catch { - // Ref persistence is a convenience cache. Do not break memory search/source reads. + return undefined; } } +/** + * Derive the compact handle for an id. + * + * A hash prefix — NOT a prefix of the id itself. The previous algorithm kept the + * first 10 hex-looking characters of the id, which scavenged them from arbitrary + * positions across the string. For structured ids such as + * `md-ingest:personal::::::::local/:CLAUDE.md:` every record that + * shares the constant prefix collapsed onto the SAME ref (the discriminating + * sha256 sits past the 10th kept character), so one ref mapped to dozens of + * records and resolution silently picked whichever was newest. + * + * A digest gives a uniform distribution for ANY id shape. Being a pure function + * of the id also means the handle is reproducible: the same memory always yields + * the same handle across sessions, restarts and machines, so the ref→id table is + * a REBUILDABLE index rather than the only source of truth (a random handle + * would be permanently dead if that table were ever lost). md5 is used purely + * as a fast non-cryptographic digest here, never for security. + */ export function makeMemoryShortRef(kind: MemoryShortRefKind, id: string): string { - const compact = id.replace(/[^a-f0-9]/gi, '').slice(0, 10) || id.slice(0, 10); + const digest = createHash('md5').update(id, 'utf8').digest(); + const compact = encodeBase32(digest).slice(0, MEMORY_SHORT_REF_LENGTH); return `${refPrefix(kind)}:${compact}`; } -export function registerMemoryShortRef(entry: MemoryShortRefEntry): string { - ensurePersistedLoaded(); +function registerMemoryShortRefWithoutPersist(entry: MemoryShortRefEntry): string { const ref = normalizeRef(makeMemoryShortRef(entry.kind, entry.id)); const bucket = entriesByRef.get(ref) ?? []; const nextEntry = { ...entry, lastSeenAt: entry.lastSeenAt ?? Date.now() }; @@ -154,28 +578,176 @@ export function registerMemoryShortRef(entry: MemoryShortRefEntry): string { || !sameNamespace(existing.namespace, entry.namespace)); next.push(nextEntry); entriesByRef.set(ref, next); + return ref; +} + +export function registerMemoryShortRef(entry: MemoryShortRefEntry): string { + ensurePersistedLoaded(); + const ref = registerMemoryShortRefWithoutPersist(entry); pruneShortRefs(); - persistShortRefs(); + persistShortRefs([{ ref, entry }]); return ref; } +/** + * Batch variant for surfaces that register many refs at once (startup/recall + * memory injection registers every injected item so the agent can redeem the + * ref via get_memory_sources). Persists ONCE instead of per entry — the + * per-entry path rewrites the whole cache file, which would otherwise mean N + * full-file writes on the session send path. + */ +export function registerMemoryShortRefs(entries: readonly MemoryShortRefEntry[]): string[] { + ensurePersistedLoaded(); + const refs = entries.map((entry) => registerMemoryShortRefWithoutPersist(entry)); + if (refs.length > 0) { + pruneShortRefs(); + persistShortRefs(refs.map((ref, index) => ({ ref, entry: entries[index]! }))); + } + return refs; +} + +/** + * Every record in this namespace that derived `ref` — normally one. + * + * Registration keys entries by (kind, id, namespace), so two survivors mean a + * genuine digest collision. Rather than silently answering with one of them, + * read paths can surface both and let the caller pick; a 65-bit collision should + * never occur, so it is also counted and warned about. + */ +export function resolveMemoryShortRefCandidates(ref: string, namespace: ContextNamespace): MemoryShortRefEntry[] { + ensurePersistedLoaded(); + const bucket = entriesByRef.get(normalizeRef(ref)); + if (!bucket || bucket.length === 0) return []; + const sameNs = bucket.filter((entry) => sameNamespace(entry.namespace, namespace)); + if (sameNs.length > 1) { + incrementCounter('mem.short_ref.collision', { kind: sameNs[0]!.kind }); + warnOncePerHour('mem.short_ref.collision', { + ref: normalizeRef(ref), + ids: sameNs.map((entry) => entry.id).slice(0, 4), + }); + } + return sameNs; +} + export function resolveMemoryShortRef(ref: string, namespace?: ContextNamespace): MemoryShortRefEntry | undefined { ensurePersistedLoaded(); const bucket = entriesByRef.get(normalizeRef(ref)); if (!bucket || bucket.length === 0) return undefined; - const exact = namespace ? newestEntry(bucket.filter((entry) => sameNamespace(entry.namespace, namespace))) : undefined; - if (exact) return exact; + if (namespace) { + const sameNs = resolveMemoryShortRefCandidates(ref, namespace); + // More than one survivor means two different records genuinely derived the + // same handle. Callers that ACT on the result (archive/delete/update) must + // never operate on a guess, so this strict accessor refuses. Read paths can + // use resolveMemoryShortRefCandidates and present the alternatives instead. + if (sameNs.length > 1) return undefined; + const exact = sameNs[0]; + if (exact) return exact; + } + // Cross-namespace isolation: a handle registered under another namespace must + // NOT resolve here, even when it is the only entry for this ref. Callers other + // than get_memory_sources (archive/delete/update) also resolve refs, so this + // stays the strict boundary rather than deferring to a downstream check. if (namespace) return undefined; return bucket.length === 1 ? bucket[0] : undefined; } +/** + * Hydrate one handle from SQLite after an in-process miss. + * + * The memory MCP server is a separate, long-lived process. Startup/recall can + * persist new handles from the daemon after the MCP process has populated its + * local Map, so a one-time warm-load cannot keep the two processes coherent. + * Querying by exact ref uses the memory_short_refs primary-key prefix and avoids + * rescanning the bounded warm-load window on every miss. + */ +async function hydrateMemoryShortRefFromStore(ref: string): Promise { + if (shortRefStorePath()) return 0; + const normalizedRef = normalizeRef(ref); + if (!normalizedRef) return 0; + const existing = persistedRefHydrations.get(normalizedRef); + if (existing) return existing; + const hydration = (async () => { + try { + const rows = await getContextStoreClient() + .run>>('listMemoryShortRefsByRef', [normalizedRef]); + if (!Array.isArray(rows)) { + reportShortRefFailure('hydrate_ref', new Error('listMemoryShortRefsByRef returned a non-array response'), { + ref: normalizedRef, + }); + return 0; + } + // This ref is being actively redeemed. Promote hydrated rows in the + // process-local LRU before pruning so an old persisted handle cannot be + // inserted and immediately evicted while the index is at its 10k cap. + const merged = mergePersistedShortRefRows(rows, { touchLastSeenAt: true }); + reportDiscardedShortRefRows('hydrate_ref', merged.discarded); + pruneShortRefs(); + return merged.loaded; + } catch (error) { + reportShortRefFailure('hydrate_ref', error, { ref: normalizedRef }); + return 0; + } + })(); + persistedRefHydrations.set(normalizedRef, hydration); + try { + return await hydration; + } finally { + if (persistedRefHydrations.get(normalizedRef) === hydration) { + persistedRefHydrations.delete(normalizedRef); + } + } +} + +/** + * Async resolver for MCP reads. It preserves the synchronous resolver's exact + * namespace and collision semantics, adding only an indexed SQLite retry when + * this process has no candidate in the caller namespace. + */ +export async function resolveMemoryShortRefCandidatesWithStore( + ref: string, + namespace: ContextNamespace, +): Promise { + const candidates = resolveMemoryShortRefCandidates(ref, namespace); + if (candidates.length > 0) return candidates; + await hydrateMemoryShortRefFromStore(ref); + return resolveMemoryShortRefCandidates(ref, namespace); +} + +/** + * Strict async resolver for MCP mutations. Ambiguous handles and handles from a + * different namespace remain unresolved; the store lookup never authorizes a + * cross-namespace fallback. + */ +export async function resolveMemoryShortRefWithStore( + ref: string, + namespace?: ContextNamespace, +): Promise { + const resolved = resolveMemoryShortRef(ref, namespace); + if (resolved) return resolved; + await hydrateMemoryShortRefFromStore(ref); + return resolveMemoryShortRef(ref, namespace); +} + +/** + * Force several records onto one handle. A 65-bit digest collision cannot be + * produced by registering real ids, so the ambiguous-handle paths would + * otherwise be untestable above the resolver. + */ +export function seedMemoryShortRefCollisionForTests(ref: string, entries: readonly MemoryShortRefEntry[]): void { + ensurePersistedLoaded(); + entriesByRef.set(normalizeRef(ref), entries.map((entry) => ({ ...entry, lastSeenAt: entry.lastSeenAt ?? Date.now() }))); +} + export function resetMemoryShortRefsForTests(): void { entriesByRef.clear(); + persistedRefHydrations.clear(); + shortRefHealth = undefined; persistedLoaded = true; } export function reloadMemoryShortRefsForTests(): void { entriesByRef.clear(); + persistedRefHydrations.clear(); persistedLoaded = false; ensurePersistedLoaded(); } diff --git a/src/daemon/command-handler.ts b/src/daemon/command-handler.ts index c0ed4e7c0..1c2648747 100644 --- a/src/daemon/command-handler.ts +++ b/src/daemon/command-handler.ts @@ -129,6 +129,13 @@ import { resolveWindowsUpgradeRunnerPath, } from '../util/windows-upgrade-script.js'; import { buildBashSharpRepair } from '../util/sharp-repair-script.js'; +import { + buildPosixUpgradeLayoutRecoveryScript, + parsePosixUpgradeFailureStatus, + POSIX_UPGRADE_INSTALL_FAILURE_EXIT_CODE, +} from '../util/posix-upgrade-layout-recovery.js'; +import { warnOncePerHour } from '../util/rate-limited-warn.js'; +import { getDefaultUpgradeBlockedOutbox } from './upgrade-blocked-outbox.js'; import { encodeVbsAsUtf16, encodeCmdAsUtf8Bom } from '../util/windows-launch-artifacts.js'; import { registerTempFile, removeTrackedTempFile } from '../store/temp-file-store.js'; import { sanitizeProjectName } from '../../shared/sanitize-project-name.js'; @@ -156,7 +163,11 @@ import { getCodexRuntimeConfig } from '../agent/codex-runtime-config.js'; import { mergeCodexDisplayMetadata } from '../agent/codex-display.js'; import { P2P_TERMINAL_RUN_STATUSES } from '../../shared/p2p-status.js'; import { DAEMON_MSG } from '../../shared/daemon-events.js'; -import { DAEMON_UPGRADE_TARGET_LATEST, normalizeDaemonUpgradeTargetVersion } from '../../shared/daemon-upgrade.js'; +import { + DAEMON_UPGRADE_BLOCK_REASON, + DAEMON_UPGRADE_TARGET_LATEST, + normalizeDaemonUpgradeTargetVersion, +} from '../../shared/daemon-upgrade.js'; import { CC_PRESET_MSG, normalizeCcPresetName, type CcPreset } from '../../shared/cc-presets.js'; import { MEMORY_MCP_PROVIDER_IDS, @@ -962,6 +973,7 @@ import { QWEN_MODEL_IDS } from '../../shared/qwen-models.js'; import { getQwenRuntimeConfig } from '../agent/qwen-runtime-config.js'; import { getQwenDisplayMetadata } from '../agent/provider-display.js'; import { buildRelatedPastWorkText, buildStartupProjectMemoryText } from '../../shared/memory-recall-format.js'; +import { attachMemoryShortRefs } from '../context/memory-recall-refs.js'; import { getQwenOAuthQuotaUsageLabel, recordQwenOAuthRequest } from '../agent/provider-quota.js'; import { listProviderSessions as listProviderSessionsImpl } from './provider-sessions.js'; import { buildMemoryContextTimelinePayload, buildMemoryContextStatusPayload } from './memory-context-timeline.js'; @@ -1627,9 +1639,13 @@ function dispatchWebCommand(cmd: Record, serverLink: ServerLink case DAEMON_COMMAND_TYPES.DAEMON_UPGRADE: try { const normalizedTarget = normalizeDaemonUpgradeTargetVersion(cmd.targetVersion); + const upgradeId = typeof cmd.upgradeId === 'string' && cmd.upgradeId.length > 0 && cmd.upgradeId.length <= 128 + ? cmd.upgradeId + : undefined; void handleDaemonUpgrade( normalizedTarget === DAEMON_UPGRADE_TARGET_LATEST ? undefined : normalizedTarget, serverLink, + upgradeId, ); } catch { logger.warn({ targetVersion: cmd.targetVersion }, 'daemon.upgrade rejected invalid targetVersion'); @@ -6088,12 +6104,40 @@ async function handleAskAnswer(cmd: Record, serverLink: ServerL if (result === 'sent') { emitTransportUserMessageEvent(sessionName, answer); } else { + // DEADLOCK BREAK. `queuePlacement: 'front'` only wins against other + // QUEUED messages — it still waits for the active turn to settle. But + // the active turn is usually the very turn that is paused ON this + // question, so it can only settle once the answer arrives: the answer + // waits for the turn, the turn waits for the answer, and the user sees + // "answered, nothing happened". Providers without + // `answerPendingQuestion` (everything except claude-code-sdk today) + // hit this on every dialog. + // + // A dialog answer is a control response, not an ordinary queued send + // (see CLAUDE.md: approval/feedback responses must use the priority + // path and must not be blocked by the ordinary send queue). `cancel()` + // is exactly that priority path: it cuts in line, settles the active + // turn LOCALLY without waiting on the provider, keeps queued messages + // intact, and then drains — so our front entry becomes the next turn + // immediately. This is the "force-interrupt that re-steers the model" + // the branch above always intended. + // + // The drain emits the visible `user.message` for the queued entry, so + // do NOT emit it here as well or the answer renders twice. timelineEmitter.emit( sessionName, 'session.state', buildTransportQueueSessionStatePayload(sessionName, 'queued', 'command_handler_ask_answer'), { source: 'daemon', confidence: 'high' }, ); + try { + await runtime.cancel(); + } catch (err) { + logger.warn( + { err, sessionName }, + 'ask.answer: force-interrupt of the paused turn failed; answer stays queued at the front', + ); + } } } else { // Timed out / already self-continued and no live runtime snapshot is @@ -6866,7 +6910,11 @@ async function resolveUpgradeRegistry(): Promise<{ base: string; explicit: boole return { base, explicit: base !== INSTALLER_OFFICIAL_NPM_REGISTRY }; } -async function handleDaemonUpgrade(targetVersion?: string, serverLink?: ServerLink): Promise { +async function handleDaemonUpgrade( + targetVersion?: string, + serverLink?: ServerLink, + upgradeId?: string, +): Promise { const UPGRADE_MEMORY_FREEZE_TTL_MS = 15 * 60 * 1000; // ── Opt-out: forcibly disable the daemon's self-upgrade ─────────────────── @@ -7169,6 +7217,7 @@ async function handleDaemonUpgrade(targetVersion?: string, serverLink?: ServerLi const scriptDir = mkdtempSync(join(tmpdir(), 'imcodes-upgrade-')); const logFile = join(scriptDir, 'upgrade.log'); + const statusFile = join(scriptDir, 'upgrade-status.json'); const scriptPath = join(scriptDir, 'upgrade.sh'); // Build the platform-specific restart command. // We always restart regardless of whether npm install succeeded, so the daemon @@ -7338,10 +7387,23 @@ launchctl load -w "${plist}"`; LOG="${logFile}" SCRIPT_DIR="${scriptDir}" +UPGRADE_STATUS_FILE="${statusFile}" CLEANUP_AFTER_SEC=${CLEANUP_AFTER_SEC} REGISTRY_ARG="${registryArg}" log() { echo "[$(date '+%Y-%m-%dT%H:%M:%S%z')] $*" >> "$LOG"; } +${buildPosixUpgradeLayoutRecoveryScript()} + +write_install_failure_status() { + local retry_reason="$1" + local attempts="$2" + local exit_code="$3" + local status_tmp="$UPGRADE_STATUS_FILE.tmp.$$" + printf '{"state":"blocked","reason":"${DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED}","retryReason":"%s","attempts":%s,"exitCode":%s}\\n' \ + "$retry_reason" "$attempts" "$exit_code" > "$status_tmp" 2>>"$LOG" || return 1 + mv "$status_tmp" "$UPGRADE_STATUS_FILE" 2>>"$LOG" +} + schedule_self_cleanup() { if [ -z "$SCRIPT_DIR" ] || [ ! -d "$SCRIPT_DIR" ]; then return 0 @@ -7529,6 +7591,12 @@ GLOBAL_ROOT=$(eval "$NPM_RUN root -g" 2>>"$LOG") log "[step 1] global root: $GLOBAL_ROOT" GLOBAL_PKG="$GLOBAL_ROOT/imcodes" +# npm's rename destination is created before the incoming package's +# preinstall hook can run, so the package-level cleanup cannot heal this +# failure. Remove interrupted reify leftovers here, before npm starts. +cleanup_stale_imcodes_staging_dirs "$GLOBAL_ROOT" "$UPGRADE_LOCK_STALE_AFTER_SEC" \ + || log "[step 1.5] stale npm staging cleanup was incomplete; install may retry targeted recovery" + # Remove existing npm link if any — it shadows install and prevents real upgrade. if [ -L "$GLOBAL_PKG" ]; then log "[step 1] removing pre-existing npm link: $GLOBAL_PKG -> $(readlink "$GLOBAL_PKG")" @@ -7580,6 +7648,7 @@ INSTALL_OUT="${scriptDir}/install-attempt.log" INSTALL_RC=1 ATTEMPT=0 MAX_ATTEMPTS=5 +RETRY_REASON="not-classified" # Indexed sequentially with $ATTEMPT (1-based), so element 0 is unused. # 15s / 30s / 60s / 120s keeps the common npm publish-CDN window quick # without stretching a bad target into a 10-minute local stall. @@ -7604,6 +7673,7 @@ while [ "$ATTEMPT" -lt "$MAX_ATTEMPTS" ]; do VIEW_RC=$? cat "$INSTALL_OUT" >> "$LOG" if [ "$VIEW_RC" -ne 0 ] && is_etarget_output "$INSTALL_OUT"; then + RETRY_REASON="target-not-visible" log "[step 2] ${pkgSpec} not visible in registry yet" if [ "$ATTEMPT" -ge "$MAX_ATTEMPTS" ]; then log "[step 2] target never became visible across $MAX_ATTEMPTS attempts — giving up before heavyweight install" @@ -7642,6 +7712,12 @@ while [ "$ATTEMPT" -lt "$MAX_ATTEMPTS" ]; do elif is_transient_npm_output "$INSTALL_OUT"; then IS_RETRYABLE=1 RETRY_REASON="transient-network" + elif is_recoverable_layout_output "$INSTALL_OUT" "$GLOBAL_ROOT"; then + IS_RETRYABLE=1 + RETRY_REASON="stale-staging-dir" + if ! recover_stale_layout_from_output "$INSTALL_OUT" "$GLOBAL_ROOT"; then + log "[step 2] targeted stale staging cleanup failed; retrying remains bounded" + fi fi if [ "$IS_RETRYABLE" -ne 1 ]; then log "[step 2] non-retryable npm failure — not retrying. Tail of npm output:" @@ -7658,9 +7734,11 @@ while [ "$ATTEMPT" -lt "$MAX_ATTEMPTS" ]; do done if [ "$INSTALL_RC" -ne 0 ]; then log "[step 2] install FAILED after $ATTEMPT attempts (final exit $INSTALL_RC) — keeping current daemon running" + write_install_failure_status "$RETRY_REASON" "$ATTEMPT" "$INSTALL_RC" \ + || log "[step 2] failed to write upgrade status marker: $UPGRADE_STATUS_FILE" log "=== upgrade aborted ===" schedule_self_cleanup - exit 0 + exit ${POSIX_UPGRADE_INSTALL_FAILURE_EXIT_CODE} fi log "[step 2] install succeeded after $ATTEMPT attempt(s)" @@ -7957,6 +8035,40 @@ schedule_self_cleanup detached: true, stdio: 'ignore', }); + child.on('exit', (code, signal) => { + if (code !== POSIX_UPGRADE_INSTALL_FAILURE_EXIT_CODE) return; + let status = null; + try { + status = parsePosixUpgradeFailureStatus(readFileSync(statusFile, 'utf8')); + } catch { + // The exit code is still authoritative even if the diagnostic marker + // could not be read (for example, the tmp directory was removed). + } + const diagnostic = { + targetVersion: targetVersion ?? DAEMON_UPGRADE_TARGET_LATEST, + log: logFile, + retryReason: status?.retryReason ?? 'unknown', + exitCode: status?.exitCode ?? code, + ...(signal ? { signal } : {}), + ...(status?.attempts !== undefined ? { attempts: status.attempts } : {}), + }; + warnOncePerHour('daemon.upgrade.install_failed', diagnostic); + const blockedMessage = { + type: DAEMON_MSG.UPGRADE_BLOCKED, + reason: DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED, + failureId: randomUUID(), + ...(upgradeId ? { upgradeId } : {}), + fromVersion: DAEMON_VERSION, + ...diagnostic, + ts: Date.now(), + } as const; + void getDefaultUpgradeBlockedOutbox() + .report(blockedMessage, (message) => serverLink?.trySend?.(message) ?? false) + .catch((err) => { + logger.warn({ err, failureId: blockedMessage.failureId }, 'daemon.upgrade: failed to persist install failure'); + }); + releaseUpgradeMemoryFreeze(); + }); child.unref(); logger.info({ log: logFile }, 'daemon.upgrade: upgrade script spawned, will restart in ~3 s'); @@ -10574,11 +10686,28 @@ async function handlePersonalMemoryQuery(cmd: Record, serverLin includeArchived, }; const projects = await getContextStoreClient().run('listMemoryProjectSummaries', [summaryArgs]); + // The browser's manual "sync recent summaries" path must consume handles + // issued by the daemon. It previously derived a legacy 10-character value in + // Web code without registering it, so the displayed `proj:` token was dead. + // Keep ordinary management-list reads side-effect free; only the explicit + // sync caller asks us to register and persist handles. + const referencedRecords = cmd.includeShortRefs === true + ? attachMemoryShortRefs(records.map((record) => ({ + ...record, + type: 'processed' as const, + }))) + : []; + const responseRecords = referencedRecords.length > 0 + ? records.map((record, index) => ({ + ...record, + ...(referencedRecords[index]?.ref ? { ref: referencedRecords[index].ref } : {}), + })) + : records; serverLink.send({ type: MEMORY_WS.PERSONAL_RESPONSE, requestId, stats, - records, + records: responseRecords, pendingRecords, projects, }); @@ -12438,8 +12567,8 @@ async function prependLocalMemory( const semanticHitIds = finalItems.filter((item) => item.type === 'processed').map((item) => item.id); const hitIds = [...summaryItems.map((item) => item.id), ...semanticHitIds]; const sections: string[] = []; - if (summaryItems.length > 0) sections.push(buildStartupProjectMemoryText(summaryItems)); - if (finalItems.length > 0) sections.push(buildRelatedPastWorkText(finalItems)); + if (summaryItems.length > 0) sections.push(buildStartupProjectMemoryText(attachMemoryShortRefs(summaryItems))); + if (finalItems.length > 0) sections.push(buildRelatedPastWorkText(attachMemoryShortRefs(finalItems))); const injectedText = sections.join('\n\n'); const timelineItems = [...summaryItems, ...finalItems]; const timelinePayload = buildMemoryContextTimelinePayload(query, timelineItems); diff --git a/src/daemon/disk-usage.ts b/src/daemon/disk-usage.ts new file mode 100644 index 000000000..c95f181be --- /dev/null +++ b/src/daemon/disk-usage.ts @@ -0,0 +1,133 @@ +/** + * Cross-platform disk-capacity collection for the `daemon.stats` status bar. + * + * Efficiency: uses the `fs.statfs` syscall (libuv → statvfs / GetDiskFreeSpaceEx) + * — no `df`/`wmic` subprocess. Mount enumeration is a cheap file read on Linux + * (/proc/self/mounts), a readdir on macOS (/Volumes), and 26 fail-fast statfs + * probes on Windows (drive letters). Results are cached with a short TTL and + * refreshed in the background so the synchronous send path never blocks. + */ +import { statfs, readFile, readdir } from 'node:fs/promises'; +import type { DiskUsage } from '../../shared/disk-usage.js'; +import logger from '../util/logger.js'; + +// Pseudo / virtual filesystems to skip on Linux (fstype column of /proc/self/mounts). +const LINUX_PSEUDO_FSTYPES = new Set([ + 'proc', 'sysfs', 'devtmpfs', 'devpts', 'tmpfs', 'ramfs', 'securityfs', 'pstore', + 'bpf', 'cgroup', 'cgroup2', 'autofs', 'mqueue', 'debugfs', 'tracefs', 'hugetlbfs', + 'fusectl', 'configfs', 'binfmt_misc', 'nsfs', 'overlay', 'squashfs', 'efivarfs', + 'rpc_pipefs', 'selinuxfs', 'fuse.gvfsd-fuse', 'fuse.portal', 'fuse.snapfuse', + 'fuse.snapd', 'ramfs', 'devfs', +]); + +const DISK_CACHE_TTL_MS = 15_000; +// A statfs on a wedged network mount (e.g. a stale SMB/NFS volume under macOS +// /Volumes) can hang indefinitely; cap it so one bad mount can't stall the +// whole refresh (which would otherwise leave `refreshing` stuck and freeze the +// snapshot forever). Linux already excludes network fs (non-/dev/ devices). +const STATFS_TIMEOUT_MS = 2_000; +let cache: { at: number; disks: DiskUsage[] } | null = null; +let refreshing = false; + +function withTimeout(promise: Promise, ms: number): Promise { + return new Promise((resolve) => { + const timer = setTimeout(() => resolve(null), ms); + promise.then( + (value) => { clearTimeout(timer); resolve(value); }, + () => { clearTimeout(timer); resolve(null); }, + ); + }); +} + +async function usageForMount(mount: string): Promise { + try { + const s = await withTimeout(statfs(mount), STATFS_TIMEOUT_MS); + if (!s) return null; + const bsize = Number(s.bsize); + const totalBytes = Number(s.blocks) * bsize; + if (!Number.isFinite(totalBytes) || totalBytes <= 0) return null; + const usedBytes = Math.max(0, totalBytes - Number(s.bfree) * bsize); + const usedPercent = Math.min(100, Math.max(0, Math.round((usedBytes / totalBytes) * 100))); + return { mount, totalBytes, usedBytes, usedPercent }; + } catch { + // Non-existent drive letter (Windows), unmounted volume, permission — skip. + return null; + } +} + +async function enumerateMounts(): Promise { + if (process.platform === 'linux') { + const txt = await readFile('/proc/self/mounts', 'utf8').catch(() => ''); + const mounts: string[] = []; + const seenDev = new Set(); + for (const line of txt.split('\n')) { + const parts = line.split(' '); + const dev = parts[0]; + const mount = parts[1]; + const fstype = parts[2]; + if (!dev || !mount || !fstype) continue; + if (!dev.startsWith('/dev/')) continue; // real block devices only + if (LINUX_PSEUDO_FSTYPES.has(fstype)) continue; + if (seenDev.has(dev)) continue; // one entry per device (skip bind mounts) + seenDev.add(dev); + mounts.push(mount.replace(/\\040/g, ' ')); // /proc escapes spaces as \040 + } + return mounts.length ? mounts : ['/']; + } + if (process.platform === 'darwin') { + // No /proc on macOS. Root plus mounted volumes covers real disks; firmlink + // duplicates (/, /System/Volumes/Data) collapse in the dedupe below. + const mounts = ['/']; + const vols = await readdir('/Volumes').catch(() => [] as string[]); + for (const v of vols) mounts.push(`/Volumes/${v}`); + return mounts; + } + if (process.platform === 'win32') { + const drives: string[] = []; + for (let c = 65; c <= 90; c += 1) drives.push(`${String.fromCharCode(c)}:\\`); + return drives; // statfs fails fast for absent drives; usageForMount filters them out + } + return ['/']; +} + +/** Refresh the cached disk snapshot via fast statfs syscalls. Never throws. */ +export async function refreshDiskUsage(): Promise { + if (refreshing) return cache?.disks ?? []; + refreshing = true; + try { + const mounts = await enumerateMounts(); + const results = await Promise.all(mounts.map(usageForMount)); + const disks: DiskUsage[] = []; + const seen = new Set(); + for (const d of results) { + if (!d) continue; + // Collapse duplicates — macOS firmlinks and bind mounts report the same + // size+used against the same physical volume. + const key = `${d.totalBytes}:${d.usedBytes}`; + if (seen.has(key)) continue; + seen.add(key); + disks.push(d); + } + disks.sort((a, b) => b.usedPercent - a.usedPercent || b.totalBytes - a.totalBytes); + cache = { at: Date.now(), disks }; + return disks; + } catch (err) { + logger.debug({ err }, 'disk-usage: refresh failed'); + return cache?.disks ?? []; + } finally { + refreshing = false; + } +} + +/** + * Non-blocking accessor for the synchronous stats send path. Returns the cached + * snapshot and kicks off a background refresh when it's stale (or absent). The + * first call returns [] and warms the cache; the next stats tick carries data. + * Call `refreshDiskUsage()` once at startup to populate the first message. + */ +export function getDiskUsage(): DiskUsage[] { + if (!cache || Date.now() - cache.at > DISK_CACHE_TTL_MS) { + void refreshDiskUsage(); + } + return cache?.disks ?? []; +} diff --git a/src/daemon/file-transfer-handler.ts b/src/daemon/file-transfer-handler.ts index 7e3ae8a87..0a8037ca7 100644 --- a/src/daemon/file-transfer-handler.ts +++ b/src/daemon/file-transfer-handler.ts @@ -13,6 +13,7 @@ import logger from '../util/logger.js'; import { FILE_TRANSFER_LIMITS, FILE_TRANSFER_MSG, + FILE_TRANSFER_UPLOAD_ERROR_CODE, FILE_PATH_HANDLE_ERROR, type AttachmentRef, type FileUploadRequest, @@ -250,13 +251,23 @@ async function finalizeUploadedFile(params: { logger.info({ uploadId, filename, size }, 'File upload complete'); } +/** ENOSPC — the disk filled up mid-write. libuv surfaces it as error.code on + * both POSIX and Windows; the message text is a defensive fallback. */ +function isNoSpaceError(err: unknown): boolean { + if ((err as { code?: unknown } | null)?.code === 'ENOSPC') return true; + const msg = (err instanceof Error ? err.message : String(err)).toLowerCase(); + return msg.includes('enospc') || msg.includes('no space left'); +} + function sendUploadError(serverLink: FileTransferSender, uploadId: string, filename: string | undefined, err: unknown): void { const errMsg = err instanceof Error ? err.message : String(err); - logger.error({ uploadId, filename, err }, 'File upload failed'); + const code = isNoSpaceError(err) ? FILE_TRANSFER_UPLOAD_ERROR_CODE.INSUFFICIENT_CAPACITY : undefined; + logger.error({ uploadId, filename, err, code }, 'File upload failed'); const response: FileUploadError = { type: 'file.upload_error', uploadId, message: errMsg, + ...(code ? { code } : {}), }; serverLink.send(response); } diff --git a/src/daemon/latency-tracer.ts b/src/daemon/latency-tracer.ts index 6800e8c84..2fe83137a 100644 --- a/src/daemon/latency-tracer.ts +++ b/src/daemon/latency-tracer.ts @@ -4,6 +4,7 @@ import { homedir, loadavg } from 'node:os'; import { PerformanceObserver, monitorEventLoopDelay, performance } from 'node:perf_hooks'; import logger from '../util/logger.js'; import { MSG_COMMAND_ACK } from '../../shared/ack-protocol.js'; +import { DAEMON_MSG } from '../../shared/daemon-events.js'; import { TIMELINE_MESSAGES } from '../../shared/timeline-protocol.js'; import { TRANSPORT_EVENT, TRANSPORT_MSG } from '../../shared/transport-events.js'; @@ -643,6 +644,8 @@ export function classifyServerSendPlane(msgType: string | undefined): ServerSend || msgType === 'session.idle' || msgType === 'daemon.hello' || msgType === 'daemon.stats' + || msgType === DAEMON_MSG.UPGRADE_BLOCKED + || msgType === DAEMON_MSG.UPGRADE_BLOCKED_SYNC || msgType === 'heartbeat' // Live timeline events carry the chat stream/typewriter updates and // session.state transitions. They must bypass bulk history/data replay. diff --git a/src/daemon/lifecycle.ts b/src/daemon/lifecycle.ts index 6ac4b3bac..29f818905 100644 --- a/src/daemon/lifecycle.ts +++ b/src/daemon/lifecycle.ts @@ -1,12 +1,13 @@ import { loadStore, flushStore, listSessions, getSession, upsertSession, removeSession, type SessionRecord } from '../store/session-store.js'; -import { restoreFromStore, setSessionEventCallback, setSessionPersistCallback, restartSession, respawnSession, initOnStartup, rebuildProviderRoutes, getTransportRuntime, unregisterProviderRoute } from '../agent/session-manager.js'; +import { restoreFromStore, setSessionEventCallback, setSessionPersistCallback, restartSession, respawnSession, initOnStartup, rebuildProviderRoutes, getTransportRuntime, unregisterProviderRoute, resyncTransportSessionStatesAfterLinkRestore } from '../agent/session-manager.js'; import { sessionExists, isPaneAlive, BACKEND, killSession } from '../agent/tmux.js'; import { detectRepo } from '../repo/detector.js'; import { repoCache, RepoCache } from '../repo/cache.js'; -import { ServerLink } from './server-link.js'; +import { ServerLink, setServerLinkReconnectResyncHandler } from './server-link.js'; import { handleWebCommand, setRouterContext, refreshCodexQuotaMetadata, refreshClaudeSdkSubQuotaMetadata } from './command-handler.js'; import { dispatchSessionMessageByName } from './session-dispatch.js'; import { initFileTransfer, startCleanupTimer } from './file-transfer-handler.js'; +import { loadMemoryShortRefsFromStore } from '../context/memory-short-ref.js'; import { notifySessionIdle, listP2pRuns, serializeP2pRun } from './p2p-orchestrator.js'; import { isP2pParticipantMemoryNoise } from './p2p-memory-filter.js'; import { handlePreviewBinaryFrame } from './preview-relay.js'; @@ -62,7 +63,7 @@ import { import { buildWorkerSessionSyncPlan, type WorkerSessionSyncPlanInput } from './worker-session-sync-plan.js'; import { buildTransportQueueSnapshotPayload } from './transport-queue-projection.js'; import { getStaleSessionCompressionRun, resolveSessionCompressionWatchRuns } from '../context/summary-compressor.js'; -import { normalizeActivityGeneration } from '../../shared/session-activity-types.js'; +import { isServerLinkResyncStatePayload, normalizeActivityGeneration } from '../../shared/session-activity-types.js'; import type { TransportRuntimeDiagnosticSnapshot } from '../agent/transport-session-runtime.js'; function latestAssistantTextFromEvents(events: Array<{ type?: unknown; payload?: unknown }>): string | undefined { @@ -540,6 +541,14 @@ export async function startup(): Promise { startCleanupTimer(); logger.info('File transfer initialized'); + // Warm the memory short-ref index so handles injected before this restart + // still resolve. Resolution is synchronous (it runs inside render paths), so + // the durable map is read once here rather than per lookup. Best-effort: a + // cold index only means a handle re-registers on its next injection. + void loadMemoryShortRefsFromStore() + .then((loaded) => { if (loaded > 0) logger.info({ loaded }, 'Memory short-ref index warmed'); }) + .catch(() => { /* non-fatal: handles are a pure function of the id */ }); + // Clean up old timeline files (>7 days) and truncate oversized ones. // // Both calls walk every JSONL file in ~/.imcodes/timeline. With a backlog @@ -684,6 +693,14 @@ export async function startup(): Promise { let scheduleServerLinkRestoreBroadcast: (() => void) | null = null; if (creds) { serverLink = new ServerLink({ workerUrl: workerUrl!, serverId, token }); + // Heal the exact loss window observed on deck_sub_26624c1t: an + // authoritative `session.state: idle` emitted while the ServerLink socket + // was down is silently dropped (control-plane, no replay), leaving the + // server + every browser rendering the session "working" forever. After + // each RE-connect, re-broadcast every transport session's current state. + setServerLinkReconnectResyncHandler(() => { + resyncTransportSessionStatesAfterLinkRestore(); + }); serverLink.onMessage((msg) => { handleWebCommand(msg, serverLink!); }); @@ -814,6 +831,13 @@ export async function startup(): Promise { // Wire timeline idle events → P2P orchestrator + queued message drain (covers all agent types: CC, codex, gemini, etc.) timelineEmitter.on((e) => { + // A link-restore resync re-states what the daemon already knew so the server + // and browsers can catch up. It is NOT a fresh idle transition: running the + // side effects below for it made every reconnect drain the queue of every + // transport session at once, delivering messages that had been queued for + // days and flooding the user with notifications. Remote consumers still get + // the event; local listeners must ignore it. + if (e.type === 'session.state' && isServerLinkResyncStatePayload(e.payload)) return; liveContextIngestion.handleTimelineEvent(e); if (e.type === 'session.state' && (e.payload as Record).state === 'idle') { notifySessionIdle(e.sessionId); diff --git a/src/daemon/memory-context-timeline.ts b/src/daemon/memory-context-timeline.ts index ada98d0a5..dc5407ac8 100644 --- a/src/daemon/memory-context-timeline.ts +++ b/src/daemon/memory-context-timeline.ts @@ -6,6 +6,7 @@ import type { MemoryContextTimelineStatus, } from '../shared/timeline/types.js'; import { buildRelatedPastWorkText } from '../../shared/memory-recall-format.js'; +import { attachMemoryShortRefs } from '../context/memory-recall-refs.js'; import type { ContextAuthorityDecision, MemoryRecallInjectionSurface, @@ -31,8 +32,14 @@ export function buildMemoryContextTimelinePayload( .map((item) => ({ ...item, text: item.text.trim() })) .filter((item) => item.text); if (items.length === 0 && preferenceItems.length === 0) return null; - const timelineItems: MemoryContextTimelineItem[] = items.map((item) => ({ + // Register once, then reuse the exact same handles in both the agent-facing + // injected text and the user-facing timeline payload. Mapping first used to + // drop `type`, which made observation evidence look like a projection in the + // reconstructed injectedText and left the web card with no ref at all. + const referencedItems = attachMemoryShortRefs(items); + const timelineItems: MemoryContextTimelineItem[] = referencedItems.map((item) => ({ id: item.id, + ...(item.ref ? { ref: item.ref } : {}), projectId: item.projectId, ...('scope' in item && item.scope ? { scope: item.scope } : {}), ...('enterpriseId' in item && item.enterpriseId ? { enterpriseId: item.enterpriseId } : {}), @@ -46,7 +53,7 @@ export function buildMemoryContextTimelinePayload( relevanceScore: item.relevanceScore, })); const injectedText = options?.injectedText - ?? (timelineItems.length > 0 ? buildRelatedPastWorkText(timelineItems) : undefined); + ?? (referencedItems.length > 0 ? buildRelatedPastWorkText(referencedItems) : undefined); return { ...(query ? { query } : {}), ...(injectedText ? { injectedText } : {}), diff --git a/src/daemon/memory-inject.ts b/src/daemon/memory-inject.ts index c8b6da822..0775f2685 100644 --- a/src/daemon/memory-inject.ts +++ b/src/daemon/memory-inject.ts @@ -16,6 +16,7 @@ import { AGENT_SEND_DOCS } from './imcodes-workflow-docs.js'; import type { MemorySearchResultItem } from '../context/memory-search.js'; import { selectStartupMemoryForBootstrap } from '../context/memory-recall-client.js'; import { buildStartupProjectMemoryText } from '../../shared/memory-recall-format.js'; +import { attachMemoryShortRefs } from '../context/memory-recall-refs.js'; import logger from '../util/logger.js'; import { warnOncePerHour } from '../util/rate-limited-warn.js'; import { incrementCounter } from '../util/metrics.js'; @@ -139,7 +140,7 @@ export async function injectGeminiMemoryWithTimeline( export async function readProcessedMemory(projectName: string): Promise { const items = await readProcessedMemoryItems(projectName); if (items.length === 0) return null; - return buildStartupProjectMemoryText(items); + return buildStartupProjectMemoryText(attachMemoryShortRefs(items)); } export async function readProcessedMemoryItems(projectName: string): Promise { @@ -170,7 +171,7 @@ export async function buildSessionBootstrapContextWithItems( ): Promise<{ text: string; items: MemorySearchResultItem[] }> { const projectContext = await readProjectMemory(cwd); const items = await readProcessedMemoryItems(projectName); - const processedMemory = items.length > 0 ? buildStartupProjectMemoryText(items) : null; + const processedMemory = items.length > 0 ? buildStartupProjectMemoryText(attachMemoryShortRefs(items)) : null; const parts: string[] = []; if (projectContext) parts.push(projectContext); if (processedMemory) parts.push(processedMemory); diff --git a/src/daemon/memory-mcp-caller.ts b/src/daemon/memory-mcp-caller.ts index 4757f4b95..c71116fb3 100644 --- a/src/daemon/memory-mcp-caller.ts +++ b/src/daemon/memory-mcp-caller.ts @@ -1,4 +1,5 @@ import type { ContextNamespace } from '../../shared/context-types.js'; +import { LEGACY_DAEMON_LOCAL_USER_ID } from '../../shared/memory-namespace.js'; import { MEMORY_MCP_ENV_KEYS, type MemoryMcpEnvSource } from '../../shared/memory-mcp-env.js'; import { isMemoryScope, validateMemoryScopeIdentity } from '../../shared/memory-scope.js'; import { isValidImcodesSessionName } from '../../shared/session-scope.js'; @@ -23,7 +24,8 @@ export class MemoryMcpCallerEnvError extends Error { } } -const DAEMON_LOCAL_MEMORY_USER_ID = 'daemon-local'; +// Alias of the shared sentinel rather than a fourth copy of the literal. +const DAEMON_LOCAL_MEMORY_USER_ID = LEGACY_DAEMON_LOCAL_USER_ID; function optionalString(value: string | undefined): string | null { const trimmed = value?.trim(); diff --git a/src/daemon/memory-mcp-tools.ts b/src/daemon/memory-mcp-tools.ts index 8e6d5f96a..fc92e83ad 100644 --- a/src/daemon/memory-mcp-tools.ts +++ b/src/daemon/memory-mcp-tools.ts @@ -94,7 +94,14 @@ import { getContextStoreClient } from '../store/context-store-worker-client.js'; import { listSessions as listStoredSessions, loadStore, type SessionRecord } from '../store/session-store.js'; import { dispatchDestroyExecutionClone, dispatchSendMessage, dispatchSendStop, listSendTargets, type SendMessageCloneRequest, type SendToolDeps } from './send-tool.js'; import { cronMcpCreate, cronMcpCreateSelf, cronMcpDelete, cronMcpList, cronMcpUpdate, cronMcpUpdateSelf, type CronMcpClientOptions } from './cron-mcp-client.js'; -import { registerMemoryShortRef, resolveMemoryShortRef } from '../context/memory-short-ref.js'; +import { + registerMemoryShortRef, + resolveMemoryShortRefCandidatesWithStore, + resolveMemoryShortRefWithStore, +} from '../context/memory-short-ref.js'; + +/** Upper bound on records expanded for one colliding handle. */ +const AMBIGUOUS_REF_CANDIDATE_CAP = 4; import { GitOriginRepositoryIdentityService } from '../agent/repository-identity-service.js'; import { ALIAS_MCP_TOOLS, toAliasMetadata, type AliasMcpToolName } from '../../shared/alias-types.js'; import { @@ -759,13 +766,16 @@ function canManageProjectionNamespace(projectionNamespace: ContextNamespace, cal return !projectionUserId || projectionUserId === LEGACY_DAEMON_LOCAL_USER_ID || projectionUserId === callerUserId; } -function resolveProjectionRefArg(args: Record, namespace: ContextNamespace): string | ToolResult { +async function resolveProjectionRefArg( + args: Record, + namespace: ContextNamespace, +): Promise { const projectionId = stringArg(args, 'projectionId'); const ref = stringArg(args, 'ref'); if (projectionId && ref) return error(MCP_ERROR_REASONS.VALIDATION_FAILED, 'ref cannot be combined with projectionId'); if (projectionId) return projectionId; if (!ref) return error(MCP_ERROR_REASONS.VALIDATION_FAILED, 'projectionId or ref is required'); - const resolved = resolveMemoryShortRef(ref, namespace); + const resolved = await resolveMemoryShortRefWithStore(ref, namespace); if (!resolved || resolved.kind !== 'projection') { return error(MCP_ERROR_REASONS.PROJECTION_UNAVAILABLE, 'projection is not available in the caller namespace'); } @@ -952,8 +962,56 @@ export function createMemoryMcpToolHandlers(caller: McpRuntimeCaller, deps: Memo } if (!projectId) return emptySources(); if (ref) { - const resolved = resolveMemoryShortRef(ref, scopedCaller.namespace); - if (!resolved) return { status: 'ok', ref, sourceEventCount: 0, sources: [] }; + const candidates = await resolveMemoryShortRefCandidatesWithStore(ref, scopedCaller.namespace); + if (candidates.length === 0) return { status: 'ok', ref, sourceEventCount: 0, sources: [] }; + // A digest collision (two records deriving one handle) should never + // happen at 65 bits. If it does, answering with one of them would be a + // silently wrong memory — but refusing outright throws away information + // the caller can use. Expand every candidate and let the caller judge, + // flagged so it is never mistaken for a single authoritative answer. + if (candidates.length > 1) { + // The single-candidate path below rejects a kind that disagrees with + // the ref; the ambiguous path must enforce the same contract rather + // than quietly ignoring the caller's constraint. + if (kind && candidates.some((candidate) => candidate.kind !== kind)) { + return error(MCP_ERROR_REASONS.VALIDATION_FAILED, 'source lookup kind does not match the supplied ref'); + } + const expanded = []; + for (const candidate of candidates.slice(0, AMBIGUOUS_REF_CANDIDATE_CAP)) { + const identity = candidate.kind === 'observation' + ? { kind: candidate.kind, observationId: candidate.id } + : { kind: candidate.kind, projectionId: candidate.id }; + try { + if (candidate.kind === 'observation') { + expanded.push({ ...identity, ...(await memoryGetSources({ observationId: candidate.id, kind: 'observation' }, scopedCaller)) }); + continue; + } + const candidateResult = await orchestrator(candidate.id, scopedCaller); + if (candidateResult.status === 'error') { + expanded.push({ ...identity, unavailable: candidateResult.message }); + continue; + } + const { status: _candidateStatus, ...candidatePayload } = candidateResult; + expanded.push({ ...identity, ...candidatePayload }); + } catch { + expanded.push({ ...identity, unavailable: 'expansion failed' }); + } + } + return { + status: 'ok', + ref, + ambiguousRef: true, + // Expansion is bounded, so say how many records the handle actually + // covers. Reporting only the expanded subset while calling it every + // match would send the caller away believing it had seen them all. + candidateCount: candidates.length, + truncated: candidates.length > expanded.length, + candidates: expanded, + sourceEventCount: 0, + sources: [], + }; + } + const resolved = candidates[0]!; if (kind && kind !== resolved.kind) { return error(MCP_ERROR_REASONS.VALIDATION_FAILED, 'source lookup kind does not match the supplied ref'); } @@ -990,7 +1048,7 @@ export function createMemoryMcpToolHandlers(caller: McpRuntimeCaller, deps: Memo if (gate) return gate; const args = pickAllowedMcpArgs(input, ['projectionId', 'ref']); const scopedCaller = scopedCallerForDeps(caller, deps); - const projectionId = resolveProjectionRefArg(args, scopedCaller.namespace); + const projectionId = await resolveProjectionRefArg(args, scopedCaller.namespace); if (typeof projectionId !== 'string') return projectionId; const projection = await loadManageableProjection(projectionId, scopedCaller, getProcessedProjectionById); if (isToolResultValue(projection)) return projection; @@ -1003,7 +1061,7 @@ export function createMemoryMcpToolHandlers(caller: McpRuntimeCaller, deps: Memo if (gate) return gate; const args = pickAllowedMcpArgs(input, ['projectionId', 'ref']); const scopedCaller = scopedCallerForDeps(caller, deps); - const projectionId = resolveProjectionRefArg(args, scopedCaller.namespace); + const projectionId = await resolveProjectionRefArg(args, scopedCaller.namespace); if (typeof projectionId !== 'string') return projectionId; const projection = await loadManageableProjection(projectionId, scopedCaller, getProcessedProjectionById); if (isToolResultValue(projection)) return projection; @@ -1016,7 +1074,7 @@ export function createMemoryMcpToolHandlers(caller: McpRuntimeCaller, deps: Memo if (gate) return gate; const args = pickAllowedMcpArgs(input, ['projectionId', 'ref']); const scopedCaller = scopedCallerForDeps(caller, deps); - const projectionId = resolveProjectionRefArg(args, scopedCaller.namespace); + const projectionId = await resolveProjectionRefArg(args, scopedCaller.namespace); if (typeof projectionId !== 'string') return projectionId; const projection = await loadManageableProjection(projectionId, scopedCaller, getProcessedProjectionById); if (isToolResultValue(projection)) return projection; @@ -1031,7 +1089,7 @@ export function createMemoryMcpToolHandlers(caller: McpRuntimeCaller, deps: Memo const text = stringArg(args, 'text'); if (!text) return error(MCP_ERROR_REASONS.VALIDATION_FAILED, 'text is required'); const scopedCaller = scopedCallerForDeps(caller, deps); - const projectionId = resolveProjectionRefArg(args, scopedCaller.namespace); + const projectionId = await resolveProjectionRefArg(args, scopedCaller.namespace); if (typeof projectionId !== 'string') return projectionId; const projection = await loadManageableProjection(projectionId, scopedCaller, getProcessedProjectionById); if (isToolResultValue(projection)) return projection; @@ -1054,7 +1112,7 @@ export function createMemoryMcpToolHandlers(caller: McpRuntimeCaller, deps: Memo return error(MCP_ERROR_REASONS.VALIDATION_FAILED, 'feedback must be not_relevant or relevant'); } const scopedCaller = scopedCallerForDeps(caller, deps); - const projectionId = resolveProjectionRefArg(args, scopedCaller.namespace); + const projectionId = await resolveProjectionRefArg(args, scopedCaller.namespace); if (typeof projectionId !== 'string') return projectionId; const projection = await loadManageableProjection(projectionId, scopedCaller, getProcessedProjectionById); if (isToolResultValue(projection)) return projection; diff --git a/src/daemon/server-link.ts b/src/daemon/server-link.ts index 770be16ab..f7174cd35 100644 --- a/src/daemon/server-link.ts +++ b/src/daemon/server-link.ts @@ -3,11 +3,21 @@ import { performance } from 'node:perf_hooks'; import type { TimelineEvent } from './timeline-event.js'; import logger from '../util/logger.js'; import { DAEMON_VERSION } from '../util/version.js'; +import { DAEMON_MSG } from '../../shared/daemon-events.js'; +import { + DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION, + DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL, +} from '../../shared/daemon-upgrade.js'; import { setTransportRelaySend } from './transport-relay.js'; import { setProviderRegistryServerLink } from '../agent/provider-registry.js'; import { getDefaultAckOutbox } from './ack-outbox.js'; +import { getDefaultUpgradeBlockedOutbox } from './upgrade-blocked-outbox.js'; import { getEmbeddingStatus } from '../context/embedding.js'; import type { EmbeddingStatus } from '../../shared/embedding-status.js'; +import type { DiskUsage } from '../../shared/disk-usage.js'; +import { getDiskUsage, refreshDiskUsage } from './disk-usage.js'; +import { getMemoryShortRefHealth } from '../context/memory-short-ref.js'; +import type { MemoryShortRefHealth } from '../../shared/memory-short-ref-health.js'; import { recordDaemonServerLinkStatus } from '../util/daemon-status.js'; import { P2P_WORKFLOW_IMPLEMENTATION_CAPABILITY_V1, @@ -19,7 +29,7 @@ import { P2P_WORKFLOW_MSG } from '../../shared/p2p-workflow-messages.js'; import { SESSION_GROUP_CLONE_CAPABILITY_V1 } from '../../shared/session-group-clone.js'; import { EXECUTION_CLONE_CAPABILITY_V1 } from '../../shared/execution-clone.js'; import { GIT_REMOTE_CLONE_CAPABILITY_V1 } from '../../shared/git-remote-url.js'; -import { TIMELINE_PROTOCOL_CAPABILITY, TIMELINE_PROTOCOL_REVISION } from '../../shared/timeline-protocol.js'; +import { TIMELINE_MESSAGES, TIMELINE_PROTOCOL_CAPABILITY, TIMELINE_PROTOCOL_REVISION } from '../../shared/timeline-protocol.js'; import { FILE_TRANSFER_UPLOAD_FETCH_CAPABILITY, FILE_TRANSFER_DOWNLOAD_STREAM_CAPABILITY, @@ -32,6 +42,11 @@ import { stringifyForServerSend, } from './latency-tracer.js'; import { getDaemonBuildInfo } from './build-info.js'; +import { incrementCounter } from '../util/metrics.js'; +import { + TIMELINE_DELIVERY_METRICS, + countableTimelineEventType, +} from '../../shared/timeline-delivery-telemetry.js'; interface SystemStats { cpu: number; @@ -46,10 +61,17 @@ interface SystemStats { * goes stale across reconnects. See `getEmbeddingStatus` for state * semantics. */ embedding: EmbeddingStatus; + /** Mounted-filesystem capacity for the desktop status bar (mobile ignores it). */ + disks: DiskUsage[]; + /** Sticky memory-handle persistence failure, omitted while healthy. Rides the + * heartbeat because the counter and the daemon log both stay on the machine + * whose disk is usually what failed. */ + shortRefHealth?: MemoryShortRefHealth; } /** Collect lightweight system stats for daemon.stats messages. */ function collectSystemStats(): SystemStats { + const shortRefHealth = getMemoryShortRefHealth(); const memTotal = os.totalmem(); const memFree = os.freemem(); const [load1, load5, load15] = os.loadavg(); @@ -65,6 +87,8 @@ function collectSystemStats(): SystemStats { load15: +load15.toFixed(2), uptime: os.uptime(), embedding: getEmbeddingStatus(), + disks: getDiskUsage(), + ...(shortRefHealth ? { shortRefHealth } : {}), }; } @@ -150,6 +174,10 @@ const WATCHDOG_MS = 10_000; // check connection health every 10s // getOpenSocketSilenceMs() still prevents this from false-reconnecting a healthy // socket during the daemon's own load spikes (it reports 0 silence then). const SILENT_CONNECTION_RECYCLE_MS = 30_000; +// Throttle for the dropped-timeline-event warning. The counter records every +// drop; the log line is a human-facing heartbeat so a flapping link is visible +// in daemon.log without one line per lost message. +const TIMELINE_DROP_LOG_THROTTLE_MS = 30_000; // Event-loop stall guard. Under heavy local load the daemon's event loop can // freeze for tens of seconds. The silence-based reconnects measure // `now - lastPong`, which during a freeze blames the SERVER for the daemon's @@ -159,6 +187,17 @@ const SILENT_CONNECTION_RECYCLE_MS = 30_000; // silence checks stand down until inbound can actually be read again. const LOOP_PROBE_MS = 1_000; const EVENT_LOOP_STALL_THRESHOLD_MS = 3_000; // probe overdue by >3 s ⇒ loop stalled + +/** + * Invoked (async, best-effort) after every successful RE-connect so the session + * layer can re-broadcast authoritative per-session state that live control-plane + * relays dropped while the link was down. Registered from daemon startup + * (lifecycle) to avoid a server-link → session-manager import cycle. + */ +let serverLinkReconnectResyncHandler: (() => void) | null = null; +export function setServerLinkReconnectResyncHandler(handler: (() => void) | null): void { + serverLinkReconnectResyncHandler = handler; +} const DAEMON_STATIC_CAPABILITIES = [ SESSION_GROUP_CLONE_CAPABILITY_V1, // Distinct from session-group-clone — gates the dedicated execution-clone @@ -250,6 +289,13 @@ export class ServerLink { private backoffMs = INITIAL_BACKOFF_MS; private stopping = false; private reconnecting = false; + /** True once this link has completed at least one successful open. Gates the + * reconnect state-resync so a daemon's FIRST connect (startup already runs + * its own full session sync) does not double-broadcast. */ + private hadConnectedBefore = false; + /** Running total of content-bearing timeline events dropped while link down. */ + private timelineDropsWhileLinkDown = 0; + private lastTimelineDropLogAt = 0; private lastPong = 0; // timestamp of last received message (any message counts as proof of life) private seq = 0; private readonly workerUrl: string; @@ -346,7 +392,14 @@ export class ServerLink { }); // Send auth handshake immediately — server closes the socket if this is not // the first message or if credentials are invalid (5s timeout enforced server-side). - ws.send(JSON.stringify({ type: 'auth', serverId: this.serverId, token: this.token, daemonVersion: this.daemonVersion })); + ws.send(JSON.stringify({ + type: 'auth', + serverId: this.serverId, + token: this.token, + daemonVersion: this.daemonVersion, + [DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.AUTH_REVISION_FIELD]: + DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION, + })); this.sendDaemonHello(); // Wire transport relay so provider callbacks can send events to browsers via this socket. setTransportRelaySend((msg) => { @@ -370,6 +423,28 @@ export class ServerLink { outbox.flushOnReconnect(sender).catch((err) => { logger.warn({ err }, 'AckOutbox flush on reconnect failed'); }); + const sendUpgradeBlockedFrame = (message: unknown): boolean => { + if (this.ws !== ws) return false; + return this.trySend(message); + }; + getDefaultUpgradeBlockedOutbox() + .flushOnReconnect(sendUpgradeBlockedFrame, this.daemonVersion) + .then(() => { + // Connection-bound barrier: the server must not arm an automatic + // upgrade until this daemon has finished reading and replaying its + // persisted terminal blocker. Binding the sender to `ws` prevents a + // delayed flush from an old connection from certifying a replacement. + sendUpgradeBlockedFrame({ + type: DAEMON_MSG.UPGRADE_BLOCKED_SYNC, + revision: DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION, + }); + }) + .catch((err) => { + // Fail closed. Without a successful outbox read we cannot prove that + // the previous install did not terminally fail, so no sync-complete + // frame is sent and the server keeps automatic upgrade gated. + logger.warn({ err }, 'UpgradeBlockedOutbox flush on reconnect failed'); + }); // Resume the data-plane drain after reconnect. Anything that piled up // in `dataPlaneSendQueue` while the link was down is now safe to send @@ -377,6 +452,25 @@ export class ServerLink { // sit there until the next enqueue happened to schedule another flush. this.flushDataPlaneAfterReconnect(); + // Control-plane messages (live timeline events, incl. the authoritative + // `session.state: idle`) are DROPPED by trySend while the socket is not + // OPEN — there is no queue or replay for them. A turn that settles inside + // an outage window therefore leaves the server and every browser showing + // "working" forever. On every RE-connect, let the session layer + // re-broadcast each transport session's current authoritative state so + // the lost snapshot heals within one round-trip. + if (this.hadConnectedBefore && serverLinkReconnectResyncHandler) { + const handler = serverLinkReconnectResyncHandler; + setImmediate(() => { + try { + handler(); + } catch (err) { + logger.warn({ err }, 'ServerLink: reconnect state resync handler failed'); + } + }); + } + this.hadConnectedBefore = true; + // Refresh the supervisor global-defaults cache on every (re)connect so // user edits to "Global custom instructions" land in the daemon within // one WS round-trip, not next restart. See `supervisor-defaults-cache.ts`. @@ -422,6 +516,22 @@ export class ServerLink { } try { const msg = JSON.parse(event.data); + if ( + msg?.type === DAEMON_MSG.UPGRADE_BLOCKED_ACK + && typeof msg.failureId === 'string' + && ( + msg.disposition === DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.ACCEPTED + || msg.disposition === DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.OBSOLETE + || msg.disposition === DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.SUPERSEDED + ) + ) { + void getDefaultUpgradeBlockedOutbox() + .acknowledge(msg.failureId, msg.disposition) + .catch((err) => { + logger.warn({ err, failureId: msg.failureId }, 'UpgradeBlockedOutbox ack handling failed'); + }); + return; + } if (msg?.type === 'heartbeat_ack') { // Heartbeat acks are the CLI/status proof-of-life source. They must // not be throttled behind a just-written heartbeat-sent record, or @@ -476,6 +586,15 @@ export class ServerLink { // since the daemon must never die from transient disconnects. // Callers that need delivery confirmation should check isConnected() // or await a response event before acting on `send()`. + // + // "Silently" used to be literal. Live timeline events are control-plane, so + // this branch is where a chat message vanishes with no queue and no replay + // — the browser then only recovers it via a no-lower-bound backfill. Now + // that activation/reconnect heals that automatically, an increase in the + // underlying drop rate would be invisible, so count it. Only + // content-bearing events are counted; `agent.status` fires ~1/s during a + // turn and would bury the signal. + this.recordTimelineDropWhileLinkDown(msg); return false; } try { @@ -741,6 +860,34 @@ export class ServerLink { }); } + /** + * Count a content-bearing timeline event lost because the link was down. + * + * Periodically warns with the running total so a flapping link shows up in the + * daemon log as an accumulating number instead of silence. The counter itself + * is the durable signal; the log line is for humans reading `daemon.log`. + */ + private recordTimelineDropWhileLinkDown(msg: unknown): void { + if (messageTypeOf(msg) !== TIMELINE_MESSAGES.EVENT) return; + const eventType = countableTimelineEventType((msg as { event?: unknown } | null)?.event); + if (!eventType) return; + incrementCounter(TIMELINE_DELIVERY_METRICS.DAEMON_LINK_DOWN_DROPPED, { eventType }); + this.timelineDropsWhileLinkDown += 1; + const now = Date.now(); + if (now - this.lastTimelineDropLogAt < TIMELINE_DROP_LOG_THROTTLE_MS) return; + this.lastTimelineDropLogAt = now; + logger.warn({ + droppedTotal: this.timelineDropsWhileLinkDown, + eventType, + readyState: this.ws?.readyState ?? 'no-socket', + }, 'ServerLink: dropped content-bearing timeline event while link was down (browser must heal via backfill)'); + } + + /** Total content-bearing timeline events dropped while the link was down. */ + get timelineDropCountWhileLinkDown(): number { + return this.timelineDropsWhileLinkDown; + } + /** Reports whether the underlying WebSocket is currently OPEN. */ isConnected(): boolean { return !!this.ws && this.ws.readyState === WebSocket.OPEN; @@ -816,6 +963,8 @@ export class ServerLink { }, 10_000); } }, HEARTBEAT_MS); + // Warm the disk-usage cache so the first stats message already carries it. + void refreshDiskUsage(); // Stats updates more frequently than heartbeat this.statsTimer = setInterval(() => { if (this.ws?.readyState === WebSocket.OPEN) { diff --git a/src/daemon/supervision-automation.ts b/src/daemon/supervision-automation.ts index 5451968ed..5a7ae05aa 100644 --- a/src/daemon/supervision-automation.ts +++ b/src/daemon/supervision-automation.ts @@ -94,6 +94,9 @@ interface ActiveTaskRunState { // as soon as either automation or the current session delegates the audit, // and stays false while waiting for the reply. requiresAudit: boolean; + // Sticky safety gate set by REWORK. A broker decision cannot clear it; only + // a later matching peer-audit PASS may authorize repository finalization. + freshAuditRequiredAfterRework: boolean; continueLoops: number; continueStreakCount: number; lastContinueBucket?: string; @@ -217,19 +220,19 @@ function classifyContinueBucket(decision: { nextAction?: string; gap?: string; r return text.slice(0, 120); } -const REPOSITORY_FINALIZATION_ACTION_RE = /(?:\b(?:git\s+(?:add|commit|push)|commit|push|stage|staging)\b|提交|推送|暂存)/iu; -const SUBSTANTIVE_PRE_AUDIT_ACTION_RE = /(?:\b(?:test|tests|testing|typecheck|lint|build|verify|verification|validate|validation|fix|repair|implement|edit|modify|update|write|refactor|deploy|release|restart)\b|测试|类型检查|构建|验证|修复|实现|修改|更新|编写|重构|部署|发布|重启)/iu; +const REPOSITORY_FINALIZATION_ACTION_RE = /(?:\b(?:git\s+(?:add|commit|push|merge)|commit|push|stage|staging|merge|release|deploy|publish)\b|提交|推送|暂存|合并|发布|部署|上线)/iu; +const SUBSTANTIVE_PRE_AUDIT_ACTION_RE = /(?:\b(?:test|tests|testing|typecheck|lint|build|verify|verification|validate|validation|fix|repair|implement|edit|modify|update|write|refactor|restart)\b|测试|类型检查|构建|验证|修复|实现|修改|更新|编写|重构|重启)/iu; const COMPLETED_PRE_AUDIT_WORK_RE = /(?:\b(?:implementation|fix(?:es)?|coding|changes?|tests?|testing|typecheck|lint|build|verification|validation)\b[\s\S]{0,80}\b(?:complete|completed|done|finished|pass(?:ed)?)\b|(?:修复|实现|代码|改动|测试|验证|检查|类型检查|构建)[\s\S]{0,60}(?:已完成|已经完成|均已完成|全部完成|完成并通过|已通过|验证通过|测试通过))/iu; const PENDING_PRE_AUDIT_WORK_RE = /(?:\b(?:still|yet|remaining|pending|missing|failed?|incomplete|need(?:s)?\s+to|must)\b[\s\S]{0,50}\b(?:implementation|fix(?:es)?|tests?|testing|typecheck|lint|build|verification|validation)\b|\b(?:implementation|fix(?:es)?|tests?|testing|typecheck|lint|build|verification|validation)\b[\s\S]{0,50}\b(?:remain(?:s|ing)?|pending|missing|fail(?:ed|ing)?|incomplete|not\s+(?:done|complete)|need(?:s)?|required)\b|(?:仍|还|尚|待|未|缺少|失败)[\s\S]{0,30}(?:测试|验证|修复|实现|构建|类型检查)|(?:测试|验证|修复|实现|构建|类型检查)[\s\S]{0,30}(?:未完成|仍需|还需|待处理|失败|缺失|未通过))/iu; -const POST_AUDIT_REPOSITORY_FINALIZATION_ACTION = 'Peer-audit has passed. Finalize only the already-audited repository changes: stage the intended task files, commit them, and push the current branch. Do not request or start another audit.'; +const POST_AUDIT_REPOSITORY_FINALIZATION_ACTION = 'Peer-audit has passed. Perform only the already-audited repository or delivery finalization requested for this task (stage/commit/push, merge, release, publish, or deploy as applicable). Do not perform additional implementation work. Do not request or start another audit.'; type RepositoryFinalizationClassification = 'none' | 'finalization_only' | 'completion_evidenced_mixed'; /** * `supervised_audit` must review the implementation before repository * finalization. Only hold an action whose imperative next step is purely - * stage/commit/push work. Any instruction that also asks for tests, fixes, - * implementation, build, deployment, or another substantive mutation stays + * stage/commit/push/merge/release/deploy work. Any instruction that also asks + * for tests, fixes, implementation, build, or another substantive mutation stays * in the normal pre-audit continue loop. Audit/review words are deliberately * not substantive here: "commit after peer-audit PASS" describes the gate * this function is deciding to start, rather than work the target session @@ -748,6 +751,7 @@ class SupervisionAutomation { userText: text, phase: 'execution', requiresAudit: snapshot.mode === SUPERVISION_MODE.SUPERVISED_AUDIT, + freshAuditRequiredAfterRework: false, continueLoops: 0, continueStreakCount: 0, evaluating: false, @@ -1139,7 +1143,7 @@ class SupervisionAutomation { const latest = this.activeRuns.get(run.sessionName); if (!latest || latest.generation !== run.generation || latest.phase !== evaluatedPhase) return; latest.evaluating = false; - latest.requiresAudit = decision.requiresAudit !== false; + latest.requiresAudit = latest.freshAuditRequiredAfterRework || decision.requiresAudit !== false; switch (decision.decision) { case 'complete': { @@ -1220,7 +1224,7 @@ class SupervisionAutomation { const guardedNextAction = latest.phase === 'execution' && latest.snapshot.mode === SUPERVISION_MODE.SUPERVISED_AUDIT && hasRepositoryFinalizationAction(decision) - ? 'Complete only the remaining substantive implementation or validation work described by the supervisor. Do not stage, commit, or push; repository finalization is deferred until peer-audit PASS.' + ? 'Complete only the remaining substantive implementation or validation work described by the supervisor. Do not stage, commit, or push; do not merge, release, publish, or deploy. Repository and delivery finalization are deferred until peer-audit PASS.' : decision.nextAction; await this.dispatchContinue(latest, { reason: decision.reason, @@ -1414,6 +1418,7 @@ class SupervisionAutomation { if (verdict === 'PASS') { this.emitOrchestratedAuditResult(current, 'pass', undefined, findings); current.auditAttemptId = undefined; + current.freshAuditRequiredAfterRework = false; if (current.deferredFinalization) { current.phase = 'finalizing'; current.ignoreIdleUntilPostAuditTurnActivity = options.settledWithoutIdle === true; @@ -1443,6 +1448,7 @@ class SupervisionAutomation { const reworkBrief = buildReworkBrief(current, findings); current.phase = 'execution'; current.requiresAudit = true; + current.freshAuditRequiredAfterRework = true; current.ignoreIdleUntilPostAuditTurnActivity = options.settledWithoutIdle === true; current.evaluating = false; current.sawAssistantOutput = false; diff --git a/src/daemon/supervision-prompts.ts b/src/daemon/supervision-prompts.ts index 095159d59..b16d53a8b 100644 --- a/src/daemon/supervision-prompts.ts +++ b/src/daemon/supervision-prompts.ts @@ -67,11 +67,12 @@ function buildAuditBeforeFinalizationRule(request: SupervisionBrokerRequest): st if (request.snapshot?.mode !== SUPERVISION_MODE.SUPERVISED_AUDIT) return ''; return [ 'Audit-order rule for supervised_audit:', - '- Peer audit MUST finish before repository commit/push finalization.', - '- If implementation and validation are complete and the ONLY remaining action is git add/commit/push, return continue with that exact finalization-only nextAction; the daemon will hold it until peer-audit PASS instead of sending it now.', - '- If fixes, tests, typecheck, lint, build, validation, documentation changes, deployment, or any other substantive work remains, return continue normally so that work happens before audit.', - '- Never combine substantive pre-audit work and post-audit commit/push in one nextAction.', - '- If both the assistant response and your reason say implementation/validation are already complete, NEVER invent generic "remaining implementation or validation" work. Return only the concrete git add/commit/push finalization nextAction.', + '- Peer audit MUST finish before repository or delivery finalization, including git add/commit/push, merge, release, publish, and deploy actions.', + '- A REWORK verdict means the previous audit did NOT pass and grants no repository-finalization authority. After applying the requested fixes and validations, require a fresh matching peer audit and a new PASS before any git add/commit/push, merge, release, publish, or deploy action.', + '- If implementation and validation are complete and the ONLY remaining action is finalization such as git add/commit/push, merge, release, publish, or deploy, return continue with that exact finalization-only nextAction; the daemon will hold it until peer-audit PASS instead of sending it now.', + '- If fixes, tests, typecheck, lint, build, validation, documentation changes, or any other non-finalization substantive work remains, return continue normally so that work happens before audit.', + '- Never combine substantive pre-audit work and post-audit finalization in one nextAction.', + '- If both the assistant response and your reason say implementation/validation are already complete, NEVER invent generic "remaining implementation or validation" work. Return only the concrete repository or delivery finalization nextAction (git add/commit/push, merge, release, publish, or deploy as applicable).', '- Do not ask the target session to arrange or resend the audit in a normal continue decision. The daemon emits a separate orchestration prompt containing the exact auditor session ID and reply-enabled send command, exactly once.', ].join('\n'); } @@ -394,5 +395,8 @@ export function buildReworkBriefPrompt( verdictText, '', 'Apply the required fixes and continue the same task.', + 'This REWORK verdict means the previous audit did not pass. It is not authorization to finalize the repository.', + 'After fixing and validating the change, stop before repository finalization and report that the implementation is ready for a fresh peer audit.', + 'Do not stage, commit, push, merge, release, publish, or deploy until a new matching peer audit returns PASS.', ].join('\n'); } diff --git a/src/daemon/upgrade-blocked-outbox.ts b/src/daemon/upgrade-blocked-outbox.ts new file mode 100644 index 000000000..dd59850a5 --- /dev/null +++ b/src/daemon/upgrade-blocked-outbox.ts @@ -0,0 +1,197 @@ +/** + * Crash-safe single-record outbox for detached daemon-upgrade failures. + * + * The detached npm process can finish while ServerLink is disconnected. + * A plain `serverLink.send()` drops that control-plane message forever, so + * persist the latest terminal install failure before attempting delivery and + * replay it after the next authenticated WebSocket connection. + */ + +import { mkdir, readFile, rename, unlink, writeFile } from 'node:fs/promises'; +import { homedir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { DAEMON_MSG } from '../../shared/daemon-events.js'; +import { + DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION, + DAEMON_UPGRADE_BLOCK_REASON, + type DaemonUpgradeBlockedAckDisposition, +} from '../../shared/daemon-upgrade.js'; +import logger from '../util/logger.js'; + +const UPGRADE_BLOCKED_OUTBOX_TTL_MS = 7 * 24 * 60 * 60 * 1000; +const DEFAULT_FILE = join(homedir(), '.imcodes', 'pending-upgrade-blocked.json'); + +export interface UpgradeInstallFailedMessage { + type: typeof DAEMON_MSG.UPGRADE_BLOCKED; + reason: typeof DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED; + failureId: string; + upgradeId?: string; + fromVersion: string; + targetVersion: string; + retryReason: string; + attempts?: number; + exitCode: number; + log: string; + signal?: string; + ts: number; + acknowledgedAt?: number; +} + +export type UpgradeBlockedSender = (message: UpgradeInstallFailedMessage) => boolean; + +function boundedString(value: unknown, max: number): string | null { + return typeof value === 'string' && value.length > 0 && value.length <= max ? value : null; +} + +export function parseUpgradeInstallFailedMessage(raw: string): UpgradeInstallFailedMessage | null { + try { + const value = JSON.parse(raw) as Record; + if (value.type !== DAEMON_MSG.UPGRADE_BLOCKED) return null; + if (value.reason !== DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED) return null; + const failureId = boundedString(value.failureId, 128); + const upgradeId = value.upgradeId === undefined ? null : boundedString(value.upgradeId, 128); + const fromVersion = boundedString(value.fromVersion, 128); + const targetVersion = boundedString(value.targetVersion, 128); + const retryReason = boundedString(value.retryReason, 64); + const log = boundedString(value.log, 4096); + if (!failureId || (value.upgradeId !== undefined && !upgradeId) || !fromVersion || !targetVersion || !retryReason || !log) return null; + if (!/^[a-z][a-z0-9-]{0,63}$/.test(retryReason)) return null; + if (!Number.isInteger(value.exitCode) || (value.exitCode as number) < 1 || (value.exitCode as number) > 255) return null; + if (!Number.isInteger(value.ts) || (value.ts as number) < 1) return null; + if (value.attempts !== undefined && (!Number.isInteger(value.attempts) || (value.attempts as number) < 1 || (value.attempts as number) > 100)) return null; + if (value.signal !== undefined && typeof value.signal !== 'string') return null; + if (value.acknowledgedAt !== undefined && (!Number.isInteger(value.acknowledgedAt) || (value.acknowledgedAt as number) < 1)) return null; + return { + type: DAEMON_MSG.UPGRADE_BLOCKED, + reason: DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED, + failureId, + ...(upgradeId ? { upgradeId } : {}), + fromVersion, + targetVersion, + retryReason, + ...(value.attempts === undefined ? {} : { attempts: value.attempts as number }), + exitCode: value.exitCode as number, + log, + ...(typeof value.signal === 'string' ? { signal: value.signal } : {}), + ts: value.ts as number, + ...(value.acknowledgedAt === undefined ? {} : { acknowledgedAt: value.acknowledgedAt as number }), + }; + } catch { + return null; + } +} + +export class UpgradeBlockedOutbox { + private operation: Promise = Promise.resolve(); + + constructor( + private readonly filePath = DEFAULT_FILE, + private readonly now: () => number = Date.now, + ) {} + + /** + * Persist before send. A false/throwing sender leaves the record on disk for + * reconnect replay. A successful WebSocket enqueue does not remove the + * terminal blocker: it must survive server restarts until this daemon moves + * off fromVersion or the server explicitly declares the target obsolete. + */ + report(message: UpgradeInstallFailedMessage, send: UpgradeBlockedSender): Promise { + return this.serialize(async () => { + await this.persist(message); + return this.tryDeliver(message, send); + }); + } + + /** Replay one still-relevant terminal failure after authenticated reconnect. */ + flushOnReconnect(send: UpgradeBlockedSender, currentDaemonVersion: string): Promise { + return this.serialize(async () => { + const message = await this.load(); + if (!message) return false; + if ( + message.fromVersion !== currentDaemonVersion + || this.now() - message.ts > UPGRADE_BLOCKED_OUTBOX_TTL_MS + ) { + await this.clear(); + return false; + } + return this.tryDeliver(message, send); + }); + } + + /** + * Record that the server applied the blocker. Keep accepted blockers on disk + * so a later server restart can rebuild its in-memory terminal target gate. + * A changed server target makes the old failure obsolete and safe to clear. + */ + acknowledge(failureId: string, disposition: DaemonUpgradeBlockedAckDisposition): Promise { + return this.serialize(async () => { + const message = await this.load(); + if (!message || message.failureId !== failureId) return false; + if ( + disposition === DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.OBSOLETE + || disposition === DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.SUPERSEDED + ) { + await this.clear(); + return true; + } + if (message.acknowledgedAt === undefined) { + await this.persist({ ...message, acknowledgedAt: this.now() }); + } + return true; + }); + } + + private serialize(task: () => Promise): Promise { + const result = this.operation.then(task, task); + this.operation = result.then(() => undefined, () => undefined); + return result; + } + + private async persist(message: UpgradeInstallFailedMessage): Promise { + await mkdir(dirname(this.filePath), { recursive: true }); + const tmp = `${this.filePath}.tmp`; + await writeFile(tmp, `${JSON.stringify(message)}\n`, { encoding: 'utf8', mode: 0o600 }); + await rename(tmp, this.filePath); + } + + private async load(): Promise { + try { + const message = parseUpgradeInstallFailedMessage(await readFile(this.filePath, 'utf8')); + if (message) return message; + logger.warn({ file: this.filePath }, 'UpgradeBlockedOutbox: discarding invalid record'); + await this.clear(); + return null; + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return null; + throw err; + } + } + + private async tryDeliver(message: UpgradeInstallFailedMessage, send: UpgradeBlockedSender): Promise { + try { + return send(message); + } catch (err) { + logger.warn({ err, failureId: message.failureId }, 'UpgradeBlockedOutbox: send failed; retaining for reconnect'); + return false; + } + } + + private async clear(): Promise { + try { + await unlink(this.filePath); + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err; + } + } +} + +let defaultOutbox: UpgradeBlockedOutbox | null = null; + +export function getDefaultUpgradeBlockedOutbox(): UpgradeBlockedOutbox { + if (!defaultOutbox) defaultOutbox = new UpgradeBlockedOutbox(); + return defaultOutbox; +} + +export function __resetDefaultUpgradeBlockedOutboxForTests(): void { + defaultOutbox = null; +} diff --git a/src/shared/timeline/types.ts b/src/shared/timeline/types.ts index cd817ad96..2f85e7856 100644 --- a/src/shared/timeline/types.ts +++ b/src/shared/timeline/types.ts @@ -164,6 +164,8 @@ export interface PeerAuditStatusTimelinePayload { export interface MemoryContextTimelineItem { id: string; + /** Redeemable projection/observation handle shown to users and agents. */ + ref?: string; projectId: string; scope?: string; enterpriseId?: string; diff --git a/src/store/context-store.ts b/src/store/context-store.ts index 578e1a602..f7b39148c 100644 --- a/src/store/context-store.ts +++ b/src/store/context-store.ts @@ -445,6 +445,21 @@ function ensureDb(): DatabaseSyncInstance { ); CREATE INDEX IF NOT EXISTS idx_pinned_namespace ON context_pinned_notes(namespace_key); + -- ref → id map for the compact memory handles injected into agent context. + -- Durable so a handle still resolves after a daemon restart. Replaces a + -- whole-file JSON rewrite per registration whose failures were swallowed — + -- a full disk silently stopped persisting handles entirely. + CREATE TABLE IF NOT EXISTS memory_short_refs ( + ref TEXT NOT NULL, + kind TEXT NOT NULL, + id TEXT NOT NULL, + namespace_key TEXT NOT NULL DEFAULT '', + namespace_json TEXT, + last_seen_at INTEGER NOT NULL, + PRIMARY KEY (ref, kind, id, namespace_key) + ); + CREATE INDEX IF NOT EXISTS idx_memory_short_refs_seen ON memory_short_refs(last_seen_at); + CREATE TABLE IF NOT EXISTS context_replication_state ( namespace_key TEXT PRIMARY KEY, pending_projection_ids_json TEXT NOT NULL, @@ -2739,6 +2754,80 @@ export function listPinnedNotes(namespaceKey: string): PinnedNote[] { })); } +export interface MemoryShortRefRow { + ref: string; + kind: string; + id: string; + namespaceKey: string; + namespaceJson: string | null; + lastSeenAt: number; +} + +/** Persist memory handles. Batched into one transaction: the in-memory index + * registers a whole injected block at once. */ +export function upsertMemoryShortRefs(rows: readonly MemoryShortRefRow[]): number { + if (rows.length === 0) return 0; + const database = ensureDb(); + const statement = database.prepare(` + INSERT INTO memory_short_refs (ref, kind, id, namespace_key, namespace_json, last_seen_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(ref, kind, id, namespace_key) DO UPDATE SET + namespace_json = excluded.namespace_json, + last_seen_at = excluded.last_seen_at + `); + // IMMEDIATE, like every other write transaction here: the worker and the + // startup cold-fallback path can both hold a write connection, and a deferred + // transaction upgrading to a write under WAL fails with SQLITE_BUSY instead of + // waiting for the lock. + database.exec('BEGIN IMMEDIATE'); + try { + for (const row of rows) { + statement.run(row.ref, row.kind, row.id, row.namespaceKey, row.namespaceJson, row.lastSeenAt); + } + database.exec('COMMIT'); + } catch (error) { + try { database.exec('ROLLBACK'); } catch { /* a failing rollback must not mask the original error */ } + throw error; + } + return rows.length; +} + +/** Newest-first so a bounded warm-load keeps the most recently used handles. */ +export function listMemoryShortRefs(limit?: number): MemoryShortRefRow[] { + const database = ensureDb(); + const sql = 'SELECT * FROM memory_short_refs ORDER BY last_seen_at DESC' + + (typeof limit === 'number' && limit > 0 ? ' LIMIT ?' : ''); + const statement = database.prepare(sql); + const rows = (typeof limit === 'number' && limit > 0 ? statement.all(limit) : statement.all()) as Array>; + return mapMemoryShortRefRows(rows); +} + +/** + * Exact lookup used when another process persisted a handle after this + * process's in-memory short-ref index was populated. The primary key starts + * with `ref`, so this is an indexed point lookup rather than another warm-load. + */ +export function listMemoryShortRefsByRef(ref: string): MemoryShortRefRow[] { + const database = ensureDb(); + const rows = database.prepare(` + SELECT * FROM memory_short_refs + WHERE ref = ? + ORDER BY last_seen_at DESC + `).all(ref) as Array>; + return mapMemoryShortRefRows(rows); +} + +function mapMemoryShortRefRows(rows: Array>): MemoryShortRefRow[] { + return rows.map((row) => ({ + ref: String(row.ref), + kind: String(row.kind), + id: String(row.id), + namespaceKey: String(row.namespace_key ?? ''), + namespaceJson: row.namespace_json == null ? null : String(row.namespace_json), + lastSeenAt: Number(row.last_seen_at), + })); +} + export function getStagedEvent(id: string): LocalContextEvent | undefined { const database = ensureDb(); const row = database.prepare('SELECT * FROM context_staged_events WHERE id = ?').get(id) as Record | undefined; diff --git a/src/util/base32.ts b/src/util/base32.ts new file mode 100644 index 000000000..83119cf93 --- /dev/null +++ b/src/util/base32.ts @@ -0,0 +1,28 @@ +/** + * Minimal RFC 4648 base32 encoder, lowercase and unpadded. + * + * Node has no built-in base32 (`Buffer.toString` covers hex/base64/base64url + * only), and pulling a dependency for ~15 lines isn't worth it. Output is + * lowercase because callers normalize handles to lowercase; emitting it directly + * avoids a case round-trip. The alphabet omits 0/1/8/9, so base32 handles stay + * reasonably unambiguous when read off a screen. + */ +const BASE32_ALPHABET = 'abcdefghijklmnopqrstuvwxyz234567'; + +export function encodeBase32(bytes: Uint8Array): string { + let out = ''; + let value = 0; + let bits = 0; + for (const byte of bytes) { + // `bits` is drained below 5 on every iteration, so `value` never exceeds + // 20 significant bits — well inside the 32-bit range of `<<`/`>>>`. + value = (value << 8) | byte; + bits += 8; + while (bits >= 5) { + out += BASE32_ALPHABET[(value >>> (bits - 5)) & 31]; + bits -= 5; + } + } + if (bits > 0) out += BASE32_ALPHABET[(value << (5 - bits)) & 31]; + return out; +} diff --git a/src/util/metrics.ts b/src/util/metrics.ts index 548564706..d641e9371 100644 --- a/src/util/metrics.ts +++ b/src/util/metrics.ts @@ -1,36 +1,17 @@ -export type MetricLabels = Record; - -const counters = new Map(); -const MAX_COUNTERS = 1000; - -function labelsKey(labels?: MetricLabels): string { - if (!labels) return ''; - const entries = Object.entries(labels) - .filter(([, value]) => typeof value === 'string') - .sort(([a], [b]) => a.localeCompare(b)); - return entries.map(([key, value]) => `${key}=${value}`).join(','); -} - -function counterKey(name: string, labels?: MetricLabels): string { - const suffix = labelsKey(labels); - return suffix ? `${name}{${suffix}}` : name; -} - -export function incrementCounter(name: string, labels?: MetricLabels): void { - if (!name) return; - const key = counterKey(name, labels); - if (!counters.has(key) && counters.size >= MAX_COUNTERS) return; - counters.set(key, (counters.get(key) ?? 0) + 1); -} - -export function getCounter(name: string, labels?: MetricLabels): number { - return counters.get(counterKey(name, labels)) ?? 0; -} - -export function snapshotCounters(): Record { - return Object.fromEntries(counters.entries()); -} - -export function resetMetricsForTests(): void { - counters.clear(); -} +/** + * Daemon-facing entry point for in-process counters. + * + * The implementation lives in `shared/metrics.ts` so the daemon and the server + * cannot drift apart again (they already had, once: only the server copy grew + * `addCounter`). This file stays so the many existing `../util/metrics.js` + * imports keep working. + */ +export { + addCounter, + counterKeyCount, + getCounter, + incrementCounter, + resetMetricsForTests, + snapshotCounters, +} from '../../shared/metrics.js'; +export type { MetricLabels } from '../../shared/metrics.js'; diff --git a/src/util/posix-upgrade-layout-recovery.ts b/src/util/posix-upgrade-layout-recovery.ts new file mode 100644 index 000000000..5288020b0 --- /dev/null +++ b/src/util/posix-upgrade-layout-recovery.ts @@ -0,0 +1,175 @@ +import { DAEMON_UPGRADE_BLOCK_REASON } from '../../shared/daemon-upgrade.js'; + +export const POSIX_UPGRADE_INSTALL_FAILURE_EXIT_CODE = 75; + +export interface PosixUpgradeFailureStatus { + state: 'blocked'; + reason: typeof DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED; + retryReason: string; + attempts: number; + exitCode: number; +} + +/** + * Parse the status marker written by the detached POSIX upgrade script. + * Keep this strict: the marker crosses a process boundary and is later + * forwarded to the server as diagnostics. + */ +export function parsePosixUpgradeFailureStatus(raw: string): PosixUpgradeFailureStatus | null { + try { + const value = JSON.parse(raw) as Record; + if (value.state !== 'blocked' || value.reason !== DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED) return null; + if (typeof value.retryReason !== 'string' || !/^[a-z][a-z0-9-]{0,63}$/.test(value.retryReason)) return null; + if (!Number.isInteger(value.attempts) || (value.attempts as number) < 1 || (value.attempts as number) > 100) return null; + if (!Number.isInteger(value.exitCode) || (value.exitCode as number) < 1 || (value.exitCode as number) > 255) return null; + return { + state: 'blocked', + reason: DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED, + retryReason: value.retryReason, + attempts: value.attempts as number, + exitCode: value.exitCode as number, + }; + } catch { + return null; + } +} + +/** + * Bash helpers embedded verbatim in the detached Linux/macOS upgrade script. + * + * npm atomically renames the installed global package to a sibling named + * `.imcodes-` during reify. If an earlier install was interrupted, + * that deterministic staging destination can remain non-empty and every later + * install fails before the new package's preinstall hook can run. + */ +export function buildPosixUpgradeLayoutRecoveryScript(): string { + return ` +npm_error_field() { + local output_file="$1" + local wanted="$2" + awk -v wanted="$wanted" ' + $1 == "npm" && ($2 == "error" || $2 == "ERR!") && $3 == wanted { + $1 = ""; $2 = ""; $3 = "" + sub(/^[[:space:]]+/, "") + value = $0 + } + END { + if (value != "") print value + } + ' "$output_file" 2>/dev/null +} + +is_safe_global_root() { + local global_root="$1" + case "$global_root" in + /*) ;; + *) return 1 ;; + esac + [ "$global_root" != "/" ] && [ -d "$global_root" ] +} + +is_imcodes_staging_path() { + local global_root="$1" + local candidate="$2" + local suffix + is_safe_global_root "$global_root" || return 1 + case "$candidate" in + "$global_root"/.imcodes-*) ;; + *) return 1 ;; + esac + suffix=\${candidate#"$global_root"/.imcodes-} + case "$suffix" in + ''|*/*) return 1 ;; + esac + return 0 +} + +cleanup_stale_imcodes_staging_dirs() { + local global_root="$1" + local min_age_seconds="$2" + local stale + local modified_at + local now + local age + local cleanup_failed=0 + if ! is_safe_global_root "$global_root"; then + log "[step 1.5] refusing stale staging cleanup for unsafe global root: \${global_root:-}" + return 1 + fi + case "$min_age_seconds" in + ''|*[!0-9]*) + log "[step 1.5] refusing stale staging cleanup with invalid age threshold: \${min_age_seconds:-}" + return 1 + ;; + esac + now=$(date +%s) + for stale in "$global_root"/.imcodes-*; do + if [ ! -e "$stale" ] && [ ! -L "$stale" ]; then + continue + fi + if ! is_imcodes_staging_path "$global_root" "$stale"; then + log "[step 1.5] skipped unexpected staging path: $stale" + cleanup_failed=1 + continue + fi + # The daemon upgrade lock cannot coordinate with a user running npm + # manually. A freshly-created .imcodes-* directory may therefore belong + # to an active external npm reify and must not be removed by this broad + # sweep. Recent collisions are handled later from npm's exact rename + # error, where the reported destination proves which path blocked us. + modified_at=$(stat -c %Y "$stale" 2>/dev/null || stat -f %m "$stale" 2>/dev/null || true) + case "$modified_at" in + ''|*[!0-9]*) + log "[step 1.5] preserving staging directory with unknown age: $stale" + continue + ;; + esac + age=$((now - modified_at)) + if [ "$age" -lt 0 ] || [ "$age" -lt "$min_age_seconds" ]; then + log "[step 1.5] preserving recent npm staging directory (age \${age}s): $stale" + continue + fi + log "[step 1.5] removing stale npm staging directory: $stale" + if ! rm -rf -- "$stale"; then + log "[step 1.5] failed to remove stale npm staging directory: $stale" + cleanup_failed=1 + fi + done + return "$cleanup_failed" +} + +is_recoverable_layout_output() { + local output_file="$1" + local global_root="$2" + local error_code + local error_syscall + local error_path + local error_dest + error_code=$(npm_error_field "$output_file" code) + error_syscall=$(npm_error_field "$output_file" syscall) + error_path=$(npm_error_field "$output_file" path) + error_dest=$(npm_error_field "$output_file" dest) + case "$error_code" in + ENOTEMPTY|EEXIST) ;; + *) return 1 ;; + esac + [ "$error_syscall" = "rename" ] || return 1 + [ "$error_path" = "$global_root/imcodes" ] || return 1 + is_imcodes_staging_path "$global_root" "$error_dest" +} + +recover_stale_layout_from_output() { + local output_file="$1" + local global_root="$2" + local error_dest + is_recoverable_layout_output "$output_file" "$global_root" || return 1 + error_dest=$(npm_error_field "$output_file" dest) + if [ ! -e "$error_dest" ] && [ ! -L "$error_dest" ]; then + log "[step 2] stale npm staging destination already absent: $error_dest" + return 0 + fi + log "[step 2] removing npm staging destination from rename failure: $error_dest" + rm -rf -- "$error_dest" +} +`; +} diff --git a/test/agent/runtime-context-bootstrap.test.ts b/test/agent/runtime-context-bootstrap.test.ts index 83b06b0b1..ffb70acfe 100644 --- a/test/agent/runtime-context-bootstrap.test.ts +++ b/test/agent/runtime-context-bootstrap.test.ts @@ -4,7 +4,7 @@ import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { SKILL_REGISTRY_FILE_NAME, SKILL_REGISTRY_SCHEMA_VERSION, makeSkillUri } from '../../shared/skill-registry-types.js'; import { configureSharedContextRuntime } from '../../src/context/shared-context-runtime.js'; -import { resetMemoryShortRefsForTests, resolveMemoryShortRef } from '../../src/context/memory-short-ref.js'; +import { makeMemoryShortRef, resetMemoryShortRefsForTests, resolveMemoryShortRef } from '../../src/context/memory-short-ref.js'; import { ensureContextNamespace, writeContextObservation, writeProcessedProjection } from '../../src/store/context-store.js'; import { cleanupIsolatedSharedContextDb, createIsolatedSharedContextDb } from '../util/shared-context-db.js'; @@ -583,8 +583,11 @@ describe('resolveTransportContextBootstrap', () => { { summary: 'Important architecture memory', projectionClass: 'durable_memory_candidate' }, { summary: 'Recent startup memory', projectionClass: 'recent_summary' }, ]); - expect(startup?.injectedText).toContain('[important] Important architecture memory'); - expect(startup?.injectedText).toContain('[recent] Recent startup memory'); + // Each injected line carries a redeemable handle between the label and the + // summary, so the agent can call get_memory_sources when the summary alone + // isn't enough. base32 handle alphabet is a-z2-7. + expect(startup?.injectedText).toMatch(/\[important\] \(proj:[a-z2-7]{13}\) Important architecture memory/); + expect(startup?.injectedText).toMatch(/\[recent\] \(proj:[a-z2-7]{13}\) Recent startup memory/); }); it('injects only active or promoted observations as a lightweight startup index', async () => { @@ -647,8 +650,8 @@ describe('resolveTransportContextBootstrap', () => { }), ]); expect(startup?.injectedText).toContain(''); - const activeRef = `obs:${active.id.slice(0, 10)}`; - const promotedRef = `obs:${promoted.id.slice(0, 10)}`; + const activeRef = makeMemoryShortRef('observation', active.id); + const promotedRef = makeMemoryShortRef('observation', promoted.id); expect(startup?.injectedText).toContain(`ref: ${activeRef}`); expect(startup?.injectedText).toContain(`ref: ${promotedRef}`); expect(startup?.injectedText).not.toContain(active.id); diff --git a/test/agent/session-manager-state-resync.test.ts b/test/agent/session-manager-state-resync.test.ts new file mode 100644 index 000000000..99438c817 --- /dev/null +++ b/test/agent/session-manager-state-resync.test.ts @@ -0,0 +1,224 @@ +/** + * ServerLink reconnect state resync — regression for the deck_sub_26624c1t + * incident (2026-07-24 23:46 → 01:43): + * + * Live `timeline.event` relays are control-plane and are silently DROPPED by + * `ServerLink.trySend` while the socket is not OPEN. CC2's turn settled during + * a link outage, so its authoritative `session.state: idle` (seq 2271, + * decisionReason `activity_reconciler_clear`) never reached the server — every + * browser kept rendering "Agent working…" for two hours and held queued + * composer sends, while the daemon-local store/timeline were idle all along. + * + * `resyncTransportSessionStatesAfterLinkRestore()` re-broadcasts each transport + * session's CURRENT authoritative state after the link comes back. These tests + * pin: (1) the resync idle payload satisfies the shared authoritative-idle + * shape validator (a weak idle would be overridden client-side and NOT heal the + * stuck "working" footer); (2) running sessions re-broadcast running with + * activity evidence; (3) the session record is re-pushed through the persist + * callback so a store PUT lost in the same outage heals too. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../../src/util/logger.js', () => ({ + default: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); + +import { + resyncTransportSessionStatesAfterLinkRestore, + setSessionPersistCallback, +} from '../../src/agent/session-manager.js'; +import { timelineEmitter } from '../../src/daemon/timeline-emitter.js'; +import { TIMELINE_SUPPRESS_PUSH_FIELD } from '../../shared/push-notifications.js'; +import { + SESSION_STATE_DECISION_REASON_SERVER_LINK_RESYNC, + isAuthoritativeIdlePayloadShape, + isServerLinkResyncStatePayload, +} from '../../shared/session-activity-types.js'; +import { upsertSession, removeSession } from '../../src/store/session-store.js'; +import type { TransportSessionRuntime } from '../../src/agent/transport-session-runtime.js'; + +const IDLE_SESSION = 'deck_storecheckresync_w1'; +const RUNNING_SESSION = 'deck_storecheckresync_w2'; + +function makeRuntime(overrides: { + status: string; + blockingWorkCount?: number; + activeToolCount?: number; + busyReasons?: string[]; +}): TransportSessionRuntime { + return { + getStatus: () => overrides.status, + getDiagnosticSnapshot: () => ({ + activityGeneration: { scope: 'session', sessionName: IDLE_SESSION, generation: 3 }, + blockingWorkCount: overrides.blockingWorkCount ?? 0, + activeToolCount: overrides.activeToolCount ?? 0, + busyReasons: overrides.busyReasons ?? [], + }), + lastProviderError: undefined, + } as unknown as TransportSessionRuntime; +} + +describe('resyncTransportSessionStatesAfterLinkRestore', () => { + let emitSpy: ReturnType; + + beforeEach(() => { + // Intercept BEFORE any emit so nothing is persisted to timeline JSONL and + // nothing leaves this test (hygiene: shared/test-session-guard.ts names + // are used anyway as belt-and-braces). + emitSpy = vi.spyOn(timelineEmitter, 'emit').mockImplementation(() => null as never); + }); + + afterEach(() => { + emitSpy.mockRestore(); + setSessionPersistCallback(async () => {}); + removeSession(IDLE_SESSION); + removeSession(RUNNING_SESSION); + }); + + it('re-broadcasts an idle transport session as a FULL authoritative idle (not a weak idle)', () => { + const emitted = resyncTransportSessionStatesAfterLinkRestore([ + [IDLE_SESSION, makeRuntime({ status: 'idle' })], + ]); + + expect(emitted).toBe(1); + expect(emitSpy).toHaveBeenCalledTimes(1); + const [sessionName, type, payload, opts] = emitSpy.mock.calls[0] as unknown as [ + string, + string, + Record, + { eventId?: string }, + ]; + expect(sessionName).toBe(IDLE_SESSION); + expect(type).toBe('session.state'); + expect(payload.state).toBe('idle'); + expect(payload.decisionReason).toBe('server_link_resync'); + // The critical contract: a resync idle MUST satisfy the shared + // authoritative-idle validator. A weak idle is demoted client-side + // (isWeakIdlePayload) and cannot un-stick a "working" footer. + expect(isAuthoritativeIdlePayloadShape(payload)).toBe(true); + // Stable per-epoch eventId so a double-fired resync updates in place. + expect(opts.eventId).toMatch(new RegExp(`^transport-state-resync:${IDLE_SESSION}:\\d+$`)); + }); + + it('re-broadcasts a running transport session as running with activity evidence', () => { + const emitted = resyncTransportSessionStatesAfterLinkRestore([ + [RUNNING_SESSION, makeRuntime({ status: 'streaming', blockingWorkCount: 2, busyReasons: ['runtime_dispatch'] })], + ]); + + expect(emitted).toBe(1); + const [, , payload] = emitSpy.mock.calls[0] as unknown as [string, string, Record]; + expect(payload.state).toBe('running'); + expect(payload.blockingWorkCount).toBe(2); + expect(payload.busyReasons).toEqual(['runtime_dispatch']); + }); + + it('treats idle-with-blocking-work as running (same override as the live path)', () => { + resyncTransportSessionStatesAfterLinkRestore([ + [RUNNING_SESSION, makeRuntime({ status: 'idle', blockingWorkCount: 1 })], + ]); + const [, , payload] = emitSpy.mock.calls[0] as unknown as [string, string, Record]; + expect(payload.state).toBe('running'); + }); + + it('re-pushes the session record through the persist callback so a lost store PUT heals', async () => { + upsertSession({ + name: IDLE_SESSION, + projectName: 'storecheckresync', + role: 'w1', + agentType: 'claude-code-sdk', + runtimeType: 'transport', + state: 'idle', + } as never); + const persisted: string[] = []; + setSessionPersistCallback(async (_record, name) => { + persisted.push(name); + }); + + resyncTransportSessionStatesAfterLinkRestore([ + [IDLE_SESSION, makeRuntime({ status: 'idle' })], + ]); + + expect(persisted).toEqual([IDLE_SESSION]); + }); + + it('keeps going when one runtime throws and reports only successful sessions', () => { + const broken = { + getStatus: () => { + throw new Error('boom'); + }, + } as unknown as TransportSessionRuntime; + const emitted = resyncTransportSessionStatesAfterLinkRestore([ + ['deck_storecheckresync_w3', broken], + [IDLE_SESSION, makeRuntime({ status: 'idle' })], + ]); + expect(emitted).toBe(1); + }); + + it('is a no-op with no live transport runtimes', () => { + expect(resyncTransportSessionStatesAfterLinkRestore([])).toBe(0); + expect(emitSpy).not.toHaveBeenCalled(); + }); +}); + +describe('server-link resync payload marking', () => { + /** + * Incident: after a server/daemon reconnect the user got hundreds of push + * notifications, some quoting messages queued two days earlier. + * + * lifecycle.ts wires `timelineEmitter.on(...)` so that ANY + * `session.state: idle` runs `notifySessionIdle()` + `drainQueue()`. The + * resync re-broadcasts idle for EVERY transport session on EVERY reconnect, + * so each reconnect replayed a "turn finished" edge across all of them, + * draining every stale queue at once. Every agent answered, every answer fired + * an idle hook, and the phone got flooded. + * + * The resync payload must therefore be identifiable so daemon-local listeners + * can ignore it while the server and browsers still receive it. + */ + it('marks resync state payloads so local listeners can skip them', () => { + const emitSpy = vi.spyOn(timelineEmitter, 'emit').mockImplementation(() => null as never); + try { + resyncTransportSessionStatesAfterLinkRestore([ + [IDLE_SESSION, makeRuntime({ status: 'idle' })], + ]); + const [, , payload] = emitSpy.mock.calls[0] as unknown as [string, string, Record]; + expect(payload.state).toBe('idle'); + expect(payload.decisionReason).toBe(SESSION_STATE_DECISION_REASON_SERVER_LINK_RESYNC); + expect(isServerLinkResyncStatePayload(payload)).toBe(true); + } finally { + emitSpy.mockRestore(); + } + }); + + it('marks the resync idle as push-suppressed so a link restore cannot fan notifications', () => { + // The 43-deployment storm: a gateway blip dropped four daemons' sockets, all + // reconnected, and 276ms later ~90 push dispatches went out inside 500ms — + // one "Task complete" per idle session, quoting long-finished work. The + // server's PUSH_TIMELINE_EVENT_MAX_AGE_MS guard cannot catch these because a + // resync event is newly minted even though the state it reports is old, so + // the suppression has to be explicit. + // Local spy: this describe block has no shared one (see the sibling test). + const emitSpy = vi.spyOn(timelineEmitter, 'emit').mockImplementation(() => null as never); + try { + resyncTransportSessionStatesAfterLinkRestore([ + [IDLE_SESSION, makeRuntime({ status: 'idle' })], + ]); + const [, , payload] = emitSpy.mock.calls[0] as unknown as [string, string, Record]; + expect(payload[TIMELINE_SUPPRESS_PUSH_FIELD]).toBe(true); + // Suppressing the push must NOT weaken the idle: the whole point of the + // resync is to un-stick a "working" footer, which a weak idle cannot do. + expect(isAuthoritativeIdlePayloadShape(payload)).toBe(true); + } finally { + emitSpy.mockRestore(); + } + }); + + it('does not mark an ordinary live idle as a resync', () => { + // The live onStatusChange path uses `activity_reconciler_clear`; only that + // one may drive drainQueue / notifySessionIdle. + expect(isServerLinkResyncStatePayload({ state: 'idle', decisionReason: 'activity_reconciler_clear' })).toBe(false); + expect(isServerLinkResyncStatePayload({ state: 'idle' })).toBe(false); + expect(isServerLinkResyncStatePayload(null)).toBe(false); + expect(isServerLinkResyncStatePayload('server_link_resync')).toBe(false); + }); +}); diff --git a/test/context/memory-recall-refs-namespace.test.ts b/test/context/memory-recall-refs-namespace.test.ts new file mode 100644 index 000000000..e4fabe76a --- /dev/null +++ b/test/context/memory-recall-refs-namespace.test.ts @@ -0,0 +1,280 @@ +/** + * A handle injected into an agent's context must be redeemable by that agent. + * + * Live failure this pins: the ⚠️ "记忆句柄未保存" warning with 302 handle rows + * sitting in the database. The handles WERE persisted — under the wrong + * namespace. Registration derived it from recall-item fields, and recall items + * carry `scope` + `projectId` but no `userId`, so injected handles landed under + * `userId: ''`. The MCP server resolves personal / user_private memory as + * `userId: 'daemon-local'`, and `resolveMemoryShortRef` refuses a cross-namespace + * match on purpose, so `get_memory_sources` answered `sources: []` for every + * injected ref while the very same session's search-provided refs redeemed fine. + * + * Verified against the real store before the fix: the four injected refs I probed + * were all present with namespace_key `["personal","","","",""]`, while + * the working search-path refs read `["personal","daemon-local","","",""]` + * — one field apart. 290 of 302 rows (96%) carried the empty spelling, so nearly + * every handle ever minted on that machine was unreachable. + * + * Worse than unresolvable: `personal` declares + * `requiredIdentityFields: ['user_id', 'project_id']`, so those rows FAILED + * validation and the warm loader discarded them on every daemon start. The handles + * were being destroyed, not just missed. + * + * The owner is therefore not optional for owner-private memory — on a single-user + * device it is merely implicit. So the fix has two halves, and either one alone + * leaves the bug half-alive: + * 1. registration makes the owner explicit, so new rows are policy-valid; + * 2. the warm loader backfills the owner on already-stored owner-less rows + * instead of dropping them, which is what rescues the existing 290. + * + * Sharing does not transfer ownership, so SHARED scopes keep a real owner and stay + * isolated — the sentinel stands for "this device's owner", never for "anybody". + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +const { runMock } = vi.hoisted(() => ({ runMock: vi.fn() })); + +vi.mock('../../src/store/context-store-worker-client.js', () => ({ + getContextStoreClient: () => ({ run: runMock }), +})); + +import { attachMemoryShortRefs } from '../../src/context/memory-recall-refs.js'; +import { buildMemoryContextTimelinePayload } from '../../src/daemon/memory-context-timeline.js'; +import { + loadMemoryShortRefsFromStore, + resetMemoryShortRefsForTests, + resolveMemoryShortRef, +} from '../../src/context/memory-short-ref.js'; +import { LEGACY_DAEMON_LOCAL_USER_ID } from '../../shared/memory-namespace.js'; +import type { ContextNamespace } from '../../shared/context-types.js'; + +const PROJECT = 'github-im4codes/im4codes/imcodes'; + +/** Exactly what getDefaultMcpServers hands the MCP server for a personal scope. */ +const RESOLVER_NAMESPACE: ContextNamespace = { + scope: 'personal', + userId: LEGACY_DAEMON_LOCAL_USER_ID, + projectId: PROJECT, +}; + +describe('injected memory handles resolve for the agent they were injected into', () => { + let dir: string; + let priorPath: string | undefined; + let priorLegacy: string | undefined; + + beforeEach(() => { + priorPath = process.env.IMCODES_MEMORY_SHORT_REF_PATH; + priorLegacy = process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH; + dir = mkdtempSync(join(tmpdir(), 'imc-recall-refs-')); + // Keep the store path unset so persistence goes through the mocked client, + // and point the legacy file at a temp dir so the developer's real cache is + // never read. + delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; + process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = join(dir, 'legacy.json'); + runMock.mockReset(); + runMock.mockResolvedValue(undefined); + resetMemoryShortRefsForTests(); + }); + + afterEach(() => { + if (priorPath === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_PATH = priorPath; + if (priorLegacy === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = priorLegacy; + rmSync(dir, { recursive: true, force: true }); + }); + + it('redeems a personal projection handle even though the recall item carries no userId', () => { + // The injected shape, verbatim: scope + projectId, no userId. + const [item] = attachMemoryShortRefs([ + { id: 'proj-id-1', type: 'processed', scope: 'personal', projectId: PROJECT }, + ]); + + expect(item.ref).toBeTruthy(); + const resolved = resolveMemoryShortRef(item.ref!, RESOLVER_NAMESPACE); + expect(resolved?.id).toBe('proj-id-1'); + expect(resolved?.kind).toBe('projection'); + }); + + it('redeems an observation handle from the same injected shape', () => { + const [item] = attachMemoryShortRefs([ + { id: 'obs-id-1', type: 'observation', scope: 'user_private' }, + ]); + + expect(item.ref).toBeTruthy(); + const resolved = resolveMemoryShortRef(item.ref!, { + scope: 'user_private', + userId: LEGACY_DAEMON_LOCAL_USER_ID, + }); + expect(resolved?.id).toBe('obs-id-1'); + expect(resolved?.kind).toBe('observation'); + }); + + it('preserves the same observation handle in injected text and the web timeline payload', () => { + const payload = buildMemoryContextTimelinePayload('remember this observation', [{ + id: 'obs-timeline-1', + type: 'observation', + scope: 'user_private', + projectId: PROJECT, + summary: 'The user prefers compact audit reports', + }]); + + const item = payload?.items[0]; + expect(item?.ref).toMatch(/^obs:[a-z2-7]{13}$/); + expect(payload?.injectedText).toContain(`(${item?.ref})`); + expect(payload?.injectedText).not.toContain('(proj:'); + }); + + it('still redeems after a daemon restart, via the row that actually reached the store', async () => { + // The in-memory index would mask a namespace mismatch for the life of the + // process, so assert the round trip: persist, drop the index, warm-load from + // exactly what was written, resolve again. + const [item] = attachMemoryShortRefs([ + { id: 'proj-id-2', type: 'processed', scope: 'personal', projectId: PROJECT }, + ]); + const upsert = runMock.mock.calls.find(([name]) => name === 'upsertMemoryShortRefs'); + expect(upsert).toBeTruthy(); + const written = (upsert![1] as [Array>])[0]; + expect(written).toHaveLength(1); + + resetMemoryShortRefsForTests(); + runMock.mockReset(); + // `listMemoryShortRefs` in the store already maps snake_case columns to + // camelCase, so the warm loader receives exactly the rows that were written. + // (Re-converting them to snake_case here made this test fail against correct + // product code — the mock, not the loader, was wrong.) + runMock.mockImplementation(async (name: string) => ( + name === 'listMemoryShortRefs' ? written : undefined + )); + await loadMemoryShortRefsFromStore(); + + expect(resolveMemoryShortRef(item.ref!, RESOLVER_NAMESPACE)?.id).toBe('proj-id-2'); + }); + + it('rescues an already-stored owner-less row instead of discarding it at startup', async () => { + // The 290 rows written before registration filled the owner in. `personal` + // declares requiredIdentityFields ['user_id','project_id'], so validation + // rejected them and the warm loader dropped every one on each daemon start — + // the handles were not merely unresolvable, they were destroyed. A device has + // one owner, so an owner-less owner-private row is unambiguous. + resetMemoryShortRefsForTests(); + runMock.mockReset(); + runMock.mockImplementation(async (name: string) => ( + name === 'listMemoryShortRefs' + ? [{ + ref: 'proj:legacyownerless', + kind: 'projection', + id: 'legacy-id-1', + namespaceKey: JSON.stringify(['personal', '', PROJECT, '', '']), + namespaceJson: JSON.stringify({ scope: 'personal', projectId: PROJECT }), + lastSeenAt: Date.now(), + }] + : undefined + )); + + const loaded = await loadMemoryShortRefsFromStore(); + expect(loaded).toBe(1); + expect(resolveMemoryShortRef('proj:legacyownerless', RESOLVER_NAMESPACE)?.id).toBe('legacy-id-1'); + }); + + it('rescues a row whose owner was whitespace, not just absent or empty', async () => { + // Found by the second independent audit, with a real-SQLite probe: the + // normalizer treats a whitespace-only owner as missing, so such a row IS + // backfilled — but the loader used to rebuild the pre-backfill key by ASSUMING + // the owner had been `''`, so a row that stored `" "` in both its JSON and its + // key was discarded anyway. The pre-backfill namespace is now carried through + // verbatim instead of reconstructed, which removes the assumption. + resetMemoryShortRefsForTests(); + runMock.mockReset(); + runMock.mockImplementation(async (name: string) => ( + name === 'listMemoryShortRefs' + ? [{ + ref: 'proj:whitespaceowner', + kind: 'projection', + id: 'whitespace-id-1', + namespaceKey: JSON.stringify(['personal', ' ', PROJECT, '', '']), + namespaceJson: JSON.stringify({ scope: 'personal', projectId: PROJECT, userId: ' ' }), + lastSeenAt: Date.now(), + }] + : undefined + )); + + expect(await loadMemoryShortRefsFromStore()).toBe(1); + expect(resolveMemoryShortRef('proj:whitespaceowner', RESOLVER_NAMESPACE)?.id) + .toBe('whitespace-id-1'); + }); + + it('does not excuse an owner-less KEY on a row that carries an explicit owner', async () => { + // The rescue must only forgive rows that actually needed it. Offering the + // owner-less key shape to every row also excused a genuine key/JSON + // disagreement — the exact corruption this integrity check exists to catch. + // (Found by the independent audit of the first version of this fix.) + resetMemoryShortRefsForTests(); + runMock.mockReset(); + runMock.mockImplementation(async (name: string) => ( + name === 'listMemoryShortRefs' + ? [{ + ref: 'proj:mismatchedowner', + kind: 'projection', + id: 'mismatch-id-1', + // Key says nobody owns it; JSON says user-42 does. Inconsistent. + namespaceKey: JSON.stringify(['personal', '', PROJECT, '', '']), + namespaceJson: JSON.stringify({ scope: 'personal', projectId: PROJECT, userId: 'user-42' }), + lastSeenAt: Date.now(), + }] + : undefined + )); + + expect(await loadMemoryShortRefsFromStore()).toBe(0); + expect(resolveMemoryShortRef('proj:mismatchedowner', { + scope: 'personal', projectId: PROJECT, userId: 'user-42', + })).toBeUndefined(); + }); + + it('keeps SHARED scopes isolated by owner — sharing does not transfer ownership', () => { + // The narrow boundary: the owner is only implied for owner-private scopes. A + // shared record can genuinely belong to somebody else, so collapsing owners + // there would let one owner's handle resolve as another's. + const [item] = attachMemoryShortRefs([ + { id: 'proj-id-3', type: 'processed', scope: 'project_shared', projectId: PROJECT, userId: 'user-42' }, + ]); + + expect(item.ref).toBeTruthy(); + expect(resolveMemoryShortRef(item.ref!, { + scope: 'project_shared', projectId: PROJECT, userId: 'user-42', + })?.id).toBe('proj-id-3'); + // A different owner must NOT redeem it. + expect(resolveMemoryShortRef(item.ref!, { + scope: 'project_shared', projectId: PROJECT, userId: 'someone-else', + })).toBeUndefined(); + }); + + it('keeps an explicit real owner distinct from the daemon-local one', () => { + // Filling the owner in must not flatten real user ids together: the sentinel + // stands in for "this device's owner", not for "anybody". + const [item] = attachMemoryShortRefs([ + { id: 'proj-id-4', type: 'processed', scope: 'personal', projectId: PROJECT, userId: 'user-42' }, + ]); + + expect(resolveMemoryShortRef(item.ref!, { + scope: 'personal', projectId: PROJECT, userId: 'user-42', + })?.id).toBe('proj-id-4'); + expect(resolveMemoryShortRef(item.ref!, RESOLVER_NAMESPACE)).toBeUndefined(); + }); + + it('still isolates the dimensions that DO carry information', () => { + // Dropping the owner must not turn into "namespace no longer matters". + const [item] = attachMemoryShortRefs([ + { id: 'proj-id-5', type: 'processed', scope: 'personal', projectId: PROJECT }, + ]); + + expect(resolveMemoryShortRef(item.ref!, { scope: 'personal', projectId: 'other/project' })) + .toBeUndefined(); + expect(resolveMemoryShortRef(item.ref!, { scope: 'project_shared', projectId: PROJECT })) + .toBeUndefined(); + }); +}); diff --git a/test/context/memory-short-ref-health.test.ts b/test/context/memory-short-ref-health.test.ts new file mode 100644 index 000000000..3cdba1c10 --- /dev/null +++ b/test/context/memory-short-ref-health.test.ts @@ -0,0 +1,108 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const { runMock, incrementCounterMock, warnOncePerHourMock } = vi.hoisted(() => ({ + runMock: vi.fn(), + incrementCounterMock: vi.fn(), + warnOncePerHourMock: vi.fn(), +})); + +// Both existing signals are stubbed out entirely: the point is that the failure +// is still reportable when neither of them can carry it. The counter is a +// process-local map that a restart clears, and the throttled warning ends up in +// the daemon log — on the same disk whose exhaustion is the failure being +// reported, with write errors swallowed. +vi.mock('../../src/store/context-store-worker-client.js', () => ({ + getContextStoreClient: () => ({ run: runMock }), +})); +vi.mock('../../src/util/metrics.js', () => ({ incrementCounter: incrementCounterMock })); +vi.mock('../../src/util/rate-limited-warn.js', () => ({ warnOncePerHour: warnOncePerHourMock })); + +import { + getMemoryShortRefHealth, + loadMemoryShortRefsFromStore, + registerMemoryShortRefs, + resetMemoryShortRefsForTests, +} from '../../src/context/memory-short-ref.js'; + +describe('memory short refs — persistence failure leaves the process', () => { + let priorPath: string | undefined; + let priorLegacy: string | undefined; + + beforeEach(() => { + priorPath = process.env.IMCODES_MEMORY_SHORT_REF_PATH; + priorLegacy = process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH; + delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; + process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = '/nonexistent/imcodes-test/legacy.json'; + vi.clearAllMocks(); + resetMemoryShortRefsForTests(); + }); + + afterEach(() => { + resetMemoryShortRefsForTests(); + if (priorPath === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_PATH = priorPath; + if (priorLegacy === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = priorLegacy; + }); + + const entry = { + kind: 'projection' as const, + id: 'fb7a7af3-3185-45a4-ac47-b26a57142353', + namespace: { scope: 'personal' as const, userId: 'u1', projectId: 'p1' }, + }; + + it('reports nothing while persistence is healthy', async () => { + runMock.mockResolvedValue(1); + registerMemoryShortRefs([entry]); + await vi.waitFor(() => expect(runMock).toHaveBeenCalled()); + expect(getMemoryShortRefHealth()).toBeUndefined(); + }); + + it('exposes a disk-full write failure through a channel that is neither the store nor the log', async () => { + runMock.mockRejectedValue(Object.assign(new Error('ENOSPC: no space left on device'), { code: 'ENOSPC' })); + + registerMemoryShortRefs([entry]); + await vi.waitFor(() => expect(getMemoryShortRefHealth()).toBeDefined()); + + expect(getMemoryShortRefHealth()).toMatchObject({ + stage: 'persist_store', + failures: 1, + lastError: expect.stringContaining('ENOSPC'), + }); + expect(getMemoryShortRefHealth()!.lastFailureAt).toBeGreaterThan(0); + }); + + it('stays set after the failure so a later reader still sees it', async () => { + // Sticky, not edge-triggered: whoever reconnects after the incident has to + // be able to learn that handles stopped persisting. + runMock.mockRejectedValue(new Error('ENOSPC: no space left on device')); + registerMemoryShortRefs([entry]); + await vi.waitFor(() => expect(getMemoryShortRefHealth()).toBeDefined()); + + // Any number of later reads report the same standing failure. + expect(getMemoryShortRefHealth()).toBeDefined(); + expect(getMemoryShortRefHealth()).toBeDefined(); + + // A second failure accumulates rather than resetting, so a stuck disk is + // distinguishable from a single blip. + registerMemoryShortRefs([{ ...entry, id: 'second-id' }]); + await vi.waitFor(() => expect(getMemoryShortRefHealth()!.failures).toBeGreaterThan(1)); + }); + + it('reports a failed warm-load too, not only writes', async () => { + runMock.mockRejectedValue(new Error('context_store_unavailable')); + await loadMemoryShortRefsFromStore(); + expect(getMemoryShortRefHealth()).toMatchObject({ stage: 'warm_load' }); + }); + it('reports discarded rows too, since those handles are lost the same way', async () => { + // A row dropped as unusable means that memory is no longer reachable by + // handle after a restart — the same loss as a failed write, so it belongs on + // the same off-box signal rather than only in machine-local counters. + runMock.mockImplementation(async (op: string) => (op === 'listMemoryShortRefs' + ? [{ ref: 'proj:aaaaaaaaaaaaa', kind: 'projection', id: 'x', namespaceKey: '', namespaceJson: '{bad', lastSeenAt: 1 }] + : 1)); + + await loadMemoryShortRefsFromStore(); + expect(getMemoryShortRefHealth()).toMatchObject({ stage: 'discarded_warm_load' }); + }); +}); diff --git a/test/context/memory-short-ref-legacy-key.test.ts b/test/context/memory-short-ref-legacy-key.test.ts new file mode 100644 index 000000000..fc5d93a56 --- /dev/null +++ b/test/context/memory-short-ref-legacy-key.test.ts @@ -0,0 +1,177 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { resetContextStoreClientForTests } from '../../src/store/context-store-worker-client.js'; +import { upsertMemoryShortRefs } from '../../src/store/context-store.js'; +import { + loadMemoryShortRefsFromStore, + makeMemoryShortRef, + resetMemoryShortRefsForTests, + resolveMemoryShortRef, +} from '../../src/context/memory-short-ref.js'; +import { cleanupIsolatedSharedContextDb, createIsolatedSharedContextDb } from '../util/shared-context-db.js'; + +/** + * Exercised against a real SQLite store rather than a mocked client: the whole + * point is how node:sqlite round-trips a namespace key, which a mock cannot + * reproduce. + * + * Keys used to be NUL-separated. The bytes store fine and the primary key is + * unaffected. Older node:sqlite versions returned only the leading scope, + * while current versions return the complete NUL-containing JS string. + * Verifying either legacy shape only against the current JSON-tuple key would + * discard every handle written before the format changed. + */ +describe('memory short refs — legacy NUL-separated namespace keys', () => { + let tempDir: string; + let priorPath: string | undefined; + let priorLegacy: string | undefined; + + const namespace = { scope: 'personal' as const, userId: 'u1', projectId: 'p1' }; + const NUL = String.fromCharCode(0); + const legacyKey = [namespace.scope, namespace.userId, namespace.projectId, '', ''].join(NUL); + + beforeEach(async () => { + priorPath = process.env.IMCODES_MEMORY_SHORT_REF_PATH; + priorLegacy = process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH; + delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; + process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = '/nonexistent/imcodes-test/legacy.json'; + tempDir = await createIsolatedSharedContextDb('memory-short-ref-legacy-key'); + resetMemoryShortRefsForTests(); + }); + + afterEach(async () => { + resetMemoryShortRefsForTests(); + resetContextStoreClientForTests(); + if (priorPath === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_PATH = priorPath; + if (priorLegacy === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = priorLegacy; + await cleanupIsolatedSharedContextDb(tempDir); + }); + + it('still resolves a handle stored under the old NUL-separated key', async () => { + const id = 'fb7a7af3-3185-45a4-ac47-b26a57142353'; + const ref = makeMemoryShortRef('projection', id); + upsertMemoryShortRefs([{ + ref, kind: 'projection', id, + namespaceKey: legacyKey, + namespaceJson: JSON.stringify(namespace), + lastSeenAt: 1, + }]); + + await expect(loadMemoryShortRefsFromStore()).resolves.toBeGreaterThanOrEqual(1); + expect(resolveMemoryShortRef(ref, namespace)).toMatchObject({ id }); + }); + + it('still accepts the truncated read-back shape from older node:sqlite versions', async () => { + const id = 'eeeeeeee-1111-2222-3333-444444444444'; + const ref = makeMemoryShortRef('projection', id); + upsertMemoryShortRefs([{ + ref, kind: 'projection', id, + namespaceKey: namespace.scope, + namespaceJson: JSON.stringify(namespace), + lastSeenAt: 1, + }]); + + await expect(loadMemoryShortRefsFromStore()).resolves.toBeGreaterThanOrEqual(1); + expect(resolveMemoryShortRef(ref, namespace)).toMatchObject({ id }); + }); + + it('keeps two legacy rows that differ only by namespace apart', async () => { + // Accepting either legacy key shape must not let one namespace's handle + // answer for another's. + const id = 'aaaaaaaa-1111-2222-3333-444444444444'; + const ref = makeMemoryShortRef('projection', id); + const other = { scope: 'personal' as const, userId: 'u2', projectId: 'p2' }; + upsertMemoryShortRefs([ + { ref, kind: 'projection', id, namespaceKey: legacyKey, namespaceJson: JSON.stringify(namespace), lastSeenAt: 1 }, + { + ref, kind: 'projection', id, + namespaceKey: [other.scope, other.userId, other.projectId, '', ''].join(NUL), + namespaceJson: JSON.stringify(other), + lastSeenAt: 2, + }, + ]); + + await loadMemoryShortRefsFromStore(); + expect(resolveMemoryShortRef(ref, namespace)).toMatchObject({ id }); + expect(resolveMemoryShortRef(ref, other)).toMatchObject({ id }); + // A third, unrelated namespace must still miss. + expect(resolveMemoryShortRef(ref, { scope: 'personal', userId: 'u3', projectId: 'p3' } as never)).toBeUndefined(); + }); + + it('discards a row whose namespace column is present but unparseable', async () => { + // An empty key plus broken JSON previously decoded as "no namespace" and + // loaded, leaving a handle no namespaced caller can reach. + const id = 'bbbbbbbb-1111-2222-3333-444444444444'; + const ref = makeMemoryShortRef('projection', id); + upsertMemoryShortRefs([{ + ref, kind: 'projection', id, + namespaceKey: '', + namespaceJson: '{bad-json', + lastSeenAt: 1, + }]); + + await expect(loadMemoryShortRefsFromStore()).resolves.toBe(0); + expect(resolveMemoryShortRef(ref)).toBeUndefined(); + }); +}); + +describe('memory short refs — namespace values that name nothing reachable', () => { + let tempDir: string; + let priorPath: string | undefined; + let priorLegacy: string | undefined; + + beforeEach(async () => { + priorPath = process.env.IMCODES_MEMORY_SHORT_REF_PATH; + priorLegacy = process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH; + delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; + process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = '/nonexistent/imcodes-test/legacy.json'; + tempDir = await createIsolatedSharedContextDb('memory-short-ref-identity'); + resetMemoryShortRefsForTests(); + }); + + afterEach(async () => { + resetMemoryShortRefsForTests(); + resetContextStoreClientForTests(); + if (priorPath === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_PATH = priorPath; + if (priorLegacy === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = priorLegacy; + await cleanupIsolatedSharedContextDb(tempDir); + }); + + // Each row below pairs the namespace with the key that namespaceStorageKey + // would produce for it, so the key-consistency check agrees and ONLY the + // scope/identity rules can reject the row. Earlier tests always used a valid + // key, which meant a mismatch masked whether identity was checked at all. + const keyFor = (ns: Record) => JSON.stringify([ + ns.scope, ns.userId ?? '', ns.projectId ?? '', ns.workspaceId ?? '', ns.enterpriseId ?? '', + ]); + + it('discards a personal-scope row missing the identity that scope requires', async () => { + const id = 'cccccccc-1111-2222-3333-444444444444'; + const ref = makeMemoryShortRef('projection', id); + const ns = { scope: 'personal', userId: 'u1' }; // no projectId + upsertMemoryShortRefs([{ + ref, kind: 'projection', id, + namespaceKey: keyFor(ns), namespaceJson: JSON.stringify(ns), lastSeenAt: 1, + }]); + + await expect(loadMemoryShortRefsFromStore()).resolves.toBe(0); + expect(resolveMemoryShortRef(ref, ns as never)).toBeUndefined(); + }); + + it('discards a row whose namespace column holds the JSON text null', async () => { + // Parses successfully, so a parse-failure guard alone lets it through; it + // still describes no namespace. + const id = 'dddddddd-1111-2222-3333-444444444444'; + const ref = makeMemoryShortRef('projection', id); + upsertMemoryShortRefs([{ + ref, kind: 'projection', id, + namespaceKey: '', namespaceJson: 'null', lastSeenAt: 1, + }]); + + await expect(loadMemoryShortRefsFromStore()).resolves.toBe(0); + expect(resolveMemoryShortRef(ref)).toBeUndefined(); + }); +}); diff --git a/test/context/memory-short-ref-namespace.test.ts b/test/context/memory-short-ref-namespace.test.ts new file mode 100644 index 000000000..a8d5e7e42 --- /dev/null +++ b/test/context/memory-short-ref-namespace.test.ts @@ -0,0 +1,123 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +const { runMock, incrementCounterMock, warnOncePerHourMock } = vi.hoisted(() => ({ + runMock: vi.fn(), + incrementCounterMock: vi.fn(), + warnOncePerHourMock: vi.fn(), +})); + +vi.mock('../../src/store/context-store-worker-client.js', () => ({ + getContextStoreClient: () => ({ run: runMock }), +})); +vi.mock('../../src/util/metrics.js', () => ({ incrementCounter: incrementCounterMock })); +vi.mock('../../src/util/rate-limited-warn.js', () => ({ warnOncePerHour: warnOncePerHourMock })); + +import { + loadMemoryShortRefsFromStore, + resetMemoryShortRefsForTests, + resolveMemoryShortRef, +} from '../../src/context/memory-short-ref.js'; + +/** + * A stored namespace that cannot be fully validated must take the whole row + * down. Loading it namespace-less instead yields an entry that looks healthy but + * can never resolve for a namespaced caller, and counts as loaded — the handle + * disappears with no signal, which is precisely the failure this module exists + * to prevent. An earlier check only required `scope` to be a non-empty string, + * so an illegal scope or a corrupt identity field slipped through. + */ +describe('memory short refs — invalid namespaces are discarded, not degraded', () => { + let priorPath: string | undefined; + let priorLegacy: string | undefined; + let dir: string; + + beforeEach(() => { + priorPath = process.env.IMCODES_MEMORY_SHORT_REF_PATH; + priorLegacy = process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH; + dir = mkdtempSync(join(tmpdir(), 'imc-ns-')); + delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; + process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = join(dir, 'legacy.json'); + vi.clearAllMocks(); + resetMemoryShortRefsForTests(); + }); + + afterEach(() => { + resetMemoryShortRefsForTests(); + if (priorPath === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_PATH = priorPath; + if (priorLegacy === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = priorLegacy; + rmSync(dir, { recursive: true, force: true }); + }); + + const validNamespace = { scope: 'personal', userId: 'u1', projectId: 'p1' }; + /** Storage key is the JSON tuple written by namespaceStorageKey(). */ + const validKey = JSON.stringify(['personal', 'u1', 'p1', '', '']); + + const invalidNamespaces: Array<[string, unknown]> = [ + ['illegal scope value', { scope: 'not_a_scope', userId: 'u1' }], + ['non-string identity field', { scope: 'personal', userId: 42 }], + ['non-object namespace', 'personal'], + ]; + + for (const [label, namespace] of invalidNamespaces) { + it(`discards a store row with an ${label}`, async () => { + runMock.mockImplementation(async (op: string) => (op === 'listMemoryShortRefs' + ? [{ ref: 'proj:aaaaaaaaaaaaa', kind: 'projection', id: 'store-row', namespaceKey: validKey, namespaceJson: JSON.stringify(namespace), lastSeenAt: 1 }] + : 1)); + + await expect(loadMemoryShortRefsFromStore()).resolves.toBe(0); + expect(resolveMemoryShortRef('proj:aaaaaaaaaaaaa')).toBeUndefined(); + expect(incrementCounterMock).toHaveBeenCalledWith('mem.short_ref.discarded_row', { source: 'warm_load' }); + }); + + it(`discards a legacy JSON row with an ${label}`, async () => { + writeFileSync(join(dir, 'legacy.json'), JSON.stringify({ + schemaVersion: 2, + entries: [{ ref: 'proj:bbbbbbbbbbbbb', kind: 'projection', id: 'legacy-row', namespace }], + }), 'utf8'); + runMock.mockImplementation(async (op: string) => (op === 'listMemoryShortRefs' ? [] : 1)); + + await loadMemoryShortRefsFromStore(); + expect(resolveMemoryShortRef('proj:bbbbbbbbbbbbb')).toBeUndefined(); + expect(incrementCounterMock).toHaveBeenCalledWith('mem.short_ref.discarded_row', { source: 'legacy_file' }); + }); + } + + it('discards a store row whose namespace_key disagrees with its namespace_json', async () => { + // The key says the row is scoped while the JSON says it is not: the row's + // identity is corrupt, and loading it would file the handle under the wrong + // namespace. + runMock.mockImplementation(async (op: string) => (op === 'listMemoryShortRefs' + ? [{ ref: 'proj:ccccccccccccc', kind: 'projection', id: 'mismatch', namespaceKey: validKey, namespaceJson: null, lastSeenAt: 1 }] + : 1)); + + await expect(loadMemoryShortRefsFromStore()).resolves.toBe(0); + expect(incrementCounterMock).toHaveBeenCalledWith('mem.short_ref.discarded_row', { source: 'warm_load' }); + }); + + it('loads a row whose namespace is valid and consistent', async () => { + runMock.mockImplementation(async (op: string) => (op === 'listMemoryShortRefs' + ? [{ ref: 'proj:ddddddddddddd', kind: 'projection', id: 'ok-row', namespaceKey: validKey, namespaceJson: JSON.stringify(validNamespace), lastSeenAt: 1 }] + : 1)); + + await expect(loadMemoryShortRefsFromStore()).resolves.toBe(1); + expect(resolveMemoryShortRef('proj:ddddddddddddd', validNamespace as never)).toMatchObject({ id: 'ok-row' }); + expect(incrementCounterMock).not.toHaveBeenCalledWith('mem.short_ref.discarded_row', expect.anything()); + }); + + it('rejects a schemaVersion 1 legacy file wholesale', async () => { + // Handles from the previous derivation denote different records now. + writeFileSync(join(dir, 'legacy.json'), JSON.stringify({ + schemaVersion: 1, + entries: [{ ref: 'proj:deeaca10cd', kind: 'projection', id: 'v1-row', namespace: validNamespace }], + }), 'utf8'); + runMock.mockImplementation(async (op: string) => (op === 'listMemoryShortRefs' ? [] : 1)); + + await expect(loadMemoryShortRefsFromStore()).resolves.toBe(0); + expect(resolveMemoryShortRef('proj:deeaca10cd')).toBeUndefined(); + }); +}); diff --git a/test/context/memory-short-ref-persist-failure.test.ts b/test/context/memory-short-ref-persist-failure.test.ts new file mode 100644 index 000000000..be5132a37 --- /dev/null +++ b/test/context/memory-short-ref-persist-failure.test.ts @@ -0,0 +1,143 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const { runMock, incrementCounterMock, warnOncePerHourMock, writeFileSyncMock, mkdirSyncMock } = vi.hoisted(() => ({ + runMock: vi.fn(), + incrementCounterMock: vi.fn(), + warnOncePerHourMock: vi.fn(), + writeFileSyncMock: vi.fn(), + mkdirSyncMock: vi.fn(), +})); + +// The production-route test below strips VITEST/NODE_ENV so it exercises the +// real routing. Should the routing regress, the file branch would reactivate and +// write to the runner's actual home directory — a red test must never touch a +// developer's real data, and the true-negative procedure for this file would +// trigger exactly that. Stub the writes so the failure state is hermetic, which +// also lets the test assert positively that no file was written. +vi.mock('node:fs', () => ({ + writeFileSync: writeFileSyncMock, + mkdirSync: mkdirSyncMock, + readFileSync: () => { throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }); }, +})); + +vi.mock('../../src/store/context-store-worker-client.js', () => ({ + getContextStoreClient: () => ({ run: runMock }), +})); +vi.mock('../../src/util/metrics.js', () => ({ incrementCounter: incrementCounterMock })); +vi.mock('../../src/util/rate-limited-warn.js', () => ({ warnOncePerHour: warnOncePerHourMock })); + +import { + registerMemoryShortRefs, + resetMemoryShortRefsForTests, + resolveMemoryShortRef, +} from '../../src/context/memory-short-ref.js'; + +/** + * The whole point of moving handles off the JSON file was that its write errors + * were swallowed, so a full disk silently stopped persisting handles and nobody + * found out until they died on the next restart. Persisting to the store must + * not reintroduce that: a failed write is still non-fatal for the running + * process, but it has to be observable. + */ +describe('memory short refs — persistence failures stay observable', () => { + let priorPath: string | undefined; + + beforeEach(() => { + priorPath = process.env.IMCODES_MEMORY_SHORT_REF_PATH; + delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; // select the store path + vi.clearAllMocks(); + resetMemoryShortRefsForTests(); + }); + + afterEach(() => { + resetMemoryShortRefsForTests(); + if (priorPath === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_PATH = priorPath; + }); + + const entry = { + kind: 'projection' as const, + id: 'fb7a7af3-3185-45a4-ac47-b26a57142353', + namespace: { scope: 'personal' as const, userId: 'user-1', projectId: 'repo-1' }, + }; + + it('reports a counter and a rate-limited warning when the store write fails', async () => { + runMock.mockRejectedValue(new Error('context_store_unavailable')); + + const refs = registerMemoryShortRefs([entry]); + expect(refs).toHaveLength(1); + await vi.waitFor(() => expect(incrementCounterMock).toHaveBeenCalled()); + + expect(incrementCounterMock).toHaveBeenCalledWith( + 'mem.short_ref.persist_failure', + { stage: 'persist_store' }, + ); + expect(warnOncePerHourMock).toHaveBeenCalledWith( + 'mem.short_ref.persist_failure.persist_store', + expect.objectContaining({ error: 'context_store_unavailable' }), + ); + }); + + it('routes a production process to the store, not a JSON file', async () => { + // Regression: the path resolver returned ~/.imcodes/memory-short-refs.json + // for any process that wasn't a test, which inverted the routing — real + // daemons kept writing the JSON file while only tests exercised the store, + // so the migration and its failure reporting never ran where they mattered. + // + // The ambient VITEST flag makes a plain assertion here useless: it sends the + // OLD code down the store branch too, and the test passes either way. Drop + // the test markers so this exercises the real production route. + const priorVitest = process.env.VITEST; + const priorNodeEnv = process.env.NODE_ENV; + delete process.env.VITEST; + process.env.NODE_ENV = 'production'; + try { + runMock.mockResolvedValue(1); + registerMemoryShortRefs([entry]); + await vi.waitFor(() => expect(runMock).toHaveBeenCalled()); + expect(runMock).toHaveBeenCalledWith('upsertMemoryShortRefs', [ + expect.arrayContaining([expect.objectContaining({ id: entry.id, kind: 'projection' })]), + ]); + // The file branch must stay dormant without an explicit path override. + expect(writeFileSyncMock).not.toHaveBeenCalled(); + } finally { + if (priorVitest === undefined) delete process.env.VITEST; + else process.env.VITEST = priorVitest; + if (priorNodeEnv === undefined) delete process.env.NODE_ENV; + else process.env.NODE_ENV = priorNodeEnv; + } + }); + + it('keeps registration non-fatal and in-memory resolution working when the write fails', async () => { + runMock.mockRejectedValue(new Error('SQLITE_BUSY')); + // Registration must not throw: the caller is a synchronous render path. + let refs: string[] = []; + expect(() => { refs = registerMemoryShortRefs([entry]); }).not.toThrow(); + await vi.waitFor(() => expect(warnOncePerHourMock).toHaveBeenCalled()); + // The unpersisted handle still resolves for the life of this process. + expect(resolveMemoryShortRef(refs[0]!, entry.namespace)).toMatchObject({ id: entry.id }); + }); + + it('reports a failed warm-load instead of returning an ambiguous zero', async () => { + // A warm-load failure leaves every handle issued before this restart + // unresolvable for the whole process, and `0` alone is indistinguishable + // from "nothing was stored". + runMock.mockRejectedValue(new Error('context_store_unavailable')); + const { loadMemoryShortRefsFromStore } = await import('../../src/context/memory-short-ref.js'); + + await expect(loadMemoryShortRefsFromStore()).resolves.toBe(0); + expect(incrementCounterMock).toHaveBeenCalledWith( + 'mem.short_ref.persist_failure', + { stage: 'warm_load' }, + ); + }); + + it('stays quiet on a successful write', async () => { + runMock.mockResolvedValue(1); + registerMemoryShortRefs([entry]); + await Promise.resolve(); + await Promise.resolve(); + expect(incrementCounterMock).not.toHaveBeenCalled(); + expect(warnOncePerHourMock).not.toHaveBeenCalled(); + }); +}); diff --git a/test/context/memory-short-ref-store.test.ts b/test/context/memory-short-ref-store.test.ts new file mode 100644 index 000000000..9a874aeb7 --- /dev/null +++ b/test/context/memory-short-ref-store.test.ts @@ -0,0 +1,231 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { resetContextStoreClientForTests } from '../../src/store/context-store-worker-client.js'; +import { + listMemoryShortRefs, + listMemoryShortRefsByRef, + upsertMemoryShortRefs, +} from '../../src/store/context-store.js'; +import { readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { + loadMemoryShortRefsFromStore, + makeMemoryShortRef, + registerMemoryShortRefs, + resetMemoryShortRefsForTests, + resolveMemoryShortRef, + resolveMemoryShortRefCandidatesWithStore, + resolveMemoryShortRefWithStore, + seedMemoryShortRefCollisionForTests, +} from '../../src/context/memory-short-ref.js'; +import { cleanupIsolatedSharedContextDb, createIsolatedSharedContextDb } from '../util/shared-context-db.js'; + +/** + * Durable handle map. Handles used to persist by rewriting a whole JSON file on + * every registration, with the write error swallowed — once the disk filled up + * the file silently stopped being written, so every handle issued after that + * point died on the next daemon restart. Persist incrementally to the context + * store instead, and warm the in-memory index from it at startup. + */ +describe('memory short refs — durable store persistence', () => { + let tempDir: string; + let priorPath: string | undefined; + let priorLegacyPath: string | undefined; + + beforeEach(async () => { + // Unset so the store (not the JSON file) is the persistence target. + priorPath = process.env.IMCODES_MEMORY_SHORT_REF_PATH; + delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; + // Warm-load also imports the retired JSON cache. Point it at a path that + // does not exist so these assertions never read the developer's real file. + priorLegacyPath = process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH; + process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = '/nonexistent/imcodes-test/legacy-short-refs.json'; + tempDir = await createIsolatedSharedContextDb('memory-short-ref-store'); + resetMemoryShortRefsForTests(); + }); + + afterEach(async () => { + resetMemoryShortRefsForTests(); + resetContextStoreClientForTests(); + if (priorPath === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_PATH = priorPath; + if (priorLegacyPath === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = priorLegacyPath; + await cleanupIsolatedSharedContextDb(tempDir); + }); + + const namespace = { scope: 'personal' as const, userId: 'user-1', projectId: 'repo-1' }; + + it('persists registered handles to the store and resolves them after a restart', async () => { + const [ref] = registerMemoryShortRefs([ + { kind: 'projection', id: 'fb7a7af3-3185-45a4-ac47-b26a57142353', namespace }, + { kind: 'observation', id: '2cf7602fb9d586fa5377f22b879ca97b3ba4c5acbdd32b9df36ad15e2df0a8c6', namespace }, + ]); + expect(ref).toMatch(/^proj:[a-z2-7]{13}$/); + + // Registration is fire-and-forget; let the store write settle. + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(listMemoryShortRefs().length).toBeGreaterThanOrEqual(2); + + // Simulate a daemon restart: in-memory index gone, store intact. + resetMemoryShortRefsForTests(); + expect(resolveMemoryShortRef(ref!, namespace)).toBeUndefined(); + + const loaded = await loadMemoryShortRefsFromStore(); + expect(loaded).toBeGreaterThanOrEqual(2); + expect(resolveMemoryShortRef(ref!, namespace)).toMatchObject({ + kind: 'projection', + id: 'fb7a7af3-3185-45a4-ac47-b26a57142353', + }); + }); + + it('re-registering the same memory upserts instead of accumulating rows', async () => { + const entry = { kind: 'projection' as const, id: 'dddddddddd-1111-2222-3333-444444444444', namespace }; + registerMemoryShortRefs([entry]); + await new Promise((resolve) => setTimeout(resolve, 50)); + const afterFirst = listMemoryShortRefs().length; + + // The handle is a pure function of the id, so a repeat injection reuses the + // same primary key rather than growing the table. + registerMemoryShortRefs([entry]); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(listMemoryShortRefs().length).toBe(afterFirst); + }); + + it('imports handles from the retired JSON cache and moves them into the store', async () => { + // The JSON file is a read-only fallback during the migration: handles it + // still holds are valid, so carry them over rather than stranding them until + // their memory is injected again. The file itself is never written or + // deleted — it just stops being needed. + const legacyPath = join(tempDir, 'legacy-short-refs.json'); + process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = legacyPath; + const legacyId = '45663d84-d3bb-4f9a-bf65-eba630b45d66'; + const legacyRef = makeMemoryShortRef('projection', legacyId); + writeFileSync(legacyPath, JSON.stringify({ + schemaVersion: 2, + entries: [{ ref: legacyRef, kind: 'projection', id: legacyId, namespace, lastSeenAt: 123 }], + }), 'utf8'); + + const before = readFileSync(legacyPath, 'utf8'); + const loaded = await loadMemoryShortRefsFromStore(); + expect(loaded).toBeGreaterThanOrEqual(1); + expect(resolveMemoryShortRef(legacyRef, namespace)).toMatchObject({ id: legacyId }); + + // Imported handles land in the store, so the next start no longer needs the file. + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(listMemoryShortRefs().some((row) => row.id === legacyId)).toBe(true); + // And the file is left exactly as it was. + expect(readFileSync(legacyPath, 'utf8')).toBe(before); + }); + + it('keeps cross-namespace isolation after a store warm-load', async () => { + const [ref] = registerMemoryShortRefs([ + { kind: 'projection', id: 'cccccccccc-1111-2222-3333-444444444444', namespace }, + ]); + await new Promise((resolve) => setTimeout(resolve, 50)); + resetMemoryShortRefsForTests(); + await loadMemoryShortRefsFromStore(); + + expect(resolveMemoryShortRef(ref!, { ...namespace, projectId: 'other-repo' })).toBeUndefined(); + expect(resolveMemoryShortRef(ref!, namespace)).toMatchObject({ id: 'cccccccccc-1111-2222-3333-444444444444' }); + }); + + it('hydrates an exact ref persisted after this process cache was populated', async () => { + const localId = 'local-cache-entry'; + const remoteId = 'persisted-by-another-process'; + const remoteRef = makeMemoryShortRef('projection', remoteId); + + // Populate this process's Map first, then write a different ref straight to + // SQLite as another daemon/MCP process would. A one-time warm-load cannot + // observe this later write. + seedMemoryShortRefCollisionForTests(makeMemoryShortRef('projection', localId), [{ + kind: 'projection', + id: localId, + namespace, + }]); + upsertMemoryShortRefs([{ + ref: remoteRef, + kind: 'projection', + id: remoteId, + namespaceKey: JSON.stringify([ + namespace.scope, + namespace.userId, + namespace.projectId, + '', + '', + ]), + namespaceJson: JSON.stringify(namespace), + lastSeenAt: Date.now(), + }]); + + expect(resolveMemoryShortRef(remoteRef, namespace)).toBeUndefined(); + expect(listMemoryShortRefsByRef(remoteRef)).toHaveLength(1); + + await expect(resolveMemoryShortRefCandidatesWithStore(remoteRef, namespace)) + .resolves.toEqual([ + expect.objectContaining({ kind: 'projection', id: remoteId }), + ]); + await expect(resolveMemoryShortRefWithStore(remoteRef, namespace)) + .resolves.toMatchObject({ kind: 'projection', id: remoteId }); + }); + + it('does not cross namespaces while hydrating an exact persisted ref', async () => { + const id = 'persisted-private-entry'; + const ref = makeMemoryShortRef('projection', id); + upsertMemoryShortRefs([{ + ref, + kind: 'projection', + id, + namespaceKey: JSON.stringify([ + namespace.scope, + namespace.userId, + namespace.projectId, + '', + '', + ]), + namespaceJson: JSON.stringify(namespace), + lastSeenAt: Date.now(), + }]); + + await expect(resolveMemoryShortRefCandidatesWithStore(ref, { + ...namespace, + userId: 'other-user', + })).resolves.toEqual([]); + await expect(resolveMemoryShortRefWithStore(ref, { + ...namespace, + projectId: 'other-repo', + })).resolves.toBeUndefined(); + await expect(resolveMemoryShortRefWithStore(ref, namespace)) + .resolves.toMatchObject({ id }); + }); + + it('keeps an old hydrated ref long enough to resolve when the local LRU is full', async () => { + for (let index = 0; index < 10_000; index += 1) { + seedMemoryShortRefCollisionForTests(`proj:cached${String(index).padStart(7, '0')}`, [{ + kind: 'projection', + id: `cached-${index}`, + namespace, + lastSeenAt: 1_000 + index, + }]); + } + const id = 'old-persisted-ref-at-lru-cap'; + const ref = makeMemoryShortRef('projection', id); + upsertMemoryShortRefs([{ + ref, + kind: 'projection', + id, + namespaceKey: JSON.stringify([ + namespace.scope, + namespace.userId, + namespace.projectId, + '', + '', + ]), + namespaceJson: JSON.stringify(namespace), + lastSeenAt: 1, + }]); + + expect(resolveMemoryShortRef(ref, namespace)).toBeUndefined(); + await expect(resolveMemoryShortRefWithStore(ref, namespace)) + .resolves.toMatchObject({ id }); + }); +}); diff --git a/test/context/memory-short-ref.test.ts b/test/context/memory-short-ref.test.ts index d3afa6b8f..34fb2fa07 100644 --- a/test/context/memory-short-ref.test.ts +++ b/test/context/memory-short-ref.test.ts @@ -1,3 +1,4 @@ +import { writeFileSync } from 'node:fs'; import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -8,6 +9,7 @@ import { reloadMemoryShortRefsForTests, resetMemoryShortRefsForTests, resolveMemoryShortRef, + resolveMemoryShortRefCandidates, } from '../../src/context/memory-short-ref.js'; describe('memory short refs', () => { @@ -28,9 +30,24 @@ describe('memory short refs', () => { await rm(tempDir, { recursive: true, force: true }); }); + // Golden values: pin the derivation (md5 → base32, first 13 chars) so an + // accidental change to the algorithm invalidating every issued handle fails + // here loudly instead of silently. it('builds compact deterministic refs from full memory ids', () => { - expect(makeMemoryShortRef('observation', 'aaaaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee')).toBe('obs:aaaaaaaaaa'); - expect(makeMemoryShortRef('projection', '1111111111-2222-3333-4444-555555555555')).toBe('proj:1111111111'); + expect(makeMemoryShortRef('observation', 'aaaaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee')).toBe('obs:vd34zyurs535g'); + expect(makeMemoryShortRef('projection', '1111111111-2222-3333-4444-555555555555')).toBe('proj:bp27impgwfo27'); + }); + + it('derives handles from the whole id, not the leading hex-ish characters', () => { + // Regression: the previous derivation kept the first 10 hex-looking chars of + // the id, scavenged from anywhere in the string. Structured ids that share a + // constant prefix therefore collapsed onto ONE handle (their discriminating + // sha256 sat past the 10th kept character), and resolution then returned + // whichever colliding record was newest — i.e. the wrong memory, silently. + const base = 'md-ingest:personal::::::::local/10cd7355ec31:CLAUDE.md:'; + const first = makeMemoryShortRef('projection', `${base}5f4045310f20b47e48090f0d7bc67020305d68f12ea66e9920a800321f279b42`); + const second = makeMemoryShortRef('projection', `${base}bff091dac14fa9e1c6405660f45a0db626ccafd19276af083a49439ce4bbd659`); + expect(first).not.toBe(second); }); it('survives daemon restart by reloading the local short-ref cache', () => { @@ -55,17 +72,25 @@ describe('memory short refs', () => { }); }); + // A digest collision can no longer be produced through registerMemoryShortRef + // (that was only reachable because the old derivation collided on a shared id + // prefix), so seed the persisted cache directly to keep the ambiguity paths + // covered. They still guard real 65-bit collisions. + function seedColliding(ref: string, entries: ReadonlyArray>): void { + writeFileSync( + join(tempDir, 'refs.json'), + JSON.stringify({ schemaVersion: 2, entries: entries.map((entry) => ({ ref, ...entry })) }), + 'utf8', + ); + reloadMemoryShortRefsForTests(); + } + it('does not guess when a short ref is ambiguous across namespaces', () => { - const ref = registerMemoryShortRef({ - kind: 'projection', - id: 'bbbbbbbbbb-1111-2222-3333-444444444444', - namespace: { scope: 'user_private', userId: 'user-1', projectId: 'repo-a' }, - }); - registerMemoryShortRef({ - kind: 'projection', - id: 'bbbbbbbbbb-9999-8888-7777-666666666666', - namespace: { scope: 'user_private', userId: 'user-1', projectId: 'repo-b' }, - }); + const ref = 'proj:collide000000'; + seedColliding(ref, [ + { kind: 'projection', id: 'bbbbbbbbbb-1111-2222-3333-444444444444', namespace: { scope: 'user_private', userId: 'user-1', projectId: 'repo-a' } }, + { kind: 'projection', id: 'bbbbbbbbbb-9999-8888-7777-666666666666', namespace: { scope: 'user_private', userId: 'user-1', projectId: 'repo-b' } }, + ]); expect(resolveMemoryShortRef(ref)).toBeUndefined(); expect(resolveMemoryShortRef(ref, { scope: 'user_private', userId: 'user-1', projectId: 'repo-b' })).toMatchObject({ @@ -86,29 +111,42 @@ describe('memory short refs', () => { }); }); - it('resolves same-namespace short-ref conflicts to the newest seen entry', () => { + it('surfaces every record behind a colliding handle so a reader can choose', () => { + // Refusing outright would throw away information the caller can use, so read + // paths get all matches; only callers that ACT on the result (see the strict + // accessor below) refuse. const namespace = { scope: 'user_private' as const, userId: 'user-1', projectId: 'repo-1' }; - const ref = registerMemoryShortRef({ - kind: 'observation', - id: 'cccccccccc-1111-2222-3333-444444444444', - namespace, - lastSeenAt: 100, - }); - registerMemoryShortRef({ - kind: 'observation', - id: 'cccccccccc-9999-8888-7777-666666666666', - namespace, - lastSeenAt: 200, - }); + const ref = 'obs:collide000001'; + seedColliding(ref, [ + { kind: 'observation', id: 'aaaa1111-2222-3333-4444-555555555555', namespace, lastSeenAt: 100 }, + { kind: 'observation', id: 'bbbb9999-8888-7777-6666-666666666666', namespace, lastSeenAt: 200 }, + ]); + + const candidates = resolveMemoryShortRefCandidates(ref, namespace); + expect(candidates.map((entry) => entry.id).sort()).toEqual([ + 'aaaa1111-2222-3333-4444-555555555555', + 'bbbb9999-8888-7777-6666-666666666666', + ]); + }); - expect(resolveMemoryShortRef(ref, namespace)).toMatchObject({ - id: 'cccccccccc-9999-8888-7777-666666666666', - }); + it('refuses to resolve a same-namespace collision instead of guessing the newest', () => { + // Two different records deriving one handle used to resolve to whichever was + // seen last, handing back the wrong memory with no signal. That was tolerable + // when the derivation collided routinely; with a 65-bit digest a same- + // namespace collision should never occur, so it is treated as unresolvable + // (and reported) rather than silently answered with the wrong record. + const namespace = { scope: 'user_private' as const, userId: 'user-1', projectId: 'repo-1' }; + const ref = 'obs:collide000000'; + seedColliding(ref, [ + { kind: 'observation', id: 'cccccccccc-1111-2222-3333-444444444444', namespace, lastSeenAt: 100 }, + { kind: 'observation', id: 'cccccccccc-9999-8888-7777-666666666666', namespace, lastSeenAt: 200 }, + ]); - reloadMemoryShortRefsForTests(); + expect(resolveMemoryShortRef(ref, namespace)).toBeUndefined(); - expect(resolveMemoryShortRef(ref, namespace)).toMatchObject({ - id: 'cccccccccc-9999-8888-7777-666666666666', - }); + // Still unresolvable after a reload — the collision is a property of the + // data, not of this process's in-memory state. + reloadMemoryShortRefsForTests(); + expect(resolveMemoryShortRef(ref, namespace)).toBeUndefined(); }); }); diff --git a/test/daemon/codex-watcher-bootstrap.test.ts b/test/daemon/codex-watcher-bootstrap.test.ts index 5a4bb71cf..240cb5fb0 100644 --- a/test/daemon/codex-watcher-bootstrap.test.ts +++ b/test/daemon/codex-watcher-bootstrap.test.ts @@ -51,6 +51,7 @@ vi.mock('../../src/daemon/timeline-emitter.js', () => ({ })); import { ensureSessionFile } from '../../src/daemon/codex-watcher.js'; +import { makeMemoryShortRef } from '../../src/context/memory-short-ref.js'; describe('ensureSessionFile', () => { beforeEach(() => { @@ -107,7 +108,7 @@ describe('ensureSessionFile', () => { 'memory.context', expect.objectContaining({ reason: 'startup', - injectedText: '[Related past work]\n\n- [proj] Fix websocket reconnect loop\n', + injectedText: `[Related past work]\n\n- [proj] (${makeMemoryShortRef('projection', 'mem-1')}) Fix websocket reconnect loop\n`, }), expect.any(Object), ); diff --git a/test/daemon/command-handler-memory-context.test.ts b/test/daemon/command-handler-memory-context.test.ts index 45b794b25..bfe5f1b37 100644 --- a/test/daemon/command-handler-memory-context.test.ts +++ b/test/daemon/command-handler-memory-context.test.ts @@ -4,6 +4,7 @@ import { mkdir, mkdtemp, rm, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { promisify } from 'node:util'; +import { makeMemoryShortRef } from '../../src/context/memory-short-ref.js'; const execFileAsync = promisify(execFile); @@ -604,6 +605,7 @@ describe('handleWebCommand memory context timeline', () => { type: MEMORY_WS.PERSONAL_QUERY, requestId: 'personal-list', projectId: 'github.com/acme/repo', + includeShortRefs: true, [MEMORY_MANAGEMENT_CONTEXT_FIELD]: { actorId: 'user-bob', userId: 'user-bob', @@ -644,6 +646,7 @@ describe('handleWebCommand memory context timeline', () => { requestId: 'personal-list', records: [expect.objectContaining({ id: 'bob-proj', + ref: makeMemoryShortRef('projection', 'bob-proj'), summary: 'Bob private project memory', ownerUserId: 'user-bob', createdByUserId: 'user-bob', @@ -1399,10 +1402,13 @@ describe('handleWebCommand memory context timeline', () => { expect.objectContaining({ relatedToEventId: 'evt-user-1', query: 'Fix reconnect issues in websocket client', - injectedText: '[Related past work]\n\n- [codedeck] Fix websocket reconnect loop\n', + // Each line carries a redeemable handle so the agent can call + // get_memory_sources when the summary alone isn't enough. + injectedText: `[Related past work]\n\n- [codedeck] (${makeMemoryShortRef('projection', 'mem-1')}) Fix websocket reconnect loop\n`, items: [ expect.objectContaining({ id: 'mem-1', + ref: makeMemoryShortRef('projection', 'mem-1'), projectId: 'codedeck', relevanceScore: 0.812, hitCount: 4, diff --git a/test/daemon/command-handler-transport-queue.test.ts b/test/daemon/command-handler-transport-queue.test.ts index afd588234..c7e054a1b 100644 --- a/test/daemon/command-handler-transport-queue.test.ts +++ b/test/daemon/command-handler-transport-queue.test.ts @@ -878,10 +878,12 @@ describe('handleWebCommand transport queue behavior', () => { now: 102, }); const send = vi.fn(() => 'queued'); + const cancel = vi.fn(async () => {}); getProviderMock.mockReturnValue({ id: 'mock-provider' }); getTransportRuntimeMock.mockReturnValue({ providerSessionId: 'route-transport', send, + cancel, pendingCount: 3, pendingMessages: ['selected option', 'queued msg', 'queued msg 2'], pendingEntries: [ @@ -929,6 +931,73 @@ describe('handleWebCommand transport queue behavior', () => { const answerStateCall = emitMock.mock.calls.find((call) => call[0] === 'deck_transport_brain' && call[1] === 'session.state'); expect(answerStateCall?.[2]).not.toHaveProperty('pendingCount'); expect(answerStateCall?.[2]).not.toHaveProperty('pendingMessages'); + // Front placement alone only beats other QUEUED messages — it still waits + // for the active turn, which is the very turn paused on this question. The + // answer must therefore force-interrupt it (priority path) or the two wait + // on each other forever. + expect(cancel).toHaveBeenCalledTimes(1); + }); + + it('force-interrupts the paused turn so a queued ask.answer cannot deadlock', async () => { + // Regression: user answers the dialog and nothing happens. `send` returns + // 'queued' because the question-paused turn still counts as active work, so + // the answer sits at the queue front waiting for a turn that can only end + // once the answer is delivered. Every provider WITHOUT + // `answerPendingQuestion` (codex-sdk, gemini, qwen, opencode, copilot, + // grok, kimi, openclaw) took this path on every dialog. + const send = vi.fn(() => 'queued'); + const cancel = vi.fn(async () => {}); + getProviderMock.mockReturnValue({ id: 'mock-provider' }); + getTransportRuntimeMock.mockReturnValue({ + providerSessionId: 'route-transport', + send, + cancel, + pendingCount: 1, + pendingMessages: ['picked B'], + pendingEntries: [{ clientMessageId: 'ans', text: 'picked B' }], + pendingVersion: 1, + }); + + handleWebCommand({ + type: 'ask.answer', + sessionName: 'deck_transport_brain', + answer: 'picked B', + }, serverLink as any); + await flushAsync(); + + expect(send).toHaveBeenCalledWith('picked B', undefined, undefined, undefined, { queuePlacement: 'front' }); + expect(cancel).toHaveBeenCalledTimes(1); + // The drain emits the visible user.message for the queued entry; emitting it + // here too would render the answer twice. + const userMessageEmits = emitMock.mock.calls.filter((call) => call[1] === 'user.message'); + expect(userMessageEmits).toHaveLength(0); + }); + + it('does not interrupt anything when the ask.answer was delivered directly', async () => { + // Idle session: `send` returns 'sent', so there is no paused turn to break + // out of. Cancelling here would abort unrelated work for no reason. + const send = vi.fn(() => 'sent'); + const cancel = vi.fn(async () => {}); + getProviderMock.mockReturnValue({ id: 'mock-provider' }); + getTransportRuntimeMock.mockReturnValue({ + providerSessionId: 'route-transport', + send, + cancel, + pendingCount: 0, + pendingMessages: [], + pendingEntries: [], + pendingVersion: 0, + }); + + handleWebCommand({ + type: 'ask.answer', + sessionName: 'deck_transport_brain', + answer: 'picked A', + }, serverLink as any); + await flushAsync(); + + expect(send).toHaveBeenCalledTimes(1); + expect(cancel).not.toHaveBeenCalled(); }); it('undo_queued_message deletes a SQLite-only queued orphan when the runtime in-memory queue is empty', async () => { diff --git a/test/daemon/disk-usage.test.ts b/test/daemon/disk-usage.test.ts new file mode 100644 index 000000000..3b9066c91 --- /dev/null +++ b/test/daemon/disk-usage.test.ts @@ -0,0 +1,131 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; + +const { statfsMock, readFileMock, readdirMock } = vi.hoisted(() => ({ + statfsMock: vi.fn(), + readFileMock: vi.fn(), + readdirMock: vi.fn(), +})); + +vi.mock('node:fs/promises', () => ({ + statfs: statfsMock, + readFile: readFileMock, + readdir: readdirMock, +})); +vi.mock('../../src/util/logger.js', () => ({ + default: { debug: vi.fn(), error: vi.fn(), warn: vi.fn(), info: vi.fn() }, +})); + +import { refreshDiskUsage } from '../../src/daemon/disk-usage.js'; + +const originalPlatform = process.platform; +function setPlatform(p: NodeJS.Platform) { + Object.defineProperty(process, 'platform', { value: p, configurable: true }); +} + +/** bfree defaults to bavail; used = (blocks - bfree) * bsize. */ +function mockStatfs(map: Record) { + statfsMock.mockImplementation(async (mount: string) => { + const s = map[mount]; + if (!s) throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }); + return { type: 0, bsize: s.bsize, blocks: s.blocks, bfree: s.bfree, bavail: s.bfree, files: 0, ffree: 0 }; + }); +} + +describe('disk-usage collector', () => { + beforeEach(() => { + vi.clearAllMocks(); + setPlatform('linux'); + }); + afterEach(() => { + setPlatform(originalPlatform); + }); + + it('parses /proc/mounts, skips pseudo fs, dedups by device, sorts fullest first, computes usage', async () => { + readFileMock.mockResolvedValue([ + 'proc /proc proc rw 0 0', + 'sysfs /sys sysfs rw 0 0', + 'tmpfs /run tmpfs rw 0 0', + 'cgroup2 /sys/fs/cgroup cgroup2 rw 0 0', + '/dev/sda1 / ext4 rw 0 0', + '/dev/sda1 /var/lib/docker ext4 rw 0 0', // same device — bind mount, deduped + '/dev/sdb1 /data ext4 rw 0 0', + '', + ].join('\n')); + mockStatfs({ + '/': { bsize: 4096, blocks: 1000, bfree: 100 }, // 90% used + '/data': { bsize: 4096, blocks: 2000, bfree: 1000 }, // 50% used + }); + + const disks = await refreshDiskUsage(); + + expect(disks).toHaveLength(2); // pseudo skipped, device deduped + expect(disks[0]).toMatchObject({ mount: '/', usedPercent: 90 }); // fullest first + expect(disks[1]).toMatchObject({ mount: '/data', usedPercent: 50 }); + expect(disks[0].totalBytes).toBe(4096 * 1000); + expect(disks[0].usedBytes).toBe(4096 * 900); + }); + + it('collapses distinct devices that report identical size+used (macOS firmlinks)', async () => { + readFileMock.mockResolvedValue([ + '/dev/disk1 / apfs rw 0 0', + '/dev/disk2 /System/Volumes/Data apfs rw 0 0', + '', + ].join('\n')); + mockStatfs({ + '/': { bsize: 4096, blocks: 1000, bfree: 100 }, + '/System/Volumes/Data': { bsize: 4096, blocks: 1000, bfree: 100 }, + }); + + const disks = await refreshDiskUsage(); + expect(disks).toHaveLength(1); + }); + + it('drops mounts whose statfs fails and clamps a 0/undefined total away', async () => { + readFileMock.mockResolvedValue([ + '/dev/sda1 / ext4 rw 0 0', + '/dev/sdc1 /broken ext4 rw 0 0', + '/dev/sdd1 /zero ext4 rw 0 0', + '', + ].join('\n')); + statfsMock.mockImplementation(async (mount: string) => { + if (mount === '/') return { type: 0, bsize: 4096, blocks: 500, bfree: 250, bavail: 250, files: 0, ffree: 0 }; + if (mount === '/zero') return { type: 0, bsize: 4096, blocks: 0, bfree: 0, bavail: 0, files: 0, ffree: 0 }; + throw Object.assign(new Error('EACCES'), { code: 'EACCES' }); + }); + + const disks = await refreshDiskUsage(); + expect(disks).toHaveLength(1); + expect(disks[0]).toMatchObject({ mount: '/', usedPercent: 50 }); + }); + + it('falls back to root when /proc/mounts is empty on linux', async () => { + readFileMock.mockResolvedValue(''); + mockStatfs({ '/': { bsize: 4096, blocks: 100, bfree: 40 } }); // 60% used + + const disks = await refreshDiskUsage(); + expect(disks).toEqual([{ mount: '/', totalBytes: 4096 * 100, usedBytes: 4096 * 60, usedPercent: 60 }]); + }); + + it('skips a mount whose statfs hangs and still returns the healthy ones', async () => { + vi.useFakeTimers(); + try { + readFileMock.mockResolvedValue([ + '/dev/sda1 / ext4 rw 0 0', + '/dev/sde1 /hung ext4 rw 0 0', + '', + ].join('\n')); + statfsMock.mockImplementation((mount: string) => + mount === '/' + ? Promise.resolve({ type: 0, bsize: 4096, blocks: 100, bfree: 50, bavail: 50, files: 0, ffree: 0 }) + : new Promise(() => {}), // never resolves — a wedged network mount + ); + const pending = refreshDiskUsage(); + await vi.advanceTimersByTimeAsync(2100); // past STATFS_TIMEOUT_MS + const disks = await pending; + expect(disks).toHaveLength(1); + expect(disks[0].mount).toBe('/'); + } finally { + vi.useRealTimers(); + } + }); +}); diff --git a/test/daemon/memory-inject-startup.test.ts b/test/daemon/memory-inject-startup.test.ts index a7bf6030c..7b4a50167 100644 --- a/test/daemon/memory-inject-startup.test.ts +++ b/test/daemon/memory-inject-startup.test.ts @@ -52,6 +52,7 @@ vi.mock('../../src/context/memory-recall-core.js', async (importOriginal) => ({ import { injectGeminiMemoryWithTimeline } from '../../src/daemon/memory-inject.js'; import { getSummarySyncFingerprints, resetAllSummarySyncHistories } from '../../src/context/summary-sync-history.js'; +import { makeMemoryShortRef } from '../../src/context/memory-short-ref.js'; describe('injectGeminiMemoryWithTimeline', () => { beforeEach(() => { @@ -111,7 +112,7 @@ describe('injectGeminiMemoryWithTimeline', () => { 'memory.context', expect.objectContaining({ reason: 'startup', - injectedText: '[Related past work]\n\n- [proj] Fix websocket reconnect loop\n', + injectedText: `[Related past work]\n\n- [proj] (${makeMemoryShortRef('projection', 'mem-1')}) Fix websocket reconnect loop\n`, items: [ expect.objectContaining({ id: 'mem-1', diff --git a/test/daemon/memory-mcp-tools-schema-firewall.test.ts b/test/daemon/memory-mcp-tools-schema-firewall.test.ts index d4f89b6f8..62a16c30e 100644 --- a/test/daemon/memory-mcp-tools-schema-firewall.test.ts +++ b/test/daemon/memory-mcp-tools-schema-firewall.test.ts @@ -1,4 +1,7 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import type { ContextNamespace, ProcessedContextProjection } from '../../shared/context-types.js'; import { MCP_FEATURE_FLAGS_BY_NAME } from '../../shared/memory-mcp-feature-flags.js'; import { MEMORY_FEATURE_FLAGS_BY_NAME, type MemoryFeatureFlag } from '../../shared/feature-flags.js'; @@ -7,7 +10,9 @@ import { MCP_ERROR_REASONS } from '../../shared/memory-mcp-errors.js'; import { MEMORY_MCP_DEGRADED_REASON } from '../../shared/memory-ws.js'; import { createMemoryMcpToolHandlers } from '../../src/daemon/memory-mcp-tools.js'; import type { McpRuntimeCaller } from '../../src/daemon/memory-mcp-caller.js'; -import { registerMemoryShortRef, resetMemoryShortRefsForTests } from '../../src/context/memory-short-ref.js'; +vi.mock('../../src/util/rate-limited-warn.js', () => ({ warnOncePerHour: vi.fn() })); + +import { makeMemoryShortRef, registerMemoryShortRef, resetMemoryShortRefsForTests, seedMemoryShortRefCollisionForTests } from '../../src/context/memory-short-ref.js'; import type { SessionRecord } from '../../src/store/session-store.js'; function caller(overrides: Partial = {}): McpRuntimeCaller { @@ -56,8 +61,36 @@ function projection(overrides: Partial = {}): Proces } describe('memory MCP tool schema firewall', () => { + let shortRefDir: string; + let priorShortRefPath: string | undefined; + let priorLegacyPath: string | undefined; + beforeEach(() => { + // Registering a handle now persists it, and the default target is the + // context store — which would build a real SQLite database and WAL in the + // runner's home directory just from exercising these handlers. Point + // persistence at a scratch file, and the legacy cache at a path that does + // not exist. The collision cases also emit a throttled warning, so the + // warning module is stubbed above to keep that content out of the daemon + // log. Two writes remain outside this suite's reach and are NOT claimed to + // be isolated: the daemon logger creates its file when the module is + // imported, and one pre-existing observation-expansion case below builds the + // context store. + shortRefDir = mkdtempSync(join(tmpdir(), 'imc-mcp-shortref-')); + priorShortRefPath = process.env.IMCODES_MEMORY_SHORT_REF_PATH; + priorLegacyPath = process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH; + process.env.IMCODES_MEMORY_SHORT_REF_PATH = join(shortRefDir, 'refs.json'); + process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = join(shortRefDir, 'absent-legacy.json'); + resetMemoryShortRefsForTests(); + }); + + afterEach(() => { resetMemoryShortRefsForTests(); + if (priorShortRefPath === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_PATH = priorShortRefPath; + if (priorLegacyPath === undefined) delete process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH; + else process.env.IMCODES_MEMORY_SHORT_REF_LEGACY_PATH = priorLegacyPath; + rmSync(shortRefDir, { recursive: true, force: true }); }); it('submits peer audit replies only through the strict structured dependency', async () => { @@ -260,6 +293,81 @@ describe('memory MCP tool schema firewall', () => { expect(archiveMemory).toHaveBeenCalledWith('proj-ref'); }); + it('rejects a kind that disagrees with an ambiguous ref instead of expanding it', async () => { + // The single-candidate path validates the requested kind; the ambiguous + // path returned candidates regardless, silently dropping the caller's + // constraint. Resolver-level tests could not see this — it needs the handler. + const namespace: ContextNamespace = { scope: 'personal', userId: 'user-1', projectId: 'repo-1' }; + const ref = registerMemoryShortRef({ kind: 'projection', id: 'collide-a', namespace }); + registerMemoryShortRef({ kind: 'projection', id: 'collide-b', namespace }); + // Force both onto one handle so the ambiguous branch is exercised. + seedMemoryShortRefCollisionForTests(ref, [ + { kind: 'projection', id: 'collide-a', namespace }, + { kind: 'projection', id: 'collide-b', namespace }, + ]); + const getProcessedProjectionById = vi.fn(() => projection({ id: 'collide-a', namespace })); + const handlers = createMemoryMcpToolHandlers(caller({ namespace }), { + getProcessedProjectionById, + isMemoryFeatureEnabled: () => true, + }); + + await expect(handlers[MEMORY_MCP_TOOL_NAMES.GET_MEMORY_SOURCES]({ ref, kind: 'observation' })).resolves.toMatchObject({ + status: 'error', + reason: MCP_ERROR_REASONS.VALIDATION_FAILED, + }); + }); + + it('declares how many records an ambiguous ref covers when expansion is bounded', async () => { + // Expansion is capped, so a caller told it received "every match" would stop + // looking while the answer sat in an omitted record. + // + // The orchestrator MUST be injected: without it the handler falls through to + // the real one, which builds an actual SQLite store (plus WAL and a log) in + // the runner's home directory. + const namespace: ContextNamespace = { scope: 'personal', userId: 'user-1', projectId: 'repo-1' }; + const ids = ['c1', 'c2', 'c3', 'c4', 'c5']; + const ref = registerMemoryShortRef({ kind: 'projection', id: ids[0]!, namespace }); + seedMemoryShortRefCollisionForTests(ref, ids.map((id) => ({ kind: 'projection' as const, id, namespace }))); + const getMemorySourcesOrchestrator = vi.fn(async (projectionId: string) => ({ + status: 'ok' as const, + projectionId, + sourceEventCount: 1, + sources: [{ eventId: `evt-${projectionId}`, status: 'ok', content: `body ${projectionId}` }], + })); + const handlers = createMemoryMcpToolHandlers(caller({ namespace }), { + getMemorySourcesOrchestrator, + isMemoryFeatureEnabled: () => true, + }); + + const result = await handlers[MEMORY_MCP_TOOL_NAMES.GET_MEMORY_SOURCES]({ ref }) as Record; + expect(result).toMatchObject({ status: 'ok', ambiguousRef: true, candidateCount: 5, truncated: true }); + const candidates = result.candidates as Array>; + // Exactly the cap, carrying real expanded content — `length < 5` alone would + // also pass on an empty array. + expect(candidates).toHaveLength(4); + expect(candidates.map((candidate) => candidate.projectionId)).toEqual(['c1', 'c2', 'c3', 'c4']); + expect(candidates[0]).toMatchObject({ kind: 'projection', sources: [expect.objectContaining({ content: 'body c1' })] }); + }); + + it('marks an ambiguous ref untruncated when every candidate fits', async () => { + const namespace: ContextNamespace = { scope: 'personal', userId: 'user-1', projectId: 'repo-1' }; + const ref = registerMemoryShortRef({ kind: 'projection', id: 'pair-a', namespace }); + seedMemoryShortRefCollisionForTests(ref, [ + { kind: 'projection', id: 'pair-a', namespace }, + { kind: 'projection', id: 'pair-b', namespace }, + ]); + const handlers = createMemoryMcpToolHandlers(caller({ namespace }), { + getMemorySourcesOrchestrator: vi.fn(async (projectionId: string) => ({ + status: 'ok' as const, projectionId, sourceEventCount: 0, sources: [], + })), + isMemoryFeatureEnabled: () => true, + }); + + const result = await handlers[MEMORY_MCP_TOOL_NAMES.GET_MEMORY_SOURCES]({ ref }) as Record; + expect(result).toMatchObject({ ambiguousRef: true, candidateCount: 2, truncated: false }); + expect((result.candidates as unknown[])).toHaveLength(2); + }); + it('short-circuits memory disabled gates before backend calls', async () => { const searchMemory = vi.fn(); const savePreference = vi.fn(async () => ({ status: 'ok' })); @@ -435,7 +543,7 @@ describe('memory MCP tool schema firewall', () => { items: [ { projectionId, - ref: 'proj:1111111111', + ref: makeMemoryShortRef('projection', projectionId), recordKind: 'projection', sourceLookup: { tool: 'get_memory_sources', kind: 'projection', projectionId }, summary: 'MCP provider readiness fixed for Gemini, Copilot, and Qwen.', @@ -457,7 +565,7 @@ describe('memory MCP tool schema firewall', () => { limit: 5, })); await expect(handlers[MEMORY_MCP_TOOL_NAMES.GET_MEMORY_SOURCES]({ - ref: 'proj:1111111111', + ref: makeMemoryShortRef('projection', projectionId), kind: 'projection', })).resolves.toMatchObject({ status: 'ok', @@ -502,7 +610,7 @@ describe('memory MCP tool schema firewall', () => { items: [ { observationId, - ref: 'obs:aaaaaaaaaa', + ref: makeMemoryShortRef('observation', observationId), recordKind: 'observation', sourceLookup: { tool: 'get_memory_sources', kind: 'observation', observationId }, observationClass: 'note', @@ -518,7 +626,7 @@ describe('memory MCP tool schema firewall', () => { serverId: 'attacker-srv', }); await expect(handlers[MEMORY_MCP_TOOL_NAMES.GET_MEMORY_SOURCES]({ - ref: 'obs:aaaaaaaaaa', + ref: makeMemoryShortRef('observation', observationId), kind: 'observation', })).resolves.toMatchObject({ status: 'ok', diff --git a/test/daemon/memory-recall-integration.test.ts b/test/daemon/memory-recall-integration.test.ts index 442f58161..86865a2f5 100644 --- a/test/daemon/memory-recall-integration.test.ts +++ b/test/daemon/memory-recall-integration.test.ts @@ -201,8 +201,11 @@ describe('memory recall integration', () => { const { buildSessionBootstrapContext } = await import('../../src/daemon/memory-inject.js'); const context = await buildSessionBootstrapContext('/tmp/fake-project', 'my-project'); - expect(context).toContain('[important] Persist enterprise binding decisions as durable memory'); - expect(context).toContain('[recent] Recent fix for transport memory recall cards'); + // A redeemable handle sits between the label and the summary so the agent + // can call get_memory_sources when the summary alone isn't enough. + // base32 handle alphabet is a-z2-7. + expect(context).toMatch(/\[important\] \(proj:[a-z2-7]{13}\) Persist enterprise binding decisions as durable memory/); + expect(context).toMatch(/\[recent\] \(proj:[a-z2-7]{13}\) Recent fix for transport memory recall cards/); }); it('buildSessionBootstrapContext includes "# Recent project memory" when memories exist', async () => { diff --git a/test/daemon/server-link.test.ts b/test/daemon/server-link.test.ts index 20b135c24..7c86fcd62 100644 --- a/test/daemon/server-link.test.ts +++ b/test/daemon/server-link.test.ts @@ -1,5 +1,14 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +const upgradeBlockedOutboxMock = vi.hoisted(() => ({ + flushOnReconnect: vi.fn(async () => false), + acknowledge: vi.fn(async () => true), +})); + +vi.mock('../../src/daemon/upgrade-blocked-outbox.js', () => ({ + getDefaultUpgradeBlockedOutbox: () => upgradeBlockedOutboxMock, +})); + // Mock WebSocket before importing ServerLink const mockWsInstance = { send: vi.fn(), @@ -15,11 +24,18 @@ vi.mock('../../src/util/daemon-status.js', () => ({ recordDaemonServerLinkStatus: vi.fn(), })); -import { ServerLink, __setServerLinkDataPlaneQueueConfigForTests } from '../../src/daemon/server-link.js'; +import { ServerLink, __setServerLinkDataPlaneQueueConfigForTests, setServerLinkReconnectResyncHandler } from '../../src/daemon/server-link.js'; +import { TIMELINE_DELIVERY_METRICS } from '../../shared/timeline-delivery-telemetry.js'; +import { getCounter, resetMetricsForTests } from '../../src/util/metrics.js'; import { recordDaemonServerLinkStatus } from '../../src/util/daemon-status.js'; import { TIMELINE_MESSAGES, TIMELINE_PROTOCOL_CAPABILITY } from '../../shared/timeline-protocol.js'; import { TRANSPORT_EVENT } from '../../shared/transport-events.js'; import { FILE_TRANSFER_UPLOAD_FETCH_CAPABILITY } from '../../shared/transport/file-transfer.js'; +import { DAEMON_MSG } from '../../shared/daemon-events.js'; +import { + DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION, + DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL, +} from '../../shared/daemon-upgrade.js'; const recordDaemonServerLinkStatusMock = vi.mocked(recordDaemonServerLinkStatus); @@ -28,6 +44,7 @@ describe('ServerLink', () => { beforeEach(() => { vi.clearAllMocks(); + upgradeBlockedOutboxMock.flushOnReconnect.mockResolvedValue(false); link = new ServerLink({ workerUrl: 'wss://test.workers.dev', serverId: 'srv-123', @@ -54,6 +71,76 @@ describe('ServerLink', () => { ); }); + it('replays one persisted terminal upgrade failure after the link opens', async () => { + link.connect(); + const openHandler = mockWsInstance.addEventListener.mock.calls.find(([type]) => type === 'open')?.[1] as + | (() => void) + | undefined; + expect(openHandler).toBeDefined(); + + openHandler?.(); + + expect(upgradeBlockedOutboxMock.flushOnReconnect).toHaveBeenCalledTimes(1); + expect(upgradeBlockedOutboxMock.flushOnReconnect).toHaveBeenCalledWith( + expect.any(Function), + link.daemonVersion, + ); + }); + + it('advertises blocker-sync support and certifies the connection only after outbox replay settles', async () => { + let resolveFlush!: (value: boolean) => void; + upgradeBlockedOutboxMock.flushOnReconnect.mockImplementationOnce( + () => new Promise((resolve) => { resolveFlush = resolve; }), + ); + + link.connect(); + const openHandler = mockWsInstance.addEventListener.mock.calls.find(([type]) => type === 'open')?.[1] as + | (() => void) + | undefined; + openHandler?.(); + + const beforeFlush = mockWsInstance.send.mock.calls.map(([raw]) => JSON.parse(String(raw)) as Record); + expect(beforeFlush[0]).toMatchObject({ + type: 'auth', + [DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.AUTH_REVISION_FIELD]: + DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION, + }); + expect(beforeFlush.some((message) => message.type === DAEMON_MSG.UPGRADE_BLOCKED_SYNC)).toBe(false); + + resolveFlush(false); + await Promise.resolve(); + await Promise.resolve(); + + expect(mockWsInstance.send.mock.calls.map(([raw]) => JSON.parse(String(raw)))).toContainEqual(expect.objectContaining({ + type: DAEMON_MSG.UPGRADE_BLOCKED_SYNC, + revision: DAEMON_UPGRADE_BLOCKED_SYNC_PROTOCOL.REVISION, + })); + }); + + it('records the server acknowledgement without exposing it to ordinary message handlers', async () => { + link.connect(); + const messageHandler = mockWsInstance.addEventListener.mock.calls.find(([type]) => type === 'message')?.[1] as + | ((event: MessageEvent) => void) + | undefined; + const ordinaryHandler = vi.fn(); + link.onMessage(ordinaryHandler); + + messageHandler?.({ + data: JSON.stringify({ + type: DAEMON_MSG.UPGRADE_BLOCKED_ACK, + failureId: 'failure-1', + disposition: DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.ACCEPTED, + }), + } as MessageEvent); + await Promise.resolve(); + + expect(upgradeBlockedOutboxMock.acknowledge).toHaveBeenCalledWith( + 'failure-1', + DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.ACCEPTED, + ); + expect(ordinaryHandler).not.toHaveBeenCalled(); + }); + it('send() silently drops messages when not connected (fire-and-forget safe)', () => { // The daemon must never die from transient disconnects — ServerLink.send() // is best-effort and must not throw. Callers that need delivery @@ -343,4 +430,101 @@ describe('ServerLink', () => { // The previous WS instance must have been explicitly closed. expect(mockWsInstance.close).toHaveBeenCalledTimes(1); }); + + it('counts content-bearing timeline events dropped while the link is down', () => { + // This branch is where a chat message vanishes: live timeline events are + // control-plane, so a non-OPEN socket drops them with no queue and no replay. + // The browser now heals it via a no-lower-bound backfill, so without this + // counter a rising drop rate would be masked by the recovery. + resetMetricsForTests(); + mockWsInstance.readyState = 3; // CLOSED + + const before = getCounter(TIMELINE_DELIVERY_METRICS.DAEMON_LINK_DOWN_DROPPED, { eventType: 'assistant.text' }); + expect(link.send({ + type: TIMELINE_MESSAGES.EVENT, + event: { type: 'assistant.text', eventId: 'e1', sessionId: 's', ts: 1, payload: { text: 'lost' } }, + })).toBeUndefined(); + expect(getCounter(TIMELINE_DELIVERY_METRICS.DAEMON_LINK_DOWN_DROPPED, { eventType: 'assistant.text' })) + .toBe(before + 1); + expect(link.timelineDropCountWhileLinkDown).toBe(1); + }); + + it('does not count high-frequency status chatter as a lost message', () => { + // agent.status fires ~1/s during a turn; counting it would bury the signal. + resetMetricsForTests(); + mockWsInstance.readyState = 3; + + link.send({ + type: TIMELINE_MESSAGES.EVENT, + event: { type: 'agent.status', eventId: 'e2', sessionId: 's', ts: 2, payload: { status: 'thinking' } }, + }); + link.send({ type: 'heartbeat' }); + + expect(getCounter(TIMELINE_DELIVERY_METRICS.DAEMON_LINK_DOWN_DROPPED, { eventType: 'agent.status' })).toBe(0); + expect(link.timelineDropCountWhileLinkDown).toBe(0); + }); + + it('counts nothing while the link is OPEN', () => { + resetMetricsForTests(); + mockWsInstance.readyState = 1; // OPEN + link.connect(); + link.send({ + type: TIMELINE_MESSAGES.EVENT, + event: { type: 'assistant.text', eventId: 'e3', sessionId: 's', ts: 3, payload: { text: 'delivered' } }, + }); + expect(link.timelineDropCountWhileLinkDown).toBe(0); + }); + + it('fires the reconnect state-resync handler on RE-connect only, not the first connect', async () => { + // Regression for deck_sub_26624c1t: live timeline events (incl. the + // authoritative session.state idle) are control-plane and silently dropped + // while the socket is down — with no replay, a turn that settles during an + // outage leaves every browser rendering "working" forever. The open handler + // must invoke the registered resync hook after every reconnect so the + // session layer can re-broadcast current authoritative states. The FIRST + // connect must NOT fire it (daemon startup already runs a full session sync). + const resync = vi.fn(); + setServerLinkReconnectResyncHandler(resync); + vi.useFakeTimers(); + try { + link.connect(); + const openHandler = mockWsInstance.addEventListener.mock.calls.find(([type]) => type === 'open')?.[1] as + | (() => void) + | undefined; + expect(openHandler).toBeDefined(); + + // First successful open — startup path, no resync. + openHandler?.(); + await vi.advanceTimersByTimeAsync(0); // flush the scheduled setImmediate + expect(resync).not.toHaveBeenCalled(); + + // Re-connect (same socket object in this harness; the handler's stale-ws + // guard passes because this.ws is unchanged). + openHandler?.(); + await vi.advanceTimersByTimeAsync(0); + expect(resync).toHaveBeenCalledTimes(1); + } finally { + setServerLinkReconnectResyncHandler(null); + } + }); + + it('survives a throwing resync handler without breaking the open handshake', async () => { + setServerLinkReconnectResyncHandler(() => { + throw new Error('resync boom'); + }); + vi.useFakeTimers(); + try { + link.connect(); + const openHandler = mockWsInstance.addEventListener.mock.calls.find(([type]) => type === 'open')?.[1] as + | (() => void) + | undefined; + openHandler?.(); + openHandler?.(); + await vi.advanceTimersByTimeAsync(0); + // No throw escaped to the test — the handler failure is contained. + expect(link.isConnected()).toBe(true); + } finally { + setServerLinkReconnectResyncHandler(null); + } + }); }); diff --git a/test/daemon/supervision-automation.test.ts b/test/daemon/supervision-automation.test.ts index cfa54fbdd..b1547721b 100644 --- a/test/daemon/supervision-automation.test.ts +++ b/test/daemon/supervision-automation.test.ts @@ -98,6 +98,25 @@ async function waitForRunPhase(phase: 'execution' | 'auditing' | 'finalizing', t } } +/** Wait until the automation has finished the run (no active run left). + * Same real-check-queue yield as `waitForRunPhase`, so a terminal run cannot + * be asserted before the automation has actually torn it down. */ +async function waitForRunEnd(timeoutMs = 10_000) { + const deadline = performance.now() + timeoutMs; + while (supervisionAutomation.getActiveRun('deck_supervision_brain') !== undefined) { + if (performance.now() >= deadline) return; + await new Promise((resolve) => setImmediate(resolve)); + } +} + +async function waitForTransportSendCount(expectedCount: number, timeoutMs = 10_000) { + const deadline = performance.now() + timeoutMs; + while (mockTransportRuntime.send.mock.calls.length < expectedCount) { + if (performance.now() >= deadline) return; + await new Promise((resolve) => setImmediate(resolve)); + } +} + let projectDir: string | null = null; beforeEach(() => { @@ -1183,14 +1202,201 @@ describe('SupervisionAutomation', () => { await sleep(25); expect(mockTransportRuntime.send).toHaveBeenCalledTimes(2); - expect(String(mockTransportRuntime.send.mock.calls[1]?.[0])).toContain('Audit verdict: REWORK'); - expect(String(mockTransportRuntime.send.mock.calls[1]?.[0])).not.toContain('Commit the completed changes and push to origin/dev.'); + const reworkPrompt = String(mockTransportRuntime.send.mock.calls[1]?.[0]); + expect(reworkPrompt).toContain('Audit verdict: REWORK'); + expect(reworkPrompt).toContain('Do not stage, commit, push, merge, release, publish, or deploy until a new matching peer audit returns PASS.'); + expect(reworkPrompt).not.toContain('Commit the completed changes and push to origin/dev.'); expect(supervisionAutomation.getActiveRun('deck_supervision_brain')).toMatchObject({ phase: 'execution', reworkDispatches: 1, }); }); + it('requires a fresh PASS after REWORK before releasing deferred commit and push', async () => { + const snapshot = await seedSession('supervised_audit', false, 1); + mockSupervisionDecide + .mockResolvedValueOnce({ + decision: 'continue', + reason: 'only repository finalization remains', + confidence: 0.9, + gap: 'changes are uncommitted', + nextAction: 'Commit the completed changes and push to origin/dev.', + }) + .mockResolvedValueOnce({ + decision: 'complete', + reason: 'the rework and validation are complete', + confidence: 0.9, + requiresAudit: false, + }); + + supervisionAutomation.init(); + supervisionAutomation.registerTaskIntent( + 'deck_supervision_brain', + 'cmd-rework-fresh-pass-before-push', + 'implement the feature', + snapshot, + ); + beginRun('cmd-rework-fresh-pass-before-push', 'implement the feature'); + completeTurn('Implementation and validation are complete.'); + await waitForRunPhase('auditing'); + + completeDelegatedAudit('REWORK', 'Add the missing regression coverage.'); + await waitForRunPhase('execution'); + expect(mockTransportRuntime.send).toHaveBeenCalledTimes(2); + + completeTurn('The requested rework and validation are complete; no repository finalization was performed.'); + await waitForRunPhase('auditing'); + expect(mockTransportRuntime.send).toHaveBeenCalledTimes(3); + expect(String(mockTransportRuntime.send.mock.calls[2]?.[0])).toContain('imcodes send --reply'); + expect(mockTransportRuntime.send.mock.calls.some((call) => + String(call[0]).includes('Commit the completed changes and push to origin/dev.'))).toBe(false); + expect(supervisionAutomation.getActiveRun('deck_supervision_brain')).toMatchObject({ + phase: 'auditing', + freshAuditRequiredAfterRework: true, + deferredFinalization: { + nextAction: 'Commit the completed changes and push to origin/dev.', + }, + }); + + completeDelegatedAudit('PASS', 'The corrected implementation and regression coverage pass.'); + await waitForRunPhase('finalizing'); + expect(mockTransportRuntime.send).toHaveBeenCalledTimes(4); + expect(String(mockTransportRuntime.send.mock.calls[3]?.[0])).toContain( + 'Commit the completed changes and push to origin/dev.', + ); + expect(supervisionAutomation.getActiveRun('deck_supervision_brain')).toMatchObject({ + phase: 'finalizing', + freshAuditRequiredAfterRework: false, + }); + }); + + it.each([ + ['merge', 'Merge the repaired branch into master.'], + ['release', 'Create the release for the repaired change.'], + ['publish', 'Publish the repaired package.'], + ['deploy', 'Deploy the repaired change to production.'], + ['Chinese merge', '将当前分支合并到 master。'], + ['Chinese release', '发布当前版本。'], + ['Chinese deploy', '部署当前版本到生产环境。'], + ['Chinese go-live', '将当前版本上线。'], + ])('holds %s finalization after REWORK until a fresh PASS', async (_kind, finalizationAction) => { + const snapshot = await seedSession('supervised_audit', false, 1); + mockSupervisionDecide + .mockResolvedValueOnce({ + decision: 'complete', + reason: 'the initial implementation is ready for audit', + confidence: 0.9, + requiresAudit: true, + }) + .mockResolvedValueOnce({ + decision: 'continue', + reason: 'the requested rework and validation are complete; only delivery finalization remains', + confidence: 0.9, + requiresAudit: false, + gap: 'the repaired change has not been finalized', + nextAction: finalizationAction, + }); + + supervisionAutomation.init(); + supervisionAutomation.registerTaskIntent( + 'deck_supervision_brain', + `cmd-rework-${_kind}-fresh-pass`, + 'implement and deliver the feature', + snapshot, + ); + beginRun(`cmd-rework-${_kind}-fresh-pass`, 'implement and deliver the feature'); + completeTurn('The initial implementation and validation are complete.'); + await waitForRunPhase('auditing'); + + completeDelegatedAudit('REWORK', 'Repair the audit finding before delivery.'); + await waitForRunPhase('execution'); + completeTurn('The audit finding is repaired and validation passes. No finalization was performed.'); + await waitForRunPhase('auditing'); + + expect(mockTransportRuntime.send).toHaveBeenCalledTimes(3); + expect(String(mockTransportRuntime.send.mock.calls[2]?.[0])).toContain('imcodes send --reply'); + expect(mockTransportRuntime.send.mock.calls.some((call) => + String(call[0]).includes(finalizationAction))).toBe(false); + expect(supervisionAutomation.getActiveRun('deck_supervision_brain')).toMatchObject({ + phase: 'auditing', + freshAuditRequiredAfterRework: true, + deferredFinalization: { nextAction: finalizationAction }, + }); + + completeDelegatedAudit('PASS', 'The repaired change passes the fresh audit.'); + await waitForRunPhase('finalizing'); + expect(mockTransportRuntime.send).toHaveBeenCalledTimes(4); + expect(String(mockTransportRuntime.send.mock.calls[3]?.[0])).toContain(finalizationAction); + expect(supervisionAutomation.getActiveRun('deck_supervision_brain')).toMatchObject({ + phase: 'finalizing', + freshAuditRequiredAfterRework: false, + }); + }); + + it('strips mixed pre-audit validation and publish/deploy finalization until fresh PASS', async () => { + const snapshot = await seedSession('supervised_audit', false, 1); + const mixedAction = 'Run the focused tests, then deploy and publish the repaired release.'; + const finalizationAction = 'Deploy and publish the repaired release.'; + mockSupervisionDecide + .mockResolvedValueOnce({ + decision: 'complete', + reason: 'the initial implementation is ready for audit', + confidence: 0.9, + requiresAudit: true, + }) + .mockResolvedValueOnce({ + decision: 'continue', + reason: 'focused validation remains before delivery finalization', + confidence: 0.9, + requiresAudit: false, + gap: 'the focused tests have not run', + nextAction: mixedAction, + }) + .mockResolvedValueOnce({ + decision: 'continue', + reason: 'the repaired implementation and focused tests are complete; only delivery finalization remains', + confidence: 0.9, + requiresAudit: false, + gap: 'the repaired release has not been delivered', + nextAction: finalizationAction, + }); + + supervisionAutomation.init(); + supervisionAutomation.registerTaskIntent( + 'deck_supervision_brain', + 'cmd-rework-mixed-publish-deploy', + 'implement and deliver the feature', + snapshot, + ); + beginRun('cmd-rework-mixed-publish-deploy', 'implement and deliver the feature'); + completeTurn('The initial implementation is complete.'); + await waitForRunPhase('auditing'); + + completeDelegatedAudit('REWORK', 'Repair the finding and run focused tests.'); + await waitForRunPhase('execution'); + completeTurn('The finding is repaired, but the focused tests still need to run.'); + await waitForTransportSendCount(3); + + expect(mockTransportRuntime.send).toHaveBeenCalledTimes(3); + const preAuditContinue = String(mockTransportRuntime.send.mock.calls[2]?.[0]); + expect(preAuditContinue).toContain('Complete only the remaining substantive implementation or validation work'); + expect(preAuditContinue).toContain('Do not stage, commit, or push; do not merge, release, publish, or deploy.'); + expect(preAuditContinue).not.toContain(mixedAction); + expect(preAuditContinue).not.toContain(finalizationAction); + + completeTurn('The repaired implementation and focused tests now pass; no finalization was performed.'); + await waitForRunPhase('auditing'); + expect(mockTransportRuntime.send).toHaveBeenCalledTimes(4); + expect(String(mockTransportRuntime.send.mock.calls[3]?.[0])).toContain('imcodes send --reply'); + expect(mockTransportRuntime.send.mock.calls.some((call) => + String(call[0]).includes(finalizationAction))).toBe(false); + + completeDelegatedAudit('PASS', 'The repaired implementation and focused tests pass audit.'); + await waitForRunPhase('finalizing'); + expect(mockTransportRuntime.send).toHaveBeenCalledTimes(5); + expect(String(mockTransportRuntime.send.mock.calls[4]?.[0])).toContain(finalizationAction); + }); + it('keeps ordinary supervised commit and push continuation immediate', async () => { const snapshot = await seedSession('supervised'); mockSupervisionDecide.mockResolvedValueOnce({ @@ -1848,10 +2054,18 @@ describe('SupervisionAutomation', () => { supervisionAutomation.registerTaskIntent('deck_supervision_brain', 'cmd-loop-one', 'implement the feature', snapshot); beginRun('cmd-loop-one', 'implement the feature'); + // Each step waits for the phase it depends on instead of a fixed sleep. + // Dispatching the audit is async (broker decision + filesystem baseline + // discovery); sleep(25) covered that locally but not on a loaded CI runner. + // Completing the delegated audit before the run reached `auditing` derailed + // the sequence, and call[1] then held the audit-orchestration prompt rather + // than the rework brief — the macOS CI failure this replaces. completeTurn('implemented the feature'); - await sleep(25); + await waitForRunPhase('auditing'); + expect(mockTransportRuntime.send).toHaveBeenCalledTimes(1); + completeDelegatedAudit('REWORK', 'first audit needs fixes'); - await sleep(25); + await waitForRunPhase('execution'); expect(mockTransportRuntime.send).toHaveBeenCalledTimes(2); expect(String(mockTransportRuntime.send.mock.calls[1]?.[0])).toContain('Audit verdict: REWORK'); expect(supervisionAutomation.getActiveRun('deck_supervision_brain')).toMatchObject({ @@ -1860,13 +2074,16 @@ describe('SupervisionAutomation', () => { }); completeTurn('implemented the requested rework'); - await sleep(25); + await waitForRunPhase('auditing'); expect(mockTransportRuntime.send).toHaveBeenCalledTimes(3); expect(String(mockTransportRuntime.send.mock.calls[2]?.[0])).toContain('imcodes send --reply'); + + // maxAuditLoops = 1, so the second REWORK must end the run WITHOUT another + // rework dispatch. Wait for teardown, then assert the count never grew. completeDelegatedAudit('REWORK', 'second audit still needs fixes'); - await sleep(25); - expect(mockTransportRuntime.send).toHaveBeenCalledTimes(3); + await waitForRunEnd(); expect(supervisionAutomation.getActiveRun('deck_supervision_brain')).toBeUndefined(); + expect(mockTransportRuntime.send).toHaveBeenCalledTimes(3); }); it('ignores deprecated combo auditMode and still starts exactly one lightweight peer audit', async () => { diff --git a/test/daemon/supervision-prompts.test.ts b/test/daemon/supervision-prompts.test.ts index 250197246..3b2d2e2e9 100644 --- a/test/daemon/supervision-prompts.test.ts +++ b/test/daemon/supervision-prompts.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest'; import { normalizeSessionSupervisionSnapshot, SUPERVISION_MODE } from '../../shared/supervision-config.js'; import { buildPeerAuditBriefV1, + buildReworkBriefPrompt, buildSupervisionContinuePrompt, buildSupervisionDecisionPrompt, buildSupervisionDecisionRepairPrompt, @@ -126,10 +127,14 @@ describe('supervision prompts', () => { assistantResponse: 'Implementation and tests are complete; changes are not committed.', }); - expect(prompt).toContain('Peer audit MUST finish before repository commit/push finalization.'); + expect(prompt).toContain('Peer audit MUST finish before repository or delivery finalization'); + expect(prompt).toContain('A REWORK verdict means the previous audit did NOT pass'); + expect(prompt).toContain('require a fresh matching peer audit and a new PASS before any git add/commit/push'); + expect(prompt).toContain('merge, release, publish, or deploy'); expect(prompt).toContain('the daemon will hold it until peer-audit PASS instead of sending it now'); - expect(prompt).toContain('Never combine substantive pre-audit work and post-audit commit/push in one nextAction.'); + expect(prompt).toContain('Never combine substantive pre-audit work and post-audit finalization in one nextAction.'); expect(prompt).toContain('NEVER invent generic "remaining implementation or validation" work'); + expect(prompt).toContain('Return only the concrete repository or delivery finalization nextAction (git add/commit/push, merge, release, publish, or deploy as applicable).'); expect(prompt).toContain('exact auditor session ID and reply-enabled send command, exactly once'); expect(prompt).toContain('"requiresAudit":true'); expect(prompt).toContain('Set false for ordinary read-only checks, status queries, lookups, explanations, simple verification, and read-only review/audit.'); @@ -139,6 +144,20 @@ describe('supervision prompts', () => { expect(prompt).toContain('A task that starts as a check but proceeds to modify/fix something requires audit unless its matching audit is already pending or passed.'); }); + it('forbids repository finalization after REWORK until a fresh peer audit passes', () => { + const prompt = buildReworkBriefPrompt( + 'deck_supervision_brain', + 'Implement and deliver the fix', + 'The first implementation is ready.', + 'The auditor found a missing regression test.', + ); + + expect(prompt).toContain('This REWORK verdict means the previous audit did not pass.'); + expect(prompt).toContain('stop before repository finalization'); + expect(prompt).toContain('ready for a fresh peer audit'); + expect(prompt).toContain('Do not stage, commit, push, merge, release, publish, or deploy until a new matching peer audit returns PASS.'); + }); + it('does NOT include IM.codes workflow background in the continue prompt', () => { // Regression guard. The continue prompt is sent to the TARGET session's // chat, not to the supervisor judge. Injecting the IM.codes capability diff --git a/test/daemon/upgrade-blocked-outbox.test.ts b/test/daemon/upgrade-blocked-outbox.test.ts new file mode 100644 index 000000000..f3fd17be4 --- /dev/null +++ b/test/daemon/upgrade-blocked-outbox.test.ts @@ -0,0 +1,90 @@ +import { existsSync, mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { DAEMON_MSG } from '../../shared/daemon-events.js'; +import { DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION } from '../../shared/daemon-upgrade.js'; +import { UpgradeBlockedOutbox, type UpgradeInstallFailedMessage } from '../../src/daemon/upgrade-blocked-outbox.js'; + +const tempDirs: string[] = []; + +function makeFixture(now = 10_000) { + const dir = mkdtempSync(join(tmpdir(), 'imcodes-upgrade-blocked-outbox-')); + tempDirs.push(dir); + const file = join(dir, 'pending.json'); + const message: UpgradeInstallFailedMessage = { + type: DAEMON_MSG.UPGRADE_BLOCKED, + reason: 'install_failed', + failureId: 'failure-1', + upgradeId: 'upgrade-1', + fromVersion: '2026.7.3157-dev.3556', + targetVersion: '2026.7.3192-dev.3593', + retryReason: 'stale-staging-dir', + attempts: 5, + exitCode: 217, + log: '/tmp/imcodes-upgrade-test/upgrade.log', + ts: now, + }; + return { file, message, now }; +} + +afterEach(() => { + vi.restoreAllMocks(); + while (tempDirs.length > 0) rmSync(tempDirs.pop()!, { recursive: true, force: true }); +}); + +describe('UpgradeBlockedOutbox', () => { + it('retains an accepted terminal blocker so a later server restart can rebuild the target gate', async () => { + const { file, message, now } = makeFixture(); + const first = new UpgradeBlockedOutbox(file, () => now); + + expect(await first.report(message, () => false)).toBe(false); + expect(existsSync(file)).toBe(true); + + const afterRestart = new UpgradeBlockedOutbox(file, () => now + 1); + const send = vi.fn(() => true); + expect(await afterRestart.flushOnReconnect(send, message.fromVersion)).toBe(true); + expect(send).toHaveBeenCalledTimes(1); + expect(send).toHaveBeenCalledWith(message); + expect(existsSync(file)).toBe(true); + + expect(await afterRestart.acknowledge( + message.failureId, + DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.ACCEPTED, + )).toBe(true); + expect(existsSync(file)).toBe(true); + + const afterServerRestart = new UpgradeBlockedOutbox(file, () => now + 2); + expect(await afterServerRestart.flushOnReconnect(send, message.fromVersion)).toBe(true); + expect(send).toHaveBeenCalledTimes(2); + + expect(await afterServerRestart.acknowledge( + message.failureId, + DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.OBSOLETE, + )).toBe(true); + expect(existsSync(file)).toBe(false); + }); + + it('clears a persisted failure superseded by a newer explicit manual upgrade', async () => { + const { file, message, now } = makeFixture(); + const outbox = new UpgradeBlockedOutbox(file, () => now); + await outbox.report(message, () => false); + + expect(await outbox.acknowledge( + message.failureId, + DAEMON_UPGRADE_BLOCKED_ACK_DISPOSITION.SUPERSEDED, + )).toBe(true); + expect(existsSync(file)).toBe(false); + }); + + it('clears an obsolete failure after a later daemon version starts', async () => { + const { file, message, now } = makeFixture(); + const outbox = new UpgradeBlockedOutbox(file, () => now); + await outbox.report(message, () => false); + + const send = vi.fn(() => true); + expect(await outbox.flushOnReconnect(send, '2026.7.3192-dev.3593')).toBe(false); + expect(send).not.toHaveBeenCalled(); + expect(existsSync(file)).toBe(false); + }); +}); diff --git a/test/e2e/daemon-upgrade-gate.test.ts b/test/e2e/daemon-upgrade-gate.test.ts index c4b8e292b..1a44846d0 100644 --- a/test/e2e/daemon-upgrade-gate.test.ts +++ b/test/e2e/daemon-upgrade-gate.test.ts @@ -30,18 +30,26 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { DAEMON_MSG } from '../../shared/daemon-events.js'; +import { DAEMON_UPGRADE_BLOCK_REASON } from '../../shared/daemon-upgrade.js'; const mocks = vi.hoisted(() => { const store = new Map>(); const runtimes = new Map string; sending: boolean; pendingCount: number }>(); const spawnCalls: Array<{ command: string; args: readonly string[] }> = []; + const spawnedChildren: Array<{ + command: string; + emit: (event: string, ...args: unknown[]) => void; + }> = []; const writeCalls: Array<{ path: string; data: string }> = []; + const upgradeBlockedReport = vi.fn(async () => false); let compressionState = { active: false, activeCount: 0, queued: 0, idle: true }; return { store, runtimes, spawnCalls, + spawnedChildren, writeCalls, + upgradeBlockedReport, getCompressionState: () => compressionState, setCompressionState: (next: typeof compressionState) => { compressionState = next; }, }; @@ -62,6 +70,12 @@ vi.mock('../../src/store/session-store.js', () => ({ updateSessionState: vi.fn(), })); +vi.mock('../../src/daemon/upgrade-blocked-outbox.js', () => ({ + getDefaultUpgradeBlockedOutbox: () => ({ + report: mocks.upgradeBlockedReport, + }), +})); + vi.mock('../../src/agent/session-manager.js', async (importOriginal) => { const actual = await importOriginal(); return { @@ -78,12 +92,25 @@ vi.mock('../../src/agent/session-manager.js', async (importOriginal) => { // caring about platform. function captureSpawn(command: string, args: readonly string[]) { mocks.spawnCalls.push({ command, args }); - return { + const handlers = new Map void>>(); + const child = { unref: vi.fn(), - on: vi.fn(), + on: vi.fn((event: string, handler: (...args: unknown[]) => void) => { + const existing = handlers.get(event) ?? []; + existing.push(handler); + handlers.set(event, existing); + return child; + }), stdout: { on: vi.fn() }, stderr: { on: vi.fn() }, } as any; + mocks.spawnedChildren.push({ + command, + emit: (event, ...eventArgs) => { + for (const handler of handlers.get(event) ?? []) handler(...eventArgs); + }, + }); + return child; } vi.mock('child_process', async (importOriginal) => { @@ -303,6 +330,8 @@ describe('daemon.upgrade gate (e2e regression for 3389fab2)', () => { mocks.store.clear(); mocks.runtimes.clear(); mocks.spawnCalls.length = 0; + mocks.spawnedChildren.length = 0; + mocks.upgradeBlockedReport.mockClear(); mocks.setCompressionState({ active: false, activeCount: 0, queued: 0, idle: true }); }); @@ -573,6 +602,7 @@ skipOnWindows('daemon.upgrade — Linux/macOS upgrade.sh contract', () => { mocks.store.clear(); mocks.runtimes.clear(); mocks.spawnCalls.length = 0; + mocks.spawnedChildren.length = 0; mocks.writeCalls.length = 0; }); @@ -580,16 +610,27 @@ skipOnWindows('daemon.upgrade — Linux/macOS upgrade.sh contract', () => { vi.clearAllMocks(); }); - async function captureUpgradeScript(): Promise { + async function captureUpgradeAttempt(): Promise<{ + script: string; + serverLink: { send: ReturnType }; + }> { const serverLink = { send: vi.fn() } as { send: ReturnType }; - handleWebCommand({ type: 'daemon.upgrade', targetVersion: '99.99.99-test' } as any, serverLink as any); + handleWebCommand({ + type: 'daemon.upgrade', + targetVersion: '99.99.99-test', + upgradeId: 'upgrade-attempt-test', + } as any, serverLink as any); await waitForCondition( () => mocks.writeCalls.some((c) => c.path.endsWith('upgrade.sh')), 5000, ); const script = mocks.writeCalls.find((c) => c.path.endsWith('upgrade.sh')); if (!script) throw new Error('upgrade.sh was never written'); - return script.data; + return { script: script.data, serverLink }; + } + + async function captureUpgradeScript(): Promise { + return (await captureUpgradeAttempt()).script; } it('captures the real install exit code (regression: pre-fix logged "(exit 0)" on every failure)', async () => { @@ -626,6 +667,57 @@ skipOnWindows('daemon.upgrade — Linux/macOS upgrade.sh contract', () => { expect(sh).toMatch(/tail -20 "\$INSTALL_OUT"/); }); + it('cleans stale npm staging before install and retries only matching rename collisions', async () => { + const sh = await captureUpgradeScript(); + + const cleanupIdx = sh.indexOf('cleanup_stale_imcodes_staging_dirs "$GLOBAL_ROOT"'); + const installIdx = sh.indexOf('install -g --ignore-scripts --prefer-online'); + expect(cleanupIdx).toBeGreaterThan(-1); + expect(installIdx).toBeGreaterThan(-1); + expect(cleanupIdx).toBeLessThan(installIdx); + expect(sh).toContain('is_recoverable_layout_output "$INSTALL_OUT" "$GLOBAL_ROOT"'); + expect(sh).toContain('recover_stale_layout_from_output "$INSTALL_OUT" "$GLOBAL_ROOT"'); + expect(sh).toContain('RETRY_REASON="stale-staging-dir"'); + expect(sh).toMatch(/ENOTEMPTY\|EEXIST/); + expect(sh).toContain('[ "$error_syscall" = "rename" ]'); + expect(sh).toContain('[ "$error_path" = "$global_root/imcodes" ]'); + }); + + it('reports a final detached npm install failure to the server with bounded diagnostics', async () => { + const realFs = await vi.importActual('node:fs'); + const statusDir = '/tmp/imcodes-upgrade-gate-test'; + const statusPath = `${statusDir}/upgrade-status.json`; + realFs.mkdirSync(statusDir, { recursive: true }); + try { + const { serverLink } = await captureUpgradeAttempt(); + const bashChild = mocks.spawnedChildren.find((entry) => entry.command === '/bin/bash'); + expect(bashChild).toBeDefined(); + realFs.writeFileSync(statusPath, JSON.stringify({ + state: 'blocked', + reason: DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED, + retryReason: 'stale-staging-dir', + attempts: 5, + exitCode: 217, + })); + + bashChild!.emit('exit', 75, null); + + await vi.waitFor(() => expect(mocks.upgradeBlockedReport).toHaveBeenCalledWith(expect.objectContaining({ + type: DAEMON_MSG.UPGRADE_BLOCKED, + reason: DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED, + upgradeId: 'upgrade-attempt-test', + retryReason: 'stale-staging-dir', + attempts: 5, + exitCode: 217, + }), expect.any(Function))); + expect(serverLink.send).not.toHaveBeenCalledWith(expect.objectContaining({ + reason: DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED, + })); + } finally { + realFs.rmSync(statusPath, { force: true }); + } + }); + it('uses an atomic single-flight lock before touching the global npm install', async () => { const sh = await captureUpgradeScript(); diff --git a/test/e2e/memory-mcp-interface.test.ts b/test/e2e/memory-mcp-interface.test.ts index 9ee11d108..49ea4d94d 100644 --- a/test/e2e/memory-mcp-interface.test.ts +++ b/test/e2e/memory-mcp-interface.test.ts @@ -13,13 +13,17 @@ import { import { ALIAS_MCP_TOOLS } from '../../shared/alias-types.js'; import { MEMORY_FEATURE_FLAGS_BY_NAME, memoryFeatureFlagEnvKey } from '../../shared/feature-flags.js'; import { MEMORY_MCP_ENV_KEYS, buildMemoryMcpServerEnv } from '../../shared/memory-mcp-env.js'; +import { makeMemoryShortRef } from '../../src/context/memory-short-ref.js'; import { createMemoryMcpToolHandlers } from '../../src/daemon/memory-mcp-tools.js'; import type { McpRuntimeCaller } from '../../src/daemon/memory-mcp-caller.js'; import { archiveEventsForMaterialization, + ensureContextNamespace, listContextObservations, recordContextEvent, resetContextStoreForTests, + upsertMemoryShortRefs, + writeContextObservation, writeProcessedProjection, } from '../../src/store/context-store.js'; import { cleanupIsolatedSharedContextDb, createIsolatedSharedContextDb } from '../util/shared-context-db.js'; @@ -199,7 +203,7 @@ describe('memory MCP interface e2e', () => { })); expect(search).toMatchObject({ status: 'ok' }); const items = search.items as Array>; - const expectedRef = `obs:${observationId.replace(/[^a-f0-9]/gi, '').slice(0, 10)}`; + const expectedRef = makeMemoryShortRef('observation', observationId); expect(items[0]).toMatchObject({ observationId, ref: expectedRef, @@ -252,6 +256,115 @@ describe('memory MCP interface e2e', () => { }); }); + it('redeems and mutates refs persisted after the independent MCP process starts', async () => { + // The MCP project scoping layer intentionally maps user_private to the + // project-local personal namespace before resolving refs. + const lateRefNamespace = { + scope: 'personal' as const, + userId: USER_ID, + projectId: PROJECT_NAME, + }; + const readable = writeProcessedProjection({ + namespace: lateRefNamespace, + class: 'recent_summary', + sourceEventIds: ['evt-late-ref-readable'], + summary: 'projection persisted after MCP startup remains directly redeemable', + content: { ownerUserId: USER_ID, eventCount: 1 }, + origin: 'chat_compacted', + createdAt: 1_100, + updatedAt: 1_100, + }); + const manageable = writeProcessedProjection({ + namespace: lateRefNamespace, + class: 'recent_summary', + sourceEventIds: ['evt-late-ref-manageable'], + summary: 'projection persisted after MCP startup remains manageable by ref', + content: { ownerUserId: USER_ID, eventCount: 1 }, + origin: 'chat_compacted', + createdAt: 1_200, + updatedAt: 1_200, + }); + const observationNamespace = ensureContextNamespace(lateRefNamespace, 1_250); + const observation = writeContextObservation({ + namespaceId: observationNamespace.id, + scope: lateRefNamespace.scope, + class: 'note', + origin: 'agent_learned', + fingerprint: 'late-cross-process-observation-ref', + content: { text: 'observation persisted after MCP startup remains directly redeemable' }, + text: 'observation persisted after MCP startup remains directly redeemable', + sourceEventIds: ['evt-late-ref-observation'], + state: 'active', + now: 1_250, + }); + const readableRef = makeMemoryShortRef('projection', readable.id); + const manageableRef = makeMemoryShortRef('projection', manageable.id); + const observationRef = makeMemoryShortRef('observation', observation.id); + + await withStdioClient(childEnv(), async (client) => { + // Ensure the child is fully started before another process writes these + // rows. No search/list call is allowed to register them in the child Map. + await client.listTools(); + upsertMemoryShortRefs([ + { ref: readableRef, kind: 'projection', id: readable.id }, + { ref: manageableRef, kind: 'projection', id: manageable.id }, + { ref: observationRef, kind: 'observation', id: observation.id }, + ].map((entry, index) => ({ + ...entry, + namespaceKey: JSON.stringify([ + lateRefNamespace.scope, + lateRefNamespace.userId, + lateRefNamespace.projectId, + '', + '', + ]), + namespaceJson: JSON.stringify(lateRefNamespace), + lastSeenAt: Date.now() + index, + }))); + + const sources = structured(await client.callTool({ + name: MEMORY_MCP_TOOL_NAMES.GET_MEMORY_SOURCES, + arguments: { ref: readableRef, kind: 'projection' }, + })); + expect(sources).toMatchObject({ + status: 'ok', + projectionId: readable.id, + sourceEventCount: 1, + projectionSource: expect.objectContaining({ + eventId: 'evt-late-ref-readable', + content: readable.summary, + }), + }); + + const observationSources = structured(await client.callTool({ + name: MEMORY_MCP_TOOL_NAMES.GET_MEMORY_SOURCES, + arguments: { ref: observationRef, kind: 'observation' }, + })); + expect(observationSources).toMatchObject({ + status: 'ok', + observationId: observation.id, + sourceEventCount: 1, + sources: [ + expect.objectContaining({ + eventId: 'evt-late-ref-observation', + status: 'observation', + content: 'observation persisted after MCP startup remains directly redeemable', + }), + ], + }); + + const archived = structured(await client.callTool({ + name: MEMORY_MCP_TOOL_NAMES.ARCHIVE_MEMORY, + arguments: { ref: manageableRef }, + })); + expect(archived).toMatchObject({ + status: 'ok', + projectionId: manageable.id, + changed: true, + }); + }); + }); + it('finds a projection by MCP search and expands summary fallback by projectionId or short ref', async () => { const projection = writeProcessedProjection({ namespace, @@ -275,7 +388,7 @@ describe('memory MCP interface e2e', () => { const listedItems = listed.items as Array>; expect(listedItems.find((item) => item.projectionId === projection.id)).toMatchObject({ projectionId: projection.id, - ref: `proj:${projection.id.replace(/[^a-f0-9]/gi, '').slice(0, 10)}`, + ref: makeMemoryShortRef('projection', projection.id), recordKind: 'projection', projectionClass: 'recent_summary', sourceLookup: { @@ -305,7 +418,7 @@ describe('memory MCP interface e2e', () => { }, }); expect(['exact', 'semantic', 'trigram']).toContain(hit?.matchKind); - const expectedRef = `proj:${projection.id.replace(/[^a-f0-9]/gi, '').slice(0, 10)}`; + const expectedRef = makeMemoryShortRef('projection', projection.id); expect(hit?.ref).toBe(expectedRef); const sources = structured(await client.callTool({ diff --git a/test/shared/memory-mcp-contracts.test.ts b/test/shared/memory-mcp-contracts.test.ts index 2192a61a2..57932d0e4 100644 --- a/test/shared/memory-mcp-contracts.test.ts +++ b/test/shared/memory-mcp-contracts.test.ts @@ -134,6 +134,21 @@ describe('memory MCP shared contracts', () => { } }); + it('tells the model an ambiguous ref is bounded and that empty sources is not "no memory"', () => { + // This description is what the model actually reads — registerMemoryMcpTools + // publishes it verbatim. It previously promised "every match" while the + // implementation expanded at most four, so a caller could stop looking with + // the answer sitting in an omitted record; and an ambiguous reply carries an + // empty `sources`, which reads as "no memory" unless the text says otherwise. + const getSources = MEMORY_MCP_TOOL_CONTRACTS[MEMORY_MCP_TOOL_NAMES.GET_MEMORY_SOURCES]; + + expect(getSources.description).toContain('up to four'); + expect(getSources.description).not.toMatch(/every match/i); + expect(getSources.description).toContain('candidateCount'); + expect(getSources.description).toContain('truncated'); + expect(getSources.description).toMatch(/not.*no memory/i); + }); + it('documents when search_memory results should be expanded through get_memory_sources', () => { const search = MEMORY_MCP_TOOL_CONTRACTS[MEMORY_MCP_TOOL_NAMES.SEARCH_MEMORY]; const getSources = MEMORY_MCP_TOOL_CONTRACTS[MEMORY_MCP_TOOL_NAMES.GET_MEMORY_SOURCES]; diff --git a/test/shared/metrics.test.ts b/test/shared/metrics.test.ts new file mode 100644 index 000000000..06d1f714e --- /dev/null +++ b/test/shared/metrics.test.ts @@ -0,0 +1,72 @@ +/** + * The counter store had no test at all, and it had already silently drifted into + * two copies with different APIs. These tests pin the two properties that matter: + * the cardinality cap actually holds, and the daemon/server entry points are the + * SAME store rather than two that merely look alike. + */ +import { beforeEach, describe, expect, it } from 'vitest'; + +import { + addCounter, + counterKeyCount, + getCounter, + incrementCounter, + resetMetricsForTests, + snapshotCounters, +} from '../../shared/metrics.js'; +import * as daemonMetrics from '../../src/util/metrics.js'; + +describe('shared metrics store', () => { + beforeEach(() => { + resetMetricsForTests(); + }); + + it('counts per name+labels combination, order-independently', () => { + incrementCounter('a.b', { x: '1', y: '2' }); + incrementCounter('a.b', { y: '2', x: '1' }); + expect(getCounter('a.b', { x: '1', y: '2' })).toBe(2); + // A different label value is a different series, not the same counter. + expect(getCounter('a.b', { x: '9', y: '2' })).toBe(0); + expect(getCounter('a.b')).toBe(0); + }); + + it('ignores non-positive and non-finite amounts instead of corrupting a total', () => { + addCounter('c', 5); + addCounter('c', 0); + addCounter('c', -3); + addCounter('c', Number.NaN); + addCounter('c', Number.POSITIVE_INFINITY); + expect(getCounter('c')).toBe(5); + }); + + it('caps distinct keys at 10k, but keeps counting keys it already knows', () => { + for (let i = 0; i < 10_000; i++) incrementCounter('bulk', { i: String(i) }); + expect(counterKeyCount()).toBe(10_000); + + // Past the cap a NEW key is dropped... + incrementCounter('bulk', { i: 'overflow' }); + expect(getCounter('bulk', { i: 'overflow' })).toBe(0); + expect(counterKeyCount()).toBe(10_000); + + // ...while an EXISTING key still accumulates, so established series do not + // freeze just because something noisy filled the map. + incrementCounter('bulk', { i: '0' }); + expect(getCounter('bulk', { i: '0' })).toBe(2); + }); + + it('exposes the daemon entry point as the same store, not a second copy', () => { + // The drift that motivated consolidation: two modules with one API each. + daemonMetrics.incrementCounter('shared.store.check'); + expect(getCounter('shared.store.check')).toBe(1); + expect(daemonMetrics.getCounter('shared.store.check')).toBe(1); + expect(typeof daemonMetrics.addCounter).toBe('function'); + }); + + it('snapshots as a plain object and is fully cleared by the test reset', () => { + incrementCounter('snap', { k: 'v' }); + expect(snapshotCounters()).toEqual({ 'snap{k=v}': 1 }); + resetMetricsForTests(); + expect(snapshotCounters()).toEqual({}); + expect(counterKeyCount()).toBe(0); + }); +}); diff --git a/test/shared/timeline-delivery-telemetry.test.ts b/test/shared/timeline-delivery-telemetry.test.ts new file mode 100644 index 000000000..1bdd2325b --- /dev/null +++ b/test/shared/timeline-delivery-telemetry.test.ts @@ -0,0 +1,79 @@ +/** + * The healing fix (activation / reconnect requesting a no-lower-bound window) + * removes the user-visible symptom of dropped timeline events, which is exactly + * why the drop itself has to become a number. These tests pin the classifier so + * the counters cannot silently start counting the wrong thing: + * + * - content-bearing events (a lost one leaves a hole in the chat) are counted; + * - status/telemetry chatter is NOT — `agent.status` fires about once a second + * during a turn and would bury the signal under noise. + */ +import { describe, expect, it } from 'vitest'; + +import { + TIMELINE_DELIVERY_METRICS, + isContentBearingTimelineEvent, + timelineEventTypeOf, +} from '../../shared/timeline-delivery-telemetry.js'; + +describe('timeline delivery telemetry classifier', () => { + it('counts events whose loss leaves a visible hole in the chat', () => { + for (const type of [ + 'user.message', + 'assistant.text', + 'tool.call', + 'tool.result', + 'file.change', + 'ask.question', + 'peer_audit.result', + ]) { + expect(isContentBearingTimelineEvent({ type })).toBe(true); + } + }); + + it('ignores high-frequency status/telemetry so it cannot drown the signal', () => { + for (const type of [ + 'agent.status', + 'usage.update', + 'session.state', + 'assistant.thinking', + 'memory.context', + 'memory.compression', + 'transport.queue.snapshot', + 'command.ack', + ]) { + expect(isContentBearingTimelineEvent({ type })).toBe(false); + } + }); + + it('treats unknown and malformed events as non-counting (fail closed)', () => { + // A future noisy event type must not inflate the drop series by default. + expect(isContentBearingTimelineEvent({ type: 'something.new' })).toBe(false); + expect(isContentBearingTimelineEvent({})).toBe(false); + expect(isContentBearingTimelineEvent(null)).toBe(false); + expect(isContentBearingTimelineEvent(undefined)).toBe(false); + expect(isContentBearingTimelineEvent('assistant.text')).toBe(false); + expect(isContentBearingTimelineEvent([{ type: 'assistant.text' }])).toBe(false); + }); + + it('reads the inner event type out of a timeline.event envelope', () => { + expect(timelineEventTypeOf({ type: 'timeline.event', event: { type: 'assistant.text' } })) + .toBe('assistant.text'); + expect(timelineEventTypeOf({ type: 'timeline.event' })).toBeUndefined(); + expect(timelineEventTypeOf({ type: 'timeline.event', event: null })).toBeUndefined(); + expect(timelineEventTypeOf(null)).toBeUndefined(); + }); + + it('keeps one shared counter name per drop site so daemon and server agree', () => { + // Daemon and server both emit these; drifting names would split the series + // and hide a regression in plain sight. + expect(TIMELINE_DELIVERY_METRICS.DAEMON_LINK_DOWN_DROPPED) + .toBe('timeline.delivery.daemon_link_down_dropped'); + expect(TIMELINE_DELIVERY_METRICS.SERVER_NO_SUBSCRIBER_DROPPED) + .toBe('timeline.delivery.server_no_subscriber_dropped'); + expect(TIMELINE_DELIVERY_METRICS.SERVER_DELIVERED) + .toBe('timeline.delivery.server_delivered'); + expect(new Set(Object.values(TIMELINE_DELIVERY_METRICS)).size) + .toBe(Object.values(TIMELINE_DELIVERY_METRICS).length); + }); +}); diff --git a/test/shared/wire-protocol-contract.test.ts b/test/shared/wire-protocol-contract.test.ts index f442ff3e4..9f72d7212 100644 --- a/test/shared/wire-protocol-contract.test.ts +++ b/test/shared/wire-protocol-contract.test.ts @@ -68,6 +68,8 @@ describe('shared daemon/server/web wire protocol contracts', () => { RECONNECTED: 'daemon.reconnected', DISCONNECTED: 'daemon.disconnected', UPGRADE_BLOCKED: 'daemon.upgrade_blocked', + UPGRADE_BLOCKED_ACK: 'daemon.upgrade_blocked_ack', + UPGRADE_BLOCKED_SYNC: 'daemon.upgrade_blocked_sync', UPGRADING: 'daemon.upgrading', MACHINE_EXEC_CHUNK: 'machine.exec_chunk', MACHINE_EXEC_RESULT: 'machine.exec_result', diff --git a/test/util/posix-upgrade-layout-recovery.test.ts b/test/util/posix-upgrade-layout-recovery.test.ts new file mode 100644 index 000000000..65416d317 --- /dev/null +++ b/test/util/posix-upgrade-layout-recovery.test.ts @@ -0,0 +1,184 @@ +import { mkdtempSync, mkdirSync, rmSync, utimesSync, writeFileSync, existsSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { afterEach, describe, expect, it } from 'vitest'; +import { + buildPosixUpgradeLayoutRecoveryScript, + parsePosixUpgradeFailureStatus, +} from '../../src/util/posix-upgrade-layout-recovery.js'; + +const describePosix = process.platform === 'win32' ? describe.skip : describe; +const tempDirs: string[] = []; + +function makeTempDir(): string { + const dir = mkdtempSync(join(tmpdir(), 'imcodes-upgrade-layout-')); + tempDirs.push(dir); + return dir; +} + +function runRecoveryHarness(body: string, env: Record) { + return spawnSync('bash', ['-c', ` +set -u +log() { :; } +${buildPosixUpgradeLayoutRecoveryScript()} +${body} +`], { + encoding: 'utf8', + env: { ...process.env, ...env }, + }); +} + +afterEach(() => { + while (tempDirs.length > 0) { + rmSync(tempDirs.pop()!, { recursive: true, force: true }); + } +}); + +describePosix('POSIX daemon-upgrade npm layout recovery', () => { + it('removes only old interrupted .imcodes-* siblings before npm install', () => { + const root = makeTempDir(); + const current = join(root, 'imcodes'); + const staleA = join(root, '.imcodes-Vuo7WXWs'); + const staleB = join(root, '.imcodes-other'); + const recent = join(root, '.imcodes-active-install'); + mkdirSync(current); + mkdirSync(join(staleA, 'nested'), { recursive: true }); + mkdirSync(staleB); + mkdirSync(recent); + writeFileSync(join(staleA, 'nested', 'leftover'), 'stale'); + const old = new Date(Date.now() - 2 * 60 * 60_000); + utimesSync(staleA, old, old); + utimesSync(staleB, old, old); + + const result = runRecoveryHarness('cleanup_stale_imcodes_staging_dirs "$ROOT" 1800', { ROOT: root }); + + expect(result.status, result.stderr).toBe(0); + expect(existsSync(current)).toBe(true); + expect(existsSync(staleA)).toBe(false); + expect(existsSync(staleB)).toBe(false); + expect(existsSync(recent)).toBe(true); + }); + + it('preserves a recent staging directory that may belong to an active external npm install', () => { + const root = makeTempDir(); + const active = join(root, '.imcodes-active-install'); + mkdirSync(active); + writeFileSync(join(active, 'in-progress'), 'active'); + + const result = runRecoveryHarness('cleanup_stale_imcodes_staging_dirs "$ROOT" 1800', { ROOT: root }); + + expect(result.status, result.stderr).toBe(0); + expect(existsSync(active)).toBe(true); + }); + + it.each(['ENOTEMPTY', 'EEXIST'])('classifies %s rename collisions and removes the exact stale destination', (code) => { + const root = makeTempDir(); + const current = join(root, 'imcodes'); + const stale = join(root, '.imcodes-Vuo7WXWs'); + const output = join(root, 'npm-output.log'); + mkdirSync(current); + mkdirSync(stale); + writeFileSync(join(stale, 'leftover'), 'stale'); + writeFileSync(output, [ + `npm error code ${code}`, + 'npm error syscall rename', + `npm error path ${current}`, + `npm error dest ${stale}`, + `npm error ${code}: directory not empty`, + '', + ].join('\n')); + + const result = runRecoveryHarness( + 'is_recoverable_layout_output "$OUTPUT" "$ROOT" && recover_stale_layout_from_output "$OUTPUT" "$ROOT"', + { ROOT: root, OUTPUT: output }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(existsSync(current)).toBe(true); + expect(existsSync(stale)).toBe(false); + }); + + it.each([ + ['EACCES', 'rename'], + ['E401', 'rename'], + ['ENOTEMPTY', 'unlink'], + ])('does not retry or clean a genuine non-layout failure (%s/%s)', (code, syscall) => { + const root = makeTempDir(); + const current = join(root, 'imcodes'); + const stale = join(root, '.imcodes-Vuo7WXWs'); + const output = join(root, 'npm-output.log'); + mkdirSync(current); + mkdirSync(stale); + writeFileSync(join(stale, 'must-stay'), 'data'); + writeFileSync(output, [ + `npm error code ${code}`, + `npm error syscall ${syscall}`, + `npm error path ${current}`, + `npm error dest ${stale}`, + '', + ].join('\n')); + + const result = runRecoveryHarness( + 'if is_recoverable_layout_output "$OUTPUT" "$ROOT"; then exit 42; fi', + { ROOT: root, OUTPUT: output }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(existsSync(stale)).toBe(true); + }); + + it('rejects a rename destination outside the global package root', () => { + const root = makeTempDir(); + const outsideRoot = makeTempDir(); + const current = join(root, 'imcodes'); + const outside = join(outsideRoot, '.imcodes-Vuo7WXWs'); + const output = join(root, 'npm-output.log'); + mkdirSync(current); + mkdirSync(outside); + writeFileSync(output, [ + 'npm error code ENOTEMPTY', + 'npm error syscall rename', + `npm error path ${current}`, + `npm error dest ${outside}`, + '', + ].join('\n')); + + const result = runRecoveryHarness( + 'if is_recoverable_layout_output "$OUTPUT" "$ROOT"; then exit 42; fi', + { ROOT: root, OUTPUT: output }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(existsSync(outside)).toBe(true); + }); +}); + +describe('POSIX daemon-upgrade failure marker', () => { + it('accepts the bounded install-failure status emitted by the detached script', () => { + expect(parsePosixUpgradeFailureStatus(JSON.stringify({ + state: 'blocked', + reason: 'install_failed', + retryReason: 'stale-staging-dir', + attempts: 3, + exitCode: 217, + }))).toEqual({ + state: 'blocked', + reason: 'install_failed', + retryReason: 'stale-staging-dir', + attempts: 3, + exitCode: 217, + }); + }); + + it('rejects malformed or unbounded failure markers', () => { + expect(parsePosixUpgradeFailureStatus('not-json')).toBeNull(); + expect(parsePosixUpgradeFailureStatus(JSON.stringify({ + state: 'blocked', + reason: 'install_failed', + retryReason: '../../escape', + attempts: 0, + exitCode: 999, + }))).toBeNull(); + }); +}); diff --git a/web/src/app.tsx b/web/src/app.tsx index ce453278d..7aa0fa657 100644 --- a/web/src/app.tsx +++ b/web/src/app.tsx @@ -3486,42 +3486,6 @@ export function App() { } } } - if (msg.type === DAEMON_MSG.UPGRADE_BLOCKED) { - const now = Date.now(); - if (!shouldShowDaemonUpgradeBlockedToast(lastUpgradeBlockedToastRef.current, msg.reason, now)) return; - lastUpgradeBlockedToastRef.current = { reason: msg.reason, shownAt: now }; - const message = trans(daemonUpgradeBlockedToastKey(msg.reason)); - const id = Date.now() + Math.random(); - setToasts((prev) => [...prev, { - id, - sessionName: '', - project: '', - kind: 'notification', - title: trans('toast.upgrade_blocked_title'), - message, - }]); - setTimeout(() => setToasts((prev) => prev.filter((x) => x.id !== id)), 8000); - } - if (msg.type === DAEMON_MSG.DISCONNECTED) { - // Mark projection stale immediately — that's just a data-freshness - // hint, not the user-facing status badge. But do NOT flip the - // "Daemon Offline" badge yet: the server side still has a - // RECONNECT_GRACE_MS window during which the daemon can reconnect - // and inflight commands are replayed without surfacing any failure. - // Matching that grace period here prevents the badge from flashing - // on every pod restart / brief network blip while the user's turn - // is actually landing fine. If the daemon does stay gone, the - // server will broadcast MSG_DAEMON_OFFLINE (no reconnect event) and - // this timer fires, putting the badge into the Daemon-Offline - // state. RECONNECTED / session_list clear the timer below. - watchProjectionStore.setSnapshotStatus('stale'); - if (daemonOfflineGraceTimerRef.current) clearTimeout(daemonOfflineGraceTimerRef.current); - daemonOfflineGraceTimerRef.current = setTimeout(() => { - daemonOfflineGraceTimerRef.current = null; - setDaemonOnline(false); - setServers((prev) => markServerOffline(prev, selectedServerId)); - }, RECONNECT_GRACE_MS); - } { // Daemon-upgrade badge lifecycle (pure reducer): UPGRADING begins it // (so the version shows "upgrading…" through the restart instead of a @@ -3547,6 +3511,43 @@ export function App() { } } } + if (msg.type === DAEMON_MSG.UPGRADE_BLOCKED) { + const now = Date.now(); + if (shouldShowDaemonUpgradeBlockedToast(lastUpgradeBlockedToastRef.current, msg.reason, now)) { + lastUpgradeBlockedToastRef.current = { reason: msg.reason, shownAt: now }; + const message = trans(daemonUpgradeBlockedToastKey(msg.reason)); + const id = Date.now() + Math.random(); + setToasts((prev) => [...prev, { + id, + sessionName: '', + project: '', + kind: 'notification', + title: trans('toast.upgrade_blocked_title'), + message, + }]); + setTimeout(() => setToasts((prev) => prev.filter((x) => x.id !== id)), 8000); + } + } + if (msg.type === DAEMON_MSG.DISCONNECTED) { + // Mark projection stale immediately — that's just a data-freshness + // hint, not the user-facing status badge. But do NOT flip the + // "Daemon Offline" badge yet: the server side still has a + // RECONNECT_GRACE_MS window during which the daemon can reconnect + // and inflight commands are replayed without surfacing any failure. + // Matching that grace period here prevents the badge from flashing + // on every pod restart / brief network blip while the user's turn + // is actually landing fine. If the daemon does stay gone, the + // server will broadcast MSG_DAEMON_OFFLINE (no reconnect event) and + // this timer fires, putting the badge into the Daemon-Offline + // state. RECONNECTED / session_list clear the timer below. + watchProjectionStore.setSnapshotStatus('stale'); + if (daemonOfflineGraceTimerRef.current) clearTimeout(daemonOfflineGraceTimerRef.current); + daemonOfflineGraceTimerRef.current = setTimeout(() => { + daemonOfflineGraceTimerRef.current = null; + setDaemonOnline(false); + setServers((prev) => markServerOffline(prev, selectedServerId)); + }, RECONNECT_GRACE_MS); + } if (msg.type === MSG_DAEMON_ONLINE || msg.type === DAEMON_MSG.RECONNECTED) { void checkForAppUpdate(); if (daemonOfflineGraceTimerRef.current) { diff --git a/web/src/components/ChatView.tsx b/web/src/components/ChatView.tsx index 1382bb110..552f32eef 100644 --- a/web/src/components/ChatView.tsx +++ b/web/src/components/ChatView.tsx @@ -16,6 +16,7 @@ import type { } from '../ws-client.js'; import type { FileChangeBatch, FileChangePatch } from '@shared/file-change.js'; import { FS_READ_ERROR_CODES } from '@shared/fs-read-error-codes.js'; +import { isInsufficientCapacityError } from '../upload-error.js'; import { SDK_SUBAGENT_DETAIL_KIND, SDK_SUBAGENT_DIAGNOSTIC, @@ -29,7 +30,11 @@ import { isHtmlPreviewPath, type HtmlPreviewViewMode } from '@shared/html-previe import { FileBrowser, type FileBrowserPreviewRequest } from './file-browser-lazy.js'; import { ChatMarkdown } from './ChatMarkdown.js'; import { AgentTodoList } from './AgentTodoList.js'; -import { computeFollowThresholds } from './chat-follow-thresholds.js'; +import { + CHAT_MOUNT_SETTLE_MS, + CHAT_MOUNT_SETTLE_TICK_MS, + computeFollowThresholds, +} from './chat-follow-thresholds.js'; import type { ChatLocalImagePreviewLoader, ChatLocalImagePreviewResult } from './ChatLocalImagePreview.js'; import { HtmlFullscreenPreview, openHtmlPreviewInNewWindow, type HtmlFullscreenPreviewState } from './HtmlFullscreenPreview.js'; import { isLikelyDomainPath, renderChatPathActions, type ChatPathDownloadHandler } from '../chat-path-actions.js'; @@ -1360,6 +1365,10 @@ export function ChatView({ events, loading, refreshing = false, historyStatus, l const ctxMenuRef = useRef(null); const [revealingOlder, setRevealingOlder] = useState(false); const revealingOlderTimerRef = useRef | null>(null); + // Scroll anchor preservation: pre-prepend scrollHeight, restored after the + // taller list commits. Declared with the reveal state it serves (both the + // local render-limit reveal and the HTTP load-older prepend set it). + const scrollAnchorRef = useRef<{ scrollHeight: number } | null>(null); // Timestamp when ctx menu was opened — clicks within 400ms are synthetic (from long-press release) const menuOpenedAtRef = useRef(0); // Zoomed text modal — opened by double-tap on a chat bubble on touch devices. @@ -1821,6 +1830,18 @@ export function ChatView({ events, loading, refreshing = false, historyStatus, l }; const revealHiddenOlderItems = () => { + // Raising the render limit mounts CHAT_RENDER_ITEM_INCREMENT more items + // ABOVE the viewport. Without an anchor the browser keeps `scrollTop`, so + // the same offset now points that much higher in the list and the reading + // position lurches toward older history — the "scrolled to the end and it + // jumps up a section" report. Worse, it lands the view back inside the + // `scrollTop < 100` trigger zone, so the next cooldown tick reveals again + // and jumps again. Capture the pre-reveal height here (not at the call + // sites) so every caller — the scroll auto-trigger and the manual + // "load older" button — is anchored identically; the restore effect below + // re-adds the delta. + const scrollEl = scrollRef.current; + if (scrollEl) scrollAnchorRef.current = { scrollHeight: scrollEl.scrollHeight }; if (revealingOlderTimerRef.current) clearTimeout(revealingOlderTimerRef.current); setRevealingOlder(true); setRenderItemLimit((limit) => limit + CHAT_RENDER_ITEM_INCREMENT); @@ -1995,7 +2016,21 @@ export function ChatView({ events, loading, refreshing = false, historyStatus, l // Stamp when the pin banner toggles so the ResizeObserver can tell its own // ~60px height shift apart from a genuine viewport resize (sub-session bar, // keyboard) and skip re-pinning to bottom for the former. See bannerToggleAtRef. + // + // Compare against the previous value instead of just running on the dep change: + // an effect keyed on `[pinnedAboveViewport]` ALSO runs on mount, which stamped a + // 300ms suppression window over the most layout-unstable moment there is — the + // first frames after a page refresh, when `--vvh` is applied from an effect, the + // composer rehydrates its saved draft, and the sub-session bar runs its 200ms + // max-height transition. Every one of those shrinks `.chat-view` while + // `scrollTop` stays put (the list sets `overflow-anchor: none`, so the engine + // never compensates), and the suppressed ResizeObserver consumed the change + // without re-pinning — leaving the view permanently a little above the bottom + // after every reload. Mount is not a toggle, so it must not stamp. + const prevPinnedAboveViewportRef = useRef(pinnedAboveViewport); useEffect(() => { + if (prevPinnedAboveViewportRef.current === pinnedAboveViewport) return; + prevPinnedAboveViewportRef.current = pinnedAboveViewport; bannerToggleAtRef.current = Date.now(); }, [pinnedAboveViewport]); @@ -2111,8 +2146,22 @@ export function ChatView({ events, loading, refreshing = false, historyStatus, l // Scroll auto-trigger for Load Older const lastLoadOlderAtRef = useRef(0); const LOAD_OLDER_COOLDOWN_MS = 1000; - // Scroll anchor preservation: save scrollHeight before prepend, restore after - const scrollAnchorRef = useRef<{ scrollHeight: number } | null>(null); + + // Mirror `loadingOlder` into a ref so the mount-settle interval can read the + // CURRENT value. Adding the prop to that effect's deps instead would restart + // the settle window every time a load-older starts or finishes, which is both + // wasteful and wrong (the window is meant to track the mount, not history + // fetches), and reading the captured prop would leave it permanently stale. + const loadingOlderRef = useRef(loadingOlder); + loadingOlderRef.current = loadingOlder; + + // Latest `events` identity, for the mount-settle interval to tell a LAYOUT + // settle apart from a DATA change. Anything driven by new events is owned by + // the content-update layout effect and its follow rules (which deliberately + // decline to scroll for some updates); the settle window must never + // second-guess that, only handle height moving under unchanged data. + const latestEventsRef = useRef(events); + latestEventsRef.current = events; // Pause "stick to bottom" follow mode. Shared by handleScroll's distance // threshold and the explicit wheel/touch up-gesture handlers below. @@ -2252,6 +2301,54 @@ export function ChatView({ events, loading, refreshing = false, historyStatus, l return () => ro.disconnect(); }, [preview]); + // Hold the bottom through the post-mount layout settle. + // + // The ResizeObserver above only reacts to `clientHeight`, so anything that + // changes CONTENT height after the initial pin is invisible to it — and the + // blocks that mount ABOVE every message are the worst case, because growth + // above the viewport with `scrollTop` frozen slides the whole log down and the + // user ends up looking at older content: the reported "refresh 后上弹". + // Culprits that need no images at all: the "Load older" row appearing when + // `hasOlderHistory` flips after the first history response, the + // `.chat-load-older-status` row, `AgentTodoList` (first child of the scroller), + // and the tool-chooser banner gated on an async `usePref` fetch. None of them + // change `renderedRevision`, so the content-update layout effect never re-pins + // either, and `overflow-anchor: none` means the engine will not help. + // + // So for a bounded window after mount/session-change, re-assert the bottom + // whenever total height changes. Deliberately narrow: it only acts while follow + // is still engaged (a gentle scroll-up disengages it and this goes inert + // immediately — the exact complaint behind the earlier Safari jitter fixes), it + // never engages follow itself, it stays out of the way of the load-older anchor, + // and it only fires on an actual height change so a quiet mount costs nothing. + useEffect(() => { + const el = scrollRef.current; + if (!el) return; + let lastScrollHeight = el.scrollHeight; + let seenEvents = latestEventsRef.current; + const deadline = Date.now() + CHAT_MOUNT_SETTLE_MS; + const timer = setInterval(() => { + if (Date.now() > deadline) { clearInterval(timer); return; } + const following = preview || autoScrollRef.current; + if (!following) { clearInterval(timer); return; } + if (loadingOlderRef.current || scrollAnchorRef.current) return; + const nextScrollHeight = el.scrollHeight; + // A DATA change is not ours to act on: the content-update layout effect + // owns those and deliberately declines to follow some of them (a + // non-rendered status update must not yank the viewport). Re-baseline and + // stand down, so this window only ever reacts to height moving on its own. + if (latestEventsRef.current !== seenEvents) { + seenEvents = latestEventsRef.current; + lastScrollHeight = nextScrollHeight; + return; + } + if (nextScrollHeight === lastScrollHeight) return; + lastScrollHeight = nextScrollHeight; + scrollToBottom(false); + }, CHAT_MOUNT_SETTLE_TICK_MS); + return () => clearInterval(timer); + }, [sessionId, preview]); + // Touch gesture mode is based on pointer coarseness only. Narrow desktop // windows still need native selection and the Copy/Quote popup. const isTouchDevice = useTouchChatGestures(); @@ -3120,7 +3217,8 @@ function AttachmentDownloadButton({ const handleError = (err: unknown) => { const msg = err instanceof Error ? err.message : String(err); - if (msg.includes('daemon_offline') || msg.includes('503')) setError(t('upload.daemon_offline')); + if (isInsufficientCapacityError(msg)) setError(t('upload.insufficient_capacity')); + else if (msg.includes('daemon_offline') || msg.includes('503')) setError(t('upload.daemon_offline')); else if (msg.includes('410') || msg.includes('expired')) setError(t('upload.download_expired')); else if (msg.includes('404')) setError(t('upload.download_expired')); else setError(t('upload.upload_failed')); @@ -3745,6 +3843,9 @@ const MemoryContextEvent = memo(function MemoryContextEvent({ event }: { event:
{item.summary}
+ {item.ref && ( + {item.ref} + )} {item.projectId} {score && {t('chat.memory_context_score', { score })}} {typeof item.hitCount === 'number' && item.hitCount > 0 ? ( diff --git a/web/src/components/SessionControls.tsx b/web/src/components/SessionControls.tsx index 37c78f3f6..baa33f74b 100644 --- a/web/src/components/SessionControls.tsx +++ b/web/src/components/SessionControls.tsx @@ -23,6 +23,7 @@ import { insertMachineMarkerAtCaret } from '../util/machine-insert.js'; import { buildMachineSendExtra } from '../util/machine-send.js'; import { matchInlineMachineTrigger, stripInlineMachineTrigger } from '../util/machine-trigger.js'; import { parseAliasMarkers } from '@shared/alias-types.js'; +import { isInsufficientCapacityError } from '../upload-error.js'; import { MobileDpad, DPAD_ARROW_SEQUENCES } from './MobileDpad.js'; import { P2pConfigPanel, buildP2pWorkflowLaunchEnvelopeFromConfig } from './P2pConfigPanel.js'; import { useExecutionRouting } from '../hooks/useExecutionRouting.js'; @@ -4019,7 +4020,9 @@ export function SessionControls({ ws, activeSession, connected: connectedProp, i console.error('[upload] failed:', err); const body = err instanceof Error ? err.message : String(err); let errorMessage: string; - if (body.includes('daemon_offline')) { + if (isInsufficientCapacityError(body)) { + errorMessage = t('upload.insufficient_capacity'); + } else if (body.includes('daemon_offline')) { errorMessage = t('upload.daemon_offline'); } else if (body.includes('file_too_large')) { errorMessage = t('upload.file_too_large', { max: MAX_UPLOAD_SIZE_MB }); diff --git a/web/src/components/SubSessionBar.tsx b/web/src/components/SubSessionBar.tsx index b99e32953..598a8e96b 100644 --- a/web/src/components/SubSessionBar.tsx +++ b/web/src/components/SubSessionBar.tsx @@ -24,6 +24,8 @@ import { EmbeddingStatusIcon } from './EmbeddingStatusIcon.js'; import { SharedStateIndicator } from './SharedStateIndicator.js'; import type { SharedStateSummary } from '../tab-sharing-ui.js'; import type { EmbeddingStatus } from '@shared/embedding-status.js'; +import type { DiskUsage } from '@shared/disk-usage.js'; +import type { MemoryShortRefHealth } from '@shared/memory-short-ref-health.js'; import { formatDaemonVersionMobile, formatDaemonVersionShort } from '../util/format-version.js'; import { isAuthoritativeUsageContextWindowSource, type UsageContextWindowSource } from '@shared/usage-context-window.js'; import { resolveQuickAgentDelegationModel } from '../quick-agent-delegation-model.js'; @@ -48,6 +50,8 @@ interface DaemonStats { load15: number; uptime: number; embedding?: EmbeddingStatus | null; + disks?: DiskUsage[] | null; + shortRefHealth?: MemoryShortRefHealth | null; } type DiscussionSummary = P2pProgressDiscussion & { @@ -174,6 +178,75 @@ function formatUptime(seconds: number): string { return d > 0 ? `${d}d ${h}h` : `${h}h`; } +/** Format a used/total byte pair in a shared GB or TB unit — never smaller + * units, so disk sizes always read as e.g. "440/460GB" or "1.2/2.0TB". */ +function formatDiskPair(usedBytes: number, totalBytes: number): string { + const useTb = totalBytes >= 1024 ** 4; + const div = useTb ? 1024 ** 4 : 1024 ** 3; + const unit = useTb ? 'TB' : 'GB'; + const fmt = (b: number) => { + const v = b / div; + return v >= 100 ? v.toFixed(0) : v.toFixed(1); + }; + return `${fmt(usedBytes)}/${fmt(totalBytes)}${unit}`; +} + +function shortMountLabel(mount: string): string { + if (mount === '/') return '/'; + const seg = mount.replace(/[\\/]+$/, '').split(/[/\\]/).filter(Boolean).pop(); + return seg || mount; +} + +function diskUsageColor(usedPercent: number): string | undefined { + return usedPercent >= 90 ? '#f87171' : usedPercent >= 75 ? '#fbbf24' : undefined; +} + +/** + * Desktop-only disk-capacity readout for the daemon stats strip. Shows up to two + * partitions inline; with more than two, only the fullest shows inline (plus a + * "+N" hint) and every partition is listed in the hover tooltip. Mobile clients + * receive no disk data, so this renders nothing there. + */ +/** + * Memory handles stopped persisting. The daemon can only report this over the + * socket — its counter is process-local and its log shares the disk that is + * usually what failed — so this marker is the one place an operator sees it. + */ +function renderShortRefAlert(health: MemoryShortRefHealth | null | undefined, t: (k: string, v?: Record) => string): JSX.Element | null { + if (!health) return null; + const when = new Date(health.lastFailureAt).toLocaleString(); + return ( + + ⚠️ + {t('memory.short_ref_failure_short')} + + ); +} + +function renderDiskStats(disks: DiskUsage[] | null | undefined): JSX.Element | null { + if (!disks || disks.length === 0) return null; + const tooltip = disks + .map((d) => `${shortMountLabel(d.mount)} ${formatDiskPair(d.usedBytes, d.totalBytes)} (${d.usedPercent}%)`) + .join('\n'); + const inline = disks.length <= 2 ? disks : disks.slice(0, 1); + return ( + + {inline.map((d, i) => ( + + {i > 0 ? ' ' : ''} + 💾 + {formatDiskPair(d.usedBytes, d.totalBytes)} + + ))} + {disks.length > 2 && +{disks.length - 1}} + + ); +} + function formatLocalDateTime(timestamp: number): string { const date = new Date(timestamp); const pad = (value: number) => String(value).padStart(2, '0'); @@ -801,6 +874,10 @@ export function SubSessionBar({ subSessions, openIds, maximizedIds, desktopLayou // icon falls through to its "unknown" rendering instead of // showing a misleading "ready". embedding: (msg as { embedding?: EmbeddingStatus | null }).embedding ?? null, + // Older daemons don't ship `disks`; null means "no data" so the + // desktop strip simply hides the readout (and mobile never shows it). + disks: msg.disks ?? null, + shortRefHealth: msg.shortRefHealth ?? null, }); } }); @@ -900,6 +977,18 @@ export function SubSessionBar({ subSessions, openIds, maximizedIds, desktopLayou Load {stats.load1} + {desktopLayoutCapable && stats.disks && stats.disks.length > 0 && ( + <> + · + {renderDiskStats(stats.disks)} + + )} + {stats.shortRefHealth && ( + <> + · + {renderShortRefAlert(stats.shortRefHealth, t)} + + )} · · @@ -935,6 +1024,10 @@ export function SubSessionBar({ subSessions, openIds, maximizedIds, desktopLayou {' '} ≡{Number(stats.load1).toFixed(1)} {' '} + {desktopLayoutCapable && stats.disks && stats.disks.length > 0 && ( + <>{renderDiskStats(stats.disks)}{' '} + )} + {stats.shortRefHealth && (<>{renderShortRefAlert(stats.shortRefHealth, t)}{' '})} {desktopLayoutCapable && ( <> diff --git a/web/src/components/chat-follow-thresholds.ts b/web/src/components/chat-follow-thresholds.ts index 5fbed2dc0..4c83dd9d9 100644 --- a/web/src/components/chat-follow-thresholds.ts +++ b/web/src/components/chat-follow-thresholds.ts @@ -22,6 +22,26 @@ * threshold would need more than half the achievable range, both thresholds * are rescaled to that range (0.5× / 0.2×), keeping disengage > reengage. */ +/** + * How long after mount / session-change the chat keeps re-asserting the bottom + * while follow is engaged, and how often it checks. + * + * A page refresh finishes painting long before the layout settles: blocks mount + * ABOVE the messages (the "Load older" row once `hasOlderHistory` resolves, the + * loading-older status row, the agent todo list, the tool-chooser banner behind an + * async preference fetch), and growth above the viewport with `scrollTop` frozen + * slides the whole log down — the user is left looking at older content. The + * ResizeObserver cannot see it (it only watches `clientHeight`) and the chat sets + * `overflow-anchor: none`, so nothing else corrects it. + * + * The window has to outlast the slowest of those settles — the `.subsession-bar` + * max-height transition alone is 200ms and the preference fetch is a network + * round-trip — while staying short enough that it is unambiguously "the refresh", + * not an ongoing behaviour that could fight the user later. + */ +export const CHAT_MOUNT_SETTLE_MS = 1_500; +export const CHAT_MOUNT_SETTLE_TICK_MS = 80; + export interface FollowThresholds { disengageThreshold: number; reengageThreshold: number; diff --git a/web/src/daemon-upgrade-blocked.ts b/web/src/daemon-upgrade-blocked.ts index d4485a0b2..72bef141e 100644 --- a/web/src/daemon-upgrade-blocked.ts +++ b/web/src/daemon-upgrade-blocked.ts @@ -1,3 +1,5 @@ +import { DAEMON_UPGRADE_BLOCK_REASON } from '@shared/daemon-upgrade.js'; + export const DAEMON_UPGRADE_BLOCKED_TOAST_THROTTLE_MS = 15 * 60_000; export type DaemonUpgradeBlockedToastState = { @@ -14,6 +16,7 @@ export type DaemonUpgradeBlockedToastKey = | 'toast.upgrade_blocked_session_busy' | 'toast.upgrade_blocked_cooldown_active' | 'toast.upgrade_blocked_toolchain_unavailable' + | 'toast.upgrade_blocked_install_failed' | 'toast.upgrade_blocked_unknown'; /** @@ -31,6 +34,7 @@ export function daemonUpgradeBlockedToastKey(reason: string): DaemonUpgradeBlock case 'session_busy': return 'toast.upgrade_blocked_session_busy'; case 'cooldown_active': return 'toast.upgrade_blocked_cooldown_active'; case 'toolchain_unavailable': return 'toast.upgrade_blocked_toolchain_unavailable'; + case DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED: return 'toast.upgrade_blocked_install_failed'; default: return 'toast.upgrade_blocked_unknown'; } } diff --git a/web/src/hooks/useTimeline.ts b/web/src/hooks/useTimeline.ts index 4180c32f6..4fa07370d 100644 --- a/web/src/hooks/useTimeline.ts +++ b/web/src/hooks/useTimeline.ts @@ -3202,7 +3202,19 @@ export function useTimeline( // 15s does NOT refire on every focus/visibility/appStateChange tick. // App-resume's resetCooldowns:true path explicitly clears the map // so a real foreground from background still bypasses this. - fireHttpBackfillRef.current(0, { phase: 'refresh', cooldownMs: ACTIVE_REFRESH_COOLDOWN_MS }); + // `manualLatestWindow` (no lower bound), NOT the cheap tail catch-up. + // Activation follows a window in which live events were provably dropped: + // the server relays timeline events ONLY to currently-subscribed viewers + // (bridge.ts sendJsonToSessionSubscribers — no subscriber means silent + // discard, no queue, no replay), and a backgrounded app has no + // subscription at all. A push notification arriving is itself proof the + // app was backgrounded. A tail-anchored backfill asks for + // `(localNewest - 1ms, …]`, so the moment ANY newer event lands (and + // `agent.status`/`usage.update` are cursor-eligible and fire every second + // during a turn) the dropped events sit below the cursor and become + // permanently unreachable — the user is left with only the ↻ button. This + // is the same reasoning the seed/IDB paths above already use. + fireHttpBackfillRef.current(0, { phase: 'refresh', cooldownMs: ACTIVE_REFRESH_COOLDOWN_MS, mode: 'manualLatestWindow' }); }; window.addEventListener(ACTIVE_TIMELINE_REFRESH_EVENT, handler); return () => window.removeEventListener(ACTIVE_TIMELINE_REFRESH_EVENT, handler); @@ -3670,7 +3682,14 @@ export function useTimeline( }); setRefreshing(true); sendForwardHistoryRequest('refresh', buildForwardHistoryArgs(MAX_MEMORY_EVENTS)); - fireHttpBackfillRef.current(600, { phase: 'refresh', visible: true }); + // `manualLatestWindow`: a daemon reconnect means the daemon→server link + // was down, and live timeline events are control-plane — `trySend` + // drops them silently with no queue and no replay, and the reconnect + // resync re-broadcasts only `session.state`, never assistant.text / + // tool.* / user.message. So events were almost certainly lost below the + // local tail, where a tail-anchored `(localNewest - 1ms, …]` request can + // never reach them. + fireHttpBackfillRef.current(600, { phase: 'refresh', visible: true, mode: 'manualLatestWindow' }); } } // ── Browser WS disconnected: reset in-flight pagination to prevent stuck state ── diff --git a/web/src/i18n/locales/en.json b/web/src/i18n/locales/en.json index ee5966445..f549c0352 100644 --- a/web/src/i18n/locales/en.json +++ b/web/src/i18n/locales/en.json @@ -860,6 +860,7 @@ "upgrade_blocked_session_busy": "A session still has active work. Wait until it goes idle before upgrading the daemon.", "upgrade_blocked_cooldown_active": "The daemon was upgraded recently. Automatic upgrade will retry after the cooldown.", "upgrade_blocked_toolchain_unavailable": "Node is missing on the daemon host (toolchain removed). Reinstall Node — auto-upgrade is disabled until then.", + "upgrade_blocked_install_failed": "Automatic upgrade failed during npm install. Check the daemon host's upgrade log; it will remain on the current version.", "upgrade_blocked_unknown": "The daemon is temporarily busy. Automatic upgrade will retry." }, "discussion": { @@ -1552,6 +1553,7 @@ "upload_file": "Upload file", "uploading": "Uploading...", "upload_failed": "Upload failed", + "insufficient_capacity": "Insufficient disk space on the server", "file_too_large": "File too large (max {{max}}MB)", "long_text_attached": "Large pasted text attached as {{name}}", "long_text_requires_attachment": "Paste is too large for inline input here. Upload it as a file instead.", @@ -2599,7 +2601,9 @@ "layerDiagnostics": "Skill layer diagnostics", "enforced": "Enforced", "additive": "Additive" - } + }, + "short_ref_failure_short": "memory handles not saved", + "short_ref_failure_detail": "Memory handles stopped being saved ({{stage}}, {{failures}} failure(s), last {{when}}): {{error}}. Handles issued now will not resolve after a daemon restart." }, "todos": { "title": "Tasks", diff --git a/web/src/i18n/locales/es.json b/web/src/i18n/locales/es.json index b16db9b3e..2aa159b4d 100644 --- a/web/src/i18n/locales/es.json +++ b/web/src/i18n/locales/es.json @@ -860,6 +860,7 @@ "upgrade_blocked_session_busy": "Una sesión todavía tiene trabajo activo. Espera a que quede inactiva antes de actualizar el daemon.", "upgrade_blocked_cooldown_active": "El daemon se actualizó recientemente. La actualización automática volverá a intentarlo después del tiempo de espera.", "upgrade_blocked_toolchain_unavailable": "Falta Node en el host del daemon (toolchain eliminado). Reinstala Node; la actualización automática está deshabilitada hasta entonces.", + "upgrade_blocked_install_failed": "La actualización automática falló durante la instalación de npm. Revisa el registro de actualización del host; el daemon seguirá usando la versión actual.", "upgrade_blocked_unknown": "El daemon está ocupado temporalmente. La actualización automática volverá a intentarlo." }, "discussion": { @@ -1552,6 +1553,7 @@ "upload_file": "Subir archivo", "uploading": "Subiendo...", "upload_failed": "Error al subir", + "insufficient_capacity": "Espacio en disco insuficiente en el servidor", "file_too_large": "Archivo demasiado grande (máx. {{max}}MB)", "long_text_attached": "El texto pegado largo se adjuntó como {{name}}", "long_text_requires_attachment": "Este pegado es demasiado grande para el cuadro de texto. Súbelo como archivo.", @@ -2599,7 +2601,9 @@ "layerDiagnostics": "Diagnóstico de capas de habilidades", "enforced": "Obligatoria", "additive": "Aditiva" - } + }, + "short_ref_failure_short": "identificadores de memoria sin guardar", + "short_ref_failure_detail": "Los identificadores de memoria dejaron de guardarse ({{stage}}, {{failures}} fallo(s), último {{when}}): {{error}}. Los emitidos ahora no se resolverán tras reiniciar el daemon." }, "todos": { "title": "Tareas", diff --git a/web/src/i18n/locales/ja.json b/web/src/i18n/locales/ja.json index d4c9dbe2b..a31384614 100644 --- a/web/src/i18n/locales/ja.json +++ b/web/src/i18n/locales/ja.json @@ -860,6 +860,7 @@ "upgrade_blocked_session_busy": "セッションに実行中の処理があります。idle になってから daemon をアップグレードしてください。", "upgrade_blocked_cooldown_active": "daemon は最近アップグレードされました。クールダウン後に自動アップグレードを再試行します。", "upgrade_blocked_toolchain_unavailable": "daemon ホストの Node が見つかりません（ツールチェーンが削除されました）。Node を再インストールしてください。それまで自動アップグレードは無効です。", + "upgrade_blocked_install_failed": "npm のインストール中に自動アップグレードが失敗しました。daemon ホストのアップグレードログを確認してください。現在のバージョンで実行を続けます。", "upgrade_blocked_unknown": "daemon は一時的にビジーです。自動アップグレードを再試行します。" }, "discussion": { @@ -1551,6 +1552,7 @@ "upload_file": "ファイルをアップロード", "uploading": "アップロード中...", "upload_failed": "アップロード失敗", + "insufficient_capacity": "サーバーのディスク容量が不足しています", "file_too_large": "ファイルが大きすぎます（最大 {{max}}MB）", "long_text_attached": "長い貼り付けテキストを {{name}} として添付しました", "long_text_requires_attachment": "この貼り付け内容は入力欄に直接入れるには長すぎます。ファイルとしてアップロードしてください。", @@ -2598,7 +2600,9 @@ "layerDiagnostics": "スキルレイヤー診断", "enforced": "強制", "additive": "追加" - } + }, + "short_ref_failure_short": "メモリハンドル未保存", + "short_ref_failure_detail": "メモリハンドルの保存が停止しました({{stage}}、{{failures}} 件、最終 {{when}}):{{error}}。現在発行されたハンドルは daemon 再起動後に解決できません。" }, "todos": { "title": "タスク", diff --git a/web/src/i18n/locales/ko.json b/web/src/i18n/locales/ko.json index 39078da32..82a97f9fe 100644 --- a/web/src/i18n/locales/ko.json +++ b/web/src/i18n/locales/ko.json @@ -860,6 +860,7 @@ "upgrade_blocked_session_busy": "세션에 아직 활성 작업이 있습니다. idle 상태가 된 뒤 daemon을 업그레이드하세요.", "upgrade_blocked_cooldown_active": "daemon이 최근 업그레이드되었습니다. 대기 시간이 끝나면 자동 업그레이드를 다시 시도합니다.", "upgrade_blocked_toolchain_unavailable": "daemon 호스트에 Node가 없습니다(툴체인 제거됨). Node를 다시 설치하세요. 그 전까지 자동 업그레이드가 비활성화됩니다.", + "upgrade_blocked_install_failed": "npm 설치 중 자동 업그레이드가 실패했습니다. daemon 호스트의 업그레이드 로그를 확인하세요. 현재 버전으로 계속 실행됩니다.", "upgrade_blocked_unknown": "daemon이 일시적으로 사용 중입니다. 자동 업그레이드를 다시 시도합니다." }, "discussion": { @@ -1551,6 +1552,7 @@ "upload_file": "파일 업로드", "uploading": "업로드 중...", "upload_failed": "업로드 실패", + "insufficient_capacity": "서버의 디스크 공간이 부족합니다", "file_too_large": "파일이 너무 큽니다 (최대 {{max}}MB)", "long_text_attached": "긴 붙여넣기 텍스트가 {{name}} 첨부파일로 추가되었습니다", "long_text_requires_attachment": "이 붙여넣기 내용은 입력창에 직접 넣기엔 너무 깁니다. 파일로 업로드하세요.", @@ -2598,7 +2600,9 @@ "layerDiagnostics": "스킬 계층 진단", "enforced": "강제", "additive": "추가" - } + }, + "short_ref_failure_short": "메모리 핸들 저장 실패", + "short_ref_failure_detail": "메모리 핸들 저장이 중단되었습니다({{stage}}, {{failures}}회, 최근 {{when}}): {{error}}. 지금 발급된 핸들은 daemon 재시작 후 해석되지 않습니다." }, "todos": { "title": "작업", diff --git a/web/src/i18n/locales/ru.json b/web/src/i18n/locales/ru.json index 94f2da5be..911c6f226 100644 --- a/web/src/i18n/locales/ru.json +++ b/web/src/i18n/locales/ru.json @@ -860,6 +860,7 @@ "upgrade_blocked_session_busy": "В сессии ещё выполняется работа. Дождитесь состояния idle перед обновлением daemon.", "upgrade_blocked_cooldown_active": "daemon недавно обновлялся. Автообновление повторится после периода ожидания.", "upgrade_blocked_toolchain_unavailable": "На хосте daemon отсутствует Node (toolchain удалён). Переустановите Node — автообновление отключено до этого.", + "upgrade_blocked_install_failed": "Автообновление завершилось с ошибкой во время установки npm. Проверьте журнал обновления на хосте daemon; работа продолжится на текущей версии.", "upgrade_blocked_unknown": "daemon временно занят. Автообновление повторит попытку." }, "discussion": { @@ -1551,6 +1552,7 @@ "upload_file": "Загрузить файл", "uploading": "Загрузка...", "upload_failed": "Ошибка загрузки", + "insufficient_capacity": "Недостаточно места на диске сервера", "file_too_large": "Файл слишком большой (макс. {{max}}МБ)", "long_text_attached": "Большой вставленный текст прикреплен как {{name}}", "long_text_requires_attachment": "Эта вставка слишком большая для поля ввода. Загрузите ее как файл.", @@ -2598,7 +2600,9 @@ "layerDiagnostics": "Диагностика слоев навыков", "enforced": "Обязательный", "additive": "Добавочный" - } + }, + "short_ref_failure_short": "дескрипторы памяти не сохраняются", + "short_ref_failure_detail": "Дескрипторы памяти перестали сохраняться ({{stage}}, ошибок: {{failures}}, последняя {{when}}): {{error}}. Выданные сейчас не разрешатся после перезапуска daemon." }, "todos": { "title": "Задачи", diff --git a/web/src/i18n/locales/zh-CN.json b/web/src/i18n/locales/zh-CN.json index 20598b6e9..a35a7f332 100644 --- a/web/src/i18n/locales/zh-CN.json +++ b/web/src/i18n/locales/zh-CN.json @@ -860,6 +860,7 @@ "upgrade_blocked_session_busy": "仍有会话正在执行任务。等它回到 idle 再升级 daemon。", "upgrade_blocked_cooldown_active": "daemon 最近刚完成升级，冷却结束后会自动重试。", "upgrade_blocked_toolchain_unavailable": "daemon 主机上的 Node 已丢失（工具链被删除）。请重新安装 Node——在此之前自动升级已禁用。", + "upgrade_blocked_install_failed": "npm 安装期间自动升级失败。请检查 daemon 主机上的升级日志；daemon 将继续运行当前版本。", "upgrade_blocked_unknown": "daemon 暂时繁忙，稍后会自动重试升级。" }, "discussion": { @@ -1552,6 +1553,7 @@ "upload_file": "上传文件", "uploading": "上传中...", "upload_failed": "上传失败", + "insufficient_capacity": "服务器磁盘空间不足", "file_too_large": "文件过大（最大 {{max}}MB）", "long_text_attached": "超长粘贴内容已作为 {{name}} 附件添加", "long_text_requires_attachment": "这段粘贴内容太长，不能直接放进输入框，请改为文件上传。", @@ -2599,7 +2601,9 @@ "layerDiagnostics": "技能层级诊断", "enforced": "强制", "additive": "附加" - } + }, + "short_ref_failure_short": "记忆句柄未保存", + "short_ref_failure_detail": "记忆句柄已停止保存({{stage}},{{failures}} 次失败,最近 {{when}}):{{error}}。现在签发的句柄在 daemon 重启后将无法解析。" }, "todos": { "title": "任务清单", diff --git a/web/src/i18n/locales/zh-TW.json b/web/src/i18n/locales/zh-TW.json index 17b3f7ccb..e048730e6 100644 --- a/web/src/i18n/locales/zh-TW.json +++ b/web/src/i18n/locales/zh-TW.json @@ -860,6 +860,7 @@ "upgrade_blocked_session_busy": "仍有工作階段正在執行任務。等它回到 idle 再升級 daemon。", "upgrade_blocked_cooldown_active": "daemon 最近剛完成升級，冷卻結束後會自動重試。", "upgrade_blocked_toolchain_unavailable": "daemon 主機上的 Node 已遺失（工具鏈被移除）。請重新安裝 Node——在此之前自動升級已停用。", + "upgrade_blocked_install_failed": "npm 安裝期間自動升級失敗。請檢查 daemon 主機上的升級日誌；daemon 將繼續執行目前版本。", "upgrade_blocked_unknown": "daemon 暫時忙碌，稍後會自動重試升級。" }, "discussion": { @@ -1552,6 +1553,7 @@ "upload_file": "上傳檔案", "uploading": "上傳中...", "upload_failed": "上傳失敗", + "insufficient_capacity": "伺服器磁碟空間不足", "file_too_large": "檔案過大（最大 {{max}}MB）", "long_text_attached": "過長貼上內容已作為 {{name}} 附件加入", "long_text_requires_attachment": "這段貼上內容太長，不能直接放進輸入框，請改成檔案上傳。", @@ -2599,7 +2601,9 @@ "layerDiagnostics": "技能層級診斷", "enforced": "強制", "additive": "附加" - } + }, + "short_ref_failure_short": "記憶句柄未儲存", + "short_ref_failure_detail": "記憶句柄已停止儲存({{stage}},{{failures}} 次失敗,最近 {{when}}):{{error}}。現在簽發的句柄在 daemon 重啟後將無法解析。" }, "todos": { "title": "任務清單", diff --git a/web/src/memory-summary-sync.ts b/web/src/memory-summary-sync.ts index cc21838de..6c9c58e0a 100644 --- a/web/src/memory-summary-sync.ts +++ b/web/src/memory-summary-sync.ts @@ -47,15 +47,25 @@ function newestRecords(records: ContextMemoryRecordView[], projectId: string, li function mergeMemoryRecordsByPriority(recordGroups: ContextMemoryRecordView[][]): ContextMemoryRecordView[] { const merged: ContextMemoryRecordView[] = []; - const seen = new Set(); + const indexByKey = new Map(); for (const records of recordGroups) { for (const record of records) { const summary = record.summary.trim(); const key = record.id ? `id:${record.id}` : `summary:${record.projectId}:${record.projectionClass}:${summary}`; - if (seen.has(key)) continue; - seen.add(key); + const existingIndex = indexByKey.get(key); + if (existingIndex !== undefined) { + // Cloud stays authoritative for summary content/order, while a matching + // local daemon result can enrich it with the persisted short ref that + // only the daemon is able to issue. + const existing = merged[existingIndex]; + if (existing && !existing.ref && record.ref) { + merged[existingIndex] = { ...existing, ref: record.ref }; + } + continue; + } + indexByKey.set(key, merged.length); merged.push(record); } } @@ -109,6 +119,7 @@ export function localPersonalMemorySummarySource( projectId: input.projectId, projectionClass: input.projectionClass, limit: input.limit, + includeShortRefs: true, }); } catch (error) { finish(() => reject(error)); @@ -116,11 +127,6 @@ export function localPersonalMemorySummarySource( }); } -function projectionRef(projectionId: string): string { - const compact = projectionId.replace(/[^a-f0-9]/gi, '').slice(0, 10) || projectionId.slice(0, 10); - return `proj:${compact.toLowerCase()}`; -} - function sourceLookupLine(projectionId: string): string { return `sourceLookup: ${JSON.stringify({ tool: 'get_memory_sources', @@ -185,7 +191,8 @@ export async function buildMemorySummarySyncMessage( remainingSummaryChars -= trimmed.text.length; truncated = truncated || trimmed.truncated; const project = record.projectId ? `[${record.projectId}] ` : ''; - lines.push(`${lines.length + 1}. [ref: ${projectionRef(record.id)}] ${project}${trimmed.text}\n ${sourceLookupLine(record.id)}`); + const ref = record.ref ? `[ref: ${record.ref}] ` : ''; + lines.push(`${lines.length + 1}. ${ref}${project}${trimmed.text}\n ${sourceLookupLine(record.id)}`); } if (lines.length === 0) return null; diff --git a/web/src/styles.css b/web/src/styles.css index 519c9e195..d12f5003a 100644 --- a/web/src/styles.css +++ b/web/src/styles.css @@ -3121,6 +3121,7 @@ html.native-ios-mac .chat-dl-btn { .chat-memory-context-item-summary { font-size: 12px; line-height: 1.5; color: #e2e8f0; white-space: pre-wrap; word-break: break-word; } .chat-memory-context-item-meta { display: flex; flex-wrap: wrap; gap: 6px; } .chat-memory-context-chip { display: inline-flex; align-items: center; padding: 2px 7px; border-radius: 999px; background: rgba(30,41,59,0.9); border: 1px solid #334155; color: #94a3b8; font-size: 10px; } +.chat-memory-context-ref { color: #c4b5fd; font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; user-select: all; } .chat-memory-context-chip-muted { color: #64748b; } .chat-memory-context-collapse-bottom { align-self: flex-start; background: none; border: 1px solid #334155; color: #94a3b8; border-radius: 6px; padding: 6px 10px; font-size: 11px; cursor: pointer; transition: color 0.15s, border-color 0.15s, background 0.15s; } .chat-memory-context-collapse-bottom:hover { color: #e2e8f0; border-color: #6366f1; background: rgba(99,102,241,0.08); } diff --git a/web/src/thinking-utils.ts b/web/src/thinking-utils.ts index f0b7998e0..06a92fbbd 100644 --- a/web/src/thinking-utils.ts +++ b/web/src/thinking-utils.ts @@ -7,9 +7,14 @@ * Only assistant.text, user.message, and session.state=idle end thinking. * Tool calls, status updates, and other events are skipped (don't end thinking). */ -import { isAuthoritativeCleanIdlePayload, isWorkingSessionState, reduceTimelineActivity } from '../../shared/session-activity-types.js'; +import { + TIMELINE_METADATA_EVENT_TYPES, + isAuthoritativeCleanIdlePayload, + isWorkingSessionState, + reduceTimelineActivity, +} from '../../shared/session-activity-types.js'; -const THINKING_SKIP_TYPES = new Set([ +const THINKING_SKIP_TYPES = new Set([ 'agent.status', 'usage.update', 'tool.call', @@ -17,6 +22,9 @@ const THINKING_SKIP_TYPES = new Set([ 'mode.state', 'terminal.snapshot', 'command.ack', + // Metadata/telemetry never ends a thinking sequence — see + // TIMELINE_METADATA_EVENT_TYPES. + ...TIMELINE_METADATA_EVENT_TYPES, ]); export function getActiveThinkingTs(events: Array<{ type: string; ts: number; payload?: Record }>): number | null { diff --git a/web/src/timeline-running.ts b/web/src/timeline-running.ts index 3432201a7..eabee5529 100644 --- a/web/src/timeline-running.ts +++ b/web/src/timeline-running.ts @@ -1,19 +1,29 @@ import type { TimelineEvent } from '../../src/shared/timeline/types.js'; import { SDK_SUBAGENT_DETAIL_KIND } from '../../shared/sdk-subagent-status.js'; -import { isAuthoritativeCleanIdlePayload, isWorkingSessionState, reduceTimelineActivity } from '../../shared/session-activity-types.js'; +import { + TIMELINE_METADATA_EVENT_TYPES, + isAuthoritativeCleanIdlePayload, + isWorkingSessionState, + reduceTimelineActivity, +} from '../../shared/session-activity-types.js'; const RUNNING_TIMELINE_EVENT_TYPES = new Set([ 'assistant.thinking', 'tool.call', ]); -const NEUTRAL_TAIL_EVENT_TYPES = new Set([ +const NEUTRAL_TAIL_EVENT_TYPES = new Set([ 'agent.status', 'usage.update', 'tool.result', 'mode.state', 'terminal.snapshot', 'command.ack', + // Pure metadata/telemetry (memory.context, file.change, queue snapshots, …). + // These must never terminate the tail scan: `memory.context` is the tail for + // the whole pre-first-tool thinking window of every context-injecting send, + // and stopping there reported "no active turn" while the agent was working. + ...TIMELINE_METADATA_EVENT_TYPES, ]); type TimelineTailEvent = Pick & { diff --git a/web/src/upload-error.ts b/web/src/upload-error.ts new file mode 100644 index 000000000..d872473a2 --- /dev/null +++ b/web/src/upload-error.ts @@ -0,0 +1,15 @@ +import { FILE_TRANSFER_UPLOAD_ERROR_CODE } from '@shared/transport/file-transfer.js'; + +/** + * True when a file-transfer error indicates the daemon ran out of disk space + * (ENOSPC). The daemon tags such errors with a shared code, the server responds + * HTTP 507, and the daemon's raw ENOSPC text rides along in the message — so an + * ApiError surfaces as e.g. "API 507: insufficient_capacity". Match any of them. + * Shared by the upload composer and the attachment download/preview button. + */ +export function isInsufficientCapacityError(body: string): boolean { + return body.includes(FILE_TRANSFER_UPLOAD_ERROR_CODE.INSUFFICIENT_CAPACITY) + || body.includes('507') + || body.includes('ENOSPC') + || body.includes('no space left'); +} diff --git a/web/src/util/daemon-upgrade-status.ts b/web/src/util/daemon-upgrade-status.ts index 25c1a6a4b..48decd5eb 100644 --- a/web/src/util/daemon-upgrade-status.ts +++ b/web/src/util/daemon-upgrade-status.ts @@ -11,6 +11,8 @@ export type DaemonUpgradingState = { targetVersion: string } | null; * version (empty string when the daemon didn't supply a usable one). * - `MSG_DAEMON_ONLINE` / `DAEMON_MSG.RECONNECTED` → the (possibly upgraded) * daemon is back, so clear the badge (`null`). + * - `DAEMON_MSG.UPGRADE_BLOCKED` → the detached upgrade stopped and the + * current daemon remains active, so clear the badge immediately. * - anything else → `undefined`, meaning "not relevant, keep current state". * * Returning a discriminated `undefined` (vs `null`) lets the caller distinguish @@ -23,7 +25,11 @@ export function nextDaemonUpgradingState( if (msgType === DAEMON_MSG.UPGRADING) { return { targetVersion: typeof rawTargetVersion === 'string' ? rawTargetVersion : '' }; } - if (msgType === MSG_DAEMON_ONLINE || msgType === DAEMON_MSG.RECONNECTED) { + if ( + msgType === MSG_DAEMON_ONLINE + || msgType === DAEMON_MSG.RECONNECTED + || msgType === DAEMON_MSG.UPGRADE_BLOCKED + ) { return null; } return undefined; diff --git a/web/src/ws-client.ts b/web/src/ws-client.ts index 5180658e9..e2278cc0e 100644 --- a/web/src/ws-client.ts +++ b/web/src/ws-client.ts @@ -8,10 +8,13 @@ import { apiFetch, ApiError } from './api.js'; import type { TimelineEvent } from '../../src/shared/timeline/types.js'; import { REPO_MSG } from '@shared/repo-types.js'; import { DAEMON_MSG } from '@shared/daemon-events.js'; +import { DAEMON_UPGRADE_BLOCK_REASON } from '@shared/daemon-upgrade.js'; import type { ResourceChangedMessage } from '@shared/resource-events.js'; import { P2P_CONFIG_MSG } from '@shared/p2p-config-events.js'; import { P2P_WORKFLOW_MSG, isP2pWorkflowRequestId } from '@shared/p2p-workflow-messages.js'; import { TRANSPORT_EVENT } from '@shared/transport-events.js'; +import type { DiskUsage } from '@shared/disk-usage.js'; +import type { MemoryShortRefHealth } from '@shared/memory-short-ref-health.js'; import { P2P_CAPABILITY_FRESHNESS_TTL_MS } from '@shared/p2p-workflow-constants.js'; import { TRANSPORT_MSG } from '@shared/transport-events.js'; import { DAEMON_COMMAND_TYPES } from '@shared/daemon-command-types.js'; @@ -151,7 +154,8 @@ export type ServerMessage = | 'transport_busy' | 'session_busy' | 'cooldown_active' - | 'toolchain_unavailable'; + | 'toolchain_unavailable' + | typeof DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED; activeRunIds?: string[]; activeSessionNames?: string[]; blockedSessions?: TransportUpgradeBlockedSession[]; @@ -161,6 +165,15 @@ export type ServerMessage = lastUpgradeAt?: number | null; nodeBinPresent?: boolean; npmAvailable?: boolean; + failureId?: string; + fromVersion?: string; + targetVersion?: string; + retryReason?: string; + attempts?: number; + exitCode?: number; + log?: string; + signal?: string; + ts?: number; } | { type: 'daemon.error'; kind: 'uncaughtException' | 'unhandledRejection' | 'warning'; message: string; stack?: string; ts: number } | { type: 'session_list'; daemonVersion?: string | null; sessions: Array<{ name: string; sessionInstanceId?: string; runtimeEpoch?: string; project: string; role: string; agentType: string; providerId?: string; agentVersion?: string; state: string; error?: string | null; projectDir?: string; runtimeType?: 'process' | 'transport'; label?: string; description?: string; userCreated?: boolean; ccPreset?: string | null; qwenModel?: string; requestedModel?: string; activeModel?: string; qwenAuthType?: string; qwenAuthLimit?: string; qwenAvailableModels?: string[]; copilotAvailableModels?: string[]; cursorAvailableModels?: string[]; codexAvailableModels?: string[]; modelDisplay?: string; planLabel?: string; permissionLabel?: string; quotaLabel?: string; quotaUsageLabel?: string; quotaMeta?: import('../../shared/provider-quota.js').ProviderQuotaMeta | null; effort?: import('../../shared/effort-levels.js').TransportEffortLevel; contextNamespace?: import('../../shared/session-context-bootstrap.js').SessionContextBootstrapState['contextNamespace']; contextNamespaceDiagnostics?: string[]; contextRemoteProcessedFreshness?: import('../../shared/context-types.js').ContextFreshness; contextLocalProcessedFreshness?: import('../../shared/context-types.js').ContextFreshness; contextRetryExhausted?: boolean; contextSharedPolicyOverride?: import('../../shared/context-types.js').SharedScopePolicyOverride; transportConfig?: Record | null; transportPendingMessages?: string[]; transportPendingMessageEntries?: TransportPendingMessageEntry[]; queueEpoch?: string; queueAuthorityId?: string; failedMessageEntries?: TransportPendingMessageEntry[]; pendingMessageVersion?: number; transportPendingMessageVersion?: number }> } @@ -186,7 +199,7 @@ export type ServerMessage = | { type: 'discussion.done'; discussionId: string; filePath: string; conclusion: string } | { type: 'discussion.error'; discussionId?: string; requestId?: string; error: string } | { type: 'discussion.list'; discussions: Array<{ id: string; requestId?: string; topic: string; state: string; currentRound: number; maxRounds: number; completedHops?: number; totalHops?: number; currentSpeaker?: string; conclusion?: string; filePath?: string }> } - | { type: 'daemon.stats'; daemonVersion?: string | null; cpu: number; memUsed: number; memTotal: number; load1: number; load5: number; load15: number; uptime: number } + | { type: 'daemon.stats'; daemonVersion?: string | null; cpu: number; memUsed: number; memTotal: number; load1: number; load5: number; load15: number; uptime: number; disks?: DiskUsage[]; shortRefHealth?: MemoryShortRefHealth } | FsLsResponse | FsReadResponse | FsGitStatusResponse diff --git a/web/test/app-resume-refresh.test.tsx b/web/test/app-resume-refresh.test.tsx index b38c27be2..2001d1a2f 100644 --- a/web/test/app-resume-refresh.test.tsx +++ b/web/test/app-resume-refresh.test.tsx @@ -99,10 +99,17 @@ describe('native app resume refresh chain', () => { expect(reconnectNow).toHaveBeenCalledWith(true); expect(fetchSpy).toHaveBeenCalledTimes(1); + // Resume MUST drop the lower bound. While the app was backgrounded it had no + // session subscription, so the server discarded its timeline events outright + // (no queue, no replay). Those events sit BELOW the local tail, so the old + // `afterTs: ` request could never reach them — that is exactly the + // "notification arrived but the chat is empty until I press ↻" bug. This test + // asserts the resume chain fires; the cursor value it used to pin was just + // whatever the tail-anchored implementation produced. expect(fetchSpy).toHaveBeenCalledWith( serverId, sessionName, - expect.objectContaining({ afterTs: 999 }), + expect.objectContaining({ afterTs: undefined }), ); removeListener(); @@ -177,10 +184,11 @@ describe('native app resume refresh chain', () => { expect(reconnectNow).toHaveBeenCalledWith(true); expect(fetchSpy).toHaveBeenCalledTimes(1); + // Same contract as above: a resume backfill carries no lower bound. expect(fetchSpy).toHaveBeenCalledWith( serverId, sessionName, - expect.objectContaining({ afterTs: 999 }), + expect.objectContaining({ afterTs: undefined }), ); }); }); diff --git a/web/test/components/ChatView-refresh-repin.test.tsx b/web/test/components/ChatView-refresh-repin.test.tsx new file mode 100644 index 000000000..d5b41c29a --- /dev/null +++ b/web/test/components/ChatView-refresh-repin.test.tsx @@ -0,0 +1,248 @@ +/** + * @vitest-environment jsdom + * + * "After refreshing in Safari the chat still sits a little above the bottom, + * even when I was scrolled all the way down." + * + * The chat list sets `overflow-anchor: none`, so the browser never compensates + * when something shrinks the viewport — only the ResizeObserver re-pin does. But + * the banner-toggle suppression it consults was stamped by an effect keyed on + * `[pinnedAboveViewport]`, and such an effect ALSO runs on mount, so the first + * 300ms after a refresh had re-pinning disabled. That is exactly the window in + * which `--vvh` is applied from an effect, the composer rehydrates its saved + * draft, and the sub-session bar runs its 200ms max-height transition. Worse, the + * observer records the new height BEFORE checking the suppression, so the change + * was consumed for good: no later resize could recover it. + * + * These two tests pin both halves of the contract — a genuine post-mount resize + * must re-pin, and a real banner toggle must still NOT. + */ +import { h } from 'preact'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { render, cleanup, act, fireEvent, waitFor } from '@testing-library/preact'; + +vi.mock('react-i18next', () => ({ + useTranslation: () => ({ t: (key: string, fallback?: string) => fallback ?? key }), +})); +vi.mock('../../src/components/ChatMarkdown.js', () => ({ + ChatMarkdown: ({ text }: { text: string }) =>
{text}
, +})); +vi.mock('../../src/components/FileBrowser.js', () => ({ FileBrowser: () => null })); +vi.mock('../../src/components/FloatingPanel.js', () => ({ + FloatingPanel: ({ children }: { children?: preact.ComponentChildren }) =>
{children}
, +})); +vi.mock('../../src/hooks/usePref.js', () => ({ + parseBooleanish: (raw: unknown) => (raw === true || raw === 'true' ? true : raw === false || raw === 'false' ? false : null), + usePref: () => ({ + value: true, rawValue: true, loaded: true, loading: false, stale: false, + error: null, save: async () => undefined, set: () => undefined, reload: async () => true, + }), +})); + +import { ChatView } from '../../src/components/ChatView.js'; +import type { TimelineEvent } from '../../src/ws-client.js'; + +/** Controllable ResizeObserver — jsdom has none, and we need to fire it on demand. */ +const resizeCallbacks: ResizeObserverCallback[] = []; +class FakeResizeObserver { + constructor(cb: ResizeObserverCallback) { resizeCallbacks.push(cb); } + observe(): void {} + unobserve(): void {} + disconnect(): void {} +} +function fireResize(): void { + for (const cb of resizeCallbacks) cb([], {} as ResizeObserver); +} + +/** Minimal IntersectionObserver fake so we can drive the pin banner on/off. */ +type IOCallback = (entries: IntersectionObserverEntry[]) => void; +const ioInstances: Array<{ fire: (e: Array>) => void }> = []; +class FakeIntersectionObserver { + private callback: IOCallback; + private target: Element | null = null; + constructor(callback: IOCallback) { + this.callback = callback; + const self = this; + ioInstances.push({ + fire: (partial) => { + self.callback(partial.map((e) => ({ + target: self.target, + isIntersecting: false, + intersectionRatio: 0, + intersectionRect: {} as DOMRectReadOnly, + boundingClientRect: { bottom: 0, top: 0 } as DOMRectReadOnly, + rootBounds: { top: 0, bottom: 500 } as DOMRectReadOnly, + time: 0, + ...e, + })) as IntersectionObserverEntry[]); + }, + }); + } + observe(target: Element): void { this.target = target; } + unobserve(): void { this.target = null; } + disconnect(): void { this.target = null; } + takeRecords(): IntersectionObserverEntry[] { return []; } +} + +function userEvent(eventId: string, text: string, ts: number): TimelineEvent { + return { + eventId, type: 'user.message', ts, epoch: 1, seq: ts, + sessionId: 'deck_repin_brain', source: 'daemon', confidence: 'high', + payload: { text }, + } as unknown as TimelineEvent; +} +function assistantEvent(eventId: string, text: string, ts: number): TimelineEvent { + return { + eventId, type: 'assistant.text', ts, epoch: 1, seq: ts, + sessionId: 'deck_repin_brain', source: 'daemon', confidence: 'high', + payload: { text, streaming: false }, + } as unknown as TimelineEvent; +} + +const EVENTS = [ + userEvent('u1', 'why is the chat not at the bottom after refresh', 1000), + assistantEvent('a1', 'investigating the pin path', 2000), +]; + +/** + * Renders, waits for the initial bottom pin, then hands back a harness whose + * `clientHeight` can shrink like a real post-refresh layout settle. + */ +async function renderPinnedChat() { + const { container } = render( + , + ); + const scrollEl = container.querySelector('.chat-view') as HTMLDivElement; + let scrollTopValue = 0; + let clientHeightValue = 200; + let scrollHeightValue = 1200; + Object.defineProperty(scrollEl, 'scrollTop', { + configurable: true, + get: () => scrollTopValue, + set: (v: number) => { scrollTopValue = v; }, + }); + Object.defineProperty(scrollEl, 'scrollHeight', { configurable: true, get: () => scrollHeightValue }); + Object.defineProperty(scrollEl, 'clientHeight', { configurable: true, get: () => clientHeightValue }); + + // Initial pin to the bottom (this is the state the user is in before refresh). + await waitFor(() => expect(scrollTopValue).toBe(1200)); + + // Drain the frames mount already scheduled before handing control back. The + // session-change effect queues `requestAnimationFrame(() => scrollToBottom(true))`, + // and under full-suite parallel load that callback can land AFTER a test has + // set up its scenario — silently re-pinning to the bottom and making an + // unrelated assertion fail. Wait until the position stops changing on its own. + await act(async () => { + for (let i = 0; i < 3; i++) { + await new Promise((resolve) => { requestAnimationFrame(() => resolve()); }); + } + await new Promise((resolve) => { setTimeout(resolve, 30); }); + }); + expect(scrollTopValue).toBe(1200); + + return { + container, + scrollEl, + get scrollTop() { return scrollTopValue; }, + /** Simulate the browser leaving the view above the bottom after a shrink. */ + setScrollTop: (v: number) => { scrollTopValue = v; }, + shrinkViewport: (to: number) => { clientHeightValue = to; }, + /** A block mounting ABOVE the messages makes the whole list taller. */ + growContent: (to: number) => { scrollHeightValue = to; }, + /** Drive a real user scroll-up so follow mode disengages. */ + scrollUpTo: (v: number) => { + scrollTopValue = v; + fireEvent.scroll(scrollEl); + }, + }; +} + +describe('ChatView — post-refresh re-pin to bottom', () => { + beforeEach(() => { + resizeCallbacks.length = 0; + ioInstances.length = 0; + vi.stubGlobal('ResizeObserver', FakeResizeObserver as unknown as typeof ResizeObserver); + vi.stubGlobal('IntersectionObserver', FakeIntersectionObserver as unknown as typeof IntersectionObserver); + }); + + afterEach(() => { + cleanup(); + vi.unstubAllGlobals(); + }); + + it('re-pins to the bottom when the viewport shrinks right after mount', async () => { + const chat = await renderPinnedChat(); + + // The post-refresh settle: --vvh lands / composer draft rehydrates / the + // sub-session bar finishes its transition, so the pane loses height and the + // browser leaves the reading position above the bottom. + chat.shrinkViewport(140); + chat.setScrollTop(1040); + act(() => { fireResize(); }); + + // Must snap back to the bottom. Before the fix the mount-stamped suppression + // swallowed this resize entirely and the view stayed at 1040 forever. + await waitFor(() => expect(chat.scrollTop).toBe(1200)); + }); + + it('still does NOT re-pin for the pin banner toggling its own height', async () => { + const chat = await renderPinnedChat(); + + // A real toggle: the last user bubble goes above the viewport, so the banner + // mounts and steals ~60px. Re-pinning here would snap the user back down, + // which re-hides the banner and starts the height-oscillation jitter loop. + await waitFor(() => expect(ioInstances.length).toBeGreaterThan(0)); + + // The suppression is a 300ms WALL-CLOCK window, so freeze the clock before + // the toggle stamps it. Otherwise this test just races the machine: under + // full-suite parallel load the gap between the stamp and the resize exceeded + // 300ms, the suppression expired, and the failure looked like a product bug + // when it was only the test being slow. + const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(1_700_000_000_000); + try { + act(() => { + ioInstances[ioInstances.length - 1].fire([{ + isIntersecting: false, + boundingClientRect: { bottom: -10, top: -30 } as DOMRectReadOnly, + rootBounds: { top: 0, bottom: 500 } as DOMRectReadOnly, + }]); + }); + expect(chat.container.querySelector('.chat-pinned-last-sent')).not.toBeNull(); + + chat.shrinkViewport(140); + chat.setScrollTop(600); // user is reading up here; must not be yanked + act(() => { fireResize(); }); + + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(chat.scrollTop).toBe(600); + } finally { + nowSpy.mockRestore(); + } + }); + + it('re-pins when a block mounts ABOVE the messages and grows the list (no images involved)', async () => { + const chat = await renderPinnedChat(); + + // The reported case with no images in the chat at all: the "Load older" row + // appears once `hasOlderHistory` resolves from the first history response, the + // agent todo list mounts, the tool-chooser banner lands behind its async pref + // fetch. All sit above every message, so the list gets taller ABOVE the + // viewport while `clientHeight` never changes — the ResizeObserver is blind to + // it and `overflow-anchor: none` means the browser will not compensate. + chat.growContent(1320); + + await waitFor(() => expect(chat.scrollTop).toBe(1320), { timeout: 1000 }); + }); + + it('leaves the view alone if the user scrolled up before the content grew', async () => { + const chat = await renderPinnedChat(); + + // A gentle scroll-up must win — this is precisely the "auto-update fights my + // scrolling" complaint behind the earlier Safari jitter fixes. + chat.scrollUpTo(400); + chat.growContent(1320); + + await new Promise((resolve) => setTimeout(resolve, 300)); + expect(chat.scrollTop).toBe(400); + }); +}); diff --git a/web/test/components/ChatView.test.tsx b/web/test/components/ChatView.test.tsx index ec301d725..c92e92ab5 100644 --- a/web/test/components/ChatView.test.tsx +++ b/web/test/components/ChatView.test.tsx @@ -215,7 +215,7 @@ describe('ChatView', () => { expect(screen.getByText('message-0')).toBeTruthy(); }); - it('reveals locally cached older messages at the top without preserving the previous anchor', async () => { + it('reveals locally cached older messages at the top while preserving the reading position', async () => { const events = Array.from({ length: CHAT_INITIAL_RENDER_ITEM_LIMIT + 10 }, (_, index) => ({ eventId: `user-${index}`, type: 'user.message', @@ -256,7 +256,64 @@ describe('ChatView', () => { await waitFor(() => { expect(screen.getByText('message-0')).toBeTruthy(); }); - expect(scrollTopValue).toBe(0); + // Mounting the older items grows scrollHeight 1200 → 1800 ABOVE the + // viewport. The anchor must re-add that 600px delta so the message the user + // was reading stays under their eyes. Leaving scrollTop at 0 (the previous + // behaviour) silently teleported the viewport a whole reveal-chunk back into + // older history — reported on mobile as "scrolled to the end and it jumps up + // a section". The revealed content is still reachable by scrolling up. + expect(scrollTopValue).toBe(600); + }); + + it('does not move the viewport when a near-top scroll auto-reveals older items', async () => { + // Mobile repro: the auto-trigger fires at scrollTop < 100, so the user is + // near — but not at — the top. Without anchoring, the reveal jumped them up + // by the whole added height AND left them inside the < 100 trigger zone, so + // the next cooldown tick jumped again. Anchoring must both hold the position + // and carry them out of the trigger zone. + const events = Array.from({ length: CHAT_INITIAL_RENDER_ITEM_LIMIT + 10 }, (_, index) => ({ + eventId: `user-${index}`, + type: 'user.message', + ts: 1_700_000_000_000 + index, + payload: { text: `message-${index}` }, + })); + + const { container } = render( + , + ); + const scrollEl = container.querySelector('.chat-view') as HTMLDivElement; + let scrollTopValue = 0; + Object.defineProperty(scrollEl, 'scrollTop', { + configurable: true, + get: () => scrollTopValue, + set: (value) => { scrollTopValue = value; }, + }); + Object.defineProperty(scrollEl, 'scrollHeight', { + configurable: true, + get: () => (container.textContent?.includes('message-0') ? 5000 : 1200), + }); + Object.defineProperty(scrollEl, 'clientHeight', { configurable: true, value: 200 }); + + await waitFor(() => { + expect(scrollTopValue).toBe(1200); + }); + + scrollTopValue = 50; // near the top, inside the auto-reveal trigger zone + fireEvent.wheel(scrollEl, { deltaY: -20 }); + fireEvent.scroll(scrollEl); + + await waitFor(() => { + expect(screen.getByText('message-0')).toBeTruthy(); + }); + // 50 + (5000 - 1200) = 3850: same content under the viewport, and safely + // out of the < 100 re-trigger zone so the reveal cannot loop. + expect(scrollTopValue).toBe(3850); + expect(scrollTopValue).toBeGreaterThanOrEqual(100); }); it('collapses sent user messages longer than ten hard lines by default', () => { @@ -1655,6 +1712,7 @@ describe('ChatView', () => { items: [ { id: 'mem-transport-1', + ref: 'proj:transportrefa', projectId: 'repo-1', summary: 'Fixed transport recall visibility', relevanceScore: 0.91, @@ -1675,6 +1733,8 @@ describe('ChatView', () => { await waitFor(() => { expect(container.textContent).toContain('Fixed transport recall visibility'); + expect(container.textContent).toContain('proj:transportrefa'); + expect(container.querySelector('.chat-memory-context-ref')?.textContent).toBe('proj:transportrefa'); expect(container.querySelector('.chat-memory-context')?.getAttribute('data-related-to')).toBe('evt-user-transport'); }); }); diff --git a/web/test/components/SubSessionBar.test.tsx b/web/test/components/SubSessionBar.test.tsx index 8b1967c15..6b7569b5c 100644 --- a/web/test/components/SubSessionBar.test.tsx +++ b/web/test/components/SubSessionBar.test.tsx @@ -1,6 +1,8 @@ /** * @vitest-environment jsdom */ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { h } from 'preact'; import { act, cleanup, fireEvent, render, waitFor } from '@testing-library/preact'; @@ -13,6 +15,10 @@ vi.mock('react-i18next', () => ({ if (key === 'subsessionBar.p2p_discussions') return 'Team discussions'; if (key === 'repo.info_title') return 'Repository information'; if (key === 'subsessionBar.scheduled_tasks') return 'Scheduled Tasks'; + if (key === 'memory.short_ref_failure_short') return 'memory handles failing'; + if (key === 'memory.short_ref_failure_detail') { + return `stage=${vars?.stage} failures=${vars?.failures} at=${vars?.when} error=${vars?.error}`; + } return key; }, }), @@ -49,6 +55,9 @@ import { SubSessionBar } from '../../src/components/SubSessionBar.js'; import { reorderSubSessions } from '../../src/api.js'; import type { SubSession } from '../../src/hooks/useSubSessions.js'; import { SUBSESSION_ACCENT_COLORS } from '../../src/subsession-accent-colors.js'; +import { SUPPORTED_LOCALES } from '../../src/i18n/locales/index.js'; + +const LOCALE_DIR = join(process.cwd().endsWith('/web') ? process.cwd() : join(process.cwd(), 'web'), 'src/i18n/locales'); function makeSubSession(overrides: Partial = {}): SubSession { return { @@ -1056,4 +1065,104 @@ describe('SubSessionBar', () => { }); }); + // The health record already travelled daemon -> bridge -> browser; these cover the + // last hop, where it either becomes something an operator sees or dies silently. + const failingHealth = { + stage: 'persist_store', + failures: 3, + lastFailureAt: new Date(2026, 4, 12, 7, 8, 9).getTime(), + lastError: 'ENOSPC: no space left on device', + }; + + function renderBarWithStats(collapsed: boolean, statsWs: ReturnType) { + return render( + , + ); + } + + it.each([['collapsed', true], ['expanded', false]] as const)( + 'surfaces a short-ref persistence failure in the %s status bar', + async (_label, collapsed) => { + const statsWs = makeStatsWs(); + const view = renderBarWithStats(collapsed, statsWs); + await waitFor(() => expect(statsWs.ws.onMessage).toHaveBeenCalled()); + + // A healthy daemon must not show the marker, otherwise it is noise and + // gets ignored exactly when it matters. + act(() => { statsWs.emit(daemonStatsMessage); }); + expect(view.container.querySelector('.daemon-stat-shortref-alert')).toBeNull(); + + act(() => { statsWs.emit({ ...daemonStatsMessage, shortRefHealth: failingHealth }); }); + + const alert = view.container.querySelector('.daemon-stat-shortref-alert') as HTMLElement; + expect(alert).toBeTruthy(); + expect(alert.textContent).toContain('memory handles failing'); + // The tooltip has to name the stage, the count and the underlying error; + // "something is wrong" is not actionable at 3am. + expect(alert.title).toContain('persist_store'); + expect(alert.title).toContain('failures=3'); + expect(alert.title).toContain('ENOSPC'); + }, + ); + + it('keeps the marker visible on mobile, unlike the disk readout', async () => { + // Disk usage is desktop-only by request, but a failure that silently breaks + // memory recall should not depend on which client happens to be open. + const statsWs = makeStatsWs(); + const view = render( + , + ); + await waitFor(() => expect(statsWs.ws.onMessage).toHaveBeenCalled()); + act(() => { + statsWs.emit({ + ...daemonStatsMessage, + disks: [{ mount: '/', usedBytes: 5 * 1024 ** 4, totalBytes: 8 * 1024 ** 4, usedPercent: 62 }], + shortRefHealth: failingHealth, + }); + }); + + expect(view.container.querySelector('.daemon-stat-disk')).toBeNull(); + expect(view.container.querySelector('.daemon-stat-shortref-alert')).toBeTruthy(); + }); + + it('ships both alert strings in every locale, with the placeholders the tooltip passes', () => { + // The t() mock above would happily render a key that does not exist in + // production, so check the real locale files rather than trusting it. + for (const locale of SUPPORTED_LOCALES) { + const raw = JSON.parse(readFileSync(join(LOCALE_DIR, `${locale}.json`), 'utf8')) as Record>; + const short = raw.memory?.short_ref_failure_short; + const detail = raw.memory?.short_ref_failure_detail; + expect(short, `${locale}: memory.short_ref_failure_short`).toBeTruthy(); + expect(detail, `${locale}: memory.short_ref_failure_detail`).toBeTruthy(); + for (const placeholder of ['stage', 'failures', 'when', 'error']) { + expect(detail, `${locale}: {{${placeholder}}}`).toContain(`{{${placeholder}}}`); + } + } + }); }); diff --git a/web/test/daemon-upgrade-blocked.test.ts b/web/test/daemon-upgrade-blocked.test.ts index fca8f7322..e044c8061 100644 --- a/web/test/daemon-upgrade-blocked.test.ts +++ b/web/test/daemon-upgrade-blocked.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from 'vitest'; +import { DAEMON_UPGRADE_BLOCK_REASON } from '../../shared/daemon-upgrade.js'; import { DAEMON_UPGRADE_BLOCKED_TOAST_THROTTLE_MS, daemonUpgradeBlockedToastKey, @@ -15,6 +16,7 @@ describe('daemon upgrade blocked toast', () => { expect(daemonUpgradeBlockedToastKey('session_busy')).toBe('toast.upgrade_blocked_session_busy'); expect(daemonUpgradeBlockedToastKey('cooldown_active')).toBe('toast.upgrade_blocked_cooldown_active'); expect(daemonUpgradeBlockedToastKey('toolchain_unavailable')).toBe('toast.upgrade_blocked_toolchain_unavailable'); + expect(daemonUpgradeBlockedToastKey(DAEMON_UPGRADE_BLOCK_REASON.INSTALL_FAILED)).toBe('toast.upgrade_blocked_install_failed'); expect(daemonUpgradeBlockedToastKey('future_reason')).toBe('toast.upgrade_blocked_unknown'); }); diff --git a/web/test/daemon-upgrade-status.test.ts b/web/test/daemon-upgrade-status.test.ts index 9a0c1ab76..8b2d42e8e 100644 --- a/web/test/daemon-upgrade-status.test.ts +++ b/web/test/daemon-upgrade-status.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect, vi } from 'vitest'; import { DAEMON_MSG } from '@shared/daemon-events.js'; import { MSG_DAEMON_ONLINE, MSG_DAEMON_OFFLINE } from '@shared/ack-protocol.js'; +import appSource from '../src/app.tsx?raw'; import { nextDaemonUpgradingState, daemonUpgradingLabel, @@ -21,15 +22,15 @@ describe('nextDaemonUpgradingState', () => { expect(nextDaemonUpgradingState(DAEMON_MSG.UPGRADING, null as unknown)).toEqual({ targetVersion: '' }); }); - it('clears the badge when the daemon comes back (online / reconnected)', () => { + it('clears the badge when the daemon comes back or reports the upgrade blocked', () => { expect(nextDaemonUpgradingState(MSG_DAEMON_ONLINE)).toBeNull(); expect(nextDaemonUpgradingState(DAEMON_MSG.RECONNECTED)).toBeNull(); + expect(nextDaemonUpgradingState(DAEMON_MSG.UPGRADE_BLOCKED)).toBeNull(); }); it('leaves the badge untouched (undefined) for unrelated messages', () => { // undefined is the "keep current state" sentinel — distinct from null (clear). expect(nextDaemonUpgradingState(DAEMON_MSG.DISCONNECTED)).toBeUndefined(); - expect(nextDaemonUpgradingState(DAEMON_MSG.UPGRADE_BLOCKED)).toBeUndefined(); expect(nextDaemonUpgradingState(MSG_DAEMON_OFFLINE)).toBeUndefined(); expect(nextDaemonUpgradingState('daemon.stats')).toBeUndefined(); expect(nextDaemonUpgradingState('session_list')).toBeUndefined(); @@ -40,6 +41,17 @@ describe('nextDaemonUpgradingState', () => { // stays up until the new version reconnects. So DISCONNECTED is a no-op. expect(nextDaemonUpgradingState(DAEMON_MSG.DISCONNECTED)).toBeUndefined(); }); + + it('applies the badge reducer before toast throttling can skip blocker presentation', () => { + const reducerIndex = appSource.indexOf('const upgradingNext = nextDaemonUpgradingState('); + const blockerToastIndex = appSource.indexOf('if (msg.type === DAEMON_MSG.UPGRADE_BLOCKED)'); + + expect(reducerIndex).toBeGreaterThan(-1); + expect(blockerToastIndex).toBeGreaterThan(-1); + expect(reducerIndex).toBeLessThan(blockerToastIndex); + expect(appSource.slice(blockerToastIndex, blockerToastIndex + 1200)) + .not.toContain('if (!shouldShowDaemonUpgradeBlockedToast'); + }); }); describe('daemonUpgradingLabel', () => { diff --git a/web/test/memory-summary-sync.test.ts b/web/test/memory-summary-sync.test.ts index 6628d716e..57496f24a 100644 --- a/web/test/memory-summary-sync.test.ts +++ b/web/test/memory-summary-sync.test.ts @@ -38,8 +38,9 @@ describe('memory summary sync message', () => { }); expect(message).toContain('SYNC ONLY'); expect(message).toContain('Recent summaries (2/5, max 7200 chars):'); - expect(message).toContain('1. [ref: proj:aaaaaaaaaa] [repo-1] Newest summary'); - expect(message).toContain('2. [ref: proj:1111111111] [repo-1] Older summary'); + expect(message).toContain('1. [repo-1] Newest summary'); + expect(message).toContain('2. [repo-1] Older summary'); + expect(message).not.toContain('[ref:'); expect(message).toContain('"tool":"get_memory_sources"'); expect(message).toContain('"kind":"projection"'); expect(message).toContain('"projectionId":"aaaaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"'); @@ -64,8 +65,8 @@ describe('memory summary sync message', () => { it('merges cloud and local summaries while keeping cloud records first for duplicates', async () => { const localSource = vi.fn().mockResolvedValue({ records: [ - { id: 'duplicate-summary', projectId: 'repo-1', projectionClass: 'recent_summary', summary: 'Local duplicate summary', updatedAt: 700 }, - { id: 'local-summary', projectId: 'repo-1', projectionClass: 'recent_summary', summary: 'Local only summary', updatedAt: 600 }, + { id: 'duplicate-summary', ref: 'proj:duplicateaaaa', projectId: 'repo-1', projectionClass: 'recent_summary', summary: 'Local duplicate summary', updatedAt: 700 }, + { id: 'local-summary', ref: 'proj:localsummaryaa', projectId: 'repo-1', projectionClass: 'recent_summary', summary: 'Local only summary', updatedAt: 600 }, ], }); getPersonalCloudMemory.mockResolvedValueOnce({ @@ -92,6 +93,8 @@ describe('memory summary sync message', () => { expect(message).toContain('Local only summary'); expect(message).toContain('Cloud duplicate summary'); expect(message).toContain('Cloud only summary'); + expect(message).toContain('[ref: proj:duplicateaaaa] [repo-1] Cloud duplicate summary'); + expect(message).toContain('[ref: proj:localsummaryaa] [repo-1] Local only summary'); expect(message).not.toContain('Local duplicate summary'); }); @@ -144,7 +147,7 @@ describe('memory summary sync message', () => { requestId: message.requestId, stats: {}, records: [ - { id: 'local-ws-summary', projectId: 'repo-1', projectionClass: 'recent_summary', summary: 'Local ws summary', updatedAt: 700 }, + { id: 'local-ws-summary', ref: 'proj:localwssumaaa', projectId: 'repo-1', projectionClass: 'recent_summary', summary: 'Local ws summary', updatedAt: 700 }, ], }); } @@ -160,6 +163,7 @@ describe('memory summary sync message', () => { projectId: 'repo-1', projectionClass: 'recent_summary', limit: 4, + includeShortRefs: true, }); expect(ws.send).toHaveBeenCalledWith(expect.objectContaining({ @@ -170,6 +174,7 @@ describe('memory summary sync message', () => { limit: 4, })); expect(view?.records[0].summary).toBe('Local ws summary'); + expect(view?.records[0].ref).toBe('proj:localwssumaaa'); expect(handlers.size).toBe(0); }); @@ -198,8 +203,9 @@ describe('memory summary sync message', () => { limit: 10, }); expect(message).toContain('Recent summaries (10/10, max 7200 chars):'); - expect(message).toContain('1. [ref: proj:a12] [repo-1] Summary 12'); - expect(message).toContain('10. [ref: proj:a3] [repo-1] Summary 3'); + expect(message).toContain('1. [repo-1] Summary 12'); + expect(message).toContain('10. [repo-1] Summary 3'); + expect(message).not.toContain('[ref:'); expect(message).not.toContain('Summary 2'); }); diff --git a/web/test/thinking-utils.test.ts b/web/test/thinking-utils.test.ts index edd100dab..109beac8d 100644 --- a/web/test/thinking-utils.test.ts +++ b/web/test/thinking-utils.test.ts @@ -104,6 +104,26 @@ describe('getActiveThinkingTs', () => { { type: 'agent.status', ts: 12, payload: { label: 'still thinking' } }, ] as any)).toBe(10); }); + + it('walks through metadata events so the thinking timer survives memory injection', () => { + // memory.context / memory.compression / file.change are telemetry: they must + // not end a thinking sequence. Before the fix, memory.context terminated the + // scan and the elapsed-thinking timer silently disappeared. + expect(getActiveThinkingTs([ + { type: 'assistant.thinking', ts: 10 }, + { type: 'memory.context', ts: 11, payload: { query: 'x' } }, + { type: 'file.change', ts: 12, payload: { batch: {} } }, + { type: 'agent.status', ts: 13, payload: { label: 'Thinking (1.9k tokens)' } }, + ] as any)).toBe(10); + }); + + it('still ends thinking on real user/assistant turn boundaries', () => { + expect(getActiveThinkingTs([ + { type: 'assistant.thinking', ts: 10 }, + { type: 'assistant.text', ts: 11, payload: { text: 'answered', streaming: false } }, + { type: 'memory.context', ts: 12, payload: {} }, + ] as any)).toBe(null); + }); }); describe('getActiveStatusText', () => { diff --git a/web/test/timeline-running.test.ts b/web/test/timeline-running.test.ts index 95105af1a..f874ec3a0 100644 --- a/web/test/timeline-running.test.ts +++ b/web/test/timeline-running.test.ts @@ -222,4 +222,60 @@ describe('isRunningTimelineEvent', () => { { type: 'usage.update', payload: { model: 'gpt-5.5' } }, ] as any)).toBe(true); }); + + /** + * Regression for deck_sub_704c1g42 (2026-07-25 08:53:32 → 08:54:06): the + * daemon emitted `session.state: running`, then the agent thought for 34 s + * emitting only `agent.status` labels. The tail therefore ended at the + * `memory.context` of that send, which used to fall through to `return false` + * — reporting "no active turn" for the whole window. `SubSessionWindow` then + * fed that into `resolveTimelineBackedSessionState`, whose stale-running guard + * overrode the timeline's `running` with the store snapshot, and the footer + * rendered IDLE while the agent was demonstrably editing files. + */ + it('stays active through the memory.context tail of a context-injecting send', () => { + const events = [ + { type: 'session.state', payload: authoritativeIdlePayload }, + { type: 'command.ack', payload: { commandId: 'cmd-x', status: 'accepted' } }, + { type: 'assistant.thinking', payload: { text: '' } }, + { type: 'session.state', payload: { state: 'running', blockingWorkCount: 3 } }, + { type: 'user.message', payload: { text: '远程桌面只干一件事那里 字幕缺了下载安装的字幕吧' } }, + { type: 'memory.context', payload: { query: 'x', injectedText: '[Related past work]' } }, + { type: 'agent.status', payload: { status: 'requesting', label: null } }, + { type: 'agent.status', payload: { status: 'thinking', label: 'Thinking (1.9k tokens)' } }, + ]; + expect(hasActiveTimelineTurn(events as any)).toBe(true); + }); + + it('stays active when telemetry metadata is the tail after real work', () => { + // file.change / memory.compression / queue snapshots are pure metadata and + // must not be read as "the turn ended". + for (const tail of [ + { type: 'file.change', payload: { batch: {} } }, + { type: 'memory.compression', payload: { outcome: 'success' } }, + { type: 'transport.queue.snapshot', payload: {} }, + { type: 'peer_audit.status', payload: {} }, + ]) { + expect(hasActiveTimelineTurn([ + { type: 'session.state', payload: { state: 'running', blockingWorkCount: 1 } }, + { type: 'user.message', payload: { text: 'go' } }, + { type: 'memory.context', payload: {} }, + tail, + ] as any)).toBe(true); + } + }); + + it('still reports inactive when metadata trails an authoritative clean idle', () => { + // The fix must not resurrect a settled turn: walking through metadata has to + // land on the authoritative idle and report no active work. + expect(hasActiveTimelineTurn([ + { type: 'user.message', payload: { text: 'go' } }, + { type: 'tool.call', payload: { toolCallId: 'A', tool: 'Bash' } }, + { type: 'tool.result', payload: { toolCallId: 'A', terminalStatus: 'succeeded' } }, + { type: 'assistant.text', payload: { text: 'done', streaming: false } }, + { type: 'session.state', payload: authoritativeIdlePayload }, + { type: 'memory.compression', payload: { outcome: 'success' } }, + { type: 'memory.context', payload: {} }, + ] as any)).toBe(false); + }); }); diff --git a/web/test/upload-error.test.ts b/web/test/upload-error.test.ts new file mode 100644 index 000000000..fbd554066 --- /dev/null +++ b/web/test/upload-error.test.ts @@ -0,0 +1,23 @@ +import { describe, it, expect } from 'vitest'; +import { isInsufficientCapacityError } from '../src/upload-error.js'; + +describe('isInsufficientCapacityError', () => { + it('matches the shared code, 507 status, and raw ENOSPC text', () => { + // The exact ApiError message the upload catch receives (status + parsed code). + expect(isInsufficientCapacityError('API 507: insufficient_capacity')).toBe(true); + // Raw JSON body fallback (server response body, no code parsed). + expect(isInsufficientCapacityError('{"error":"insufficient_capacity","message":"ENOSPC: no space left on device"}')).toBe(true); + // Bare daemon ENOSPC error text. + expect(isInsufficientCapacityError('ENOSPC: no space left on device, write')).toBe(true); + expect(isInsufficientCapacityError('no space left on device')).toBe(true); + expect(isInsufficientCapacityError('API 507: unmapped')).toBe(true); + }); + + it('does not match unrelated file-transfer errors', () => { + expect(isInsufficientCapacityError('API 503: daemon_offline')).toBe(false); + expect(isInsufficientCapacityError('API 413: file_too_large')).toBe(false); + expect(isInsufficientCapacityError('upload_failed')).toBe(false); + expect(isInsufficientCapacityError('API 500: internal error')).toBe(false); + expect(isInsufficientCapacityError('')).toBe(false); + }); +}); diff --git a/web/test/use-timeline-http-backfill.test.ts b/web/test/use-timeline-http-backfill.test.ts index cb7700646..b4c6dacfb 100644 --- a/web/test/use-timeline-http-backfill.test.ts +++ b/web/test/use-timeline-http-backfill.test.ts @@ -1147,6 +1147,88 @@ describe('useTimeline — HTTP backfill on WS reconnect', () => { expect(fetchSpy).toHaveBeenCalledTimes(1); }); + it('recovers events dropped while backgrounded: activation asks for the full newest window', async () => { + // The "push notification arrived but the chat shows nothing until I hit ↻" + // bug. The server relays timeline events ONLY to currently-subscribed + // viewers, so a backgrounded app (exactly the state a push proves) has its + // events silently discarded — no queue, no replay. On activation the + // backfill therefore MUST drop the lower bound: a tail-anchored + // `(localNewest - 1ms, …]` request can never reach events that were dropped + // below the local tail, and `agent.status`/`usage.update` push that cursor + // forward every second during a turn, so the gap goes permanently invisible. + const sessionName = `deck_bg_gap_${Date.now()}`; + const serverId = `srv-bg-gap-${Date.now()}`; + + const localTail: TimelineEvent = { + eventId: `${sessionName}-tail`, + sessionId: sessionName, + ts: 5_000, + epoch: 1, + seq: 20, + source: 'daemon', + confidence: 'high', + type: 'assistant.text', + payload: { text: 'local-tail' }, + }; + // Emitted while the app was backgrounded, i.e. BELOW the local tail ts. + const droppedWhileBackgrounded: TimelineEvent = { + eventId: `${sessionName}-dropped`, + sessionId: sessionName, + ts: 4_000, + epoch: 1, + seq: 18, + source: 'daemon', + confidence: 'high', + type: 'assistant.text', + payload: { text: 'dropped-while-backgrounded' }, + }; + ingestTimelineEventForCache(serverId, sessionName, localTail); + fetchSpy.mockResolvedValue({ + events: [localTail, droppedWhileBackgrounded], + epoch: 1, + hasMore: false, + nextCursor: null, + }); + + const ws: WsClient = { + connected: true, + onMessage: () => () => {}, + sendTimelineReplayRequest: vi.fn(() => 'replay-bg'), + sendTimelineHistoryRequest: vi.fn(() => 'history-bg'), + } as unknown as WsClient; + + function Probe() { + const { events } = useTimeline(sessionName, ws, serverId); + return h('div', { 'data-testid': 'probe' }, events.map((e) => String(e.payload.text ?? '')).join('|')); + } + + vi.useFakeTimers({ shouldAdvanceTime: true }); + render(h(Probe)); + await waitFor(() => { + expect(screen.getByTestId('probe').textContent).toContain('local-tail'); + }); + await act(async () => { await vi.advanceTimersByTimeAsync(300); }); + fetchSpy.mockClear(); + + __resetBackfillCooldownsForTests(); + await act(async () => { + window.dispatchEvent(new CustomEvent(ACTIVE_TIMELINE_REFRESH_EVENT)); + }); + await act(async () => { await vi.advanceTimersByTimeAsync(50); }); + + expect(fetchSpy).toHaveBeenCalled(); + // No activation fetch may carry a numeric lower bound — that is the bug. + expect(fetchSpy).not.toHaveBeenCalledWith( + serverId, + sessionName, + expect.objectContaining({ afterTs: expect.any(Number) }), + ); + // End-to-end: the dropped event is visible without a manual ↻. + await waitFor(() => { + expect(screen.getByTestId('probe').textContent).toContain('dropped-while-backgrounded'); + }); + }); + it('uses HTTP-backfilled command.ack to mark daemon receipt while keeping the optimistic send pending', async () => { const sessionName = `deck_http_backfill_ack_${Date.now()}`; const serverId = `srv-http-ack-${Date.now()}`;