From e2a0eda4fa25c073cf0331bec23b6cdd3e6333af Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Mon, 27 Jul 2026 04:40:20 +0200 Subject: [PATCH 01/17] feat(storage): archived cleanup with quarantine default (phase 2) Add preview-first archived-session cleanup (oldest N%) with quarantine under CODEX_HOME/.trash and optional permanent delete. Fold Phase 1 hygiene: immutable comment, skip .trash in scan, drop dead scanStorage imports, and document Storage on the dashboard guide. --- .../src/content/docs/guides/web-dashboard.md | 1 + gui/src/i18n/de.ts | 18 +- gui/src/i18n/en.ts | 18 +- gui/src/i18n/ja.ts | 18 +- gui/src/i18n/ko.ts | 18 +- gui/src/i18n/ru.ts | 18 +- gui/src/i18n/zh.ts | 18 +- gui/src/pages/Storage.tsx | 204 ++++++++- src/server/management-api.ts | 1 - .../management/agent-settings-routes.ts | 1 - src/server/management/combo-routes.ts | 1 - src/server/management/config-routes.ts | 1 - src/server/management/logs-usage-routes.ts | 34 ++ src/server/management/model-routes.ts | 1 - src/server/management/oauth-account-routes.ts | 1 - src/server/management/provider-routes.ts | 1 - src/server/management/shared.ts | 1 - src/storage/cleanup.ts | 417 ++++++++++++++++++ src/storage/scanner.ts | 4 +- tests/api-storage-cleanup.test.ts | 117 +++++ tests/storage-cleanup.test.ts | 146 ++++++ tests/storage-scanner.test.ts | 13 + 22 files changed, 1031 insertions(+), 21 deletions(-) create mode 100644 src/storage/cleanup.ts create mode 100644 tests/api-storage-cleanup.test.ts create mode 100644 tests/storage-cleanup.test.ts diff --git a/docs-site/src/content/docs/guides/web-dashboard.md b/docs-site/src/content/docs/guides/web-dashboard.md index 3ddeae488..2fec2442f 100644 --- a/docs-site/src/content/docs/guides/web-dashboard.md +++ b/docs-site/src/content/docs/guides/web-dashboard.md @@ -39,6 +39,7 @@ bun run dev:gui | **Models** | Toggle native GPT and routed models, set provider allowlists and context caps, choose v1/base/v2, and configure the v2 thread limit. Configured providers stay visible as zero-model groups when discovery is off or returns no rows. | | **Logs** | Auto-refresh recent requests with tokens, requested effort and (when available) effective outbound effort, resolved model, provider, status, request id, duration, and error details. The detail view includes the exact reasoning wire field when the adapter emits one. Filter by opaque conversation/session id (when the client sends one) to total tokens and estimated list-price cost for the currently loaded Logs ring. | | **Usage / Debug** | Inspect token-usage coverage and trends, or enable opt-in provider transport and usage-extraction diagnostics. | +| **Storage** | Read-only CODEX_HOME disk breakdown (sessions, archives, DBs, attachments). Optional archived cleanup: preview the oldest N%, then quarantine to `CODEX_HOME/.trash` (default) or permanently delete behind an explicit checkbox. Active sessions stay read-only. Cleanup is refused while Codex holds `state_*.sqlite` locked. | | **Stop** | Gracefully stop the proxy and installed background service, restore native Codex, and exit (`POST /api/stop`). | ### Linking to a section diff --git a/gui/src/i18n/de.ts b/gui/src/i18n/de.ts index 38630dceb..92fe15a24 100644 --- a/gui/src/i18n/de.ts +++ b/gui/src/i18n/de.ts @@ -864,7 +864,7 @@ export const de: Record = { "nav.storage": "Speicher", "storage.title": "Speicher", - "storage.subtitle": "Schreibgeschützte Übersicht, wohin der CODEX_HOME-Speicher geht. Nichts auf dieser Seite löscht etwas.", + "storage.subtitle": "Diagnostics for CODEX_HOME disk use. Archived cleanup below can quarantine or permanently delete oldest archived sessions — active sessions stay read-only.", "storage.loading": "Speicher wird gescannt…", "storage.empty": "CODEX_HOME ist leer oder fehlt — nichts zu berichten.", "storage.error": "Speicher-Scan fehlgeschlagen. Prüfe, ob CODEX_HOME auf ein gültiges Verzeichnis zeigt.", @@ -888,6 +888,22 @@ export const de: Record = { "storage.bucket.attachments": "Anhänge", "storage.bucket.deletion_manifests": "Lösch-Manifeste", "storage.bucket.other": "Sonstiges", + "storage.cleanup.title": "Archived cleanup", + "storage.cleanup.help": "Remove the oldest archived sessions by percentage. Active sessions are never touched. Quarantine is the default — files move to CODEX_HOME/.trash.", + "storage.cleanup.slider": "Oldest archived percent", + "storage.cleanup.percent": "Oldest {percent}%", + "storage.cleanup.preview": "Preview", + "storage.cleanup.confirmTitle": "Confirm archived cleanup", + "storage.cleanup.confirmBody": "This will process {count} archived file(s) (~{size}), the oldest {percent}%.", + "storage.cleanup.moreFiles": "…and {n} more", + "storage.cleanup.permanent": "Delete permanently (skip quarantine)", + "storage.cleanup.permanentWarn": "Permanent delete cannot be undone.", + "storage.cleanup.quarantineNote": "Files move to .trash under CODEX_HOME. Restore is not available in this release — keep the trash folder until you are sure.", + "storage.cleanup.cancel": "Cancel", + "storage.cleanup.confirmQuarantine": "Quarantine", + "storage.cleanup.confirmPermanent": "Delete permanently", + "storage.cleanup.doneQuarantine": "Quarantined {count} file(s) ({size}).", + "storage.cleanup.donePermanent": "Permanently deleted {count} file(s) ({size}).", "modal.back": "Zurück", "modal.badge.oauth": "OAuth", "modal.customProvider": "Benutzerdefinierter Anbieter", diff --git a/gui/src/i18n/en.ts b/gui/src/i18n/en.ts index cc30bf3e5..03ea3c214 100644 --- a/gui/src/i18n/en.ts +++ b/gui/src/i18n/en.ts @@ -613,7 +613,7 @@ export const en = { "nav.storage": "Storage", "storage.title": "Storage", - "storage.subtitle": "Read-only view of where your CODEX_HOME disk goes. Nothing on this page deletes anything.", + "storage.subtitle": "Diagnostics for CODEX_HOME disk use. Archived cleanup below can quarantine or permanently delete oldest archived sessions — active sessions stay read-only.", "storage.loading": "Scanning storage…", "storage.empty": "CODEX_HOME is empty or missing — nothing to report.", "storage.error": "Storage scan failed. Check that CODEX_HOME points at a valid directory.", @@ -637,6 +637,22 @@ export const en = { "storage.bucket.attachments": "Attachments", "storage.bucket.deletion_manifests": "Deletion manifests", "storage.bucket.other": "Other", + "storage.cleanup.title": "Archived cleanup", + "storage.cleanup.help": "Remove the oldest archived sessions by percentage. Active sessions are never touched. Quarantine is the default — files move to CODEX_HOME/.trash.", + "storage.cleanup.slider": "Oldest archived percent", + "storage.cleanup.percent": "Oldest {percent}%", + "storage.cleanup.preview": "Preview", + "storage.cleanup.confirmTitle": "Confirm archived cleanup", + "storage.cleanup.confirmBody": "This will process {count} archived file(s) (~{size}), the oldest {percent}%.", + "storage.cleanup.moreFiles": "…and {n} more", + "storage.cleanup.permanent": "Delete permanently (skip quarantine)", + "storage.cleanup.permanentWarn": "Permanent delete cannot be undone.", + "storage.cleanup.quarantineNote": "Files move to .trash under CODEX_HOME. Restore is not available in this release — keep the trash folder until you are sure.", + "storage.cleanup.cancel": "Cancel", + "storage.cleanup.confirmQuarantine": "Quarantine", + "storage.cleanup.confirmPermanent": "Delete permanently", + "storage.cleanup.doneQuarantine": "Quarantined {count} file(s) ({size}).", + "storage.cleanup.donePermanent": "Permanently deleted {count} file(s) ({size}).", // add-provider modal "modal.addNamed": "Add: {label}", diff --git a/gui/src/i18n/ja.ts b/gui/src/i18n/ja.ts index 3573457c7..3d4d8aed8 100644 --- a/gui/src/i18n/ja.ts +++ b/gui/src/i18n/ja.ts @@ -577,7 +577,7 @@ export const ja: Record = { "nav.storage": "ストレージ", "storage.title": "ストレージ", - "storage.subtitle": "CODEX_HOME ディスクの使用先の読み取り専用ビューです。このページでは何も削除しません。", + "storage.subtitle": "Diagnostics for CODEX_HOME disk use. Archived cleanup below can quarantine or permanently delete oldest archived sessions — active sessions stay read-only.", "storage.loading": "ストレージをスキャン中…", "storage.empty": "CODEX_HOME が空か存在しません — 報告するものはありません。", "storage.error": "ストレージのスキャンに失敗しました。CODEX_HOME が有効なディレクトリを指しているか確認してください。", @@ -601,6 +601,22 @@ export const ja: Record = { "storage.bucket.attachments": "添付", "storage.bucket.deletion_manifests": "削除マニフェスト", "storage.bucket.other": "その他", + "storage.cleanup.title": "Archived cleanup", + "storage.cleanup.help": "Remove the oldest archived sessions by percentage. Active sessions are never touched. Quarantine is the default — files move to CODEX_HOME/.trash.", + "storage.cleanup.slider": "Oldest archived percent", + "storage.cleanup.percent": "Oldest {percent}%", + "storage.cleanup.preview": "Preview", + "storage.cleanup.confirmTitle": "Confirm archived cleanup", + "storage.cleanup.confirmBody": "This will process {count} archived file(s) (~{size}), the oldest {percent}%.", + "storage.cleanup.moreFiles": "…and {n} more", + "storage.cleanup.permanent": "Delete permanently (skip quarantine)", + "storage.cleanup.permanentWarn": "Permanent delete cannot be undone.", + "storage.cleanup.quarantineNote": "Files move to .trash under CODEX_HOME. Restore is not available in this release — keep the trash folder until you are sure.", + "storage.cleanup.cancel": "Cancel", + "storage.cleanup.confirmQuarantine": "Quarantine", + "storage.cleanup.confirmPermanent": "Delete permanently", + "storage.cleanup.doneQuarantine": "Quarantined {count} file(s) ({size}).", + "storage.cleanup.donePermanent": "Permanently deleted {count} file(s) ({size}).", // add-provider modal "modal.addNamed": "追加: {label}", diff --git a/gui/src/i18n/ko.ts b/gui/src/i18n/ko.ts index 462161977..52877e5ec 100644 --- a/gui/src/i18n/ko.ts +++ b/gui/src/i18n/ko.ts @@ -884,7 +884,7 @@ export const ko: Record = { "nav.storage": "저장소", "storage.title": "저장소", - "storage.subtitle": "CODEX_HOME 디스크 사용처를 읽기 전용으로 보여줍니다. 이 페이지에서는 아무것도 삭제되지 않습니다.", + "storage.subtitle": "Diagnostics for CODEX_HOME disk use. Archived cleanup below can quarantine or permanently delete oldest archived sessions — active sessions stay read-only.", "storage.loading": "저장소 스캔 중…", "storage.empty": "CODEX_HOME이 비어 있거나 없습니다 — 표시할 내용이 없습니다.", "storage.error": "저장소 스캔에 실패했습니다. CODEX_HOME이 올바른 디렉터리를 가리키는지 확인하세요.", @@ -908,6 +908,22 @@ export const ko: Record = { "storage.bucket.attachments": "첨부 파일", "storage.bucket.deletion_manifests": "삭제 매니페스트", "storage.bucket.other": "기타", + "storage.cleanup.title": "Archived cleanup", + "storage.cleanup.help": "Remove the oldest archived sessions by percentage. Active sessions are never touched. Quarantine is the default — files move to CODEX_HOME/.trash.", + "storage.cleanup.slider": "Oldest archived percent", + "storage.cleanup.percent": "Oldest {percent}%", + "storage.cleanup.preview": "Preview", + "storage.cleanup.confirmTitle": "Confirm archived cleanup", + "storage.cleanup.confirmBody": "This will process {count} archived file(s) (~{size}), the oldest {percent}%.", + "storage.cleanup.moreFiles": "…and {n} more", + "storage.cleanup.permanent": "Delete permanently (skip quarantine)", + "storage.cleanup.permanentWarn": "Permanent delete cannot be undone.", + "storage.cleanup.quarantineNote": "Files move to .trash under CODEX_HOME. Restore is not available in this release — keep the trash folder until you are sure.", + "storage.cleanup.cancel": "Cancel", + "storage.cleanup.confirmQuarantine": "Quarantine", + "storage.cleanup.confirmPermanent": "Delete permanently", + "storage.cleanup.doneQuarantine": "Quarantined {count} file(s) ({size}).", + "storage.cleanup.donePermanent": "Permanently deleted {count} file(s) ({size}).", "modal.back": "뒤로", "modal.badge.oauth": "OAuth", "modal.customProvider": "사용자 지정 프로바이더", diff --git a/gui/src/i18n/ru.ts b/gui/src/i18n/ru.ts index 462c42e8d..1614102aa 100644 --- a/gui/src/i18n/ru.ts +++ b/gui/src/i18n/ru.ts @@ -612,7 +612,7 @@ export const ru: Record = { "nav.storage": "Хранилище", "storage.title": "Хранилище", - "storage.subtitle": "Обзор того, чем занято место в CODEX_HOME, — только для чтения. Эта страница ничего не удаляет.", + "storage.subtitle": "Diagnostics for CODEX_HOME disk use. Archived cleanup below can quarantine or permanently delete oldest archived sessions — active sessions stay read-only.", "storage.loading": "Сканирование хранилища…", "storage.empty": "CODEX_HOME пуст или отсутствует — показывать нечего.", "storage.error": "Не удалось просканировать хранилище. Убедитесь, что CODEX_HOME указывает на корректный каталог.", @@ -636,6 +636,22 @@ export const ru: Record = { "storage.bucket.attachments": "Вложения", "storage.bucket.deletion_manifests": "Манифесты удаления", "storage.bucket.other": "Прочее", + "storage.cleanup.title": "Archived cleanup", + "storage.cleanup.help": "Remove the oldest archived sessions by percentage. Active sessions are never touched. Quarantine is the default — files move to CODEX_HOME/.trash.", + "storage.cleanup.slider": "Oldest archived percent", + "storage.cleanup.percent": "Oldest {percent}%", + "storage.cleanup.preview": "Preview", + "storage.cleanup.confirmTitle": "Confirm archived cleanup", + "storage.cleanup.confirmBody": "This will process {count} archived file(s) (~{size}), the oldest {percent}%.", + "storage.cleanup.moreFiles": "…and {n} more", + "storage.cleanup.permanent": "Delete permanently (skip quarantine)", + "storage.cleanup.permanentWarn": "Permanent delete cannot be undone.", + "storage.cleanup.quarantineNote": "Files move to .trash under CODEX_HOME. Restore is not available in this release — keep the trash folder until you are sure.", + "storage.cleanup.cancel": "Cancel", + "storage.cleanup.confirmQuarantine": "Quarantine", + "storage.cleanup.confirmPermanent": "Delete permanently", + "storage.cleanup.doneQuarantine": "Quarantined {count} file(s) ({size}).", + "storage.cleanup.donePermanent": "Permanently deleted {count} file(s) ({size}).", // add-provider modal "modal.addNamed": "Добавить: {label}", diff --git a/gui/src/i18n/zh.ts b/gui/src/i18n/zh.ts index 90a0c077c..135f1d7e9 100644 --- a/gui/src/i18n/zh.ts +++ b/gui/src/i18n/zh.ts @@ -884,7 +884,7 @@ export const zh: Record = { "nav.storage": "存储", "storage.title": "存储", - "storage.subtitle": "只读视图,显示 CODEX_HOME 磁盘的使用情况。此页面不会删除任何内容。", + "storage.subtitle": "Diagnostics for CODEX_HOME disk use. Archived cleanup below can quarantine or permanently delete oldest archived sessions — active sessions stay read-only.", "storage.loading": "正在扫描存储…", "storage.empty": "CODEX_HOME 为空或不存在——没有可显示的内容。", "storage.error": "存储扫描失败。请检查 CODEX_HOME 是否指向有效目录。", @@ -908,6 +908,22 @@ export const zh: Record = { "storage.bucket.attachments": "附件", "storage.bucket.deletion_manifests": "删除清单", "storage.bucket.other": "其他", + "storage.cleanup.title": "Archived cleanup", + "storage.cleanup.help": "Remove the oldest archived sessions by percentage. Active sessions are never touched. Quarantine is the default — files move to CODEX_HOME/.trash.", + "storage.cleanup.slider": "Oldest archived percent", + "storage.cleanup.percent": "Oldest {percent}%", + "storage.cleanup.preview": "Preview", + "storage.cleanup.confirmTitle": "Confirm archived cleanup", + "storage.cleanup.confirmBody": "This will process {count} archived file(s) (~{size}), the oldest {percent}%.", + "storage.cleanup.moreFiles": "…and {n} more", + "storage.cleanup.permanent": "Delete permanently (skip quarantine)", + "storage.cleanup.permanentWarn": "Permanent delete cannot be undone.", + "storage.cleanup.quarantineNote": "Files move to .trash under CODEX_HOME. Restore is not available in this release — keep the trash folder until you are sure.", + "storage.cleanup.cancel": "Cancel", + "storage.cleanup.confirmQuarantine": "Quarantine", + "storage.cleanup.confirmPermanent": "Delete permanently", + "storage.cleanup.doneQuarantine": "Quarantined {count} file(s) ({size}).", + "storage.cleanup.donePermanent": "Permanently deleted {count} file(s) ({size}).", "modal.back": "返回", "modal.badge.oauth": "OAuth", "modal.customProvider": "自定义提供方", diff --git a/gui/src/pages/Storage.tsx b/gui/src/pages/Storage.tsx index 38c86ef66..814b737f6 100644 --- a/gui/src/pages/Storage.tsx +++ b/gui/src/pages/Storage.tsx @@ -28,7 +28,23 @@ interface StorageReport { error?: string; } -// Known scanner bucket keys → localized labels; unknown future keys fall back to the API label. +interface CleanupPreview { + percent: number; + count: number; + bytes: number; + candidates: Array<{ relPath: string; bytes: number }>; +} + +interface CleanupResult { + ok: boolean; + mode: "quarantine" | "permanent"; + count: number; + bytes: number; + trashDir?: string; + error?: string; + message?: string; +} + const BUCKET_TKEYS: Record = { sessions: "storage.bucket.sessions", archived_sessions: "storage.bucket.archived_sessions", @@ -39,6 +55,8 @@ const BUCKET_TKEYS: Record = { other: "storage.bucket.other", }; +const PRESETS = [10, 25, 50] as const; + function bucketLabel(bucket: StorageBucket, t: TFn): string { const tkey = BUCKET_TKEYS[bucket.key]; return tkey ? t(tkey) : bucket.label; @@ -111,6 +129,176 @@ function LargestFilesPanel({ buckets, locale, t }: { buckets: StorageBucket[]; l ); } +function ArchivedCleanupPanel({ + apiBase, + locale, + t, + onDone, +}: { + apiBase: string; + locale: Locale; + t: TFn; + onDone: () => void; +}) { + const [percent, setPercent] = useState(25); + const [preview, setPreview] = useState(null); + const [confirmOpen, setConfirmOpen] = useState(false); + const [permanent, setPermanent] = useState(false); + const [busy, setBusy] = useState(false); + const [status, setStatus] = useState(null); + const [error, setError] = useState(null); + + const runPreview = async () => { + setBusy(true); + setError(null); + setStatus(null); + try { + const res = await fetch(`${apiBase}/api/storage/cleanup/preview`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ percent }), + }); + const json = await res.json() as CleanupPreview & { error?: string }; + if (!res.ok) throw new Error(json.error ?? "preview failed"); + setPreview(json); + setConfirmOpen(true); + } catch (e) { + setError(e instanceof Error ? e.message : String(e)); + } finally { + setBusy(false); + } + }; + + const runCleanup = async () => { + if (!preview) return; + setBusy(true); + setError(null); + try { + const res = await fetch(`${apiBase}/api/storage/cleanup`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ percent: preview.percent, mode: permanent ? "permanent" : "quarantine" }), + }); + const json = await res.json() as CleanupResult; + if (!res.ok || !json.ok) { + throw new Error(json.message ?? json.error ?? "cleanup failed"); + } + setConfirmOpen(false); + setPreview(null); + setPermanent(false); + setStatus( + permanent + ? t("storage.cleanup.donePermanent", { count: String(json.count), size: formatBytes(json.bytes, locale) }) + : t("storage.cleanup.doneQuarantine", { count: String(json.count), size: formatBytes(json.bytes, locale) }), + ); + onDone(); + } catch (e) { + setError(e instanceof Error ? e.message : String(e)); + } finally { + setBusy(false); + } + }; + + return ( +
+

{t("storage.cleanup.title")}

+

{t("storage.cleanup.help")}

+ +
+ +
+ {PRESETS.map(p => ( + + ))} +
+ +
+ + {status &&

{status}

} + {error &&

{error}

} + + {confirmOpen && preview && ( +
!busy && setConfirmOpen(false)} + onKeyDown={e => { if (e.key === "Escape" && !busy) setConfirmOpen(false); }} + > +
e.stopPropagation()} onKeyDown={e => e.stopPropagation()}> +

{t("storage.cleanup.confirmTitle")}

+

+ {t("storage.cleanup.confirmBody", { + count: String(preview.count), + size: formatBytes(preview.bytes, locale), + percent: String(preview.percent), + })} +

+ {preview.candidates.length > 0 && ( +
    + {preview.candidates.slice(0, 8).map(c => ( +
  • {c.relPath}
  • + ))} + {preview.candidates.length > 8 && ( +
  • {t("storage.cleanup.moreFiles", { n: String(preview.candidates.length - 8) })}
  • + )} +
+ )} + +

+ {permanent ? t("storage.cleanup.permanentWarn") : t("storage.cleanup.quarantineNote")} +

+
+ + +
+
+
+ )} +
+ ); +} + export default function Storage({ apiBase }: { apiBase: string }) { const { t, locale } = useI18n(); const [data, setData] = useState(null); @@ -130,22 +318,17 @@ export default function Storage({ apiBase }: { apiBase: string }) { if (signal?.aborted || generation !== loadGenerationRef.current) return; setData(null); } finally { - // Only the current request may clear loading — a superseded abort must not - // settle the UI while a newer fetch is still in flight. if (generation === loadGenerationRef.current) setLoading(false); } }, [apiBase]); useEffect(() => { const controller = new AbortController(); - // Deferred a tick (same pattern as Usage.tsx) so the effect never sets state synchronously. const timeout = window.setTimeout(() => { void fetchStorage(controller.signal); }, 0); return () => { window.clearTimeout(timeout); - // Invalidate before abort so a superseded request's finally cannot clear - // loading in the gap before the deferred replacement increments generation. loadGenerationRef.current += 1; controller.abort(); }; @@ -153,6 +336,7 @@ export default function Storage({ apiBase }: { apiBase: string }) { const failed = !loading && (!data || data.error !== undefined); const empty = !loading && !failed && data!.total.fileCount === 0; + const archivedCount = data?.buckets.find(b => b.key === "archived_sessions")?.fileCount ?? 0; return ( <> @@ -179,6 +363,14 @@ export default function Storage({ apiBase }: { apiBase: string }) { + {archivedCount > 0 && ( + void fetchStorage()} + /> + )} )} diff --git a/src/server/management-api.ts b/src/server/management-api.ts index 43d8edb2a..52e2021b3 100644 --- a/src/server/management-api.ts +++ b/src/server/management-api.ts @@ -33,7 +33,6 @@ import { clearThreadAccountMap } from "../codex/routing"; import { primeCodexPoolQuotas } from "../codex/auth-api"; import { DEFAULT_PROVIDER_CONTEXT_CAP, globalContextCapValue, providerContextCap, providerContextCaps, setAllProviderContextCaps, setGlobalContextCapValue, setProviderContextCap } from "../providers/context-cap"; import { resolveCodexHomeDir } from "../codex/home"; -import { scanStorage } from "../storage/scanner"; import { readUsageEntries } from "../usage/log"; import { getUsageDebugLogEntries } from "../usage/debug"; import { parseRange, parseUsageSurface, summarizeUsage } from "../usage/summary"; diff --git a/src/server/management/agent-settings-routes.ts b/src/server/management/agent-settings-routes.ts index 9b131f9ed..686311f2e 100644 --- a/src/server/management/agent-settings-routes.ts +++ b/src/server/management/agent-settings-routes.ts @@ -33,7 +33,6 @@ import { clearThreadAccountMap } from "../../codex/routing"; import { primeCodexPoolQuotas } from "../../codex/auth-api"; import { DEFAULT_PROVIDER_CONTEXT_CAP, globalContextCapValue, providerContextCap, providerContextCaps, setAllProviderContextCaps, setGlobalContextCapValue, setProviderContextCap } from "../../providers/context-cap"; import { resolveCodexHomeDir } from "../../codex/home"; -import { scanStorage } from "../../storage/scanner"; import { readUsageEntries } from "../../usage/log"; import { getUsageDebugLogEntries } from "../../usage/debug"; import { parseRange, parseUsageSurface, summarizeUsage } from "../../usage/summary"; diff --git a/src/server/management/combo-routes.ts b/src/server/management/combo-routes.ts index f43df9fcd..f7942e87a 100644 --- a/src/server/management/combo-routes.ts +++ b/src/server/management/combo-routes.ts @@ -33,7 +33,6 @@ import { clearThreadAccountMap } from "../../codex/routing"; import { primeCodexPoolQuotas } from "../../codex/auth-api"; import { DEFAULT_PROVIDER_CONTEXT_CAP, globalContextCapValue, providerContextCap, providerContextCaps, setAllProviderContextCaps, setGlobalContextCapValue, setProviderContextCap } from "../../providers/context-cap"; import { resolveCodexHomeDir } from "../../codex/home"; -import { scanStorage } from "../../storage/scanner"; import { readUsageEntries } from "../../usage/log"; import { getUsageDebugLogEntries } from "../../usage/debug"; import { parseRange, parseUsageSurface, summarizeUsage } from "../../usage/summary"; diff --git a/src/server/management/config-routes.ts b/src/server/management/config-routes.ts index 4703d0625..db829c13e 100644 --- a/src/server/management/config-routes.ts +++ b/src/server/management/config-routes.ts @@ -34,7 +34,6 @@ import { clearThreadAccountMap } from "../../codex/routing"; import { primeCodexPoolQuotas } from "../../codex/auth-api"; import { DEFAULT_PROVIDER_CONTEXT_CAP, globalContextCapValue, providerContextCap, providerContextCaps, setAllProviderContextCaps, setGlobalContextCapValue, setProviderContextCap } from "../../providers/context-cap"; import { resolveCodexHomeDir } from "../../codex/home"; -import { scanStorage } from "../../storage/scanner"; import { readUsageEntries } from "../../usage/log"; import { getUsageDebugLogEntries } from "../../usage/debug"; import { parseRange, parseUsageSurface, summarizeUsage } from "../../usage/summary"; diff --git a/src/server/management/logs-usage-routes.ts b/src/server/management/logs-usage-routes.ts index e86c1fad1..95cf505ef 100644 --- a/src/server/management/logs-usage-routes.ts +++ b/src/server/management/logs-usage-routes.ts @@ -34,6 +34,7 @@ import { primeCodexPoolQuotas } from "../../codex/auth-api"; import { DEFAULT_PROVIDER_CONTEXT_CAP, globalContextCapValue, providerContextCap, providerContextCaps, setAllProviderContextCaps, setGlobalContextCapValue, setProviderContextCap } from "../../providers/context-cap"; import { resolveCodexHomeDir } from "../../codex/home"; import { scanStorage } from "../../storage/scanner"; +import { executeArchivedCleanup, previewArchivedCleanup, type CleanupMode } from "../../storage/cleanup"; import { currentUsageLogRevision, readUsageSnapshotForManagement, @@ -232,5 +233,38 @@ export async function handleLogsUsageRoutes(ctx: ManagementContext): Promise 100) { + return jsonResponse({ error: "percent must be a number between 0 and 100" }, 400); + } + return jsonResponse(previewArchivedCleanup(percent)); + } + + if (url.pathname === "/api/storage/cleanup" && req.method === "POST") { + let body: { percent?: unknown; mode?: unknown }; + try { body = await req.json(); } catch { return jsonResponse({ error: "invalid JSON body" }, 400); } + const percent = typeof body?.percent === "number" ? body.percent : Number.NaN; + if (!Number.isFinite(percent) || percent < 0 || percent > 100) { + return jsonResponse({ error: "percent must be a number between 0 and 100" }, 400); + } + const mode = body?.mode; + if (mode !== "quarantine" && mode !== "permanent") { + return jsonResponse({ error: "mode must be quarantine or permanent" }, 400); + } + const result = executeArchivedCleanup({ percent, mode: mode as CleanupMode }); + if (!result.ok && result.error === "codex_busy") { + return jsonResponse({ + ...result, + message: "Codex is using state.sqlite — try again after quitting Codex.", + }, 409); + } + if (!result.ok) return jsonResponse(result, 500); + return jsonResponse(result); + } + return null; } diff --git a/src/server/management/model-routes.ts b/src/server/management/model-routes.ts index 77457767b..ce845d5f3 100644 --- a/src/server/management/model-routes.ts +++ b/src/server/management/model-routes.ts @@ -35,7 +35,6 @@ import { clearThreadAccountMap } from "../../codex/routing"; import { primeCodexPoolQuotas } from "../../codex/auth-api"; import { DEFAULT_PROVIDER_CONTEXT_CAP, globalContextCapValue, providerContextCap, providerContextCaps, setAllProviderContextCaps, setGlobalContextCapValue, setProviderContextCap } from "../../providers/context-cap"; import { resolveCodexHomeDir } from "../../codex/home"; -import { scanStorage } from "../../storage/scanner"; import { readUsageEntries } from "../../usage/log"; import { getUsageDebugLogEntries } from "../../usage/debug"; import { parseRange, parseUsageSurface, summarizeUsage } from "../../usage/summary"; diff --git a/src/server/management/oauth-account-routes.ts b/src/server/management/oauth-account-routes.ts index 7eb8d9b06..70ac002a3 100644 --- a/src/server/management/oauth-account-routes.ts +++ b/src/server/management/oauth-account-routes.ts @@ -33,7 +33,6 @@ import { clearThreadAccountMap } from "../../codex/routing"; import { primeCodexPoolQuotas } from "../../codex/auth-api"; import { DEFAULT_PROVIDER_CONTEXT_CAP, globalContextCapValue, providerContextCap, providerContextCaps, setAllProviderContextCaps, setGlobalContextCapValue, setProviderContextCap } from "../../providers/context-cap"; import { resolveCodexHomeDir } from "../../codex/home"; -import { scanStorage } from "../../storage/scanner"; import { readUsageEntries } from "../../usage/log"; import { getUsageDebugLogEntries } from "../../usage/debug"; import { parseRange, parseUsageSurface, summarizeUsage } from "../../usage/summary"; diff --git a/src/server/management/provider-routes.ts b/src/server/management/provider-routes.ts index f2e16d259..7d8cd65c6 100644 --- a/src/server/management/provider-routes.ts +++ b/src/server/management/provider-routes.ts @@ -34,7 +34,6 @@ import { primeCodexPoolQuotas } from "../../codex/auth-api"; import { getProviderDiscoveryStatus } from "../../codex/model-cache"; import { DEFAULT_PROVIDER_CONTEXT_CAP, globalContextCapValue, providerContextCap, providerContextCaps, setAllProviderContextCaps, setGlobalContextCapValue, setProviderContextCap } from "../../providers/context-cap"; import { resolveCodexHomeDir } from "../../codex/home"; -import { scanStorage } from "../../storage/scanner"; import { readUsageEntries } from "../../usage/log"; import { getUsageDebugLogEntries } from "../../usage/debug"; import { parseRange, parseUsageSurface, summarizeUsage } from "../../usage/summary"; diff --git a/src/server/management/shared.ts b/src/server/management/shared.ts index c0e56e31d..8451dd16f 100644 --- a/src/server/management/shared.ts +++ b/src/server/management/shared.ts @@ -33,7 +33,6 @@ import { clearThreadAccountMap } from "../../codex/routing"; import { primeCodexPoolQuotas } from "../../codex/auth-api"; import { DEFAULT_PROVIDER_CONTEXT_CAP, globalContextCapValue, providerContextCap, providerContextCaps, setAllProviderContextCaps, setGlobalContextCapValue, setProviderContextCap } from "../../providers/context-cap"; import { resolveCodexHomeDir } from "../../codex/home"; -import { scanStorage } from "../../storage/scanner"; import { readUsageEntries } from "../../usage/log"; import { getUsageDebugLogEntries } from "../../usage/debug"; import { parseRange, parseUsageSurface, summarizeUsage } from "../../usage/summary"; diff --git a/src/storage/cleanup.ts b/src/storage/cleanup.ts new file mode 100644 index 000000000..5b771f81b --- /dev/null +++ b/src/storage/cleanup.ts @@ -0,0 +1,417 @@ +/** + * Phase 2 archived-session cleanup (issue #42 Option A). + * + * Preview + execute for files under `archived_sessions/` only. Active `sessions/` + * are never touched. Default mode quarantines into `CODEX_HOME/.trash//`; + * permanent delete is opt-in. DB reconciliation probes writability first — on + * SQLITE_BUSY nothing is moved or deleted. + */ +import { + existsSync, + mkdirSync, + readdirSync, + readFileSync, + renameSync, + statSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { basename, join, relative, resolve, sep } from "node:path"; +import { Database } from "bun:sqlite"; +import { resolveCodexHomeDir } from "../codex/home"; + +export const ARCHIVED_SESSIONS_DIR = "archived_sessions"; +export const TRASH_DIR = ".trash"; + +export type CleanupMode = "quarantine" | "permanent"; + +export interface ArchivedCandidate { + /** Path relative to CODEX_HOME, forward-slash separated. */ + relPath: string; + absPath: string; + bytes: number; + mtimeMs: number; +} + +export interface CleanupPreview { + codexHome: string; + percent: number; + count: number; + bytes: number; + candidates: ArchivedCandidate[]; +} + +export interface CleanupManifestEntry { + relPath: string; + bytes: number; + mtimeMs: number; + threadId?: string; + rolloutPath?: string; + archived?: number | null; +} + +export interface CleanupResult { + ok: boolean; + mode: CleanupMode; + percent: number; + count: number; + bytes: number; + trashDir?: string; + error?: string; + removedPaths: string[]; +} + +const STATE_DB_FILE = /^state_(\d+)\.sqlite$/; + +function clampPercent(percent: unknown): number { + if (typeof percent !== "number" || !Number.isFinite(percent)) return 0; + return Math.max(0, Math.min(100, Math.floor(percent))); +} + +function toForwardSlash(p: string): string { + return p.split(sep).join("/"); +} + +function newestStateDb(codexHome: string): string | null { + let best: string | null = null; + let bestVersion = -1; + let names: string[] = []; + try { + names = readdirSync(codexHome); + } catch { + return null; + } + for (const name of names) { + const match = name.match(STATE_DB_FILE); + if (!match) continue; + const version = Number(match[1]); + if (version > bestVersion) { + bestVersion = version; + best = name; + } + } + return best ? join(codexHome, best) : null; +} + +/** List archived rollout files oldest-first. Never walks `sessions/`. */ +export function listArchivedCandidates(codexHome: string): ArchivedCandidate[] { + const dir = join(codexHome, ARCHIVED_SESSIONS_DIR); + let names: string[] = []; + try { + names = readdirSync(dir); + } catch { + return []; + } + const out: ArchivedCandidate[] = []; + for (const name of names) { + if (!name.endsWith(".jsonl")) continue; + const absPath = join(dir, name); + try { + const st = statSync(absPath); + if (!st.isFile()) continue; + out.push({ + relPath: `${ARCHIVED_SESSIONS_DIR}/${name}`, + absPath, + bytes: st.size, + mtimeMs: st.mtimeMs, + }); + } catch { + /* vanished mid-scan */ + } + } + out.sort((a, b) => a.mtimeMs - b.mtimeMs || a.relPath.localeCompare(b.relPath)); + return out; +} + +export function selectOldestPercent(candidates: ArchivedCandidate[], percent: number): ArchivedCandidate[] { + const pct = clampPercent(percent); + if (pct <= 0 || candidates.length === 0) return []; + if (pct >= 100) return [...candidates]; + const n = Math.max(1, Math.floor((candidates.length * pct) / 100)); + return candidates.slice(0, n); +} + +export function previewArchivedCleanup( + percent: number, + codexHome: string = resolveCodexHomeDir(), +): CleanupPreview { + const all = listArchivedCandidates(codexHome); + const selected = selectOldestPercent(all, percent); + return { + codexHome, + percent: clampPercent(percent), + count: selected.length, + bytes: selected.reduce((sum, c) => sum + c.bytes, 0), + candidates: selected, + }; +} + +function openStateDbWritable(stateDbPath: string, busyTimeoutMs = 100): Database { + const db = new Database(stateDbPath); + try { + db.exec(`PRAGMA busy_timeout = ${busyTimeoutMs}`); + } catch { + /* older sqlite */ + } + return db; +} + +function isBusyError(error: unknown): boolean { + const msg = error instanceof Error ? error.message : String(error); + const code = (error as { code?: string })?.code ?? ""; + return ( + code === "SQLITE_BUSY" || + code === "SQLITE_LOCKED" || + /SQLITE_BUSY|SQLITE_LOCKED|database is locked|locked/i.test(msg) + ); +} + +/** True when the threads table can be written (BEGIN IMMEDIATE succeeds). */ +export function probeStateDbWritable(codexHome: string, busyTimeoutMs = 100): { ok: true; path: string } | { ok: false; error: string } { + const path = newestStateDb(codexHome); + if (!path || !existsSync(path)) return { ok: true, path: path ?? "" }; // no DB → FS-only cleanup + let db: Database | undefined; + try { + db = openStateDbWritable(path, busyTimeoutMs); + db.exec("BEGIN IMMEDIATE"); + db.exec("ROLLBACK"); + return { ok: true, path }; + } catch (error) { + if (isBusyError(error)) return { ok: false, error: "codex_busy" }; + // Missing threads table or corrupt DB: still allow FS cleanup; DB step will soft-skip. + return { ok: true, path }; + } finally { + try { db?.close(); } catch { /* */ } + } +} + +function rolloutPathMatches(candidate: ArchivedCandidate, rolloutPath: string, codexHome: string): boolean { + const raw = rolloutPath.replace(/\\/g, "/"); + const fileName = basename(candidate.relPath); + if (raw === candidate.relPath) return true; + if (raw.endsWith(`/${fileName}`) || raw.endsWith(`\\${fileName}`) || basename(raw) === fileName) return true; + try { + const abs = raw.includes(":") || raw.startsWith("/") ? resolve(raw) : resolve(codexHome, raw); + if (resolve(abs) === resolve(candidate.absPath)) return true; + } catch { + /* ignore */ + } + return false; +} + +interface ThreadSnapshot { + id: string; + rollout_path: string; + archived: number | null; +} + +function loadMatchingThreads(db: Database, candidates: ArchivedCandidate[], codexHome: string): ThreadSnapshot[] { + let rows: Array<{ id: string; rollout_path: string; archived?: number | null }> = []; + try { + rows = db.query<{ id: string; rollout_path: string; archived: number | null }, []>( + `SELECT id, rollout_path, archived FROM threads`, + ).all(); + } catch { + try { + rows = db.query<{ id: string; rollout_path: string }, []>( + `SELECT id, rollout_path FROM threads`, + ).all().map(r => ({ ...r, archived: null })); + } catch { + return []; + } + } + return rows + .filter(row => candidates.some(c => rolloutPathMatches(c, row.rollout_path, codexHome))) + .map(row => ({ + id: row.id, + rollout_path: row.rollout_path, + archived: row.archived ?? null, + })); +} + +function deleteThreadsAndEdges(db: Database, threadIds: string[]): void { + if (threadIds.length === 0) return; + const placeholders = threadIds.map(() => "?").join(","); + db.run(`DELETE FROM threads WHERE id IN (${placeholders})`, threadIds); + try { + db.run(`DELETE FROM thread_spawn_edges WHERE parent_id IN (${placeholders}) OR child_id IN (${placeholders})`, [ + ...threadIds, + ...threadIds, + ]); + } catch { + try { + db.run(`DELETE FROM thread_spawn_edges WHERE thread_id IN (${placeholders})`, threadIds); + } catch { + /* table absent or unknown schema — ignore */ + } + } +} + +function reconcileThreads( + stateDbPath: string, + candidates: ArchivedCandidate[], + codexHome: string, + busyTimeoutMs: number, +): { threads: ThreadSnapshot[]; error?: string } { + if (!stateDbPath || !existsSync(stateDbPath)) return { threads: [] }; + let db: Database | undefined; + try { + db = openStateDbWritable(stateDbPath, busyTimeoutMs); + const threads = loadMatchingThreads(db, candidates, codexHome); + db.exec("BEGIN IMMEDIATE"); + try { + deleteThreadsAndEdges(db, threads.map(t => t.id)); + db.exec("COMMIT"); + } catch (error) { + try { db.exec("ROLLBACK"); } catch { /* */ } + throw error; + } + return { threads }; + } catch (error) { + if (isBusyError(error)) return { threads: [], error: "codex_busy" }; + return { threads: [], error: error instanceof Error ? error.message : String(error) }; + } finally { + try { db?.close(); } catch { /* */ } + } +} + +export interface ExecuteCleanupOptions { + percent: number; + mode: CleanupMode; + codexHome?: string; + /** Test-only: shrink busy_timeout so lock tests fail fast. */ + busyTimeoutMs?: number; + now?: number; +} + +/** + * Execute archived cleanup. Probes DB writability first — on `codex_busy` returns + * without touching the filesystem. + */ +export function executeArchivedCleanup(options: ExecuteCleanupOptions): CleanupResult { + const codexHome = options.codexHome ?? resolveCodexHomeDir(); + const mode = options.mode; + const percent = clampPercent(options.percent); + const busyTimeoutMs = options.busyTimeoutMs ?? 100; + const preview = previewArchivedCleanup(percent, codexHome); + const empty: CleanupResult = { + ok: true, + mode, + percent, + count: 0, + bytes: 0, + removedPaths: [], + }; + if (preview.candidates.length === 0) return empty; + + if (mode !== "quarantine" && mode !== "permanent") { + return { ...empty, ok: false, error: "invalid_mode" }; + } + + const probe = probeStateDbWritable(codexHome, busyTimeoutMs); + if (!probe.ok) { + return { + ok: false, + mode, + percent, + count: 0, + bytes: 0, + removedPaths: [], + error: probe.error, + }; + } + + // Load matching thread snapshots while the DB is writable, then FS, then delete rows. + // Quarantine keeps files recoverable if the DB step races into SQLITE_BUSY. + let threadSnapshots: ThreadSnapshot[] = []; + if (probe.path && existsSync(probe.path)) { + let db: Database | undefined; + try { + db = openStateDbWritable(probe.path, busyTimeoutMs); + threadSnapshots = loadMatchingThreads(db, preview.candidates, codexHome); + } catch (error) { + if (isBusyError(error)) { + return { + ok: false, + mode, + percent, + count: 0, + bytes: 0, + removedPaths: [], + error: "codex_busy", + }; + } + } finally { + try { db?.close(); } catch { /* */ } + } + } + + const removedPaths: string[] = []; + let bytes = 0; + const epoch = options.now ?? Date.now(); + let trashDir: string | undefined; + let dbError: string | undefined; + + if (mode === "quarantine") { + trashDir = join(codexHome, TRASH_DIR, String(epoch)); + mkdirSync(trashDir, { recursive: true }); + const manifestEntries: CleanupManifestEntry[] = []; + for (const candidate of preview.candidates) { + const dest = join(trashDir, basename(candidate.relPath)); + renameSync(candidate.absPath, dest); + removedPaths.push(candidate.relPath); + bytes += candidate.bytes; + const thread = threadSnapshots.find(t => rolloutPathMatches(candidate, t.rollout_path, codexHome)); + manifestEntries.push({ + relPath: candidate.relPath, + bytes: candidate.bytes, + mtimeMs: candidate.mtimeMs, + ...(thread + ? { threadId: thread.id, rolloutPath: thread.rollout_path, archived: thread.archived } + : {}), + }); + } + writeFileSync( + join(trashDir, "manifest.json"), + JSON.stringify({ + quarantinedAt: epoch, + mode: "quarantine", + percent, + entries: manifestEntries, + }, null, 2), + ); + const dbResult = reconcileThreads(probe.path, preview.candidates, codexHome, busyTimeoutMs); + dbError = dbResult.error; + } else { + // Permanent: reconcile DB first so a lock cannot leave files deleted with rows intact. + const dbResult = reconcileThreads(probe.path, preview.candidates, codexHome, busyTimeoutMs); + if (dbResult.error === "codex_busy") { + return { + ok: false, + mode, + percent, + count: 0, + bytes: 0, + removedPaths: [], + error: "codex_busy", + }; + } + dbError = dbResult.error; + for (const candidate of preview.candidates) { + unlinkSync(candidate.absPath); + removedPaths.push(candidate.relPath); + bytes += candidate.bytes; + } + } + + return { + ok: true, + mode, + percent, + count: removedPaths.length, + bytes, + ...(trashDir ? { trashDir: toForwardSlash(relative(codexHome, trashDir) || trashDir) } : {}), + ...(dbError ? { error: dbError } : {}), + removedPaths, + }; +} diff --git a/src/storage/scanner.ts b/src/storage/scanner.ts index ba220ca70..546d461c5 100644 --- a/src/storage/scanner.ts +++ b/src/storage/scanner.ts @@ -16,7 +16,7 @@ const IMMUTABLE_READONLY_FLAGS = constants.SQLITE_OPEN_READONLY | constants.SQLI /** * Read-only CODEX_HOME storage scanner — Phase 1 of the Storage page epic * (devlog/_plan/500_storage-page-session-cleanup). Pure measurement: sizes via - * fs.stat walks, DB row counts via short-timeout readonly opens that degrade to + * fs.stat walks, DB row counts via immutable readonly opens that degrade to * null on lock/corruption. Performs zero writes under CODEX_HOME. */ @@ -199,6 +199,8 @@ export function scanStorage(codexHome: string = resolveCodexHomeDir()): StorageR continue; } if (stat.isDirectory()) { + // Quarantine trash (Phase 2) must not inflate "other" or totals. + if (name === ".trash") continue; walkFiles(full, name, files[DIR_BUCKETS[name] ?? "other"]); } else if (stat.isFile()) { const key: StorageBucketKey = STATE_DB_FILE.test(name) ? "state_db" : LOGS_DB_FILE.test(name) ? "logs_db" : "other"; diff --git a/tests/api-storage-cleanup.test.ts b/tests/api-storage-cleanup.test.ts new file mode 100644 index 000000000..c4ef3e87b --- /dev/null +++ b/tests/api-storage-cleanup.test.ts @@ -0,0 +1,117 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { Database } from "bun:sqlite"; +import { existsSync, mkdirSync, mkdtempSync, rmSync, utimesSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { saveConfig } from "../src/config"; +import { startServer } from "../src/server"; +import type { OcxConfig } from "../src/types"; +import { installIsolatedCodexHome, type IsolatedCodexHome } from "./helpers/isolated-codex-home"; + +let testDir = ""; +let previousHome: string | undefined; +let isolatedCodexHome: IsolatedCodexHome | null = null; + +function baseConfig(): OcxConfig { + return { + port: 0, + hostname: "127.0.0.1", + defaultProvider: "openai", + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://api.openai.com/v1", + authMode: "forward", + }, + }, + } as OcxConfig; +} + +function seedArchived(codexHome: string): void { + mkdirSync(join(codexHome, "archived_sessions")); + writeFileSync(join(codexHome, "archived_sessions", "rollout-old.jsonl"), "o".repeat(100)); + writeFileSync(join(codexHome, "archived_sessions", "rollout-new.jsonl"), "n".repeat(200)); + utimesSync(join(codexHome, "archived_sessions", "rollout-old.jsonl"), new Date("2026-01-01"), new Date("2026-01-01")); + utimesSync(join(codexHome, "archived_sessions", "rollout-new.jsonl"), new Date("2026-06-01"), new Date("2026-06-01")); + const db = new Database(join(codexHome, "state_5.sqlite")); + db.exec(`CREATE TABLE threads (id TEXT PRIMARY KEY, rollout_path TEXT NOT NULL, archived INTEGER)`); + db.exec(`INSERT INTO threads VALUES + ('told','archived_sessions/rollout-old.jsonl',1), + ('tnew','archived_sessions/rollout-new.jsonl',1) + `); + db.close(); +} + +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; + isolatedCodexHome = installIsolatedCodexHome("ocx-api-storage-cleanup-codex-"); + testDir = mkdtempSync(join(tmpdir(), "ocx-api-storage-cleanup-")); + process.env.OPENCODEX_HOME = testDir; + saveConfig(baseConfig()); +}); + +afterEach(() => { + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + isolatedCodexHome?.restore(); + isolatedCodexHome = null; + if (testDir) rmSync(testDir, { recursive: true, force: true }); + testDir = ""; +}); + +describe("POST /api/storage/cleanup", () => { + test("preview returns oldest percent without mutating", async () => { + seedArchived(isolatedCodexHome!.path); + const server = startServer(0); + try { + const res = await fetch(new URL("/api/storage/cleanup/preview", server.url), { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ percent: 50 }), + }); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.count).toBe(1); + expect(body.candidates[0].relPath).toBe("archived_sessions/rollout-old.jsonl"); + expect(existsSync(join(isolatedCodexHome!.path, "archived_sessions", "rollout-old.jsonl"))).toBe(true); + } finally { + await server.stop(true); + } + }); + + test("quarantine mode moves files and returns trashDir", async () => { + seedArchived(isolatedCodexHome!.path); + const server = startServer(0); + try { + const res = await fetch(new URL("/api/storage/cleanup", server.url), { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ percent: 50, mode: "quarantine" }), + }); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.ok).toBe(true); + expect(body.mode).toBe("quarantine"); + expect(body.count).toBe(1); + expect(body.trashDir).toContain(".trash/"); + expect(existsSync(join(isolatedCodexHome!.path, "archived_sessions", "rollout-old.jsonl"))).toBe(false); + expect(existsSync(join(isolatedCodexHome!.path, "archived_sessions", "rollout-new.jsonl"))).toBe(true); + } finally { + await server.stop(true); + } + }); + + test("rejects invalid mode", async () => { + const server = startServer(0); + try { + const res = await fetch(new URL("/api/storage/cleanup", server.url), { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ percent: 10, mode: "yeet" }), + }); + expect(res.status).toBe(400); + } finally { + await server.stop(true); + } + }); +}); diff --git a/tests/storage-cleanup.test.ts b/tests/storage-cleanup.test.ts new file mode 100644 index 000000000..38032b373 --- /dev/null +++ b/tests/storage-cleanup.test.ts @@ -0,0 +1,146 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { Database } from "bun:sqlite"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, utimesSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + executeArchivedCleanup, + listArchivedCandidates, + previewArchivedCleanup, + selectOldestPercent, +} from "../src/storage/cleanup"; + +const OLD = new Date("2026-01-01T00:00:00Z"); +const MID = new Date("2026-02-01T00:00:00Z"); +const NEW = new Date("2026-03-01T00:00:00Z"); + +let home = ""; + +afterEach(() => { + if (home) { + try { rmSync(home, { recursive: true, force: true }); } catch { /* */ } + home = ""; + } +}); + +function buildHome(): string { + const dir = mkdtempSync(join(tmpdir(), "ocx-cleanup-")); + mkdirSync(join(dir, "sessions", "2026", "05", "27"), { recursive: true }); + writeFileSync(join(dir, "sessions", "2026", "05", "27", "rollout-active.jsonl"), "ACTIVE".repeat(20)); + + mkdirSync(join(dir, "archived_sessions")); + writeFileSync(join(dir, "archived_sessions", "rollout-old.jsonl"), "OLD".repeat(10)); + writeFileSync(join(dir, "archived_sessions", "rollout-mid.jsonl"), "MID".repeat(20)); + writeFileSync(join(dir, "archived_sessions", "rollout-new.jsonl"), "NEW".repeat(30)); + utimesSync(join(dir, "archived_sessions", "rollout-old.jsonl"), OLD, OLD); + utimesSync(join(dir, "archived_sessions", "rollout-mid.jsonl"), MID, MID); + utimesSync(join(dir, "archived_sessions", "rollout-new.jsonl"), NEW, NEW); + + const db = new Database(join(dir, "state_5.sqlite")); + db.exec(`CREATE TABLE threads ( + id TEXT PRIMARY KEY, + rollout_path TEXT NOT NULL, + archived INTEGER, + archived_at INTEGER + )`); + db.exec(`INSERT INTO threads VALUES + ('active','sessions/2026/05/27/rollout-active.jsonl',0,NULL), + ('told','archived_sessions/rollout-old.jsonl',1,1), + ('tmid','archived_sessions/rollout-mid.jsonl',1,2), + ('tnew','archived_sessions/rollout-new.jsonl',1,3) + `); + db.close(); + return dir; +} + +describe("previewArchivedCleanup", () => { + test("lists archived files oldest-first and ignores active sessions", () => { + home = buildHome(); + const listed = listArchivedCandidates(home); + expect(listed.map(c => c.relPath)).toEqual([ + "archived_sessions/rollout-old.jsonl", + "archived_sessions/rollout-mid.jsonl", + "archived_sessions/rollout-new.jsonl", + ]); + expect(listed.some(c => c.relPath.includes("sessions/2026"))).toBe(false); + }); + + test("percent selects oldest subset", () => { + home = buildHome(); + const all = listArchivedCandidates(home); + expect(selectOldestPercent(all, 0)).toEqual([]); + expect(selectOldestPercent(all, 34).map(c => c.relPath)).toEqual([ + "archived_sessions/rollout-old.jsonl", + ]); + expect(selectOldestPercent(all, 100)).toHaveLength(3); + const preview = previewArchivedCleanup(50, home); + expect(preview.count).toBe(1); + expect(preview.candidates[0]!.relPath).toBe("archived_sessions/rollout-old.jsonl"); + expect(preview.bytes).toBe(preview.candidates[0]!.bytes); + }); +}); + +describe("executeArchivedCleanup", () => { + test("quarantine moves files to .trash and removes matching threads", () => { + home = buildHome(); + const result = executeArchivedCleanup({ + percent: 50, + mode: "quarantine", + codexHome: home, + now: 1_700_000_000_000, + }); + expect(result.ok).toBe(true); + expect(result.count).toBe(1); + expect(result.removedPaths).toEqual(["archived_sessions/rollout-old.jsonl"]); + expect(existsSync(join(home, "archived_sessions", "rollout-old.jsonl"))).toBe(false); + expect(existsSync(join(home, "sessions", "2026", "05", "27", "rollout-active.jsonl"))).toBe(true); + expect(existsSync(join(home, "archived_sessions", "rollout-mid.jsonl"))).toBe(true); + const trashFile = join(home, ".trash", "1700000000000", "rollout-old.jsonl"); + expect(existsSync(trashFile)).toBe(true); + const manifest = JSON.parse(readFileSync(join(home, ".trash", "1700000000000", "manifest.json"), "utf8")); + expect(manifest.entries[0].threadId).toBe("told"); + + const db = new Database(join(home, "state_5.sqlite"), { readonly: true }); + const ids = db.query<{ id: string }, []>("SELECT id FROM threads ORDER BY id").all().map(r => r.id); + db.close(); + expect(ids).toEqual(["active", "tmid", "tnew"]); + }); + + test("permanent deletes files and threads without creating trash", () => { + home = buildHome(); + const result = executeArchivedCleanup({ percent: 100, mode: "permanent", codexHome: home }); + expect(result.ok).toBe(true); + expect(result.count).toBe(3); + expect(existsSync(join(home, "archived_sessions", "rollout-old.jsonl"))).toBe(false); + expect(existsSync(join(home, "archived_sessions", "rollout-new.jsonl"))).toBe(false); + expect(existsSync(join(home, ".trash"))).toBe(false); + expect(existsSync(join(home, "sessions", "2026", "05", "27", "rollout-active.jsonl"))).toBe(true); + + const db = new Database(join(home, "state_5.sqlite"), { readonly: true }); + const ids = db.query<{ id: string }, []>("SELECT id FROM threads").all().map(r => r.id); + db.close(); + expect(ids).toEqual(["active"]); + }); + + test("codex_busy probe skips all filesystem mutations", () => { + home = buildHome(); + const locker = new Database(join(home, "state_5.sqlite")); + locker.exec("BEGIN EXCLUSIVE"); + try { + const result = executeArchivedCleanup({ + percent: 100, + mode: "quarantine", + codexHome: home, + busyTimeoutMs: 1, + }); + expect(result.ok).toBe(false); + expect(result.error).toBe("codex_busy"); + expect(result.count).toBe(0); + expect(existsSync(join(home, "archived_sessions", "rollout-old.jsonl"))).toBe(true); + expect(existsSync(join(home, ".trash"))).toBe(false); + } finally { + locker.exec("ROLLBACK"); + locker.close(); + } + }); +}); diff --git a/tests/storage-scanner.test.ts b/tests/storage-scanner.test.ts index 05e1f93c0..c906b313b 100644 --- a/tests/storage-scanner.test.ts +++ b/tests/storage-scanner.test.ts @@ -254,4 +254,17 @@ describe("scanStorage", () => { expect(after.get(path)).toEqual(stat); } }, 15_000); + + test("skips .trash quarantine directory (not counted in other or totals)", () => { + fixtureHome = buildFixtureHome(); + const withoutTrash = scanStorage(fixtureHome); + mkdirSync(join(fixtureHome, ".trash", "123"), { recursive: true }); + writeFileSync(join(fixtureHome, ".trash", "123", "rollout-quarantined.jsonl"), "q".repeat(5000)); + const withTrash = scanStorage(fixtureHome); + expect(withTrash.total.bytes).toBe(withoutTrash.total.bytes); + expect(withTrash.total.fileCount).toBe(withoutTrash.total.fileCount); + expect(bucket(withTrash, "other").bytes).toBe(bucket(withoutTrash, "other").bytes); + const otherPaths = (bucket(withTrash, "other").largest ?? []).map(e => e.path); + expect(otherPaths.some(p => p.includes(".trash"))).toBe(false); + }, 15_000); }); From 888e434fa33d597af061be31ab3681a9897ba86d Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Mon, 27 Jul 2026 04:56:30 +0200 Subject: [PATCH 02/17] fix(storage): address CodeRabbit review on cleanup APIs and i18n Strip host paths from preview JSON, harden FS/DB error mapping, localize cleanup copy, and cover permanent/busy API paths. --- .../src/content/docs/guides/web-dashboard.md | 2 +- .../content/docs/ja/guides/web-dashboard.md | 1 + .../content/docs/ko/guides/web-dashboard.md | 1 + .../content/docs/ru/guides/web-dashboard.md | 1 + .../docs/zh-cn/guides/web-dashboard.md | 1 + gui/src/i18n/de.ts | 36 ++--- gui/src/i18n/en.ts | 2 + gui/src/i18n/ja.ts | 36 ++--- gui/src/i18n/ko.ts | 36 ++--- gui/src/i18n/ru.ts | 36 ++--- gui/src/i18n/zh.ts | 36 ++--- gui/src/pages/Storage.tsx | 31 ++++- src/server/management/logs-usage-routes.ts | 31 +++-- src/storage/cleanup.ts | 130 ++++++++++++++---- src/storage/scanner.ts | 3 +- tests/api-storage-cleanup.test.ts | 47 +++++++ 16 files changed, 300 insertions(+), 130 deletions(-) diff --git a/docs-site/src/content/docs/guides/web-dashboard.md b/docs-site/src/content/docs/guides/web-dashboard.md index 2fec2442f..41b5f24b3 100644 --- a/docs-site/src/content/docs/guides/web-dashboard.md +++ b/docs-site/src/content/docs/guides/web-dashboard.md @@ -39,7 +39,7 @@ bun run dev:gui | **Models** | Toggle native GPT and routed models, set provider allowlists and context caps, choose v1/base/v2, and configure the v2 thread limit. Configured providers stay visible as zero-model groups when discovery is off or returns no rows. | | **Logs** | Auto-refresh recent requests with tokens, requested effort and (when available) effective outbound effort, resolved model, provider, status, request id, duration, and error details. The detail view includes the exact reasoning wire field when the adapter emits one. Filter by opaque conversation/session id (when the client sends one) to total tokens and estimated list-price cost for the currently loaded Logs ring. | | **Usage / Debug** | Inspect token-usage coverage and trends, or enable opt-in provider transport and usage-extraction diagnostics. | -| **Storage** | Read-only CODEX_HOME disk breakdown (sessions, archives, DBs, attachments). Optional archived cleanup: preview the oldest N%, then quarantine to `CODEX_HOME/.trash` (default) or permanently delete behind an explicit checkbox. Active sessions stay read-only. Cleanup is refused while Codex holds `state_*.sqlite` locked. | +| **Storage** | Read-only CODEX_HOME disk breakdown (sessions, archives, DBs, attachments). Optional archived cleanup: preview the oldest N%, then quarantine to `CODEX_HOME/.trash` (default) or permanently delete behind an explicit checkbox. Active sessions stay read-only. Cleanup is refused while Codex holds `state_*.sqlite` locked. Quarantined files are **not** restorable from the dashboard — recover manually from `.trash//` using `manifest.json` if needed. | | **Stop** | Gracefully stop the proxy and installed background service, restore native Codex, and exit (`POST /api/stop`). | ### Linking to a section diff --git a/docs-site/src/content/docs/ja/guides/web-dashboard.md b/docs-site/src/content/docs/ja/guides/web-dashboard.md index 26d466b8d..931ce4584 100644 --- a/docs-site/src/content/docs/ja/guides/web-dashboard.md +++ b/docs-site/src/content/docs/ja/guides/web-dashboard.md @@ -39,6 +39,7 @@ bun run dev:gui | **モデル** | ネイティブ GPT とルーティングモデルをオン/オフし、プロバイダー許可リストとコンテキスト上限、v1/base/v2、v2 スレッド数を設定します。 | | **ログ** | トークン、要求された強度と(利用可能な場合は)実際に送信された強度、実際のモデル、プロバイダー、状態、リクエスト ID、所要時間、エラー詳細を含む最近のリクエストを自動更新します。アダプターが reasoning パラメーターを送信した場合、詳細表示に正確な wire field も表示されます。 | | **使用量 / デバッグ** | トークン使用量の測定範囲と推移を見るか、オプションのプロバイダートランスポート/使用量抽出診断をオンにします。 | +| **ストレージ** | CODEX_HOME のディスク内訳(セッション、アーカイブ、DB、添付)を読み取り専用で表示。任意のアーカイブクリーンアップ: 最古 N% をプレビューし、既定では `CODEX_HOME/.trash` へ隔離、または明示チェックで完全削除。アクティブセッションは読み取り専用。`state_*.sqlite` がロック中は拒否。隔離分はダッシュボードから復元不可 — 必要なら `.trash//` と `manifest.json` から手動復旧。 | | **停止** | プロキシとインストールされたバックグラウンドサービスを正常終了しネイティブ Codex を復元した後終了します(`POST /api/stop`)。 | ### セクションへのリンク diff --git a/docs-site/src/content/docs/ko/guides/web-dashboard.md b/docs-site/src/content/docs/ko/guides/web-dashboard.md index db208c3d3..95a04edbe 100644 --- a/docs-site/src/content/docs/ko/guides/web-dashboard.md +++ b/docs-site/src/content/docs/ko/guides/web-dashboard.md @@ -39,6 +39,7 @@ bun run dev:gui | **Models** | 네이티브 GPT와 라우팅 모델을 켜고 끄고, 프로바이더 allowlist와 컨텍스트 상한, v1/base/v2, v2 thread 수를 설정합니다. | | **Logs** | 토큰, 요청한 강도와 (사용 가능한 경우) 실제 전송 강도, 실제 모델, 프로바이더, 상태, 요청 id, 소요 시간, 오류 상세가 포함된 최근 요청을 자동 갱신합니다. 어댑터가 reasoning 매개변수를 전송한 경우 상세 보기에 정확한 wire field도 표시됩니다. 클라이언트가 보낸 불투명 대화/세션 id로 필터하면 현재 로드된 Logs 링의 토큰·추정 정가 합계를 볼 수 있습니다. | | **Usage / Debug** | 토큰 사용량의 측정 범위와 추이를 보거나, 선택적 프로바이더 전송/사용량 추출 진단을 켭니다. | +| **Storage** | CODEX_HOME 디스크 사용량(세션, 보관, DB, 첨부)을 읽기 전용으로 표시합니다. 선택적 보관 정리: 가장 오래된 N%를 미리본 뒤 기본으로 `CODEX_HOME/.trash`에 격리하거나, 명시 체크 후 영구 삭제합니다. 활성 세션은 읽기 전용입니다. Codex가 `state_*.sqlite`를 잠그면 거절합니다. 격리 파일은 대시보드에서 복원할 수 없습니다 — 필요하면 `.trash//`와 `manifest.json`으로 수동 복구하세요. | | **Stop** | 프록시와 설치된 백그라운드 서비스를 정상 종료하고 네이티브 Codex를 복원한 뒤 끝냅니다(`POST /api/stop`). | ### 섹션으로 바로 가기 diff --git a/docs-site/src/content/docs/ru/guides/web-dashboard.md b/docs-site/src/content/docs/ru/guides/web-dashboard.md index cafaf4196..d456167b3 100644 --- a/docs-site/src/content/docs/ru/guides/web-dashboard.md +++ b/docs-site/src/content/docs/ru/guides/web-dashboard.md @@ -39,6 +39,7 @@ bun run dev:gui | **Models** | Включение и отключение нативных GPT и маршрутизируемых моделей, настройка списков разрешённых провайдеров и лимитов контекста, выбор v1/base/v2 и настройка лимита потоков v2. | | **Logs** | Автообновляемый список недавних запросов: токены, запрошенный и, когда доступен, фактически отправленный уровень рассуждений, фактическая модель, провайдер, статус, id запроса, длительность и подробности ошибок. Если адаптер отправляет параметр рассуждений, в подробностях также отображается точное wire-поле. Можно фильтровать по непрозрачному id диалога/сессии (если клиент его передаёт) и суммировать токены и оценочную стоимость по прайс-листу в пределах загруженного кольца Logs. | | **Usage / Debug** | Просмотр покрытия и трендов расхода токенов либо включение опциональной диагностики транспорта провайдеров и извлечения данных об использовании. | +| **Storage** | Только чтение разбивки диска CODEX_HOME (сессии, архивы, БД, вложения). Опциональная очистка архива: предпросмотр самых старых N%, затем карантин в `CODEX_HOME/.trash` (по умолчанию) или безвозвратное удаление по явному флажку. Активные сессии только для чтения. Очистка отклоняется, пока Codex держит блокировку `state_*.sqlite`. Из карантина **нельзя** восстановить через панель — вручную из `.trash//` и `manifest.json`. | | **Stop** | Корректная остановка прокси и установленного фонового сервиса, восстановление нативного Codex и выход (`POST /api/stop`). | ### Ссылки на разделы diff --git a/docs-site/src/content/docs/zh-cn/guides/web-dashboard.md b/docs-site/src/content/docs/zh-cn/guides/web-dashboard.md index b096602dc..1cfc233a4 100644 --- a/docs-site/src/content/docs/zh-cn/guides/web-dashboard.md +++ b/docs-site/src/content/docs/zh-cn/guides/web-dashboard.md @@ -38,6 +38,7 @@ bun run dev:gui | **Models** | 开关原生 GPT 与路由模型,配置 provider allowlist、上下文上限、v1/base/v2 以及 v2 thread 数量。 | | **Logs** | 自动刷新近期请求,显示 token、请求强度以及(可用时)实际发送强度、实际模型、provider、状态、request id、耗时和错误详情。适配器发送 reasoning 参数时,详情中还会显示准确的 wire field。可按不透明会话/对话 ID(客户端提供时)筛选,并对当前已加载的 Logs 环形缓冲合计 token 与估算标价成本。 | | **Usage / Debug** | 查看 token usage 覆盖率与趋势,或启用可选的 provider transport 和 usage 提取诊断。 | +| **Storage** | 只读查看 CODEX_HOME 磁盘占用(会话、归档、数据库、附件)。可选归档清理:预览最旧 N%,默认隔离到 `CODEX_HOME/.trash`,或勾选后永久删除。活动会话保持只读。Codex 锁定 `state_*.sqlite` 时拒绝清理。隔离文件**不能**从仪表盘恢复——如需恢复请手动查看 `.trash//` 与 `manifest.json`。 | | **Stop** | 优雅地停止代理和已安装的后台服务,恢复原生 Codex 并退出(`POST /api/stop`)。 | ### 链接到某个部分 diff --git a/gui/src/i18n/de.ts b/gui/src/i18n/de.ts index 92fe15a24..e4ec69cf2 100644 --- a/gui/src/i18n/de.ts +++ b/gui/src/i18n/de.ts @@ -864,7 +864,7 @@ export const de: Record = { "nav.storage": "Speicher", "storage.title": "Speicher", - "storage.subtitle": "Diagnostics for CODEX_HOME disk use. Archived cleanup below can quarantine or permanently delete oldest archived sessions — active sessions stay read-only.", + "storage.subtitle": "Diagnose zur CODEX_HOME-Festplattennutzung. Die Archivbereinigung unten kann älteste archivierte Sitzungen in Quarantäne legen oder dauerhaft löschen — aktive Sitzungen bleiben schreibgeschützt.", "storage.loading": "Speicher wird gescannt…", "storage.empty": "CODEX_HOME ist leer oder fehlt — nichts zu berichten.", "storage.error": "Speicher-Scan fehlgeschlagen. Prüfe, ob CODEX_HOME auf ein gültiges Verzeichnis zeigt.", @@ -888,22 +888,24 @@ export const de: Record = { "storage.bucket.attachments": "Anhänge", "storage.bucket.deletion_manifests": "Lösch-Manifeste", "storage.bucket.other": "Sonstiges", - "storage.cleanup.title": "Archived cleanup", - "storage.cleanup.help": "Remove the oldest archived sessions by percentage. Active sessions are never touched. Quarantine is the default — files move to CODEX_HOME/.trash.", - "storage.cleanup.slider": "Oldest archived percent", - "storage.cleanup.percent": "Oldest {percent}%", - "storage.cleanup.preview": "Preview", - "storage.cleanup.confirmTitle": "Confirm archived cleanup", - "storage.cleanup.confirmBody": "This will process {count} archived file(s) (~{size}), the oldest {percent}%.", - "storage.cleanup.moreFiles": "…and {n} more", - "storage.cleanup.permanent": "Delete permanently (skip quarantine)", - "storage.cleanup.permanentWarn": "Permanent delete cannot be undone.", - "storage.cleanup.quarantineNote": "Files move to .trash under CODEX_HOME. Restore is not available in this release — keep the trash folder until you are sure.", - "storage.cleanup.cancel": "Cancel", - "storage.cleanup.confirmQuarantine": "Quarantine", - "storage.cleanup.confirmPermanent": "Delete permanently", - "storage.cleanup.doneQuarantine": "Quarantined {count} file(s) ({size}).", - "storage.cleanup.donePermanent": "Permanently deleted {count} file(s) ({size}).", + "storage.cleanup.title": "Archivbereinigung", + "storage.cleanup.help": "Entfernt die ältesten archivierten Sitzungen nach Prozentsatz. Aktive Sitzungen werden nie angefasst. Standard ist Quarantäne — Dateien wandern nach CODEX_HOME/.trash.", + "storage.cleanup.slider": "Ältester Archivanteil", + "storage.cleanup.percent": "Älteste {percent}%", + "storage.cleanup.preview": "Vorschau", + "storage.cleanup.confirmTitle": "Archivbereinigung bestätigen", + "storage.cleanup.confirmBody": "Es werden {count} archivierte Datei(en) (~{size}) verarbeitet, die ältesten {percent}%.", + "storage.cleanup.moreFiles": "…und {n} weitere", + "storage.cleanup.permanent": "Dauerhaft löschen (ohne Quarantäne)", + "storage.cleanup.permanentWarn": "Dauerhaftes Löschen kann nicht rückgängig gemacht werden.", + "storage.cleanup.quarantineNote": "Dateien wandern nach .trash unter CODEX_HOME. Wiederherstellen gibt es in dieser Version nicht — behalte den Ordner, bis du sicher bist.", + "storage.cleanup.cancel": "Abbrechen", + "storage.cleanup.confirmQuarantine": "In Quarantäne", + "storage.cleanup.confirmPermanent": "Dauerhaft löschen", + "storage.cleanup.doneQuarantine": "{count} Datei(en) in Quarantäne ({size}).", + "storage.cleanup.donePermanent": "{count} Datei(en) dauerhaft gelöscht ({size}).", + "storage.cleanup.previewFailed": "Vorschau fehlgeschlagen.", + "storage.cleanup.cleanupFailed": "Bereinigung fehlgeschlagen.", "modal.back": "Zurück", "modal.badge.oauth": "OAuth", "modal.customProvider": "Benutzerdefinierter Anbieter", diff --git a/gui/src/i18n/en.ts b/gui/src/i18n/en.ts index 03ea3c214..0c9dd1621 100644 --- a/gui/src/i18n/en.ts +++ b/gui/src/i18n/en.ts @@ -653,6 +653,8 @@ export const en = { "storage.cleanup.confirmPermanent": "Delete permanently", "storage.cleanup.doneQuarantine": "Quarantined {count} file(s) ({size}).", "storage.cleanup.donePermanent": "Permanently deleted {count} file(s) ({size}).", + "storage.cleanup.previewFailed": "Preview failed.", + "storage.cleanup.cleanupFailed": "Cleanup failed.", // add-provider modal "modal.addNamed": "Add: {label}", diff --git a/gui/src/i18n/ja.ts b/gui/src/i18n/ja.ts index 3d4d8aed8..fb56f774d 100644 --- a/gui/src/i18n/ja.ts +++ b/gui/src/i18n/ja.ts @@ -577,7 +577,7 @@ export const ja: Record = { "nav.storage": "ストレージ", "storage.title": "ストレージ", - "storage.subtitle": "Diagnostics for CODEX_HOME disk use. Archived cleanup below can quarantine or permanently delete oldest archived sessions — active sessions stay read-only.", + "storage.subtitle": "CODEX_HOME のディスク使用診断。下のアーカイブクリーンアップで古いアーカイブを隔離または完全削除できます — アクティブセッションは読み取り専用のままです。", "storage.loading": "ストレージをスキャン中…", "storage.empty": "CODEX_HOME が空か存在しません — 報告するものはありません。", "storage.error": "ストレージのスキャンに失敗しました。CODEX_HOME が有効なディレクトリを指しているか確認してください。", @@ -601,22 +601,24 @@ export const ja: Record = { "storage.bucket.attachments": "添付", "storage.bucket.deletion_manifests": "削除マニフェスト", "storage.bucket.other": "その他", - "storage.cleanup.title": "Archived cleanup", - "storage.cleanup.help": "Remove the oldest archived sessions by percentage. Active sessions are never touched. Quarantine is the default — files move to CODEX_HOME/.trash.", - "storage.cleanup.slider": "Oldest archived percent", - "storage.cleanup.percent": "Oldest {percent}%", - "storage.cleanup.preview": "Preview", - "storage.cleanup.confirmTitle": "Confirm archived cleanup", - "storage.cleanup.confirmBody": "This will process {count} archived file(s) (~{size}), the oldest {percent}%.", - "storage.cleanup.moreFiles": "…and {n} more", - "storage.cleanup.permanent": "Delete permanently (skip quarantine)", - "storage.cleanup.permanentWarn": "Permanent delete cannot be undone.", - "storage.cleanup.quarantineNote": "Files move to .trash under CODEX_HOME. Restore is not available in this release — keep the trash folder until you are sure.", - "storage.cleanup.cancel": "Cancel", - "storage.cleanup.confirmQuarantine": "Quarantine", - "storage.cleanup.confirmPermanent": "Delete permanently", - "storage.cleanup.doneQuarantine": "Quarantined {count} file(s) ({size}).", - "storage.cleanup.donePermanent": "Permanently deleted {count} file(s) ({size}).", + "storage.cleanup.title": "アーカイブのクリーンアップ", + "storage.cleanup.help": "古いアーカイブセッションを割合で削除します。アクティブセッションには触れません。既定は隔離で、ファイルは CODEX_HOME/.trash へ移動します。", + "storage.cleanup.slider": "古いアーカイブの割合", + "storage.cleanup.percent": "古い {percent}%", + "storage.cleanup.preview": "プレビュー", + "storage.cleanup.confirmTitle": "アーカイブクリーンアップの確認", + "storage.cleanup.confirmBody": "アーカイブ {count} 件(約 {size})、古い {percent}% を処理します。", + "storage.cleanup.moreFiles": "…ほか {n} 件", + "storage.cleanup.permanent": "完全に削除する(隔離しない)", + "storage.cleanup.permanentWarn": "完全削除は元に戻せません。", + "storage.cleanup.quarantineNote": "ファイルは CODEX_HOME 下の .trash へ移動します。この版ではダッシュボードから復元できません — 確信できるまでゴミ箱を残してください。", + "storage.cleanup.cancel": "キャンセル", + "storage.cleanup.confirmQuarantine": "隔離する", + "storage.cleanup.confirmPermanent": "完全に削除", + "storage.cleanup.doneQuarantine": "{count} 件を隔離しました({size})。", + "storage.cleanup.donePermanent": "{count} 件を完全削除しました({size})。", + "storage.cleanup.previewFailed": "プレビューに失敗しました。", + "storage.cleanup.cleanupFailed": "クリーンアップに失敗しました。", // add-provider modal "modal.addNamed": "追加: {label}", diff --git a/gui/src/i18n/ko.ts b/gui/src/i18n/ko.ts index 52877e5ec..9795cf0b4 100644 --- a/gui/src/i18n/ko.ts +++ b/gui/src/i18n/ko.ts @@ -884,7 +884,7 @@ export const ko: Record = { "nav.storage": "저장소", "storage.title": "저장소", - "storage.subtitle": "Diagnostics for CODEX_HOME disk use. Archived cleanup below can quarantine or permanently delete oldest archived sessions — active sessions stay read-only.", + "storage.subtitle": "CODEX_HOME 디스크 사용 진단. 아래 보관 정리로 가장 오래된 보관 세션을 격리하거나 영구 삭제할 수 있습니다 — 활성 세션은 읽기 전용입니다.", "storage.loading": "저장소 스캔 중…", "storage.empty": "CODEX_HOME이 비어 있거나 없습니다 — 표시할 내용이 없습니다.", "storage.error": "저장소 스캔에 실패했습니다. CODEX_HOME이 올바른 디렉터리를 가리키는지 확인하세요.", @@ -908,22 +908,24 @@ export const ko: Record = { "storage.bucket.attachments": "첨부 파일", "storage.bucket.deletion_manifests": "삭제 매니페스트", "storage.bucket.other": "기타", - "storage.cleanup.title": "Archived cleanup", - "storage.cleanup.help": "Remove the oldest archived sessions by percentage. Active sessions are never touched. Quarantine is the default — files move to CODEX_HOME/.trash.", - "storage.cleanup.slider": "Oldest archived percent", - "storage.cleanup.percent": "Oldest {percent}%", - "storage.cleanup.preview": "Preview", - "storage.cleanup.confirmTitle": "Confirm archived cleanup", - "storage.cleanup.confirmBody": "This will process {count} archived file(s) (~{size}), the oldest {percent}%.", - "storage.cleanup.moreFiles": "…and {n} more", - "storage.cleanup.permanent": "Delete permanently (skip quarantine)", - "storage.cleanup.permanentWarn": "Permanent delete cannot be undone.", - "storage.cleanup.quarantineNote": "Files move to .trash under CODEX_HOME. Restore is not available in this release — keep the trash folder until you are sure.", - "storage.cleanup.cancel": "Cancel", - "storage.cleanup.confirmQuarantine": "Quarantine", - "storage.cleanup.confirmPermanent": "Delete permanently", - "storage.cleanup.doneQuarantine": "Quarantined {count} file(s) ({size}).", - "storage.cleanup.donePermanent": "Permanently deleted {count} file(s) ({size}).", + "storage.cleanup.title": "보관 정리", + "storage.cleanup.help": "가장 오래된 보관 세션을 비율로 제거합니다. 활성 세션은 건드리지 않습니다. 기본은 격리이며 파일은 CODEX_HOME/.trash로 이동합니다.", + "storage.cleanup.slider": "오래된 보관 비율", + "storage.cleanup.percent": "오래된 {percent}%", + "storage.cleanup.preview": "미리보기", + "storage.cleanup.confirmTitle": "보관 정리 확인", + "storage.cleanup.confirmBody": "보관 파일 {count}개(약 {size}), 오래된 {percent}%를 처리합니다.", + "storage.cleanup.moreFiles": "…외 {n}개", + "storage.cleanup.permanent": "영구 삭제(격리 건너뛰기)", + "storage.cleanup.permanentWarn": "영구 삭제는 되돌릴 수 없습니다.", + "storage.cleanup.quarantineNote": "파일은 CODEX_HOME 아래 .trash로 이동합니다. 이 버전에서는 대시보드에서 복원할 수 없습니다 — 확신이 들 때까지 휴지통을 유지하세요.", + "storage.cleanup.cancel": "취소", + "storage.cleanup.confirmQuarantine": "격리", + "storage.cleanup.confirmPermanent": "영구 삭제", + "storage.cleanup.doneQuarantine": "파일 {count}개를 격리했습니다({size}).", + "storage.cleanup.donePermanent": "파일 {count}개를 영구 삭제했습니다({size}).", + "storage.cleanup.previewFailed": "미리보기에 실패했습니다.", + "storage.cleanup.cleanupFailed": "정리에 실패했습니다.", "modal.back": "뒤로", "modal.badge.oauth": "OAuth", "modal.customProvider": "사용자 지정 프로바이더", diff --git a/gui/src/i18n/ru.ts b/gui/src/i18n/ru.ts index 1614102aa..24188e20c 100644 --- a/gui/src/i18n/ru.ts +++ b/gui/src/i18n/ru.ts @@ -612,7 +612,7 @@ export const ru: Record = { "nav.storage": "Хранилище", "storage.title": "Хранилище", - "storage.subtitle": "Diagnostics for CODEX_HOME disk use. Archived cleanup below can quarantine or permanently delete oldest archived sessions — active sessions stay read-only.", + "storage.subtitle": "Диагностика диска CODEX_HOME. Ниже — очистка архива: карантин или безвозвратное удаление старых архивных сессий; активные сессии остаются только для чтения.", "storage.loading": "Сканирование хранилища…", "storage.empty": "CODEX_HOME пуст или отсутствует — показывать нечего.", "storage.error": "Не удалось просканировать хранилище. Убедитесь, что CODEX_HOME указывает на корректный каталог.", @@ -636,22 +636,24 @@ export const ru: Record = { "storage.bucket.attachments": "Вложения", "storage.bucket.deletion_manifests": "Манифесты удаления", "storage.bucket.other": "Прочее", - "storage.cleanup.title": "Archived cleanup", - "storage.cleanup.help": "Remove the oldest archived sessions by percentage. Active sessions are never touched. Quarantine is the default — files move to CODEX_HOME/.trash.", - "storage.cleanup.slider": "Oldest archived percent", - "storage.cleanup.percent": "Oldest {percent}%", - "storage.cleanup.preview": "Preview", - "storage.cleanup.confirmTitle": "Confirm archived cleanup", - "storage.cleanup.confirmBody": "This will process {count} archived file(s) (~{size}), the oldest {percent}%.", - "storage.cleanup.moreFiles": "…and {n} more", - "storage.cleanup.permanent": "Delete permanently (skip quarantine)", - "storage.cleanup.permanentWarn": "Permanent delete cannot be undone.", - "storage.cleanup.quarantineNote": "Files move to .trash under CODEX_HOME. Restore is not available in this release — keep the trash folder until you are sure.", - "storage.cleanup.cancel": "Cancel", - "storage.cleanup.confirmQuarantine": "Quarantine", - "storage.cleanup.confirmPermanent": "Delete permanently", - "storage.cleanup.doneQuarantine": "Quarantined {count} file(s) ({size}).", - "storage.cleanup.donePermanent": "Permanently deleted {count} file(s) ({size}).", + "storage.cleanup.title": "Очистка архива", + "storage.cleanup.help": "Удаляет самые старые архивные сессии по проценту. Активные сессии не затрагиваются. По умолчанию — карантин: файлы перемещаются в CODEX_HOME/.trash.", + "storage.cleanup.slider": "Доля самых старых архивов", + "storage.cleanup.percent": "Самые старые {percent}%", + "storage.cleanup.preview": "Предпросмотр", + "storage.cleanup.confirmTitle": "Подтвердить очистку архива", + "storage.cleanup.confirmBody": "Будет обработано {count} архивных файл(ов) (~{size}), самые старые {percent}%.", + "storage.cleanup.moreFiles": "…и ещё {n}", + "storage.cleanup.permanent": "Удалить навсегда (без карантина)", + "storage.cleanup.permanentWarn": "Безвозвратное удаление нельзя отменить.", + "storage.cleanup.quarantineNote": "Файлы перемещаются в .trash под CODEX_HOME. Восстановление из панели в этом выпуске недоступно — сохраняйте корзину, пока не уверены.", + "storage.cleanup.cancel": "Отмена", + "storage.cleanup.confirmQuarantine": "В карантин", + "storage.cleanup.confirmPermanent": "Удалить навсегда", + "storage.cleanup.doneQuarantine": "В карантин: {count} файл(ов) ({size}).", + "storage.cleanup.donePermanent": "Удалено навсегда: {count} файл(ов) ({size}).", + "storage.cleanup.previewFailed": "Не удалось выполнить предпросмотр.", + "storage.cleanup.cleanupFailed": "Не удалось выполнить очистку.", // add-provider modal "modal.addNamed": "Добавить: {label}", diff --git a/gui/src/i18n/zh.ts b/gui/src/i18n/zh.ts index 135f1d7e9..20619c49b 100644 --- a/gui/src/i18n/zh.ts +++ b/gui/src/i18n/zh.ts @@ -884,7 +884,7 @@ export const zh: Record = { "nav.storage": "存储", "storage.title": "存储", - "storage.subtitle": "Diagnostics for CODEX_HOME disk use. Archived cleanup below can quarantine or permanently delete oldest archived sessions — active sessions stay read-only.", + "storage.subtitle": "CODEX_HOME 磁盘占用诊断。下方归档清理可将最旧归档会话隔离或永久删除——活动会话保持只读。", "storage.loading": "正在扫描存储…", "storage.empty": "CODEX_HOME 为空或不存在——没有可显示的内容。", "storage.error": "存储扫描失败。请检查 CODEX_HOME 是否指向有效目录。", @@ -908,22 +908,24 @@ export const zh: Record = { "storage.bucket.attachments": "附件", "storage.bucket.deletion_manifests": "删除清单", "storage.bucket.other": "其他", - "storage.cleanup.title": "Archived cleanup", - "storage.cleanup.help": "Remove the oldest archived sessions by percentage. Active sessions are never touched. Quarantine is the default — files move to CODEX_HOME/.trash.", - "storage.cleanup.slider": "Oldest archived percent", - "storage.cleanup.percent": "Oldest {percent}%", - "storage.cleanup.preview": "Preview", - "storage.cleanup.confirmTitle": "Confirm archived cleanup", - "storage.cleanup.confirmBody": "This will process {count} archived file(s) (~{size}), the oldest {percent}%.", - "storage.cleanup.moreFiles": "…and {n} more", - "storage.cleanup.permanent": "Delete permanently (skip quarantine)", - "storage.cleanup.permanentWarn": "Permanent delete cannot be undone.", - "storage.cleanup.quarantineNote": "Files move to .trash under CODEX_HOME. Restore is not available in this release — keep the trash folder until you are sure.", - "storage.cleanup.cancel": "Cancel", - "storage.cleanup.confirmQuarantine": "Quarantine", - "storage.cleanup.confirmPermanent": "Delete permanently", - "storage.cleanup.doneQuarantine": "Quarantined {count} file(s) ({size}).", - "storage.cleanup.donePermanent": "Permanently deleted {count} file(s) ({size}).", + "storage.cleanup.title": "归档清理", + "storage.cleanup.help": "按百分比移除最旧的归档会话。不会触碰活动会话。默认隔离——文件移至 CODEX_HOME/.trash。", + "storage.cleanup.slider": "最旧归档百分比", + "storage.cleanup.percent": "最旧 {percent}%", + "storage.cleanup.preview": "预览", + "storage.cleanup.confirmTitle": "确认归档清理", + "storage.cleanup.confirmBody": "将处理 {count} 个归档文件(约 {size}),即最旧的 {percent}%。", + "storage.cleanup.moreFiles": "…以及另外 {n} 个", + "storage.cleanup.permanent": "永久删除(跳过隔离)", + "storage.cleanup.permanentWarn": "永久删除无法撤销。", + "storage.cleanup.quarantineNote": "文件会移到 CODEX_HOME 下的 .trash。本版本无法从仪表盘恢复——确定前请保留该文件夹。", + "storage.cleanup.cancel": "取消", + "storage.cleanup.confirmQuarantine": "隔离", + "storage.cleanup.confirmPermanent": "永久删除", + "storage.cleanup.doneQuarantine": "已隔离 {count} 个文件({size})。", + "storage.cleanup.donePermanent": "已永久删除 {count} 个文件({size})。", + "storage.cleanup.previewFailed": "预览失败。", + "storage.cleanup.cleanupFailed": "清理失败。", "modal.back": "返回", "modal.badge.oauth": "OAuth", "modal.customProvider": "自定义提供方", diff --git a/gui/src/pages/Storage.tsx b/gui/src/pages/Storage.tsx index 814b737f6..0f2b36119 100644 --- a/gui/src/pages/Storage.tsx +++ b/gui/src/pages/Storage.tsx @@ -147,6 +147,22 @@ function ArchivedCleanupPanel({ const [busy, setBusy] = useState(false); const [status, setStatus] = useState(null); const [error, setError] = useState(null); + const cancelRef = useRef(null); + const previousFocusRef = useRef(null); + + useEffect(() => { + if (!confirmOpen) return; + previousFocusRef.current = document.activeElement as HTMLElement | null; + cancelRef.current?.focus(); + const onKey = (e: KeyboardEvent) => { + if (e.key === "Escape" && !busy) setConfirmOpen(false); + }; + window.addEventListener("keydown", onKey); + return () => { + window.removeEventListener("keydown", onKey); + previousFocusRef.current?.focus(); + }; + }, [confirmOpen, busy]); const runPreview = async () => { setBusy(true); @@ -159,7 +175,7 @@ function ArchivedCleanupPanel({ body: JSON.stringify({ percent }), }); const json = await res.json() as CleanupPreview & { error?: string }; - if (!res.ok) throw new Error(json.error ?? "preview failed"); + if (!res.ok) throw new Error(json.error ?? t("storage.cleanup.previewFailed")); setPreview(json); setConfirmOpen(true); } catch (e) { @@ -181,7 +197,7 @@ function ArchivedCleanupPanel({ }); const json = await res.json() as CleanupResult; if (!res.ok || !json.ok) { - throw new Error(json.message ?? json.error ?? "cleanup failed"); + throw new Error(json.message ?? json.error ?? t("storage.cleanup.cleanupFailed")); } setConfirmOpen(false); setPreview(null); @@ -246,9 +262,8 @@ function ArchivedCleanupPanel({ aria-modal="true" aria-labelledby="storage-cleanup-confirm-title" onClick={() => !busy && setConfirmOpen(false)} - onKeyDown={e => { if (e.key === "Escape" && !busy) setConfirmOpen(false); }} > -
e.stopPropagation()} onKeyDown={e => e.stopPropagation()}> +
e.stopPropagation()}>

{t("storage.cleanup.confirmTitle")}

{t("storage.cleanup.confirmBody", { @@ -280,7 +295,13 @@ function ArchivedCleanupPanel({ {permanent ? t("storage.cleanup.permanentWarn") : t("storage.cleanup.quarantineNote")}

-
{status &&

{status}

} - {error &&

{error}

} + {error && !confirmOpen &&

{error}

} {confirmOpen && preview && (
!busy && setConfirmOpen(false)} > -
e.stopPropagation()}> +
e.stopPropagation()}>

{t("storage.cleanup.confirmTitle")}

{t("storage.cleanup.confirmBody", { @@ -294,6 +300,7 @@ function ArchivedCleanupPanel({

{permanent ? t("storage.cleanup.permanentWarn") : t("storage.cleanup.quarantineNote")}

+ {error &&

{error}

}
))}
@@ -294,7 +302,7 @@ function ArchivedCleanupPanel({ role="dialog" aria-modal="true" aria-labelledby="storage-cleanup-confirm-title" - onClick={() => !busy && setConfirmOpen(false)} + onClick={() => !busy && closeConfirm()} >
e.stopPropagation()}>

{t("storage.cleanup.confirmTitle")}

@@ -311,7 +319,7 @@ function ArchivedCleanupPanel({
  • {c.relPath}
  • ))} {preview.count > 8 && ( -
  • {t("storage.cleanup.moreFiles", { n: String(preview.count - 8) })}
  • +
  • {t("storage.cleanup.moreFiles", { n: String(Math.max(0, preview.count - 8)) })}
  • )} )} @@ -334,7 +342,7 @@ function ArchivedCleanupPanel({ type="button" className="btn btn-ghost" disabled={busy} - onClick={() => setConfirmOpen(false)} + onClick={() => closeConfirm()} > {t("storage.cleanup.cancel")} From 8f0ed9a9fa1ad011b27e5b32e90a50d6f76a7554 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Mon, 27 Jul 2026 06:21:04 +0200 Subject: [PATCH 09/17] fix(storage): reconcile logs/goals/memories on archived cleanup Match Codex delete_threads_strict across per-thread stores, fail closed on threads read errors before mutating files, and cover permanent multi-DB cleanup. --- src/storage/cleanup.ts | 282 +++++++++++++++++++++++++++------- tests/storage-cleanup.test.ts | 156 ++++++++++++++++++- 2 files changed, 378 insertions(+), 60 deletions(-) diff --git a/src/storage/cleanup.ts b/src/storage/cleanup.ts index 55f79c4db..7f4421629 100644 --- a/src/storage/cleanup.ts +++ b/src/storage/cleanup.ts @@ -6,8 +6,10 @@ * permanent delete is opt-in. * * Execution is bound to a preview digest. All candidates are staged first; any FS - * or DB failure rolls staged moves back. DB reconciliation runs in one transaction - * with foreign keys enabled. Success never carries soft `dbWarning` / `failedPaths`. + * or DB failure rolls staged moves back. DB reconciliation follows Codex + * `delete_threads_strict` order across the per-thread stores (`logs_*`, + * `memories_*`, `goals_*`, then `state_*`) and never soft-succeeds with + * `dbWarning` / `failedPaths`. */ import { createHash } from "node:crypto"; import { @@ -82,8 +84,15 @@ export interface CleanupResult { } const STATE_DB_FILE = /^state_(\d+)\.sqlite$/; +const LOGS_DB_FILE = /^logs_(\d+)\.sqlite$/; +const GOALS_DB_FILE = /^goals_(\d+)\.sqlite$/; +const MEMORIES_DB_FILE = /^memories_(\d+)\.sqlite$/; const JSONL_SUFFIX = ".jsonl"; const ZST_SUFFIX = ".jsonl.zst"; +const JOB_KIND_MEMORY_STAGE1 = "memory_stage1"; +const JOB_KIND_MEMORY_CONSOLIDATE_GLOBAL = "memory_consolidate_global"; +const MEMORY_CONSOLIDATION_JOB_KEY = "global"; +const DEFAULT_RETRY_REMAINING = 3; function clampPercent(percent: unknown): number { if (typeof percent !== "number" || !Number.isFinite(percent)) return 0; @@ -106,7 +115,8 @@ function isRolloutFileName(name: string): boolean { return name.endsWith(ZST_SUFFIX) || name.endsWith(JSONL_SUFFIX); } -function newestStateDb(codexHome: string): string | null { +/** Newest `prefix_N.sqlite` under CODEX_HOME, or null when absent. */ +function newestVersionedDb(codexHome: string, pattern: RegExp): string | null { let best: string | null = null; let bestVersion = -1; let names: string[] = []; @@ -116,7 +126,7 @@ function newestStateDb(codexHome: string): string | null { return null; } for (const name of names) { - const match = name.match(STATE_DB_FILE); + const match = name.match(pattern); if (!match) continue; const version = Number(match[1]); if (version > bestVersion) { @@ -127,6 +137,26 @@ function newestStateDb(codexHome: string): string | null { return best ? join(codexHome, best) : null; } +function newestStateDb(codexHome: string): string | null { + return newestVersionedDb(codexHome, STATE_DB_FILE); +} + +interface RuntimeDbPaths { + state: string | null; + logs: string | null; + goals: string | null; + memories: string | null; +} + +function discoverRuntimeDbPaths(codexHome: string): RuntimeDbPaths { + return { + state: newestVersionedDb(codexHome, STATE_DB_FILE), + logs: newestVersionedDb(codexHome, LOGS_DB_FILE), + goals: newestVersionedDb(codexHome, GOALS_DB_FILE), + memories: newestVersionedDb(codexHome, MEMORIES_DB_FILE), + }; +} + /** * Normalize a DB `rollout_path` to a CODEX_HOME-relative forward-slash path, then * to the logical `.jsonl` form. Returns null when the path is not under @@ -255,8 +285,8 @@ export function previewArchivedCleanup( }; } -function openStateDbWritable(stateDbPath: string, busyTimeoutMs = 100): Database { - const db = new Database(stateDbPath); +function openDbWritable(dbPath: string, busyTimeoutMs = 100): Database { + const db = new Database(dbPath); try { db.exec(`PRAGMA busy_timeout = ${busyTimeoutMs}`); } catch { @@ -285,25 +315,42 @@ function mapDbError(error: unknown): CleanupErrorCode { return "db_reconcile_failed"; } -/** True when the threads table can be written (BEGIN IMMEDIATE succeeds). */ -export function probeStateDbWritable(codexHome: string, busyTimeoutMs = 100): { ok: true; path: string } | { ok: false; error: CleanupErrorCode } { - const path = newestStateDb(codexHome); - if (!path || !existsSync(path)) return { ok: true, path: path ?? "" }; +/** Probe a single DB with BEGIN IMMEDIATE; missing path is a no-op success. */ +function probeDbWritable( + path: string | null, + busyTimeoutMs: number, +): { ok: true } | { ok: false; error: CleanupErrorCode } { + if (!path || !existsSync(path)) return { ok: true }; let db: Database | undefined; try { - db = openStateDbWritable(path, busyTimeoutMs); + db = openDbWritable(path, busyTimeoutMs); db.exec("BEGIN IMMEDIATE"); db.exec("ROLLBACK"); - return { ok: true, path }; + return { ok: true }; } catch (error) { if (isBusyError(error)) return { ok: false, error: "codex_busy" }; - // Corrupt / unreadable DB must block cleanup so we never orphan rows or files. return { ok: false, error: "db_reconcile_failed" }; } finally { try { db?.close(); } catch { /* */ } } } +/** + * True when every present Codex runtime DB can be written (BEGIN IMMEDIATE). + * Busy / corrupt stores abort cleanup before any filesystem mutation. + */ +export function probeStateDbWritable( + codexHome: string, + busyTimeoutMs = 100, +): { ok: true; path: string } | { ok: false; error: CleanupErrorCode } { + const paths = discoverRuntimeDbPaths(codexHome); + for (const path of [paths.state, paths.logs, paths.goals, paths.memories]) { + const probed = probeDbWritable(path, busyTimeoutMs); + if (!probed.ok) return probed; + } + return { ok: true, path: paths.state ?? "" }; +} + interface ThreadSnapshot { id: string; rollout_path: string; @@ -311,33 +358,30 @@ interface ThreadSnapshot { history_mode?: string | null; } +/** + * Load archived threads matching the candidate set. + * Optional columns are detected via PRAGMA; missing `threads` / query failures throw + * so callers map to `db_reconcile_failed` / `codex_busy` instead of treating them as empty. + */ function loadMatchingThreads(db: Database, candidates: ArchivedCandidate[], codexHome: string): ThreadSnapshot[] { - const logicalSet = new Set(candidates.map(c => c.relPath)); - let rows: Array<{ id: string; rollout_path: string; archived?: number | null; history_mode?: string | null }> = []; - try { - rows = db.query<{ id: string; rollout_path: string; archived: number | null; history_mode: string | null }, []>( - `SELECT id, rollout_path, archived, history_mode FROM threads`, - ).all(); - } catch { - try { - rows = db.query<{ id: string; rollout_path: string; archived: number | null }, []>( - `SELECT id, rollout_path, archived FROM threads`, - ).all().map(r => ({ ...r, history_mode: null })); - } catch { - try { - rows = db.query<{ id: string; rollout_path: string }, []>( - `SELECT id, rollout_path FROM threads`, - ).all().map(r => ({ ...r, archived: null, history_mode: null })); - } catch { - return []; - } - } + if (!tableExists(db, "threads")) { + throw new Error("missing_threads_table"); } + const logicalSet = new Set(candidates.map(c => c.relPath)); + const hasArchived = columnExists(db, "threads", "archived"); + const hasHistoryMode = columnExists(db, "threads", "history_mode"); + const selectCols = ["id", "rollout_path"]; + if (hasArchived) selectCols.push("archived"); + if (hasHistoryMode) selectCols.push("history_mode"); + const rows = db.query< + { id: string; rollout_path: string; archived?: number | null; history_mode?: string | null }, + [] + >(`SELECT ${selectCols.join(", ")} FROM threads`).all(); return rows .filter(row => { // When the archived column is present, only archived=1 rows may be deleted. - if (row.archived !== null && row.archived !== undefined && Number(row.archived) !== 1) { + if (hasArchived && Number(row.archived ?? 0) !== 1) { return false; } const normalized = normalizeArchivedRolloutPath(row.rollout_path, codexHome); @@ -346,8 +390,8 @@ function loadMatchingThreads(db: Database, candidates: ArchivedCandidate[], code .map(row => ({ id: row.id, rollout_path: row.rollout_path, - archived: row.archived ?? null, - history_mode: row.history_mode ?? null, + archived: hasArchived ? (row.archived ?? null) : null, + history_mode: hasHistoryMode ? (row.history_mode ?? null) : null, })); } @@ -397,14 +441,10 @@ function findReferencedHistory( } function tableExists(db: Database, name: string): boolean { - try { - const row = db.query<{ name: string }, [string]>( - `SELECT name FROM sqlite_master WHERE type='table' AND name=?`, - ).get(name); - return Boolean(row); - } catch { - return false; - } + const row = db.query<{ name: string }, [string]>( + `SELECT name FROM sqlite_master WHERE type='table' AND name=?`, + ).get(name); + return Boolean(row); } function columnExists(db: Database, table: string, column: string): boolean { @@ -420,7 +460,11 @@ function deleteThreadsAndDependents(db: Database, threadIds: string[]): void { if (threadIds.length === 0) return; const placeholders = threadIds.map(() => "?").join(","); - // Real Codex schema columns. + // Upstream deletes dynamic tools before spawn edges before threads. + if (tableExists(db, "thread_dynamic_tools")) { + db.run(`DELETE FROM thread_dynamic_tools WHERE thread_id IN (${placeholders})`, threadIds); + } + if (tableExists(db, "thread_spawn_edges")) { db.run( `DELETE FROM thread_spawn_edges WHERE parent_thread_id IN (${placeholders}) OR child_thread_id IN (${placeholders})`, @@ -428,11 +472,6 @@ function deleteThreadsAndDependents(db: Database, threadIds: string[]): void { ); } - // Explicit dependent cleanup even when FK pragma is ignored by older builds. - if (tableExists(db, "thread_dynamic_tools")) { - db.run(`DELETE FROM thread_dynamic_tools WHERE thread_id IN (${placeholders})`, threadIds); - } - db.run(`DELETE FROM threads WHERE id IN (${placeholders})`, threadIds); } @@ -445,6 +484,114 @@ interface ReconcileErr { error: CleanupErrorCode; } +function withWritableDb( + path: string, + busyTimeoutMs: number, + body: (db: Database) => void, +): { ok: true } | ReconcileErr { + let db: Database | undefined; + try { + db = openDbWritable(path, busyTimeoutMs); + db.exec("BEGIN IMMEDIATE"); + try { + body(db); + db.exec("COMMIT"); + return { ok: true }; + } catch (error) { + try { db.exec("ROLLBACK"); } catch { /* */ } + throw error; + } + } catch (error) { + return { ok: false, error: mapDbError(error) }; + } finally { + try { db?.close(); } catch { /* */ } + } +} + +/** Codex logs_*.sqlite: DELETE FROM logs WHERE thread_id IN (...). */ +function deleteLogsForThreads( + logsDbPath: string | null, + threadIds: string[], + busyTimeoutMs: number, +): { ok: true } | ReconcileErr { + if (!logsDbPath || !existsSync(logsDbPath) || threadIds.length === 0) return { ok: true }; + return withWritableDb(logsDbPath, busyTimeoutMs, db => { + if (!tableExists(db, "logs")) throw new Error("missing_logs_table"); + const placeholders = threadIds.map(() => "?").join(","); + db.run(`DELETE FROM logs WHERE thread_id IN (${placeholders})`, threadIds); + }); +} + +/** + * Codex memories_*.sqlite: remove stage1 outputs and memory_stage1 jobs, and enqueue + * global consolidation when a deleted output was selected for phase 2 (upstream). + */ +function deleteMemoriesForThreads( + memoriesDbPath: string | null, + threadIds: string[], + busyTimeoutMs: number, +): { ok: true } | ReconcileErr { + if (!memoriesDbPath || !existsSync(memoriesDbPath) || threadIds.length === 0) return { ok: true }; + return withWritableDb(memoriesDbPath, busyTimeoutMs, db => { + if (!tableExists(db, "stage1_outputs")) throw new Error("missing_stage1_outputs_table"); + const placeholders = threadIds.map(() => "?").join(","); + let selectedForPhase2 = 0; + if (columnExists(db, "stage1_outputs", "selected_for_phase2")) { + const row = db.query<{ n: number }, string[]>( + `SELECT COUNT(*) AS n FROM stage1_outputs + WHERE thread_id IN (${placeholders}) AND selected_for_phase2 != 0`, + ).get(...threadIds); + selectedForPhase2 = row?.n ?? 0; + } + db.run(`DELETE FROM stage1_outputs WHERE thread_id IN (${placeholders})`, threadIds); + if (tableExists(db, "jobs")) { + db.run( + `DELETE FROM jobs WHERE kind = ? AND job_key IN (${placeholders})`, + [JOB_KIND_MEMORY_STAGE1, ...threadIds], + ); + if (selectedForPhase2 > 0) { + const now = Math.floor(Date.now() / 1000); + db.run( + `INSERT INTO jobs ( + kind, job_key, status, worker_id, ownership_token, started_at, finished_at, + lease_until, retry_at, retry_remaining, last_error, input_watermark, last_success_watermark + ) VALUES (?, ?, 'pending', NULL, NULL, NULL, NULL, NULL, NULL, ?, NULL, ?, 0) + ON CONFLICT(kind, job_key) DO UPDATE SET + status = CASE WHEN jobs.status = 'running' THEN 'running' ELSE 'pending' END, + retry_at = CASE WHEN jobs.status = 'running' THEN jobs.retry_at ELSE NULL END, + retry_remaining = max(jobs.retry_remaining, excluded.retry_remaining), + input_watermark = CASE + WHEN excluded.input_watermark > COALESCE(jobs.input_watermark, 0) + THEN excluded.input_watermark + ELSE COALESCE(jobs.input_watermark, 0) + 1 + END`, + [JOB_KIND_MEMORY_CONSOLIDATE_GLOBAL, MEMORY_CONSOLIDATION_JOB_KEY, DEFAULT_RETRY_REMAINING, now], + ); + } + } + }); +} + +/** Codex goals_*.sqlite: DELETE FROM thread_goals (cascades continuation deferrals). */ +function deleteGoalsForThreads( + goalsDbPath: string | null, + threadIds: string[], + busyTimeoutMs: number, +): { ok: true } | ReconcileErr { + if (!goalsDbPath || !existsSync(goalsDbPath) || threadIds.length === 0) return { ok: true }; + return withWritableDb(goalsDbPath, busyTimeoutMs, db => { + if (!tableExists(db, "thread_goals")) throw new Error("missing_thread_goals_table"); + const placeholders = threadIds.map(() => "?").join(","); + if (tableExists(db, "thread_goal_continuation_deferrals")) { + db.run( + `DELETE FROM thread_goal_continuation_deferrals WHERE thread_id IN (${placeholders})`, + threadIds, + ); + } + db.run(`DELETE FROM thread_goals WHERE thread_id IN (${placeholders})`, threadIds); + }); +} + /** Load matching archived threads and refuse referenced history — no deletes yet. */ function loadThreadsForCleanup( stateDbPath: string, @@ -455,7 +602,7 @@ function loadThreadsForCleanup( if (!stateDbPath || !existsSync(stateDbPath)) return { ok: true, threads: [] }; let db: Database | undefined; try { - db = openStateDbWritable(stateDbPath, busyTimeoutMs); + db = openDbWritable(stateDbPath, busyTimeoutMs); const threads = loadMatchingThreads(db, candidates, codexHome); if (findReferencedHistory(db, threads)) { return { ok: false, error: "referenced_history" }; @@ -468,17 +615,33 @@ function loadThreadsForCleanup( } } -/** Delete matching threads in one transaction. Call only after the recovery manifest exists. */ -function deleteThreadsTransaction( - stateDbPath: string, +/** + * Reconcile all Codex per-thread stores for the matched archived candidates. + * Order matches upstream `delete_threads_strict`: logs → memories → goals → state. + */ +function reconcileDeletedThreads( + paths: RuntimeDbPaths, candidates: ArchivedCandidate[], codexHome: string, busyTimeoutMs: number, ): ReconcileOk | ReconcileErr { - if (!stateDbPath || !existsSync(stateDbPath)) return { ok: true, threads: [] }; + if (!paths.state || !existsSync(paths.state)) return { ok: true, threads: [] }; + + // Fresh id set before satellite deletes (state rows still present for retryability). + const preflight = loadThreadsForCleanup(paths.state, candidates, codexHome, busyTimeoutMs); + if (!preflight.ok) return preflight; + const threadIds = preflight.threads.map(t => t.id); + + const logs = deleteLogsForThreads(paths.logs, threadIds, busyTimeoutMs); + if (!logs.ok) return logs; + const memories = deleteMemoriesForThreads(paths.memories, threadIds, busyTimeoutMs); + if (!memories.ok) return memories; + const goals = deleteGoalsForThreads(paths.goals, threadIds, busyTimeoutMs); + if (!goals.ok) return goals; + let db: Database | undefined; try { - db = openStateDbWritable(stateDbPath, busyTimeoutMs); + db = openDbWritable(paths.state, busyTimeoutMs); db.exec("BEGIN IMMEDIATE"); try { // Re-validate under the write lock; the preflight result may be stale (TOCTOU). @@ -670,13 +833,14 @@ export function executeArchivedCleanup(options: ExecuteCleanupOptions): CleanupR }; } + const paths = discoverRuntimeDbPaths(codexHome); const probe = probeStateDbWritable(codexHome, busyTimeoutMs); if (!probe.ok) { return fail(mode, percent, probe.error); } // Preflight referenced-history / matching while DB is free, before any rename. - const loaded = loadThreadsForCleanup(probe.path, preview.candidates, codexHome, busyTimeoutMs); + const loaded = loadThreadsForCleanup(paths.state ?? "", preview.candidates, codexHome, busyTimeoutMs); if (!loaded.ok) { return fail(mode, percent, loaded.error); } @@ -731,8 +895,8 @@ export function executeArchivedCleanup(options: ExecuteCleanupOptions): CleanupR return fail(mode, percent, "fs_failed", rolled.restored ? undefined : { trashDir }); } - const deleted = deleteThreadsTransaction( - probe.path, + const deleted = reconcileDeletedThreads( + paths, preview.candidates, codexHome, busyTimeoutMs, diff --git a/tests/storage-cleanup.test.ts b/tests/storage-cleanup.test.ts index 5d02c92d9..013eb7d32 100644 --- a/tests/storage-cleanup.test.ts +++ b/tests/storage-cleanup.test.ts @@ -33,7 +33,87 @@ afterEach(() => { } }); -function buildHome(opts?: { withSpawnEdges?: boolean; withDynamicTools?: boolean }): string { +function seedSatelliteStores(dir: string): void { + const logs = new Database(join(dir, "logs_2.sqlite")); + logs.exec(`CREATE TABLE logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts INTEGER NOT NULL, + ts_nanos INTEGER NOT NULL DEFAULT 0, + level TEXT NOT NULL, + target TEXT NOT NULL, + thread_id TEXT, + estimated_bytes INTEGER NOT NULL DEFAULT 0 + )`); + logs.exec(`INSERT INTO logs (ts, level, target, thread_id, estimated_bytes) VALUES + (1,'INFO','t','told',10), + (2,'INFO','t','tmid',10), + (3,'INFO','t','active',10), + (4,'INFO','t','tnew',10)`); + logs.close(); + + const goals = new Database(join(dir, "goals_1.sqlite")); + goals.exec(`CREATE TABLE thread_goals ( + thread_id TEXT PRIMARY KEY NOT NULL, + goal_id TEXT NOT NULL, + objective TEXT NOT NULL, + status TEXT NOT NULL, + tokens_used INTEGER NOT NULL DEFAULT 0, + time_used_seconds INTEGER NOT NULL DEFAULT 0, + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL + )`); + goals.exec(`CREATE TABLE thread_goal_continuation_deferrals ( + thread_id TEXT PRIMARY KEY NOT NULL REFERENCES thread_goals(thread_id) ON DELETE CASCADE + )`); + goals.exec(`INSERT INTO thread_goals VALUES + ('told','g1','old','complete',0,0,1,1), + ('tmid','g2','mid','active',0,0,2,2), + ('active','g3','live','active',0,0,3,3)`); + goals.exec(`INSERT INTO thread_goal_continuation_deferrals VALUES ('tmid')`); + goals.close(); + + const memories = new Database(join(dir, "memories_1.sqlite")); + memories.exec(`CREATE TABLE stage1_outputs ( + thread_id TEXT PRIMARY KEY, + source_updated_at INTEGER NOT NULL, + raw_memory TEXT NOT NULL, + rollout_summary TEXT NOT NULL, + generated_at INTEGER NOT NULL, + selected_for_phase2 INTEGER NOT NULL DEFAULT 0 + )`); + memories.exec(`CREATE TABLE jobs ( + kind TEXT NOT NULL, + job_key TEXT NOT NULL, + status TEXT NOT NULL, + worker_id TEXT, + ownership_token TEXT, + started_at INTEGER, + finished_at INTEGER, + lease_until INTEGER, + retry_at INTEGER, + retry_remaining INTEGER NOT NULL, + last_error TEXT, + input_watermark INTEGER, + last_success_watermark INTEGER, + PRIMARY KEY (kind, job_key) + )`); + memories.exec(`INSERT INTO stage1_outputs VALUES + ('told',1,'m-old','s-old',1,0), + ('tmid',2,'m-mid','s-mid',2,1), + ('active',3,'m-live','s-live',3,0)`); + memories.exec(`INSERT INTO jobs VALUES + ('memory_stage1','told','done',NULL,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL), + ('memory_stage1','tmid','done',NULL,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL), + ('memory_stage1','active','done',NULL,NULL,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL), + ('memory_consolidate_global','global','done',NULL,NULL,NULL,NULL,NULL,NULL,3,NULL,0,0)`); + memories.close(); +} + +function buildHome(opts?: { + withSpawnEdges?: boolean; + withDynamicTools?: boolean; + withSatelliteStores?: boolean; +}): string { const dir = mkdtempSync(join(tmpdir(), "ocx-cleanup-")); mkdirSync(join(dir, "sessions", "2026", "05", "27"), { recursive: true }); writeFileSync(join(dir, "sessions", "2026", "05", "27", "rollout-active.jsonl"), "ACTIVE".repeat(20)); @@ -81,6 +161,7 @@ function buildHome(opts?: { withSpawnEdges?: boolean; withDynamicTools?: boolean db.exec(`INSERT INTO thread_dynamic_tools VALUES ('told',0,'tool','d','{}')`); } db.close(); + if (opts?.withSatelliteStores) seedSatelliteStores(dir); return dir; } @@ -449,4 +530,77 @@ describe("executeArchivedCleanup", () => { db.close(); expect(ids).toEqual(["active"]); }); + + test("permanent cleanup removes logs, goals, and memory rows for deleted threads", () => { + home = buildHome({ withSatelliteStores: true }); + const result = runWithDigest(100, "permanent", home); + expect(result.ok).toBe(true); + expect(result.count).toBe(3); + + const logs = new Database(join(home, "logs_2.sqlite"), { readonly: true }); + const logThreads = logs.query<{ thread_id: string }, []>( + "SELECT thread_id FROM logs ORDER BY thread_id", + ).all().map(r => r.thread_id); + logs.close(); + expect(logThreads).toEqual(["active"]); + + const goals = new Database(join(home, "goals_1.sqlite"), { readonly: true }); + const goalThreads = goals.query<{ thread_id: string }, []>( + "SELECT thread_id FROM thread_goals ORDER BY thread_id", + ).all().map(r => r.thread_id); + const deferrals = goals.query<{ thread_id: string }, []>( + "SELECT thread_id FROM thread_goal_continuation_deferrals", + ).all(); + goals.close(); + expect(goalThreads).toEqual(["active"]); + expect(deferrals).toEqual([]); + + const memories = new Database(join(home, "memories_1.sqlite"), { readonly: true }); + const stage1 = memories.query<{ thread_id: string }, []>( + "SELECT thread_id FROM stage1_outputs ORDER BY thread_id", + ).all().map(r => r.thread_id); + const stage1Jobs = memories.query<{ job_key: string }, []>( + "SELECT job_key FROM jobs WHERE kind='memory_stage1' ORDER BY job_key", + ).all().map(r => r.job_key); + const consolidate = memories.query<{ status: string }, []>( + "SELECT status FROM jobs WHERE kind='memory_consolidate_global' AND job_key='global'", + ).get(); + memories.close(); + expect(stage1).toEqual(["active"]); + expect(stage1Jobs).toEqual(["active"]); + // tmid was selected_for_phase2, so upstream enqueues/ refreshes global consolidation. + expect(consolidate?.status).toBe("pending"); + + const state = new Database(join(home, "state_5.sqlite"), { readonly: true }); + const ids = state.query<{ id: string }, []>("SELECT id FROM threads").all().map(r => r.id); + state.close(); + expect(ids).toEqual(["active"]); + }); + + test("threads read failure leaves every file and database unchanged", () => { + home = buildHome({ withSatelliteStores: true }); + // Present state DB with no readable threads table → fail closed before FS/DB mutation. + rmSync(join(home, "state_5.sqlite"), { force: true }); + const broken = new Database(join(home, "state_5.sqlite")); + broken.exec(`CREATE TABLE not_threads (id TEXT PRIMARY KEY)`); + broken.exec(`INSERT INTO not_threads VALUES ('x')`); + broken.close(); + + const beforeLogs = readFileSync(join(home, "logs_2.sqlite")); + const beforeGoals = readFileSync(join(home, "goals_1.sqlite")); + const beforeMemories = readFileSync(join(home, "memories_1.sqlite")); + const beforeState = readFileSync(join(home, "state_5.sqlite")); + + const result = runWithDigest(100, "permanent", home); + expect(result.ok).toBe(false); + expect(result.error).toBe("db_reconcile_failed"); + expect(existsSync(join(home, "archived_sessions", "rollout-old.jsonl"))).toBe(true); + expect(existsSync(join(home, "archived_sessions", "rollout-mid.jsonl"))).toBe(true); + expect(existsSync(join(home, "archived_sessions", "rollout-new.jsonl"))).toBe(true); + expect(existsSync(join(home, ".trash"))).toBe(false); + expect(Buffer.compare(beforeLogs, readFileSync(join(home, "logs_2.sqlite")))).toBe(0); + expect(Buffer.compare(beforeGoals, readFileSync(join(home, "goals_1.sqlite")))).toBe(0); + expect(Buffer.compare(beforeMemories, readFileSync(join(home, "memories_1.sqlite")))).toBe(0); + expect(Buffer.compare(beforeState, readFileSync(join(home, "state_5.sqlite")))).toBe(0); + }); }); From b9af3fd543589d72b6ca8267f3bbd5d1e4df5336 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Mon, 27 Jul 2026 06:36:35 +0200 Subject: [PATCH 10/17] fix(storage): make multi-DB cleanup recoverable under state lock Freeze thread IDs before satellite deletes, snapshot logs/memories/goals for rollback, and keep the recovery trash path when satellite restore fails. --- src/storage/cleanup.ts | 424 ++++++++++++++++++++++++++-------- tests/storage-cleanup.test.ts | 79 +++++++ 2 files changed, 413 insertions(+), 90 deletions(-) diff --git a/src/storage/cleanup.ts b/src/storage/cleanup.ts index 7f4421629..4838d8d4b 100644 --- a/src/storage/cleanup.ts +++ b/src/storage/cleanup.ts @@ -6,10 +6,10 @@ * permanent delete is opt-in. * * Execution is bound to a preview digest. All candidates are staged first; any FS - * or DB failure rolls staged moves back. DB reconciliation follows Codex - * `delete_threads_strict` order across the per-thread stores (`logs_*`, - * `memories_*`, `goals_*`, then `state_*`) and never soft-succeeds with - * `dbWarning` / `failedPaths`. + * or DB failure rolls staged moves back. DB reconciliation freezes thread IDs under + * the state write lock, then mutates `logs_*` → `memories_*` → `goals_*` → `state_*` + * with satellite snapshots so later failures can restore already-deleted rows. + * Success never carries soft `dbWarning` / `failedPaths`. */ import { createHash } from "node:crypto"; import { @@ -482,67 +482,129 @@ interface ReconcileOk { interface ReconcileErr { ok: false; error: CleanupErrorCode; + /** True when satellite rows were mutated and could not all be restored. */ + satelliteRestoreFailed?: boolean; } -function withWritableDb( - path: string, - busyTimeoutMs: number, - body: (db: Database) => void, -): { ok: true } | ReconcileErr { - let db: Database | undefined; - try { - db = openDbWritable(path, busyTimeoutMs); - db.exec("BEGIN IMMEDIATE"); - try { - body(db); - db.exec("COMMIT"); - return { ok: true }; - } catch (error) { - try { db.exec("ROLLBACK"); } catch { /* */ } - throw error; - } - } catch (error) { - return { ok: false, error: mapDbError(error) }; - } finally { - try { db?.close(); } catch { /* */ } +type SqlRow = Record; + +interface SatelliteBackup { + threadIds: string[]; + logs?: { path: string; rows: SqlRow[] }; + memories?: { + path: string; + stage1: SqlRow[]; + stage1Jobs: SqlRow[]; + consolidateJob: SqlRow | null; + consolidateTouched: boolean; + }; + goals?: { + path: string; + goals: SqlRow[]; + deferrals: SqlRow[]; + }; +} + +interface ReconcileTestHooks { + failAfterLogsMutation?: boolean; + failAfterMemoriesMutation?: boolean; + failAfterGoalsMutation?: boolean; + failBeforeStateCommit?: boolean; + failSatelliteRestore?: boolean; +} + +const SATELLITE_BACKUP_FILE = "satellite-backup.json"; + +function quoteIdent(name: string): string { + return `"${name.replaceAll('"', '""')}"`; +} + +function selectRows(db: Database, sql: string, params: Array): SqlRow[] { + return db.query>(sql).all(...params) as SqlRow[]; +} + +function insertRows(db: Database, table: string, rows: SqlRow[]): void { + for (const row of rows) { + const cols = Object.keys(row); + if (cols.length === 0) continue; + db.run( + `INSERT INTO ${quoteIdent(table)} (${cols.map(quoteIdent).join(", ")}) VALUES (${cols.map(() => "?").join(", ")})`, + cols.map(c => row[c] as string | number | bigint | null | Uint8Array), + ); } } -/** Codex logs_*.sqlite: DELETE FROM logs WHERE thread_id IN (...). */ -function deleteLogsForThreads( +function writeSatelliteBackup(stageDir: string, backup: SatelliteBackup): void { + writeFileSync(join(stageDir, SATELLITE_BACKUP_FILE), JSON.stringify(backup), "utf8"); +} + +function clearSatelliteBackup(stageDir: string): void { + try { unlinkSync(join(stageDir, SATELLITE_BACKUP_FILE)); } catch { /* */ } +} + +/** + * Snapshot + delete logs rows for frozen thread IDs. + * Returns null when the logs DB is absent. + */ +function mutateLogsWithBackup( logsDbPath: string | null, threadIds: string[], busyTimeoutMs: number, -): { ok: true } | ReconcileErr { - if (!logsDbPath || !existsSync(logsDbPath) || threadIds.length === 0) return { ok: true }; - return withWritableDb(logsDbPath, busyTimeoutMs, db => { +): SatelliteBackup["logs"] { + if (!logsDbPath || !existsSync(logsDbPath) || threadIds.length === 0) return undefined; + let db: Database | undefined; + try { + db = openDbWritable(logsDbPath, busyTimeoutMs); + db.exec("BEGIN IMMEDIATE"); if (!tableExists(db, "logs")) throw new Error("missing_logs_table"); const placeholders = threadIds.map(() => "?").join(","); + const rows = selectRows(db, `SELECT * FROM logs WHERE thread_id IN (${placeholders})`, threadIds); db.run(`DELETE FROM logs WHERE thread_id IN (${placeholders})`, threadIds); - }); + db.exec("COMMIT"); + return { path: logsDbPath, rows }; + } catch (error) { + try { db?.exec("ROLLBACK"); } catch { /* */ } + throw error; + } finally { + try { db?.close(); } catch { /* */ } + } } -/** - * Codex memories_*.sqlite: remove stage1 outputs and memory_stage1 jobs, and enqueue - * global consolidation when a deleted output was selected for phase 2 (upstream). - */ -function deleteMemoriesForThreads( +function mutateMemoriesWithBackup( memoriesDbPath: string | null, threadIds: string[], busyTimeoutMs: number, -): { ok: true } | ReconcileErr { - if (!memoriesDbPath || !existsSync(memoriesDbPath) || threadIds.length === 0) return { ok: true }; - return withWritableDb(memoriesDbPath, busyTimeoutMs, db => { +): SatelliteBackup["memories"] { + if (!memoriesDbPath || !existsSync(memoriesDbPath) || threadIds.length === 0) return undefined; + let db: Database | undefined; + try { + db = openDbWritable(memoriesDbPath, busyTimeoutMs); + db.exec("BEGIN IMMEDIATE"); if (!tableExists(db, "stage1_outputs")) throw new Error("missing_stage1_outputs_table"); const placeholders = threadIds.map(() => "?").join(","); + const stage1 = selectRows( + db, + `SELECT * FROM stage1_outputs WHERE thread_id IN (${placeholders})`, + threadIds, + ); + let stage1Jobs: SqlRow[] = []; + let consolidateJob: SqlRow | null = null; + let consolidateTouched = false; let selectedForPhase2 = 0; if (columnExists(db, "stage1_outputs", "selected_for_phase2")) { - const row = db.query<{ n: number }, string[]>( - `SELECT COUNT(*) AS n FROM stage1_outputs - WHERE thread_id IN (${placeholders}) AND selected_for_phase2 != 0`, - ).get(...threadIds); - selectedForPhase2 = row?.n ?? 0; + selectedForPhase2 = stage1.filter(r => Number(r.selected_for_phase2 ?? 0) !== 0).length; + } + if (tableExists(db, "jobs")) { + stage1Jobs = selectRows( + db, + `SELECT * FROM jobs WHERE kind = ? AND job_key IN (${placeholders})`, + [JOB_KIND_MEMORY_STAGE1, ...threadIds], + ); + consolidateJob = db.query( + `SELECT * FROM jobs WHERE kind = ? AND job_key = ?`, + ).get(JOB_KIND_MEMORY_CONSOLIDATE_GLOBAL, MEMORY_CONSOLIDATION_JOB_KEY) as SqlRow | null; } + db.run(`DELETE FROM stage1_outputs WHERE thread_id IN (${placeholders})`, threadIds); if (tableExists(db, "jobs")) { db.run( @@ -550,6 +612,7 @@ function deleteMemoriesForThreads( [JOB_KIND_MEMORY_STAGE1, ...threadIds], ); if (selectedForPhase2 > 0) { + consolidateTouched = true; const now = Math.floor(Date.now() / 1000); db.run( `INSERT INTO jobs ( @@ -569,27 +632,153 @@ function deleteMemoriesForThreads( ); } } - }); + db.exec("COMMIT"); + return { path: memoriesDbPath, stage1, stage1Jobs, consolidateJob, consolidateTouched }; + } catch (error) { + try { db?.exec("ROLLBACK"); } catch { /* */ } + throw error; + } finally { + try { db?.close(); } catch { /* */ } + } } -/** Codex goals_*.sqlite: DELETE FROM thread_goals (cascades continuation deferrals). */ -function deleteGoalsForThreads( +function mutateGoalsWithBackup( goalsDbPath: string | null, threadIds: string[], busyTimeoutMs: number, -): { ok: true } | ReconcileErr { - if (!goalsDbPath || !existsSync(goalsDbPath) || threadIds.length === 0) return { ok: true }; - return withWritableDb(goalsDbPath, busyTimeoutMs, db => { +): SatelliteBackup["goals"] { + if (!goalsDbPath || !existsSync(goalsDbPath) || threadIds.length === 0) return undefined; + let db: Database | undefined; + try { + db = openDbWritable(goalsDbPath, busyTimeoutMs); + db.exec("BEGIN IMMEDIATE"); if (!tableExists(db, "thread_goals")) throw new Error("missing_thread_goals_table"); const placeholders = threadIds.map(() => "?").join(","); + const goals = selectRows( + db, + `SELECT * FROM thread_goals WHERE thread_id IN (${placeholders})`, + threadIds, + ); + let deferrals: SqlRow[] = []; if (tableExists(db, "thread_goal_continuation_deferrals")) { + deferrals = selectRows( + db, + `SELECT * FROM thread_goal_continuation_deferrals WHERE thread_id IN (${placeholders})`, + threadIds, + ); db.run( `DELETE FROM thread_goal_continuation_deferrals WHERE thread_id IN (${placeholders})`, threadIds, ); } db.run(`DELETE FROM thread_goals WHERE thread_id IN (${placeholders})`, threadIds); - }); + db.exec("COMMIT"); + return { path: goalsDbPath, goals, deferrals }; + } catch (error) { + try { db?.exec("ROLLBACK"); } catch { /* */ } + throw error; + } finally { + try { db?.close(); } catch { /* */ } + } +} + +/** Restore previously snapshotted satellite rows. Returns false on any restore failure. */ +function restoreSatelliteBackup( + backup: SatelliteBackup, + busyTimeoutMs: number, + failRestore = false, +): boolean { + if (failRestore) return false; + try { + if (backup.logs) { + const restored = withWritableDb(backup.logs.path, busyTimeoutMs, db => { + if (!tableExists(db, "logs")) throw new Error("missing_logs_table"); + // Clear any partial survivors for these ids, then re-insert the snapshot. + if (backup.threadIds.length > 0) { + const placeholders = backup.threadIds.map(() => "?").join(","); + db.run(`DELETE FROM logs WHERE thread_id IN (${placeholders})`, backup.threadIds); + } + insertRows(db, "logs", backup.logs!.rows); + }); + if (!restored.ok) return false; + } + if (backup.memories) { + const mem = backup.memories; + const restored = withWritableDb(mem.path, busyTimeoutMs, db => { + if (!tableExists(db, "stage1_outputs")) throw new Error("missing_stage1_outputs_table"); + if (backup.threadIds.length > 0) { + const placeholders = backup.threadIds.map(() => "?").join(","); + db.run(`DELETE FROM stage1_outputs WHERE thread_id IN (${placeholders})`, backup.threadIds); + if (tableExists(db, "jobs")) { + db.run( + `DELETE FROM jobs WHERE kind = ? AND job_key IN (${placeholders})`, + [JOB_KIND_MEMORY_STAGE1, ...backup.threadIds], + ); + } + } + insertRows(db, "stage1_outputs", mem.stage1); + if (tableExists(db, "jobs")) { + insertRows(db, "jobs", mem.stage1Jobs); + if (mem.consolidateTouched) { + db.run( + `DELETE FROM jobs WHERE kind = ? AND job_key = ?`, + [JOB_KIND_MEMORY_CONSOLIDATE_GLOBAL, MEMORY_CONSOLIDATION_JOB_KEY], + ); + if (mem.consolidateJob) insertRows(db, "jobs", [mem.consolidateJob]); + } + } + }); + if (!restored.ok) return false; + } + if (backup.goals) { + const g = backup.goals; + const restored = withWritableDb(g.path, busyTimeoutMs, db => { + if (!tableExists(db, "thread_goals")) throw new Error("missing_thread_goals_table"); + if (backup.threadIds.length > 0) { + const placeholders = backup.threadIds.map(() => "?").join(","); + if (tableExists(db, "thread_goal_continuation_deferrals")) { + db.run( + `DELETE FROM thread_goal_continuation_deferrals WHERE thread_id IN (${placeholders})`, + backup.threadIds, + ); + } + db.run(`DELETE FROM thread_goals WHERE thread_id IN (${placeholders})`, backup.threadIds); + } + insertRows(db, "thread_goals", g.goals); + if (tableExists(db, "thread_goal_continuation_deferrals")) { + insertRows(db, "thread_goal_continuation_deferrals", g.deferrals); + } + }); + if (!restored.ok) return false; + } + return true; + } catch { + return false; + } +} + +function withWritableDb( + path: string, + busyTimeoutMs: number, + body: (db: Database) => void, +): { ok: true } | ReconcileErr { + let db: Database | undefined; + try { + db = openDbWritable(path, busyTimeoutMs); + db.exec("BEGIN IMMEDIATE"); + try { + body(db); + db.exec("COMMIT"); + return { ok: true }; + } catch (error) { + try { db.exec("ROLLBACK"); } catch { /* */ } + throw error; + } + } catch (error) { + return { ok: false, error: mapDbError(error) }; + } finally { + try { db?.close(); } catch { /* */ } + } } /** Load matching archived threads and refuse referenced history — no deletes yet. */ @@ -617,50 +806,90 @@ function loadThreadsForCleanup( /** * Reconcile all Codex per-thread stores for the matched archived candidates. - * Order matches upstream `delete_threads_strict`: logs → memories → goals → state. + * + * Freezes the thread-ID set under the state write lock, then mutates satellites + * (logs → memories → goals) with durable snapshots. Any later failure restores + * satellite rows before the caller restores staged files. */ function reconcileDeletedThreads( paths: RuntimeDbPaths, candidates: ArchivedCandidate[], codexHome: string, busyTimeoutMs: number, + stageDir: string, + hooks?: ReconcileTestHooks, ): ReconcileOk | ReconcileErr { if (!paths.state || !existsSync(paths.state)) return { ok: true, threads: [] }; - // Fresh id set before satellite deletes (state rows still present for retryability). - const preflight = loadThreadsForCleanup(paths.state, candidates, codexHome, busyTimeoutMs); - if (!preflight.ok) return preflight; - const threadIds = preflight.threads.map(t => t.id); - - const logs = deleteLogsForThreads(paths.logs, threadIds, busyTimeoutMs); - if (!logs.ok) return logs; - const memories = deleteMemoriesForThreads(paths.memories, threadIds, busyTimeoutMs); - if (!memories.ok) return memories; - const goals = deleteGoalsForThreads(paths.goals, threadIds, busyTimeoutMs); - if (!goals.ok) return goals; + let stateDb: Database | undefined; + let backup: SatelliteBackup | undefined; + let satellitesMutated = false; + + const failWithRestore = (error: CleanupErrorCode, mapped?: CleanupErrorCode): ReconcileErr => { + const code = mapped ?? error; + let satelliteRestoreFailed = false; + if (satellitesMutated && backup) { + satelliteRestoreFailed = !restoreSatelliteBackup( + backup, + busyTimeoutMs, + Boolean(hooks?.failSatelliteRestore), + ); + // Keep on-disk backup + manifest when restore cannot complete. + if (!satelliteRestoreFailed) clearSatelliteBackup(stageDir); + } else { + clearSatelliteBackup(stageDir); + } + return { + ok: false, + error: code, + ...(satelliteRestoreFailed ? { satelliteRestoreFailed: true } : {}), + }; + }; - let db: Database | undefined; try { - db = openDbWritable(paths.state, busyTimeoutMs); - db.exec("BEGIN IMMEDIATE"); - try { - // Re-validate under the write lock; the preflight result may be stale (TOCTOU). - const threads = loadMatchingThreads(db, candidates, codexHome); - if (findReferencedHistory(db, threads)) { - db.exec("ROLLBACK"); - return { ok: false, error: "referenced_history" }; - } - deleteThreadsAndDependents(db, threads.map(t => t.id)); - db.exec("COMMIT"); - return { ok: true, threads }; - } catch (error) { - try { db.exec("ROLLBACK"); } catch { /* */ } - throw error; + stateDb = openDbWritable(paths.state, busyTimeoutMs); + stateDb.exec("BEGIN IMMEDIATE"); + + // Freeze the exact delete set under the write lock before any satellite mutation. + const threads = loadMatchingThreads(stateDb, candidates, codexHome); + if (findReferencedHistory(stateDb, threads)) { + stateDb.exec("ROLLBACK"); + return { ok: false, error: "referenced_history" }; } + const threadIds = threads.map(t => t.id); + backup = { threadIds }; + + // Snapshot+delete satellites while the state lock is held so the ID set cannot drift. + backup.logs = mutateLogsWithBackup(paths.logs, threadIds, busyTimeoutMs); + writeSatelliteBackup(stageDir, backup); + satellitesMutated = Boolean(backup.logs); + if (hooks?.failAfterLogsMutation) throw new Error("test_fail_after_logs"); + + backup.memories = mutateMemoriesWithBackup(paths.memories, threadIds, busyTimeoutMs); + writeSatelliteBackup(stageDir, backup); + satellitesMutated = satellitesMutated || Boolean(backup.memories); + if (hooks?.failAfterMemoriesMutation) throw new Error("test_fail_after_memories"); + + backup.goals = mutateGoalsWithBackup(paths.goals, threadIds, busyTimeoutMs); + writeSatelliteBackup(stageDir, backup); + satellitesMutated = satellitesMutated || Boolean(backup.goals); + if (hooks?.failAfterGoalsMutation) throw new Error("test_fail_after_goals"); + + // Re-check under the same lock before committing state deletes. + if (findReferencedHistory(stateDb, threads)) { + stateDb.exec("ROLLBACK"); + return failWithRestore("referenced_history"); + } + deleteThreadsAndDependents(stateDb, threadIds); + if (hooks?.failBeforeStateCommit) throw new Error("test_fail_before_state_commit"); + stateDb.exec("COMMIT"); + clearSatelliteBackup(stageDir); + return { ok: true, threads }; } catch (error) { - return { ok: false, error: mapDbError(error) }; + try { stateDb?.exec("ROLLBACK"); } catch { /* */ } + return failWithRestore("db_reconcile_failed", mapDbError(error)); } finally { - try { db?.close(); } catch { /* */ } + try { stateDb?.close(); } catch { /* */ } } } @@ -758,6 +987,15 @@ function removeStageIfEmpty(stageDir: string, remaining: StagedFile[]): void { try { rmSync(stageDir, { recursive: true, force: true }); } catch { /* */ } } +function removeEmptyTrashRoot(codexHome: string): void { + try { + const trashRoot = join(codexHome, TRASH_DIR); + if (existsSync(trashRoot) && readdirSync(trashRoot).length === 0) { + rmSync(trashRoot, { recursive: true, force: true }); + } + } catch { /* */ } +} + function trashRelPath(codexHome: string, stageDir: string): string { return toForwardSlash(relative(codexHome, stageDir) || stageDir); } @@ -776,6 +1014,11 @@ export interface ExecuteCleanupOptions { failManifestWrite?: boolean; failPurgeBasenames?: string[]; failRollbackBasenames?: string[]; + failAfterLogsMutation?: boolean; + failAfterMemoriesMutation?: boolean; + failAfterGoalsMutation?: boolean; + failBeforeStateCommit?: boolean; + failSatelliteRestore?: boolean; }; } @@ -900,12 +1143,18 @@ export function executeArchivedCleanup(options: ExecuteCleanupOptions): CleanupR preview.candidates, codexHome, busyTimeoutMs, + stageDir, + options._test, ); if (!deleted.ok) { const rolled = rollbackStaged(stageResult.staged, { failBasenames: failRollback }); - // Keep the stage (and manifest) whenever anything remains unrestored. - removeStageIfEmpty(stageDir, rolled.remaining); - return fail(mode, percent, deleted.error, rolled.restored ? undefined : { trashDir }); + // Keep the stage (and recovery manifest) when files or satellite DB rows remain unrestored. + const keepTrash = Boolean(deleted.satelliteRestoreFailed) || !rolled.restored; + if (!keepTrash) { + removeStageIfEmpty(stageDir, rolled.remaining); + removeEmptyTrashRoot(codexHome); + } + return fail(mode, percent, deleted.error, keepTrash ? { trashDir } : undefined); } const removedPaths = preview.candidates.map(c => c.relPath); @@ -958,12 +1207,7 @@ export function executeArchivedCleanup(options: ExecuteCleanupOptions): CleanupR try { rmSync(stageDir, { recursive: true, force: true }); } catch { /* empty dir */ } // Drop an empty `.trash` root so permanent cleanup leaves no quarantine tree behind. - try { - const trashRoot = join(codexHome, TRASH_DIR); - if (existsSync(trashRoot) && readdirSync(trashRoot).length === 0) { - rmSync(trashRoot, { recursive: true, force: true }); - } - } catch { /* */ } + removeEmptyTrashRoot(codexHome); return { ok: true, diff --git a/tests/storage-cleanup.test.ts b/tests/storage-cleanup.test.ts index 013eb7d32..1a728b952 100644 --- a/tests/storage-cleanup.test.ts +++ b/tests/storage-cleanup.test.ts @@ -177,6 +177,11 @@ function runWithDigest( failManifestWrite?: boolean; failPurgeBasenames?: string[]; failRollbackBasenames?: string[]; + failAfterLogsMutation?: boolean; + failAfterMemoriesMutation?: boolean; + failAfterGoalsMutation?: boolean; + failBeforeStateCommit?: boolean; + failSatelliteRestore?: boolean; }; }, ) { @@ -603,4 +608,78 @@ describe("executeArchivedCleanup", () => { expect(Buffer.compare(beforeMemories, readFileSync(join(home, "memories_1.sqlite")))).toBe(0); expect(Buffer.compare(beforeState, readFileSync(join(home, "state_5.sqlite")))).toBe(0); }); + + test("injected satellite/state failures restore every file and database", () => { + const hooks = [ + { failAfterLogsMutation: true }, + { failAfterMemoriesMutation: true }, + { failAfterGoalsMutation: true }, + { failBeforeStateCommit: true }, + ] as const; + + for (const hook of hooks) { + home = buildHome({ withSatelliteStores: true }); + const files = { + old: readFileSync(join(home, "archived_sessions", "rollout-old.jsonl")), + mid: readFileSync(join(home, "archived_sessions", "rollout-mid.jsonl")), + neu: readFileSync(join(home, "archived_sessions", "rollout-new.jsonl")), + }; + const logsDb = new Database(join(home, "logs_2.sqlite"), { readonly: true }); + const logs = logsDb.query("SELECT id, ts, level, target, thread_id, estimated_bytes FROM logs ORDER BY id").all(); + logsDb.close(); + const goalsDb = new Database(join(home, "goals_1.sqlite"), { readonly: true }); + const goals = goalsDb.query("SELECT thread_id, goal_id, objective, status FROM thread_goals ORDER BY thread_id").all(); + const deferrals = goalsDb.query("SELECT thread_id FROM thread_goal_continuation_deferrals ORDER BY thread_id").all(); + goalsDb.close(); + const memDb = new Database(join(home, "memories_1.sqlite"), { readonly: true }); + const stage1 = memDb.query("SELECT thread_id, raw_memory, rollout_summary, selected_for_phase2 FROM stage1_outputs ORDER BY thread_id").all(); + const jobs = memDb.query("SELECT kind, job_key, status FROM jobs ORDER BY kind, job_key").all(); + memDb.close(); + const stateDb = new Database(join(home, "state_5.sqlite"), { readonly: true }); + const threads = stateDb.query("SELECT id, rollout_path, archived FROM threads ORDER BY id").all(); + stateDb.close(); + + const result = runWithDigest(100, "permanent", home, { now: 93, _test: { ...hook } }); + expect(result.ok).toBe(false); + expect(result.error).toBe("db_reconcile_failed"); + expect(result.trashDir).toBeUndefined(); + expect(existsSync(join(home, ".trash"))).toBe(false); + expect(Buffer.compare(files.old, readFileSync(join(home, "archived_sessions", "rollout-old.jsonl")))).toBe(0); + expect(Buffer.compare(files.mid, readFileSync(join(home, "archived_sessions", "rollout-mid.jsonl")))).toBe(0); + expect(Buffer.compare(files.neu, readFileSync(join(home, "archived_sessions", "rollout-new.jsonl")))).toBe(0); + + const logsAfter = new Database(join(home, "logs_2.sqlite"), { readonly: true }); + expect(logsAfter.query("SELECT id, ts, level, target, thread_id, estimated_bytes FROM logs ORDER BY id").all()).toEqual(logs); + logsAfter.close(); + const goalsAfter = new Database(join(home, "goals_1.sqlite"), { readonly: true }); + expect(goalsAfter.query("SELECT thread_id, goal_id, objective, status FROM thread_goals ORDER BY thread_id").all()).toEqual(goals); + expect(goalsAfter.query("SELECT thread_id FROM thread_goal_continuation_deferrals ORDER BY thread_id").all()).toEqual(deferrals); + goalsAfter.close(); + const memAfter = new Database(join(home, "memories_1.sqlite"), { readonly: true }); + expect(memAfter.query("SELECT thread_id, raw_memory, rollout_summary, selected_for_phase2 FROM stage1_outputs ORDER BY thread_id").all()).toEqual(stage1); + expect(memAfter.query("SELECT kind, job_key, status FROM jobs ORDER BY kind, job_key").all()).toEqual(jobs); + memAfter.close(); + const stateAfter = new Database(join(home, "state_5.sqlite"), { readonly: true }); + expect(stateAfter.query("SELECT id, rollout_path, archived FROM threads ORDER BY id").all()).toEqual(threads); + stateAfter.close(); + + rmSync(home, { recursive: true, force: true }); + home = ""; + } + }); + + test("satellite restore failure keeps recovery trashDir and manifest", () => { + home = buildHome({ withSatelliteStores: true }); + const result = runWithDigest(100, "permanent", home, { + now: 94, + _test: { failBeforeStateCommit: true, failSatelliteRestore: true }, + }); + expect(result.ok).toBe(false); + expect(result.error).toBe("db_reconcile_failed"); + expect(result.trashDir).toBe(".trash/94"); + expect(existsSync(join(home, ".trash", "94", "manifest.json"))).toBe(true); + expect(existsSync(join(home, ".trash", "94", "satellite-backup.json"))).toBe(true); + // Files are still restored; trash is kept for DB recovery metadata. + expect(existsSync(join(home, "archived_sessions", "rollout-old.jsonl"))).toBe(true); + }); }); From 7209c279550859624f788293aac78eff0d99cd93 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Mon, 27 Jul 2026 06:48:52 +0200 Subject: [PATCH 11/17] fix(storage): persist full satellite backup before any delete commit Snapshot logs/memories/goals first, write satellite-backup.json, then mutate. Backup-write failures leave DBs and rollouts untouched; cover single-store homes. --- src/storage/cleanup.ts | 188 ++++++++++++++++++++++++---------- tests/storage-cleanup.test.ts | 109 +++++++++++++++++++- 2 files changed, 239 insertions(+), 58 deletions(-) diff --git a/src/storage/cleanup.ts b/src/storage/cleanup.ts index 4838d8d4b..fe1548443 100644 --- a/src/storage/cleanup.ts +++ b/src/storage/cleanup.ts @@ -6,10 +6,11 @@ * permanent delete is opt-in. * * Execution is bound to a preview digest. All candidates are staged first; any FS - * or DB failure rolls staged moves back. DB reconciliation freezes thread IDs under - * the state write lock, then mutates `logs_*` → `memories_*` → `goals_*` → `state_*` - * with satellite snapshots so later failures can restore already-deleted rows. - * Success never carries soft `dbWarning` / `failedPaths`. + * Freezes the thread-ID set under the state write lock, persists a complete + * satellite-backup.json before any satellite delete commit, then mutates + * `logs_*` → `memories_*` → `goals_*` → `state_*`. Later failures restore + * satellite rows before staged files. Success never carries soft `dbWarning` / + * `failedPaths`. */ import { createHash } from "node:crypto"; import { @@ -511,6 +512,7 @@ interface ReconcileTestHooks { failAfterGoalsMutation?: boolean; failBeforeStateCommit?: boolean; failSatelliteRestore?: boolean; + failSatelliteBackupWrite?: boolean; } const SATELLITE_BACKUP_FILE = "satellite-backup.json"; @@ -542,11 +544,8 @@ function clearSatelliteBackup(stageDir: string): void { try { unlinkSync(join(stageDir, SATELLITE_BACKUP_FILE)); } catch { /* */ } } -/** - * Snapshot + delete logs rows for frozen thread IDs. - * Returns null when the logs DB is absent. - */ -function mutateLogsWithBackup( +/** Read-only snapshot of logs rows that would be deleted. */ +function snapshotLogs( logsDbPath: string | null, threadIds: string[], busyTimeoutMs: number, @@ -555,22 +554,16 @@ function mutateLogsWithBackup( let db: Database | undefined; try { db = openDbWritable(logsDbPath, busyTimeoutMs); - db.exec("BEGIN IMMEDIATE"); if (!tableExists(db, "logs")) throw new Error("missing_logs_table"); const placeholders = threadIds.map(() => "?").join(","); const rows = selectRows(db, `SELECT * FROM logs WHERE thread_id IN (${placeholders})`, threadIds); - db.run(`DELETE FROM logs WHERE thread_id IN (${placeholders})`, threadIds); - db.exec("COMMIT"); return { path: logsDbPath, rows }; - } catch (error) { - try { db?.exec("ROLLBACK"); } catch { /* */ } - throw error; } finally { try { db?.close(); } catch { /* */ } } } -function mutateMemoriesWithBackup( +function snapshotMemories( memoriesDbPath: string | null, threadIds: string[], busyTimeoutMs: number, @@ -579,7 +572,6 @@ function mutateMemoriesWithBackup( let db: Database | undefined; try { db = openDbWritable(memoriesDbPath, busyTimeoutMs); - db.exec("BEGIN IMMEDIATE"); if (!tableExists(db, "stage1_outputs")) throw new Error("missing_stage1_outputs_table"); const placeholders = threadIds.map(() => "?").join(","); const stage1 = selectRows( @@ -589,7 +581,6 @@ function mutateMemoriesWithBackup( ); let stage1Jobs: SqlRow[] = []; let consolidateJob: SqlRow | null = null; - let consolidateTouched = false; let selectedForPhase2 = 0; if (columnExists(db, "stage1_outputs", "selected_for_phase2")) { selectedForPhase2 = stage1.filter(r => Number(r.selected_for_phase2 ?? 0) !== 0).length; @@ -604,15 +595,101 @@ function mutateMemoriesWithBackup( `SELECT * FROM jobs WHERE kind = ? AND job_key = ?`, ).get(JOB_KIND_MEMORY_CONSOLIDATE_GLOBAL, MEMORY_CONSOLIDATION_JOB_KEY) as SqlRow | null; } + return { + path: memoriesDbPath, + stage1, + stage1Jobs, + consolidateJob, + consolidateTouched: selectedForPhase2 > 0, + }; + } finally { + try { db?.close(); } catch { /* */ } + } +} + +function snapshotGoals( + goalsDbPath: string | null, + threadIds: string[], + busyTimeoutMs: number, +): SatelliteBackup["goals"] { + if (!goalsDbPath || !existsSync(goalsDbPath) || threadIds.length === 0) return undefined; + let db: Database | undefined; + try { + db = openDbWritable(goalsDbPath, busyTimeoutMs); + if (!tableExists(db, "thread_goals")) throw new Error("missing_thread_goals_table"); + const placeholders = threadIds.map(() => "?").join(","); + const goals = selectRows( + db, + `SELECT * FROM thread_goals WHERE thread_id IN (${placeholders})`, + threadIds, + ); + let deferrals: SqlRow[] = []; + if (tableExists(db, "thread_goal_continuation_deferrals")) { + deferrals = selectRows( + db, + `SELECT * FROM thread_goal_continuation_deferrals WHERE thread_id IN (${placeholders})`, + threadIds, + ); + } + return { path: goalsDbPath, goals, deferrals }; + } finally { + try { db?.close(); } catch { /* */ } + } +} + +/** Build the complete satellite backup while holding the state write lock (read-only on satellites). */ +function snapshotSatelliteBackup( + paths: RuntimeDbPaths, + threadIds: string[], + busyTimeoutMs: number, +): SatelliteBackup { + return { + threadIds, + logs: snapshotLogs(paths.logs, threadIds, busyTimeoutMs), + memories: snapshotMemories(paths.memories, threadIds, busyTimeoutMs), + goals: snapshotGoals(paths.goals, threadIds, busyTimeoutMs), + }; +} + +function commitLogsDelete( + section: NonNullable, + threadIds: string[], + busyTimeoutMs: number, +): void { + let db: Database | undefined; + try { + db = openDbWritable(section.path, busyTimeoutMs); + db.exec("BEGIN IMMEDIATE"); + if (!tableExists(db, "logs")) throw new Error("missing_logs_table"); + const placeholders = threadIds.map(() => "?").join(","); + db.run(`DELETE FROM logs WHERE thread_id IN (${placeholders})`, threadIds); + db.exec("COMMIT"); + } catch (error) { + try { db?.exec("ROLLBACK"); } catch { /* */ } + throw error; + } finally { + try { db?.close(); } catch { /* */ } + } +} +function commitMemoriesDelete( + section: NonNullable, + threadIds: string[], + busyTimeoutMs: number, +): void { + let db: Database | undefined; + try { + db = openDbWritable(section.path, busyTimeoutMs); + db.exec("BEGIN IMMEDIATE"); + if (!tableExists(db, "stage1_outputs")) throw new Error("missing_stage1_outputs_table"); + const placeholders = threadIds.map(() => "?").join(","); db.run(`DELETE FROM stage1_outputs WHERE thread_id IN (${placeholders})`, threadIds); if (tableExists(db, "jobs")) { db.run( `DELETE FROM jobs WHERE kind = ? AND job_key IN (${placeholders})`, [JOB_KIND_MEMORY_STAGE1, ...threadIds], ); - if (selectedForPhase2 > 0) { - consolidateTouched = true; + if (section.consolidateTouched) { const now = Math.floor(Date.now() / 1000); db.run( `INSERT INTO jobs ( @@ -633,7 +710,6 @@ function mutateMemoriesWithBackup( } } db.exec("COMMIT"); - return { path: memoriesDbPath, stage1, stage1Jobs, consolidateJob, consolidateTouched }; } catch (error) { try { db?.exec("ROLLBACK"); } catch { /* */ } throw error; @@ -642,30 +718,18 @@ function mutateMemoriesWithBackup( } } -function mutateGoalsWithBackup( - goalsDbPath: string | null, +function commitGoalsDelete( + section: NonNullable, threadIds: string[], busyTimeoutMs: number, -): SatelliteBackup["goals"] { - if (!goalsDbPath || !existsSync(goalsDbPath) || threadIds.length === 0) return undefined; +): void { let db: Database | undefined; try { - db = openDbWritable(goalsDbPath, busyTimeoutMs); + db = openDbWritable(section.path, busyTimeoutMs); db.exec("BEGIN IMMEDIATE"); if (!tableExists(db, "thread_goals")) throw new Error("missing_thread_goals_table"); const placeholders = threadIds.map(() => "?").join(","); - const goals = selectRows( - db, - `SELECT * FROM thread_goals WHERE thread_id IN (${placeholders})`, - threadIds, - ); - let deferrals: SqlRow[] = []; if (tableExists(db, "thread_goal_continuation_deferrals")) { - deferrals = selectRows( - db, - `SELECT * FROM thread_goal_continuation_deferrals WHERE thread_id IN (${placeholders})`, - threadIds, - ); db.run( `DELETE FROM thread_goal_continuation_deferrals WHERE thread_id IN (${placeholders})`, threadIds, @@ -673,7 +737,6 @@ function mutateGoalsWithBackup( } db.run(`DELETE FROM thread_goals WHERE thread_id IN (${placeholders})`, threadIds); db.exec("COMMIT"); - return { path: goalsDbPath, goals, deferrals }; } catch (error) { try { db?.exec("ROLLBACK"); } catch { /* */ } throw error; @@ -807,9 +870,9 @@ function loadThreadsForCleanup( /** * Reconcile all Codex per-thread stores for the matched archived candidates. * - * Freezes the thread-ID set under the state write lock, then mutates satellites - * (logs → memories → goals) with durable snapshots. Any later failure restores - * satellite rows before the caller restores staged files. + * Freezes the thread-ID set under the state write lock, persists a complete + * satellite backup, then mutates satellites (logs → memories → goals). Any later + * failure restores satellite rows before the caller restores staged files. */ function reconcileDeletedThreads( paths: RuntimeDbPaths, @@ -857,23 +920,35 @@ function reconcileDeletedThreads( return { ok: false, error: "referenced_history" }; } const threadIds = threads.map(t => t.id); - backup = { threadIds }; - // Snapshot+delete satellites while the state lock is held so the ID set cannot drift. - backup.logs = mutateLogsWithBackup(paths.logs, threadIds, busyTimeoutMs); - writeSatelliteBackup(stageDir, backup); - satellitesMutated = Boolean(backup.logs); - if (hooks?.failAfterLogsMutation) throw new Error("test_fail_after_logs"); - - backup.memories = mutateMemoriesWithBackup(paths.memories, threadIds, busyTimeoutMs); - writeSatelliteBackup(stageDir, backup); - satellitesMutated = satellitesMutated || Boolean(backup.memories); - if (hooks?.failAfterMemoriesMutation) throw new Error("test_fail_after_memories"); + // Snapshot every satellite first, then persist the complete backup before any delete commit. + backup = snapshotSatelliteBackup(paths, threadIds, busyTimeoutMs); + try { + if (hooks?.failSatelliteBackupWrite) { + throw new Error("test_fail_satellite_backup_write"); + } + writeSatelliteBackup(stageDir, backup); + } catch { + stateDb.exec("ROLLBACK"); + clearSatelliteBackup(stageDir); + return { ok: false, error: "fs_failed" }; + } - backup.goals = mutateGoalsWithBackup(paths.goals, threadIds, busyTimeoutMs); - writeSatelliteBackup(stageDir, backup); - satellitesMutated = satellitesMutated || Boolean(backup.goals); - if (hooks?.failAfterGoalsMutation) throw new Error("test_fail_after_goals"); + if (backup.logs) { + commitLogsDelete(backup.logs, threadIds, busyTimeoutMs); + satellitesMutated = true; + if (hooks?.failAfterLogsMutation) throw new Error("test_fail_after_logs"); + } + if (backup.memories) { + commitMemoriesDelete(backup.memories, threadIds, busyTimeoutMs); + satellitesMutated = true; + if (hooks?.failAfterMemoriesMutation) throw new Error("test_fail_after_memories"); + } + if (backup.goals) { + commitGoalsDelete(backup.goals, threadIds, busyTimeoutMs); + satellitesMutated = true; + if (hooks?.failAfterGoalsMutation) throw new Error("test_fail_after_goals"); + } // Re-check under the same lock before committing state deletes. if (findReferencedHistory(stateDb, threads)) { @@ -1019,6 +1094,7 @@ export interface ExecuteCleanupOptions { failAfterGoalsMutation?: boolean; failBeforeStateCommit?: boolean; failSatelliteRestore?: boolean; + failSatelliteBackupWrite?: boolean; }; } diff --git a/tests/storage-cleanup.test.ts b/tests/storage-cleanup.test.ts index 1a728b952..b8318a9e5 100644 --- a/tests/storage-cleanup.test.ts +++ b/tests/storage-cleanup.test.ts @@ -33,7 +33,7 @@ afterEach(() => { } }); -function seedSatelliteStores(dir: string): void { +function seedLogsStore(dir: string): void { const logs = new Database(join(dir, "logs_2.sqlite")); logs.exec(`CREATE TABLE logs ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -50,7 +50,9 @@ function seedSatelliteStores(dir: string): void { (3,'INFO','t','active',10), (4,'INFO','t','tnew',10)`); logs.close(); +} +function seedGoalsStore(dir: string): void { const goals = new Database(join(dir, "goals_1.sqlite")); goals.exec(`CREATE TABLE thread_goals ( thread_id TEXT PRIMARY KEY NOT NULL, @@ -71,7 +73,9 @@ function seedSatelliteStores(dir: string): void { ('active','g3','live','active',0,0,3,3)`); goals.exec(`INSERT INTO thread_goal_continuation_deferrals VALUES ('tmid')`); goals.close(); +} +function seedMemoriesStore(dir: string): void { const memories = new Database(join(dir, "memories_1.sqlite")); memories.exec(`CREATE TABLE stage1_outputs ( thread_id TEXT PRIMARY KEY, @@ -109,10 +113,17 @@ function seedSatelliteStores(dir: string): void { memories.close(); } +function seedSatelliteStores(dir: string): void { + seedLogsStore(dir); + seedGoalsStore(dir); + seedMemoriesStore(dir); +} + function buildHome(opts?: { withSpawnEdges?: boolean; withDynamicTools?: boolean; withSatelliteStores?: boolean; + satellites?: "all" | "logs" | "memories" | "goals"; }): string { const dir = mkdtempSync(join(tmpdir(), "ocx-cleanup-")); mkdirSync(join(dir, "sessions", "2026", "05", "27"), { recursive: true }); @@ -161,7 +172,11 @@ function buildHome(opts?: { db.exec(`INSERT INTO thread_dynamic_tools VALUES ('told',0,'tool','d','{}')`); } db.close(); - if (opts?.withSatelliteStores) seedSatelliteStores(dir); + const satellites = opts?.satellites ?? (opts?.withSatelliteStores ? "all" : undefined); + if (satellites === "all") seedSatelliteStores(dir); + else if (satellites === "logs") seedLogsStore(dir); + else if (satellites === "memories") seedMemoriesStore(dir); + else if (satellites === "goals") seedGoalsStore(dir); return dir; } @@ -182,6 +197,7 @@ function runWithDigest( failAfterGoalsMutation?: boolean; failBeforeStateCommit?: boolean; failSatelliteRestore?: boolean; + failSatelliteBackupWrite?: boolean; }; }, ) { @@ -682,4 +698,93 @@ describe("executeArchivedCleanup", () => { // Files are still restored; trash is kept for DB recovery metadata. expect(existsSync(join(home, "archived_sessions", "rollout-old.jsonl"))).toBe(true); }); + + test("satellite-backup write failure leaves every database and rollout unchanged", () => { + home = buildHome({ withSatelliteStores: true }); + const files = { + old: readFileSync(join(home, "archived_sessions", "rollout-old.jsonl")), + mid: readFileSync(join(home, "archived_sessions", "rollout-mid.jsonl")), + neu: readFileSync(join(home, "archived_sessions", "rollout-new.jsonl")), + }; + const logsDb = new Database(join(home, "logs_2.sqlite"), { readonly: true }); + const logs = logsDb.query("SELECT id, ts, level, target, thread_id, estimated_bytes FROM logs ORDER BY id").all(); + logsDb.close(); + const goalsDb = new Database(join(home, "goals_1.sqlite"), { readonly: true }); + const goals = goalsDb.query("SELECT thread_id, goal_id, objective, status FROM thread_goals ORDER BY thread_id").all(); + const deferrals = goalsDb.query("SELECT thread_id FROM thread_goal_continuation_deferrals ORDER BY thread_id").all(); + goalsDb.close(); + const memDb = new Database(join(home, "memories_1.sqlite"), { readonly: true }); + const stage1 = memDb.query("SELECT thread_id, raw_memory, rollout_summary, selected_for_phase2 FROM stage1_outputs ORDER BY thread_id").all(); + const jobs = memDb.query("SELECT kind, job_key, status FROM jobs ORDER BY kind, job_key").all(); + memDb.close(); + const stateDb = new Database(join(home, "state_5.sqlite"), { readonly: true }); + const threads = stateDb.query("SELECT id, rollout_path, archived FROM threads ORDER BY id").all(); + stateDb.close(); + + const result = runWithDigest(100, "permanent", home, { + now: 95, + _test: { failSatelliteBackupWrite: true }, + }); + expect(result.ok).toBe(false); + expect(result.error).toBe("fs_failed"); + expect(result.trashDir).toBeUndefined(); + expect(existsSync(join(home, ".trash"))).toBe(false); + expect(Buffer.compare(files.old, readFileSync(join(home, "archived_sessions", "rollout-old.jsonl")))).toBe(0); + expect(Buffer.compare(files.mid, readFileSync(join(home, "archived_sessions", "rollout-mid.jsonl")))).toBe(0); + expect(Buffer.compare(files.neu, readFileSync(join(home, "archived_sessions", "rollout-new.jsonl")))).toBe(0); + + const logsAfter = new Database(join(home, "logs_2.sqlite"), { readonly: true }); + expect(logsAfter.query("SELECT id, ts, level, target, thread_id, estimated_bytes FROM logs ORDER BY id").all()).toEqual(logs); + logsAfter.close(); + const goalsAfter = new Database(join(home, "goals_1.sqlite"), { readonly: true }); + expect(goalsAfter.query("SELECT thread_id, goal_id, objective, status FROM thread_goals ORDER BY thread_id").all()).toEqual(goals); + expect(goalsAfter.query("SELECT thread_id FROM thread_goal_continuation_deferrals ORDER BY thread_id").all()).toEqual(deferrals); + goalsAfter.close(); + const memAfter = new Database(join(home, "memories_1.sqlite"), { readonly: true }); + expect(memAfter.query("SELECT thread_id, raw_memory, rollout_summary, selected_for_phase2 FROM stage1_outputs ORDER BY thread_id").all()).toEqual(stage1); + expect(memAfter.query("SELECT kind, job_key, status FROM jobs ORDER BY kind, job_key").all()).toEqual(jobs); + memAfter.close(); + const stateAfter = new Database(join(home, "state_5.sqlite"), { readonly: true }); + expect(stateAfter.query("SELECT id, rollout_path, archived FROM threads ORDER BY id").all()).toEqual(threads); + stateAfter.close(); + }); + + test("permanent cleanup works with logs-only satellite store", () => { + home = buildHome({ satellites: "logs" }); + const result = runWithDigest(100, "permanent", home); + expect(result.ok).toBe(true); + const logs = new Database(join(home, "logs_2.sqlite"), { readonly: true }); + expect(logs.query("SELECT thread_id FROM logs ORDER BY thread_id").all().map(r => r.thread_id)).toEqual(["active"]); + logs.close(); + expect(existsSync(join(home, "goals_1.sqlite"))).toBe(false); + expect(existsSync(join(home, "memories_1.sqlite"))).toBe(false); + const state = new Database(join(home, "state_5.sqlite"), { readonly: true }); + expect(state.query("SELECT id FROM threads").all().map(r => r.id)).toEqual(["active"]); + state.close(); + }); + + test("permanent cleanup works with memories-only satellite store", () => { + home = buildHome({ satellites: "memories" }); + const result = runWithDigest(100, "permanent", home); + expect(result.ok).toBe(true); + const memories = new Database(join(home, "memories_1.sqlite"), { readonly: true }); + expect(memories.query("SELECT thread_id FROM stage1_outputs ORDER BY thread_id").all().map(r => r.thread_id)).toEqual(["active"]); + expect(memories.query("SELECT job_key FROM jobs WHERE kind='memory_stage1' ORDER BY job_key").all().map(r => r.job_key)).toEqual(["active"]); + expect(memories.query("SELECT status FROM jobs WHERE kind='memory_consolidate_global' AND job_key='global'").get()?.status).toBe("pending"); + memories.close(); + expect(existsSync(join(home, "logs_2.sqlite"))).toBe(false); + expect(existsSync(join(home, "goals_1.sqlite"))).toBe(false); + }); + + test("permanent cleanup works with goals-only satellite store", () => { + home = buildHome({ satellites: "goals" }); + const result = runWithDigest(100, "permanent", home); + expect(result.ok).toBe(true); + const goals = new Database(join(home, "goals_1.sqlite"), { readonly: true }); + expect(goals.query("SELECT thread_id FROM thread_goals ORDER BY thread_id").all().map(r => r.thread_id)).toEqual(["active"]); + expect(goals.query("SELECT thread_id FROM thread_goal_continuation_deferrals").all()).toEqual([]); + goals.close(); + expect(existsSync(join(home, "logs_2.sqlite"))).toBe(false); + expect(existsSync(join(home, "memories_1.sqlite"))).toBe(false); + }); }); From f545c24f49ee3ad63643857e4d7b8b06af166935 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Mon, 27 Jul 2026 07:22:22 +0200 Subject: [PATCH 12/17] fix(storage): hold satellite write locks through snapshot and delete Acquire BEGIN IMMEDIATE on logs/memories/goals before snapshotting, persist backup before committing satellite deletes, and restore snapshotted PK rows with conflict-safe inserts. Add regression for concurrent log insert and memory-job update surviving injected state-commit failure. --- src/storage/cleanup.ts | 517 +++++++++++++++++++--------------- tests/storage-cleanup.test.ts | 50 ++++ 2 files changed, 336 insertions(+), 231 deletions(-) diff --git a/src/storage/cleanup.ts b/src/storage/cleanup.ts index fe1548443..a3ec9ba0b 100644 --- a/src/storage/cleanup.ts +++ b/src/storage/cleanup.ts @@ -513,6 +513,8 @@ interface ReconcileTestHooks { failBeforeStateCommit?: boolean; failSatelliteRestore?: boolean; failSatelliteBackupWrite?: boolean; + /** Runs after satellite deletes are committed, before state thread deletion. */ + afterSatelliteMutations?: () => void; } const SATELLITE_BACKUP_FILE = "satellite-backup.json"; @@ -525,17 +527,54 @@ function selectRows(db: Database, sql: string, params: Array): return db.query>(sql).all(...params) as SqlRow[]; } -function insertRows(db: Database, table: string, rows: SqlRow[]): void { +function insertRowsConflictIgnore(db: Database, table: string, rows: SqlRow[]): void { for (const row of rows) { const cols = Object.keys(row); if (cols.length === 0) continue; db.run( - `INSERT INTO ${quoteIdent(table)} (${cols.map(quoteIdent).join(", ")}) VALUES (${cols.map(() => "?").join(", ")})`, + `INSERT INTO ${quoteIdent(table)} (${cols.map(quoteIdent).join(", ")}) VALUES (${cols.map(() => "?").join(", ")}) ON CONFLICT DO NOTHING`, cols.map(c => row[c] as string | number | bigint | null | Uint8Array), ); } } +function updateRowFromSnapshot( + db: Database, + table: string, + row: SqlRow, + pkCols: string[], +): void { + const cols = Object.keys(row).filter(c => !pkCols.includes(c)); + if (cols.length === 0) return; + const sets = cols.map(c => `${quoteIdent(c)} = ?`).join(", "); + const where = pkCols.map(c => `${quoteIdent(c)} = ?`).join(" AND "); + db.run( + `UPDATE ${quoteIdent(table)} SET ${sets} WHERE ${where}`, + [ + ...cols.map(c => row[c] as string | number | bigint | null | Uint8Array), + ...pkCols.map(c => row[c] as string | number | bigint | null | Uint8Array), + ], + ); +} + +/** Revert delete-time consolidate enqueue; preserve concurrent post-commit status changes. */ +function restoreConsolidateGlobalJob(db: Database, snapshot: SqlRow): void { + const kind = String(snapshot.kind); + const jobKey = String(snapshot.job_key); + const existing = db.query<{ status: string }, [string, string]>( + "SELECT status FROM jobs WHERE kind = ? AND job_key = ?", + ).get(kind, jobKey); + if (!existing) { + insertRowsConflictIgnore(db, "jobs", [snapshot]); + return; + } + const snapStatus = String(snapshot.status ?? ""); + const currentStatus = String(existing.status ?? ""); + if (currentStatus === "pending" && snapStatus !== "pending") { + updateRowFromSnapshot(db, "jobs", snapshot, ["kind", "job_key"]); + } +} + function writeSatelliteBackup(stageDir: string, backup: SatelliteBackup): void { writeFileSync(join(stageDir, SATELLITE_BACKUP_FILE), JSON.stringify(backup), "utf8"); } @@ -544,208 +583,250 @@ function clearSatelliteBackup(stageDir: string): void { try { unlinkSync(join(stageDir, SATELLITE_BACKUP_FILE)); } catch { /* */ } } -/** Read-only snapshot of logs rows that would be deleted. */ -function snapshotLogs( - logsDbPath: string | null, - threadIds: string[], +interface SatelliteWriteLock { + path: string; + db: Database; +} + +interface SatelliteWriteLocks { + logs?: SatelliteWriteLock; + memories?: SatelliteWriteLock; + goals?: SatelliteWriteLock; +} + +/** Deterministic order: logs → memories → goals. Each present DB gets BEGIN IMMEDIATE. */ +function beginSatelliteWriteLocks( + paths: RuntimeDbPaths, busyTimeoutMs: number, +): SatelliteWriteLocks { + const begin = (path: string | null): SatelliteWriteLock | undefined => { + if (!path || !existsSync(path)) return undefined; + const db = openDbWritable(path, busyTimeoutMs); + db.exec("BEGIN IMMEDIATE"); + return { path, db }; + }; + return { + logs: begin(paths.logs), + memories: begin(paths.memories), + goals: begin(paths.goals), + }; +} + +function rollbackSatelliteLock(lock: SatelliteWriteLock | undefined): void { + if (!lock) return; + try { lock.db.exec("ROLLBACK"); } catch { /* */ } + try { lock.db.close(); } catch { /* */ } +} + +function rollbackAllSatelliteLocks(locks: SatelliteWriteLocks): void { + rollbackSatelliteLock(locks.logs); + rollbackSatelliteLock(locks.memories); + rollbackSatelliteLock(locks.goals); + locks.logs = undefined; + locks.memories = undefined; + locks.goals = undefined; +} + +function commitSatelliteLock(lock: SatelliteWriteLock | undefined): void { + if (!lock) return; + lock.db.exec("COMMIT"); + lock.db.close(); +} + +function snapshotLogsInTx( + db: Database, + path: string, + threadIds: string[], ): SatelliteBackup["logs"] { - if (!logsDbPath || !existsSync(logsDbPath) || threadIds.length === 0) return undefined; - let db: Database | undefined; - try { - db = openDbWritable(logsDbPath, busyTimeoutMs); - if (!tableExists(db, "logs")) throw new Error("missing_logs_table"); - const placeholders = threadIds.map(() => "?").join(","); - const rows = selectRows(db, `SELECT * FROM logs WHERE thread_id IN (${placeholders})`, threadIds); - return { path: logsDbPath, rows }; - } finally { - try { db?.close(); } catch { /* */ } - } + if (threadIds.length === 0) return undefined; + if (!tableExists(db, "logs")) throw new Error("missing_logs_table"); + const placeholders = threadIds.map(() => "?").join(","); + const rows = selectRows(db, `SELECT * FROM logs WHERE thread_id IN (${placeholders})`, threadIds); + return { path, rows }; } -function snapshotMemories( - memoriesDbPath: string | null, +function snapshotMemoriesInTx( + db: Database, + path: string, threadIds: string[], - busyTimeoutMs: number, ): SatelliteBackup["memories"] { - if (!memoriesDbPath || !existsSync(memoriesDbPath) || threadIds.length === 0) return undefined; - let db: Database | undefined; - try { - db = openDbWritable(memoriesDbPath, busyTimeoutMs); - if (!tableExists(db, "stage1_outputs")) throw new Error("missing_stage1_outputs_table"); - const placeholders = threadIds.map(() => "?").join(","); - const stage1 = selectRows( + if (threadIds.length === 0) return undefined; + if (!tableExists(db, "stage1_outputs")) throw new Error("missing_stage1_outputs_table"); + const placeholders = threadIds.map(() => "?").join(","); + const stage1 = selectRows( + db, + `SELECT * FROM stage1_outputs WHERE thread_id IN (${placeholders})`, + threadIds, + ); + let stage1Jobs: SqlRow[] = []; + let consolidateJob: SqlRow | null = null; + let selectedForPhase2 = 0; + if (columnExists(db, "stage1_outputs", "selected_for_phase2")) { + selectedForPhase2 = stage1.filter(r => Number(r.selected_for_phase2 ?? 0) !== 0).length; + } + if (tableExists(db, "jobs")) { + stage1Jobs = selectRows( db, - `SELECT * FROM stage1_outputs WHERE thread_id IN (${placeholders})`, - threadIds, + `SELECT * FROM jobs WHERE kind = ? AND job_key IN (${placeholders})`, + [JOB_KIND_MEMORY_STAGE1, ...threadIds], ); - let stage1Jobs: SqlRow[] = []; - let consolidateJob: SqlRow | null = null; - let selectedForPhase2 = 0; - if (columnExists(db, "stage1_outputs", "selected_for_phase2")) { - selectedForPhase2 = stage1.filter(r => Number(r.selected_for_phase2 ?? 0) !== 0).length; - } - if (tableExists(db, "jobs")) { - stage1Jobs = selectRows( - db, - `SELECT * FROM jobs WHERE kind = ? AND job_key IN (${placeholders})`, - [JOB_KIND_MEMORY_STAGE1, ...threadIds], - ); - consolidateJob = db.query( - `SELECT * FROM jobs WHERE kind = ? AND job_key = ?`, - ).get(JOB_KIND_MEMORY_CONSOLIDATE_GLOBAL, MEMORY_CONSOLIDATION_JOB_KEY) as SqlRow | null; - } - return { - path: memoriesDbPath, - stage1, - stage1Jobs, - consolidateJob, - consolidateTouched: selectedForPhase2 > 0, - }; - } finally { - try { db?.close(); } catch { /* */ } + consolidateJob = db.query( + `SELECT * FROM jobs WHERE kind = ? AND job_key = ?`, + ).get(JOB_KIND_MEMORY_CONSOLIDATE_GLOBAL, MEMORY_CONSOLIDATION_JOB_KEY) as SqlRow | null; } + return { + path, + stage1, + stage1Jobs, + consolidateJob, + consolidateTouched: selectedForPhase2 > 0, + }; } -function snapshotGoals( - goalsDbPath: string | null, +function snapshotGoalsInTx( + db: Database, + path: string, threadIds: string[], - busyTimeoutMs: number, ): SatelliteBackup["goals"] { - if (!goalsDbPath || !existsSync(goalsDbPath) || threadIds.length === 0) return undefined; - let db: Database | undefined; - try { - db = openDbWritable(goalsDbPath, busyTimeoutMs); - if (!tableExists(db, "thread_goals")) throw new Error("missing_thread_goals_table"); - const placeholders = threadIds.map(() => "?").join(","); - const goals = selectRows( + if (threadIds.length === 0) return undefined; + if (!tableExists(db, "thread_goals")) throw new Error("missing_thread_goals_table"); + const placeholders = threadIds.map(() => "?").join(","); + const goals = selectRows( + db, + `SELECT * FROM thread_goals WHERE thread_id IN (${placeholders})`, + threadIds, + ); + let deferrals: SqlRow[] = []; + if (tableExists(db, "thread_goal_continuation_deferrals")) { + deferrals = selectRows( db, - `SELECT * FROM thread_goals WHERE thread_id IN (${placeholders})`, + `SELECT * FROM thread_goal_continuation_deferrals WHERE thread_id IN (${placeholders})`, threadIds, ); - let deferrals: SqlRow[] = []; - if (tableExists(db, "thread_goal_continuation_deferrals")) { - deferrals = selectRows( - db, - `SELECT * FROM thread_goal_continuation_deferrals WHERE thread_id IN (${placeholders})`, - threadIds, - ); - } - return { path: goalsDbPath, goals, deferrals }; - } finally { - try { db?.close(); } catch { /* */ } } + return { path, goals, deferrals }; } -/** Build the complete satellite backup while holding the state write lock (read-only on satellites). */ -function snapshotSatelliteBackup( - paths: RuntimeDbPaths, +/** Snapshot every present satellite under its write lock (rows stable until commit). */ +function snapshotSatelliteBackupInLocks( + locks: SatelliteWriteLocks, threadIds: string[], - busyTimeoutMs: number, ): SatelliteBackup { - return { - threadIds, - logs: snapshotLogs(paths.logs, threadIds, busyTimeoutMs), - memories: snapshotMemories(paths.memories, threadIds, busyTimeoutMs), - goals: snapshotGoals(paths.goals, threadIds, busyTimeoutMs), - }; + const backup: SatelliteBackup = { threadIds }; + if (locks.logs) { + backup.logs = snapshotLogsInTx(locks.logs.db, locks.logs.path, threadIds); + } + if (locks.memories) { + backup.memories = snapshotMemoriesInTx(locks.memories.db, locks.memories.path, threadIds); + } + if (locks.goals) { + backup.goals = snapshotGoalsInTx(locks.goals.db, locks.goals.path, threadIds); + } + return backup; } -function commitLogsDelete( - section: NonNullable, - threadIds: string[], - busyTimeoutMs: number, -): void { - let db: Database | undefined; - try { - db = openDbWritable(section.path, busyTimeoutMs); - db.exec("BEGIN IMMEDIATE"); - if (!tableExists(db, "logs")) throw new Error("missing_logs_table"); - const placeholders = threadIds.map(() => "?").join(","); - db.run(`DELETE FROM logs WHERE thread_id IN (${placeholders})`, threadIds); - db.exec("COMMIT"); - } catch (error) { - try { db?.exec("ROLLBACK"); } catch { /* */ } - throw error; - } finally { - try { db?.close(); } catch { /* */ } - } +function deleteLogsInTx(db: Database, rows: SqlRow[]): void { + if (rows.length === 0) return; + if (!tableExists(db, "logs")) throw new Error("missing_logs_table"); + const ids = rows.map(r => r.id).filter(id => id !== null && id !== undefined); + if (ids.length === 0) return; + const placeholders = ids.map(() => "?").join(","); + db.run(`DELETE FROM logs WHERE id IN (${placeholders})`, ids as Array); } -function commitMemoriesDelete( +function deleteMemoriesInTx( + db: Database, section: NonNullable, - threadIds: string[], - busyTimeoutMs: number, ): void { - let db: Database | undefined; - try { - db = openDbWritable(section.path, busyTimeoutMs); - db.exec("BEGIN IMMEDIATE"); - if (!tableExists(db, "stage1_outputs")) throw new Error("missing_stage1_outputs_table"); - const placeholders = threadIds.map(() => "?").join(","); - db.run(`DELETE FROM stage1_outputs WHERE thread_id IN (${placeholders})`, threadIds); - if (tableExists(db, "jobs")) { + if (!tableExists(db, "stage1_outputs")) throw new Error("missing_stage1_outputs_table"); + const stage1Ids = section.stage1.map(r => String(r.thread_id)); + if (stage1Ids.length > 0) { + const placeholders = stage1Ids.map(() => "?").join(","); + db.run(`DELETE FROM stage1_outputs WHERE thread_id IN (${placeholders})`, stage1Ids); + } + if (tableExists(db, "jobs")) { + const jobKeys = section.stage1Jobs.map(r => String(r.job_key)); + if (jobKeys.length > 0) { + const placeholders = jobKeys.map(() => "?").join(","); db.run( `DELETE FROM jobs WHERE kind = ? AND job_key IN (${placeholders})`, - [JOB_KIND_MEMORY_STAGE1, ...threadIds], + [JOB_KIND_MEMORY_STAGE1, ...jobKeys], + ); + } + if (section.consolidateTouched) { + const now = Math.floor(Date.now() / 1000); + db.run( + `INSERT INTO jobs ( + kind, job_key, status, worker_id, ownership_token, started_at, finished_at, + lease_until, retry_at, retry_remaining, last_error, input_watermark, last_success_watermark + ) VALUES (?, ?, 'pending', NULL, NULL, NULL, NULL, NULL, NULL, ?, NULL, ?, 0) + ON CONFLICT(kind, job_key) DO UPDATE SET + status = CASE WHEN jobs.status = 'running' THEN 'running' ELSE 'pending' END, + retry_at = CASE WHEN jobs.status = 'running' THEN jobs.retry_at ELSE NULL END, + retry_remaining = max(jobs.retry_remaining, excluded.retry_remaining), + input_watermark = CASE + WHEN excluded.input_watermark > COALESCE(jobs.input_watermark, 0) + THEN excluded.input_watermark + ELSE COALESCE(jobs.input_watermark, 0) + 1 + END`, + [JOB_KIND_MEMORY_CONSOLIDATE_GLOBAL, MEMORY_CONSOLIDATION_JOB_KEY, DEFAULT_RETRY_REMAINING, now], ); - if (section.consolidateTouched) { - const now = Math.floor(Date.now() / 1000); - db.run( - `INSERT INTO jobs ( - kind, job_key, status, worker_id, ownership_token, started_at, finished_at, - lease_until, retry_at, retry_remaining, last_error, input_watermark, last_success_watermark - ) VALUES (?, ?, 'pending', NULL, NULL, NULL, NULL, NULL, NULL, ?, NULL, ?, 0) - ON CONFLICT(kind, job_key) DO UPDATE SET - status = CASE WHEN jobs.status = 'running' THEN 'running' ELSE 'pending' END, - retry_at = CASE WHEN jobs.status = 'running' THEN jobs.retry_at ELSE NULL END, - retry_remaining = max(jobs.retry_remaining, excluded.retry_remaining), - input_watermark = CASE - WHEN excluded.input_watermark > COALESCE(jobs.input_watermark, 0) - THEN excluded.input_watermark - ELSE COALESCE(jobs.input_watermark, 0) + 1 - END`, - [JOB_KIND_MEMORY_CONSOLIDATE_GLOBAL, MEMORY_CONSOLIDATION_JOB_KEY, DEFAULT_RETRY_REMAINING, now], - ); - } } - db.exec("COMMIT"); - } catch (error) { - try { db?.exec("ROLLBACK"); } catch { /* */ } - throw error; - } finally { - try { db?.close(); } catch { /* */ } } } -function commitGoalsDelete( +function deleteGoalsInTx( + db: Database, section: NonNullable, - threadIds: string[], - busyTimeoutMs: number, ): void { - let db: Database | undefined; + if (!tableExists(db, "thread_goals")) throw new Error("missing_thread_goals_table"); + const deferralIds = section.deferrals.map(r => String(r.thread_id)); + if (deferralIds.length > 0 && tableExists(db, "thread_goal_continuation_deferrals")) { + const placeholders = deferralIds.map(() => "?").join(","); + db.run( + `DELETE FROM thread_goal_continuation_deferrals WHERE thread_id IN (${placeholders})`, + deferralIds, + ); + } + const goalIds = section.goals.map(r => String(r.thread_id)); + if (goalIds.length > 0) { + const placeholders = goalIds.map(() => "?").join(","); + db.run(`DELETE FROM thread_goals WHERE thread_id IN (${placeholders})`, goalIds); + } +} + +/** Delete snapshotted primary-key rows and commit each satellite write transaction. */ +function deleteAndCommitSatellites( + locks: SatelliteWriteLocks, + backup: SatelliteBackup, + hooks?: ReconcileTestHooks, +): void { try { - db = openDbWritable(section.path, busyTimeoutMs); - db.exec("BEGIN IMMEDIATE"); - if (!tableExists(db, "thread_goals")) throw new Error("missing_thread_goals_table"); - const placeholders = threadIds.map(() => "?").join(","); - if (tableExists(db, "thread_goal_continuation_deferrals")) { - db.run( - `DELETE FROM thread_goal_continuation_deferrals WHERE thread_id IN (${placeholders})`, - threadIds, - ); + if (locks.logs && backup.logs) { + deleteLogsInTx(locks.logs.db, backup.logs.rows); + commitSatelliteLock(locks.logs); + locks.logs = undefined; + if (hooks?.failAfterLogsMutation) throw new Error("test_fail_after_logs"); + } + if (locks.memories && backup.memories) { + deleteMemoriesInTx(locks.memories.db, backup.memories); + commitSatelliteLock(locks.memories); + locks.memories = undefined; + if (hooks?.failAfterMemoriesMutation) throw new Error("test_fail_after_memories"); + } + if (locks.goals && backup.goals) { + deleteGoalsInTx(locks.goals.db, backup.goals); + commitSatelliteLock(locks.goals); + locks.goals = undefined; + if (hooks?.failAfterGoalsMutation) throw new Error("test_fail_after_goals"); } - db.run(`DELETE FROM thread_goals WHERE thread_id IN (${placeholders})`, threadIds); - db.exec("COMMIT"); } catch (error) { - try { db?.exec("ROLLBACK"); } catch { /* */ } + rollbackAllSatelliteLocks(locks); throw error; - } finally { - try { db?.close(); } catch { /* */ } } } -/** Restore previously snapshotted satellite rows. Returns false on any restore failure. */ +/** Restore only snapshotted rows; concurrent inserts/updates after commit stay intact. */ function restoreSatelliteBackup( backup: SatelliteBackup, busyTimeoutMs: number, @@ -756,12 +837,7 @@ function restoreSatelliteBackup( if (backup.logs) { const restored = withWritableDb(backup.logs.path, busyTimeoutMs, db => { if (!tableExists(db, "logs")) throw new Error("missing_logs_table"); - // Clear any partial survivors for these ids, then re-insert the snapshot. - if (backup.threadIds.length > 0) { - const placeholders = backup.threadIds.map(() => "?").join(","); - db.run(`DELETE FROM logs WHERE thread_id IN (${placeholders})`, backup.threadIds); - } - insertRows(db, "logs", backup.logs!.rows); + insertRowsConflictIgnore(db, "logs", backup.logs!.rows); }); if (!restored.ok) return false; } @@ -769,25 +845,11 @@ function restoreSatelliteBackup( const mem = backup.memories; const restored = withWritableDb(mem.path, busyTimeoutMs, db => { if (!tableExists(db, "stage1_outputs")) throw new Error("missing_stage1_outputs_table"); - if (backup.threadIds.length > 0) { - const placeholders = backup.threadIds.map(() => "?").join(","); - db.run(`DELETE FROM stage1_outputs WHERE thread_id IN (${placeholders})`, backup.threadIds); - if (tableExists(db, "jobs")) { - db.run( - `DELETE FROM jobs WHERE kind = ? AND job_key IN (${placeholders})`, - [JOB_KIND_MEMORY_STAGE1, ...backup.threadIds], - ); - } - } - insertRows(db, "stage1_outputs", mem.stage1); + insertRowsConflictIgnore(db, "stage1_outputs", mem.stage1); if (tableExists(db, "jobs")) { - insertRows(db, "jobs", mem.stage1Jobs); - if (mem.consolidateTouched) { - db.run( - `DELETE FROM jobs WHERE kind = ? AND job_key = ?`, - [JOB_KIND_MEMORY_CONSOLIDATE_GLOBAL, MEMORY_CONSOLIDATION_JOB_KEY], - ); - if (mem.consolidateJob) insertRows(db, "jobs", [mem.consolidateJob]); + insertRowsConflictIgnore(db, "jobs", mem.stage1Jobs); + if (mem.consolidateTouched && mem.consolidateJob) { + restoreConsolidateGlobalJob(db, mem.consolidateJob); } } }); @@ -797,19 +859,9 @@ function restoreSatelliteBackup( const g = backup.goals; const restored = withWritableDb(g.path, busyTimeoutMs, db => { if (!tableExists(db, "thread_goals")) throw new Error("missing_thread_goals_table"); - if (backup.threadIds.length > 0) { - const placeholders = backup.threadIds.map(() => "?").join(","); - if (tableExists(db, "thread_goal_continuation_deferrals")) { - db.run( - `DELETE FROM thread_goal_continuation_deferrals WHERE thread_id IN (${placeholders})`, - backup.threadIds, - ); - } - db.run(`DELETE FROM thread_goals WHERE thread_id IN (${placeholders})`, backup.threadIds); - } - insertRows(db, "thread_goals", g.goals); + insertRowsConflictIgnore(db, "thread_goals", g.goals); if (tableExists(db, "thread_goal_continuation_deferrals")) { - insertRows(db, "thread_goal_continuation_deferrals", g.deferrals); + insertRowsConflictIgnore(db, "thread_goal_continuation_deferrals", g.deferrals); } }); if (!restored.ok) return false; @@ -887,6 +939,7 @@ function reconcileDeletedThreads( let stateDb: Database | undefined; let backup: SatelliteBackup | undefined; let satellitesMutated = false; + let satelliteLocks: SatelliteWriteLocks | undefined; const failWithRestore = (error: CleanupErrorCode, mapped?: CleanupErrorCode): ReconcileErr => { const code = mapped ?? error; @@ -921,45 +974,46 @@ function reconcileDeletedThreads( } const threadIds = threads.map(t => t.id); - // Snapshot every satellite first, then persist the complete backup before any delete commit. - backup = snapshotSatelliteBackup(paths, threadIds, busyTimeoutMs); + satelliteLocks = beginSatelliteWriteLocks(paths, busyTimeoutMs); try { - if (hooks?.failSatelliteBackupWrite) { - throw new Error("test_fail_satellite_backup_write"); + backup = snapshotSatelliteBackupInLocks(satelliteLocks, threadIds); + try { + if (hooks?.failSatelliteBackupWrite) { + throw new Error("test_fail_satellite_backup_write"); + } + writeSatelliteBackup(stageDir, backup); + } catch { + rollbackAllSatelliteLocks(satelliteLocks); + stateDb.exec("ROLLBACK"); + clearSatelliteBackup(stageDir); + return { ok: false, error: "fs_failed" }; } - writeSatelliteBackup(stageDir, backup); - } catch { - stateDb.exec("ROLLBACK"); - clearSatelliteBackup(stageDir); - return { ok: false, error: "fs_failed" }; - } - if (backup.logs) { - commitLogsDelete(backup.logs, threadIds, busyTimeoutMs); - satellitesMutated = true; - if (hooks?.failAfterLogsMutation) throw new Error("test_fail_after_logs"); - } - if (backup.memories) { - commitMemoriesDelete(backup.memories, threadIds, busyTimeoutMs); - satellitesMutated = true; - if (hooks?.failAfterMemoriesMutation) throw new Error("test_fail_after_memories"); - } - if (backup.goals) { - commitGoalsDelete(backup.goals, threadIds, busyTimeoutMs); - satellitesMutated = true; - if (hooks?.failAfterGoalsMutation) throw new Error("test_fail_after_goals"); - } + const hasSatelliteWork = Boolean(backup.logs || backup.memories || backup.goals); + if (hasSatelliteWork) { + satellitesMutated = true; + deleteAndCommitSatellites(satelliteLocks, backup, hooks); + } else { + rollbackAllSatelliteLocks(satelliteLocks); + } + satelliteLocks = undefined; - // Re-check under the same lock before committing state deletes. - if (findReferencedHistory(stateDb, threads)) { - stateDb.exec("ROLLBACK"); - return failWithRestore("referenced_history"); + if (hooks?.afterSatelliteMutations) hooks.afterSatelliteMutations(); + + // Re-check under the same lock before committing state deletes. + if (findReferencedHistory(stateDb, threads)) { + stateDb.exec("ROLLBACK"); + return failWithRestore("referenced_history"); + } + deleteThreadsAndDependents(stateDb, threadIds); + if (hooks?.failBeforeStateCommit) throw new Error("test_fail_before_state_commit"); + stateDb.exec("COMMIT"); + clearSatelliteBackup(stageDir); + return { ok: true, threads }; + } catch (error) { + if (satelliteLocks) rollbackAllSatelliteLocks(satelliteLocks); + throw error; } - deleteThreadsAndDependents(stateDb, threadIds); - if (hooks?.failBeforeStateCommit) throw new Error("test_fail_before_state_commit"); - stateDb.exec("COMMIT"); - clearSatelliteBackup(stageDir); - return { ok: true, threads }; } catch (error) { try { stateDb?.exec("ROLLBACK"); } catch { /* */ } return failWithRestore("db_reconcile_failed", mapDbError(error)); @@ -1095,6 +1149,7 @@ export interface ExecuteCleanupOptions { failBeforeStateCommit?: boolean; failSatelliteRestore?: boolean; failSatelliteBackupWrite?: boolean; + afterSatelliteMutations?: () => void; }; } diff --git a/tests/storage-cleanup.test.ts b/tests/storage-cleanup.test.ts index b8318a9e5..c4295c1d7 100644 --- a/tests/storage-cleanup.test.ts +++ b/tests/storage-cleanup.test.ts @@ -198,6 +198,7 @@ function runWithDigest( failBeforeStateCommit?: boolean; failSatelliteRestore?: boolean; failSatelliteBackupWrite?: boolean; + afterSatelliteMutations?: () => void; }; }, ) { @@ -787,4 +788,53 @@ describe("executeArchivedCleanup", () => { expect(existsSync(join(home, "logs_2.sqlite"))).toBe(false); expect(existsSync(join(home, "memories_1.sqlite"))).toBe(false); }); + + test("concurrent satellite writes after mutation are preserved on restore", () => { + home = buildHome({ withSatelliteStores: true }); + const result = runWithDigest(100, "permanent", home, { + now: 96, + _test: { + failBeforeStateCommit: true, + afterSatelliteMutations: () => { + const logs = new Database(join(home, "logs_2.sqlite")); + logs.exec( + `INSERT INTO logs (ts, level, target, thread_id, estimated_bytes) VALUES (99,'INFO','t','concurrent-insert',99)`, + ); + logs.close(); + const mem = new Database(join(home, "memories_1.sqlite")); + mem.exec( + `UPDATE jobs SET status='running' WHERE kind='memory_consolidate_global' AND job_key='global'`, + ); + mem.close(); + }, + }, + }); + expect(result.ok).toBe(false); + expect(result.error).toBe("db_reconcile_failed"); + + const logs = new Database(join(home, "logs_2.sqlite"), { readonly: true }); + expect(logs.query("SELECT thread_id FROM logs WHERE thread_id='concurrent-insert'").get()).toBeTruthy(); + expect( + logs.query("SELECT COUNT(*) AS n FROM logs WHERE thread_id IN ('told','tmid','tnew')").get()?.n, + ).toBe(3); + expect(logs.query("SELECT thread_id FROM logs WHERE thread_id='active'").get()).toBeTruthy(); + logs.close(); + + const memories = new Database(join(home, "memories_1.sqlite"), { readonly: true }); + expect( + memories.query( + "SELECT status FROM jobs WHERE kind='memory_consolidate_global' AND job_key='global'", + ).get()?.status, + ).toBe("running"); + expect( + memories.query("SELECT thread_id FROM stage1_outputs ORDER BY thread_id").all().map(r => r.thread_id), + ).toEqual(["active", "tmid", "told"]); + memories.close(); + + const state = new Database(join(home, "state_5.sqlite"), { readonly: true }); + expect(state.query("SELECT id FROM threads ORDER BY id").all().map(r => r.id)).toEqual([ + "active", "tmid", "tnew", "told", + ]); + state.close(); + }); }); From 9a9653e22c54c2df77eab465662c3bf251657c3e Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Mon, 27 Jul 2026 07:31:19 +0200 Subject: [PATCH 13/17] fix(storage): make satellite lock acquisition and consolidate restore race-safe Roll back and close all acquired satellite write locks when any BEGIN IMMEDIATE fails. Restore the global consolidation job only when the live row still matches cleanup's post-delete image, and add regressions for busy final-satellite rollback and concurrent enqueue watermark updates. --- src/storage/cleanup.ts | 100 ++++++++++++++++++++++++---------- tests/storage-cleanup.test.ts | 74 +++++++++++++++++++++++++ 2 files changed, 146 insertions(+), 28 deletions(-) diff --git a/src/storage/cleanup.ts b/src/storage/cleanup.ts index a3ec9ba0b..e2534dc8d 100644 --- a/src/storage/cleanup.ts +++ b/src/storage/cleanup.ts @@ -498,6 +498,8 @@ interface SatelliteBackup { stage1Jobs: SqlRow[]; consolidateJob: SqlRow | null; consolidateTouched: boolean; + /** Row image after deleteMemoriesInTx; set before memories commit (in-memory only). */ + consolidatePostImage?: SqlRow | null; }; goals?: { path: string; @@ -557,21 +559,50 @@ function updateRowFromSnapshot( ); } -/** Revert delete-time consolidate enqueue; preserve concurrent post-commit status changes. */ -function restoreConsolidateGlobalJob(db: Database, snapshot: SqlRow): void { - const kind = String(snapshot.kind); - const jobKey = String(snapshot.job_key); - const existing = db.query<{ status: string }, [string, string]>( - "SELECT status FROM jobs WHERE kind = ? AND job_key = ?", - ).get(kind, jobKey); - if (!existing) { - insertRowsConflictIgnore(db, "jobs", [snapshot]); +function normalizeSqlValue( + v: string | number | bigint | null | Uint8Array | undefined, +): string { + if (v === null || v === undefined) return ""; + if (typeof v === "bigint") return v.toString(); + if (v instanceof Uint8Array) return Buffer.from(v).toString("base64"); + return String(v); +} + +function sqlRowEqual(a: SqlRow, b: SqlRow): boolean { + const keys = new Set([...Object.keys(a), ...Object.keys(b)]); + for (const key of keys) { + if (normalizeSqlValue(a[key]) !== normalizeSqlValue(b[key])) return false; + } + return true; +} + +function readConsolidateGlobalJob(db: Database): SqlRow | null { + if (!tableExists(db, "jobs")) return null; + return db.query( + "SELECT * FROM jobs WHERE kind = ? AND job_key = ?", + ).get(JOB_KIND_MEMORY_CONSOLIDATE_GLOBAL, MEMORY_CONSOLIDATION_JOB_KEY) as SqlRow | null; +} + +/** Revert delete-time enqueue only when the row still matches cleanup's post-delete image. */ +function restoreConsolidateGlobalJob( + db: Database, + snapshot: SqlRow | null, + postImage: SqlRow | null | undefined, +): void { + if (!postImage) return; + const current = readConsolidateGlobalJob(db); + if (!current) { + if (snapshot) insertRowsConflictIgnore(db, "jobs", [snapshot]); return; } - const snapStatus = String(snapshot.status ?? ""); - const currentStatus = String(existing.status ?? ""); - if (currentStatus === "pending" && snapStatus !== "pending") { + if (!sqlRowEqual(current, postImage)) return; + if (snapshot) { updateRowFromSnapshot(db, "jobs", snapshot, ["kind", "job_key"]); + } else { + db.run( + "DELETE FROM jobs WHERE kind = ? AND job_key = ?", + [JOB_KIND_MEMORY_CONSOLIDATE_GLOBAL, MEMORY_CONSOLIDATION_JOB_KEY], + ); } } @@ -599,17 +630,29 @@ function beginSatelliteWriteLocks( paths: RuntimeDbPaths, busyTimeoutMs: number, ): SatelliteWriteLocks { - const begin = (path: string | null): SatelliteWriteLock | undefined => { - if (!path || !existsSync(path)) return undefined; - const db = openDbWritable(path, busyTimeoutMs); - db.exec("BEGIN IMMEDIATE"); - return { path, db }; - }; - return { - logs: begin(paths.logs), - memories: begin(paths.memories), - goals: begin(paths.goals), - }; + const locks: SatelliteWriteLocks = {}; + const order: Array<{ key: "logs" | "memories" | "goals"; path: string | null }> = [ + { key: "logs", path: paths.logs }, + { key: "memories", path: paths.memories }, + { key: "goals", path: paths.goals }, + ]; + try { + for (const { key, path } of order) { + if (!path || !existsSync(path)) continue; + const db = openDbWritable(path, busyTimeoutMs); + try { + db.exec("BEGIN IMMEDIATE"); + locks[key] = { path, db }; + } catch (error) { + try { db.close(); } catch { /* */ } + throw error; + } + } + return locks; + } catch (error) { + rollbackAllSatelliteLocks(locks); + throw error; + } } function rollbackSatelliteLock(lock: SatelliteWriteLock | undefined): void { @@ -670,9 +713,7 @@ function snapshotMemoriesInTx( `SELECT * FROM jobs WHERE kind = ? AND job_key IN (${placeholders})`, [JOB_KIND_MEMORY_STAGE1, ...threadIds], ); - consolidateJob = db.query( - `SELECT * FROM jobs WHERE kind = ? AND job_key = ?`, - ).get(JOB_KIND_MEMORY_CONSOLIDATE_GLOBAL, MEMORY_CONSOLIDATION_JOB_KEY) as SqlRow | null; + consolidateJob = readConsolidateGlobalJob(db); } return { path, @@ -810,6 +851,9 @@ function deleteAndCommitSatellites( } if (locks.memories && backup.memories) { deleteMemoriesInTx(locks.memories.db, backup.memories); + if (backup.memories.consolidateTouched) { + backup.memories.consolidatePostImage = readConsolidateGlobalJob(locks.memories.db); + } commitSatelliteLock(locks.memories); locks.memories = undefined; if (hooks?.failAfterMemoriesMutation) throw new Error("test_fail_after_memories"); @@ -848,8 +892,8 @@ function restoreSatelliteBackup( insertRowsConflictIgnore(db, "stage1_outputs", mem.stage1); if (tableExists(db, "jobs")) { insertRowsConflictIgnore(db, "jobs", mem.stage1Jobs); - if (mem.consolidateTouched && mem.consolidateJob) { - restoreConsolidateGlobalJob(db, mem.consolidateJob); + if (mem.consolidateTouched) { + restoreConsolidateGlobalJob(db, mem.consolidateJob, mem.consolidatePostImage); } } }); diff --git a/tests/storage-cleanup.test.ts b/tests/storage-cleanup.test.ts index c4295c1d7..061772b32 100644 --- a/tests/storage-cleanup.test.ts +++ b/tests/storage-cleanup.test.ts @@ -379,6 +379,43 @@ describe("executeArchivedCleanup", () => { } }); + test("busy final satellite lock rolls back earlier satellite write locks", () => { + home = buildHome({ withSatelliteStores: true }); + const goalsLocker = new Database(join(home, "goals_1.sqlite")); + goalsLocker.exec("BEGIN EXCLUSIVE"); + try { + const result = runWithDigest(100, "permanent", home, { busyTimeoutMs: 1 }); + expect(result.ok).toBe(false); + expect(result.error).toBe("codex_busy"); + expect(existsSync(join(home, "archived_sessions", "rollout-old.jsonl"))).toBe(true); + expect(existsSync(join(home, ".trash"))).toBe(false); + } finally { + goalsLocker.exec("ROLLBACK"); + goalsLocker.close(); + } + + const logs = new Database(join(home, "logs_2.sqlite")); + logs.exec( + `INSERT INTO logs (ts, level, target, thread_id, estimated_bytes) VALUES (99,'INFO','t','writable-after-busy',99)`, + ); + logs.close(); + const memories = new Database(join(home, "memories_1.sqlite")); + memories.exec( + `INSERT INTO stage1_outputs VALUES ('writable-after-busy',1,'m','s',1,0)`, + ); + memories.close(); + expect( + new Database(join(home, "logs_2.sqlite"), { readonly: true }) + .query("SELECT thread_id FROM logs WHERE thread_id='writable-after-busy'") + .get(), + ).toBeTruthy(); + expect( + new Database(join(home, "memories_1.sqlite"), { readonly: true }) + .query("SELECT thread_id FROM stage1_outputs WHERE thread_id='writable-after-busy'") + .get(), + ).toBeTruthy(); + }); + test("rolls back staged renames when a later rename fails", () => { home = buildHome(); const fresh = previewArchivedCleanup(100, home); @@ -837,4 +874,41 @@ describe("executeArchivedCleanup", () => { ]); state.close(); }); + + test("concurrent consolidate enqueue watermark change is preserved on restore", () => { + home = buildHome({ withSatelliteStores: true }); + const result = runWithDigest(100, "permanent", home, { + now: 97, + _test: { + failBeforeStateCommit: true, + afterSatelliteMutations: () => { + const mem = new Database(join(home, "memories_1.sqlite")); + mem.exec( + `UPDATE jobs SET input_watermark = 99999 + WHERE kind='memory_consolidate_global' AND job_key='global' AND status='pending'`, + ); + mem.close(); + }, + }, + }); + expect(result.ok).toBe(false); + expect(result.error).toBe("db_reconcile_failed"); + + const memories = new Database(join(home, "memories_1.sqlite"), { readonly: true }); + const consolidate = memories.query<{ + status: string; + input_watermark: number | null; + }, []>( + "SELECT status, input_watermark FROM jobs WHERE kind='memory_consolidate_global' AND job_key='global'", + ).get(); + memories.close(); + expect(consolidate?.status).toBe("pending"); + expect(Number(consolidate?.input_watermark ?? 0)).toBe(99999); + + const state = new Database(join(home, "state_5.sqlite"), { readonly: true }); + expect(state.query("SELECT id FROM threads ORDER BY id").all().map(r => r.id)).toEqual([ + "active", "tmid", "tnew", "told", + ]); + state.close(); + }); }); From df23423c1ea8f796a7d2f2cd4c4b709f9b97e3b5 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Mon, 27 Jul 2026 07:58:26 +0200 Subject: [PATCH 14/17] test(storage): close read-only SQLite handles in busy-satellite regression Store logs and memories read-only Database instances in variables and close them in finally blocks so Windows CI does not leak file locks. --- tests/storage-cleanup.test.ts | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/tests/storage-cleanup.test.ts b/tests/storage-cleanup.test.ts index 061772b32..bafdecc66 100644 --- a/tests/storage-cleanup.test.ts +++ b/tests/storage-cleanup.test.ts @@ -404,16 +404,24 @@ describe("executeArchivedCleanup", () => { `INSERT INTO stage1_outputs VALUES ('writable-after-busy',1,'m','s',1,0)`, ); memories.close(); - expect( - new Database(join(home, "logs_2.sqlite"), { readonly: true }) - .query("SELECT thread_id FROM logs WHERE thread_id='writable-after-busy'") - .get(), - ).toBeTruthy(); - expect( - new Database(join(home, "memories_1.sqlite"), { readonly: true }) - .query("SELECT thread_id FROM stage1_outputs WHERE thread_id='writable-after-busy'") - .get(), - ).toBeTruthy(); + + let logsRead: Database | undefined; + let memoriesRead: Database | undefined; + try { + logsRead = new Database(join(home, "logs_2.sqlite"), { readonly: true }); + memoriesRead = new Database(join(home, "memories_1.sqlite"), { readonly: true }); + expect( + logsRead.query("SELECT thread_id FROM logs WHERE thread_id='writable-after-busy'").get(), + ).toBeTruthy(); + expect( + memoriesRead.query( + "SELECT thread_id FROM stage1_outputs WHERE thread_id='writable-after-busy'", + ).get(), + ).toBeTruthy(); + } finally { + try { logsRead?.close(); } catch { /* */ } + try { memoriesRead?.close(); } catch { /* */ } + } }); test("rolls back staged renames when a later rename fails", () => { From 8fe960cba732ea31bc3e021cc7cd05cd2b5cd847 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Mon, 27 Jul 2026 08:19:37 +0200 Subject: [PATCH 15/17] fix(storage): address Codex review on cleanup safety and recovery Bind digest per physical file stats, create exclusive stage dirs with private permissions, journal manifest before staging, persist consolidate post-image before memories commit, batch large SQL IN clauses, validate archive paths, and localize GUI transport errors. --- gui/src/pages/Storage.tsx | 19 +++- src/storage/cleanup.ts | 171 ++++++++++++++++++++++++++-------- tests/storage-cleanup.test.ts | 10 +- 3 files changed, 153 insertions(+), 47 deletions(-) diff --git a/gui/src/pages/Storage.tsx b/gui/src/pages/Storage.tsx index fdef20c4b..d02f09a78 100644 --- a/gui/src/pages/Storage.tsx +++ b/gui/src/pages/Storage.tsx @@ -198,6 +198,21 @@ function ArchivedCleanupPanel({ percent: new Intl.NumberFormat(locale, { style: "percent", maximumFractionDigits: 0 }).format(value / 100), }); + const localizedCatch = (e: unknown, fallback: string): string => { + if (!(e instanceof Error)) return fallback; + const msg = e.message; + if ( + msg === "Failed to fetch" + || msg.includes("NetworkError") + || msg.includes("network error") + || msg.includes("JSON") + || msg.includes("Unexpected end of") + ) { + return fallback; + } + return msg || fallback; + }; + const runPreview = async () => { setBusy(true); setError(null); @@ -213,7 +228,7 @@ function ArchivedCleanupPanel({ setPreview(json); setConfirmOpen(true); } catch (e) { - setError(e instanceof Error ? e.message : String(e)); + setError(localizedCatch(e, t("storage.cleanup.previewFailed"))); } finally { setBusy(false); } @@ -250,7 +265,7 @@ function ArchivedCleanupPanel({ onDone(); } catch (e) { // Keep the dialog open (except stale_preview) so the failure is visible. - setError(e instanceof Error ? e.message : String(e)); + setError(localizedCatch(e, t("storage.cleanup.cleanupFailed"))); } finally { setBusy(false); } diff --git a/src/storage/cleanup.ts b/src/storage/cleanup.ts index e2534dc8d..4f390c237 100644 --- a/src/storage/cleanup.ts +++ b/src/storage/cleanup.ts @@ -22,6 +22,7 @@ import { statSync, unlinkSync, writeFileSync, + chmodSync, } from "node:fs"; import { basename, isAbsolute, join, relative, resolve, sep } from "node:path"; import { Database } from "bun:sqlite"; @@ -51,6 +52,8 @@ export interface ArchivedCandidate { mtimeMs: number; /** All physical files for this logical rollout (`.jsonl` and/or `.jsonl.zst`). */ physicalRelPaths: string[]; + /** Per-physical-file metadata bound into the preview digest. */ + physicalFiles: Array<{ relPath: string; bytes: number; mtimeMs: number }>; } export interface CleanupPreview { @@ -94,6 +97,48 @@ const JOB_KIND_MEMORY_STAGE1 = "memory_stage1"; const JOB_KIND_MEMORY_CONSOLIDATE_GLOBAL = "memory_consolidate_global"; const MEMORY_CONSOLIDATION_JOB_KEY = "global"; const DEFAULT_RETRY_REMAINING = 3; +/** Chunk size for `IN (...)` binds; spawn-edge checks bind each id twice. */ +const SQLITE_ID_CHUNK = 200; + +function chmodPrivatePath(path: string, mode: number): void { + try { chmodSync(path, mode); } catch { /* best-effort (e.g. Windows ACLs) */ } +} + +function writePrivateFile(path: string, content: string): void { + writeFileSync(path, content, "utf8"); + chmodPrivatePath(path, 0o600); +} + +function chunkIds(ids: string[], chunkSize: number): string[][] { + const chunks: string[][] = []; + for (let i = 0; i < ids.length; i += chunkSize) chunks.push(ids.slice(i, i + chunkSize)); + return chunks; +} + +/** Create `.trash/` exclusively; suffix on collision. */ +function createExclusiveStageDir(codexHome: string, epoch: number): string { + const trashRoot = join(codexHome, TRASH_DIR); + mkdirSync(trashRoot, { recursive: true }); + chmodPrivatePath(trashRoot, 0o700); + for (let attempt = 0; attempt < 100; attempt++) { + const name = attempt === 0 ? String(epoch) : `${epoch}-${attempt}`; + const stageDir = join(trashRoot, name); + try { + mkdirSync(stageDir); + chmodPrivatePath(stageDir, 0o700); + return stageDir; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EEXIST") continue; + throw error; + } + } + throw new Error("stage_dir_collision"); +} + +function isSafeArchiveFileName(name: string): boolean { + if (name.includes("/") || name.includes("\\") || name.includes("..")) return false; + return isRolloutFileName(name); +} function clampPercent(percent: unknown): number { if (typeof percent !== "number" || !Number.isFinite(percent)) return 0; @@ -192,7 +237,10 @@ export function normalizeArchivedRolloutPath(rolloutPath: string, codexHome: str export function computePreviewDigest(candidates: ArchivedCandidate[], percent: number): string { const lines = candidates .map(c => { - const physical = [...c.physicalRelPaths].sort().join(","); + const physical = [...c.physicalFiles] + .sort((a, b) => a.relPath.localeCompare(b.relPath)) + .map(f => `${f.relPath}|${f.bytes}|${Math.trunc(f.mtimeMs)}`) + .join(","); return `${c.relPath}|${c.bytes}|${Math.trunc(c.mtimeMs)}|${physical}`; }) .sort(); @@ -218,7 +266,7 @@ export function listArchivedCandidates(codexHome: string): ArchivedCandidate[] { const groups = new Map(); for (const name of names) { - if (!isRolloutFileName(name)) continue; + if (!isSafeArchiveFileName(name)) continue; const absPath = join(dir, name); try { const st = statSync(absPath); @@ -255,6 +303,7 @@ export function listArchivedCandidates(codexHome: string): ArchivedCandidate[] { bytes: acc.files.reduce((sum, f) => sum + f.bytes, 0), mtimeMs: Math.min(...acc.files.map(f => f.mtimeMs)), physicalRelPaths: acc.files.map(f => f.relPath), + physicalFiles: acc.files.map(f => ({ relPath: f.relPath, bytes: f.bytes, mtimeMs: f.mtimeMs })), }); } out.sort((a, b) => a.mtimeMs - b.mtimeMs || a.relPath.localeCompare(b.relPath)); @@ -416,14 +465,16 @@ function findReferencedHistory( // Spawn edges that cross the delete boundary keep history reachable. if (tableExists(db, "thread_spawn_edges")) { - const placeholders = ids.map(() => "?").join(","); - const edges = db.query<{ parent_thread_id: string; child_thread_id: string }, string[]>( - `SELECT parent_thread_id, child_thread_id FROM thread_spawn_edges - WHERE parent_thread_id IN (${placeholders}) OR child_thread_id IN (${placeholders})`, - ).all(...ids, ...ids); - for (const edge of edges) { - if (!idSet.has(edge.parent_thread_id) || !idSet.has(edge.child_thread_id)) { - return true; + for (const chunk of chunkIds(ids, SQLITE_ID_CHUNK)) { + const placeholders = chunk.map(() => "?").join(","); + const edges = db.query<{ parent_thread_id: string; child_thread_id: string }, string[]>( + `SELECT parent_thread_id, child_thread_id FROM thread_spawn_edges + WHERE parent_thread_id IN (${placeholders}) OR child_thread_id IN (${placeholders})`, + ).all(...chunk, ...chunk); + for (const edge of edges) { + if (!idSet.has(edge.parent_thread_id) || !idSet.has(edge.child_thread_id)) { + return true; + } } } } @@ -431,11 +482,13 @@ function findReferencedHistory( // Other threads that list one of ours as forked_from / parent (when columns exist). for (const column of ["forked_from_id", "parent_thread_id", "source_thread_id"] as const) { if (!columnExists(db, "threads", column)) continue; - const placeholders = ids.map(() => "?").join(","); - const rows = db.query<{ id: string }, string[]>( - `SELECT id FROM threads WHERE ${column} IN (${placeholders})`, - ).all(...ids); - if (rows.some(r => !idSet.has(r.id))) return true; + for (const chunk of chunkIds(ids, SQLITE_ID_CHUNK * 2)) { + const placeholders = chunk.map(() => "?").join(","); + const rows = db.query<{ id: string }, string[]>( + `SELECT id FROM threads WHERE ${column} IN (${placeholders})`, + ).all(...chunk); + if (rows.some(r => !idSet.has(r.id))) return true; + } } return false; @@ -459,21 +512,29 @@ function columnExists(db: Database, table: string, column: string): boolean { function deleteThreadsAndDependents(db: Database, threadIds: string[]): void { if (threadIds.length === 0) return; - const placeholders = threadIds.map(() => "?").join(","); // Upstream deletes dynamic tools before spawn edges before threads. if (tableExists(db, "thread_dynamic_tools")) { - db.run(`DELETE FROM thread_dynamic_tools WHERE thread_id IN (${placeholders})`, threadIds); + for (const chunk of chunkIds(threadIds, SQLITE_ID_CHUNK * 2)) { + const placeholders = chunk.map(() => "?").join(","); + db.run(`DELETE FROM thread_dynamic_tools WHERE thread_id IN (${placeholders})`, chunk); + } } if (tableExists(db, "thread_spawn_edges")) { - db.run( - `DELETE FROM thread_spawn_edges WHERE parent_thread_id IN (${placeholders}) OR child_thread_id IN (${placeholders})`, - [...threadIds, ...threadIds], - ); + for (const chunk of chunkIds(threadIds, SQLITE_ID_CHUNK)) { + const placeholders = chunk.map(() => "?").join(","); + db.run( + `DELETE FROM thread_spawn_edges WHERE parent_thread_id IN (${placeholders}) OR child_thread_id IN (${placeholders})`, + [...chunk, ...chunk], + ); + } } - db.run(`DELETE FROM threads WHERE id IN (${placeholders})`, threadIds); + for (const chunk of chunkIds(threadIds, SQLITE_ID_CHUNK * 2)) { + const placeholders = chunk.map(() => "?").join(","); + db.run(`DELETE FROM threads WHERE id IN (${placeholders})`, chunk); + } } interface ReconcileOk { @@ -607,7 +668,7 @@ function restoreConsolidateGlobalJob( } function writeSatelliteBackup(stageDir: string, backup: SatelliteBackup): void { - writeFileSync(join(stageDir, SATELLITE_BACKUP_FILE), JSON.stringify(backup), "utf8"); + writePrivateFile(join(stageDir, SATELLITE_BACKUP_FILE), JSON.stringify(backup)); } function clearSatelliteBackup(stageDir: string): void { @@ -840,6 +901,7 @@ function deleteGoalsInTx( function deleteAndCommitSatellites( locks: SatelliteWriteLocks, backup: SatelliteBackup, + stageDir: string, hooks?: ReconcileTestHooks, ): void { try { @@ -853,6 +915,7 @@ function deleteAndCommitSatellites( deleteMemoriesInTx(locks.memories.db, backup.memories); if (backup.memories.consolidateTouched) { backup.memories.consolidatePostImage = readConsolidateGlobalJob(locks.memories.db); + writeSatelliteBackup(stageDir, backup); } commitSatelliteLock(locks.memories); locks.memories = undefined; @@ -1036,7 +1099,7 @@ function reconcileDeletedThreads( const hasSatelliteWork = Boolean(backup.logs || backup.memories || backup.goals); if (hasSatelliteWork) { satellitesMutated = true; - deleteAndCommitSatellites(satelliteLocks, backup, hooks); + deleteAndCommitSatellites(satelliteLocks, backup, stageDir, hooks); } else { rollbackAllSatelliteLocks(satelliteLocks); } @@ -1069,13 +1132,21 @@ function reconcileDeletedThreads( type StagedFile = { from: string; to: string; relPath: string }; function absFromRel(codexHome: string, relPath: string): string { - return join(codexHome, ...relPath.split("/")); + if (relPath.includes("..") || isAbsolute(relPath) || /^[A-Za-z]:[\\/]/.test(relPath)) { + throw new Error("invalid_rel_path"); + } + const abs = resolve(codexHome, ...relPath.split("/")); + const homeAbs = resolve(codexHome); + const rel = toForwardSlash(relative(homeAbs, abs)); + if (!rel || rel.startsWith("..")) throw new Error("path_escape"); + return abs; } function stageCandidates( codexHome: string, candidates: ArchivedCandidate[], stageDir: string, + opts?: { blockDestBasenames?: Set }, ): { ok: true; staged: StagedFile[] } | { ok: false; staged: StagedFile[] } { const staged: StagedFile[] = []; const usedBasenames = new Set(); @@ -1092,6 +1163,9 @@ function stageCandidates( } usedBasenames.add(base); const to = join(stageDir, base); + if (opts?.blockDestBasenames?.has(base)) { + mkdirSync(to, { recursive: true }); + } renameSync(from, to); staged.push({ from, to, relPath: rel }); } @@ -1187,6 +1261,7 @@ export interface ExecuteCleanupOptions { failManifestWrite?: boolean; failPurgeBasenames?: string[]; failRollbackBasenames?: string[]; + blockStageDestBasenames?: string[]; failAfterLogsMutation?: boolean; failAfterMemoriesMutation?: boolean; failAfterGoalsMutation?: boolean; @@ -1227,6 +1302,7 @@ export function executeArchivedCleanup(options: ExecuteCleanupOptions): CleanupR const busyTimeoutMs = options.busyTimeoutMs ?? 100; const failRollback = new Set(options._test?.failRollbackBasenames ?? []); const failPurge = new Set(options._test?.failPurgeBasenames ?? []); + const blockStageDest = new Set(options._test?.blockStageDestBasenames ?? []); if (mode !== "quarantine" && mode !== "permanent") { return fail(mode, percent, "invalid_mode"); @@ -1264,16 +1340,9 @@ export function executeArchivedCleanup(options: ExecuteCleanupOptions): CleanupR } const epoch = options.now ?? Date.now(); - const stageDir = join(codexHome, TRASH_DIR, String(epoch)); + const stageDir = createExclusiveStageDir(codexHome, epoch); const trashDir = trashRelPath(codexHome, stageDir); - const stageResult = stageCandidates(codexHome, preview.candidates, stageDir); - if (!stageResult.ok) { - const rolled = rollbackStaged(stageResult.staged, { failBasenames: failRollback }); - removeStageIfEmpty(stageDir, rolled.remaining); - return fail(mode, percent, "fs_failed", rolled.restored ? undefined : { trashDir }); - } - const threadByRelPath = new Map(); for (const thread of loaded.threads) { const normalized = normalizeArchivedRolloutPath(thread.rollout_path, codexHome); @@ -1292,12 +1361,8 @@ export function executeArchivedCleanup(options: ExecuteCleanupOptions): CleanupR }; }); - // Manifest must exist before DB deletion so a mid-flight crash still has recovery metadata. - try { - if (options._test?.failManifestWrite) { - throw new Error("test_fail_manifest_write"); - } - writeFileSync( + const writeManifest = (extra: Record = {}) => { + writePrivateFile( join(stageDir, "manifest.json"), JSON.stringify({ quarantinedAt: epoch, @@ -1305,8 +1370,34 @@ export function executeArchivedCleanup(options: ExecuteCleanupOptions): CleanupR percent, digest: preview.digest, entries: manifestEntries, + ...extra, }, null, 2), ); + }; + + // Journal staged paths before the first rename so a crash mid-stage is recoverable. + try { + if (options._test?.failManifestWrite) { + throw new Error("test_fail_manifest_write"); + } + writeManifest({ staging: true }); + } catch { + removeStageIfEmpty(stageDir, []); + return fail(mode, percent, "fs_failed"); + } + + const stageResult = stageCandidates(codexHome, preview.candidates, stageDir, { + blockDestBasenames: blockStageDest.size > 0 ? blockStageDest : undefined, + }); + if (!stageResult.ok) { + const rolled = rollbackStaged(stageResult.staged, { failBasenames: failRollback }); + removeStageIfEmpty(stageDir, rolled.remaining); + return fail(mode, percent, "fs_failed", rolled.restored ? undefined : { trashDir }); + } + + // Final manifest before DB deletion so a mid-flight crash still has recovery metadata. + try { + writeManifest(); } catch { const rolled = rollbackStaged(stageResult.staged, { failBasenames: failRollback }); removeStageIfEmpty(stageDir, rolled.remaining); @@ -1353,7 +1444,7 @@ export function executeArchivedCleanup(options: ExecuteCleanupOptions): CleanupR // Overwrite the pre-commit manifest so recovery reflects what actually survived. const survivingRelPaths = new Set(purge.remaining.map(item => item.relPath)); try { - writeFileSync( + writePrivateFile( join(stageDir, "manifest.json"), JSON.stringify({ quarantinedAt: epoch, diff --git a/tests/storage-cleanup.test.ts b/tests/storage-cleanup.test.ts index bafdecc66..fa44a893f 100644 --- a/tests/storage-cleanup.test.ts +++ b/tests/storage-cleanup.test.ts @@ -429,17 +429,17 @@ describe("executeArchivedCleanup", () => { const fresh = previewArchivedCleanup(100, home); expect(fresh.count).toBe(3); - // Allow stageDir creation, then make the second candidate's destination a directory - // so renameSync fails after the first file has already moved. - mkdirSync(join(home, ".trash", "42"), { recursive: true }); - mkdirSync(join(home, ".trash", "42", "rollout-mid.jsonl")); + // Exclusive stage dir uses `.trash/42-1` when `.trash/42` already exists. + const now = 42; + mkdirSync(join(home, ".trash", String(now)), { recursive: true }); const result = executeArchivedCleanup({ percent: 100, mode: "quarantine", digest: fresh.digest, codexHome: home, - now: 42, + now, + _test: { blockStageDestBasenames: ["rollout-mid.jsonl"] }, }); expect(result.ok).toBe(false); expect(result.error).toBe("fs_failed"); From 2caecb9b4348704972a26d1797bb40b024228152 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Mon, 27 Jul 2026 08:32:49 +0200 Subject: [PATCH 16/17] fix(storage): address CodeRabbit review on chunking and errors Batch satellite snapshot/delete IN queries to stay under SQLite variable limits, return fs_failed when exclusive stage-dir creation fails, and split injected-failure regressions into test.each cases. --- src/storage/cleanup.ts | 108 +++++++++++++++++++++------------- tests/storage-cleanup.test.ts | 24 ++++---- 2 files changed, 76 insertions(+), 56 deletions(-) diff --git a/src/storage/cleanup.ts b/src/storage/cleanup.ts index 4f390c237..66d128222 100644 --- a/src/storage/cleanup.ts +++ b/src/storage/cleanup.ts @@ -744,8 +744,11 @@ function snapshotLogsInTx( ): SatelliteBackup["logs"] { if (threadIds.length === 0) return undefined; if (!tableExists(db, "logs")) throw new Error("missing_logs_table"); - const placeholders = threadIds.map(() => "?").join(","); - const rows = selectRows(db, `SELECT * FROM logs WHERE thread_id IN (${placeholders})`, threadIds); + const rows: SqlRow[] = []; + for (const chunk of chunkIds(threadIds, SQLITE_ID_CHUNK * 2)) { + const placeholders = chunk.map(() => "?").join(","); + rows.push(...selectRows(db, `SELECT * FROM logs WHERE thread_id IN (${placeholders})`, chunk)); + } return { path, rows }; } @@ -756,24 +759,29 @@ function snapshotMemoriesInTx( ): SatelliteBackup["memories"] { if (threadIds.length === 0) return undefined; if (!tableExists(db, "stage1_outputs")) throw new Error("missing_stage1_outputs_table"); - const placeholders = threadIds.map(() => "?").join(","); - const stage1 = selectRows( - db, - `SELECT * FROM stage1_outputs WHERE thread_id IN (${placeholders})`, - threadIds, - ); + const stage1: SqlRow[] = []; let stage1Jobs: SqlRow[] = []; + for (const chunk of chunkIds(threadIds, SQLITE_ID_CHUNK * 2)) { + const placeholders = chunk.map(() => "?").join(","); + stage1.push(...selectRows( + db, + `SELECT * FROM stage1_outputs WHERE thread_id IN (${placeholders})`, + chunk, + )); + if (tableExists(db, "jobs")) { + stage1Jobs.push(...selectRows( + db, + `SELECT * FROM jobs WHERE kind = ? AND job_key IN (${placeholders})`, + [JOB_KIND_MEMORY_STAGE1, ...chunk], + )); + } + } let consolidateJob: SqlRow | null = null; let selectedForPhase2 = 0; if (columnExists(db, "stage1_outputs", "selected_for_phase2")) { selectedForPhase2 = stage1.filter(r => Number(r.selected_for_phase2 ?? 0) !== 0).length; } if (tableExists(db, "jobs")) { - stage1Jobs = selectRows( - db, - `SELECT * FROM jobs WHERE kind = ? AND job_key IN (${placeholders})`, - [JOB_KIND_MEMORY_STAGE1, ...threadIds], - ); consolidateJob = readConsolidateGlobalJob(db); } return { @@ -792,19 +800,22 @@ function snapshotGoalsInTx( ): SatelliteBackup["goals"] { if (threadIds.length === 0) return undefined; if (!tableExists(db, "thread_goals")) throw new Error("missing_thread_goals_table"); - const placeholders = threadIds.map(() => "?").join(","); - const goals = selectRows( - db, - `SELECT * FROM thread_goals WHERE thread_id IN (${placeholders})`, - threadIds, - ); + const goals: SqlRow[] = []; let deferrals: SqlRow[] = []; - if (tableExists(db, "thread_goal_continuation_deferrals")) { - deferrals = selectRows( + for (const chunk of chunkIds(threadIds, SQLITE_ID_CHUNK * 2)) { + const placeholders = chunk.map(() => "?").join(","); + goals.push(...selectRows( db, - `SELECT * FROM thread_goal_continuation_deferrals WHERE thread_id IN (${placeholders})`, - threadIds, - ); + `SELECT * FROM thread_goals WHERE thread_id IN (${placeholders})`, + chunk, + )); + if (tableExists(db, "thread_goal_continuation_deferrals")) { + deferrals.push(...selectRows( + db, + `SELECT * FROM thread_goal_continuation_deferrals WHERE thread_id IN (${placeholders})`, + chunk, + )); + } } return { path, goals, deferrals }; } @@ -832,8 +843,10 @@ function deleteLogsInTx(db: Database, rows: SqlRow[]): void { if (!tableExists(db, "logs")) throw new Error("missing_logs_table"); const ids = rows.map(r => r.id).filter(id => id !== null && id !== undefined); if (ids.length === 0) return; - const placeholders = ids.map(() => "?").join(","); - db.run(`DELETE FROM logs WHERE id IN (${placeholders})`, ids as Array); + for (const chunk of chunkIds(ids as string[], SQLITE_ID_CHUNK * 2)) { + const placeholders = chunk.map(() => "?").join(","); + db.run(`DELETE FROM logs WHERE id IN (${placeholders})`, chunk as Array); + } } function deleteMemoriesInTx( @@ -842,17 +855,19 @@ function deleteMemoriesInTx( ): void { if (!tableExists(db, "stage1_outputs")) throw new Error("missing_stage1_outputs_table"); const stage1Ids = section.stage1.map(r => String(r.thread_id)); - if (stage1Ids.length > 0) { - const placeholders = stage1Ids.map(() => "?").join(","); - db.run(`DELETE FROM stage1_outputs WHERE thread_id IN (${placeholders})`, stage1Ids); + for (const chunk of chunkIds(stage1Ids, SQLITE_ID_CHUNK * 2)) { + if (chunk.length === 0) continue; + const placeholders = chunk.map(() => "?").join(","); + db.run(`DELETE FROM stage1_outputs WHERE thread_id IN (${placeholders})`, chunk); } if (tableExists(db, "jobs")) { const jobKeys = section.stage1Jobs.map(r => String(r.job_key)); - if (jobKeys.length > 0) { - const placeholders = jobKeys.map(() => "?").join(","); + for (const chunk of chunkIds(jobKeys, SQLITE_ID_CHUNK * 2)) { + if (chunk.length === 0) continue; + const placeholders = chunk.map(() => "?").join(","); db.run( `DELETE FROM jobs WHERE kind = ? AND job_key IN (${placeholders})`, - [JOB_KIND_MEMORY_STAGE1, ...jobKeys], + [JOB_KIND_MEMORY_STAGE1, ...chunk], ); } if (section.consolidateTouched) { @@ -883,17 +898,21 @@ function deleteGoalsInTx( ): void { if (!tableExists(db, "thread_goals")) throw new Error("missing_thread_goals_table"); const deferralIds = section.deferrals.map(r => String(r.thread_id)); - if (deferralIds.length > 0 && tableExists(db, "thread_goal_continuation_deferrals")) { - const placeholders = deferralIds.map(() => "?").join(","); - db.run( - `DELETE FROM thread_goal_continuation_deferrals WHERE thread_id IN (${placeholders})`, - deferralIds, - ); + if (tableExists(db, "thread_goal_continuation_deferrals")) { + for (const chunk of chunkIds(deferralIds, SQLITE_ID_CHUNK * 2)) { + if (chunk.length === 0) continue; + const placeholders = chunk.map(() => "?").join(","); + db.run( + `DELETE FROM thread_goal_continuation_deferrals WHERE thread_id IN (${placeholders})`, + chunk, + ); + } } const goalIds = section.goals.map(r => String(r.thread_id)); - if (goalIds.length > 0) { - const placeholders = goalIds.map(() => "?").join(","); - db.run(`DELETE FROM thread_goals WHERE thread_id IN (${placeholders})`, goalIds); + for (const chunk of chunkIds(goalIds, SQLITE_ID_CHUNK * 2)) { + if (chunk.length === 0) continue; + const placeholders = chunk.map(() => "?").join(","); + db.run(`DELETE FROM thread_goals WHERE thread_id IN (${placeholders})`, chunk); } } @@ -1340,7 +1359,12 @@ export function executeArchivedCleanup(options: ExecuteCleanupOptions): CleanupR } const epoch = options.now ?? Date.now(); - const stageDir = createExclusiveStageDir(codexHome, epoch); + let stageDir: string; + try { + stageDir = createExclusiveStageDir(codexHome, epoch); + } catch { + return fail(mode, percent, "fs_failed"); + } const trashDir = trashRelPath(codexHome, stageDir); const threadByRelPath = new Map(); diff --git a/tests/storage-cleanup.test.ts b/tests/storage-cleanup.test.ts index fa44a893f..30cc4d37b 100644 --- a/tests/storage-cleanup.test.ts +++ b/tests/storage-cleanup.test.ts @@ -671,15 +671,14 @@ describe("executeArchivedCleanup", () => { expect(Buffer.compare(beforeState, readFileSync(join(home, "state_5.sqlite")))).toBe(0); }); - test("injected satellite/state failures restore every file and database", () => { - const hooks = [ - { failAfterLogsMutation: true }, - { failAfterMemoriesMutation: true }, - { failAfterGoalsMutation: true }, - { failBeforeStateCommit: true }, - ] as const; - - for (const hook of hooks) { + test.each([ + ["failAfterLogsMutation", { failAfterLogsMutation: true }], + ["failAfterMemoriesMutation", { failAfterMemoriesMutation: true }], + ["failAfterGoalsMutation", { failAfterGoalsMutation: true }], + ["failBeforeStateCommit", { failBeforeStateCommit: true }], + ] as const)( + "injected %s restores every file and database", + (_name, hook) => { home = buildHome({ withSatelliteStores: true }); const files = { old: readFileSync(join(home, "archived_sessions", "rollout-old.jsonl")), @@ -724,11 +723,8 @@ describe("executeArchivedCleanup", () => { const stateAfter = new Database(join(home, "state_5.sqlite"), { readonly: true }); expect(stateAfter.query("SELECT id, rollout_path, archived FROM threads ORDER BY id").all()).toEqual(threads); stateAfter.close(); - - rmSync(home, { recursive: true, force: true }); - home = ""; - } - }); + }, + ); test("satellite restore failure keeps recovery trashDir and manifest", () => { home = buildHome({ withSatelliteStores: true }); From 3b0c85d2997a3c5eed530039539caf37fe0fed3d Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Mon, 27 Jul 2026 08:36:15 +0200 Subject: [PATCH 17/17] fix(storage): allowlist cleanup test hooks on management API Restrict wire-forwarded _test hooks via pickWireCleanupTestHooks and align runWithDigest typing with ExecuteCleanupOptions. --- src/server/management/logs-usage-routes.ts | 12 +++------- src/storage/cleanup.ts | 28 ++++++++++++++++++++++ tests/storage-cleanup.test.ts | 14 ++--------- 3 files changed, 33 insertions(+), 21 deletions(-) diff --git a/src/server/management/logs-usage-routes.ts b/src/server/management/logs-usage-routes.ts index 6478b6874..ec0bd4308 100644 --- a/src/server/management/logs-usage-routes.ts +++ b/src/server/management/logs-usage-routes.ts @@ -34,7 +34,7 @@ import { primeCodexPoolQuotas } from "../../codex/auth-api"; import { DEFAULT_PROVIDER_CONTEXT_CAP, globalContextCapValue, providerContextCap, providerContextCaps, setAllProviderContextCaps, setGlobalContextCapValue, setProviderContextCap } from "../../providers/context-cap"; import { resolveCodexHomeDir } from "../../codex/home"; import { scanStorage } from "../../storage/scanner"; -import { executeArchivedCleanup, previewArchivedCleanup, type CleanupMode } from "../../storage/cleanup"; +import { executeArchivedCleanup, pickWireCleanupTestHooks, previewArchivedCleanup, type CleanupMode } from "../../storage/cleanup"; import { currentUsageLogRevision, readUsageSnapshotForManagement, @@ -274,14 +274,8 @@ export async function handleLogsUsageRoutes(ctx: ManagementContext): Promise, + "afterSatelliteMutations" +>; + +function isStringArray(v: unknown): v is string[] { + return Array.isArray(v) && v.every(e => typeof e === "string"); +} + +/** Pick only allowlisted serializable hooks; drops afterSatelliteMutations and unknown keys. */ +export function pickWireCleanupTestHooks(raw: unknown): CleanupWireTestHooks | undefined { + if (!raw || typeof raw !== "object") return undefined; + const o = raw as Record; + const out: CleanupWireTestHooks = {}; + if (typeof o.failManifestWrite === "boolean") out.failManifestWrite = o.failManifestWrite; + if (isStringArray(o.failPurgeBasenames)) out.failPurgeBasenames = o.failPurgeBasenames; + if (isStringArray(o.failRollbackBasenames)) out.failRollbackBasenames = o.failRollbackBasenames; + if (isStringArray(o.blockStageDestBasenames)) out.blockStageDestBasenames = o.blockStageDestBasenames; + if (typeof o.failAfterLogsMutation === "boolean") out.failAfterLogsMutation = o.failAfterLogsMutation; + if (typeof o.failAfterMemoriesMutation === "boolean") out.failAfterMemoriesMutation = o.failAfterMemoriesMutation; + if (typeof o.failAfterGoalsMutation === "boolean") out.failAfterGoalsMutation = o.failAfterGoalsMutation; + if (typeof o.failBeforeStateCommit === "boolean") out.failBeforeStateCommit = o.failBeforeStateCommit; + if (typeof o.failSatelliteRestore === "boolean") out.failSatelliteRestore = o.failSatelliteRestore; + if (typeof o.failSatelliteBackupWrite === "boolean") out.failSatelliteBackupWrite = o.failSatelliteBackupWrite; + return Object.keys(out).length > 0 ? out : undefined; +} + function fail( mode: CleanupMode, percent: number, diff --git a/tests/storage-cleanup.test.ts b/tests/storage-cleanup.test.ts index 30cc4d37b..b28258e00 100644 --- a/tests/storage-cleanup.test.ts +++ b/tests/storage-cleanup.test.ts @@ -18,6 +18,7 @@ import { normalizeArchivedRolloutPath, previewArchivedCleanup, selectOldestPercent, + type ExecuteCleanupOptions, } from "../src/storage/cleanup"; const OLD = new Date("2026-01-01T00:00:00Z"); @@ -188,18 +189,7 @@ function runWithDigest( busyTimeoutMs?: number; now?: number; digest?: string; - _test?: { - failManifestWrite?: boolean; - failPurgeBasenames?: string[]; - failRollbackBasenames?: string[]; - failAfterLogsMutation?: boolean; - failAfterMemoriesMutation?: boolean; - failAfterGoalsMutation?: boolean; - failBeforeStateCommit?: boolean; - failSatelliteRestore?: boolean; - failSatelliteBackupWrite?: boolean; - afterSatelliteMutations?: () => void; - }; + _test?: ExecuteCleanupOptions["_test"]; }, ) { const preview = previewArchivedCleanup(percent, codexHome);