diff --git a/.claude/worktrees/agent-a09a3e20 b/.claude/worktrees/agent-a09a3e20 new file mode 160000 index 0000000..5f35206 --- /dev/null +++ b/.claude/worktrees/agent-a09a3e20 @@ -0,0 +1 @@ +Subproject commit 5f3520658d949372276bf27be0aa8b2ff0d9193c diff --git a/.claude/worktrees/agent-ad3369ec b/.claude/worktrees/agent-ad3369ec new file mode 160000 index 0000000..5f35206 --- /dev/null +++ b/.claude/worktrees/agent-ad3369ec @@ -0,0 +1 @@ +Subproject commit 5f3520658d949372276bf27be0aa8b2ff0d9193c diff --git a/src/connectors/index.ts b/src/connectors/index.ts index 2885f8a..6a44d1c 100644 --- a/src/connectors/index.ts +++ b/src/connectors/index.ts @@ -111,11 +111,11 @@ export function deleteDbConnectorConfig(db: Database.Database, type: string): bo const CONNECTORS_DIR = join(homedir(), ".libscope", "connectors"); function ensureConnectorsDir(): void { - if (!existsSync(CONNECTORS_DIR)) { - mkdirSync(CONNECTORS_DIR, { recursive: true, mode: 0o700 }); - } else { + if (existsSync(CONNECTORS_DIR)) { // Remediate existing directories that may have permissive permissions restrictPermissions(CONNECTORS_DIR, 0o700); + } else { + mkdirSync(CONNECTORS_DIR, { recursive: true, mode: 0o700 }); } try { chmodSync(CONNECTORS_DIR, 0o700); @@ -163,13 +163,12 @@ export function hasNamedConnectorConfig(name: string): boolean { return existsSync(filePath); } -/** Delete documents with a given source_type from the database. Returns count deleted. */ -export function deleteConnectorDocuments(db: Database.Database, sourceType: string): number { - const rows = db - .prepare("SELECT id FROM documents WHERE source_type = ?") - .all(sourceType) as Array<{ id: string }>; - if (rows.length === 0) return 0; - +/** + * Delete a set of document rows and their associated chunks, embeddings, and FTS entries. + * Tolerates missing virtual tables (chunk_embeddings, chunks_fts). + */ +export function deleteDocumentRows(db: Database.Database, rows: Array<{ id: string }>): void { + const log = getLogger(); const deleteChunksFts = db.prepare( "DELETE FROM chunks_fts WHERE rowid IN (SELECT rowid FROM chunks_fts WHERE chunk_id IN (SELECT id FROM chunks WHERE document_id = ?))", ); @@ -179,28 +178,33 @@ export function deleteConnectorDocuments(db: Database.Database, sourceType: stri const deleteChunks = db.prepare("DELETE FROM chunks WHERE document_id = ?"); const deleteDoc = db.prepare("DELETE FROM documents WHERE id = ?"); - const tx = db.transaction(() => { - for (const row of rows) { - try { - deleteChunksFts.run(row.id); - } catch (err) { - getLogger().debug( - { err, documentId: row.id }, - "FTS table cleanup skipped (table may not exist)", - ); - } - try { - deleteEmbeddings.run(row.id); - } catch (err) { - getLogger().debug( - { err, documentId: row.id }, - "chunk_embeddings cleanup skipped (table may not exist)", - ); - } - deleteChunks.run(row.id); - deleteDoc.run(row.id); + for (const row of rows) { + try { + deleteChunksFts.run(row.id); + } catch (err) { + log.debug({ err, documentId: row.id }, "FTS table cleanup skipped (table may not exist)"); + } + try { + deleteEmbeddings.run(row.id); + } catch (err) { + log.debug( + { err, documentId: row.id }, + "chunk_embeddings cleanup skipped (table may not exist)", + ); } - }); + deleteChunks.run(row.id); + deleteDoc.run(row.id); + } +} + +/** Delete documents with a given source_type from the database. Returns count deleted. */ +export function deleteConnectorDocuments(db: Database.Database, sourceType: string): number { + const rows = db + .prepare("SELECT id FROM documents WHERE source_type = ?") + .all(sourceType) as Array<{ id: string }>; + if (rows.length === 0) return 0; + + const tx = db.transaction(() => deleteDocumentRows(db, rows)); tx(); return rows.length; diff --git a/src/connectors/slack.ts b/src/connectors/slack.ts index 5257a38..26dc1cd 100644 --- a/src/connectors/slack.ts +++ b/src/connectors/slack.ts @@ -5,6 +5,7 @@ import { getLogger } from "../logger.js"; import { LibScopeError, ValidationError } from "../errors.js"; import { fetchWithRetry } from "./http-utils.js"; import { startSync, completeSync, failSync } from "./sync-tracker.js"; +import { deleteDocumentRows } from "./index.js"; export interface SlackConfig { token: string; @@ -569,40 +570,9 @@ export function disconnectSlack(db: Database.Database): number { const rows = db.prepare("SELECT id FROM documents WHERE url LIKE 'slack://%'").all() as Array<{ id: string; }>; - if (rows.length === 0) return 0; - const deleteChunksFts = db.prepare( - "DELETE FROM chunks_fts WHERE rowid IN (SELECT rowid FROM chunks_fts WHERE chunk_id IN (SELECT id FROM chunks WHERE document_id = ?))", - ); - const deleteEmbeddings = db.prepare( - "DELETE FROM chunk_embeddings WHERE chunk_id IN (SELECT id FROM chunks WHERE document_id = ?)", - ); - const deleteChunks = db.prepare("DELETE FROM chunks WHERE document_id = ?"); - const deleteDoc = db.prepare("DELETE FROM documents WHERE id = ?"); - - const tx = db.transaction(() => { - for (const row of rows) { - try { - deleteChunksFts.run(row.id); - } catch (err) { - getLogger().debug( - { err, documentId: row.id }, - "FTS table cleanup skipped (table may not exist)", - ); - } - try { - deleteEmbeddings.run(row.id); - } catch (err) { - getLogger().debug( - { err, documentId: row.id }, - "chunk_embeddings cleanup skipped (table may not exist)", - ); - } - deleteChunks.run(row.id); - deleteDoc.run(row.id); - } - }); + const tx = db.transaction(() => deleteDocumentRows(db, rows)); tx(); return rows.length; diff --git a/src/core/bulk.ts b/src/core/bulk.ts index 1df1438..d00b15e 100644 --- a/src/core/bulk.ts +++ b/src/core/bulk.ts @@ -92,7 +92,9 @@ export function bulkDelete( const log = getLogger(); const ids = resolveSelector(db, selector); - if (!dryRun) { + if (dryRun) { + log.info({ count: ids.length, dryRun: true }, "Bulk delete dry run"); + } else { const deleteAll = db.transaction(() => { for (const id of ids) { deleteDocument(db, id); @@ -100,8 +102,6 @@ export function bulkDelete( }); deleteAll(); log.info({ count: ids.length, dryRun: false }, "Bulk delete completed"); - } else { - log.info({ count: ids.length, dryRun: true }, "Bulk delete dry run"); } return { affected: ids.length, documentIds: ids }; @@ -123,7 +123,9 @@ export function bulkRetag( const ids = resolveSelector(db, selector); - if (!dryRun) { + if (dryRun) { + log.info({ count: ids.length, addTags, removeTags, dryRun: true }, "Bulk retag dry run"); + } else { const retagAll = db.transaction(() => { for (const id of ids) { if (addTags && addTags.length > 0) { @@ -143,8 +145,6 @@ export function bulkRetag( }); retagAll(); log.info({ count: ids.length, addTags, removeTags, dryRun: false }, "Bulk retag completed"); - } else { - log.info({ count: ids.length, addTags, removeTags, dryRun: true }, "Bulk retag dry run"); } return { affected: ids.length, documentIds: ids }; @@ -160,7 +160,9 @@ export function bulkMove( const log = getLogger(); const ids = resolveSelector(db, selector); - if (!dryRun) { + if (dryRun) { + log.info({ count: ids.length, targetTopicId, dryRun: true }, "Bulk move dry run"); + } else { const moveAll = db.transaction(() => { const stmt = db.prepare( "UPDATE documents SET topic_id = ?, updated_at = datetime('now') WHERE id = ?", @@ -171,8 +173,6 @@ export function bulkMove( }); moveAll(); log.info({ count: ids.length, targetTopicId, dryRun: false }, "Bulk move completed"); - } else { - log.info({ count: ids.length, targetTopicId, dryRun: true }, "Bulk move dry run"); } return { affected: ids.length, documentIds: ids }; diff --git a/src/core/repo.ts b/src/core/repo.ts index 79761ae..e9a8a8b 100644 --- a/src/core/repo.ts +++ b/src/core/repo.ts @@ -105,7 +105,7 @@ export function parseRepoUrl(url: string): ParsedRepoUrl { // ── SSRF Protection ────────────────────────────────────────────────────────── async function validateHost(hostname: string): Promise { - const stripped = hostname.replace(/^\[|\]$/g, ""); + const stripped = hostname.replaceAll(/^\[|\]$/g, ""); const results = await Promise.allSettled([dns.resolve4(stripped), dns.resolve6(stripped)]); const addresses: string[] = []; diff --git a/src/core/search.ts b/src/core/search.ts index 76d8519..c0570fd 100644 --- a/src/core/search.ts +++ b/src/core/search.ts @@ -39,11 +39,12 @@ function isVectorTableError(err: unknown): boolean { /** Escape LIKE special characters so user input is treated literally. */ export function escapeLikePattern(input: string): string { + // prettier-ignore return input - .replace(/\\/g, "\\\\") - .replace(/%/g, "\\%") - .replace(/_/g, "\\_") - .replace(/\[/g, "\\["); + .replaceAll("\\", String.raw`\\`) + .replaceAll("%", String.raw`\%`) + .replaceAll("_", String.raw`\_`) + .replaceAll("[", String.raw`\[`); } export interface SearchOptions { diff --git a/src/core/spider.ts b/src/core/spider.ts index 07de1c1..5d11faa 100644 --- a/src/core/spider.ts +++ b/src/core/spider.ts @@ -260,7 +260,7 @@ function stripTags(input: string): string { pos = scanPastTag(input, open + 1); } // Collapse whitespace left behind by removed tags - return result.replace(/\s+/g, " "); + return result.replaceAll(/\s+/g, " "); } function extractTitle(html: string, url: string): string { diff --git a/src/core/topics.ts b/src/core/topics.ts index 92e18fa..59eb2af 100644 --- a/src/core/topics.ts +++ b/src/core/topics.ts @@ -31,7 +31,7 @@ export function createTopic(db: Database.Database, input: CreateTopicInput): Top const id = input.name .toLowerCase() - .replace(/[^a-z0-9]+/g, "-") + .replaceAll(/[^a-z0-9]+/g, "-") .replaceAll(/^-|-$/g, "") || randomUUID(); // Verify parent exists if provided @@ -106,11 +106,11 @@ export function listTopics(db: Database.Database, parentId?: string): Topic[] { `; const params: unknown[] = []; - if (parentId !== undefined) { + if (parentId === undefined) { + sql += " WHERE parent_id IS NULL"; + } else { sql += " WHERE parent_id = ?"; params.push(parentId); - } else { - sql += " WHERE parent_id IS NULL"; } sql += " ORDER BY name"; diff --git a/src/core/url-fetcher.ts b/src/core/url-fetcher.ts index 1377ef5..2bbe7dd 100644 --- a/src/core/url-fetcher.ts +++ b/src/core/url-fetcher.ts @@ -86,7 +86,7 @@ async function validateUrl(url: string, allowPrivateUrls = false): Promise { getLogger().error({ err, url: req.url }, "Unhandled error in web request handler"); - if (!res.headersSent) { - sendJson(res, 500, { error: "Internal server error" }); - } else { + if (res.headersSent) { req.socket.destroy(); + } else { + sendJson(res, 500, { error: "Internal server error" }); } }); }); diff --git a/tests/fixtures/helpers.ts b/tests/fixtures/helpers.ts index 6a01a33..1e6496a 100644 --- a/tests/fixtures/helpers.ts +++ b/tests/fixtures/helpers.ts @@ -57,10 +57,10 @@ export function withEnv(vars: Record, fn: () => void): void { fn(); } finally { for (const [key, original] of Object.entries(originals)) { - if (original !== undefined) { - process.env[key] = original; - } else { + if (original === undefined) { delete process.env[key]; + } else { + process.env[key] = original; } } } diff --git a/tests/unit/api.test.ts b/tests/unit/api.test.ts index f703715..a9a4285 100644 --- a/tests/unit/api.test.ts +++ b/tests/unit/api.test.ts @@ -47,16 +47,16 @@ function createMockReq(method: string, url: string, body?: unknown): IncomingMes req.url = url; req.headers = { host: "localhost:3378" }; - if (body !== undefined) { - const json = typeof body === "string" ? body : JSON.stringify(body); - req.headers["content-type"] = "application/json"; - // Push the body data asynchronously + if (body === undefined) { process.nextTick(() => { - req.push(Buffer.from(json)); req.push(null); }); } else { + const json = typeof body === "string" ? body : JSON.stringify(body); + req.headers["content-type"] = "application/json"; + // Push the body data asynchronously process.nextTick(() => { + req.push(Buffer.from(json)); req.push(null); }); } diff --git a/tests/unit/config.test.ts b/tests/unit/config.test.ts index a84d0be..2a54616 100644 --- a/tests/unit/config.test.ts +++ b/tests/unit/config.test.ts @@ -2,6 +2,17 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; import { loadConfig, validateConfig, invalidateConfigCache } from "../../src/config.js"; import type { LibScopeConfig } from "../../src/config.js"; import * as loggerModule from "../../src/logger.js"; +import { withEnv } from "../fixtures/helpers.js"; + +/** Set env vars, invalidate cache, load config, then restore env. */ +function loadConfigWithEnv(vars: Record): LibScopeConfig { + let result: LibScopeConfig | undefined; + withEnv(vars, () => { + invalidateConfigCache(); + result = loadConfig(); + }); + return result!; +} describe("config", () => { it("should return default config when no files exist", () => { @@ -21,118 +32,42 @@ describe("config", () => { }); it("should respect LIBSCOPE_EMBEDDING_PROVIDER env var", () => { - const original = process.env["LIBSCOPE_EMBEDDING_PROVIDER"]; - try { - process.env["LIBSCOPE_EMBEDDING_PROVIDER"] = "ollama"; - invalidateConfigCache(); - const config = loadConfig(); - expect(config.embedding.provider).toBe("ollama"); - } finally { - if (original !== undefined) { - process.env["LIBSCOPE_EMBEDDING_PROVIDER"] = original; - } else { - delete process.env["LIBSCOPE_EMBEDDING_PROVIDER"]; - } - } + const config = loadConfigWithEnv({ LIBSCOPE_EMBEDDING_PROVIDER: "ollama" }); + expect(config.embedding.provider).toBe("ollama"); }); it("should ignore invalid provider values from env", () => { - const original = process.env["LIBSCOPE_EMBEDDING_PROVIDER"]; - try { - process.env["LIBSCOPE_EMBEDDING_PROVIDER"] = "invalid"; - invalidateConfigCache(); - const config = loadConfig(); - // Should fall through to default since "invalid" doesn't match the switch - expect(config.embedding.provider).toBe("local"); - } finally { - if (original !== undefined) { - process.env["LIBSCOPE_EMBEDDING_PROVIDER"] = original; - } else { - delete process.env["LIBSCOPE_EMBEDDING_PROVIDER"]; - } - } + const config = loadConfigWithEnv({ LIBSCOPE_EMBEDDING_PROVIDER: "invalid" }); + expect(config.embedding.provider).toBe("local"); }); it("should pick up LIBSCOPE_OPENAI_API_KEY", () => { - const original = process.env["LIBSCOPE_OPENAI_API_KEY"]; - try { - process.env["LIBSCOPE_OPENAI_API_KEY"] = "sk-test123"; - invalidateConfigCache(); - const config = loadConfig(); - expect(config.embedding.openaiApiKey).toBe("sk-test123"); - } finally { - if (original !== undefined) { - process.env["LIBSCOPE_OPENAI_API_KEY"] = original; - } else { - delete process.env["LIBSCOPE_OPENAI_API_KEY"]; - } - } + const config = loadConfigWithEnv({ LIBSCOPE_OPENAI_API_KEY: "sk-test123" }); + expect(config.embedding.openaiApiKey).toBe("sk-test123"); }); it("should pick up LIBSCOPE_OLLAMA_URL", () => { - const original = process.env["LIBSCOPE_OLLAMA_URL"]; - try { - process.env["LIBSCOPE_OLLAMA_URL"] = "http://custom:11434"; - invalidateConfigCache(); - const config = loadConfig(); - expect(config.embedding.ollamaUrl).toBe("http://custom:11434"); - } finally { - if (original !== undefined) { - process.env["LIBSCOPE_OLLAMA_URL"] = original; - } else { - delete process.env["LIBSCOPE_OLLAMA_URL"]; - } - } + const config = loadConfigWithEnv({ LIBSCOPE_OLLAMA_URL: "http://custom:11434" }); + expect(config.embedding.ollamaUrl).toBe("http://custom:11434"); }); it("should pick up LIBSCOPE_ALLOW_PRIVATE_URLS", () => { - const original = process.env["LIBSCOPE_ALLOW_PRIVATE_URLS"]; - try { - process.env["LIBSCOPE_ALLOW_PRIVATE_URLS"] = "true"; - invalidateConfigCache(); - const config = loadConfig(); - expect(config.indexing.allowPrivateUrls).toBe(true); - } finally { - if (original !== undefined) { - process.env["LIBSCOPE_ALLOW_PRIVATE_URLS"] = original; - } else { - delete process.env["LIBSCOPE_ALLOW_PRIVATE_URLS"]; - } - } + const config = loadConfigWithEnv({ LIBSCOPE_ALLOW_PRIVATE_URLS: "true" }); + expect(config.indexing.allowPrivateUrls).toBe(true); }); it("should pick up LIBSCOPE_ALLOW_SELF_SIGNED_CERTS", () => { - const original = process.env["LIBSCOPE_ALLOW_SELF_SIGNED_CERTS"]; - try { - process.env["LIBSCOPE_ALLOW_SELF_SIGNED_CERTS"] = "1"; - invalidateConfigCache(); - const config = loadConfig(); - expect(config.indexing.allowSelfSignedCerts).toBe(true); - } finally { - if (original !== undefined) { - process.env["LIBSCOPE_ALLOW_SELF_SIGNED_CERTS"] = original; - } else { - delete process.env["LIBSCOPE_ALLOW_SELF_SIGNED_CERTS"]; - } - } + const config = loadConfigWithEnv({ LIBSCOPE_ALLOW_SELF_SIGNED_CERTS: "1" }); + expect(config.indexing.allowSelfSignedCerts).toBe(true); }); it("should pick up LIBSCOPE_LLM_PROVIDER and LIBSCOPE_LLM_MODEL", () => { - const origProvider = process.env["LIBSCOPE_LLM_PROVIDER"]; - const origModel = process.env["LIBSCOPE_LLM_MODEL"]; - try { - process.env["LIBSCOPE_LLM_PROVIDER"] = "ollama"; - process.env["LIBSCOPE_LLM_MODEL"] = "llama3"; - invalidateConfigCache(); - const config = loadConfig(); - expect(config.llm?.provider).toBe("ollama"); - expect(config.llm?.model).toBe("llama3"); - } finally { - if (origProvider !== undefined) process.env["LIBSCOPE_LLM_PROVIDER"] = origProvider; - else delete process.env["LIBSCOPE_LLM_PROVIDER"]; - if (origModel !== undefined) process.env["LIBSCOPE_LLM_MODEL"] = origModel; - else delete process.env["LIBSCOPE_LLM_MODEL"]; - } + const config = loadConfigWithEnv({ + LIBSCOPE_LLM_PROVIDER: "ollama", + LIBSCOPE_LLM_MODEL: "llama3", + }); + expect(config.llm?.provider).toBe("ollama"); + expect(config.llm?.model).toBe("llama3"); }); }); @@ -161,10 +96,10 @@ describe("validateConfig", () => { afterEach(() => { vi.restoreAllMocks(); for (const [key, val] of Object.entries(savedEnv)) { - if (val !== undefined) { - process.env[key] = val; - } else { + if (val === undefined) { delete process.env[key]; + } else { + process.env[key] = val; } } }); diff --git a/tests/unit/workspace.test.ts b/tests/unit/workspace.test.ts index 8ef1086..ef433cc 100644 --- a/tests/unit/workspace.test.ts +++ b/tests/unit/workspace.test.ts @@ -29,10 +29,10 @@ describe("workspace", () => { afterEach(() => { process.env["HOME"] = savedHome; - if (savedWsEnv !== undefined) { - process.env["LIBSCOPE_WORKSPACE"] = savedWsEnv; - } else { + if (savedWsEnv === undefined) { delete process.env["LIBSCOPE_WORKSPACE"]; + } else { + process.env["LIBSCOPE_WORKSPACE"] = savedWsEnv; } if (existsSync(tempDir)) { rmSync(tempDir, { recursive: true, force: true });