Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .claude/worktrees/agent-a09a3e20
Submodule agent-a09a3e20 added at 5f3520
1 change: 1 addition & 0 deletions .claude/worktrees/agent-ad3369ec
Submodule agent-ad3369ec added at 5f3520
66 changes: 35 additions & 31 deletions src/connectors/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -111,11 +111,11 @@ export function deleteDbConnectorConfig(db: Database.Database, type: string): bo
const CONNECTORS_DIR = join(homedir(), ".libscope", "connectors");

function ensureConnectorsDir(): void {
if (!existsSync(CONNECTORS_DIR)) {
mkdirSync(CONNECTORS_DIR, { recursive: true, mode: 0o700 });
} else {
if (existsSync(CONNECTORS_DIR)) {
// Remediate existing directories that may have permissive permissions
restrictPermissions(CONNECTORS_DIR, 0o700);
} else {
mkdirSync(CONNECTORS_DIR, { recursive: true, mode: 0o700 });
}
try {
chmodSync(CONNECTORS_DIR, 0o700);
Expand Down Expand Up @@ -163,13 +163,12 @@ export function hasNamedConnectorConfig(name: string): boolean {
return existsSync(filePath);
}

/** Delete documents with a given source_type from the database. Returns count deleted. */
export function deleteConnectorDocuments(db: Database.Database, sourceType: string): number {
const rows = db
.prepare("SELECT id FROM documents WHERE source_type = ?")
.all(sourceType) as Array<{ id: string }>;
if (rows.length === 0) return 0;

/**
* Delete a set of document rows and their associated chunks, embeddings, and FTS entries.
* Tolerates missing virtual tables (chunk_embeddings, chunks_fts).
*/
export function deleteDocumentRows(db: Database.Database, rows: Array<{ id: string }>): void {
const log = getLogger();
const deleteChunksFts = db.prepare(
"DELETE FROM chunks_fts WHERE rowid IN (SELECT rowid FROM chunks_fts WHERE chunk_id IN (SELECT id FROM chunks WHERE document_id = ?))",
);
Expand All @@ -179,28 +178,33 @@ export function deleteConnectorDocuments(db: Database.Database, sourceType: stri
const deleteChunks = db.prepare("DELETE FROM chunks WHERE document_id = ?");
const deleteDoc = db.prepare("DELETE FROM documents WHERE id = ?");

const tx = db.transaction(() => {
for (const row of rows) {
try {
deleteChunksFts.run(row.id);
} catch (err) {
getLogger().debug(
{ err, documentId: row.id },
"FTS table cleanup skipped (table may not exist)",
);
}
try {
deleteEmbeddings.run(row.id);
} catch (err) {
getLogger().debug(
{ err, documentId: row.id },
"chunk_embeddings cleanup skipped (table may not exist)",
);
}
deleteChunks.run(row.id);
deleteDoc.run(row.id);
for (const row of rows) {
try {
deleteChunksFts.run(row.id);
} catch (err) {
log.debug({ err, documentId: row.id }, "FTS table cleanup skipped (table may not exist)");
}
try {
deleteEmbeddings.run(row.id);
} catch (err) {
log.debug(
{ err, documentId: row.id },
"chunk_embeddings cleanup skipped (table may not exist)",
);
}
});
deleteChunks.run(row.id);
deleteDoc.run(row.id);
}
}

/** Delete documents with a given source_type from the database. Returns count deleted. */
export function deleteConnectorDocuments(db: Database.Database, sourceType: string): number {
const rows = db
.prepare("SELECT id FROM documents WHERE source_type = ?")
.all(sourceType) as Array<{ id: string }>;
if (rows.length === 0) return 0;

const tx = db.transaction(() => deleteDocumentRows(db, rows));
tx();

return rows.length;
Expand Down
34 changes: 2 additions & 32 deletions src/connectors/slack.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { getLogger } from "../logger.js";
import { LibScopeError, ValidationError } from "../errors.js";
import { fetchWithRetry } from "./http-utils.js";
import { startSync, completeSync, failSync } from "./sync-tracker.js";
import { deleteDocumentRows } from "./index.js";

export interface SlackConfig {
token: string;
Expand Down Expand Up @@ -569,40 +570,9 @@ export function disconnectSlack(db: Database.Database): number {
const rows = db.prepare("SELECT id FROM documents WHERE url LIKE 'slack://%'").all() as Array<{
id: string;
}>;

if (rows.length === 0) return 0;

const deleteChunksFts = db.prepare(
"DELETE FROM chunks_fts WHERE rowid IN (SELECT rowid FROM chunks_fts WHERE chunk_id IN (SELECT id FROM chunks WHERE document_id = ?))",
);
const deleteEmbeddings = db.prepare(
"DELETE FROM chunk_embeddings WHERE chunk_id IN (SELECT id FROM chunks WHERE document_id = ?)",
);
const deleteChunks = db.prepare("DELETE FROM chunks WHERE document_id = ?");
const deleteDoc = db.prepare("DELETE FROM documents WHERE id = ?");

const tx = db.transaction(() => {
for (const row of rows) {
try {
deleteChunksFts.run(row.id);
} catch (err) {
getLogger().debug(
{ err, documentId: row.id },
"FTS table cleanup skipped (table may not exist)",
);
}
try {
deleteEmbeddings.run(row.id);
} catch (err) {
getLogger().debug(
{ err, documentId: row.id },
"chunk_embeddings cleanup skipped (table may not exist)",
);
}
deleteChunks.run(row.id);
deleteDoc.run(row.id);
}
});
const tx = db.transaction(() => deleteDocumentRows(db, rows));
tx();

return rows.length;
Expand Down
18 changes: 9 additions & 9 deletions src/core/bulk.ts
Original file line number Diff line number Diff line change
Expand Up @@ -92,16 +92,16 @@ export function bulkDelete(
const log = getLogger();
const ids = resolveSelector(db, selector);

if (!dryRun) {
if (dryRun) {
log.info({ count: ids.length, dryRun: true }, "Bulk delete dry run");
} else {
const deleteAll = db.transaction(() => {
for (const id of ids) {
deleteDocument(db, id);
}
});
deleteAll();
log.info({ count: ids.length, dryRun: false }, "Bulk delete completed");
} else {
log.info({ count: ids.length, dryRun: true }, "Bulk delete dry run");
}

return { affected: ids.length, documentIds: ids };
Expand All @@ -123,7 +123,9 @@ export function bulkRetag(

const ids = resolveSelector(db, selector);

if (!dryRun) {
if (dryRun) {
log.info({ count: ids.length, addTags, removeTags, dryRun: true }, "Bulk retag dry run");
} else {
const retagAll = db.transaction(() => {
for (const id of ids) {
if (addTags && addTags.length > 0) {
Expand All @@ -143,8 +145,6 @@ export function bulkRetag(
});
retagAll();
log.info({ count: ids.length, addTags, removeTags, dryRun: false }, "Bulk retag completed");
} else {
log.info({ count: ids.length, addTags, removeTags, dryRun: true }, "Bulk retag dry run");
}

return { affected: ids.length, documentIds: ids };
Expand All @@ -160,7 +160,9 @@ export function bulkMove(
const log = getLogger();
const ids = resolveSelector(db, selector);

if (!dryRun) {
if (dryRun) {
log.info({ count: ids.length, targetTopicId, dryRun: true }, "Bulk move dry run");
} else {
const moveAll = db.transaction(() => {
const stmt = db.prepare(
"UPDATE documents SET topic_id = ?, updated_at = datetime('now') WHERE id = ?",
Expand All @@ -171,8 +173,6 @@ export function bulkMove(
});
moveAll();
log.info({ count: ids.length, targetTopicId, dryRun: false }, "Bulk move completed");
} else {
log.info({ count: ids.length, targetTopicId, dryRun: true }, "Bulk move dry run");
}

return { affected: ids.length, documentIds: ids };
Expand Down
2 changes: 1 addition & 1 deletion src/core/repo.ts
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ export function parseRepoUrl(url: string): ParsedRepoUrl {
// ── SSRF Protection ──────────────────────────────────────────────────────────

async function validateHost(hostname: string): Promise<void> {
const stripped = hostname.replace(/^\[|\]$/g, "");
const stripped = hostname.replaceAll(/^\[|\]$/g, "");
const results = await Promise.allSettled([dns.resolve4(stripped), dns.resolve6(stripped)]);

const addresses: string[] = [];
Expand Down
9 changes: 5 additions & 4 deletions src/core/search.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,12 @@ function isVectorTableError(err: unknown): boolean {

/** Escape LIKE special characters so user input is treated literally. */
export function escapeLikePattern(input: string): string {
// prettier-ignore
return input
.replace(/\\/g, "\\\\")
.replace(/%/g, "\\%")
.replace(/_/g, "\\_")
.replace(/\[/g, "\\[");
.replaceAll("\\", String.raw`\\`)
.replaceAll("%", String.raw`\%`)
.replaceAll("_", String.raw`\_`)
.replaceAll("[", String.raw`\[`);
}

export interface SearchOptions {
Expand Down
2 changes: 1 addition & 1 deletion src/core/spider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,7 @@ function stripTags(input: string): string {
pos = scanPastTag(input, open + 1);
}
// Collapse whitespace left behind by removed tags
return result.replace(/\s+/g, " ");
return result.replaceAll(/\s+/g, " ");
}

function extractTitle(html: string, url: string): string {
Expand Down
8 changes: 4 additions & 4 deletions src/core/topics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ export function createTopic(db: Database.Database, input: CreateTopicInput): Top
const id =
input.name
.toLowerCase()
.replace(/[^a-z0-9]+/g, "-")
.replaceAll(/[^a-z0-9]+/g, "-")
.replaceAll(/^-|-$/g, "") || randomUUID();

// Verify parent exists if provided
Expand Down Expand Up @@ -106,11 +106,11 @@ export function listTopics(db: Database.Database, parentId?: string): Topic[] {
`;
const params: unknown[] = [];

if (parentId !== undefined) {
if (parentId === undefined) {
sql += " WHERE parent_id IS NULL";
} else {
sql += " WHERE parent_id = ?";
params.push(parentId);
} else {
sql += " WHERE parent_id IS NULL";
}

sql += " ORDER BY name";
Expand Down
2 changes: 1 addition & 1 deletion src/core/url-fetcher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ async function validateUrl(url: string, allowPrivateUrls = false): Promise<strin
}

// Resolve hostname and check every returned address
const hostname = parsed.hostname.replace(/^\[|\]$/g, ""); // strip IPv6 brackets
const hostname = parsed.hostname.replaceAll(/^\[|\]$/g, ""); // strip IPv6 brackets
const { resolve4, resolve6 } = dns;
const results = await Promise.allSettled([resolve4(hostname), resolve6(hostname)]);

Expand Down
6 changes: 3 additions & 3 deletions src/web/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,10 @@ export function startWebServer(
}
handleRequest(db, provider, req, res).catch((err) => {
getLogger().error({ err, url: req.url }, "Unhandled error in web request handler");
if (!res.headersSent) {
sendJson(res, 500, { error: "Internal server error" });
} else {
if (res.headersSent) {
req.socket.destroy();
} else {
sendJson(res, 500, { error: "Internal server error" });
}
});
});
Expand Down
6 changes: 3 additions & 3 deletions tests/fixtures/helpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,10 @@ export function withEnv(vars: Record<string, string>, fn: () => void): void {
fn();
} finally {
for (const [key, original] of Object.entries(originals)) {
if (original !== undefined) {
process.env[key] = original;
} else {
if (original === undefined) {
delete process.env[key];
} else {
process.env[key] = original;
}
}
}
Expand Down
10 changes: 5 additions & 5 deletions tests/unit/api.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,16 +47,16 @@ function createMockReq(method: string, url: string, body?: unknown): IncomingMes
req.url = url;
req.headers = { host: "localhost:3378" };

if (body !== undefined) {
const json = typeof body === "string" ? body : JSON.stringify(body);
req.headers["content-type"] = "application/json";
// Push the body data asynchronously
if (body === undefined) {
process.nextTick(() => {
req.push(Buffer.from(json));
req.push(null);
});
} else {
const json = typeof body === "string" ? body : JSON.stringify(body);
req.headers["content-type"] = "application/json";
// Push the body data asynchronously
process.nextTick(() => {
req.push(Buffer.from(json));
req.push(null);
});
}
Expand Down
Loading
Loading