Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,12 @@ npm run build # tsc — outputs to dist/

CI runs on Node 20 and 22. CI checks: lint, format:check, typecheck, test:coverage, build. Build verifies `dist/mcp/server.js`, `dist/cli/index.js`, `dist/core/index.js` exist.

**Before every push, run this exact sequence locally:**
```bash
npm run format:check && npm run lint && npm run typecheck && npm test
```
Do not skip any step. Do not assume "pre-existing errors" — compare the lint error count against main. If your branch has MORE errors than main, CI will fail. The pre-existing error count on main is ~39 lint errors (all in parsers/rag.ts).

## Code Style & TypeScript

- **Strict mode** with `noUncheckedIndexedAccess`, `exactOptionalPropertyTypes`, `noImplicitReturns`
Expand Down
194 changes: 114 additions & 80 deletions src/cli/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -335,6 +335,119 @@ async function handlePackConflict(
return false;
}

/** Try to resolve a pack from git registries; returns true if installed. */
async function tryGitRegistryInstall(
nameOrPath: string,
opts: { fromRegistry?: string; version?: string; yes?: boolean },
db: ReturnType<typeof getDatabase>,
provider: EmbeddingProvider,
installOpts: {
batchSize: number | undefined;
resumeFrom: number | undefined;
concurrency: number | undefined;
},
reporter: ReturnType<typeof createReporter>,
): Promise<boolean> {
const isLocalFile = nameOrPath.endsWith(".json") || nameOrPath.endsWith(".json.gz");
if (isLocalFile || loadRegistries().length === 0) return false;

const { name: packName, version: specVersion } = parsePackSpecifier(nameOrPath);
const version = opts.version ?? specVersion;

const { resolved, conflict, warnings } = resolvePackFromRegistries(packName, {
version,
registryName: opts.fromRegistry,
conflictResolution: opts.fromRegistry
? { strategy: "explicit", registryName: opts.fromRegistry }
: opts.yes
? { strategy: "priority" }
: undefined,
});

for (const w of warnings) {
reporter.log(`Warning: ${w}`);
}

if (conflict && !resolved) {
const handled = await handlePackConflict(
conflict,
packName,
version,
opts,
db,
provider,
installOpts,
reporter,
);
if (handled) return true;
}

if (resolved) {
await installResolvedPack(db, provider, resolved, installOpts, reporter);
return true;
}

return false;
}

/** Execute a pack install from git registry, URL, or local file. */
async function executePackInstall(
nameOrPath: string,
opts: {
registry?: string;
fromRegistry?: string;
version?: string;
yes?: boolean;
batchSize?: string;
resumeFrom?: string;
concurrency?: string;
},
): Promise<void> {
const { db, provider } = initializeAppWithEmbedding();
const globalOpts = program.opts<ProgramOpts>();
const reporter = createReporter(globalOpts.verbose);

const batchSize = opts.batchSize ? parseIntOption(opts.batchSize, "--batch-size") : undefined;
const resumeFrom = opts.resumeFrom ? parseIntOption(opts.resumeFrom, "--resume-from") : undefined;
const concurrency = opts.concurrency
? parseIntOption(opts.concurrency, "--concurrency")
: undefined;

if (concurrency !== undefined && concurrency < 1) {
reporter.log('Error: "--concurrency" must be an integer greater than or equal to 1.');
closeDatabase();
process.exit(1);
return;
}

const installOpts = { batchSize, resumeFrom, concurrency };

try {
const installed = await tryGitRegistryInstall(
nameOrPath,
opts,
db,
provider,
installOpts,
reporter,
);
if (installed) return;

const result = await installPack(db, provider, nameOrPath, {
registryUrl: opts.registry,
...installOpts,
onProgress: (current, total, docTitle) => {
reporter.progress(current, total, docTitle);
},
});

reporter.clearProgress();
reportInstallResult(result, reporter);
} finally {
closeDatabase();
}
}

const program = new Command();

program
Expand Down Expand Up @@ -1891,86 +2004,7 @@ packCmd
concurrency?: string;
},
) => {
const { db, provider } = initializeAppWithEmbedding();
const globalOpts = program.opts<ProgramOpts>();
const reporter = createReporter(globalOpts.verbose);

const batchSize = opts.batchSize ? parseIntOption(opts.batchSize, "--batch-size") : undefined;
const resumeFrom = opts.resumeFrom
? parseIntOption(opts.resumeFrom, "--resume-from")
: undefined;
const concurrency = opts.concurrency
? parseIntOption(opts.concurrency, "--concurrency")
: undefined;

if (concurrency !== undefined && concurrency < 1) {
reporter.log('Error: "--concurrency" must be an integer greater than or equal to 1.');
closeDatabase();
process.exit(1);
return;
}

const installOpts = { batchSize, resumeFrom, concurrency };

try {
// Check if this is a local file or URL-based registry install
const isLocalFile = nameOrPath.endsWith(".json") || nameOrPath.endsWith(".json.gz");

// Try git registry resolution if not a local file and we have registries configured
if (!isLocalFile && loadRegistries().length > 0) {
const { name: packName, version: specVersion } = parsePackSpecifier(nameOrPath);
const version = opts.version ?? specVersion;

const { resolved, conflict, warnings } = resolvePackFromRegistries(packName, {
version,
registryName: opts.fromRegistry,
conflictResolution: opts.fromRegistry
? { strategy: "explicit", registryName: opts.fromRegistry }
: opts.yes
? { strategy: "priority" }
: undefined,
});

for (const w of warnings) {
reporter.log(`Warning: ${w}`);
}

if (conflict && !resolved) {
const handled = await handlePackConflict(
conflict,
packName,
version,
opts,
db,
provider,
installOpts,
reporter,
);
if (handled) return;
}

if (resolved) {
await installResolvedPack(db, provider, resolved, installOpts, reporter);
return;
}

// Fall through to URL-based registry install if git registry resolution failed
}

// Original URL-based or local file install
const result = await installPack(db, provider, nameOrPath, {
registryUrl: opts.registry,
...installOpts,
onProgress: (current, total, docTitle) => {
reporter.progress(current, total, docTitle);
},
});

reporter.clearProgress();
reportInstallResult(result, reporter);
} finally {
closeDatabase();
}
await executePackInstall(nameOrPath, opts);
},
);

Expand Down
80 changes: 47 additions & 33 deletions src/connectors/notion.ts
Original file line number Diff line number Diff line change
Expand Up @@ -328,37 +328,39 @@ function renderChildContent(children: NotionBlock[]): string | undefined {
.join("\n");
}

/** Convert a single Notion block to markdown line(s), appending to the output array. */
function convertSingleBlock(block: NotionBlock, lines: string[]): void {
const text = getBlockText(block);
const children = (block as Record<string, unknown>)["children"] as NotionBlock[] | undefined;

// Handle table specially (children are inline rows)
if (block.type === "table" && children) {
lines.push(...renderTableRows(children));
return;
}

// Try special blocks first (ones needing type-specific data extraction)
const specialLine = convertSpecialBlock(block, text);
if (specialLine === undefined) {
const simpleLine = blockToMarkdownLine(block, text);
if (simpleLine !== undefined) lines.push(simpleLine);
} else {
lines.push(specialLine);
}

// Render nested children (except table children which are handled above)
if (children && block.type !== "table") {
const indented = renderChildContent(children);
if (indented) lines.push(indented);
}
}

/** Convert an array of Notion blocks to markdown. */
export function convertNotionBlocks(blocks: NotionBlock[]): string {
const lines: string[] = [];

for (const block of blocks) {
const text = getBlockText(block);
const children = (block as Record<string, unknown>)["children"] as NotionBlock[] | undefined;

// Handle table specially (children are inline rows)
if (block.type === "table" && children) {
lines.push(...renderTableRows(children));
continue;
}

// Try special blocks first (ones needing type-specific data extraction)
const specialLine = convertSpecialBlock(block, text);
if (specialLine === undefined) {
// Simple blocks that just need text formatting
const simpleLine = blockToMarkdownLine(block, text);
if (simpleLine !== undefined) lines.push(simpleLine);
} else {
lines.push(specialLine);
}

// Render nested children (except table children which are handled above)
if (children && block.type !== "table") {
const indented = renderChildContent(children);
if (indented) lines.push(indented);
}
convertSingleBlock(block, lines);
}

return lines.join("\n");
}

Expand Down Expand Up @@ -480,6 +482,24 @@ async function syncNotionDatabase(
log.debug({ id: item.id, title: dbTitle, rows: rows.length }, "Indexed Notion database");
}

/** Sync a single search result item (page or database). */
async function syncNotionItem(
db: Database.Database,
provider: EmbeddingProvider,
token: string,
item: NotionSearchResult,
excludeSet: Set<string>,
result: NotionSyncResult,
): Promise<void> {
if (item.object === "page") {
const indexed = await syncNotionPage(db, provider, token, item);
if (indexed) result.pagesIndexed++;
} else if (item.object === "database") {
await syncNotionDatabase(db, provider, token, item, excludeSet);
result.databasesIndexed++;
}
}

/** Sync pages and databases from Notion into the knowledge base. */
export async function syncNotion(
db: Database.Database,
Expand Down Expand Up @@ -514,13 +534,7 @@ export async function syncNotion(
}

try {
if (item.object === "page") {
const indexed = await syncNotionPage(db, provider, config.token, item);
if (indexed) result.pagesIndexed++;
} else if (item.object === "database") {
await syncNotionDatabase(db, provider, config.token, item, excludeSet);
result.databasesIndexed++;
}
await syncNotionItem(db, provider, config.token, item, excludeSet, result);
} catch (err) {
const title = extractTitle(item);
const message = err instanceof Error ? err.message : String(err);
Expand Down
Loading
Loading