diff --git a/.github/scripts/i18n/README.md b/.github/scripts/i18n/README.md index bc05ea16c..f29b05dd5 100644 --- a/.github/scripts/i18n/README.md +++ b/.github/scripts/i18n/README.md @@ -28,6 +28,35 @@ Quality controls during/after a run write to `.github/i18n-logs/translate/` (gitignored): semantic mismatches reported by the model, and a truncation scan (`check-translation-truncation.ts`). +### Long pages (chunked translation) + +Very long MDX files (e.g. `tutorials/partner-nodes/pricing.mdx`) exceed model +output limits when translated in one shot. Two strategies avoid truncation: + +| Strategy | Use case | Split boundary | Incremental sync | +|----------|----------|----------------|------------------| +| `heading_sections` | Long reference pages | Level-2 `##` headings | Per-section content hash in `translationBlockHashes` | +| `update_blocks` | Changelog | `` blocks | New version labels only | + +Configure explicit paths in `translation-config.json` → `chunked_files`, or rely +on `auto_chunk` (default: body ≥ 10k chars and ≥ 4 `##` sections) to auto-enable +`heading_sections`. + +During a chunked run the script: + +1. Parses English into blocks (intro + each `##` section). +2. Compares each block’s hash to `translationBlockHashes` in the target frontmatter. +3. Translates only pending blocks (plus frontmatter when needed). +4. Checkpoints after every block so a failed run can resume. + +Example: + +```bash +pnpm translate -- tutorials/partner-nodes/pricing.mdx --lang ko +pnpm translate:check-truncation -- --lang ko +pnpm translate:repair-truncated -- --lang ko # force re-translate flagged files +``` + ## Quality review (LLM-as-a-judge) `review-i18n.ts` scores existing translations with an **independent** (and @@ -139,6 +168,7 @@ env → `frontend_locales_path` in `translation-config.json` → | File | Role | |------|------| | `translate-i18n.ts` | translation entry point | +| `chunked-translate.ts` | split/reassemble long MDX (`heading_sections`, `update_blocks`) | | `sync-glossary.mjs` | rebuild the glossary frontend mirror | | `glossary.mjs` | load glossary layers, select + inject terms | | `i18n-config.mjs` | shared path rules from `translation-config.json` | diff --git a/.github/scripts/i18n/check-translation-truncation.ts b/.github/scripts/i18n/check-translation-truncation.ts index 9258b7247..daa52e92d 100644 --- a/.github/scripts/i18n/check-translation-truncation.ts +++ b/.github/scripts/i18n/check-translation-truncation.ts @@ -24,6 +24,12 @@ import { TRUNCATION_ISSUES_JSON, TRUNCATION_ISSUES_TXT, } from "./i18n-config.mjs"; +import { + missingSectionLabels, + parseFrontmatterAndBody, + resolveChunkStrategy, + type ChunkedFileConfig, +} from "./chunked-translate.ts"; const ROOT = REPO_ROOT; const LOG_DIR = TRANSLATE_LOG_DIR; @@ -39,6 +45,8 @@ interface LangConfig { interface TranslationConfig { skip_paths: string[]; + chunked_files?: ChunkedFileConfig[]; + auto_chunk?: { min_body_chars?: number; min_sections?: number }; languages: LangConfig[]; } @@ -59,10 +67,11 @@ export interface TruncationReport { const config = loadI18nConfig() as TranslationConfig; const pathFilterOpts = { languages: config.languages, skip_paths: config.skip_paths }; +const CHUNKED_FILES = config.chunked_files ?? []; +const AUTO_CHUNK = config.auto_chunk; -function parseFrontmatterAndBody(content: string): { body: string } { - const match = content.match(/^---\n[\s\S]*?\n---\n?([\s\S]*)$/); - return { body: match ? match[1] : content }; +function parseBody(content: string): string { + return parseFrontmatterAndBody(content).body; } function countCodeFences(body: string): { open: boolean; openLang: string; openLine: number } { @@ -137,8 +146,8 @@ export function detectTruncation( enRel: string ): TruncationIssue["reasons"] { const reasons: string[] = []; - const enBody = parseFrontmatterAndBody(enContent).body; - const targetBody = parseFrontmatterAndBody(targetContent).body; + const enBody = parseBody(enContent); + const targetBody = parseBody(targetContent); const fence = countCodeFences(targetBody); if (fence.open) { @@ -171,6 +180,14 @@ export function detectTruncation( } } + const chunkStrategy = resolveChunkStrategy(enRel, enBody, CHUNKED_FILES, AUTO_CHUNK); + if (chunkStrategy === "heading_sections") { + const missingSections = missingSectionLabels(enBody, targetBody, chunkStrategy); + if (missingSections.length > 0) { + reasons.push("missing_sections"); + } + } + return reasons; } @@ -180,8 +197,8 @@ function formatDetail( targetContent: string ): string { const parts: string[] = []; - const enBody = parseFrontmatterAndBody(enContent).body; - const targetBody = parseFrontmatterAndBody(targetContent).body; + const enBody = parseBody(enContent); + const targetBody = parseBody(targetContent); const fence = countCodeFences(targetBody); if (reasons.includes("unclosed_code_fence")) { @@ -209,6 +226,12 @@ function formatDetail( const missing = enLabels.filter((l) => !targetLabels.has(l)); parts.push(`missing versions: ${missing.slice(0, 5).join(", ")}${missing.length > 5 ? "..." : ""}`); } + if (reasons.includes("missing_sections")) { + const missing = missingSectionLabels(enBody, targetBody, "heading_sections"); + parts.push( + `missing sections: ${missing.slice(0, 5).join(", ")}${missing.length > 5 ? "..." : ""}` + ); + } return parts.join("; "); } diff --git a/.github/scripts/i18n/chunked-translate.ts b/.github/scripts/i18n/chunked-translate.ts new file mode 100644 index 000000000..7f2b176b0 --- /dev/null +++ b/.github/scripts/i18n/chunked-translate.ts @@ -0,0 +1,470 @@ +/** + * Chunked MDX translation helpers. + * + * Strategies: + * - update_blocks: changelog/index.mdx — one per block + * - heading_sections: long pages — split on level-2 `##` headings + * + * Incremental sync stores per-block English hashes in frontmatter + * (`translationBlockHashes`) keyed by stable English labels. Target files + * are split by section index (headings are translated, so labels differ). + */ + +import { createHash } from "crypto"; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export type ChunkStrategy = "update_blocks" | "heading_sections"; + +export interface ChunkedFileConfig { + path: string; + strategy: ChunkStrategy; +} + +export interface AutoChunkConfig { + /** Minimum English body length (chars) to auto-enable heading_sections. */ + min_body_chars?: number; + /** Minimum number of `##` sections required for auto-chunking. */ + min_sections?: number; +} + +export interface ContentBlock { + /** Stable sync key from English (section title or `_intro`). */ + label: string; + content: string; +} + +export interface ParsedDocument { + frontmatter: string; + blocks: ContentBlock[]; +} + +export interface BlockSlot { + label: string; + content: string | null; +} + +export interface SectionSyncStatus { + upToDate: boolean; + pendingBlocks: string[]; + needsFrontmatter: boolean; + /** EN section removed — re-serialize target without re-translating unchanged blocks. */ + needsReserialize?: boolean; +} + +// --------------------------------------------------------------------------- +// Hashing +// --------------------------------------------------------------------------- + +export function blockHash(content: string): string { + return createHash("sha256").update(content).digest("hex").slice(0, 8); +} + +export function documentBlockHashes(blocks: ContentBlock[]): Record { + const out: Record = {}; + for (const b of blocks) { + if (b.label in out) { + throw new Error( + `Duplicate block label: "${b.label}". Use unique section headings for sync.` + ); + } + out[b.label] = blockHash(b.content); + } + return out; +} + +export function aggregateDocumentHash(blockHashes: Record): string { + const joined = Object.keys(blockHashes) + .sort() + .map((k) => `${k}:${blockHashes[k]}`) + .join("|"); + return blockHash(joined); +} + +// --------------------------------------------------------------------------- +// Frontmatter helpers +// --------------------------------------------------------------------------- + +export function parseFrontmatterAndBody(content: string): { frontmatter: string; body: string } { + const match = content.match(/^(---\n[\s\S]*?\n---)\n?([\s\S]*)$/); + if (!match) return { frontmatter: "", body: content }; + return { frontmatter: `${match[1]}\n`, body: match[2] }; +} + +/** Parse `translationBlockHashes` YAML map from frontmatter body. */ +export function parseBlockHashesFromFrontmatter(fmBody: string): Record { + const out: Record = {}; + const lines = fmBody.split("\n"); + let inMap = false; + for (const line of lines) { + if (/^translationBlockHashes:\s*$/.test(line)) { + inMap = true; + continue; + } + if (inMap) { + const quoted = line.match(/^\s{2}("(?:\\.|[^"\\])*"):\s*"?([a-f0-9]{8})"?\s*$/); + if (quoted) { + out[JSON.parse(quoted[1]!)] = quoted[2]!; + continue; + } + const plain = line.match(/^\s{2}([^:]+):\s*"?([a-f0-9]{8})"?\s*$/); + if (plain) { + out[plain[1]!.trim()] = plain[2]!; + continue; + } + if (/^[A-Za-z_][\w-]*:/.test(line)) { + inMap = false; + } + } + } + return out; +} + +export function formatBlockHashesYaml(blockHashes: Record): string { + const lines = ["translationBlockHashes:"]; + for (const label of Object.keys(blockHashes).sort()) { + lines.push(` ${JSON.stringify(label)}: ${blockHashes[label]}`); + } + return lines.join("\n"); +} + +function stripFrontmatterLineFieldAfterNewline(body: string, field: string): string { + return body.replace(new RegExp(`\\n${field}:.*`, "g"), ""); +} + +function stripFrontmatterLineFieldAtStart(body: string, field: string): string { + return body.replace(new RegExp(`^${field}:.*\\n?`, "m"), ""); +} + +function stripFrontmatterBlockFieldAfterNewline(body: string, field: string): string { + return body.replace( + new RegExp(`\\n${field}:[\\s\\S]*?(?=\\n[A-Za-z_][\\w-]*:|\\s*$)`), + "" + ); +} + +function stripFrontmatterBlockFieldAtStart(body: string, field: string): string { + return body.replace( + new RegExp(`^${field}:[\\s\\S]*?(?=^[A-Za-z_][\\w-]*:|\\s*$)`, "m"), + "" + ); +} + +function stripFrontmatterListFieldAfterNewline(body: string, field: string): string { + return body.replace(new RegExp(`\\n${field}:(?:\\n\\s+-.*?)*`, "g"), ""); +} + +function stripFrontmatterListFieldAtStart(body: string, field: string): string { + return body.replace(new RegExp(`^${field}:(?:\\n\\s+-.*?)*`, "g"), ""); +} + +export function stripTranslationMetaFromFrontmatter(body: string): string { + let out = body; + out = stripFrontmatterLineFieldAfterNewline(out, "translationSourceHash"); + out = stripFrontmatterLineFieldAfterNewline(out, "translationFrom"); + out = stripFrontmatterBlockFieldAfterNewline(out, "translationBlockHashes"); + out = stripFrontmatterListFieldAfterNewline(out, "translationMismatches"); + out = stripFrontmatterLineFieldAtStart(out, "translationSourceHash"); + out = stripFrontmatterLineFieldAtStart(out, "translationFrom"); + out = stripFrontmatterBlockFieldAtStart(out, "translationBlockHashes"); + out = stripFrontmatterListFieldAtStart(out, "translationMismatches"); + return out; +} + +export function setChunkedTranslationMeta( + content: string, + fileHash: string, + enPath: string, + blockHashes: Record +): string { + const metaLines = [ + `translationSourceHash: ${fileHash}`, + `translationFrom: ${enPath}`, + formatBlockHashesYaml(blockHashes), + ]; + const metaBlock = metaLines.join("\n"); + + const fmMatch = content.match(/^(---\n)([\s\S]*?)(\n---)/); + if (!fmMatch) { + return `---\n${metaBlock}\n---\n${content}`; + } + const [, open, body, close] = fmMatch; + const rest = content.slice(fmMatch[0].length); + const cleaned = stripTranslationMetaFromFrontmatter(body); + return `${open}${cleaned}\n${metaBlock}${close}${rest}`; +} + +// --------------------------------------------------------------------------- +// heading_sections — split on level-2 headings +// --------------------------------------------------------------------------- + +const H2_HEADING_RE = /^## (?![#])/; +const FENCE_RE = /^(```|~~~)/; + +function toggleFence(line: string, inFence: boolean): boolean { + return FENCE_RE.test(line.trim()) ? !inFence : inFence; +} + +function isH2SectionLine(line: string, inFence: boolean): boolean { + return !inFence && H2_HEADING_RE.test(line); +} + +export function parseHeadingSections(body: string): ContentBlock[] { + const lines = body.split("\n"); + const blocks: ContentBlock[] = []; + let introLines: string[] = []; + let current: ContentBlock | null = null; + let inFence = false; + + for (const line of lines) { + inFence = toggleFence(line, inFence); + if (isH2SectionLine(line, inFence)) { + if (current) { + blocks.push({ ...current, content: current.content.trimEnd() }); + } else if (introLines.length > 0) { + const intro = introLines.join("\n").trimEnd(); + if (intro.length > 0) { + blocks.push({ label: "_intro", content: intro }); + } + introLines = []; + } + current = { label: line.slice(3).trim(), content: line }; + } else if (current) { + current.content += `\n${line}`; + } else { + introLines.push(line); + } + } + + if (current) { + blocks.push({ ...current, content: current.content.trimEnd() }); + } else if (introLines.length > 0) { + const intro = introLines.join("\n").trimEnd(); + if (intro.length > 0) { + blocks.push({ label: "_intro", content: intro }); + } + } + + return blocks; +} + +/** Split target body into the same number of positional sections as English. */ +export function parseTargetSectionsByIndex(body: string, enBlockCount: number): string[] { + const enBlocks = parseHeadingSections(body); + if (enBlocks.length === enBlockCount) { + return enBlocks.map((b) => b.content); + } + + // Fallback: positional split using English block structure markers + const sections: string[] = []; + const lines = body.split("\n"); + let introLines: string[] = []; + let currentLines: string[] = []; + let h2Count = 0; + let inFence = false; + let hasIntro = body.trim().length > 0 && !H2_HEADING_RE.test(lines.find((l) => l.trim()) ?? ""); + + for (const line of lines) { + inFence = toggleFence(line, inFence); + if (isH2SectionLine(line, inFence)) { + if (h2Count === 0 && introLines.length > 0) { + sections.push(introLines.join("\n").trimEnd()); + introLines = []; + hasIntro = true; + } else if (currentLines.length > 0) { + sections.push(currentLines.join("\n").trimEnd()); + } + currentLines = [line]; + h2Count++; + } else if (h2Count === 0 && !hasIntro) { + currentLines.push(line); + } else if (h2Count === 0) { + introLines.push(line); + } else { + currentLines.push(line); + } + } + + if (introLines.length > 0) { + sections.unshift(introLines.join("\n").trimEnd()); + } + if (currentLines.length > 0) { + sections.push(currentLines.join("\n").trimEnd()); + } + + // Pad with empty strings if target is shorter + while (sections.length < enBlockCount) { + sections.push(""); + } + return sections.slice(0, enBlockCount); +} + +export function countH2Sections(body: string): number { + let inFence = false; + let count = 0; + for (const line of body.split("\n")) { + inFence = toggleFence(line, inFence); + if (isH2SectionLine(line, inFence)) count++; + } + return count; +} + +export function shouldAutoChunk(body: string, autoChunk?: AutoChunkConfig): boolean { + const minChars = autoChunk?.min_body_chars ?? 10_000; + const minSections = autoChunk?.min_sections ?? 4; + const sectionCount = countH2Sections(body); + return body.length >= minChars && sectionCount >= minSections; +} + +// --------------------------------------------------------------------------- +// update_blocks — changelog elements +// --------------------------------------------------------------------------- + +export function parseUpdateBlocks(body: string): ContentBlock[] { + const blocks: ContentBlock[] = []; + const re = /]*>[\s\S]*?<\/Update>/g; + let match: RegExpExecArray | null; + while ((match = re.exec(body)) !== null) { + blocks.push({ label: match[1]!, content: match[0] }); + } + return blocks; +} + +export function changelogLabelHash(blocks: ContentBlock[]): string { + return blockHash(blocks.map((b) => b.label).join("|")); +} + +// --------------------------------------------------------------------------- +// Document parsing / serialization +// --------------------------------------------------------------------------- + +export function parseDocument(content: string, strategy: ChunkStrategy): ParsedDocument { + const { frontmatter, body } = parseFrontmatterAndBody(content); + const blocks = + strategy === "update_blocks" ? parseUpdateBlocks(body) : parseHeadingSections(body); + return { frontmatter, blocks }; +} + +export function serializeChunkedDocument( + frontmatter: string, + slots: BlockSlot[], + fileHash: string, + enRel: string, + blockHashes: Record +): string { + const body = slots + .map((s) => s.content) + .filter((c): c is string => c !== null) + .join("\n\n"); + const raw = `${frontmatter}\n${body}\n`; + return setChunkedTranslationMeta(raw, fileHash, enRel, blockHashes); +} + +export function getSectionSyncStatus( + enContent: string, + existingContent: string, + strategy: ChunkStrategy, + force: boolean +): SectionSyncStatus { + const enDoc = parseDocument(enContent, strategy); + const enHashes = documentBlockHashes(enDoc.blocks); + + if (force) { + return { + upToDate: false, + pendingBlocks: enDoc.blocks.map((b) => b.label), + needsFrontmatter: true, + }; + } + + if (!existingContent.trim()) { + return { + upToDate: false, + pendingBlocks: enDoc.blocks.map((b) => b.label), + needsFrontmatter: true, + }; + } + + const existingDoc = parseDocument(existingContent, strategy); + + if (strategy === "update_blocks") { + const existingLabels = new Set(existingDoc.blocks.map((b) => b.label)); + const pendingBlocks = enDoc.blocks + .filter((b) => !existingLabels.has(b.label)) + .map((b) => b.label); + return { + upToDate: pendingBlocks.length === 0, + pendingBlocks, + needsFrontmatter: existingDoc.blocks.length === 0, + }; + } + + const existingFm = parseFrontmatterAndBody(existingContent).frontmatter; + const storedHashes = parseBlockHashesFromFrontmatter( + existingFm.replace(/^---\n/, "").replace(/\n---$/, "") + ); + + const pendingBlocks = enDoc.blocks + .filter((b) => storedHashes[b.label] !== enHashes[b.label]) + .map((b) => b.label); + + const hasStructureDrift = Object.keys(storedHashes).some((k) => !(k in enHashes)); + + return { + upToDate: pendingBlocks.length === 0 && !hasStructureDrift, + pendingBlocks, + needsFrontmatter: Object.keys(storedHashes).length === 0, + needsReserialize: hasStructureDrift && pendingBlocks.length === 0, + }; +} + +export function resolveChunkStrategy( + relPath: string, + enBody: string, + chunkedFiles: ChunkedFileConfig[], + autoChunk?: AutoChunkConfig +): ChunkStrategy | null { + const normalized = relPath.replace(/\\/g, "/"); + const configured = chunkedFiles.find((c) => c.path.replace(/\\/g, "/") === normalized); + if (configured) return configured.strategy; + if (shouldAutoChunk(enBody, autoChunk)) return "heading_sections"; + return null; +} + +export function validateTranslatedBlock( + strategy: ChunkStrategy, + enBlock: ContentBlock, + translated: string +): boolean { + if (!translated.trim()) return false; + + if (strategy === "update_blocks") { + return translated.includes(">(); function getGlossary(langCode: string): ReturnType { @@ -282,18 +301,8 @@ function sanitizeMdxFrontmatter(content: string): string { return `${open}${sanitized}${close}${content.slice(fmMatch[0].length)}`; } -function stripTranslationMetaFromFrontmatter(body: string): string { - return sanitizeFrontmatterBody( - body - .replace(/\ntranslationSourceHash:.*/, "") - .replace(/\ntranslationFrom:.*/, "") - .replace(/\ntranslationBlockHashes:[\s\S]*?(?=\n[A-Za-z_][\w-]*:|\s*$)/, "") - .replace(/\ntranslationMismatches:(?:\n\s+-.*?)*/g, "") - .replace(/^translationSourceHash:.*\n?/, "") - .replace(/^translationFrom:.*\n?/, "") - .replace(/^translationBlockHashes:[\s\S]*?(?=^[A-Za-z_][\w-]*:|\s*$)/m, "") - .replace(/^translationMismatches:(?:\n\s+-.*?)*/g, "") - ); +function stripAndSanitizeTranslationMeta(body: string): string { + return sanitizeFrontmatterBody(stripTranslationMetaFromFrontmatter(body)); } /** Inject or update translation metadata in frontmatter (hash only — mismatches go to .github/i18n-logs/translate/) */ @@ -306,7 +315,7 @@ function setTranslationMeta(content: string, hash: string, enPath: string): stri } const [, open, body, close] = fmMatch; const rest = content.slice(fmMatch[0].length); - const cleaned = stripTranslationMetaFromFrontmatter(body); + const cleaned = stripAndSanitizeTranslationMeta(body); return `${open}${cleaned}\n${metaBlock}${close}${rest}`; } @@ -526,123 +535,33 @@ function cleanModelOutput(text: string): string { } // --------------------------------------------------------------------------- -// Chunked translation ( blocks — changelog/index.mdx) +// Chunked translation (update_blocks | heading_sections) // --------------------------------------------------------------------------- -interface UpdateBlock { - label: string; - content: string; -} - -interface ParsedUpdateDocument { - frontmatter: string; - blocks: UpdateBlock[]; -} - -function isChunkedFile(relPath: string): boolean { - const normalized = relPath.replace(/\\/g, "/"); - return CHUNKED_FILES.some( - (c) => c.strategy === "update_blocks" && normalized === c.path.replace(/\\/g, "/") - ); -} - -function parseFrontmatterAndBody(content: string): { frontmatter: string; body: string } { - const match = content.match(/^(---\n[\s\S]*?\n---)\n?([\s\S]*)$/); - if (!match) return { frontmatter: "", body: content }; - return { frontmatter: `${match[1]}\n`, body: match[2] }; -} - -function parseUpdateBlocks(body: string): UpdateBlock[] { - const blocks: UpdateBlock[] = []; - const re = /]*>[\s\S]*?<\/Update>/g; - let match: RegExpExecArray | null; - while ((match = re.exec(body)) !== null) { - blocks.push({ label: match[1], content: match[0] }); - } - return blocks; -} - -function parseUpdateDocument(content: string): ParsedUpdateDocument { - const { frontmatter, body } = parseFrontmatterAndBody(content); - return { frontmatter, blocks: parseUpdateBlocks(body) }; -} - -/** Changelog sync key: ordered version labels only (content edits to old blocks are ignored). */ -function changelogLabelHash(enDoc: ParsedUpdateDocument): string { - return sourceHash(enDoc.blocks.map((b) => b.label).join("|")); -} - -interface ChunkedSyncStatus { - upToDate: boolean; - pendingBlocks: string[]; - needsFrontmatter: boolean; -} - -function getChunkedSyncStatus( - enContent: string, - existingContent: string, - force: boolean -): ChunkedSyncStatus { - const enDoc = parseUpdateDocument(enContent); - const existingDoc = existingContent ? parseUpdateDocument(existingContent) : null; - const existingLabels = new Set((existingDoc?.blocks ?? []).map((b) => b.label)); - - if (force) { - return { - upToDate: false, - pendingBlocks: enDoc.blocks.map((b) => b.label), - needsFrontmatter: true, - }; - } - - const pendingBlocks = enDoc.blocks - .filter((b) => !existingLabels.has(b.label)) - .map((b) => b.label); - - return { - upToDate: pendingBlocks.length === 0 && Boolean(existingDoc), - pendingBlocks, - needsFrontmatter: !existingDoc, - }; -} - -interface ChunkedBlockSlot { - label: string; - content: string | null; -} - -function serializeChunkedDocument( - frontmatter: string, - slots: ChunkedBlockSlot[], - fileHash: string, - enRel: string -): string { - const body = slots - .map((s) => s.content) - .filter((c): c is string => c !== null) - .join("\n\n"); - return setTranslationMeta(`${frontmatter}\n${body}\n`, fileHash, enRel); -} - async function writeChunkedCheckpoint( targetPath: string, frontmatter: string, - slots: ChunkedBlockSlot[], + slots: BlockSlot[], fileHash: string, enRel: string, + blockHashes: Record, label?: string ): Promise { await mkdir(dirname(targetPath), { recursive: true }); - await writeFile(targetPath, serializeChunkedDocument(frontmatter, slots, fileHash, enRel)); + await writeFile( + targetPath, + serializeChunkedDocument(frontmatter, slots, fileHash, enRel, blockHashes) + ); if (label) { const done = slots.filter((s) => s.content !== null).length; console.log(` Saved ${label} → disk (${done}/${slots.length} blocks)`); } } -async function translateUpdateChunkedFile( +async function translateChunkedFile( relPath: string, lang: LangConfig, + strategy: ChunkStrategy, force: boolean, enContent: string, existingContent: string, @@ -654,22 +573,63 @@ async function translateUpdateChunkedFile( blocksTranslated: number; output: string | null; }> { - const status = getChunkedSyncStatus(enContent, existingContent, force); + const status = getSectionSyncStatus(enContent, existingContent, strategy, force); if (status.upToDate) { return { mismatches: [], status: "up-to-date", blocksTranslated: 0, output: null }; } - const enDoc = parseUpdateDocument(enContent); - const existingDoc = existingContent ? parseUpdateDocument(existingContent) : null; + const enDoc = parseDocument(enContent, strategy); + const existingDoc = existingContent ? parseDocument(existingContent, strategy) : null; + const enBlockHashes = documentBlockHashes(enDoc.blocks); + const fileHash = + strategy === "update_blocks" + ? changelogLabelHash(enDoc.blocks) + : aggregateDocumentHash(enBlockHashes); + const existingByLabel = new Map( (existingDoc?.blocks ?? []).map((b) => [b.label, b.content]) ); - const fileHash = changelogLabelHash(enDoc); + const existingByIndex = + strategy === "heading_sections" && existingContent + ? parseTargetSectionsByIndex(parseFrontmatterAndBody(existingContent).body, enDoc.blocks.length) + : []; + + const slots: BlockSlot[] = enDoc.blocks.map((b, i) => { + if (!force && !status.pendingBlocks.includes(b.label)) { + const content = + strategy === "heading_sections" + ? existingByIndex[i] ?? null + : existingByLabel.get(b.label) ?? null; + return { label: b.label, content: content?.trim() ? content : null }; + } + return { label: b.label, content: null }; + }); - const slots: ChunkedBlockSlot[] = enDoc.blocks.map((b) => ({ - label: b.label, - content: !force && existingByLabel.has(b.label) ? existingByLabel.get(b.label)! : null, - })); + if (status.needsReserialize && status.pendingBlocks.length === 0) { + if (slots.every((s) => s.content?.trim())) { + const translatedFrontmatter = existingDoc?.frontmatter ?? enDoc.frontmatter; + const filledSlots: BlockSlot[] = slots.map((s) => ({ + label: s.label, + content: s.content!, + })); + const output = serializeChunkedDocument( + translatedFrontmatter, + filledSlots, + fileHash, + enRel, + enBlockHashes + ); + await mkdir(dirname(targetPath), { recursive: true }); + await writeFile(targetPath, output); + console.log(` Re-serialized after EN section removal (no re-translation)`); + return { + mismatches: [], + status: "translated", + blocksTranslated: 0, + output, + }; + } + } const allMismatches: string[] = []; let blocksTranslated = 0; @@ -692,33 +652,46 @@ async function translateUpdateChunkedFile( } allMismatches.push(...fmResult.mismatches); frontmatterDirty = true; - await writeChunkedCheckpoint(targetPath, translatedFrontmatter, slots, fileHash, enRel); + await writeChunkedCheckpoint( + targetPath, + translatedFrontmatter, + slots, + fileHash, + enRel, + enBlockHashes, + ); } else { translatedFrontmatter = existingDoc!.frontmatter; } - for (const slot of slots) { + for (let i = 0; i < slots.length; i++) { + const slot = slots[i]!; if (slot.content !== null) continue; - const enBlock = enDoc.blocks.find((b) => b.label === slot.label)!; - const existingBlock = existingByLabel.get(slot.label); + const enBlock = enDoc.blocks[i]!; + const existingBlock = + strategy === "heading_sections" + ? existingByIndex[i] ?? "" + : existingByLabel.get(slot.label) ?? ""; + + const blockTag = + strategy === "heading_sections" + ? slot.label === "_intro" + ? "intro" + : slot.label + : slot.label; - console.log(` Translating block: ${slot.label}...`); + console.log(` Translating block: ${blockTag}...`); const blockResult = IS_QWEN_MT - ? await translateWithQwenMT(enBlock.content, existingBlock ?? "", lang) - : await translateWithLLM( - enBlock.content, - existingBlock ?? "", - lang, - `${relPath}#${slot.label}` - ); + ? await translateWithQwenMT(enBlock.content, existingBlock, lang) + : await translateWithLLM(enBlock.content, existingBlock, lang, `${relPath}#${blockTag}`); let translatedBlock = cleanModelOutput(blockResult.content); translatedBlock = localizeMdxPaths(translatedBlock, lang, config.languages); - if (!translatedBlock.includes(" 0 || frontmatterDirty || status.needsFrontmatter; return { @@ -771,11 +751,15 @@ async function translateFile( const existingContent = await readFileOr(targetPath); - if (!snippetsMode && isChunkedFile(relPath)) { - console.log(` → [CHUNKED] ${lang.code}/${relPath}`); - const chunked = await translateUpdateChunkedFile( + const chunkStrategy = !snippetsMode ? resolveFileChunkStrategy(relPath, enContent) : null; + if (chunkStrategy) { + const strategyLabel = + chunkStrategy === "heading_sections" ? "SECTIONS" : "CHUNKED"; + console.log(` → [${strategyLabel}] ${lang.code}/${relPath}`); + const chunked = await translateChunkedFile( relPath, lang, + chunkStrategy, force, enContent, existingContent, @@ -984,8 +968,14 @@ async function runTranslatePhase(options: { const existing = await readFileOr(targetPath); - if (!snippetsMode && isChunkedFile(relPath)) { - const chunkedStatus = getChunkedSyncStatus(enContent, existing, false); + const chunkStrategy = !snippetsMode ? resolveFileChunkStrategy(relPath, enContent) : null; + if (chunkStrategy) { + const chunkedStatus = getSectionSyncStatus( + enContent, + existing, + chunkStrategy, + false + ); if (chunkedStatus.upToDate) upToDate.push(job); else pending.push(job); continue; @@ -1005,17 +995,26 @@ async function runTranslatePhase(options: { if (dryRun) { console.log(`Would translate (${phaseLabel}):`); for (const { relPath, lang } of pending.slice(0, 40)) { - if (!snippetsMode && isChunkedFile(relPath)) { + const chunkStrategy = !snippetsMode + ? resolveFileChunkStrategy( + relPath, + await readFileOr(makeMapping(lang, relPath, false).enPath) + ) + : null; + if (chunkStrategy) { const enContent = await readFileOr(makeMapping(lang, relPath, false).enPath); const existing = await readFileOr(makeMapping(lang, relPath, false).targetPath); - const cs = getChunkedSyncStatus(enContent, existing, false); + const cs = getSectionSyncStatus(enContent, existing, chunkStrategy, false); const parts: string[] = []; if (cs.needsFrontmatter) parts.push("frontmatter"); if (cs.pendingBlocks.length > 0) { - parts.push(`${cs.pendingBlocks.length} missing version(s)`); + const unit = + chunkStrategy === "heading_sections" ? "section(s)" : "version(s)"; + parts.push(`${cs.pendingBlocks.length} pending ${unit}`); } const detail = parts.length > 0 ? ` (${parts.join(", ")})` : ""; - console.log(` [${lang.code}] ${relPath}${detail}`); + const tag = chunkStrategy === "heading_sections" ? "SECTIONS" : "CHUNKED"; + console.log(` [${lang.code}] ${relPath} [${tag}]${detail}`); } else { console.log(` [${lang.code}] ${relPath}`); } diff --git a/.github/scripts/i18n/translation-config.json b/.github/scripts/i18n/translation-config.json index 47d667a30..b539eda6a 100644 --- a/.github/scripts/i18n/translation-config.json +++ b/.github/scripts/i18n/translation-config.json @@ -5,8 +5,16 @@ { "path": "changelog/index.mdx", "strategy": "update_blocks" + }, + { + "path": "tutorials/partner-nodes/pricing.mdx", + "strategy": "heading_sections" } ], + "auto_chunk": { + "min_body_chars": 10000, + "min_sections": 4 + }, "languages": [ { "code": "ja", @@ -28,7 +36,7 @@ } ], "preserve_terms": [ - "ComfyUI", "LoRA", "VAE", "CLIP", "checkpoint", "ControlNet", + "ComfyUI", "Comfy Cloud", "LoRA", "VAE", "CLIP", "checkpoint", "ControlNet", "KSampler", "UNet", "API", "JSON", "MDX", "GitHub", "Hugging Face", "Civitai", "ComfyUI Manager", "safetensors", "GGUF", "SDXL", "SD1.5", "Stable Diffusion", "Black Forest Labs", "Stability AI", "RealESRGAN",