Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions .github/scripts/i18n/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,35 @@ Quality controls during/after a run write to `.github/i18n-logs/translate/`
(gitignored): semantic mismatches reported by the model, and a truncation scan
(`check-translation-truncation.ts`).

### Long pages (chunked translation)

Very long MDX files (e.g. `tutorials/partner-nodes/pricing.mdx`) exceed model
output limits when translated in one shot. Two strategies avoid truncation:

| Strategy | Use case | Split boundary | Incremental sync |
|----------|----------|----------------|------------------|
| `heading_sections` | Long reference pages | Level-2 `##` headings | Per-section content hash in `translationBlockHashes` |
| `update_blocks` | Changelog | `<Update label="…">` blocks | New version labels only |

Configure explicit paths in `translation-config.json` → `chunked_files`, or rely
on `auto_chunk` (default: body ≥ 10k chars and ≥ 4 `##` sections) to auto-enable
`heading_sections`.

During a chunked run the script:

1. Parses English into blocks (intro + each `##` section).
2. Compares each block’s hash to `translationBlockHashes` in the target frontmatter.
3. Translates only pending blocks (plus frontmatter when needed).
4. Checkpoints after every block so a failed run can resume.

Example:

```bash
pnpm translate -- tutorials/partner-nodes/pricing.mdx --lang ko
pnpm translate:check-truncation -- --lang ko
pnpm translate:repair-truncated -- --lang ko # force re-translate flagged files
```

## Quality review (LLM-as-a-judge)

`review-i18n.ts` scores existing translations with an **independent** (and
Expand Down Expand Up @@ -139,6 +168,7 @@ env → `frontend_locales_path` in `translation-config.json` →
| File | Role |
|------|------|
| `translate-i18n.ts` | translation entry point |
| `chunked-translate.ts` | split/reassemble long MDX (`heading_sections`, `update_blocks`) |
| `sync-glossary.mjs` | rebuild the glossary frontend mirror |
| `glossary.mjs` | load glossary layers, select + inject terms |
| `i18n-config.mjs` | shared path rules from `translation-config.json` |
Expand Down
37 changes: 30 additions & 7 deletions .github/scripts/i18n/check-translation-truncation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,12 @@ import {
TRUNCATION_ISSUES_JSON,
TRUNCATION_ISSUES_TXT,
} from "./i18n-config.mjs";
import {
missingSectionLabels,
parseFrontmatterAndBody,
resolveChunkStrategy,
type ChunkedFileConfig,
} from "./chunked-translate.ts";

const ROOT = REPO_ROOT;
const LOG_DIR = TRANSLATE_LOG_DIR;
Expand All @@ -39,6 +45,8 @@ interface LangConfig {

interface TranslationConfig {
skip_paths: string[];
chunked_files?: ChunkedFileConfig[];
auto_chunk?: { min_body_chars?: number; min_sections?: number };
languages: LangConfig[];
}

Expand All @@ -59,10 +67,11 @@ export interface TruncationReport {

const config = loadI18nConfig() as TranslationConfig;
const pathFilterOpts = { languages: config.languages, skip_paths: config.skip_paths };
const CHUNKED_FILES = config.chunked_files ?? [];
const AUTO_CHUNK = config.auto_chunk;

function parseFrontmatterAndBody(content: string): { body: string } {
const match = content.match(/^---\n[\s\S]*?\n---\n?([\s\S]*)$/);
return { body: match ? match[1] : content };
function parseBody(content: string): string {
return parseFrontmatterAndBody(content).body;
}

function countCodeFences(body: string): { open: boolean; openLang: string; openLine: number } {
Expand Down Expand Up @@ -137,8 +146,8 @@ export function detectTruncation(
enRel: string
): TruncationIssue["reasons"] {
const reasons: string[] = [];
const enBody = parseFrontmatterAndBody(enContent).body;
const targetBody = parseFrontmatterAndBody(targetContent).body;
const enBody = parseBody(enContent);
const targetBody = parseBody(targetContent);

const fence = countCodeFences(targetBody);
if (fence.open) {
Expand Down Expand Up @@ -171,6 +180,14 @@ export function detectTruncation(
}
}

const chunkStrategy = resolveChunkStrategy(enRel, enBody, CHUNKED_FILES, AUTO_CHUNK);
if (chunkStrategy === "heading_sections") {
const missingSections = missingSectionLabels(enBody, targetBody, chunkStrategy);
if (missingSections.length > 0) {
reasons.push("missing_sections");
}
}

return reasons;
}

Expand All @@ -180,8 +197,8 @@ function formatDetail(
targetContent: string
): string {
const parts: string[] = [];
const enBody = parseFrontmatterAndBody(enContent).body;
const targetBody = parseFrontmatterAndBody(targetContent).body;
const enBody = parseBody(enContent);
const targetBody = parseBody(targetContent);
const fence = countCodeFences(targetBody);

if (reasons.includes("unclosed_code_fence")) {
Expand Down Expand Up @@ -209,6 +226,12 @@ function formatDetail(
const missing = enLabels.filter((l) => !targetLabels.has(l));
parts.push(`missing versions: ${missing.slice(0, 5).join(", ")}${missing.length > 5 ? "..." : ""}`);
}
if (reasons.includes("missing_sections")) {
const missing = missingSectionLabels(enBody, targetBody, "heading_sections");
parts.push(
`missing sections: ${missing.slice(0, 5).join(", ")}${missing.length > 5 ? "..." : ""}`
);
}
return parts.join("; ");
}

Expand Down
Loading
Loading