[WIP] Use Crowdin translations for reactjs.org

.eslintignore

@@ -2,6 +2,9 @@ node_modules/*
 
 # Ignore markdown files and examples
 content/*
+crowdin/__exported__/*
+crowdin/__translated__/*
+crowdin/__untranslated__/*
 
 # Ignore built files
 public/*

.gitignore

@@ -3,3 +3,4 @@
 .idea
 node_modules
 public
+yarn-error.log

crowdin/.gitignore

@@ -0,0 +1,3 @@
+__exported__/
+__translated__/
+languages.json

crowdin/README.md

@@ -0,0 +1,105 @@
+## How does it work?
+
+**Only content from [the `content/docs` directory](https://github.com/reactjs/reactjs.org/tree/master/content/docs) is localized. All other sections/pages remain English only.**
+
+### Downloading content from Crowdin
+
+This directory contains some JavaScript files as well as a symlink for the default language (English) that points to [the `content/docs` directory](https://github.com/reactjs/reactjs.org/tree/master/content/docs):
+```sh
+.
+└── crowdin
+   ├── __translated__ #----------------------- Initially empty, except for English
+   │   └── en-US
+   │       └── docs -> ../../../content/docs
+   ├── __untranslated__ #--------------------- Contains symlinks to untranslated content
+   │   ├── blog -> ../../content/blog
+   │   └── # ...
+   ├── config.js #---------------------------- Crowdin configuration settings
+   └── download.js #-------------------------- Node Download script
+```
+
+To retrieve translations using the Crowdin API, use the Yarn task `yarn crowdin:download`. This will download data into an `__exported__` subdirectory:
+```sh
+.
+└── crowdin
+   ├── __exported__
+   │   └── # Crowdin expoert goes here ...
+   ├── __translated__
+   │   └── # ...
+   ├── __untranslated__
+   │   └── # ...
+   └── # ...
+```
+
+Next the task identifies which languages have been translated past a certain threshold (specified by `config.js`). For these languages, the script creates symlinks in the `__translated__` subdirectory:
+```sh
+.
+└── crowdin
+   ├── __exported__
+   │   └── # ...
+   ├── __translated__
+   │   ├── en-US
+   │   │   └── docs -> ../../../content/docs
+   │   ├── es-ES
+   │   │   └── docs -> ../../__exported__/path/to/docs/es-ES/docs
+   │   └── # Other languages that pass the threshold ...
+   ├── __untranslated__
+   │   └── # ...
+   └── # ...
+```
+
+### Gatsby integration
+
+A new (local) `gatsby-plugin-crowdin` plugin has been created that knows how to create localized links to certain sections of the website (e.g. things within the translated "/docs" directory).
+
+The `gatsby-source-filesystem` plugin has been configured to read all content from the `crowdin/__translated__/` and  `crowdin/__untranslated__/` (symlinked) directories rather than `content`. This way it consumes translated content when available. (Crowdin provides default language fallbacks for pages/sections that have not yet been translated for any given locale.)
+
+This configuration is done via `gatsby-config.js`:
+```js
+{
+  resolve: 'gatsby-source-filesystem',
+  options: {
+    name: 'untranslated',
+    path: `${__dirname}/crowdin/__untranslated__/`,
+  },
+},
+{
+  resolve: 'gatsby-source-filesystem',
+  options: {
+    name: 'translated',
+    path: `${__dirname}/crowdin/__translated__/`,
+  },
+},
+```
+
+Because of the default initial symlink (`crowdin/__translated__/en-US/docs` -> `content/docs`) Gatsby will still serve English content when run locally, even if the Crowdin script has not been run. This should enable fast iteration and creation of new content.
+
+Translations can be updated by running `yarn crowdin:download` (or automatically as part of CI deployment).
+
+### Language selector
+
+The Yarn task `crowdin:update-languages` determines which translated languages have been downloaded. (This task is automatically run before `yarn dev` or `yarn build` in order to just-in-time update the list.) The task writes a list of locales to a local JSON file, `languages.json`:
+
+```sh
+.
+└── crowdin
+   ├── __exported__
+   │   └── # ...
+   ├── __translated__
+   │   └── # ...
+   ├── __untranslated__
+   │   └── # ...
+   ├── translated-languages.json # This is the list of local translations
+   └── # ...
+```
+
+This `languages.json` list is imported into a translations page (`pages/translations.js`) and used to create a list of links to translated docs.
+
+### Locale persistence
+
+By default, legacy links to docs pages (e.g. `/docs/hello-world.html`) are re-routed to a new page (`docs-language-redirect.js`) that determines which locale to redirect to (e.g. `/en-US/docs/hello-world.html`). This is done as follows:
+* First it checks `localStorage` for the user's selected language. If one is found, it is used.
+* Next it checks the user's preferred languages (using `navigator.languages`). If any have been translated, it is used.
+* Lastly it falls back to English.
+
+Each time a user visits a localized docs path, the website updates their currently selected language (in `localStorage`) so that subsequent visits (within this session or a new session) will restore their selected language.

crowdin/__translated__/.gitignore

@@ -0,0 +1 @@
+!en-US

crowdin/__translated__/en-US/docs

@@ -0,0 +1 @@
+../../../content/docs

crowdin/__untranslated__/404.md

@@ -0,0 +1 @@
+../../content/404.md

crowdin/__untranslated__/acknowledgements.yml

@@ -0,0 +1 @@
+../../content/acknowledgements.yml

crowdin/__untranslated__/authors.yml

@@ -0,0 +1 @@
+../../content/authors.yml

crowdin/__untranslated__/blog

@@ -0,0 +1 @@
+../../content/blog

crowdin/__untranslated__/community

@@ -0,0 +1 @@
+../../content/community

crowdin/__untranslated__/home

@@ -0,0 +1 @@
+../../content/home

crowdin/__untranslated__/images

@@ -0,0 +1 @@
+../../content/images

crowdin/__untranslated__/tutorial

@@ -0,0 +1 @@
+../../content/tutorial

crowdin/__untranslated__/versions.yml

@@ -0,0 +1 @@
+../../content/versions.yml

crowdin/__untranslated__/warnings

@@ -0,0 +1 @@
+../../content/warnings

crowdin/config.js

@@ -0,0 +1,11 @@
+const path = require('path');
+
+// Also relates to the crowdin.yaml file in the root directory
+module.exports = {
+  defaultLanguage: 'en',
+  downloadedRootDirectory: path.join('test-17', 'docs'),
+  key: process.env.CROWDIN_API_KEY,
+  threshold: 50,
+  url: 'https://api.crowdin.com/api/project/react',
+  whitelist: ['docs'],
+};

crowdin/download.js

@@ -0,0 +1,140 @@
+const Crowdin = require('crowdin-node');
+const {downloadedRootDirectory, key, threshold, url, whitelist} = require('./config');
+const {existsSync, mkdirSync} = require('fs');
+const {join, resolve} = require('path');
+const {symlink, lstatSync, readdirSync} = require('fs');
+
+const TRANSLATED_PATH = resolve(__dirname, '__translated__');
+const EXPORTED_PATH = resolve(__dirname, '__exported__');
+
+// Path to the "docs" folder within the downloaded Crowdin translations bundle.
+const downloadedDocsPath = resolve(
+  EXPORTED_PATH,
+  downloadedRootDirectory,
+);
+
+// Sanity check (local) Crowdin config file for expected values.
+const validateCrowdinConfig = () => {
+  const errors = [];
+  if (!key) {
+    errors.push('key: No process.env.CROWDIN_API_KEY value defined.');
+  }
+  if (!Number.isInteger(threshold)) {
+    errors.push(`threshold: Invalid translation threshold defined.`);
+  }
+  if (!downloadedRootDirectory) {
+    errors.push('downloadedRootDirectory: No root directory defined for the downloaded translations bundle.');
+  }
+  if (!url) {
+    errors.push('url: No Crowdin project URL defined.');
+  }
+  if (errors.length > 0) {
+    console.error('Invalid Crowdin config values for:\n• ' + errors.join('\n• '));
+    throw Error('Invalid Crowdin config');
+  }
+};
+
+// Download Crowdin translations (into EXPORTED_PATH),
+// Filter languages that have been sufficiently translated (based on config.threshold),
+// And setup symlinks for them (in TRANSLATED_PATH) for Gatsby to read.
+const downloadAndSymlink = () => {
+  const crowdin = new Crowdin({apiKey: key, endpointUrl: url});
+  crowdin
+    // .export() // Not sure if this should be called in the script since it could be very slow
+    // .then(() => crowdin.downloadToPath(EXPORTED_PATH))
+    .downloadToPath(EXPORTED_PATH)
+    .then(() => crowdin.getTranslationStatus())
+    .then(locales => {
+      const usableLocales = locales
+        .filter(
+          locale => locale.translated_progress > threshold,
+        )
+        .map(local => local.code);
+
+      const localeDirectories = getLanguageDirectories(downloadedDocsPath);
+      const localeToFolderMap = createLocaleToFolderMap(localeDirectories);
+
+      usableLocales.forEach(locale => {
+        const languageCode = localeToFolderMap.get(locale);
+        const rootLanguageFolder = resolve(TRANSLATED_PATH, languageCode);
+
+        if (Array.isArray(whitelist)) {
+          if (!existsSync(rootLanguageFolder)) {
+            mkdirSync(rootLanguageFolder);
+          }
+
+          // Symlink only the whitelisted subdirectories
+          whitelist.forEach(subdirectory => {
+            createSymLink(join(languageCode, subdirectory));
+          });
+        } else {
+          // Otherwise symlink the entire language export
+          createSymLink(languageCode);
+        }
+      });
+    });
+
+};
+
+// Creates a relative symlink from a downloaded translation in the current working directory
+// Note that the current working directory of this node process should be where the symlink is created
+// or else the relative paths would be incorrect
+const createSymLink = (relativePath) => {
+  const from = resolve(downloadedDocsPath, relativePath);
+  const to = resolve(TRANSLATED_PATH, relativePath);
+  symlink(from, to, err => {
+    if (!err) {
+      return;
+    }
+
+    if (err.code === 'EEXIST') {
+      // eslint-disable-next-line no-console
+      console.info(`Symlink already exists for ${to}`);
+    } else {
+      console.error(err);
+      process.exit(1);
+    }
+  });
+};
+
+// Crowdin.getTranslationStatus() provides ISO 639-1 (e.g. "fr" for French) or 639-3 (e.g. "fil" for Filipino) language codes,
+// But the folder structure of downloaded translations uses locale codes (e.g. "fr-FR" for French, "fil-PH" for the Philippines).
+// This function creates a map between language and locale code.
+const createLocaleToFolderMap = (directories) => {
+  const localeToLanguageCode = locale => locale.includes('-') ? locale.substr(0, locale.indexOf('-')) : locale;
+  const localeToFolders = new Map();
+  const localeToFolder = new Map();
+
+  for (let locale of directories) {
+    const languageCode = localeToLanguageCode(locale);
+
+    localeToFolders.set(
+      languageCode,
+      localeToFolders.has(languageCode)
+        ? localeToFolders.get(languageCode).concat(locale)
+        : [locale],
+    );
+  }
+
+  localeToFolders.forEach((folders, locale) => {
+    if (folders.length === 1) {
+      localeToFolder.set(locale, folders[0]);
+    } else {
+      for (let folder of folders) {
+        localeToFolder.set(folder, folder);
+      }
+    }
+  });
+
+  return localeToFolder;
+};
+
+// Parse downloaded translation folder to determine which langauges it contains.
+const getLanguageDirectories = source =>
+  readdirSync(source).filter(
+    name =>
+      lstatSync(join(source, name)).isDirectory() && name !== '_data',
+  );
+
+validateCrowdinConfig();
+downloadAndSymlink();

crowdin/update-languages.js

@@ -0,0 +1,22 @@
+const {readdirSync, statSync, writeFileSync} = require('fs');
+const {join, resolve} = require('path');
+
+const TRANSLATED_LANGUAGES_JSON_PATH = resolve(__dirname, 'languages.json');
+const TRANSLATED_PATH = resolve(__dirname, '__translated__');
+
+// TODO Use crowdin.yaml
+
+// Determine which languages we have translations downloaded for...
+const languages = [];
+readdirSync(TRANSLATED_PATH).forEach(entry => {
+  if (statSync(join(TRANSLATED_PATH, entry)).isDirectory()) {
+    languages.push(entry);
+  }
+});
+
+// Update the languages JSON config file.
+// This file is used to display the localization toggle UI.
+writeFileSync(
+  TRANSLATED_LANGUAGES_JSON_PATH,
+  JSON.stringify(languages),
+);

gatsby-config.js

@@ -24,6 +24,10 @@ module.exports = {
     'gatsby-plugin-netlify',
     'gatsby-plugin-glamor',
     'gatsby-plugin-react-next',
+    {
+      resolve: 'gatsby-plugin-crowdin',
+      options: {},
+    },
     'gatsby-plugin-twitter',
     {
       resolve: 'gatsby-plugin-nprogress',
@@ -34,15 +38,22 @@ module.exports = {
     {
       resolve: 'gatsby-source-filesystem',
       options: {
-        path: `${__dirname}/src/pages`,
         name: 'pages',
+        path: `${__dirname}/src/pages`,
       },
     },
     {
       resolve: 'gatsby-source-filesystem',
       options: {
-        name: 'packages',
-        path: `${__dirname}/content/`,
+        name: 'untranslated',
+        path: `${__dirname}/crowdin/__untranslated__/`,
+      },
+    },
+    {
+      resolve: 'gatsby-source-filesystem',
+      options: {
+        name: 'translated',
+        path: `${__dirname}/crowdin/__translated__/`,
       },
     },
     {
@@ -70,6 +81,10 @@ module.exports = {
               target: '_blank',
             },
           },
+          {
+            resolve: 'gatsby-plugin-crowdin',
+            options: {},
+          },
           {
             resolve: 'gatsby-remark-embed-snippet',
             options: {