Merge pull request #1310 from geichelberger/i18n-lazy-loading

Add lazy loading of lang files

.github/generate-lngs.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+declare -A country_language_map
+declare -a order
+
+# Loop through each file in the directory
+for file in ??-??.json; do
+    if [ -f "$file" ]; then
+        # Extract country name and language code from the filename
+        country=$(basename "$file" .json | cut -d '-' -f 1)
+        language_code=$(basename "$file" .json)
+
+        if [ ! "${country_language_map[$country]}" ]; then
+            order+=("$country")
+        fi
+
+        # Check if the country already exists in the map
+        if [ -n "${country_language_map[$country]}" ]; then
+            country_language_map["$country"]+=" $language_code"
+        else
+            country_language_map["$country"]="$language_code"
+        fi
+    fi
+done
+
+echo "export const languages = new Map<string, string>(["
+# Print the country-language mappings
+for i in "${!order[@]}"; do
+    country=${order[$i]}
+    languages=()
+    for language in ${country_language_map[$country]}; do
+        languages+=("$language")
+    done
+    if [ ${#languages[@]} -eq 1 ]; then
+        echo "  [\"$country\", \"${languages[0]}\"],"
+    else
+        for language in "${languages[@]}"; do
+            echo "  [\"$language\", \"$language\"],"
+        done
+    fi
+done
+echo "]);"

.github/workflows/update-translations.yml

@@ -34,10 +34,7 @@ jobs:
 
       - name: update language list
         working-directory: src/i18n/locales
-        run: |
-          echo -n '[ "' > locales.json
-          echo -n ??-??.json | sed 's/  */", "/g' >> locales.json
-          echo '" ]' >> locales.json
+        run: $GITHUB_WORKSPACE/.github/generate-lngs.sh > ../lngs-generated.ts
 
       - name: upload translations
         run: |

package.json

@@ -19,6 +19,8 @@
     "final-form": "^4.20.10",
     "i18next": "^23.7.11",
     "i18next-browser-languagedetector": "^7.2.0",
+    "i18next-chained-backend": "^4.6.2",
+    "i18next-resources-to-backend": "^1.2.0",
     "lodash": "^4.17.21",
     "luxon": "^3.4.4",
     "mui-rff": "^7.3.0",

src/i18n/LazyLoadingPlugin.ts

@@ -0,0 +1,37 @@
+import { BackendModule, InitOptions, MultiReadCallback, ReadCallback, ResourceLanguage, Services } from "i18next";
+import { languages } from "./lngs-generated";
+
+export default class LazyLoadingPlugin implements BackendModule {
+
+  type: "backend";
+
+  constructor(_services: Services, _backendOptions: object, _i18nextOptions: InitOptions<object>) {
+    this.type = "backend";
+  }
+
+  init(_services: Services, _backendOptions: object, _i18nextOptions: InitOptions<object>): void {
+    // no init needed
+  }
+
+  read(language: string, _namespace: string, callback: ReadCallback): void {
+    const lng = languages.get(language);
+    import(`./locales/${lng}.json`).then(
+      obj => {
+        callback(null, obj);
+      }
+    );
+  }
+
+  create?(_languages: readonly string[], _namespace: string, _key: string, _fallbackValue: string): void {
+    throw new Error("Method not implemented.");
+  }
+
+  readMulti?(_languages: readonly string[], _namespaces: readonly string[], _callback: MultiReadCallback): void {
+    throw new Error("Method not implemented.");
+  }
+
+  save?(_language: string, _namespace: string, _data: ResourceLanguage): void {
+    throw new Error("Method not implemented.");
+  }
+
+}

src/i18n/config.tsx

@@ -1,41 +1,36 @@
-import i18next, { InitOptions } from "i18next";
+import i18next from "i18next";
 import { initReactI18next } from "react-i18next";
 import LanguageDetector from "i18next-browser-languagedetector";
+import ChainedBackend, { ChainedBackendOptions } from "i18next-chained-backend";
+import resourcesToBackend from "i18next-resources-to-backend";
 
-import locales from "./locales.json";
+import { languages } from "./lngs-generated";
+import LazyLoadingPlugin from "./LazyLoadingPlugin";
 
-const debug = Boolean(new URLSearchParams(window.location.search).get("debug"));
-
-const resources: InitOptions["resources"] = {};
-
-const data = import.meta.glob("./locales/*.json");
-
-for (const path in data) {
-  const code = path.replace(/^.*[\\/]/, "").replace(/\..*$/, "");
-  if (!locales.some(e => e.includes(code))) {
-    continue;
-  }
-  const short = code.replace(/-.*$/, "");
-  const main = locales.filter(l => l.indexOf(short) === 0).length === 1;
 
-  data[path]().then(mod => {
-    const translation = JSON.parse(JSON.stringify(mod));
+const debug = Boolean(new URLSearchParams(window.location.search).get("debug"));
 
-    if (!main) {
-      resources[code] = { translation: translation };
-    }
-    resources[short] = { translation: translation };
-  });
-}
+const bundledResources = {
+  en: {
+    translation: import("./locales/en-US.json"),
+  },
+};
 
 i18next
+  .use(ChainedBackend)
   .use(initReactI18next)
   .use(LanguageDetector)
-  .init({
-    resources,
-    fallbackLng: ["en-US", "en"],
-    nonExplicitSupportedLngs: true,
+  .init<ChainedBackendOptions>({
+    supportedLngs: Array.from(languages.keys()),
+    fallbackLng: ["en", "en-US"],
+    nonExplicitSupportedLngs: false,
     debug: debug,
+    backend: {
+      backends: [
+        LazyLoadingPlugin,
+        resourcesToBackend(bundledResources),
+      ],
+    },
   });
 
 if (debug) {

src/i18n/i18next.d.ts

@@ -2,7 +2,7 @@
 import 'i18next';
 
 // import all namespaces (for the default language, only)
-import translation from '../i18n/locales/en-US.json';
+import translation from './locales/en-US.json';
 
 declare module 'i18next' {
   interface CustomTypeOptions {

src/i18n/lngs-generated.ts

@@ -0,0 +1,11 @@
+export const languages = new Map<string, string>([
+  ["cs", "cs-CZ"],
+  ["de", "de-DE"],
+  ["el", "el-GR"],
+  ["en", "en-US"],
+  ["es", "es-ES"],
+  ["fr", "fr-FR"],
+  ["nl", "nl-NL"],
+  ["zh-CN", "zh-CN"],
+  ["zh-TW", "zh-TW"],
+]);

src/main/Header.tsx

@@ -17,6 +17,7 @@ import { selectIsEnd } from "../redux/endSlice";
 import { checkboxMenuItem, HeaderMenuItemDef, ProtoButton, useColorScheme, WithHeaderMenu } from "@opencast/appkit";
 import { IconType } from "react-icons";
 import i18next from "i18next";
+import { languages as lngs } from "../i18n/lngs-generated";
 
 function Header() {
   const theme = useTheme();
@@ -142,10 +143,8 @@ const LanguageButton: React.FC = () => {
     }).of(language);
   };
 
-  const resourcesArray: string[] | undefined = i18next.options.resources && Object.keys(i18next.options.resources);
-
-  const languages = resourcesArray?.map(entry => {
-    return { value: entry, label: languageNames(entry) };
+  const languages = Array.from(lngs, ([key, value]) => {
+    return { value: value, label: languageNames(key) };
   });
 
   // menuItems can"t deal with languages being undefined, so we return early