Load localizations lazily (#12932)

msujew · web-flow · commit c745963ca6df · 2023-10-26T14:10:35.000+02:00
diff --git a/packages/core/src/node/i18n/localization-contribution.ts b/packages/core/src/node/i18n/localization-contribution.ts
@@ -18,7 +18,7 @@ import * as fs from 'fs-extra';
 import { inject, injectable, named } from 'inversify';
 import { ContributionProvider, isObject } from '../../common';
 import { LanguageInfo, Localization } from '../../common/i18n/localization';
-import { LocalizationProvider } from './localization-provider';
+import { LazyLocalization, LocalizationProvider } from './localization-provider';
 
 export const LocalizationContribution = Symbol('LocalizationContribution');
 
@@ -41,38 +41,42 @@ export class LocalizationRegistry {
         ));
     }
 
-    registerLocalization(localization: Localization): void {
+    registerLocalization(localization: Localization | LazyLocalization): void {
+        if (!LazyLocalization.is(localization)) {
+            localization = LazyLocalization.fromLocalization(localization);
+        }
         this.localizationProvider.addLocalizations(localization);
     }
 
     registerLocalizationFromRequire(locale: string | LanguageInfo, required: unknown): void {
         const translations = this.flattenTranslations(required);
-        this.registerLocalization(this.createLocalization(locale, translations));
+        this.registerLocalization(this.createLocalization(locale, () => Promise.resolve(translations)));
     }
 
-    async registerLocalizationFromFile(localizationPath: string, locale?: string | LanguageInfo): Promise<void> {
+    registerLocalizationFromFile(localizationPath: string, locale?: string | LanguageInfo): void {
         if (!locale) {
             locale = this.identifyLocale(localizationPath);
         }
         if (!locale) {
             throw new Error('Could not determine locale from path.');
         }
-        const translationJson = await fs.readJson(localizationPath);
-        const translations = this.flattenTranslations(translationJson);
-        this.registerLocalization(this.createLocalization(locale, translations));
+        this.registerLocalization(this.createLocalization(locale, async () => {
+            const translationJson = await fs.readJson(localizationPath);
+            return this.flattenTranslations(translationJson);
+        }));
     }
 
-    protected createLocalization(locale: string | LanguageInfo, translations: Record<string, string>): Localization {
-        let localization: Localization;
+    protected createLocalization(locale: string | LanguageInfo, translations: () => Promise<Record<string, string>>): LazyLocalization {
+        let localization: LazyLocalization;
         if (typeof locale === 'string') {
             localization = {
                 languageId: locale,
-                translations
+                getTranslations: translations
             };
         } else {
             localization = {
                 ...locale,
-                translations
+                getTranslations: translations
             };
         }
         return localization;
diff --git a/packages/core/src/node/i18n/localization-provider.ts b/packages/core/src/node/i18n/localization-provider.ts
@@ -17,24 +17,70 @@
 import { injectable } from 'inversify';
 import { nls } from '../../common/nls';
 import { LanguageInfo, Localization } from '../../common/i18n/localization';
+import { Disposable } from '../../common/disposable';
+import { isObject } from '../../common/types';
+
+/**
+ * Localization data structure that contributes its localizations asynchronously.
+ * Allows to load localizations on demand when requested by the user.
+ */
+export interface LazyLocalization extends LanguageInfo {
+    getTranslations(): Promise<Record<string, string>>;
+}
+
+export namespace LazyLocalization {
+    export function is(obj: unknown): obj is LazyLocalization {
+        return isObject<LazyLocalization>(obj) && typeof obj.languageId === 'string' && typeof obj.getTranslations === 'function';
+    }
+    export function fromLocalization(localization: Localization): LazyLocalization {
+        const {
+            languageId,
+            languageName,
+            languagePack,
+            localizedLanguageName,
+            translations
+        } = localization;
+        return {
+            languageId,
+            languageName,
+            languagePack,
+            localizedLanguageName,
+            getTranslations: () => Promise.resolve(translations)
+        };
+    }
+    export async function toLocalization(localization: LazyLocalization): Promise<Localization> {
+        const {
+            languageId,
+            languageName,
+            languagePack,
+            localizedLanguageName
+        } = localization;
+        return {
+            languageId,
+            languageName,
+            languagePack,
+            localizedLanguageName,
+            translations: await localization.getTranslations()
+        };
+    }
+}
 
 @injectable()
 export class LocalizationProvider {
 
-    protected localizations: Localization[] = [];
+    protected localizations: LazyLocalization[] = [];
     protected currentLanguage = nls.defaultLocale;
 
-    addLocalizations(...localizations: Localization[]): void {
+    addLocalizations(...localizations: LazyLocalization[]): Disposable {
         this.localizations.push(...localizations);
-    }
-
-    removeLocalizations(...localizations: Localization[]): void {
-        for (const localization of localizations) {
-            const index = this.localizations.indexOf(localization);
-            if (index >= 0) {
-                this.localizations.splice(index, 1);
+        return Disposable.create(() => {
+            for (const localization of localizations) {
+                const index = this.localizations.indexOf(localization);
+                if (index >= 0) {
+                    this.localizations.splice(index, 1);
+                }
             }
-        }
+        });
     }
 
     setCurrentLanguage(languageId: string): void {
@@ -61,12 +107,13 @@ export class LocalizationProvider {
         return Array.from(languageInfos.values()).sort((a, b) => a.languageId.localeCompare(b.languageId));
     }
 
-    loadLocalization(languageId: string): Localization {
+    async loadLocalization(languageId: string): Promise<Localization> {
         const merged: Localization = {
             languageId,
             translations: {}
         };
-        for (const localization of this.localizations.filter(e => e.languageId === languageId)) {
+        const localizations = await Promise.all(this.localizations.filter(e => e.languageId === languageId).map(LazyLocalization.toLocalization));
+        for (const localization of localizations) {
             merged.languageName ||= localization.languageName;
             merged.localizedLanguageName ||= localization.localizedLanguageName;
             merged.languagePack ||= localization.languagePack;
diff --git a/packages/plugin-ext/src/common/plugin-protocol.ts b/packages/plugin-ext/src/common/plugin-protocol.ts
@@ -643,8 +643,7 @@ export interface Localization {
 export interface Translation {
     id: string;
     path: string;
-    version: string;
-    contents: { [scope: string]: { [key: string]: string } }
+    cachedContents?: { [scope: string]: { [key: string]: string } };
 }
 
 export interface SnippetContribution {
diff --git a/packages/plugin-ext/src/hosted/node/hosted-plugin-localization-service.ts b/packages/plugin-ext/src/hosted/node/hosted-plugin-localization-service.ts
@@ -16,13 +16,13 @@
 
 import * as path from 'path';
 import * as fs from '@theia/core/shared/fs-extra';
-import { LocalizationProvider } from '@theia/core/lib/node/i18n/localization-provider';
+import { LazyLocalization, LocalizationProvider } from '@theia/core/lib/node/i18n/localization-provider';
 import { Localization } from '@theia/core/lib/common/i18n/localization';
 import { inject, injectable } from '@theia/core/shared/inversify';
 import { DeployedPlugin, Localization as PluginLocalization, PluginIdentifiers, Translation } from '../../common';
 import { EnvVariablesServer } from '@theia/core/lib/common/env-variables';
 import { BackendApplicationContribution } from '@theia/core/lib/node';
-import { Disposable, DisposableCollection, isObject, MaybePromise, nls, URI } from '@theia/core';
+import { Disposable, DisposableCollection, isObject, MaybePromise, nls, Path, URI } from '@theia/core';
 import { Deferred } from '@theia/core/lib/common/promise-util';
 import { LanguagePackBundle, LanguagePackService } from '../../common/language-pack-service';
 
@@ -70,11 +70,8 @@ export class HostedPluginLocalizationService implements BackendApplicationContri
         if (plugin.contributes?.localizations) {
             // Indicator that this plugin is a vscode language pack
             // Language packs translate Theia and some builtin vscode extensions
-            const localizations = buildLocalizations(plugin.contributes.localizations);
-            disposable.push(Disposable.create(() => {
-                this.localizationProvider.removeLocalizations(...localizations);
-            }));
-            this.localizationProvider.addLocalizations(...localizations);
+            const localizations = buildLocalizations(plugin.metadata.model.packageUri, plugin.contributes.localizations);
+            disposable.push(this.localizationProvider.addLocalizations(...localizations));
         }
         if (plugin.metadata.model.l10n || plugin.contributes?.localizations) {
             // Indicator that this plugin is a vscode language pack or has its own localization bundles
@@ -100,26 +97,29 @@ export class HostedPluginLocalizationService implements BackendApplicationContri
         const pluginId = plugin.metadata.model.id;
         const packageUri = new URI(plugin.metadata.model.packageUri);
         if (plugin.contributes?.localizations) {
+            const l10nPromises: Promise<void>[] = [];
             for (const localization of plugin.contributes.localizations) {
                 for (const translation of localization.translations) {
-                    const l10n = getL10nTranslation(translation);
-                    if (l10n) {
-                        const translatedPluginId = translation.id;
-                        const translationUri = packageUri.resolve(translation.path);
-                        const locale = localization.languageId;
-                        // We store a bundle for another extension in here
-                        // Hence we use `translatedPluginId` instead of `pluginId`
-                        this.languagePackService.storeBundle(translatedPluginId, locale, {
-                            contents: processL10nBundle(l10n),
-                            uri: translationUri.toString()
-                        });
-                        disposable.push(Disposable.create(() => {
-                            // Only dispose the deleted locale for the specific plugin
-                            this.languagePackService.deleteBundle(translatedPluginId, locale);
-                        }));
-                    }
+                    l10nPromises.push(getL10nTranslation(plugin.metadata.model.packageUri, translation).then(l10n => {
+                        if (l10n) {
+                            const translatedPluginId = translation.id;
+                            const translationUri = packageUri.resolve(translation.path);
+                            const locale = localization.languageId;
+                            // We store a bundle for another extension in here
+                            // Hence we use `translatedPluginId` instead of `pluginId`
+                            this.languagePackService.storeBundle(translatedPluginId, locale, {
+                                contents: processL10nBundle(l10n),
+                                uri: translationUri.toString()
+                            });
+                            disposable.push(Disposable.create(() => {
+                                // Only dispose the deleted locale for the specific plugin
+                                this.languagePackService.deleteBundle(translatedPluginId, locale);
+                            }));
+                        }
+                    }));
                 }
             }
+            await Promise.all(l10nPromises);
         }
         // The `l10n` field of the plugin model points to a relative directory path within the plugin
         // It is supposed to contain localization bundles that contain translations of the plugin strings into different languages
@@ -150,11 +150,13 @@ export class HostedPluginLocalizationService implements BackendApplicationContri
      */
     async localizePlugin(plugin: DeployedPlugin): Promise<DeployedPlugin> {
         const currentLanguage = this.localizationProvider.getCurrentLanguage();
-        const localization = this.localizationProvider.loadLocalization(currentLanguage);
         const pluginPath = new URI(plugin.metadata.model.packageUri).path.fsPath();
         const pluginId = plugin.metadata.model.id;
         try {
-            const translations = await loadPackageTranslations(pluginPath, currentLanguage);
+            const [localization, translations] = await Promise.all([
+                this.localizationProvider.loadLocalization(currentLanguage),
+                loadPackageTranslations(pluginPath, currentLanguage),
+            ]);
             plugin = localizePackage(plugin, translations, (key, original) => {
                 const fullKey = `${pluginId}/package/${key}`;
                 return Localization.localize(localization, fullKey, original);
@@ -218,10 +220,24 @@ export class HostedPluginLocalizationService implements BackendApplicationContri
 
 // New plugin localization logic using vscode.l10n
 
-function getL10nTranslation(translation: Translation): UnprocessedL10nBundle | undefined {
+async function getL10nTranslation(packageUri: string, translation: Translation): Promise<UnprocessedL10nBundle | undefined> {
     // 'bundle' is a special key that contains all translations for the l10n vscode API
     // If that doesn't exist, we can assume that the language pack is using the old vscode-nls API
-    return translation.contents.bundle;
+    if (translation.cachedContents) {
+        return translation.cachedContents.bundle;
+    } else {
+        const translationPath = new URI(packageUri).path.join(translation.path).fsPath();
+        try {
+            const translationJson = await fs.readJson(translationPath);
+            translation.cachedContents = translationJson?.contents;
+            return translationJson?.contents?.bundle;
+        } catch (err) {
+            console.error('Failed reading translation file from: ' + translationPath, err);
+            // Store an empty object, so we don't reattempt to load the file
+            translation.cachedContents = {};
+            return undefined;
+        }
+    }
 }
 
 async function loadPluginBundles(l10nUri: URI): Promise<Record<string, LanguagePackBundle> | undefined> {
@@ -262,28 +278,47 @@ function processL10nBundle(bundle: UnprocessedL10nBundle): Record<string, string
 
 // Old plugin localization logic for vscode-nls
 // vscode-nls was used until version 1.73 of VSCode to translate extensions
+// This style of localization is still used by vscode language packs
 
-function buildLocalizations(localizations: PluginLocalization[]): Localization[] {
-    const theiaLocalizations: Localization[] = [];
+function buildLocalizations(packageUri: string, localizations: PluginLocalization[]): LazyLocalization[] {
+    const theiaLocalizations: LazyLocalization[] = [];
+    const packagePath = new URI(packageUri).path;
     for (const localization of localizations) {
-        const theiaLocalization: Localization = {
+        let cachedLocalization: Promise<Record<string, string>> | undefined;
+        const theiaLocalization: LazyLocalization = {
             languageId: localization.languageId,
             languageName: localization.languageName,
             localizedLanguageName: localization.localizedLanguageName,
             languagePack: true,
-            translations: {}
+            async getTranslations(): Promise<Record<string, string>> {
+                cachedLocalization ??= loadTranslations(packagePath, localization.translations);
+                return cachedLocalization;
+            },
         };
-        for (const translation of localization.translations) {
-            for (const [scope, value] of Object.entries(translation.contents)) {
+        theiaLocalizations.push(theiaLocalization);
+    }
+    return theiaLocalizations;
+}
+
+async function loadTranslations(packagePath: Path, translations: Translation[]): Promise<Record<string, string>> {
+    const allTranslations = await Promise.all(translations.map(async translation => {
+        const values: Record<string, string> = {};
+        const translationPath = packagePath.join(translation.path).fsPath();
+        try {
+            const translationJson = await fs.readJson(translationPath);
+            const translationContents: Record<string, Record<string, string>> = translationJson?.contents;
+            for (const [scope, value] of Object.entries(translationContents ?? {})) {
                 for (const [key, item] of Object.entries(value)) {
                     const translationKey = buildTranslationKey(translation.id, scope, key);
-                    theiaLocalization.translations[translationKey] = item;
+                    values[translationKey] = item;
                 }
             }
+        } catch (err) {
+            console.error('Failed to load translation from: ' + translationPath, err);
         }
-        theiaLocalizations.push(theiaLocalization);
-    }
-    return theiaLocalizations;
+        return values;
+    }));
+    return Object.assign({}, ...allTranslations);
 }
 
 function buildTranslationKey(pluginId: string, scope: string, key: string): string {
diff --git a/packages/plugin-ext/src/hosted/node/scanners/scanner-theia.ts b/packages/plugin-ext/src/hosted/node/scanners/scanner-theia.ts

Original file line number	Diff line number	Diff line change
`@@ -643,8 +643,7 @@ export interface Localization {`
`643`	`643`	`export interface Translation {`
`644`	`644`	`id: string;`
`645`	`645`	`path: string;`
`646`		`- version: string;`
`647`		`- contents: { [scope: string]: { [key: string]: string } }`
	`646`	`+ cachedContents?: { [scope: string]: { [key: string]: string } };`
`648`	`647`	`}`
`649`	`648`
`650`	`649`	`export interface SnippetContribution {`