From ac4cc4a6113d850e27c35d533880871fad213e83 Mon Sep 17 00:00:00 2001 From: absidue <48293849+absidue@users.noreply.github.com> Date: Sat, 24 Sep 2022 17:06:50 +0200 Subject: [PATCH] Compress and lazy load locales (#2603) * Compress and lazy load locales * Remove index.html when loading the locales on the web * Fix locale output path for web build --- _scripts/ProcessLocalesPlugin.js | 96 +++++++++++++++++++ _scripts/webpack.main.config.js | 6 +- _scripts/webpack.renderer.config.js | 14 +++ _scripts/webpack.web.config.js | 17 +++- package.json | 2 +- .../ft-video-player/ft-video-player.js | 2 +- .../general-settings/general-settings.js | 22 +++-- src/renderer/i18n/index.js | 93 ++++++++++++++---- src/renderer/store/modules/settings.js | 6 +- .../SubscribedChannels/SubscribedChannels.js | 2 +- static/locales/activeLocales.json | 46 +++++++++ 11 files changed, 271 insertions(+), 35 deletions(-) create mode 100644 _scripts/ProcessLocalesPlugin.js create mode 100644 static/locales/activeLocales.json diff --git a/_scripts/ProcessLocalesPlugin.js b/_scripts/ProcessLocalesPlugin.js new file mode 100644 index 000000000..b96a58233 --- /dev/null +++ b/_scripts/ProcessLocalesPlugin.js @@ -0,0 +1,96 @@ +const { existsSync, readFileSync } = require('fs') +const { brotliCompressSync, constants } = require('zlib') +const { load: loadYaml } = require('js-yaml') + +class ProcessLocalesPlugin { + constructor(options = {}) { + this.compress = !!options.compress + + if (typeof options.inputDir !== 'string') { + throw new Error('ProcessLocalesPlugin: no input directory `inputDir` specified.') + } else if (!existsSync(options.inputDir)) { + throw new Error('ProcessLocalesPlugin: the specified input directory does not exist.') + } + this.inputDir = options.inputDir + + if (typeof options.outputDir !== 'string') { + throw new Error('ProcessLocalesPlugin: no output directory `outputDir` specified.') + } + this.outputDir = options.outputDir + + this.localeNames = [] + + this.loadLocales() + } + + apply(compiler) { + compiler.hooks.thisCompilation.tap('ProcessLocalesPlugin', (compilation) => { + + const { RawSource } = compiler.webpack.sources; + + compilation.hooks.processAssets.tapPromise({ + name: 'process-locales-plugin', + stage: compiler.webpack.Compilation.PROCESS_ASSETS_STAGE_ADDITIONAL + }, + async (_assets) => { + const promises = [] + + for (const { locale, data } of this.locales) { + promises.push(new Promise((resolve) => { + if (Object.prototype.hasOwnProperty.call(data, 'Locale Name')) { + delete data['Locale Name'] + } + + let filename = `${this.outputDir}/${locale}.json` + let output = JSON.stringify(data) + + if (this.compress) { + filename += '.br' + output = this.compressLocale(output) + } + + compilation.emitAsset( + filename, + new RawSource(output), + { minimized: true } + ) + + resolve() + })) + } + + await Promise.all(promises) + + this.locales = null + }) + }) + } + + loadLocales() { + this.locales = [] + + const activeLocales = JSON.parse(readFileSync(`${this.inputDir}/activeLocales.json`)) + + for (const locale of activeLocales) { + const contents = readFileSync(`${this.inputDir}/${locale}.yaml`, 'utf-8') + const data = loadYaml(contents) + + this.localeNames.push(data['Locale Name'] ?? locale) + this.locales.push({ locale, data }) + } + } + + compressLocale(data) { + const buffer = Buffer.from(data, 'utf-8') + + return brotliCompressSync(buffer, { + params: { + [constants.BROTLI_PARAM_MODE]: constants.BROTLI_MODE_TEXT, + [constants.BROTLI_PARAM_QUALITY]: constants.BROTLI_MAX_QUALITY, + [constants.BROTLI_PARAM_SIZE_HINT]: buffer.byteLength + } + }) + } +} + +module.exports = ProcessLocalesPlugin diff --git a/_scripts/webpack.main.config.js b/_scripts/webpack.main.config.js index fb1ad736e..54d4ccb00 100644 --- a/_scripts/webpack.main.config.js +++ b/_scripts/webpack.main.config.js @@ -27,7 +27,9 @@ const config = { optimization: { minimizer: [ '...', // extend webpack's list instead of overwriting it - new JsonMinimizerPlugin() + new JsonMinimizerPlugin({ + exclude: /\/locales\/.*\.json/ + }) ] }, node: { @@ -70,7 +72,7 @@ if (isDevMode) { to: path.join(__dirname, '../dist/static'), globOptions: { dot: true, - ignore: ['**/.*', '**/pwabuilder-sw.js', '**/dashFiles/**', '**/storyboards/**'], + ignore: ['**/.*', '**/locales/**', '**/pwabuilder-sw.js', '**/dashFiles/**', '**/storyboards/**'], }, }, ] diff --git a/_scripts/webpack.renderer.config.js b/_scripts/webpack.renderer.config.js index d1a9287d1..12a408bff 100644 --- a/_scripts/webpack.renderer.config.js +++ b/_scripts/webpack.renderer.config.js @@ -4,6 +4,7 @@ const HtmlWebpackPlugin = require('html-webpack-plugin') const VueLoaderPlugin = require('vue-loader/lib/plugin') const MiniCssExtractPlugin = require('mini-css-extract-plugin') const CssMinimizerPlugin = require('css-minimizer-webpack-plugin') +const ProcessLocalesPlugin = require('./ProcessLocalesPlugin') const { productName } = require('../package.json') @@ -151,6 +152,19 @@ if (isDevMode) { __static: `"${path.join(__dirname, '../static').replace(/\\/g, '\\\\')}"`, }) ) +} else { + const processLocalesPlugin = new ProcessLocalesPlugin({ + compress: true, + inputDir: path.join(__dirname, '../static/locales'), + outputDir: 'static/locales', + }) + + config.plugins.push( + processLocalesPlugin, + new webpack.DefinePlugin({ + 'process.env.LOCALE_NAMES': JSON.stringify(processLocalesPlugin.localeNames) + }) + ) } module.exports = config diff --git a/_scripts/webpack.web.config.js b/_scripts/webpack.web.config.js index bafdf9b75..1aa474a4d 100644 --- a/_scripts/webpack.web.config.js +++ b/_scripts/webpack.web.config.js @@ -6,6 +6,7 @@ const CopyWebpackPlugin = require('copy-webpack-plugin') const MiniCssExtractPlugin = require('mini-css-extract-plugin') const JsonMinimizerPlugin = require('json-minimizer-webpack-plugin') const CssMinimizerPlugin = require('css-minimizer-webpack-plugin') +const ProcessLocalesPlugin = require('./ProcessLocalesPlugin') const { productName } = require('../package.json') @@ -105,7 +106,9 @@ const config = { optimization: { minimizer: [ '...', // extend webpack's list instead of overwriting it - new JsonMinimizerPlugin(), + new JsonMinimizerPlugin({ + exclude: /\/locales\/.*\.json/ + }), new CssMinimizerPlugin() ] }, @@ -160,7 +163,17 @@ if (isDevMode) { }) ) } else { + const processLocalesPlugin = new ProcessLocalesPlugin({ + compress: false, + inputDir: path.join(__dirname, '../static/locales'), + outputDir: 'static/locales', + }) + config.plugins.push( + processLocalesPlugin, + new webpack.DefinePlugin({ + 'process.env.LOCALE_NAMES': JSON.stringify(processLocalesPlugin.localeNames) + }), new CopyWebpackPlugin({ patterns: [ { @@ -172,7 +185,7 @@ if (isDevMode) { to: path.join(__dirname, '../dist/web/static'), globOptions: { dot: true, - ignore: ['**/.*', '**/pwabuilder-sw.js', '**/dashFiles/**', '**/storyboards/**'], + ignore: ['**/.*', '**/locales/**', '**/pwabuilder-sw.js', '**/dashFiles/**', '**/storyboards/**'], }, }, ] diff --git a/package.json b/package.json index 3f8c1e7fc..0ec7bb415 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,6 @@ "http-proxy-agent": "^4.0.1", "https-proxy-agent": "^5.0.0", "jquery": "^3.6.0", - "js-yaml": "^4.1.0", "lodash.debounce": "^4.0.8", "lodash.isequal": "^4.5.0", "marked": "^4.0.17", @@ -105,6 +104,7 @@ "eslint-plugin-standard": "^5.0.0", "eslint-plugin-vue": "^7.17.0", "html-webpack-plugin": "^5.3.2", + "js-yaml": "^4.1.0", "json-minimizer-webpack-plugin": "^4.0.0", "mini-css-extract-plugin": "^2.2.2", "npm-run-all": "^4.1.5", diff --git a/src/renderer/components/ft-video-player/ft-video-player.js b/src/renderer/components/ft-video-player/ft-video-player.js index 3f728af02..68d0f7ecc 100644 --- a/src/renderer/components/ft-video-player/ft-video-player.js +++ b/src/renderer/components/ft-video-player/ft-video-player.js @@ -138,7 +138,7 @@ export default Vue.extend({ }, computed: { currentLocale: function () { - return this.$store.getters.getCurrentLocale + return this.$i18n.locale }, defaultPlayback: function () { diff --git a/src/renderer/components/general-settings/general-settings.js b/src/renderer/components/general-settings/general-settings.js index 1de312df6..8b3e832ee 100644 --- a/src/renderer/components/general-settings/general-settings.js +++ b/src/renderer/components/general-settings/general-settings.js @@ -111,7 +111,7 @@ export default Vue.extend({ }, localeOptions: function () { - return ['system'].concat(Object.keys(this.$i18n.messages)) + return ['system'].concat(this.$i18n.allLocales) }, localeNames: function () { @@ -119,14 +119,18 @@ export default Vue.extend({ this.$t('Settings.General Settings.System Default') ] - Object.entries(this.$i18n.messages).forEach(([locale, localeData]) => { - const localeName = localeData['Locale Name'] - if (typeof localeName !== 'undefined') { - names.push(localeName) - } else { - names.push(locale) - } - }) + if (process.env.NODE_ENV === 'development') { + Object.entries(this.$i18n.messages).forEach(([locale, localeData]) => { + const localeName = localeData['Locale Name'] + if (typeof localeName !== 'undefined') { + names.push(localeName) + } else { + names.push(locale) + } + }) + } else { + names.push(...process.env.LOCALE_NAMES) + } return names }, diff --git a/src/renderer/i18n/index.js b/src/renderer/i18n/index.js index df870e326..e73631035 100644 --- a/src/renderer/i18n/index.js +++ b/src/renderer/i18n/index.js @@ -1,31 +1,88 @@ import Vue from 'vue' import VueI18n from 'vue-i18n' -import yaml from 'js-yaml' import fs from 'fs' +// List of locales approved for use +import activeLocales from '../../../static/locales/activeLocales.json' + const isDev = process.env.NODE_ENV === 'development' -Vue.use(VueI18n) - -// List of locales approved for use -const activeLocales = ['en-US', 'en_GB', 'ar', 'bg', 'ca', 'cs', 'da', 'de-DE', 'el', 'es', 'es_AR', 'es-MX', 'et', 'eu', 'fi', 'fr-FR', 'gl', 'he', 'hu', 'hr', 'id', 'is', 'it', 'ja', 'ko', 'lt', 'nb_NO', 'nl', 'nn', 'pl', 'pt', 'pt-BR', 'pt-PT', 'ro', 'ru', 'sk', 'sl', 'sr', 'sv', 'tr', 'uk', 'vi', 'zh-CN', 'zh-TW'] const messages = {} -/* eslint-disable-next-line */ -const fileLocation = isDev ? 'static/locales/' : `${__dirname}/static/locales/` -// Take active locales and load respective YAML file -activeLocales.forEach((locale) => { - try { - // File location when running in dev - const doc = yaml.load(fs.readFileSync(`${fileLocation}${locale}.yaml`)) - messages[locale] = doc - } catch (e) { - console.error(e) +if (isDev) { + const { load } = require('js-yaml') + // Take active locales and load respective YAML file + activeLocales.forEach((locale) => { + try { + // File location when running in dev + const doc = load(fs.readFileSync(`static/locales/${locale}.yaml`)) + messages[locale] = doc + } catch (e) { + console.error(e) + } + }) +} + +class CustomVueI18n extends VueI18n { + constructor(options) { + super(options) + this.allLocales = activeLocales } -}) -export default new VueI18n({ + async loadLocale(locale) { + // we only lazy load locales for producation builds + if (!isDev) { + // don't need to load it if it's already loaded + if (this.availableLocales.includes(locale)) { + return + } + if (!this.allLocales.includes(locale)) { + console.error(`Unable to load unknown locale: "${locale}"`) + } + + if (process.env.IS_ELECTRON) { + const { brotliDecompressSync } = require('zlib') + // locales are only compressed in our production Electron builds + try { + // decompress brotli compressed json file and then load it + // eslint-disable-next-line node/no-path-concat + const compressed = fs.readFileSync(`${__dirname}/static/locales/${locale}.json.br`) + const data = JSON.parse(brotliDecompressSync(compressed).toString()) + this.setLocaleMessage(locale, data) + } catch (err) { + console.error(err) + } + } else { + const url = new URL(window.location.href) + url.hash = '' + if (url.pathname.endsWith('index.html')) { + url.pathname = url.pathname.replace(/index\.html$/, '') + } + + if (url.pathname) { + url.pathname += `/static/locales/${locale}.json` + } else { + url.pathname = `/static/locales/${locale}.json` + } + + const response = await fetch(url) + const data = await response.json() + this.setLocaleMessage(locale, data) + } + } + } +} + +Vue.use(CustomVueI18n) + +const i18n = new CustomVueI18n({ locale: 'en-US', fallbackLocale: { default: 'en-US' }, - messages: messages + messages }) + +if (!isDev) { + i18n.loadLocale('en-US') +} + +export default i18n diff --git a/src/renderer/store/modules/settings.js b/src/renderer/store/modules/settings.js index e3a8fdd49..20f8c0c17 100644 --- a/src/renderer/store/modules/settings.js +++ b/src/renderer/store/modules/settings.js @@ -281,7 +281,7 @@ const stateWithSideEffects = { if (value === 'system') { const systemLocaleName = (await dispatch('getSystemLocale')).replace('-', '_') // ex: en_US const systemLocaleLang = systemLocaleName.split('_')[0] // ex: en - const targetLocaleOptions = Object.keys(i18n.messages).filter((locale) => { // filter out other languages + const targetLocaleOptions = i18n.allLocales.filter((locale) => { // filter out other languages const localeLang = locale.replace('-', '_').split('_')[0] return localeLang.includes(systemLocaleLang) }).sort((a, b) => { @@ -317,6 +317,10 @@ const stateWithSideEffects = { } } + if (process.env.NODE_ENV !== 'development') { + await i18n.loadLocale(targetLocale) + } + i18n.locale = targetLocale dispatch('getRegionData', { isDev: process.env.NODE_ENV === 'development', diff --git a/src/renderer/views/SubscribedChannels/SubscribedChannels.js b/src/renderer/views/SubscribedChannels/SubscribedChannels.js index b3c3d69b0..fbe49b2f8 100644 --- a/src/renderer/views/SubscribedChannels/SubscribedChannels.js +++ b/src/renderer/views/SubscribedChannels/SubscribedChannels.js @@ -60,7 +60,7 @@ export default Vue.extend({ }, locale: function () { - return this.$store.getters.getCurrentLocale.replace('_', '-') + return this.$i18n.locale.replace('_', '-') }, backendPreference: function () { diff --git a/static/locales/activeLocales.json b/static/locales/activeLocales.json new file mode 100644 index 000000000..d376267a3 --- /dev/null +++ b/static/locales/activeLocales.json @@ -0,0 +1,46 @@ +[ + "en-US", + "en_GB", + "ar", + "bg", + "ca", + "cs", + "da", + "de-DE", + "el", + "es", + "es_AR", + "es-MX", + "et", + "eu", + "fi", + "fr-FR", + "gl", + "he", + "hu", + "hr", + "id", + "is", + "it", + "ja", + "ko", + "lt", + "nb_NO", + "nl", + "nn", + "pl", + "pt", + "pt-BR", + "pt-PT", + "ro", + "ru", + "sk", + "sl", + "sr", + "sv", + "tr", + "uk", + "vi", + "zh-CN", + "zh-TW" +]