From 3e3d0d96b598d2442899169a9f5dcbf6bbb45790 Mon Sep 17 00:00:00 2001 From: mozgzh <97530463+mozgzh@users.noreply.github.com> Date: Tue, 17 May 2022 09:24:36 -0400 Subject: [PATCH] Add language metadata (#3704) * Ensure columns correct type * add locale metadata * Ensure columns correct type * add locale metadata * Update query with metadata * Update language type * Normalize flag name across db backend and frontend * Rename migration to ensure runs in correct order * Simplify query * Refactor language logic * Add translation flag * Update on startup import * Update startup scripts * Remove migration with data * Ensure all langauges in locales are joined on * Remove debug code * Update store to use db * Update store to use db * Update migration * Remove build script for importing languages * Remove build script for importing languages * Ignore already inserted variants * Remove noisey logging * Remove last static files * Remove locales dependancy * Add test to see if build passes --- common/language.ts | 5 +- locales/native-names.json | 144 ------------------ locales/rtl.json | 12 -- locales/test.json | 0 locales/translated.json | 120 --------------- package.json | 5 +- scripts/import-locales.js | 123 --------------- server/package.json | 1 - server/src/lib/api.ts | 8 - server/src/lib/model.ts | 18 +-- server/src/lib/model/db.ts | 19 +-- server/src/lib/model/db/import-locales.ts | 95 ++++++++++-- .../20220222145730-import-variant-data.ts | 27 ++-- ...09124328-add-isContributable-to-locales.ts | 21 +++ web/src/stores/languages.ts | 45 +++--- 15 files changed, 161 insertions(+), 482 deletions(-) delete mode 100644 locales/native-names.json delete mode 100644 locales/rtl.json create mode 100644 locales/test.json delete mode 100644 locales/translated.json delete mode 100644 scripts/import-locales.js create mode 100644 server/src/lib/model/db/migrations/20220509124328-add-isContributable-to-locales.ts diff --git a/common/language.ts b/common/language.ts index e23c7cbf16ed8..c236a4c54a50a 100644 --- a/common/language.ts +++ b/common/language.ts @@ -16,7 +16,10 @@ export type Language = { id: number; name: string; sentenceCount: SentenceCount; - isContributable?: boolean; + is_contributable?: boolean; + is_translated?: boolean; + native_name: string; + text_direction: string; }; // single variant object diff --git a/locales/native-names.json b/locales/native-names.json deleted file mode 100644 index a7f16d3ed2693..0000000000000 --- a/locales/native-names.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "ab": "Аԥсуа", - "ace": "Bahsa Acèh", - "af": "af", - "am": "አማርኛ", - "an": "Aragonés", - "ar": "العربية", - "arn": "Mapuzugun", - "as": "অসমীয়া", - "ast": "Asturianu", - "az": "Azərbaycanca", - "ba": "Башҡорт", - "bas": "Basaa", - "be": "Беларуская", - "bg": "Български", - "bn": "বাংলা", - "bn-BD": "bn-BD", - "br": "Brezhoneg", - "bxr": "Буряад", - "ca": "català", - "cak": "Kaqchikel", - "ckb": "کوردیی ناوەندی", - "cnh": "Laiholh (Hakha)", - "co": "Corsu", - "cs": "Čeština", - "cv": "Чӑвашла", - "cy": "Cymraeg", - "da": "Dansk", - "de": "Deutsch", - "dsb": "Dolnoserbšćina", - "dv": "ދިވެހި", - "el": "Ελληνικά", - "en": "English", - "eo": "Esperanto", - "es": "Español", - "es-AR": "es-AR", - "es-CL": "es-CL", - "et": "eesti", - "eu": "Euskara", - "fa": "فارسی", - "ff": "Pulaar-Fulfulde", - "fi": "suomi", - "fo": "Føroyskt", - "fr": "Français", - "fy-NL": "Frysk", - "ga-IE": "Gaeilge", - "gl": "Galego", - "gn": "Guarani", - "gom": "gom", - "ha": "Hausa", - "he": "עברית", - "hi": "हिंदी", - "hr": "Hrvatski", - "hsb": "Hornjoserbšćina", - "ht": "ht", - "hu": "Magyar", - "hy-AM": "Հայերեն", - "ia": "Interlingua", - "id": "Bahasa Indonesia", - "ie": "ie", - "ig": "Ásụ̀sụ́ Ìgbò", - "is": "Íslenska", - "it": "Italiano", - "ja": "日本語", - "ka": "ქართული", - "kab": "Taqbaylit", - "kbd": "Адыгэбзэ (Къэбэрдэй)", - "kk": "Қазақ тілі", - "km": "km", - "kmr": "Kurdî (Kurmancî)", - "ko": "한국어", - "kpv": "Коми кыв", - "kw": "Kernowek", - "ky": "Кыргызча", - "lg": "Luganda", - "lij": "Lìgure", - "lt": "Lietuvių", - "lv": "Latviešu", - "mdf": "Мокшень кяль", - "mhr": "Марий", - "mk": "Македонски", - "ml": "മലയാളം", - "mn": "Монгол хэл", - "mni": "ꯃꯤꯇꯩ ꯂꯣꯟ", - "mr": "मराठी", - "ms": "Bahasa Melayu", - "mt": "Malti", - "my": "ဗမာ", - "myv": "Эрзянь кель", - "nan-tw": "臺語", - "nb-NO": "Norsk (bokmål)", - "ne-NP": "नेपाली", - "nia": "nia", - "nl": "Nederlands", - "nn-NO": "Norsk (nynorsk)", - "nyn": "nyn", - "oc": "Occitan", - "or": "ଓଡ଼ିଆ", - "pa-IN": "ਪੰਜਾਬੀ", - "pap-AW": "pap-AW", - "pl": "polski", - "ps": "ps", - "pt": "Português", - "rm-sursilv": "romontsch sursilvan", - "rm-vallader": "Rumantsch vallader", - "ro": "Română", - "ru": "Русский", - "rw": "Ikinyarwanda", - "sah": "Саха тыла", - "sat": "ᱥᱟᱱᱛᱟᱲᱤ (ᱚᱞ ᱪᱤᱠᱤ)", - "sc": "Sardu", - "scn": "Sicilianu", - "si": "සිංහල", - "sk": "slovenčina", - "skr": "سرائیکی", - "sl": "slovenščina", - "sq": "Shqip", - "sr": "Српски", - "sv-SE": "Svenska", - "sw": "Kiswahili", - "syr": "ܣܘܼܪܝܝܐ", - "ta": "தமிழ்", - "te": "తెలుగు", - "tg": "Тоҷикӣ", - "th": "ไทย", - "ti": "ትግርኛ", - "tig": "ትግረይት", - "tk": "Türkmençe", - "tl": "Tagalog", - "tok": "toki pona", - "tr": "Türkçe", - "tt": "Татар", - "ug": "ئۇيغۇرچە", - "uk": "Українська", - "ur": "اردو", - "uz": "O‘zbek", - "vec": "Vèneto", - "vi": "Việt", - "vot": "vad̕d̕a", - "yue": "粵語", - "zh-CN": "汉语（中国大陆）", - "zh-HK": "中文（香港）", - "zh-TW": "華語（台灣）" -} diff --git a/locales/rtl.json b/locales/rtl.json deleted file mode 100644 index 574c92bd37816..0000000000000 --- a/locales/rtl.json +++ /dev/null @@ -1,12 +0,0 @@ -[ - "ar", - "ckb", - "dv", - "fa", - "he", - "ps", - "skr", - "syr", - "ug", - "ur" -] diff --git a/locales/test.json b/locales/test.json new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/locales/translated.json b/locales/translated.json deleted file mode 100644 index 02077d53ccd46..0000000000000 --- a/locales/translated.json +++ /dev/null @@ -1,120 +0,0 @@ -[ - "ab", - "an", - "ar", - "as", - "ast", - "az", - "ba", - "bas", - "be", - "bg", - "bn", - "br", - "ca", - "cak", - "ckb", - "cnh", - "co", - "cs", - "cv", - "cy", - "da", - "de", - "dsb", - "dv", - "el", - "en", - "eo", - "es", - "et", - "eu", - "fa", - "ff", - "fi", - "fr", - "fy-NL", - "ga-IE", - "gl", - "gn", - "ha", - "he", - "hi", - "hi-IN", - "hr", - "hsb", - "hu", - "hy-AM", - "ia", - "id", - "ig", - "is", - "it", - "ja", - "ka", - "kab", - "kbd", - "kk", - "kmr", - "ko", - "kw", - "ky", - "lg", - "lij", - "lt", - "lv", - "mdf", - "mhr", - "mk", - "ml", - "mn", - "mr", - "mt", - "myv", - "nan-tw", - "nb-NO", - "ne-NP", - "nl", - "nn-NO", - "oc", - "or", - "pa-IN", - "pl", - "ps", - "pt", - "rm-sursilv", - "rm-vallader", - "ro", - "ru", - "rw", - "sah", - "sat", - "sc", - "sk", - "skr", - "sl", - "sq", - "sr", - "sv-SE", - "sw", - "ta", - "te", - "th", - "ti", - "tig", - "tk", - "tok", - "tr", - "tt", - "ug", - "uk", - "ur", - "uz", - "vec", - "vi", - "vot", - "yue", - "zh-CN", - "zh-HK", - "zh-TW" -] diff --git a/package.json b/package.json index a39e7606d363e..3baa4ff27488e 100644 --- a/package.json +++ b/package.json @@ -44,14 +44,13 @@ "build:server": "cd server && yarn build && cd ..", "build:common": "cd common && yarn build && cd ..", "build:maint": "cd maintenance && yarn build && cd ..", - "import-locales": "node scripts/import-locales", "lint": "eslint .", "lint:common": "eslint ./common", "lint:server": "eslint ./server", "lint:web": "cd web && yarn lint", "prettier": "prettier \"**/*.{ts,tsx}\" --write", - "start": "yarn build:common && concurrently -p \"[{name}]\" -n \"CO,BE,FE\" -c \"bgYellow.bold,bgBlue.bold,bgMagenta.bold,bgCyan.bold\" \"cd common && yarn dev\" \"cd server && yarn start\" \"cd web && yarn dev\" \"yarn import-locales\"", - "start:prod": "yarn build:common && yarn import-locales && node server/js/main.js", + "start": "yarn build:common && concurrently -p \"[{name}]\" -n \"CO,BE,FE\" -c \"bgYellow.bold,bgBlue.bold,bgMagenta.bold,bgCyan.bold\" \"cd common && yarn dev\" \"cd server && yarn start\" \"cd web && yarn dev\"", + "start:prod": "yarn build:common && node server/js/main.js", "start:maint": "cd maintenance && yarn dev", "test": "yarn build:common && concurrently --kill-others-on-fail \"cd web && yarn test\" \"cd server && SERVER_CONFIG_PATH='../config.json' yarn test\"" }, diff --git a/scripts/import-locales.js b/scripts/import-locales.js deleted file mode 100644 index 7612a0491d821..0000000000000 --- a/scripts/import-locales.js +++ /dev/null @@ -1,123 +0,0 @@ -const fs = require('fs'); -const path = require('path'); -const mysql = require('mysql2'); -const { parse } = require('@fluent/syntax'); -const fetch = require('node-fetch'); -const { promisify } = require('util'); -const { getConfig } = require('../server/js/config-helper'); -const TRANSLATED_MIN_PROGRESS = 0.75; -const CONTRIBUTABLE_MIN_SENTENCES = 5000; - -const dataPath = path.join(__dirname, '..', 'locales'); -const localeMessagesPath = path.join(__dirname, '..', 'web', 'locales'); - -const { MYSQLHOST, MYSQLUSER, MYSQLPASS, MYSQLDBNAME } = getConfig(); - -const dbConfig = { - host: MYSQLHOST, - user: MYSQLUSER, - password: MYSQLPASS, - database: MYSQLDBNAME, -}; - -function saveDataJSON(name, data) { - fs.writeFileSync( - path.join(dataPath, name + '.json'), - JSON.stringify(data, null, 2) + '\n' - ); -} - -async function fetchPontoonLanguages() { - const url = - 'https://pontoon.mozilla.org/graphql?query={project(slug:%22common-voice%22){localizations{totalStrings,approvedStrings,locale{code,name,direction}}}}'; - const response = await fetch(url); - const { data } = await response.json(); - - return data.project.localizations - .map(({ totalStrings, approvedStrings, locale }) => ({ - code: locale.code, - name: locale.name, - direction: locale.direction, - translated: approvedStrings / totalStrings, - })) - .concat({ code: 'en', name: 'English', translated: 1, direction: 'LTR' }) - .sort((l1, l2) => l1.code.localeCompare(l2.code)); -} - -async function saveToMessages(languages) { - const messagesPath = path.join(localeMessagesPath, 'en', 'messages.ftl'); - const messages = fs.readFileSync(messagesPath, 'utf-8'); - const newMessages = messages.replace( - /#\s\[Languages]([\s\S]*?)#\s\[\/]/gm, - [ - '# [Languages]', - '## Languages', - languages.map(({ code, name }) => `${code} = ${name}`).join('\n'), - '# [/]', - ].join('\n') - ); - fs.writeFileSync(messagesPath, newMessages); -} - -/** - * Saves completed locales (always merges with previously added locales) - */ -async function saveCompletedLocalesJSON(languages) { - const fileName = 'translated'; - const completedLocalesPath = path.join(dataPath, fileName + '.json'); - const existingLocales = JSON.parse( - fs.readFileSync(completedLocalesPath, 'utf-8') - ); - saveDataJSON( - fileName, - [ - ...new Set([ - ...existingLocales, - ...languages - .filter(l => l.translated >= TRANSLATED_MIN_PROGRESS) - .map(l => l.code), - ]), - ].sort() - ); -} - -async function importPontoonLocales() { - const languages = await fetchPontoonLanguages(); - await Promise.all([ - saveToMessages(languages), - saveDataJSON( - 'rtl', - languages - .filter(l => l.direction === 'RTL') - .map(l => l.code) - .sort() - ), - saveCompletedLocalesJSON(languages), - ]); -} - -async function buildLocaleNativeNameMapping() { - const locales = fs.readdirSync(localeMessagesPath); - const nativeNames = {}; - for (const locale of locales) { - const messagesPath = path.join(localeMessagesPath, locale, 'messages.ftl'); - - if (!fs.existsSync(messagesPath)) { - continue; - } - - const messages = parse(fs.readFileSync(messagesPath, 'utf-8')); - const message = messages.body.find( - message => message.id && message.id.name === locale - ); - - nativeNames[locale] = message ? message.value.elements[0].value : locale; - } - saveDataJSON('native-names', nativeNames); -} - -async function importLocales() { - await Promise.all([importPontoonLocales(), buildLocaleNativeNameMapping()]); -} - -importLocales().catch(e => console.error(e)); diff --git a/server/package.json b/server/package.json index e9bae870febf2..2683fffeb0477 100644 --- a/server/package.json +++ b/server/package.json @@ -27,7 +27,6 @@ "http-status-codes": "2.2.0", "ioredis": "4.28.5", "js-md5": "0.7.3", - "locales": "file:../locales", "lodash.omit": "4.5.0", "lodash.pick": "4.4.0", "mysql2": "1.7.0", diff --git a/server/src/lib/api.ts b/server/src/lib/api.ts index 6ccd8149d2b9d..a32a60e1fc899 100644 --- a/server/src/lib/api.ts +++ b/server/src/lib/api.ts @@ -57,18 +57,10 @@ export default class API { router.use(authMiddleware); router.get('/metrics', (request: Request, response: Response) => { - console.log('Received a metrics request', { - referer: request.header('Referer'), - query: request.query, - }); response.redirect('/'); }); router.get('/golem', (request: Request, response: Response) => { - console.log('Received a Golem request', { - referer: request.header('Referer'), - query: request.query, - }); response.redirect('/'); }); diff --git a/server/src/lib/model.ts b/server/src/lib/model.ts index 18ed03d2f7619..37a96ca76e68e 100644 --- a/server/src/lib/model.ts +++ b/server/src/lib/model.ts @@ -193,27 +193,21 @@ export default class Model { async (): Promise => { const languages = await this.db.getLanguages(); return languages.map(language => { - const isContributable = + const is_contributable = language.sentenceCount.currentCount >= language.sentenceCount.targetSentenceCount; - return { ...language, isContributable }; + return { ...language, is_contributable }; }); }, DAY ); getAllLanguages = lazyCache( - 'get-all-languages-isContributable', + 'get-all-languages', async (): Promise => { const languages = await this.db.getAllLanguages(); - return languages.map(language => { - return { - id: language.id, - name: language.name, - isContributable: language.isContributable, - }; - }); + return languages; }, DAY ); @@ -235,11 +229,11 @@ export default class Model { const allLanguages = await this.getLanguages(); const contributableLocales = allLanguages - .filter(language => language.isContributable) + .filter(language => language.is_contributable) .map(language => language.name); const inProgressLocales = allLanguages - .filter(language => !language.isContributable) + .filter(language => !language.is_contributable) .map(language => language.name); function indexCountByLanguage( diff --git a/server/src/lib/model/db.ts b/server/src/lib/model/db.ts index c3e7912fdac8f..5481a68ecebad 100644 --- a/server/src/lib/model/db.ts +++ b/server/src/lib/model/db.ts @@ -931,24 +931,11 @@ export default class DB { async getAllLanguages(): Promise { const [rows] = await this.mysql.query( - `SELECT l.id, l.name, l.target_sentence_count as target_sentence_count, count(1) as total_sentence_count + `SELECT * FROM locales l - JOIN sentences s ON s.locale_id = l.id - GROUP BY l.id` - ); - return rows.map( - (row: { - id: number; - name: string; - target_sentence_count: number; - total_sentence_count: number; - isContributable: boolean; - }) => ({ - id: row.id, - name: row.name, - isContributable: row.total_sentence_count >= row.target_sentence_count, - }) + ` ); + return rows; } async getRequestedLanguages(): Promise { diff --git a/server/src/lib/model/db/import-locales.ts b/server/src/lib/model/db/import-locales.ts index 7b40eb8328081..49fb19d43c9fd 100644 --- a/server/src/lib/model/db/import-locales.ts +++ b/server/src/lib/model/db/import-locales.ts @@ -1,40 +1,117 @@ +import { Language } from 'common'; import fetch from 'node-fetch'; import { getMySQLInstance } from './mysql'; - +const TRANSLATED_MIN_PROGRESS = 0.75; +const DEFAULT_TARGET_SENTENCE_COUNT = 5000; type Locale = { code: string; + direction: string; + name: string; + translated: boolean; }; interface PontoonData { locale: Locale; + totalStrings: number; + approvedStrings: number; } const db = getMySQLInstance(); -const fetchPontoonLanguages = async (): Promise => { +const fetchPontoonLanguages = async (): Promise => { const url = 'https://pontoon.mozilla.org/graphql?query={project(slug:%22common-voice%22){localizations{totalStrings,approvedStrings,locale{code,name,direction}}}}'; const response = await fetch(url); const { data } = await response.json(); - const localizations = data.project.localizations; + const localizations: PontoonData[] = data.project.localizations; return localizations - .map(({ locale }: PontoonData) => ({ + .map(({ totalStrings, approvedStrings, locale }) => ({ code: locale.code, + name: locale.name, + translated: approvedStrings / totalStrings, + direction: locale.direction, })) .concat({ code: 'en', name: 'English', translated: 1, direction: 'LTR' }) - .sort((l1: Locale, l2: Locale) => l1.code.localeCompare(l2.code)); + .sort((l1, l2) => l1.code.localeCompare(l2.code)); }; export async function importLocales() { console.log('Importing languages'); const locales = await fetchPontoonLanguages(); + if (locales) { - console.log('Inserting languages into database'); + const [existingLangauges] = await db.query(` + SELECT l.id, l.name, l.target_sentence_count as target_sentence_count, count(1) as total_sentence_count + FROM locales l + LEFT JOIN sentences s ON s.locale_id = l.id + GROUP BY l.id + `); + console.log(`${existingLangauges.length} Existing Languages`); - await db.query('INSERT IGNORE INTO locales (name) VALUES ?', [ - locales.map(l => [l.code]), - ]); + const allLanguages = existingLangauges.reduce((obj: any, language: any) => { + obj[language.name] = { + ...language, + hasEnoughSentences: + language.total_sentence_count >= language.target_sentence_count, + }; + return obj; + }, {}); + const newLanguageData = locales.reduce((obj, language) => { + const isTranslated = language.translated >= TRANSLATED_MIN_PROGRESS; + const hasEnoughSentences = + allLanguages[language.code]?.hasEnoughSentences || false; + + obj[language.code] = { + ...language, + target_sentence_count: + allLanguages[language.code]?.target_sentence_count || + DEFAULT_TARGET_SENTENCE_COUNT, + is_translated: isTranslated ? 1 : 0, + is_contributable: isTranslated && hasEnoughSentences ? 1 : 0, + }; + return obj; + }, {}); + await Promise.all([ + locales.map(lang => { + if (allLanguages[lang.code]) { + // this language exists in db, just update + + return db.query( + ` + UPDATE locales + SET native_name = ?, + is_contributable = ?, + is_translated = ?, + text_direction = ? + WHERE id = ? + `, + [ + lang.name, + newLanguageData[lang.code].is_contributable, + newLanguageData[lang.code].is_translated, + lang.direction, + allLanguages[lang.code].id, + ] + ); + } else { + // this is a new language, insert + return db.query( + `INSERT IGNORE INTO locales(name, target_sentence_count, native_name, is_contributable, is_translated, text_direction) VALUES (?)`, + [ + [ + lang.code, + newLanguageData[lang.code].target_sentence_count, + lang.name, + newLanguageData[lang.code].is_contributable, + newLanguageData[lang.code].is_translated, + lang.direction, + ], + ] + ); + } + }), + ]); // Make sure each language has at minimum an "unspecified" accent await db.query(` INSERT IGNORE INTO accents (locale_id, accent_name, accent_token, user_submitted) diff --git a/server/src/lib/model/db/migrations/20220222145730-import-variant-data.ts b/server/src/lib/model/db/migrations/20220222145730-import-variant-data.ts index 0aa43b93a6fef..fc9185d3719b5 100644 --- a/server/src/lib/model/db/migrations/20220222145730-import-variant-data.ts +++ b/server/src/lib/model/db/migrations/20220222145730-import-variant-data.ts @@ -109,18 +109,21 @@ export const up = async function (db: any): Promise { obj[current.name] = current.id; return obj; }, {}); - for (const row of VARIANTS) { - await db.runSql( - `INSERT INTO variants (locale_id, variant_token, variant_name) VALUES ( ${ - mappedLanguages[row['locale_name']] + - ',"' + - row['variant_token'] + - '","' + - row['variant_name'] + - '"' - })` - ); - } + + await Promise.all( + VARIANTS.map(row => { + db.runSql( + `INSERT IGNORE INTO variants (locale_id, variant_token, variant_name) VALUES ( ${ + mappedLanguages[row['locale_name']] + + ',"' + + row['variant_token'] + + '","' + + row['variant_name'] + + '"' + })` + ); + }) + ); }; export const down = async function (db: any): Promise { diff --git a/server/src/lib/model/db/migrations/20220509124328-add-isContributable-to-locales.ts b/server/src/lib/model/db/migrations/20220509124328-add-isContributable-to-locales.ts new file mode 100644 index 0000000000000..db23e8762ed7d --- /dev/null +++ b/server/src/lib/model/db/migrations/20220509124328-add-isContributable-to-locales.ts @@ -0,0 +1,21 @@ +export const up = async function (db: any): Promise { + await db.runSql( + ` + ALTER TABLE locales + ADD COLUMN native_name varchar(255) CHARACTER SET utf8mb4, + ADD COLUMN is_contributable BOOLEAN DEFAULT 0 NOT NULL, + ADD COLUMN is_translated BOOLEAN DEFAULT 0 NOT NULL, + ADD COLUMN text_direction ENUM('LTR', 'RTL', 'TTB', 'BTT') DEFAULT 'LTR' NOT NULL + ` + ); +}; + +export const down = async function (db: any): Promise { + return await db.runSql(` + ALTER TABLE locales + DROP COLUMN native_name, + DROP COLUMN is_contributable, + DROP COLUMN is_translated, + DROP COLUMN text_direction +`); +}; diff --git a/web/src/stores/languages.ts b/web/src/stores/languages.ts index c6ad7e99e939b..b3cdf500a2b23 100644 --- a/web/src/stores/languages.ts +++ b/web/src/stores/languages.ts @@ -31,16 +31,6 @@ interface LoadedAction { export type Action = LoadedAction; -async function getLocaleData(fileName: string): Promise { - const response = await fetch(`/dist/languages/${fileName}.json`); - return response.json(); -} - -async function getNativeNamesData(): Promise { - const response = await fetch(`/dist/languages/native-names.json`); - return response.json(); -} - export const actions = { loadLocalesData: () => { return async ( @@ -48,20 +38,33 @@ export const actions = { getState: () => StateTree ) => { const { api } = getState(); - - const response = await Promise.all([ - api.fetchAllLanguages(), - getNativeNamesData(), - getLocaleData('rtl'), - getLocaleData('translated'), - ]); - - const [allLanguages, nativeNames, rtlLocales, translatedLocales] = - response; + const allLanguages = await api.fetchAllLanguages(); + + //get obj of native names, default to language code + const nativeNames = allLanguages.reduce((names: any, language) => { + names[language.name] = language.native_name ?? language.name; + return names; + }, {}); + + //get array of rtl languages + const rtlLocales = allLanguages.reduce((names: any, language) => { + if (language.text_direction === 'RTL') { + names.push(language.name); + } + return names; + }, []); + + const translatedLocales = allLanguages.reduce((names: any, language) => { + if (language.is_translated) { + names.push(language.name); + } + return names; + }, []); + console.log('translatedLocales', translatedLocales); const allLocales = allLanguages.map(language => language.name); const contributableLocales = allLanguages - .filter(language => language.isContributable) + .filter(language => language.is_contributable) .map(language => language.name); dispatch({