diff --git a/documentation/frontend/reference/i18n.md b/documentation/frontend/reference/i18n.md index 1fd695c8270..36293cceb0d 100644 --- a/documentation/frontend/reference/i18n.md +++ b/documentation/frontend/reference/i18n.md @@ -51,8 +51,9 @@ processed and loaded into Nuxt via the Nuxt i18n module. JED 1.x (derived from the flattened POT files) files are converted back into the nested JSON as expected by Nuxt i18n. - This script uses the `wp-locales.json` file for the list of locales, if - downloading each locale separately and in parallel. + This script downloads all available translations in bulk as a ZIP file and + then extracts JSON files from the ZIP file. This prevents excessive calls to + GlotPress, which can be throttled and cause some locales to be missed. **Script:** `i18n:get-translations` diff --git a/frontend/package.json b/frontend/package.json index 0249d2300b1..07f7836d783 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -118,7 +118,6 @@ "@vue/test-utils": "^1.1.3", "adm-zip": "^0.5.10", "autoprefixer": "^10.4.16", - "axios-rate-limit": "^1.3.0", "babel-core": "^7.0.0-bridge.0", "babel-jest": "^26.6.3", "babel-loader": "8.2.5", diff --git a/frontend/src/locales/scripts/axios.js b/frontend/src/locales/scripts/axios.js index 0aceed7a858..f92100fca69 100644 --- a/frontend/src/locales/scripts/axios.js +++ b/frontend/src/locales/scripts/axios.js @@ -1,13 +1,7 @@ const axios = require("axios") -const rateLimit = require("axios-rate-limit") const { userAgent } = require("../../constants/user-agent") -module.exports = rateLimit( - axios.create({ - headers: { "User-Agent": userAgent }, - }), - { - maxRPS: 50, // limit GlotPress calls to 50 requests per second - } -) +module.exports = module.exports = axios.create({ + headers: { "User-Agent": userAgent }, +}) diff --git a/frontend/src/locales/scripts/get-translations.js b/frontend/src/locales/scripts/get-translations.js index 67c64fd95f1..b1faf01d377 100644 --- a/frontend/src/locales/scripts/get-translations.js +++ b/frontend/src/locales/scripts/get-translations.js @@ -11,7 +11,6 @@ const chokidar = require("chokidar") const { parseJson } = require("./read-i18n") const bulkDownload = require("./bulk-download") -const separateDownload = require("./separate-download") /** * Write `en.json` from `en.json5`. @@ -37,16 +36,11 @@ if (process.argv.includes("--watch")) { } if (!process.argv.includes("--en-only")) { - bulkDownload() - .catch((err) => { - console.error(err) - return separateDownload() - }) - .catch((err) => { - console.error(err) - console.error(":'-( Downloading translations failed.") - if (process.argv.includes("--require-complete")) { - process.exitCode = 1 - } - }) + bulkDownload().catch((err) => { + console.error(err) + console.error(":'-( Downloading translations failed.") + if (process.argv.includes("--require-complete")) { + process.exitCode = 1 + } + }) } diff --git a/frontend/src/locales/scripts/separate-download.js b/frontend/src/locales/scripts/separate-download.js deleted file mode 100644 index a4ca219afae..00000000000 --- a/frontend/src/locales/scripts/separate-download.js +++ /dev/null @@ -1,57 +0,0 @@ -const axios = require("./axios") -const { writeLocaleFile } = require("./utils") -const jed1xJsonToJson = require("./jed1x-json-to-json") - -const DOWNLOAD_BASE_URL = - "https://translate.wordpress.org/projects/meta/openverse" - -const getTranslationUrl = (locale) => - `${DOWNLOAD_BASE_URL}/${locale}/default/export-translations/` - -const fetchJed1x = async (locale) => { - try { - const res = await axios.get(getTranslationUrl(locale), { - params: { format: "jed1x" }, - }) - return res.data - } catch (err) { - return err.response.status - } -} - -const isEmpty = (obj) => Object.values(obj).every((x) => x === null) - -const fetchJed1xAll = async (locales) => { - const results = await Promise.allSettled(locales.map(fetchJed1x)) - - let succeeded = {} - let failed = {} - results.forEach(({ status, value }, index) => { - if (status === "fulfilled" && !isEmpty(value)) { - succeeded[locales[index]] = jed1xJsonToJson(value) - } else { - failed[locales[index]] = value - } - }) - await Promise.all( - Object.entries(succeeded).map((args) => writeLocaleFile(...args)) - ) - - return [Object.keys(succeeded).length, Object.keys(failed).length] -} - -const separateDownload = async () => { - console.log("Performing parallel download.") - - const localeJson = require("./wp-locales.json") - const locales = Object.values(localeJson).map((item) => item.slug) - const [succeeded, failed] = await fetchJed1xAll(locales) - - console.log(`Successfully downloaded ${succeeded} translations.`) - if (failed) { - console.log(`Failed to download ${failed} translations.`) - throw new Error("Parallel download partially failed") - } -} - -module.exports = separateDownload diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 95ad0a88bd0..0aa60bca7ac 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -76,7 +76,6 @@ importers: autoprefixer: ^10.4.16 axios: ^1.0.0 axios-mock-adapter: ^1.20.0 - axios-rate-limit: ^1.3.0 babel-core: ^7.0.0-bridge.0 babel-jest: ^26.6.3 babel-loader: 8.2.5 @@ -186,7 +185,6 @@ importers: '@vue/test-utils': 1.1.3_5bwbnhtkovckcydjgad3t2muke adm-zip: 0.5.10 autoprefixer: 10.4.16_postcss@8.4.31 - axios-rate-limit: 1.3.0_axios@1.6.1 babel-core: 7.0.0-bridge.0_@babel+core@7.22.5 babel-jest: 26.6.3_@babel+core@7.22.5 babel-loader: 8.2.5_rf5mwho5nu3s3spznxs3423x5y @@ -9131,14 +9129,6 @@ packages: is-buffer: 2.0.5 dev: false - /axios-rate-limit/1.3.0_axios@1.6.1: - resolution: {integrity: sha512-cKR5wTbU/CeeyF1xVl5hl6FlYsmzDVqxlN4rGtfO5x7J83UxKDckudsW0yW21/ZJRcO0Qrfm3fUFbhEbWTLayw==} - peerDependencies: - axios: '*' - dependencies: - axios: 1.6.1 - dev: true - /axios/1.6.1: resolution: {integrity: sha512-vfBmhDpKafglh0EldBEbVuoe7DyAavGSLWhuSm5ZSEKQnHhBf0xAAwybbNH1IkrJNGnS/VG4I5yxig1pCEXE4g==} dependencies: @@ -9147,6 +9137,7 @@ packages: proxy-from-env: 1.1.0 transitivePeerDependencies: - debug + dev: false /babel-code-frame/6.26.0: resolution: {integrity: sha512-XqYMR2dfdGMW+hd0IUZ2PwK+fGeFkOxZJ0wY+JaQAHzt1Zx8LcvpiZD2NiGkEG8qx0CfkAOr5xt76d1e8vG90g==} @@ -13014,6 +13005,7 @@ packages: peerDependenciesMeta: debug: optional: true + dev: false /for-each/0.3.3: resolution: {integrity: sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==} @@ -13100,6 +13092,7 @@ packages: asynckit: 0.4.0 combined-stream: 1.0.8 mime-types: 2.1.35 + dev: false /forwarded/0.2.0: resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} @@ -18957,6 +18950,7 @@ packages: /proxy-from-env/1.1.0: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + dev: false /prr/1.0.1: resolution: {integrity: sha512-yPw4Sng1gWghHQWj0B3ZggWUm4qVbPwPFcRG8KyxiU7J2OHFSoEHKS+EZ3fv5l1t9CyCiop6l/ZYeWbrgoQejw==}