diff --git a/ui/packages/i18n-tool/i18n.config.js b/ui/packages/i18n-tool/i18n.config.js new file mode 100644 index 00000000..eec03161 --- /dev/null +++ b/ui/packages/i18n-tool/i18n.config.js @@ -0,0 +1,36 @@ +module.exports = { + // entry for scan start + entry: ['./src'], + // files or subdirectories to be excluded from processing + exclude: [ + 'node_modules/**', + 'scripts/**', + 'dist/**', + 'vite.config.ts', + 'tailwind.config.js', + 'postcss.config.js', + '**/*.d.ts', + 'utils/i18n.tsx', + ], + // path to the original locales directory + localesDir: "./locales", + // original lang + originLang: "zh-CN", + // target lang, can a list, currently we support zh-CN、en-US + targetLangs: ["en-US"], + // [i18n keygen] + prefixKey: '', + keygenAlgorithm: 'md5', + showOriginKey: true, + // [i18n import config] + i18nImport: "import i18nInstance from '@/utils/i18n';", + i18nObject: 'i18nInstance', + i18nMethod: 't', + // [i18n translate provider] + translate: { + type: '', + appid: '', + model: '', + key: '', + } +}; diff --git a/ui/packages/i18n-tool/package.json b/ui/packages/i18n-tool/package.json new file mode 100644 index 00000000..deba4508 --- /dev/null +++ b/ui/packages/i18n-tool/package.json @@ -0,0 +1,23 @@ +{ + "name": "@karmada/i18n-tool", + "version": "1.0.0", + "bin": "src/index.js", + "dependencies": { + "@babel/generator": "^7.25.6", + "@babel/parser": "^7.25.6", + "@babel/traverse": "^7.25.6", + "@babel/types": "^7.25.6", + "@karmada/translators": "workspace:*", + "chalk": "^4.1.2", + "commander": "^12.1.0", + "debug": "^4.3.5", + "glob": "^11.0.0", + "prettier": "^3.3.2" + }, + "scripts": { + "test": "vitest" + }, + "author": "", + "license": "ISC", + "description": "" +} diff --git a/ui/packages/i18n-tool/src/index.js b/ui/packages/i18n-tool/src/index.js new file mode 100644 index 00000000..0c25ee12 --- /dev/null +++ b/ui/packages/i18n-tool/src/index.js @@ -0,0 +1,47 @@ +#!/usr/bin/env node + +const {Command} = require('commander'); +const chalk = require('chalk'); +const pkgInfo = require('../package.json'); + +const scan = require('./scan') +const init = require('./init') + + + +// Create command-line tool +const program = new Command(); +program + .name(pkgInfo.name) + .description(pkgInfo.description) + .version(pkgInfo.version, '-v, --version') + .option('-d, --directory ', 'specify folder path') + .option('-f, --file ', 'specify a single file path') + // If not specified, will look for the config in the same directory as i18n-tool + .option( + '-c, --config ', + 'specify the configuration file path, default is ./i18n.config.cjs', + './i18n.config.cjs' + ) + +program + .command('scan') + .alias('s') + .description('scan target language text from code') + .action(async () => { + const options = program.opts(); + await scan(options); + }); + +program + .command('init') + .description('init i18n.config.js for current project') + .action(async () => { + const options = program.opts(); + await init(options); + }); + +program.parseAsync(process.argv).catch((error) => { + console.error(chalk.red(error.stack)); + process.exit(1); +}); diff --git a/ui/packages/i18n-tool/src/init.js b/ui/packages/i18n-tool/src/init.js new file mode 100644 index 00000000..b1bd9e44 --- /dev/null +++ b/ui/packages/i18n-tool/src/init.js @@ -0,0 +1,16 @@ +const fs = require('node:fs'); +const path = require('node:path'); +const {getDebug} = require("./utils"); +const debug = getDebug('init'); + +async function init(cmdOptions) { + debug('execute scan method', cmdOptions); + const srcDir = path.dirname( __filename) + const i18nTemplateFilePath = path.join(srcDir, "../", "i18n.config.js") + + debug('i18nTemplateFilePath %s', i18nTemplateFilePath); + const targetI18nConfigFilePath = path.join(process.cwd(), "i18n.config.cjs") + fs.copyFileSync(i18nTemplateFilePath, targetI18nConfigFilePath) +} + +module.exports = init diff --git a/ui/packages/i18n-tool/src/options.js b/ui/packages/i18n-tool/src/options.js new file mode 100644 index 00000000..862620e6 --- /dev/null +++ b/ui/packages/i18n-tool/src/options.js @@ -0,0 +1,69 @@ +const path = require('node:path'); +const fs = require('node:fs'); +const chalk = require('chalk'); + +const defaultOptions = { + entry: ['src'], + exclude: [], + + localesDir: "src/locales", + originLang: "zh-CN", + targetLangs: ["en-US"], + + prefixKey: '', + showOriginKey: true, + keygenAlgorithm: 'md5', +}; + +/** + * return default options for i18n-tools + * @returns + */ +function getDefaultOptions() { + return defaultOptions +} + + +/** + * Try to parse i18n config from the specific file path + * @param configPath + */ +async function parseI18nConfig(configPath) { + const _configPath = generatePath(configPath) + if (!fs.existsSync(_configPath)) { + console.log( + chalk.red(`I18n config file ${configPath} not exist, please check!`), + ); + process.exit(1); + } + + const configOptions = require(_configPath) + configOptions.entry = (configOptions.entry || []).map((entryItem) => + generatePath(entryItem), + ); + return { + ...defaultOptions, + ...configOptions, + }; +} + +/** + * Generate absolute path for the input path. + * If the input path is relative path, it will expand as absolute path. + * If the input path is absolute path, it will return directly. + * @param p + * @returns absolute path of input + */ +function generatePath(p) { + const cwd = process.cwd(); + if (path.isAbsolute(p)) { + return p; + } + return path.join(cwd, p); +} + +module.exports = { + getDefaultOptions, + parseI18nConfig, + generatePath, +} diff --git a/ui/packages/i18n-tool/src/scan/ast.js b/ui/packages/i18n-tool/src/scan/ast.js new file mode 100644 index 00000000..d69d6eac --- /dev/null +++ b/ui/packages/i18n-tool/src/scan/ast.js @@ -0,0 +1,130 @@ +const babelParser = require('@babel/parser'); +const babelTraverse = require('@babel/traverse').default; +const babelGenerator = require('@babel/generator').default; +const t = require("@babel/types"); +const crypto = require('node:crypto'); + +// Method to generate unique keys using the MD5 hash algorithm +const generateKey = (text) => crypto.createHash('md5').update(text).digest('hex'); + +// Process the AST tree, detect Chinese characters, and generate i18n key-value pairs +const processAST = (tsxCode, debugMode) => { + const log = (...args) => { + if (debugMode) { + console.log(...args); + } + }; + + const ast = babelParser.parse(tsxCode, { + sourceType: 'module', + plugins: ['jsx', 'typescript'], + }); + + let CNPath = []; // Store paths of Chinese text + let i18nMap = {}; // Store key-value pairs for i18n + let i18nImported = false; // Flag to check if i18nInstance is imported + + // Traverse the AST and find the target nodes + babelTraverse(ast, { + ImportDeclaration(path) { + const sourceValue = path.node.source.value; + if (sourceValue === '@/utils/i18n') { + const specifier = path.node.specifiers.find(spec => spec.local.name === 'i18nInstance'); + if (specifier) { + i18nImported = true; + } + } + }, + StringLiteral(path) { + const value = path.node.value; + if (/[\u4e00-\u9fa5]/.test(value)) { // Check if the string contains Chinese characters + const key = generateKey(value); + if (!i18nMap[key]) { + i18nMap[key] = value; + CNPath.push({ path, key, value, type: 'StringLiteral' }); + log(`Detected Chinese text: ${value}`); + } + } + }, + TemplateLiteral(path) { + path.node.quasis.forEach((quasi) => { + const value = quasi.value.raw; + if (/[\u4e00-\u9fa5]/.test(value)) { // Check if the template contains Chinese characters + const key = generateKey(value); + if (!i18nMap[key]) { + i18nMap[key] = value; + CNPath.push({ path: quasi, key, value, type: 'TemplateLiteral' }); + log(`Detected Chinese text: ${value}`); + } + } + }); + }, + JSXText(path) { + const value = path.node.value.trim(); + if (/[\u4e00-\u9fa5]/.test(value)) { // Check if JSXText contains Chinese characters + const key = generateKey(value); + if (!i18nMap[key]) { + i18nMap[key] = value; + CNPath.push({ path, key, value, type: 'JSXText' }); + log(`Detected Chinese text: ${value}`); + } + } + }, + JSXAttribute(path) { + const value = path.node.value && path.node.value.value; + if (typeof value === 'string' && /[\u4e00-\u9fa5]/.test(value)) { // Check if JSXAttribute contains Chinese text + const key = generateKey(value); + if (!i18nMap[key]) { + i18nMap[key] = value; + CNPath.push({ path, key, value, type: 'JSXAttribute' }); + log(`Detected Chinese text: ${value}`); + } + } + } + }); + + return { ast, CNPath, i18nMap, i18nImported }; +}; + +// Generate code and add i18nInstance.t calls +const generateCode = (ast, i18nImported, CNPath) => { + CNPath.forEach(({ path, key, value, type }) => { + if(isI18nInvoke(path)) { + return + } + if (type === 'StringLiteral') { + path.replaceWith( + babelParser.parseExpression(`i18nInstance.t("${key}", "${value}")`) + ); + } else if (type === 'TemplateLiteral' ) { + path.value.raw = `\${i18nInstance.t("${key}", "${value}")}`; + } else if (type === 'JSXText') { + const jsxText = `i18nInstance.t("${key}", "${value}")` + path.replaceWith(t.JSXExpressionContainer(babelParser.parseExpression(jsxText))) + } else if (type === 'JSXAttribute') { + path.node.value = babelParser.parseExpression(`i18nInstance.t("${key}", "${value}")`); + } + }); + + let transformedCode = babelGenerator(ast).code; + if (!i18nImported) { + transformedCode = `import i18nInstance from '@/utils/i18n';\n` + transformedCode; + } + return transformedCode; +}; + +/** + * check the ast node to detect whether the ast node is already in form of i18n + * @param path + * @returns {boolean} + */ +function isI18nInvoke(path) { + if (!path || !path.parent || + !path.parent.callee || + !path.parent.callee.property || !path.parent.callee.object) return false; + const property = path.parent.callee.property.name + const object = path.parent.callee.object.name + return object === "i18nInstance" && property === "t" +} + +module.exports = { processAST, generateCode }; diff --git a/ui/packages/i18n-tool/src/scan/index.js b/ui/packages/i18n-tool/src/scan/index.js new file mode 100644 index 00000000..3527d33f --- /dev/null +++ b/ui/packages/i18n-tool/src/scan/index.js @@ -0,0 +1,106 @@ +const fs = require('node:fs'); +const path = require('node:path'); +const prettier = require('prettier'); +const {getDebug, parseFiles, buildLocalFilename} = require('../utils') +const {getDefaultOptions, parseI18nConfig} = require('../options') +const {processAST, generateCode} = require('./ast') +const {translate, updateLocale} = require('./translate') +const debug = getDebug('scan'); + +async function scan(cmdOptions) { + debug('execute scan method', cmdOptions); + let options = getDefaultOptions(); + if (!cmdOptions.config) { + debug('config option is empty, skip process of parseI18nConfig'); + } else { + options = await parseI18nConfig(cmdOptions.config); + } + debug('final i18n config is %O', options); + + // expand scan files + let targetFiles = []; + if (cmdOptions.file) { + targetFiles = [ + { + filePath: cmdOptions.file, + ext: path.extname(cmdOptions.file), + }, + ]; + debug( + 'expand entry in single file mode, expand files are: %O', + targetFiles, + ); + } else if (cmdOptions.directory) { + targetFiles = parseFiles({ + ...options, + entry: [cmdOptions.directory], + }); + debug( + 'expand entry in directory mode, expand files are: %O', + targetFiles, + ); + } else { + targetFiles = parseFiles(options); + debug( + 'expand entry in config mode, expand files are: %O', + targetFiles, + ); + } + let needUpdateLocale = true + if (cmdOptions.file || cmdOptions.directory) { + needUpdateLocale = false + } + + const cnLocalePath = buildLocalFilename(options.localesDir, options.originLang) + let existingData = {} + if (needUpdateLocale) { + existingData = JSON.parse(fs.readFileSync(cnLocalePath, 'utf8')); + } + + + const prettierConfig = await prettier.resolveConfig(path.join(process.cwd(), '.prettierrc')); + + let updatedData = {}; + for (const file of targetFiles) { + debug('Processing single file: %s', file.filePath); + const tsxCode = fs.readFileSync(file.filePath, 'utf8'); + const {ast, CNPath, i18nMap, i18nImported} = processAST(tsxCode, false); + if (CNPath.length > 0) { + const transformedCode = generateCode(ast, i18nImported, CNPath); + const formattedCode = await prettier.format(transformedCode, { + ...prettierConfig, + parser: 'typescript', + }); + + fs.writeFileSync(file.filePath, formattedCode, 'utf8'); + updatedData = { + ...updatedData, + ...i18nMap, + } + } + } + if (needUpdateLocale) { + fs.writeFileSync(cnLocalePath, JSON.stringify({ + ...existingData, + ...updatedData, + })) + } + + if (needUpdateLocale) { + for (const targetLang of options.targetLangs) { + const targetLocalePath = buildLocalFilename(options.localesDir, targetLang) + const targetLangLocale = await translate(updatedData, { + appid: options.translate.appid, + key: options.translate.key, + type: options.translate.type, + model: options.translate.model, + from: options.originLang, + to: targetLang + }) + updateLocale(targetLocalePath, targetLangLocale) + } + } + +} + +module.exports = scan diff --git a/ui/packages/i18n-tool/src/scan/translate.js b/ui/packages/i18n-tool/src/scan/translate.js new file mode 100644 index 00000000..b71cbdf6 --- /dev/null +++ b/ui/packages/i18n-tool/src/scan/translate.js @@ -0,0 +1,62 @@ +const fs = require("node:fs"); +const translators = require('@karmada/translators') +const {BaiduTranslator, DeepLTranslator, OpenAITranslator} = translators; + + +async function translate(i18nMap, translateOpts) { + const translator = initTranslator(translateOpts); + + const i18nMapKeys = Object.keys(i18nMap); + if (i18nMapKeys.length === 0) return {} + const reversedI18nMap = Object.keys(i18nMap).reduce((p, c) => { + return { + ...p, + [i18nMap[c]]: c + } + }, {}) + let {from, to} = translateOpts + + const translateList = Object.keys(reversedI18nMap) + const resp = await translator.batchTranslate({ + input: translateList, + from, + to, + }) + return resp.output.reduce((p, c) => { + const originI18nKey = reversedI18nMap[c.src] + return { + ...p, + [originI18nKey]: c.dst + } + }, {}) +} + +function initTranslator(translateOpts) { + const {type, appid, key, model} = translateOpts + + switch (type) { + case "baidu": return new BaiduTranslator(appid, key); + + case "deepl": return new DeepLTranslator(key); + + case "openal": return new OpenAITranslator(key, model); + + default: + debug('type of translate is not right'); + } + +} + +function updateLocale(localePath, newEntries) { + const existingData = JSON.parse(fs.readFileSync(localePath, 'utf8')); + const mergedData = { + ...existingData, + ...newEntries, + }; + fs.writeFileSync(localePath, JSON.stringify(mergedData, null, 2), 'utf8'); +} + +module.exports = { + translate, + updateLocale +} diff --git a/ui/packages/i18n-tool/src/utils.js b/ui/packages/i18n-tool/src/utils.js new file mode 100644 index 00000000..4efcb679 --- /dev/null +++ b/ui/packages/i18n-tool/src/utils.js @@ -0,0 +1,81 @@ +const fs = require('node:fs'); +const path = require('node:path'); +const glob = require('glob'); +const debug = require('debug'); + +const debugMap = {}; + +/** + * Return the debug instance according to the debugger name; + * If the debug instance already exist, return it directly. + * @param name + * @returns {*|debug} + */ +function getDebug(name) { + if (debugMap[name]) return debugMap[name]; + const d = debug(name); + debugMap[name] = d; + return d; +} + + +/** + * ensure the existence of dir, if the dir not exist, it will create the dir. + * @param dir + * @returns + */ +function ensureDirExist(dir) { + if (!dir) return false; + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { + recursive: true, + }); + return true; + } + return false; +} + + +/** + * + * @param options + * @returns FileItem[] + */ +function parseFiles(options) { + const getSourceFiles = (entry, exclude) => { + return glob.sync(`${entry}/**/*.{js,ts,tsx,jsx}`, { + ignore: exclude || [], + }); + }; + const {entry, exclude} = options; + const entries = [...entry]; + return entries.reduce((total, entryItem) => { + const extendedExclude = exclude.map((excludeItem) => + path.join(entryItem, excludeItem), + ); + const files = getSourceFiles(entryItem, extendedExclude).map((file) => { + return { + filePath: file, + ext: path.extname(file), + }; + }); + return total.concat(files); + }, []); +} + +/** + * build locale fileName for target language + * @param localesDir + * @param lang + * @returns {string} + */ +function buildLocalFilename(localesDir, lang) { + return path.join(localesDir, `${lang}.json`) +} + +module.exports = { + getDebug, + ensureDirExist, + parseFiles, + buildLocalFilename +} diff --git a/ui/packages/translators/.gitignore b/ui/packages/translators/.gitignore new file mode 100644 index 00000000..1bf752b1 --- /dev/null +++ b/ui/packages/translators/.gitignore @@ -0,0 +1,151 @@ + +### Node ### +# Logs +logs +*.log +npm-debug.log* + +# Runtime data +pids +*.pid +*.seed + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage + +# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (http://nodejs.org/api/addons.html) +build/Release + +# Dependency directory +# https://docs.npmjs.com/misc/faq#should-i-check-my-node-modules-folder-into-git +node_modules + +# Optional npm cache directory +.npm + +# Optional REPL history +.node_repl_history + + +### Bower ### +bower_components +.bower-cache +.bower-registry +.bower-tmp + + +### OSX ### +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + + +### Linux ### +*~ + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + + +### Windows ### +# Windows image file caches +Thumbs.db +ehthumbs.db + +# Folder config file +Desktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msm +*.msp + +# Windows shortcuts +*.lnk + + +### Dropbox ### +# Dropbox settings and caches +.dropbox +.dropbox.attr +.dropbox.cache + + +### Sass ### +.sass-cache/ +*.css.map + + +### Less ### +*.css + + +### grunt ### +# Grunt usually compiles files inside this directory +dist/ + +# Grunt usually preprocesses files such as coffeescript, compass... inside the .tmp directory +.tmp/ + + +### SublimeText ### +# cache files for sublime text +*.tmlanguage.cache +*.tmPreferences.cache +*.stTheme.cache + +# workspace files are user-specific +*.sublime-workspace + +# project files should be checked into the repository, unless a significant +# proportion of contributors will probably not be using SublimeText +# *.sublime-project + +# sftp configuration file +sftp-config.json + + +### Code ### +# Visual Studio Code - https://code.visualstudio.com/ +.settings/ +.vscode/ + +.env diff --git a/ui/packages/translators/.prettierignore b/ui/packages/translators/.prettierignore new file mode 100644 index 00000000..08ddad8c --- /dev/null +++ b/ui/packages/translators/.prettierignore @@ -0,0 +1,5 @@ +node_modules +build +coverage +dist +pnpm-* diff --git a/ui/packages/translators/.prettierrc b/ui/packages/translators/.prettierrc new file mode 100644 index 00000000..01d4b744 --- /dev/null +++ b/ui/packages/translators/.prettierrc @@ -0,0 +1,7 @@ +{ + "trailingComma": "all", + "singleQuote": true, + "printWidth": 80, + "tabWidth": 2, + "endOfLine": "auto" +} diff --git a/ui/packages/translators/README.md b/ui/packages/translators/README.md new file mode 100644 index 00000000..a9c76737 --- /dev/null +++ b/ui/packages/translators/README.md @@ -0,0 +1,25 @@ +# Usage + + +``` +import { BaiduTranslator } from '@karmada/translators' + +const appId = 'please input your appId'; +const appKey = 'please input your appKey'; +(async function() { + const translator = new BaiduTranslator(appId, appKey); + // translate single entry + const resp = await translator.translate({ + input: "word", + from: "zh", + to: "en", + }); + + // translate multi entries + const resp = await translator.batchTranslate({ + input: ["word1", "word2", "word3"], + from: "zh", + to: "en", + }); +})() +``` \ No newline at end of file diff --git a/ui/packages/translators/package.json b/ui/packages/translators/package.json new file mode 100644 index 00000000..e097c5a5 --- /dev/null +++ b/ui/packages/translators/package.json @@ -0,0 +1,28 @@ +{ + "name": "@karmada/translators", + "version": "1.0.0", + "description": "", + "exports": { + ".": { + "require": "./dist/index.js", + "import": "./dist/index.mjs" + } + }, + "scripts": { + "prepublish": "tsup --config ./tsup.config.ts", + "build": "tsup --config ./tsup.config.ts", + "test": "vitest" + }, + "keywords": [], + "author": "", + "license": "ISC", + "devDependencies": { + "tsup": "^8.1.0", + "vite": "^5.2.0", + "vitest": "^2.0.5" + }, + "dependencies": { + "got": "^14.4.2", + "openai": "^4.63.0" + } +} diff --git a/ui/packages/translators/src/baidu.ts b/ui/packages/translators/src/baidu.ts new file mode 100644 index 00000000..d7f00bf9 --- /dev/null +++ b/ui/packages/translators/src/baidu.ts @@ -0,0 +1,146 @@ +// import got from "got"; +import { Translator, TranslateInput, BatchTranslateInput, LANG } from "./base"; +import { calMD5, sleepNs } from "./utils"; + +interface BaiduTranslateResult { + error_code?: string; + error_msg?: string; + from: LANG; + to: LANG; + trans_result: Array<{ + src: string; + dst: string; + }>; +} + +const BD_LANG_MAP = { + "zh-CN": "zh", + "en-US": "en" +} + +export class BaiduTranslator implements Translator { + private readonly appId: string; + private readonly appKey: string; + private apiUrl: string = + "https://api.fanyi.baidu.com/api/trans/vip/translate"; + constructor(appId: string, appKey: string) { + this.appId = appId; + this.appKey = appKey; + } + + async translate(input: TranslateInput) { + const batchInput: BatchTranslateInput = { + input: [input.input], + from: input.from, + to: input.to, + }; + try { + const batchOutput = await this.batchTranslate(batchInput); + return { + output: batchOutput.output[0], + from: batchOutput.from, + to: batchOutput.to, + }; + } catch (e) { + throw e; + } + } + + async batchTranslate(input: BatchTranslateInput) { + const salt = new Date().getTime().toString(); + const aliasedFrom = BD_LANG_MAP[input.from] + const aliasedTo = BD_LANG_MAP[input.to] + const query = input.input.join("\n"); + const sign = calMD5(this.appId + query + salt + this.appKey); + try { + const options = { + headers: { + "Content-Type": "application/json", + }, + searchParams: new URLSearchParams({ + q: query, + appid: this.appId, + salt: salt, + from: aliasedFrom, + to: aliasedTo, + sign: sign, + }), + }; + const {default: got} = await import('got') + const data = await got + .get(this.apiUrl, options) + .json(); + if (data.error_code && data.error_msg) { + throw new Error(`[BaiduTranslator] translate err:${data.error_msg}`); + } + return { + output: data.trans_result, + from: data.from, + to: data.to, + }; + } catch (e) { + throw e; + } + } +} + +if (import.meta.vitest) { + interface BaiduTranslateContext { + translator: Translator; + } + const { it, expect, beforeEach } = import.meta.vitest; + + const appId = process.env.BD_APPID as string; + const appKey = process.env.BD_APPKEY as string; + const isValidConfig = appId && appKey; + + beforeEach(async (context) => { + // extend context + await sleepNs(1); + context.translator = new BaiduTranslator(appId, appKey); + }); + it("translate in single entry mode", async ({ + translator, + skip, + }) => { + if (!isValidConfig) { + skip(); + } + const input = "测试中文"; + const output = "Test Chinese"; + const resp = await translator.translate({ + input, + from: "zh-CN", + to: "en-US", + }); + expect(resp.from).toBe("zh"); + expect(resp.to).toBe("en"); + expect(resp.output.src).toBe(input); + expect(resp.output.dst).toBe(output); + }); + it("translate in batch entry mode", async ({ + translator, + skip, + }) => { + if (!isValidConfig) { + skip(); + } + const resp = await translator.batchTranslate({ + input: ["集群", "云原生", "多云"], + from: "zh-CN", + to: "en-US", + }); + expect(resp.from).toBe("zh"); + expect(resp.to).toBe("en"); + expect(resp.output.map((item) => item.src)).toEqual([ + "集群", + "云原生", + "多云", + ]); + expect(resp.output.map((item) => item.dst)).toEqual([ + "colony", + "Cloud native", + "cloudy", + ]); + }); +} diff --git a/ui/packages/translators/src/base.ts b/ui/packages/translators/src/base.ts new file mode 100644 index 00000000..13bacd8b --- /dev/null +++ b/ui/packages/translators/src/base.ts @@ -0,0 +1,50 @@ +export const LangDictionary = [ + { + cnName: '中文(繁体)', + enName: 'zh-CN', + canDetect: true, + }, + { + cnName: '英语', + enName: 'en-US', + canDetect: true, + }, +] + +export type LANG = 'auto' | 'zh-CN' | 'en-US' + + + +export interface TranslateInput { + input: string; + from: LANG; + to: LANG; +} +export interface TranslateOutput { + output: { + src: string; + dst: string; + }; + from: LANG; + to: LANG; +} + +export interface BatchTranslateInput { + input: string[]; + from: LANG; + to: LANG; +} +export interface BatchTranslateOutput { + output: Array<{ + src: string; + dst: string; + }>; + from: LANG; + to: LANG; +} + + +export interface Translator { + translate(input: TranslateInput): Promise; + batchTranslate(input: BatchTranslateInput): Promise; +} diff --git a/ui/packages/translators/src/deepl.ts b/ui/packages/translators/src/deepl.ts new file mode 100644 index 00000000..c97380c0 --- /dev/null +++ b/ui/packages/translators/src/deepl.ts @@ -0,0 +1,134 @@ +import {BatchTranslateInput, LANG, TranslateInput, Translator} from "./base"; +import {sleepNs} from "./utils"; + +interface DeepLTranslateResult { + translations: Array<{ + detected_source_language: string; + text: string; + }>; +} + + +const DEEPL_LANG_MAP = { + "zh-CN": "ZH", + "en-US": "EN" +} + +export class DeepLTranslator implements Translator { + private apiUrl: string = 'https://api-free.deepl.com/v2/translate'; + private authKey: string = ""; + + constructor(authKey: string) { + this.authKey = authKey; + } + + async translate(input: TranslateInput) { + const batchInput: BatchTranslateInput = { + input: [input.input], + from: input.from, + to: input.to, + }; + try { + const batchOutput = await this.batchTranslate(batchInput); + return { + output: batchOutput.output[0], + from: batchOutput.from, + to: batchOutput.to, + }; + } catch (e) { + throw e; + } + } + + async batchTranslate(input: BatchTranslateInput) { + const aliasedFrom = DEEPL_LANG_MAP[input.from] + const aliasedTo = DEEPL_LANG_MAP[input.to] + const {default: got} = await import('got') + const options = { + headers: { + "Content-Type": "application/json", + "Authorization": `DeepL-Auth-Key ${this.authKey}` + }, + json: { + "text": input.input, + "source_lang": aliasedFrom, + "target_lang": aliasedTo, + } + } + const data = await got + .post(this.apiUrl, options) + .json(); + const output = data.translations.map((item, index) => { + return { + src: input.input[index], + dst: item.text + } + }) + return { + output: output, + from: input.from, + to: input.to, + }; + } +} + + +if (import.meta.vitest) { + interface DeepLTranslateContext { + translator: Translator; + } + + const {it, expect, beforeEach} = import.meta.vitest; + + const authKey = process.env.DEEPL_AUTHKEY as string; + const isValidConfig = !!authKey; + + beforeEach(async (context) => { + await sleepNs(1); + context.translator = new DeepLTranslator(authKey); + }); + it("[DeepL]translate in single entry mode", async ({ + translator, + skip, + }) => { + if (!isValidConfig) { + skip(); + } + const input = "多集群"; + const output = "multi-cluster"; + const resp = await translator.translate({ + input, + from: "zh-CN", + to: "en-US", + }); + expect(resp.from).toBe("zh-CN"); + expect(resp.to).toBe("en-US"); + expect(resp.output.src).toBe(input); + expect(resp.output.dst).toBe(output); + }); + it("[DeepL]translate in batch entry mode", async ({ + translator, + skip, + }) => { + if (!isValidConfig) { + skip(); + } + const resp = await translator.batchTranslate({ + input: ["集群", "云原生", "多集群"], + from: "zh-CN", + to: "en-US", + }); + expect(resp.from).toBe("zh-CN"); + expect(resp.to).toBe("en-US"); + expect(resp.output.map((item) => item.src)).toEqual([ + "集群", + "云原生", + "多集群", + ]); + expect(resp.output.map((item) => item.dst)).toEqual([ + "clustering", + "cloud native", + "multi-cluster", + ]); + }); +} diff --git a/ui/packages/translators/src/index.ts b/ui/packages/translators/src/index.ts new file mode 100644 index 00000000..1409000e --- /dev/null +++ b/ui/packages/translators/src/index.ts @@ -0,0 +1,4 @@ +export * from './base'; +export * from './baidu'; +export * from './deepl'; +export * from './openai'; diff --git a/ui/packages/translators/src/openai.ts b/ui/packages/translators/src/openai.ts new file mode 100644 index 00000000..56d57646 --- /dev/null +++ b/ui/packages/translators/src/openai.ts @@ -0,0 +1,153 @@ +import OpenAI from 'openai'; +import {BatchTranslateInput, TranslateInput, Translator} from "./base"; +import {sleepNs} from "./utils"; + +const OPENAI_LANG_MAP = { + "zh-CN": "Chinese", + "en-US": "English" +} + +export class OpenAITranslator implements Translator { + private baseUrl: string = ''; + private apiKey: string = ""; + private model: string = ""; + private client: OpenAI; + + constructor(opts: { + baseUrl?: string; + apiKey: string; + model: string; + }) { + if (opts.baseUrl) { + this.baseUrl = opts.baseUrl + } + this.apiKey = opts.apiKey + this.model = opts.model + const params = { + apiKey: this.apiKey, + model: this.model + } + if (this.baseUrl) { + params["baseURL"] = this.baseUrl + } + this.client = new OpenAI(params); + } + + generateSystemPrompt(from: string, to: string) { + const aliasedFrom = OPENAI_LANG_MAP[from] + const aliasedTo = OPENAI_LANG_MAP[to] + return `You are a helpful assistant that translates ${aliasedFrom} text to ${aliasedTo} languages.` + } + + async translate(input: TranslateInput) { + const batchInput: BatchTranslateInput = { + input: [input.input], + from: input.from, + to: input.to, + }; + try { + const batchOutput = await this.batchTranslate(batchInput); + return { + output: batchOutput.output[0], + from: batchOutput.from, + to: batchOutput.to, + }; + } catch (e) { + throw e; + } + } + + async batchTranslate(input: BatchTranslateInput) { + const originLocales = input.input + const userContent = `Keep the JSON format. Do not include the english that is being translated or any notes. Only include the translated text. Make sure there are the same amount of open brackets as closed brackets. Translate the following text from Chinese to English: \n${JSON.stringify(originLocales)}` + const chatCompletion = await this.client.chat.completions.create({ + messages: [ + { + "role": "system", + "content": this.generateSystemPrompt(input.from, input.to), + }, + {role: 'user', content: userContent} + ], + model: this.model, + temperature: 0, + }) + const translations = JSON.parse(chatCompletion.choices?.[0]?.message?.content || "[]") as string[]; + const output = translations.map((item, index) => { + return { + src: input.input[index], + dst: item + } + }) + return { + output: output, + from: input.from, + to: input.to, + }; + } +} + + +if (import.meta.vitest) { + interface OpenAITranslateContext { + translator: Translator; + } + + const {it, expect, beforeEach} = import.meta.vitest; + + const apiKey = process.env.APIKEY as string; + const model = process.env.MODEL as string; + const isValidConfig = !!apiKey; + + beforeEach(async (context) => { + await sleepNs(1); + context.translator = new OpenAITranslator({ + baseUrl: "https://burn.hair/v1", + apiKey, + model + }); + }); + it("[OpenAI]translate in single entry mode", async ({ + translator, + skip, + }) => { + if (!isValidConfig) { + skip(); + } + const input = "多集群"; + const output = "Multiple clusters"; + const resp = await translator.translate({ + input, + from: "zh-CN", + to: "en-US", + }); + expect(resp.from).toBe("zh-CN"); + expect(resp.to).toBe("en-US"); + expect(resp.output.src).toBe(input); + expect(resp.output.dst).toBe(output); + }); + it("[OpenAI]translate in batch entry mode", async ({ + translator, + skip, + }) => { + if (!isValidConfig) { + skip(); + } + const resp = await translator.batchTranslate({ + input: ["集群", "云原生", "多集群"], + from: "zh-CN", + to: "en-US", + }); + expect(resp.from).toBe("zh-CN"); + expect(resp.to).toBe("en-US"); + expect(resp.output.map((item) => item.src)).toEqual([ + "集群", + "云原生", + "多集群", + ]); + expect(resp.output.map((item) => item.dst)).toEqual([ + "Cluster", + "Cloud Native", + "Multi-cluster", + ]); + }); +} diff --git a/ui/packages/translators/src/utils.ts b/ui/packages/translators/src/utils.ts new file mode 100644 index 00000000..2e21548e --- /dev/null +++ b/ui/packages/translators/src/utils.ts @@ -0,0 +1,13 @@ +import * as crypto from "crypto"; + +export function calMD5(data: string) { + const hash = crypto.createHash("md5"); + hash.update(data); + return hash.digest("hex"); +} + +export function sleepNs(n: number) { + return new Promise((resolve) => { + setTimeout(resolve, n * 1000); + }); +} diff --git a/ui/packages/translators/tsconfig.json b/ui/packages/translators/tsconfig.json new file mode 100644 index 00000000..702ef17c --- /dev/null +++ b/ui/packages/translators/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "composite": true, + "skipLibCheck": true, + "module": "ESNext", + "moduleResolution": "bundler", + "allowSyntheticDefaultImports": true, + // "strict": true, + "types": [ + "vitest/importMeta" + ] + }, + "include": ["src", "typings.d.ts"] +} diff --git a/ui/packages/translators/tsup.config.ts b/ui/packages/translators/tsup.config.ts new file mode 100644 index 00000000..7e5b76a2 --- /dev/null +++ b/ui/packages/translators/tsup.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from 'tsup'; + +export default defineConfig({ + entry: ['src/index.ts'], + splitting: false, + sourcemap: true, + clean: true, + format: ['cjs', 'esm'], + dts: false, +}); diff --git a/ui/packages/translators/typings.d.ts b/ui/packages/translators/typings.d.ts new file mode 100644 index 00000000..e69de29b diff --git a/ui/packages/translators/vite.config.mts b/ui/packages/translators/vite.config.mts new file mode 100644 index 00000000..eb06149a --- /dev/null +++ b/ui/packages/translators/vite.config.mts @@ -0,0 +1,9 @@ +/// +import { defineConfig, loadEnv } from "vite"; + +export default defineConfig({ + test: { + includeSource: ["src/**/*.{js,ts}"], + env: loadEnv("", process.cwd(), ""), + }, +});