diff --git a/.env.example b/.env.example index a6b0a5a2d304..ba3dcbb9ac71 100644 --- a/.env.example +++ b/.env.example @@ -112,6 +112,12 @@ OPENAI_API_KEY=sk-xxxxxxxxx # QWEN_API_KEY=sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +### Cloudflare Workers AI #### + +# CLOUDFLARE_API_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxx +# CLOUDFLARE_BASE_URL_OR_ACCOUNT_ID=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + + ######################################## ############ Market Service ############ ######################################## diff --git a/locales/ar/modelProvider.json b/locales/ar/modelProvider.json index c86dcf3a884a..4309bfc7bb13 100644 --- a/locales/ar/modelProvider.json +++ b/locales/ar/modelProvider.json @@ -45,6 +45,18 @@ "title": "استخدام معلومات المصادقة الخاصة بـ Bedrock المخصصة" } }, + "cloudflare": { + "apiKey": { + "desc": "يرجى ملء Cloudflare API Key", + "placeholder": "Cloudflare API Key", + "title": "Cloudflare API Key" + }, + "baseURLOrAccountID": { + "desc": "أدخل رقم حساب Cloudflare أو عنوان URL API المخصص", + "placeholder": "رقم حساب Cloudflare / عنوان URL API المخصص", + "title": "رقم حساب Cloudflare / عنوان URL API" + } + }, "ollama": { "checker": { "desc": "اختبر ما إذا تم إدخال عنوان الوكيل بشكل صحيح", diff --git a/locales/bg-BG/modelProvider.json b/locales/bg-BG/modelProvider.json index 30e481a8d1f0..58cbdf423724 100644 --- a/locales/bg-BG/modelProvider.json +++ b/locales/bg-BG/modelProvider.json @@ -45,6 +45,18 @@ "title": "Използване на персонализирана информация за удостоверяване на Bedrock" } }, + "cloudflare": { + "apiKey": { + "desc": "Моля, въведете Cloudflare API Key", + "placeholder": "Cloudflare API Key", + "title": "Cloudflare API Key" + }, + "baseURLOrAccountID": { + "desc": "Въведете ID на Cloudflare или личен API адрес", + "placeholder": "ID на Cloudflare / личен API адрес", + "title": "ID на Cloudflare / API адрес" + } + }, "ollama": { "checker": { "desc": "Тестване дали адресът на прокси е попълнен правилно", diff --git a/locales/de-DE/modelProvider.json b/locales/de-DE/modelProvider.json index 1984d79d96f1..5db4af046e8f 100644 --- a/locales/de-DE/modelProvider.json +++ b/locales/de-DE/modelProvider.json @@ -45,6 +45,18 @@ "title": "Verwenden Sie benutzerdefinierte Bedrock-Authentifizierungsinformationen" } }, + "cloudflare": { + "apiKey": { + "desc": "Bitte füllen Sie die Cloudflare API Key", + "placeholder": "Cloudflare API Key", + "title": "Cloudflare API Key" + }, + "baseURLOrAccountID": { + "desc": "Eingeben Sie die Cloudflare-Kundenkennung oder die benutzerdefinierte API-Adresse", + "placeholder": "Cloudflare-Kundenkennung / benutzerdefinierte API-Adresse", + "title": "Cloudflare-Kundenkennung / API-Adresse" + } + }, "ollama": { "checker": { "desc": "Testen Sie, ob die Proxy-Adresse korrekt eingetragen wurde", diff --git a/locales/en-US/modelProvider.json b/locales/en-US/modelProvider.json index 0dca43e992ca..790774a75989 100644 --- a/locales/en-US/modelProvider.json +++ b/locales/en-US/modelProvider.json @@ -45,6 +45,18 @@ "title": "Use Custom Bedrock Authentication Information" } }, + "cloudflare": { + "apiKey": { + "desc": "Please enter Cloudflare API Key", + "placeholder": "Cloudflare API Key", + "title": "Cloudflare API Key" + }, + "baseURLOrAccountID": { + "desc": "Enter your Cloudflare account ID or custom API address", + "placeholder": "Cloudflare Account ID / custom API URL", + "title": "Cloudflare Account ID / API Address" + } + }, "ollama": { "checker": { "desc": "Test if the proxy address is correctly filled in", diff --git a/locales/es-ES/modelProvider.json b/locales/es-ES/modelProvider.json index 3f8252390ba2..5e1a91d336cb 100644 --- a/locales/es-ES/modelProvider.json +++ b/locales/es-ES/modelProvider.json @@ -45,6 +45,18 @@ "title": "Usar información de autenticación de Bedrock personalizada" } }, + "cloudflare": { + "apiKey": { + "desc": "Por favor complete la Cloudflare API Key", + "placeholder": "Cloudflare API Key", + "title": "Cloudflare API Key" + }, + "baseURLOrAccountID": { + "desc": "Ingrese el ID de cuenta de Cloudflare o la dirección URL personalizada de API", + "placeholder": "ID de cuenta de Cloudflare / URL de API personalizada", + "title": "ID de cuenta de Cloudflare / dirección URL de API" + } + }, "ollama": { "checker": { "desc": "Prueba si la dirección del proxy de la interfaz se ha introducido correctamente", diff --git a/locales/fr-FR/modelProvider.json b/locales/fr-FR/modelProvider.json index 196204d65b7b..88244b17c664 100644 --- a/locales/fr-FR/modelProvider.json +++ b/locales/fr-FR/modelProvider.json @@ -45,6 +45,18 @@ "title": "Utiliser des informations d'authentification Bedrock personnalisées" } }, + "cloudflare": { + "apiKey": { + "desc": "Veuillez remplir l'Cloudflare API Key", + "placeholder": "Cloudflare API Key", + "title": "Cloudflare API Key" + }, + "baseURLOrAccountID": { + "desc": "Saisir l'ID de compte Cloudflare ou l'adresse API personnalisée", + "placeholder": "ID de compte Cloudflare / URL API personnalisée", + "title": "ID de compte Cloudflare / adresse API" + } + }, "ollama": { "checker": { "desc": "Vérifiez si l'adresse du proxy est correctement saisie", diff --git a/locales/it-IT/modelProvider.json b/locales/it-IT/modelProvider.json index 07810bf158db..3d1bd5ca45ed 100644 --- a/locales/it-IT/modelProvider.json +++ b/locales/it-IT/modelProvider.json @@ -45,6 +45,18 @@ "title": "Usa le informazioni di autenticazione Bedrock personalizzate" } }, + "cloudflare": { + "apiKey": { + "desc": "Compila l'Cloudflare API Key", + "placeholder": "Cloudflare API Key", + "title": "Cloudflare API Key" + }, + "baseURLOrAccountID": { + "desc": "Inserisci l'ID dell'account Cloudflare o l'indirizzo API personalizzato", + "placeholder": "ID account Cloudflare / URL API personalizzato", + "title": "ID account Cloudflare / indirizzo API" + } + }, "ollama": { "checker": { "desc": "Verifica se l'indirizzo del proxy è stato compilato correttamente", diff --git a/locales/ja-JP/modelProvider.json b/locales/ja-JP/modelProvider.json index d1ef2f8948b3..2c9250b20faf 100644 --- a/locales/ja-JP/modelProvider.json +++ b/locales/ja-JP/modelProvider.json @@ -45,6 +45,18 @@ "title": "使用カスタム Bedrock 認証情報" } }, + "cloudflare": { + "apiKey": { + "desc": "Cloudflare API Key を入力してください", + "placeholder": "Cloudflare API Key", + "title": "Cloudflare API Key" + }, + "baseURLOrAccountID": { + "desc": "Cloudflare アカウント ID またはカスタム API アドレスを入力してください。", + "placeholder": "Cloudflare アカウント ID / カスタム API URL", + "title": "Cloudflare アカウント ID / API アドレス" + } + }, "ollama": { "checker": { "desc": "プロキシアドレスが正しく入力されているかをテストします", diff --git a/locales/ko-KR/modelProvider.json b/locales/ko-KR/modelProvider.json index c1b96eee6ea7..0bafa0152c45 100644 --- a/locales/ko-KR/modelProvider.json +++ b/locales/ko-KR/modelProvider.json @@ -45,6 +45,18 @@ "title": "사용자 정의 Bedrock 인증 정보 사용" } }, + "cloudflare": { + "apiKey": { + "desc": "Cloudflare API Key 를 작성해 주세요.", + "placeholder": "Cloudflare API Key", + "title": "Cloudflare API Key" + }, + "baseURLOrAccountID": { + "desc": "클라우드 플레어 계정 ID 또는 사용자 지정 API 주소 입력", + "placeholder": "클라우드 플레어 계정 ID / 사용자 지정 API 주소", + "title": "클라우드 플레어 계정 ID / API 주소" + } + }, "ollama": { "checker": { "desc": "프록시 주소가 올바르게 입력되었는지 테스트합니다", diff --git a/locales/nl-NL/modelProvider.json b/locales/nl-NL/modelProvider.json index 9d95b27b2e9f..d78ed31d6c93 100644 --- a/locales/nl-NL/modelProvider.json +++ b/locales/nl-NL/modelProvider.json @@ -45,6 +45,18 @@ "title": "Gebruik aangepaste Bedrock-verificatiegegevens" } }, + "cloudflare": { + "apiKey": { + "desc": "Voer Cloudflare API Key in", + "placeholder": "Cloudflare API Key", + "title": "Cloudflare API Key" + }, + "baseURLOrAccountID": { + "desc": "Voer uw Cloudflare-account ID of een custom API-URL in", + "placeholder": "Cloudflare-account ID / custom API-URL", + "title": "Cloudflare-account ID / API-URL" + } + }, "ollama": { "checker": { "desc": "Test of het proxyadres correct is ingevuld", diff --git a/locales/pl-PL/modelProvider.json b/locales/pl-PL/modelProvider.json index c233755bd553..d5174c91069c 100644 --- a/locales/pl-PL/modelProvider.json +++ b/locales/pl-PL/modelProvider.json @@ -45,6 +45,18 @@ "title": "Użyj niestandardowych informacji uwierzytelniających Bedrock" } }, + "cloudflare": { + "apiKey": { + "desc": "Wprowadź klucz Cloudflare API Key", + "placeholder": "Cloudflare API Key", + "title": "Cloudflare API Key" + }, + "baseURLOrAccountID": { + "desc": "Wprowadź ID konta Cloudflare lub adres API niestandardowy", + "placeholder": "ID konta Cloudflare / adres API niestandardowy", + "title": "ID konta Cloudflare / adres API" + } + }, "ollama": { "checker": { "desc": "Test czy adres proxy jest poprawnie wypełniony", diff --git a/locales/pt-BR/modelProvider.json b/locales/pt-BR/modelProvider.json index 1521fb28dd6b..8846ac4bbe26 100644 --- a/locales/pt-BR/modelProvider.json +++ b/locales/pt-BR/modelProvider.json @@ -45,6 +45,18 @@ "title": "Usar informações de autenticação Bedrock personalizadas" } }, + "cloudflare": { + "apiKey": { + "desc": "Insira o Cloudflare API Key", + "placeholder": "Cloudflare API Key", + "title": "Cloudflare API Key" + }, + "baseURLOrAccountID": { + "desc": "Insira o ID da conta do Cloudflare ou o endereço da API personalizado", + "placeholder": "ID da conta do Cloudflare / URL da API personalizada", + "title": "ID da conta do Cloudflare / Endereço da API" + } + }, "ollama": { "checker": { "desc": "Teste se o endereço do proxy está corretamente preenchido", diff --git a/locales/ru-RU/modelProvider.json b/locales/ru-RU/modelProvider.json index d4ee8caa89e2..93750ccea50c 100644 --- a/locales/ru-RU/modelProvider.json +++ b/locales/ru-RU/modelProvider.json @@ -45,6 +45,18 @@ "title": "Использовать пользовательскую информацию аутентификации Bedrock" } }, + "cloudflare": { + "apiKey": { + "desc": "Пожалуйста, заполните Cloudflare API Key", + "placeholder": "Cloudflare API Key", + "title": "Cloudflare API Key" + }, + "baseURLOrAccountID": { + "desc": "Введите ID аккаунта Cloudflare или адрес API по умолчанию", + "placeholder": "ID аккаунта Cloudflare / адрес API по умолчанию", + "title": "ID аккаунта Cloudflare / адрес API" + } + }, "ollama": { "checker": { "desc": "Проверить правильность адреса прокси", diff --git a/locales/tr-TR/modelProvider.json b/locales/tr-TR/modelProvider.json index 549957a54c01..a4db2aad44a6 100644 --- a/locales/tr-TR/modelProvider.json +++ b/locales/tr-TR/modelProvider.json @@ -45,6 +45,18 @@ "title": "Özel Bedrock Kimlik Bilgilerini Kullan" } }, + "cloudflare": { + "apiKey": { + "desc": "Lütfen doldurun Cloudflare API Key", + "placeholder": "Cloudflare API Key", + "title": "Cloudflare API Key" + }, + "baseURLOrAccountID": { + "desc": "Cloudflare hesabınızın ID'sini veya özel API adresinizi girin", + "placeholder": "Cloudflare Hesap ID / Özel API Adresi", + "title": "Cloudflare Hesap ID / API Adresi" + } + }, "ollama": { "checker": { "desc": "Proxy adresinin doğru girilip girilmediğini test edin", diff --git a/locales/vi-VN/modelProvider.json b/locales/vi-VN/modelProvider.json index 890169ab2623..46a027430b40 100644 --- a/locales/vi-VN/modelProvider.json +++ b/locales/vi-VN/modelProvider.json @@ -45,6 +45,18 @@ "title": "Sử dụng Thông tin Xác thực Bedrock tùy chỉnh" } }, + "cloudflare": { + "apiKey": { + "desc": "Vui lòng nhập Cloudflare API Key", + "placeholder": "Cloudflare API Key", + "title": "Cloudflare API Key" + }, + "baseURLOrAccountID": { + "desc": "Nhập ID tài khoản Cloudflare hoặc địa chỉ API tùy chỉnh", + "placeholder": "ID tài khoản Cloudflare / địa chỉ API tùy chỉnh", + "title": "ID tài khoản Cloudflare / địa chỉ API" + } + }, "ollama": { "checker": { "desc": "Kiểm tra địa chỉ proxy có được nhập chính xác không", diff --git a/locales/zh-CN/modelProvider.json b/locales/zh-CN/modelProvider.json index 6e36ee95a505..701fe0aab6cf 100644 --- a/locales/zh-CN/modelProvider.json +++ b/locales/zh-CN/modelProvider.json @@ -45,6 +45,18 @@ "title": "使用自定义 Bedrock 鉴权信息" } }, + "cloudflare": { + "apiKey": { + "desc": "请填写 Cloudflare API Key", + "placeholder": "Cloudflare API Key", + "title": "Cloudflare API Key" + }, + "baseURLOrAccountID": { + "desc": "填入 Cloudflare 账户 ID 或 自定义 API 地址", + "placeholder": "Cloudflare Account ID / custom API URL", + "title": "Cloudflare 账户 ID / API 地址" + } + }, "ollama": { "checker": { "desc": "测试代理地址是否正确填写", diff --git a/locales/zh-TW/modelProvider.json b/locales/zh-TW/modelProvider.json index 3b9baacb6b07..e1c2e120dfca 100644 --- a/locales/zh-TW/modelProvider.json +++ b/locales/zh-TW/modelProvider.json @@ -45,6 +45,18 @@ "title": "使用自定義 Bedrock 驗證資訊" } }, + "cloudflare": { + "apiKey": { + "desc": "請填入 Cloudflare API Key", + "placeholder": "Cloudflare API Key", + "title": "Cloudflare API Key" + }, + "baseURLOrAccountID": { + "desc": "填入 Cloudflare 帳戶 ID 或 自定義 API 位址", + "placeholder": "Cloudflare 帳戶 ID / 自定義 API 位址", + "title": "Cloudflare 帳戶 ID / API 位址" + } + }, "ollama": { "checker": { "desc": "測試代理地址是否正確填寫", diff --git a/src/app/(main)/settings/llm/ProviderList/Cloudflare/index.tsx b/src/app/(main)/settings/llm/ProviderList/Cloudflare/index.tsx new file mode 100644 index 000000000000..f493a46068b8 --- /dev/null +++ b/src/app/(main)/settings/llm/ProviderList/Cloudflare/index.tsx @@ -0,0 +1,45 @@ +'use client'; + +import { Input } from 'antd'; +import { useTranslation } from 'react-i18next'; + +import { CloudflareProviderCard } from '@/config/modelProviders'; +import { GlobalLLMProviderKey } from '@/types/user/settings'; + +import { KeyVaultsConfigKey } from '../../const'; +import { ProviderItem } from '../../type'; +import { CloudflareBrand } from '../providers'; + +const providerKey: GlobalLLMProviderKey = 'cloudflare'; + +export const useCloudflareProvider = (): ProviderItem => { + const { t } = useTranslation('modelProvider'); + + return { + ...CloudflareProviderCard, + apiKeyItems: [ + { + children: ( + + ), + desc: t(`${providerKey}.apiKey.desc`), + label: t(`${providerKey}.apiKey.title`), + name: [KeyVaultsConfigKey, providerKey, 'apiKey'], + }, + { + children: ( + + ), + desc: t(`${providerKey}.baseURLOrAccountID.desc`), + label: t(`${providerKey}.baseURLOrAccountID.title`), + name: [KeyVaultsConfigKey, providerKey, 'baseURLOrAccountID'], + }, + ], + title: , + }; +}; diff --git a/src/app/(main)/settings/llm/ProviderList/providers.tsx b/src/app/(main)/settings/llm/ProviderList/providers.tsx index 30aa944167b1..dfa9d66fbe71 100644 --- a/src/app/(main)/settings/llm/ProviderList/providers.tsx +++ b/src/app/(main)/settings/llm/ProviderList/providers.tsx @@ -4,6 +4,7 @@ import { Anthropic, Baichuan, Claude, + Cloudflare, DeepSeek, Gemini, Google, @@ -17,6 +18,7 @@ import { Stepfun, Together, Tongyi, + WorkersAI, ZeroOne, Zhipu, } from '@lobehub/icons'; @@ -50,6 +52,7 @@ import { import { ProviderItem } from '../type'; import { useAzureProvider } from './Azure'; import { useBedrockProvider } from './Bedrock'; +import { useCloudflareProvider } from './Cloudflare'; import { useOllamaProvider } from './Ollama'; import { useOpenAIProvider } from './OpenAI'; @@ -84,11 +87,20 @@ const GoogleBrand = () => ( ); +export const CloudflareBrand = () => ( + + + + + +); + export const useProviderList = (): ProviderItem[] => { const azureProvider = useAzureProvider(); const ollamaProvider = useOllamaProvider(); const openAIProvider = useOpenAIProvider(); const bedrockProvider = useBedrockProvider(); + const cloudflareProvider = useCloudflareProvider(); return useMemo( () => [ @@ -186,7 +198,7 @@ export const useProviderList = (): ProviderItem[] => { { ...BaichuanProviderCard, docUrl: urlJoin(BASE_DOC_URL, 'baichuan'), - title: , + title: , }, { ...TaichuProviderCard, @@ -198,7 +210,11 @@ export const useProviderList = (): ProviderItem[] => { docUrl: urlJoin(BASE_DOC_URL, 'ai360'), title: , }, + { + ...cloudflareProvider, + docUrl: urlJoin(BASE_DOC_URL, 'cloudflare'), + }, ], - [azureProvider, ollamaProvider, ollamaProvider, bedrockProvider], + [azureProvider, ollamaProvider, ollamaProvider, bedrockProvider, cloudflareProvider], ); }; diff --git a/src/app/api/chat/agentRuntime.ts b/src/app/api/chat/agentRuntime.ts index e5ff047ef3fc..ed0eb65b0b78 100644 --- a/src/app/api/chat/agentRuntime.ts +++ b/src/app/api/chat/agentRuntime.ts @@ -200,6 +200,17 @@ const getLlmOptionsFromPayload = (provider: string, payload: JWTPayload) => { return { apiKey }; } + case ModelProvider.Cloudflare: { + const { CLOUDFLARE_API_KEY, CLOUDFLARE_BASE_URL_OR_ACCOUNT_ID } = getLLMConfig(); + + const apiKey = apiKeyManager.pick(payload?.apiKey || CLOUDFLARE_API_KEY); + const baseURLOrAccountID = + payload.apiKey && payload.cloudflareBaseURLOrAccountID + ? payload.cloudflareBaseURLOrAccountID + : CLOUDFLARE_BASE_URL_OR_ACCOUNT_ID; + + return { apiKey, baseURLOrAccountID }; + } } }; diff --git a/src/components/ModelProviderIcon/index.tsx b/src/components/ModelProviderIcon/index.tsx index de6270991684..19bbfc2b66db 100644 --- a/src/components/ModelProviderIcon/index.tsx +++ b/src/components/ModelProviderIcon/index.tsx @@ -5,6 +5,7 @@ import { Azure, Baichuan, Bedrock, + Cloudflare, DeepSeek, Google, Groq, @@ -134,6 +135,10 @@ const ModelProviderIcon = memo(({ provider }) => { return ; } + case ModelProvider.Cloudflare: { + return ; + } + default: { return null; } diff --git a/src/config/llm.ts b/src/config/llm.ts index b745e7a235bc..b8bdac89dc8a 100644 --- a/src/config/llm.ts +++ b/src/config/llm.ts @@ -87,6 +87,10 @@ export const getLLMConfig = () => { ENABLED_AI360: z.boolean(), AI360_API_KEY: z.string().optional(), + + ENABLED_CLOUDFLARE: z.boolean(), + CLOUDFLARE_API_KEY: z.string().optional(), + CLOUDFLARE_BASE_URL_OR_ACCOUNT_ID: z.string().optional(), }, runtimeEnv: { API_KEY_SELECT_MODE: process.env.API_KEY_SELECT_MODE, @@ -171,6 +175,11 @@ export const getLLMConfig = () => { ENABLED_AI360: !!process.env.AI360_API_KEY, AI360_API_KEY: process.env.AI360_API_KEY, + + ENABLED_CLOUDFLARE: + !!process.env.CLOUDFLARE_API_KEY && !!process.env.CLOUDFLARE_BASE_URL_OR_ACCOUNT_ID, + CLOUDFLARE_API_KEY: process.env.CLOUDFLARE_API_KEY, + CLOUDFLARE_BASE_URL_OR_ACCOUNT_ID: process.env.CLOUDFLARE_BASE_URL_OR_ACCOUNT_ID, }, }); }; diff --git a/src/config/modelProviders/cloudflare.ts b/src/config/modelProviders/cloudflare.ts new file mode 100644 index 000000000000..359a469b3713 --- /dev/null +++ b/src/config/modelProviders/cloudflare.ts @@ -0,0 +1,99 @@ +import { ModelProviderCard } from '@/types/llm'; + +// ref https://developers.cloudflare.com/workers-ai/models/#text-generation +// api https://developers.cloudflare.com/workers-ai/configuration/open-ai-compatibility +const Cloudflare: ModelProviderCard = { + chatModels: [ + { + displayName: 'deepseek-coder-6.7b-instruct-awq', + enabled: true, + id: '@hf/thebloke/deepseek-coder-6.7b-instruct-awq', + tokens: 16_384, + }, + { + displayName: 'deepseek-math-7b-instruct', + enabled: true, + id: '@hf/thebloke/deepseek-math-7b-instruct', + tokens: 4096, + }, + { + displayName: 'gemma-7b-it', + enabled: true, + id: '@hf/google/gemma-7b-it', + tokens: 2048, + }, + { + displayName: 'hermes-2-pro-mistral-7b', + enabled: true, + // functionCall: true, + id: '@hf/nousresearch/hermes-2-pro-mistral-7b', + tokens: 4096, + }, + { + displayName: 'llama-3-8b-instruct-awq', + id: '@cf/meta/llama-3-8b-instruct-awq', + tokens: 8192, + }, + { + displayName: 'llama-3.1-8b-instruct', + id: '@cf/meta/llama-3.1-8b-instruct', + tokens: 128_000, + }, + { + displayName: 'mistral-7b-instruct-v0.2', + id: '@hf/mistral/mistral-7b-instruct-v0.2', + tokens: 4096, + }, + { + displayName: 'neural-chat-7b-v3-1-awq', + enabled: true, + id: '@hf/thebloke/neural-chat-7b-v3-1-awq', + tokens: 32_768, + }, + { + displayName: 'openchat-3.5-0106', + id: '@cf/openchat/openchat-3.5-0106', + tokens: 8192, + }, + { + displayName: 'openhermes-2.5-mistral-7b-awq', + enabled: true, + id: '@hf/thebloke/openhermes-2.5-mistral-7b-awq', + tokens: 32_768, + }, + { + displayName: 'qwen1.5-14b-chat-awq', + enabled: true, + id: '@cf/qwen/qwen1.5-14b-chat-awq', + tokens: 32_768, + }, + { + displayName: 'starling-lm-7b-beta', + enabled: true, + id: '@hf/nexusflow/starling-lm-7b-beta', + tokens: 4096, + }, + { + displayName: 'zephyr-7b-beta-awq', + enabled: true, + id: '@hf/thebloke/zephyr-7b-beta-awq', + tokens: 32_768, + }, + { + description: + 'Generation over generation, Meta Llama 3 demonstrates state-of-the-art performance on a wide range of industry benchmarks and offers new capabilities, including improved reasoning.\t', + displayName: 'meta-llama-3-8b-instruct', + enabled: true, + functionCall: false, + id: '@hf/meta-llama/meta-llama-3-8b-instruct', + }, + ], + checkModel: '@hf/meta-llama/meta-llama-3-8b-instruct', + id: 'cloudflare', + modelList: { + showModelFetcher: true, + }, + name: 'Cloudflare Workers AI', +}; + +export default Cloudflare; diff --git a/src/config/modelProviders/google.ts b/src/config/modelProviders/google.ts index b56d368130bf..e3b4d91b4c6d 100644 --- a/src/config/modelProviders/google.ts +++ b/src/config/modelProviders/google.ts @@ -53,16 +53,14 @@ const Google: ModelProviderCard = { vision: true, }, { - description: - 'The best model for scaling across a wide range of tasks. This is the latest model.', + description: 'The best model for scaling across a wide range of tasks. This is the latest model.', displayName: 'Gemini 1.0 Pro', id: 'gemini-1.0-pro-latest', maxOutput: 2048, tokens: 30_720 + 2048, }, { - description: - 'The best model for scaling across a wide range of tasks. This is a stable model that supports tuning.', + description: 'The best model for scaling across a wide range of tasks. This is a stable model that supports tuning.', displayName: 'Gemini 1.0 Pro 001 (Tuning)', functionCall: true, id: 'gemini-1.0-pro-001', @@ -70,8 +68,7 @@ const Google: ModelProviderCard = { tokens: 30_720 + 2048, }, { - description: - 'The best model for scaling across a wide range of tasks. Released April 9, 2024.', + description: 'The best model for scaling across a wide range of tasks. Released April 9, 2024.', displayName: 'Gemini 1.0 Pro 002 (Tuning)', id: 'gemini-1.0-pro-002', maxOutput: 2048, diff --git a/src/config/modelProviders/index.ts b/src/config/modelProviders/index.ts index db5ae1446e8b..e5b4cdc3de74 100644 --- a/src/config/modelProviders/index.ts +++ b/src/config/modelProviders/index.ts @@ -5,6 +5,7 @@ import AnthropicProvider from './anthropic'; import AzureProvider from './azure'; import BaichuanProvider from './baichuan'; import BedrockProvider from './bedrock'; +import CloudflareProvider from './cloudflare'; import DeepSeekProvider from './deepseek'; import GoogleProvider from './google'; import GroqProvider from './groq'; @@ -45,6 +46,7 @@ export const LOBE_DEFAULT_MODEL_LIST: ChatModelCard[] = [ BaichuanProvider.chatModels, TaichuProvider.chatModels, Ai360Provider.chatModels, + CloudflareProvider.chatModels, ].flat(); export const DEFAULT_MODEL_PROVIDER_LIST = [ @@ -70,6 +72,7 @@ export const DEFAULT_MODEL_PROVIDER_LIST = [ BaichuanProvider, TaichuProvider, Ai360Provider, + CloudflareProvider, ]; export const filterEnabledModels = (provider: ModelProviderCard) => { @@ -86,6 +89,7 @@ export { default as AnthropicProviderCard } from './anthropic'; export { default as AzureProviderCard } from './azure'; export { default as BaichuanProviderCard } from './baichuan'; export { default as BedrockProviderCard } from './bedrock'; +export { default as CloudflareProviderCard } from './cloudflare'; export { default as DeepSeekProviderCard } from './deepseek'; export { default as GoogleProviderCard } from './google'; export { default as GroqProviderCard } from './groq'; diff --git a/src/const/auth.ts b/src/const/auth.ts index 1c0fd878dd35..d59b44d435e4 100644 --- a/src/const/auth.ts +++ b/src/const/auth.ts @@ -35,6 +35,8 @@ export interface JWTPayload { awsAccessKeyId?: string; awsRegion?: string; awsSecretAccessKey?: string; + + cloudflareBaseURLOrAccountID?: string; /** * user id * in client db mode it's a uuid diff --git a/src/const/settings/llm.ts b/src/const/settings/llm.ts index 1cd98e069ae8..7299986c3cb9 100644 --- a/src/const/settings/llm.ts +++ b/src/const/settings/llm.ts @@ -3,6 +3,7 @@ import { AnthropicProviderCard, BaichuanProviderCard, BedrockProviderCard, + CloudflareProviderCard, DeepSeekProviderCard, GoogleProviderCard, GroqProviderCard, @@ -45,6 +46,10 @@ export const DEFAULT_LLM_CONFIG: UserModelProviderConfig = { enabled: false, enabledModels: filterEnabledModels(BedrockProviderCard), }, + cloudflare: { + enabled: false, + enabledModels: filterEnabledModels(CloudflareProviderCard), + }, deepseek: { enabled: false, enabledModels: filterEnabledModels(DeepSeekProviderCard), diff --git a/src/libs/agent-runtime/AgentRuntime.ts b/src/libs/agent-runtime/AgentRuntime.ts index fdb28eb25b3f..41b95b7f920e 100644 --- a/src/libs/agent-runtime/AgentRuntime.ts +++ b/src/libs/agent-runtime/AgentRuntime.ts @@ -8,6 +8,7 @@ import { LobeAnthropicAI } from './anthropic'; import { LobeAzureOpenAI } from './azureOpenai'; import { LobeBaichuanAI } from './baichuan'; import { LobeBedrockAI, LobeBedrockAIParams } from './bedrock'; +import { LobeCloudflareAI, LobeCloudflareParams } from './cloudflare'; import { LobeDeepSeekAI } from './deepseek'; import { LobeGoogleAI } from './google'; import { LobeGroq } from './groq'; @@ -110,6 +111,7 @@ class AgentRuntime { azure: { apiVersion?: string; apikey?: string; endpoint?: string }; baichuan: Partial; bedrock: Partial; + cloudflare: Partial; deepseek: Partial; google: { apiKey?: string; baseURL?: string }; groq: Partial; @@ -247,8 +249,12 @@ class AgentRuntime { runtimeModel = new LobeAi360AI(params.ai360 ?? {}); break } - } + case ModelProvider.Cloudflare: { + runtimeModel = new LobeCloudflareAI(params.cloudflare ?? {}); + break; + } + } return new AgentRuntime(runtimeModel); } } diff --git a/src/libs/agent-runtime/cloudflare/index.ts b/src/libs/agent-runtime/cloudflare/index.ts new file mode 100644 index 000000000000..6dcf9de3e4b4 --- /dev/null +++ b/src/libs/agent-runtime/cloudflare/index.ts @@ -0,0 +1,221 @@ +import { ChatModelCard } from '@/types/llm'; + +import { LobeRuntimeAI } from '../BaseAI'; +import { AgentRuntimeErrorType } from '../error'; +import { ChatCompetitionOptions, ChatStreamPayload, ModelProvider } from '../types'; +import { AgentRuntimeError } from '../utils/createError'; +import { desensitizeUrl } from '../utils/desensitizeUrl'; +import { StreamingResponse } from '../utils/response'; + +const DEFAULT_BASE_URL_PREFIX = 'https://api.cloudflare.com'; + +export interface LobeCloudflareParams { + apiKey?: string; + baseURLOrAccountID?: string; +} + +function fillUrl(accountID: string): string { + return `${DEFAULT_BASE_URL_PREFIX}/client/v4/accounts/${accountID}/ai/run/`; +} + +function desensitizeAccountId(path: string): string { + return path.replace(/\/[\dA-Fa-f]{32}\//, '/****/'); +} + +function desensitizeCloudflareUrl(url: string): string { + const urlObj = new URL(url); + let { protocol, hostname, port, pathname, search } = urlObj; + if (url.startsWith(DEFAULT_BASE_URL_PREFIX)) { + return `${protocol}//${hostname}${port ? `:${port}` : ''}${desensitizeAccountId(pathname)}${search}`; + } else { + const desensitizedUrl = desensitizeUrl(`${protocol}//${hostname}${port ? `:${port}` : ''}`); + return `${desensitizedUrl}${desensitizeAccountId(pathname)}${search}`; + } +} + +const CF_PROPERTY_NAME = 'property_id'; + +function getModelBeta(model: any): boolean { + try { + const betaProperty = model['properties'].filter( + (property: any) => property[CF_PROPERTY_NAME] === 'beta', + ); + if (betaProperty.length === 1) { + return betaProperty[0]['value'] === 'true'; // This is a string now. + } + return false; + } catch { + return false; + } +} + +function getModelDisplayName(model: any, beta: boolean): string { + const modelId = model['name']; + let name = modelId.split('/').at(-1)!; + if (beta) { + name += ' (Beta)'; + } + return name; +} + +function getModelFunctionCalling(model: any): boolean { + return false; + // eslint-disable-next-line no-unreachable + try { + const fcProperty = model['properties'].filter( + (property: any) => property[CF_PROPERTY_NAME] === 'function_calling', + ); + if (fcProperty.length === 1) { + return fcProperty[0]['value'] === 'true'; + } + return false; + } catch { + return false; + } +} + +function getModelTokens(model: any): number | undefined { + try { + const tokensProperty = model['properties'].filter( + (property: any) => property[CF_PROPERTY_NAME] === 'max_total_tokens', + ); + if (tokensProperty.length === 1) { + return parseInt(tokensProperty[0]['value']); + } + return undefined; + } catch { + return undefined; + } +} + +class CloudflareStreamTransformer { + private textDecoder = new TextDecoder(); + private textEncoder = new TextEncoder(); + private buffer: string = ''; + + private parseChunk(chunk: string, controller: TransformStreamDefaultController) { + const dataPrefix = /^data: /; + const json = chunk.replace(dataPrefix, ''); + const parsedChunk = JSON.parse(json); + controller.enqueue(this.textEncoder.encode(`event: text\n`)); + controller.enqueue(this.textEncoder.encode(`data: ${JSON.stringify(parsedChunk.response)}\n\n`)); + } + + public async transform(chunk: Uint8Array, controller: TransformStreamDefaultController) { + let textChunk = this.textDecoder.decode(chunk); + if (this.buffer.trim() !== '') { + textChunk = this.buffer + textChunk; + this.buffer = ''; + } + const splits = textChunk.split('\n\n'); + for (let i = 0; i < splits.length - 1; i++) { + if (/\[DONE]/.test(splits[i].trim())) { + return; + } + this.parseChunk(splits[i], controller); + } + const lastChunk = splits.at(-1)!; + if (lastChunk.trim() !== '') { + this.buffer += lastChunk; // does not need to be trimmed. + } // else drop. + } +} + +export class LobeCloudflareAI implements LobeRuntimeAI { + baseURL: string; + accountID: string; + apiKey?: string; + + constructor({ apiKey, baseURLOrAccountID }: LobeCloudflareParams) { + if (!baseURLOrAccountID) { + throw AgentRuntimeError.createError(AgentRuntimeErrorType.InvalidProviderAPIKey); + } + if (baseURLOrAccountID.startsWith('http')) { + this.baseURL = baseURLOrAccountID.endsWith('/') + ? baseURLOrAccountID + : baseURLOrAccountID + '/'; + // Try get accountID from baseURL + this.accountID = baseURLOrAccountID.replaceAll(/^.*\/([\dA-Fa-f]{32})\/.*$/g, '$1'); + } else { + this.accountID = baseURLOrAccountID; + this.baseURL = fillUrl(baseURLOrAccountID); + } + this.apiKey = apiKey; + } + + async chat(payload: ChatStreamPayload, options?: ChatCompetitionOptions): Promise { + // Implement your logic here + // This method should handle the chat functionality using the provided payload and options + // It should return a Promise that resolves to a Response object + // You can make API calls, perform computations, or any other necessary operations + + // Example implementation: + try { + const { model, tools, ...restPayload } = payload; + const functions = tools?.map((tool) => tool.function); + const headers = options?.headers || {}; + if (this.apiKey) { + headers['Authorization'] = `Bearer ${this.apiKey}`; + } + const url = new URL(model, this.baseURL); + const response = await fetch(url, { + body: JSON.stringify({ tools: functions, ...restPayload }), + headers: { 'Content-Type': 'application/json', ...headers }, + method: 'POST', + }); + + const desensitizedEndpoint = desensitizeCloudflareUrl(this.baseURL); + + switch (response.status) { + case 400: { + throw AgentRuntimeError.chat({ + endpoint: desensitizedEndpoint, + error: response, + errorType: AgentRuntimeErrorType.ProviderBizError, + provider: ModelProvider.Cloudflare, + }); + } + } + + return StreamingResponse( + response.body!.pipeThrough(new TransformStream(new CloudflareStreamTransformer())), + ); + } catch (error) { + const desensitizedEndpoint = desensitizeCloudflareUrl(this.baseURL); + + throw AgentRuntimeError.chat({ + endpoint: desensitizedEndpoint, + error: error as any, + errorType: AgentRuntimeErrorType.ProviderBizError, + provider: ModelProvider.Cloudflare, + }); + } + } + + async models(): Promise { + const url = `${DEFAULT_BASE_URL_PREFIX}/client/v4/accounts/${this.accountID}/ai/models/search`; + const response = await fetch(url, { + headers: { + 'Authorization': `Bearer ${this.apiKey}`, + 'Content-Type': 'application/json', + }, + method: 'GET', + }); + const j = await response.json(); + const models: any[] = j['result'].filter( + (model: any) => model['task']['name'] === 'Text Generation', + ); + const chatModels: ChatModelCard[] = models.map((model) => { + const modelBeta = getModelBeta(model); + return { + description: model['description'], + displayName: getModelDisplayName(model, modelBeta), + enabled: !modelBeta, + functionCall: getModelFunctionCalling(model), + id: model['name'], + tokens: getModelTokens(model), + }; + }); + return chatModels; + } +} diff --git a/src/libs/agent-runtime/types/type.ts b/src/libs/agent-runtime/types/type.ts index b2ebbc83e82f..e9c6270fec81 100644 --- a/src/libs/agent-runtime/types/type.ts +++ b/src/libs/agent-runtime/types/type.ts @@ -27,6 +27,7 @@ export enum ModelProvider { Azure = 'azure', Baichuan = 'baichuan', Bedrock = 'bedrock', + Cloudflare = 'cloudflare', DeepSeek = 'deepseek', Google = 'google', Groq = 'groq', diff --git a/src/locales/default/modelProvider.ts b/src/locales/default/modelProvider.ts index 89a222c462f7..212892af66d0 100644 --- a/src/locales/default/modelProvider.ts +++ b/src/locales/default/modelProvider.ts @@ -21,7 +21,7 @@ export default { }, bedrock: { accessKeyId: { - desc: '填入AWS Access Key Id', + desc: '填入 AWS Access Key Id', placeholder: 'AWS Access Key Id', title: 'AWS Access Key Id', }, @@ -46,6 +46,18 @@ export default { title: '使用自定义 Bedrock 鉴权信息', }, }, + cloudflare: { + apiKey: { + desc: '请填写 Cloudflare API Key', + placeholder: 'Cloudflare API Key', + title: 'Cloudflare API Key', + }, + baseURLOrAccountID: { + desc: '填入 Cloudflare 账户 ID 或 自定义 API 地址', + placeholder: 'Cloudflare Account ID / custom API URL', + title: 'Cloudflare 账户 ID / API 地址', + }, + }, ollama: { checker: { desc: '测试代理地址是否正确填写', diff --git a/src/server/globalConfig/index.ts b/src/server/globalConfig/index.ts index a12fa0d94f03..97d78d8835f6 100644 --- a/src/server/globalConfig/index.ts +++ b/src/server/globalConfig/index.ts @@ -38,6 +38,7 @@ export const getServerGlobalConfig = () => { ENABLED_BAICHUAN, ENABLED_TAICHU, ENABLED_AI360, + ENABLED_CLOUDFLARE, ENABLED_AZURE_OPENAI, AZURE_MODEL_LIST, @@ -77,6 +78,7 @@ export const getServerGlobalConfig = () => { }, baichuan: { enabled: ENABLED_BAICHUAN }, bedrock: { enabled: ENABLED_AWS_BEDROCK }, + cloudflare: { enabled: ENABLED_CLOUDFLARE }, deepseek: { enabled: ENABLED_DEEPSEEK }, google: { enabled: ENABLED_GOOGLE }, groq: { enabled: ENABLED_GROQ }, diff --git a/src/services/_auth.ts b/src/services/_auth.ts index 6e8c98b04d9f..db500c152e6f 100644 --- a/src/services/_auth.ts +++ b/src/services/_auth.ts @@ -36,6 +36,15 @@ export const getProviderAuthPayload = (provider: string) => { return { endpoint: config?.baseURL }; } + case ModelProvider.Cloudflare: { + const config = keyVaultsConfigSelectors.cloudflareConfig(useUserStore.getState()); + + return { + apiKey: config?.apiKey, + cloudflareBaseURLOrAccountID: config?.baseURLOrAccountID, + }; + } + default: { const config = keyVaultsConfigSelectors.getVaultByProvider(provider as GlobalLLMProviderKey)( useUserStore.getState(), diff --git a/src/services/chat.ts b/src/services/chat.ts index cac91db34e14..b065a0704709 100644 --- a/src/services/chat.ts +++ b/src/services/chat.ts @@ -162,6 +162,13 @@ export function initializeWithClientStore(provider: string, payload: any) { case ModelProvider.ZeroOne: { break; } + case ModelProvider.Cloudflare: { + providerOptions = { + apikey: providerAuthPayload?.apiKey, + baseURLOrAccountID: providerAuthPayload?.cloudflareBaseURLOrAccountID, + }; + break; + } } /** diff --git a/src/store/user/slices/modelList/action.ts b/src/store/user/slices/modelList/action.ts index 9a6a90f8d74d..ac365e9e0c0d 100644 --- a/src/store/user/slices/modelList/action.ts +++ b/src/store/user/slices/modelList/action.ts @@ -106,6 +106,10 @@ export const createModelListSlice: StateCreator< const novita = draft.find((d) => d.id === ModelProvider.Novita); if (novita) novita.chatModels = mergeModels('novita', novita.chatModels); + + const cloudflare = draft.find((d) => d.id === ModelProvider.Cloudflare); + if (cloudflare) + cloudflare.chatModels = mergeModels('cloudflare', cloudflare.chatModels); }); set({ defaultModelProviderList }, false, `refreshDefaultModelList - ${params?.trigger}`); diff --git a/src/store/user/slices/modelList/selectors/keyVaults.ts b/src/store/user/slices/modelList/selectors/keyVaults.ts index 0a9bbb265b65..a564cfde4117 100644 --- a/src/store/user/slices/modelList/selectors/keyVaults.ts +++ b/src/store/user/slices/modelList/selectors/keyVaults.ts @@ -16,6 +16,7 @@ const openAIConfig = (s: UserStore) => keyVaultsSettings(s).openai || {}; const bedrockConfig = (s: UserStore) => keyVaultsSettings(s).bedrock || {}; const ollamaConfig = (s: UserStore) => keyVaultsSettings(s).ollama || {}; const azureConfig = (s: UserStore) => keyVaultsSettings(s).azure || {}; +const cloudflareConfig = (s: UserStore) => keyVaultsSettings(s).cloudflare || {}; const getVaultByProvider = (provider: GlobalLLMProviderKey) => (s: UserStore) => (keyVaultsSettings(s)[provider] || {}) as OpenAICompatibleKeyVault & AzureOpenAIKeyVault & @@ -36,6 +37,7 @@ const password = (s: UserStore) => keyVaultsSettings(s).password || ''; export const keyVaultsConfigSelectors = { azureConfig, bedrockConfig, + cloudflareConfig, getVaultByProvider, isProviderApiKeyNotEmpty, isProviderEndpointNotEmpty, diff --git a/src/store/user/slices/modelList/selectors/modelConfig.ts b/src/store/user/slices/modelList/selectors/modelConfig.ts index 8e2acb1421ca..1a6d9854e750 100644 --- a/src/store/user/slices/modelList/selectors/modelConfig.ts +++ b/src/store/user/slices/modelList/selectors/modelConfig.ts @@ -69,12 +69,14 @@ const openAIConfig = (s: UserStore) => currentLLMSettings(s).openai; const bedrockConfig = (s: UserStore) => currentLLMSettings(s).bedrock; const ollamaConfig = (s: UserStore) => currentLLMSettings(s).ollama; const azureConfig = (s: UserStore) => currentLLMSettings(s).azure; +const cloudflareConfig = (s: UserStore) => currentLLMSettings(s).cloudflare; const isAzureEnabled = (s: UserStore) => currentLLMSettings(s).azure.enabled; export const modelConfigSelectors = { azureConfig, bedrockConfig, + cloudflareConfig, currentEditingCustomModelCard, getCustomModelCard, diff --git a/src/types/user/settings/keyVaults.ts b/src/types/user/settings/keyVaults.ts index 46fc0db51254..82e0d300db1d 100644 --- a/src/types/user/settings/keyVaults.ts +++ b/src/types/user/settings/keyVaults.ts @@ -15,12 +15,18 @@ export interface AWSBedrockKeyVault { secretAccessKey?: string; } +export interface CloudflareKeyVault { + apiKey?: string; + baseURLOrAccountID?: string; +} + export interface UserKeyVaults { ai360?: OpenAICompatibleKeyVault; anthropic?: OpenAICompatibleKeyVault; azure?: AzureOpenAIKeyVault; baichuan?: OpenAICompatibleKeyVault; bedrock?: AWSBedrockKeyVault; + cloudflare?: CloudflareKeyVault; deepseek?: OpenAICompatibleKeyVault; google?: OpenAICompatibleKeyVault; groq?: OpenAICompatibleKeyVault;