diff --git a/README.md b/README.md index 7f3bef6..f5dd583 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ - + docker pull

@@ -13,23 +13,23 @@ ## Introduction -If used personally, one/new-api is too complex and has many commercial features that individuals do not need. If you don't want a complicated frontend interface and want to support more models, you can try uni-api. This is a project that manages large model APIs uniformly. It allows you to call multiple backend services through a unified API interface, converting them uniformly to OpenAI format and supporting load balancing. The currently supported backend services include: OpenAI, Anthropic, Gemini, Vertex, DeepBricks, OpenRouter, etc. +If used personally, one/new-api is overly complex, with many commercial features that individuals do not need. If you do not want a complex front-end interface and want to support more models, you can try uni-api. This is a project for unified management of large model APIs, allowing you to call multiple backend services through a unified API interface, uniformly converting them to OpenAI format and supporting load balancing. The currently supported backend services include: OpenAI, Anthropic, Gemini, Vertex, Cloudflare, DeepBricks, OpenRouter, etc. ## Features -- No front-end, purely configuration file to set up API channels. You can run your own API site just by writing a file. The documentation has detailed configuration guidelines, friendly for beginners. -- Unified management of multiple backend services, supporting OpenAI, Deepseek, DeepBricks, OpenRouter, and other API providers in the OpenAI format. Supports OpenAI Dalle-3 image generation. -- Supports Anthropic, Gemini, Vertex API simultaneously. Vertex supports both Claude and Gemini API. -- Support native tool use function calls for OpenAI, Anthropic, Gemini, Vertex. +- No front-end, pure configuration file setup for API channels. You can run your own API site just by writing a single file, and the documentation includes a detailed configuration guide, beginner-friendly. +- Unified management of multiple backend services, supporting providers such as OpenAI, Deepseek, DeepBricks, OpenRouter, and other APIs in OpenAI format. Supports OpenAI Dalle-3 image generation. +- Supports Anthropic, Gemini, Vertex API, and Cloudflare simultaneously. Vertex supports both Claude and Gemini API. +- Support for OpenAI, Anthropic, Gemini, Vertex native tool use function calls. - Supports OpenAI, Anthropic, Gemini, Vertex native image recognition API. - Supports four types of load balancing. - 1. Supports channel-level weighted load balancing, which can allocate requests based on different channel weights. By default, it is not enabled and requires channel weight configuration. - 2. Supports Vertex region-level load balancing, supports Vertex high concurrency, and can increase Gemini and Claude concurrency by up to (number of APIs * number of regions) times. Automatically enabled without additional configuration. + 1. Supports channel-level weighted load balancing, which can allocate requests based on different channel weights. Disabled by default, channel weights need to be configured. + 2. Supports Vertex regional load balancing, supports Vertex high concurrency, and can increase Gemini and Claude concurrency up to (API quantity * regional quantity) times. Automatically enabled without additional configuration. 3. Except for Vertex region-level load balancing, all APIs support channel-level sequential load balancing, enhancing the immersive translation experience. Automatically enabled without additional configuration. - 4. Support automatic API key-level polling load balancing for multiple API Keys in a single channel. -- Supports automatic retry. When an API channel response fails, automatically retry the next API channel. -- Supports fine-grained access control. Supports using wildcards to set specific models available for API key channels. -- Supports rate limiting, can set the maximum number of requests per minute, can be set as an integer, such as 2/min, 2 times per minute, 5/hour, 5 times per hour, 10/day, 10 times per day, 10/month, 10 times per month, 10/year, 10 times per year. Default is 60/min. + 4. Support automatic API key-level round-robin load balancing for multiple API keys in a single channel. +- Supports automatic retry, when an API channel response fails, automatically retry the next API channel. +- Supports fine-grained permission control. Supports using wildcards to set specific models available for API key channels. +- Supports rate limiting, allowing you to set the maximum number of requests per minute. It can be set as an integer, such as 2/min (2 times per minute), 5/hour (5 times per hour), 10/day (10 times per day), 10/month (10 times per month), 10/year (10 times per year). The default is 60/min. ## Configuration @@ -37,21 +37,21 @@ Using the api.yaml configuration file, you can configure multiple models, and ea ```yaml providers: - - provider: provider_name # Service provider name, such as openai, anthropic, gemini, openrouter, deepbricks, arbitrary name, required + - provider: provider_name # Service provider name, such as openai, anthropic, gemini, openrouter, deepbricks, any name, required base_url: https://api.your.com/v1/chat/completions # Backend service API address, required api: sk-YgS6GTi0b4bEabc4C # Provider's API Key, required - model: # At least one model + model: # At least one model is required - gpt-4o # Usable model name, required - - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, can use a short name instead of the original complex name, optional + - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, you can use a simpler name instead of the original complex name, optional - dall-e-3 - provider: anthropic base_url: https://api.anthropic.com/v1/messages - api: # Supports multiple API Keys, multiple keys automatically enable polling load balancing, at least one key, required + api: # Supports multiple API Keys, multiple keys automatically enable round-robin load balancing, at least one key, required - sk-ant-api03-bNnAOJyA-xQw_twAA - sk-ant-api02-bNnxxxx model: - - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, can use a short name instead of the original complex name, optional + - claude-3-5-sonnet-20240620: claude-3-5-sonnet # Rename model, claude-3-5-sonnet-20240620 is the provider's model name, claude-3-5-sonnet is the renamed name, you can use a simpler name instead of the original complex name, optional tools: true # Whether to support tools, such as generating code, generating documents, etc., default is true, optional - provider: gemini @@ -59,14 +59,14 @@ providers: api: AIzaSyAN2k6IRdgw model: - gemini-1.5-pro - - gemini-1.5-flash-exp-0827: gemini-1.5-flash # After renaming, the original model name gemini-1.5-flash-exp-0827 cannot be used, if you want to use the original name, you can add the original name in the model, just add the following line to use the original name + - gemini-1.5-flash-exp-0827: gemini-1.5-flash # After renaming, the original model name gemini-1.5-flash-exp-0827 cannot be used. If you want to use the original name, you can add the original name in the model, just add the line below to use the original name. - gemini-1.5-flash-exp-0827 # Add this line, both gemini-1.5-flash-exp-0827 and gemini-1.5-flash can be requested tools: true - provider: vertex - project_id: gen-lang-client-xxxxxxxxxxxxxx # Description: Your Google Cloud project ID. Format: String, usually composed of lowercase letters, numbers, and hyphens. How to obtain: You can find your project ID in the project selector of the Google Cloud Console. - private_key: "-----BEGIN PRIVATE KEY-----\nxxxxx\n-----END PRIVATE" # Description: The private key of the Google Cloud Vertex AI service account. Format: A JSON formatted string containing the private key information of the service account. How to obtain: Create a service account in the Google Cloud Console, generate a JSON formatted key file, and then set its content as the value of this environment variable. - client_email: xxxxxxxxxx@xxxxxxx.gserviceaccount.com # Description: The email address of the Google Cloud Vertex AI service account. Format: Usually a string like "service-account-name@project-id.iam.gserviceaccount.com". How to obtain: Generated when creating the service account, you can also view the service account details in the "IAM & Admin" section of the Google Cloud Console. + project_id: gen-lang-client-xxxxxxxxxxxxxx # Description: Your Google Cloud project ID. Format: String, usually consists of lowercase letters, numbers, and hyphens. How to get: You can find your project ID in the project selector of the Google Cloud Console. + private_key: "-----BEGIN PRIVATE KEY-----\nxxxxx\n-----END PRIVATE" # Description: Private key of Google Cloud Vertex AI service account. Format: A JSON formatted string containing the private key information of the service account. How to get: Create a service account in the Google Cloud Console, generate a JSON formatted key file, and set its content as the value of this environment variable. + client_email: xxxxxxxxxx@xxxxxxx.gserviceaccount.com # Description: Email address of the Google Cloud Vertex AI service account. Format: Usually a string like "service-account-name@project-id.iam.gserviceaccount.com". How to get: Generated when creating the service account, or you can view the service account details in the "IAM & Admin" section of the Google Cloud Console. model: - gemini-1.5-pro - gemini-1.5-flash @@ -75,7 +75,14 @@ providers: - claude-3-sonnet@20240229: claude-3-sonnet - claude-3-haiku@20240307: claude-3-haiku tools: true - notes: https://xxxxx.com/ # Can put the provider's website, notes, official documentation, optional + notes: https://xxxxx.com/ # You can put the provider's website, notes, official documentation, optional + + - provider: cloudflare + api: f42b3xxxxxxxxxxq4aoGAh # Cloudflare API Key, required + cf_account_id: 8ec0xxxxxxxxxxxxe721 # Cloudflare Account ID, required + model: + - '@cf/meta/llama-3.1-8b-instruct': llama-3.1-8b # Rename model, @cf/meta/llama-3.1-8b-instruct is the provider's original model name, must be enclosed in quotes otherwise YAML syntax error, llama-3.1-8b is the renamed name, you can use a simpler name instead of the original complex name, optional + - '@cf/meta/llama-3.1-8b-instruct' # Must be enclosed in quotes otherwise YAML syntax error - provider: other-provider base_url: https://api.xxx.com/v1/messages @@ -95,28 +102,28 @@ api_keys: - api: sk-pkhf60Yf0JGyJygRmXqFQyTgWUd9GZnmi3KlvowmRWpWqrhy model: - - anthropic/claude-3-5-sonnet # Usable model name, can only use the claude-3-5-sonnet model provided by the provider named anthropic. Models of other providers' claude-3-5-sonnet cannot be used. + - anthropic/claude-3-5-sonnet # Usable model name, can only use the claude-3-5-sonnet model provided by the provider named anthropic. Models with the same name from other providers cannot be used. preferences: - USE_ROUND_ROBIN: true # Whether to use polling load balancing, true to use, false to not use, default is true. When polling is enabled, each request model is requested in the order configured in the model. It is not related to the original channel order in providers. Therefore, you can set different request sequences for each API key. + USE_ROUND_ROBIN: true # Whether to use round-robin load balancing, true to use, false to not use, default is true. When enabled, each request to the model is made in the order configured in the model. This is independent of the original channel order in providers. Therefore, you can set different request orders for each API key. AUTO_RETRY: true # Whether to automatically retry, automatically retry the next provider, true to automatically retry, false to not automatically retry, default is true RATE_LIMIT: 2/min # Supports rate limiting, maximum number of requests per minute, can be set to an integer, such as 2/min, 2 times per minute, 5/hour, 5 times per hour, 10/day, 10 times per day, 10/month, 10 times per month, 10/year, 10 times per year. Default is 60/min, optional # Channel-level weighted load balancing configuration example - api: sk-KjjI60Yf0JFWtxxxxxxxxxxxxxxwmRWpWpQRo model: - - gcp1/*: 5 # The number after the colon is the weight, weights only support positive integers. - - gcp2/*: 3 # The larger the number, the greater the probability of being requested. - - gcp3/*: 2 # In this example, there are a total of 10 weights for all channels, and 5 out of 10 requests will request the gcp1/* model, 2 requests will request the gcp2/* model, and 3 requests will request the gcp3/* model. + - gcp1/*: 5 # The number after the colon is the weight, only positive integers are supported. + - gcp2/*: 3 # The larger the number, the higher the probability of the request. + - gcp3/*: 2 # In this example, there are a total of 10 weights for all channels, and out of 10 requests, 5 requests will request the gcp1/* model, 2 requests will request the gcp2/* model, and 3 requests will request the gcp3/* model. preferences: - USE_ROUND_ROBIN: true # When USE_ROUND_ROBIN must be true and there is no weight after the channels above, it will request in the original channel order, if there is weight, it will request in the weighted order. + USE_ROUND_ROBIN: true # When USE_ROUND_ROBIN must be true and there is no weight after the above channels, requests will be made in the original channel order. If there are weights, requests will be made in the weighted order. AUTO_RETRY: true ``` -## Environment variables +## Environment Variables - CONFIG_URL: The download address of the configuration file, it can be a local file or a remote file, optional -- TIMEOUT: Request timeout, default is 20 seconds. The timeout can control the time needed to switch to the next channel when a channel does not respond. Optional +- TIMEOUT: Request timeout, default is 20 seconds, the timeout can control the time needed to switch to the next channel when a channel does not respond. Optional ## Docker Local Deployment @@ -143,7 +150,7 @@ services: - ./api.yaml:/home/api.yaml ``` -CONFIG_URL is a direct link that can automatically download remote configuration files. For example, if you find it inconvenient to modify configuration files on a certain platform, you can upload the configuration file to a hosting service and provide a direct link for uni-api to download. CONFIG_URL is this direct link. +CONFIG_URL is a direct link that can automatically download remote configuration files. For instance, if you find it inconvenient to modify configuration files on a certain platform, you can upload the configuration files to a hosting service that provides a direct link for uni-api to download. CONFIG_URL is this direct link. Run Docker Compose container in the background diff --git a/README_CN.md b/README_CN.md index 01bfc3e..a813bf5 100644 --- a/README_CN.md +++ b/README_CN.md @@ -13,13 +13,13 @@ ## Introduction -如果个人使用的话，one/new-api 过于复杂，有很多个人不需要使用的商用功能，如果你不想要复杂的前端界面，有想要支持的模型多一点，可以试试 uni-api。这是一个统一管理大模型API的项目，可以通过一个统一的API接口调用多个后端服务，统一转换为 OpenAI 格式，支持负载均衡。目前支持的后端服务有：OpenAI、Anthropic、Gemini、Vertex、DeepBricks、OpenRouter 等。 +如果个人使用的话，one/new-api 过于复杂，有很多个人不需要使用的商用功能，如果你不想要复杂的前端界面，有想要支持的模型多一点，可以试试 uni-api。这是一个统一管理大模型API的项目，可以通过一个统一的API接口调用多个后端服务，统一转换为 OpenAI 格式，支持负载均衡。目前支持的后端服务有：OpenAI、Anthropic、Gemini、Vertex、cloudflare、DeepBricks、OpenRouter 等。 ## Features - 无前端，纯配置文件配置 API 渠道。只要写一个文件就能运行起一个属于自己的 API 站，文档有详细的配置指南，小白友好。 -- 统一管理多个后端服务，支持 OpenAI、Deepseek、DeepBricks、OpenRouter 等其他API 是 OpenAI 格式的提供商。支持 OpenAI Dalle-3 图像生成。 -- 同时支持 Anthropic、Gemini、Vertex API。Vertex 同时支持 Claude 和 Gemini API。 +- 统一管理多个后端服务，支持 OpenAI、Deepseek、DeepBricks、OpenRouter 等其他 API 是 OpenAI 格式的提供商。支持 OpenAI Dalle-3 图像生成。 +- 同时支持 Anthropic、Gemini、Vertex API、cloudflare。Vertex 同时支持 Claude 和 Gemini API。 - 支持 OpenAI、 Anthropic、Gemini、Vertex 原生 tool use 函数调用。 - 支持 OpenAI、Anthropic、Gemini、Vertex 原生识图 API。 - 支持四种负载均衡。 @@ -77,6 +77,13 @@ providers: tools: true notes: https://xxxxx.com/ # 可以放服务商的网址，备注信息，官方文档，选填 + - provider: cloudflare + api: f42b3xxxxxxxxxxq4aoGAh # Cloudflare API Key，必填 + cf_account_id: 8ec0xxxxxxxxxxxxe721 # Cloudflare Account ID，必填 + model: + - '@cf/meta/llama-3.1-8b-instruct': llama-3.1-8b # 重命名模型，@cf/meta/llama-3.1-8b-instruct 是服务商的原始的模型名称，必须使用引号包裹模型名，否则yaml语法错误，llama-3.1-8b 是重命名后的名字，可以使用简洁的名字代替原来复杂的名称，选填 + - '@cf/meta/llama-3.1-8b-instruct' # 必须使用引号包裹模型名，否则yaml语法错误 + - provider: other-provider base_url: https://api.xxx.com/v1/messages api: sk-bNnAOJyA-xQw_twAA diff --git a/main.py b/main.py index 1450b7c..e84998f 100644 --- a/main.py +++ b/main.py @@ -174,11 +174,14 @@ async def cleanup(self): async def process_request(request: Union[RequestModel, ImageGenerationRequest], provider: Dict, endpoint=None): url = provider['base_url'] parsed_url = urlparse(url) + # print("parsed_url", parsed_url) engine = None if parsed_url.netloc == 'generativelanguage.googleapis.com': engine = "gemini" elif parsed_url.netloc == 'aiplatform.googleapis.com': engine = "vertex" + elif parsed_url.netloc == 'api.cloudflare.com': + engine = "cloudflare" elif parsed_url.netloc == 'api.anthropic.com' or parsed_url.path.endswith("v1/messages"): engine = "claude" elif parsed_url.netloc == 'openrouter.ai': @@ -188,7 +191,8 @@ async def process_request(request: Union[RequestModel, ImageGenerationRequest], if "claude" not in provider['model'][request.model] \ and "gpt" not in provider['model'][request.model] \ - and "gemini" not in provider['model'][request.model]: + and "gemini" not in provider['model'][request.model] \ + and parsed_url.netloc != 'api.cloudflare.com': engine = "openrouter" if "claude" in provider['model'][request.model] and engine == "vertex": @@ -311,7 +315,7 @@ def get_matching_providers(self, model_name, token): # import json # for provider in provider_list: - # print(json.dumps(provider, indent=4, ensure_ascii=False)) + # print(json.dumps(provider, indent=4, ensure_ascii=False, default=circular_list_encoder)) return provider_list async def request_model(self, request: Union[RequestModel, ImageGenerationRequest], token: str, endpoint=None): diff --git a/request.py b/request.py index 2068d82..da51b9d 100644 --- a/request.py +++ b/request.py @@ -33,6 +33,8 @@ async def get_text_message(role, message, engine = None): return {"type": "text", "text": message} if "gemini" == engine or "vertex-gemini" == engine: return {"text": message} + if engine == "cloudflare": + return message raise ValueError("Unknown engine") async def get_gemini_payload(request, engine, provider): @@ -640,6 +642,55 @@ async def get_openrouter_payload(request, engine, provider): return url, headers, payload +async def get_cloudflare_payload(request, engine, provider): + headers = { + 'Content-Type': 'application/json' + } + if provider.get("api"): + headers['Authorization'] = f"Bearer {provider['api'].next()}" + + model = provider['model'][request.model] + url = "https://api.cloudflare.com/client/v4/accounts/{cf_account_id}/ai/run/{cf_model_id}".format(cf_account_id=provider['cf_account_id'], cf_model_id=model) + + msg = request.messages[-1] + messages = [] + content = None + if isinstance(msg.content, list): + for item in msg.content: + if item.type == "text": + content = await get_text_message(msg.role, item.text, engine) + else: + content = msg.content + name = msg.name + + model = provider['model'][request.model] + payload = { + "prompt": content, + } + + miss_fields = [ + 'model', + 'messages', + 'tools', + 'tool_choice', + 'temperature', + 'top_p', + 'max_tokens', + 'presence_penalty', + 'frequency_penalty', + 'n', + 'user', + 'include_usage', + 'logprobs', + 'top_logprobs' + ] + + for field, value in request.model_dump(exclude_unset=True).items(): + if field not in miss_fields and value is not None: + payload[field] = value + + return url, headers, payload + async def gpt2claude_tools_json(json_dict): import copy json_dict = copy.deepcopy(json_dict) @@ -830,6 +881,8 @@ async def get_payload(request: RequestModel, engine, provider): return await get_gpt_payload(request, engine, provider) elif engine == "openrouter": return await get_openrouter_payload(request, engine, provider) + elif engine == "cloudflare": + return await get_cloudflare_payload(request, engine, provider) elif engine == "dalle": return await get_dalle_payload(request, engine, provider) else: diff --git a/response.py b/response.py index 8ee5a98..d33d2fc 100644 --- a/response.py +++ b/response.py @@ -112,7 +112,7 @@ async def fetch_vertex_claude_response_stream(client, url, headers, payload, mod buffer += chunk while "\n" in buffer: line, buffer = buffer.split("\n", 1) - logger.info(f"{line}") + # logger.info(f"{line}") if line and '\"text\": \"' in line: try: json_data = json.loads( "{" + line + "}") @@ -143,7 +143,7 @@ async def fetch_vertex_claude_response_stream(client, url, headers, payload, mod yield sse_string yield "data: [DONE]\n\r\n" -async def fetch_gpt_response_stream(client, url, headers, payload, max_redirects=5): +async def fetch_gpt_response_stream(client, url, headers, payload): async with client.stream('POST', url, headers=headers, json=payload) as response: error_message = await check_response(response, "fetch_gpt_response_stream") if error_message: @@ -159,6 +159,31 @@ async def fetch_gpt_response_stream(client, url, headers, payload, max_redirects if line and line != "data: " and line != "data:" and not line.startswith(": "): yield line.strip() + "\n\r\n" +async def fetch_cloudflare_response_stream(client, url, headers, payload, model): + timestamp = int(datetime.timestamp(datetime.now())) + async with client.stream('POST', url, headers=headers, json=payload) as response: + error_message = await check_response(response, "fetch_gpt_response_stream") + if error_message: + yield error_message + return + + buffer = "" + async for chunk in response.aiter_text(): + buffer += chunk + while "\n" in buffer: + line, buffer = buffer.split("\n", 1) + # logger.info("line: %s", repr(line)) + if line.startswith("data:"): + line = line.lstrip("data: ") + if line == "[DONE]": + yield "data: [DONE]\n\r\n" + return + resp: dict = json.loads(line) + message = resp.get("response") + if message: + sse_string = await generate_sse_response(timestamp, model, content=message) + yield sse_string + async def fetch_claude_response_stream(client, url, headers, payload, model): timestamp = int(datetime.timestamp(datetime.now())) async with client.stream('POST', url, headers=headers, json=payload) as response: @@ -242,6 +267,9 @@ async def fetch_response_stream(client, url, headers, payload, engine, model): elif engine == "openrouter": async for chunk in fetch_gpt_response_stream(client, url, headers, payload): yield chunk + elif engine == "cloudflare": + async for chunk in fetch_cloudflare_response_stream(client, url, headers, payload, model): + yield chunk else: raise ValueError("Unknown response") except httpx.ConnectError as e: diff --git a/utils.py b/utils.py index e376ede..366a3a2 100644 --- a/utils.py +++ b/utils.py @@ -15,6 +15,8 @@ def update_config(config_data): provider['model'] = model_dict if provider.get('project_id'): provider['base_url'] = 'https://aiplatform.googleapis.com/' + if provider.get('cf_account_id'): + provider['base_url'] = 'https://api.cloudflare.com/' if provider.get('api'): if isinstance(provider.get('api'), str):