From c8341f77e1bc27c9914434fa536f985a3b4c95a5 Mon Sep 17 00:00:00 2001 From: Angel Date: Thu, 26 Sep 2024 16:38:24 -0400 Subject: [PATCH] custom model dosc --- .../_anthropic.md | 1 + .../llm-provider-integration-guides/_azure.md | 2 ++ .../_cohere.md | 2 ++ .../_llama2.md | 1 + .../_mistral.md | 1 + .../_openai.md | 2 ++ .../_anthropic.md | 1 + .../llm-provider-integration-guides/_azure.md | 1 + .../_cohere.md | 2 ++ .../_llama2.md | 2 ++ .../_mistral.md | 1 + .../_openai.md | 1 + .../plugins-hub/ai-custom-model-advanced.md | 32 +++++++++++++++++++ .../md/plugins-hub/ai-custom-model.md | 32 +++++++++++++++++++ 14 files changed, 81 insertions(+) create mode 100644 app/_includes/md/plugins-hub/ai-custom-model-advanced.md create mode 100644 app/_includes/md/plugins-hub/ai-custom-model.md diff --git a/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_anthropic.md b/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_anthropic.md index 876a0ee830ab..f99b6b054216 100644 --- a/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_anthropic.md +++ b/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_anthropic.md @@ -66,3 +66,4 @@ curl -X POST http://localhost:8000/anthropic-chat \ -H 'Content-Type: application/json' \ --data-raw '{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }' ``` +{% include_cached /md/plugins-hub/ai-custom-model-advanced.md %} \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_azure.md b/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_azure.md index df5e3d6bf822..e33775c1e536 100644 --- a/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_azure.md +++ b/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_azure.md @@ -77,3 +77,5 @@ curl -X POST http://localhost:8000/azure-chat \ -H 'Content-Type: application/json' \ --data-raw '{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }' ``` + +{% include_cached /md/plugins-hub/ai-custom-model-advanced.md %} \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_cohere.md b/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_cohere.md index 4ee33a5b4286..0dcc43eae122 100644 --- a/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_cohere.md +++ b/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_cohere.md @@ -64,3 +64,5 @@ curl -X POST http://localhost:8000/cohere-chat \ -H 'Content-Type: application/json' \ --data-raw '{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }' ``` + +{% include_cached /md/plugins-hub/ai-custom-model-advanced.md %} \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_llama2.md b/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_llama2.md index 3548e41c38af..cb7a5ff1b4d1 100644 --- a/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_llama2.md +++ b/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_llama2.md @@ -112,3 +112,4 @@ curl -X POST http://localhost:8000/llama2-chat \ -H 'Content-Type: application/json' \ --data-raw '{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }' ``` +{% include_cached /md/plugins-hub/ai-custom-model-advanced.md %} \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_mistral.md b/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_mistral.md index 24743c78d150..ec38ac8bf855 100644 --- a/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_mistral.md +++ b/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_mistral.md @@ -103,3 +103,4 @@ curl -X POST http://localhost:8000/mistral-chat \ -H 'Content-Type: application/json' \ --data-raw '{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }' ``` +{% include_cached /md/plugins-hub/ai-custom-model-advanced.md %} \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_openai.md b/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_openai.md index 43356bbfa2a0..ce12ce54e61d 100644 --- a/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_openai.md +++ b/app/_hub/kong-inc/ai-proxy-advanced/how-to/llm-provider-integration-guides/_openai.md @@ -55,6 +55,7 @@ formats: {% endplugin_example %} + ### Test the configuration Make an `llm/v1/chat` type request to test your new endpoint: @@ -64,3 +65,4 @@ curl -X POST http://localhost:8000/openai-chat \ -H 'Content-Type: application/json' \ --data-raw '{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }' ``` +{% include_cached /md/plugins-hub/ai-custom-model-advanced.md %} \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_anthropic.md b/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_anthropic.md index 6df8617d5251..4aba06abaa48 100644 --- a/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_anthropic.md +++ b/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_anthropic.md @@ -82,3 +82,4 @@ curl -X POST http://localhost:8000/anthropic-chat \ -H 'Content-Type: application/json' \ --data-raw '{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }' ``` +{% include_cached /md/plugins-hub/ai-custom-model.md %} \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_azure.md b/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_azure.md index 765f6dbf683f..656545a64656 100644 --- a/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_azure.md +++ b/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_azure.md @@ -89,3 +89,4 @@ curl -X POST http://localhost:8000/azure-chat \ -H 'Content-Type: application/json' \ --data-raw '{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }' ``` +{% include_cached /md/plugins-hub/ai-custom-model.md %} \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_cohere.md b/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_cohere.md index c53952862a36..21f09f5fd9c8 100644 --- a/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_cohere.md +++ b/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_cohere.md @@ -76,3 +76,5 @@ curl -X POST http://localhost:8000/cohere-chat \ -H 'Content-Type: application/json' \ --data-raw '{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }' ``` + +{% include_cached /md/plugins-hub/ai-custom-model.md %} \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_llama2.md b/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_llama2.md index ae7e344223ea..d7c40f0ef4af 100644 --- a/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_llama2.md +++ b/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_llama2.md @@ -123,3 +123,5 @@ curl -X POST http://localhost:8000/llama2-chat \ -H 'Content-Type: application/json' \ --data-raw '{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }' ``` + +{% include_cached /md/plugins-hub/ai-custom-model.md %} \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_mistral.md b/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_mistral.md index 05030b206825..25e764f425d9 100644 --- a/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_mistral.md +++ b/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_mistral.md @@ -115,3 +115,4 @@ curl -X POST http://localhost:8000/mistral-chat \ -H 'Content-Type: application/json' \ --data-raw '{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }' ``` +{% include_cached /md/plugins-hub/ai-custom-model.md %} \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_openai.md b/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_openai.md index 4bde4359d404..e058cf6011ee 100644 --- a/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_openai.md +++ b/app/_hub/kong-inc/ai-proxy/how-to/llm-provider-integration-guides/_openai.md @@ -76,3 +76,4 @@ curl -X POST http://localhost:8000/openai-chat \ -H 'Content-Type: application/json' \ --data-raw '{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] }' ``` +{% include_cached /md/plugins-hub/ai-custom-model.md %} \ No newline at end of file diff --git a/app/_includes/md/plugins-hub/ai-custom-model-advanced.md b/app/_includes/md/plugins-hub/ai-custom-model-advanced.md new file mode 100644 index 000000000000..dd6fedf84e46 --- /dev/null +++ b/app/_includes/md/plugins-hub/ai-custom-model-advanced.md @@ -0,0 +1,32 @@ + +For all providers, the Kong AI Proxy Advanced plugin attaches to **route** entities. + +### Custom model + +You can configure the AI Proxy Advanced Plugin using a custom model of your choice by setting the `name` and `upstream_url` when configuring the model. + + +{% plugin_example %} +plugin: kong-inc/ai-proxy-advanced +name: ai-proxy-advanced +config: + targets: + - route_type: "llm/v1/chat" + auth: + header_name: "Authorization" + header_value: "Bearer " + - model: + name: custom_model_name + provider: openai|azure|anthropic|cohere|mistral|llama2|gemini|bedrock + options: + upstream_url: http://localhost:8000/vi/chat/completions +targets: + - route +formats: + - curl + - konnect + - yaml + - kubernetes + - terraform +{% endplugin_example %} + diff --git a/app/_includes/md/plugins-hub/ai-custom-model.md b/app/_includes/md/plugins-hub/ai-custom-model.md new file mode 100644 index 000000000000..dd6fedf84e46 --- /dev/null +++ b/app/_includes/md/plugins-hub/ai-custom-model.md @@ -0,0 +1,32 @@ + +For all providers, the Kong AI Proxy Advanced plugin attaches to **route** entities. + +### Custom model + +You can configure the AI Proxy Advanced Plugin using a custom model of your choice by setting the `name` and `upstream_url` when configuring the model. + + +{% plugin_example %} +plugin: kong-inc/ai-proxy-advanced +name: ai-proxy-advanced +config: + targets: + - route_type: "llm/v1/chat" + auth: + header_name: "Authorization" + header_value: "Bearer " + - model: + name: custom_model_name + provider: openai|azure|anthropic|cohere|mistral|llama2|gemini|bedrock + options: + upstream_url: http://localhost:8000/vi/chat/completions +targets: + - route +formats: + - curl + - konnect + - yaml + - kubernetes + - terraform +{% endplugin_example %} +