From 53cc3ccd46e37000f8576cd8b0d59be224eea655 Mon Sep 17 00:00:00 2001 From: CH3CHO Date: Fri, 8 Nov 2024 18:20:56 +0800 Subject: [PATCH 1/2] feat: Enhance AI console functions 1. Support more LLM providers. 2. Move model mapping from LlmProvider to AiRoute. 3. Update the model predicates in AiRoute. --- .github/workflows/build-and-test.yaml | 2 +- backend/build.sh | 2 +- .../WasmPluginInstancesController.java | 7 +- .../sdk/constant/HigressConstants.java | 1 - .../higress/sdk/constant/Separators.java | 2 + .../{ => plugin}/BuiltInPluginName.java | 3 +- .../constant/plugin/config/AiProxyConfig.java | 33 ++ .../constant/plugin/config/KeyAuthConfig.java | 26 ++ .../plugin/config/ModelMapperConfig.java | 18 + .../plugin/config/ModelRouterConfig.java | 18 + .../higress/sdk/model/WasmPluginInstance.java | 50 +++ .../sdk/model/WasmPluginInstanceScope.java | 3 + .../sdk/model/ai/AiModelPredicate.java | 24 +- .../alibaba/higress/sdk/model/ai/AiRoute.java | 14 +- .../higress/sdk/model/ai/AiUpstream.java | 3 + .../higress/sdk/model/ai/LlmProvider.java | 2 +- .../higress/sdk/model/ai/LlmProviderType.java | 44 +++ .../sdk/model/route/RoutePredicate.java | 15 + .../model/route/RoutePredicateTypeEnum.java | 12 +- .../service/HigressServiceProviderImpl.java | 7 +- .../service/WasmPluginInstanceService.java | 9 + .../WasmPluginInstanceServiceImpl.java | 73 +++- .../ai/AbstractLlmProviderHandler.java | 144 +++++--- .../sdk/service/ai/AiRouteServiceImpl.java | 131 +++++-- .../service/ai/AzureLlmProviderHandler.java | 85 +++++ .../service/ai/DefaultLlmProviderHandler.java | 32 +- .../sdk/service/ai/LlmProviderHandler.java | 9 +- .../service/ai/LlmProviderServiceImpl.java | 94 ++--- .../service/ai/OllamaLlmProviderHandler.java | 94 +++++ .../service/consumer/ConsumerServiceImpl.java | 35 +- .../consumer/KeyAuthCredentialHandler.java | 62 ++-- .../kubernetes/KubernetesModelConverter.java | 334 +++++++++--------- .../kubernetes/crd/mcp/V1McpBridge.java | 2 + .../kubernetes/crd/wasm/MatchRule.java | 25 +- .../alibaba/higress/sdk/util/ListUtil.java | 31 ++ .../com/alibaba/higress/sdk/util/MapUtil.java | 28 ++ .../resources/plugins/model-mapper/README.md | 63 ++++ .../plugins/model-mapper/README_EN.md | 65 ++++ .../resources/plugins/model-mapper/spec.yaml | 55 +++ .../resources/plugins/model-router/README.md | 62 +++- .../plugins/model-router/README_EN.md | 74 ++-- .../resources/plugins/model-router/spec.yaml | 35 +- .../main/resources/plugins/plugins.properties | 1 + .../WasmPluginInstanceServiceTest.java | 36 +- .../KeyAuthCredentialHandlerTest.java | 70 ++-- .../KubernetesModelConverterTest.java | 20 +- 46 files changed, 1401 insertions(+), 554 deletions(-) rename backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/{ => plugin}/BuiltInPluginName.java (96%) create mode 100644 backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/plugin/config/AiProxyConfig.java create mode 100644 backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/plugin/config/KeyAuthConfig.java create mode 100644 backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/plugin/config/ModelMapperConfig.java create mode 100644 backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/plugin/config/ModelRouterConfig.java create mode 100644 backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/AzureLlmProviderHandler.java create mode 100644 backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/OllamaLlmProviderHandler.java create mode 100644 backend/sdk/src/main/java/com/alibaba/higress/sdk/util/ListUtil.java create mode 100644 backend/sdk/src/main/java/com/alibaba/higress/sdk/util/MapUtil.java create mode 100644 backend/sdk/src/main/resources/plugins/model-mapper/README.md create mode 100644 backend/sdk/src/main/resources/plugins/model-mapper/README_EN.md create mode 100644 backend/sdk/src/main/resources/plugins/model-mapper/spec.yaml diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index baae1a20..d48efb33 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -22,7 +22,7 @@ jobs: fetch-depth: 1 - name: Build Higress Console Package - run: mvn clean package -f ./backend/pom.xml + run: mvn clean package -f ./backend/pom.xml -Dpmd.language=en - name: Upload Higress Console Package uses: actions/upload-artifact@v3 diff --git a/backend/build.sh b/backend/build.sh index 732a5720..8467984d 100755 --- a/backend/build.sh +++ b/backend/build.sh @@ -6,5 +6,5 @@ fi if [ -n "$DEV" ]; then BUILD_ARGS="$BUILD_ARGS -Dapp.build.dev=$DEV" fi -./mvnw clean package -Dmaven.test.skip=true $BUILD_ARGS +./mvnw clean package -Dmaven.test.skip=true -Dpmd.language=en $BUILD_ARGS docker build -t higress-console:0.0.1 -f Dockerfile . \ No newline at end of file diff --git a/backend/console/src/main/java/com/alibaba/higress/console/controller/WasmPluginInstancesController.java b/backend/console/src/main/java/com/alibaba/higress/console/controller/WasmPluginInstancesController.java index c2df4c93..6c6865ea 100644 --- a/backend/console/src/main/java/com/alibaba/higress/console/controller/WasmPluginInstancesController.java +++ b/backend/console/src/main/java/com/alibaba/higress/console/controller/WasmPluginInstancesController.java @@ -214,8 +214,8 @@ private ResponseEntity> queryInstance(WasmPluginIns String name) { WasmPluginInstance instance = wasmPluginInstanceService.query(scope, target, name, false); if (instance == null) { - instance = WasmPluginInstance.builder().scope(scope).target(target).pluginName(name).internal(false) - .enabled(false).build(); + instance = WasmPluginInstance.builder().pluginName(name).internal(false).enabled(false).build(); + instance.setTarget(scope, target); } return ControllerUtil.buildResponseEntity(instance); } @@ -234,8 +234,7 @@ private ResponseEntity> addOrUpdateInstance(WasmPlu if (plugin == null) { throw new ValidationException("Unsupported plugin: " + name); } - instance.setScope(scope); - instance.setTarget(target); + instance.setTarget(scope, target); instance = wasmPluginInstanceService.addOrUpdate(instance); return ControllerUtil.buildResponseEntity(instance); } diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/HigressConstants.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/HigressConstants.java index be764c8c..982d76ac 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/HigressConstants.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/HigressConstants.java @@ -23,5 +23,4 @@ public class HigressConstants { public static final String INTERNAL_RESOURCE_NAME_SUFFIX = ".internal"; public static final String FALLBACK_ROUTE_NAME_SUFFIX = ".fallback"; public static final String FALLBACK_FROM_HEADER = "x-higress-fallback-from"; - public static final String MODEL_ROUTER_HEADER = "x-higress-llm-provider"; } diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/Separators.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/Separators.java index 6c2bc36a..71f50d86 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/Separators.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/Separators.java @@ -20,6 +20,8 @@ public class Separators { public static final String ASTERISK = "*"; + public static final String DOT = "."; + public static final String COMMA = ","; public static final String EQUALS_SIGN = "="; diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/BuiltInPluginName.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/plugin/BuiltInPluginName.java similarity index 96% rename from backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/BuiltInPluginName.java rename to backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/plugin/BuiltInPluginName.java index 14874f4e..3f4a6720 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/BuiltInPluginName.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/plugin/BuiltInPluginName.java @@ -10,7 +10,7 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package com.alibaba.higress.sdk.constant; +package com.alibaba.higress.sdk.constant.plugin; public final class BuiltInPluginName { @@ -29,6 +29,7 @@ public final class BuiltInPluginName { public static final String AI_QUOTA = "ai-quota"; public static final String AI_AGENT = "ai-agent"; public static final String MODEL_ROUTER = "model-router"; + public static final String MODEL_MAPPER = "model-mapper"; // Auth public static final String BASIC_AUTH = "basic-auth"; diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/plugin/config/AiProxyConfig.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/plugin/config/AiProxyConfig.java new file mode 100644 index 00000000..9525b472 --- /dev/null +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/plugin/config/AiProxyConfig.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2022-2024 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.alibaba.higress.sdk.constant.plugin.config; + +public class AiProxyConfig { + + public static final String ACTIVE_PROVIDER_ID = "activeProviderId"; + + public static final String PROVIDERS = "providers"; + public static final String PROVIDER_ID = "id"; + public static final String PROVIDER_TYPE = "type"; + public static final String PROVIDER_API_TOKENS = "apiTokens"; + + public static final String PROTOCOL = "protocol"; + + public static final String FAILOVER = "failover"; + public static final String FAILOVER_ENABLED = "enabled"; + public static final String FAILOVER_FAILURE_THRESHOLD = "failureThreshold"; + public static final String FAILOVER_SUCCESS_THRESHOLD = "successThreshold"; + public static final String FAILOVER_HEALTH_CHECK_INTERVAL = "healthCheckInterval"; + public static final String FAILOVER_HEALTH_CHECK_TIMEOUT = "healthCheckTimeout"; + public static final String FAILOVER_HEALTH_CHECK_MODEL = "healthCheckModel"; +} \ No newline at end of file diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/plugin/config/KeyAuthConfig.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/plugin/config/KeyAuthConfig.java new file mode 100644 index 00000000..2454ac2d --- /dev/null +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/plugin/config/KeyAuthConfig.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2022-2024 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.alibaba.higress.sdk.constant.plugin.config; + +public class KeyAuthConfig { + + public static final String CONSUMERS = "consumers"; + public static final String CONSUMER_NAME = "name"; + public static final String CONSUMER_CREDENTIAL = "credential"; + + public static final String KEYS = "keys"; + public static final String IN_HEADER = "in_header"; + public static final String IN_QUERY = "in_query"; + + public static final String ALLOW = "allow"; +} diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/plugin/config/ModelMapperConfig.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/plugin/config/ModelMapperConfig.java new file mode 100644 index 00000000..7fc95e4c --- /dev/null +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/plugin/config/ModelMapperConfig.java @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2022-2024 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.alibaba.higress.sdk.constant.plugin.config; + +public class ModelMapperConfig { + + public static final String MODEL_MAPPING = "modelMapping"; +} diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/plugin/config/ModelRouterConfig.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/plugin/config/ModelRouterConfig.java new file mode 100644 index 00000000..37f420d5 --- /dev/null +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/plugin/config/ModelRouterConfig.java @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2022-2024 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.alibaba.higress.sdk.constant.plugin.config; + +public class ModelRouterConfig { + + public static final String MODEL_TO_HEADER = "modelToHeader"; +} diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/WasmPluginInstance.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/WasmPluginInstance.java index e58e15ec..8ecd621c 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/WasmPluginInstance.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/WasmPluginInstance.java @@ -12,8 +12,13 @@ */ package com.alibaba.higress.sdk.model; +import java.util.HashMap; import java.util.Map; +import org.apache.commons.collections4.MapUtils; + +import com.alibaba.higress.sdk.util.MapUtil; + import io.swagger.annotations.ApiModel; import lombok.AllArgsConstructor; import lombok.Builder; @@ -29,10 +34,14 @@ public class WasmPluginInstance implements VersionedDto { private String version; + @Deprecated private WasmPluginInstanceScope scope; + @Deprecated private String target; + private Map targets; + private String pluginName; private String pluginVersion; @@ -48,4 +57,45 @@ public class WasmPluginInstance implements VersionedDto { public boolean isInternal() { return Boolean.TRUE.equals(internal); } + + public void syncDeprecatedFields() { + if (scope == null && MapUtils.isEmpty(targets)) { + return; + } + if (scope != null) { + targets = MapUtil.of(scope, target); + } else if (targets.size() == 1) { + Map.Entry entry = targets.entrySet().iterator().next(); + scope = entry.getKey(); + target = entry.getValue(); + } else { + // We don't know which one to choose, so skip syncing. + } + } + + public boolean hasScopedTarget(WasmPluginInstanceScope scope) { + return targets != null && targets.containsKey(scope); + } + + public void setGlobalTarget() { + setTarget(WasmPluginInstanceScope.GLOBAL, null); + } + + public void setTarget(WasmPluginInstanceScope scope, String target) { + if (targets == null) { + targets = new HashMap<>(); + } else { + targets.clear(); + } + targets.put(scope, target); + syncDeprecatedFields(); + } + + public void putTarget(WasmPluginInstanceScope scope, String target) { + if (targets == null) { + targets = new HashMap<>(); + } + targets.put(scope, target); + syncDeprecatedFields(); + } } diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/WasmPluginInstanceScope.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/WasmPluginInstanceScope.java index d80d08a1..decb2261 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/WasmPluginInstanceScope.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/WasmPluginInstanceScope.java @@ -13,6 +13,7 @@ package com.alibaba.higress.sdk.model; import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -39,6 +40,8 @@ public enum WasmPluginInstanceScope { */ SERVICE("service", 1000); + public static final List NON_GLOBAL_SCOPES = Arrays.asList(DOMAIN, ROUTE, SERVICE); + private static final Map ID_SCOPE_MAPPING = Arrays .stream(WasmPluginInstanceScope.values()).collect(Collectors.toMap(WasmPluginInstanceScope::getId, s -> s)); diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/AiModelPredicate.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/AiModelPredicate.java index 33812b5e..a0d9d52d 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/AiModelPredicate.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/AiModelPredicate.java @@ -12,20 +12,20 @@ */ package com.alibaba.higress.sdk.model.ai; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; +import com.alibaba.higress.sdk.exception.ValidationException; +import com.alibaba.higress.sdk.model.route.RoutePredicate; +import com.alibaba.higress.sdk.model.route.RoutePredicateTypeEnum; -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class AiModelPredicate { - - private Boolean enabled; - private String prefix; +public class AiModelPredicate extends RoutePredicate { public void validate() { + super.validate(); + RoutePredicateTypeEnum predicateType = RoutePredicateTypeEnum.fromName(this.getMatchType()); + if (predicateType == null) { + throw new ValidationException("Unknown matchType: " + this.getMatchType()); + } + if (predicateType == RoutePredicateTypeEnum.REGULAR) { + throw new ValidationException("AiModelPredicate does not support regular expression matchType"); + } } } diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/AiRoute.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/AiRoute.java index ef146a5a..21a929e1 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/AiRoute.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/AiRoute.java @@ -36,7 +36,7 @@ public class AiRoute { private String version; private List domains; private List upstreams; - private AiModelPredicate modelPredicate; + private List modelPredicates; private AiRouteAuthConfig authConfig; private AiRouteFallbackConfig fallbackConfig; @@ -44,18 +44,18 @@ public void validate() { if (StringUtils.isBlank(name)) { throw new ValidationException("name cannot be blank."); } - if (CollectionUtils.isEmpty(upstreams)){ + if (CollectionUtils.isEmpty(upstreams)) { throw new ValidationException("upstreams cannot be empty."); } upstreams.forEach(AiUpstream::validate); - if (modelPredicate != null){ - modelPredicate.validate(); - } - if (authConfig != null){ + if (authConfig != null) { authConfig.validate(); } - if (fallbackConfig != null){ + if (fallbackConfig != null) { fallbackConfig.validate(); } + if (CollectionUtils.isNotEmpty(modelPredicates)) { + modelPredicates.forEach(AiModelPredicate::validate); + } } } diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/AiUpstream.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/AiUpstream.java index b7e7f9d7..33c54b13 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/AiUpstream.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/AiUpstream.java @@ -12,6 +12,8 @@ */ package com.alibaba.higress.sdk.model.ai; +import java.util.Map; + import org.apache.commons.lang3.StringUtils; import com.alibaba.higress.sdk.exception.ValidationException; @@ -31,6 +33,7 @@ public class AiUpstream { private String provider; private Integer weight; + private Map modelMapping; public void validate() { if (StringUtils.isEmpty(provider)) { diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/LlmProvider.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/LlmProvider.java index e3d7f201..20f3cfb4 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/LlmProvider.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/LlmProvider.java @@ -33,9 +33,9 @@ public class LlmProvider { private String name; private String type; private String protocol; - private Map modelMapping; private List tokens; private TokenFailoverConfig tokenFailoverConfig; + private Map rawConfigs; public void validate(boolean forUpdate) { if (StringUtils.isBlank(name)) { diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/LlmProviderType.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/LlmProviderType.java index 3e4018a4..0f8703d1 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/LlmProviderType.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/ai/LlmProviderType.java @@ -22,4 +22,48 @@ private LlmProviderType() { public static final String OPENAI = "openai"; public static final String MOONSHOT = "moonshot"; + + public static final String AZURE = "azure"; + + public static final String AI360 = "ai360"; + + public static final String GITHUB = "github"; + + public static final String GROQ = "groq"; + + public static final String BAICHUAN = "baichuan"; + + public static final String YI = "yi"; + + public static final String DEEPSEEK = "deepseek"; + + public static final String ZHIPUAI = "zhipuai"; + + public static final String OLLAMA = "ollama"; + + public static final String CLAUDE = "claude"; + + public static final String BAIDU = "baidu"; + + public static final String HUNYUAN = "hunyuan"; + + public static final String STEPFUN = "stepfun"; + + public static final String MINIMAX = "minimax"; + + public static final String CLOUDFLARE = "cloudflare"; + + public static final String SPARK = "spark"; + + public static final String GEMINI = "gemini"; + + public static final String DEEPL = "deepl"; + + public static final String MISTRAL = "mistral"; + + public static final String COHERE = "cohere"; + + public static final String DOUBAO = "doubao"; + + public static final String COZE = "coze"; } diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/route/RoutePredicate.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/route/RoutePredicate.java index f782f4a4..c5153825 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/route/RoutePredicate.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/route/RoutePredicate.java @@ -12,6 +12,8 @@ */ package com.alibaba.higress.sdk.model.route; +import com.alibaba.higress.sdk.exception.ValidationException; + import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -31,4 +33,17 @@ public class RoutePredicate { private String matchValue; private Boolean caseSensitive; + + public void validate() { + if (this.getMatchType() == null) { + throw new ValidationException("matchType is required"); + } + RoutePredicateTypeEnum predicateType = RoutePredicateTypeEnum.fromName(this.getMatchType()); + if (predicateType == null) { + throw new ValidationException("Unknown matchType: " + this.getMatchType()); + } + if (this.getMatchValue() == null) { + throw new ValidationException("matchValue is required"); + } + } } diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/route/RoutePredicateTypeEnum.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/route/RoutePredicateTypeEnum.java index 279b7c64..cab6e534 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/route/RoutePredicateTypeEnum.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/model/route/RoutePredicateTypeEnum.java @@ -12,12 +12,13 @@ */ package com.alibaba.higress.sdk.model.route; -import org.apache.commons.lang3.StringUtils; - import java.util.Arrays; +import java.util.Locale; import java.util.Map; import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; + public enum RoutePredicateTypeEnum { /** @@ -35,6 +36,9 @@ public enum RoutePredicateTypeEnum { private final String annotationPrefix; + private static final Map NAME_MAP = + Arrays.stream(values()).collect(Collectors.toMap(RoutePredicateTypeEnum::name, t -> t)); + private static final Map ANNOTATION_PREFIX_MAP = Arrays.stream(values()).collect(Collectors.toMap(RoutePredicateTypeEnum::getAnnotationPrefix, t -> t)); @@ -46,6 +50,10 @@ public String getAnnotationPrefix() { return annotationPrefix; } + public static RoutePredicateTypeEnum fromName(String name) { + return StringUtils.isNotEmpty(name) ? NAME_MAP.get(name.toUpperCase(Locale.ROOT)) : null; + } + public static RoutePredicateTypeEnum fromAnnotationPrefix(String annotationPrefix) { return StringUtils.isNotEmpty(annotationPrefix) ? ANNOTATION_PREFIX_MAP.get(annotationPrefix) : null; } diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/HigressServiceProviderImpl.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/HigressServiceProviderImpl.java index e96a9f2e..e5be42b8 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/HigressServiceProviderImpl.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/HigressServiceProviderImpl.java @@ -55,11 +55,10 @@ class HigressServiceProviderImpl implements HigressServiceProvider { new RouteServiceImpl(kubernetesClientService, kubernetesModelConverter, wasmPluginInstanceService); domainService = new DomainServiceImpl(kubernetesClientService, kubernetesModelConverter, routeService, wasmPluginInstanceService); - consumerService = new ConsumerServiceImpl(wasmPluginService, wasmPluginInstanceService); - llmProviderService = - new LlmProviderServiceImpl(serviceSourceService, wasmPluginService, wasmPluginInstanceService); + consumerService = new ConsumerServiceImpl(wasmPluginInstanceService); + llmProviderService = new LlmProviderServiceImpl(serviceSourceService, wasmPluginInstanceService); aiRouteService = new AiRouteServiceImpl(kubernetesModelConverter, kubernetesClientService, routeService, - llmProviderService, consumerService, wasmPluginService, wasmPluginInstanceService); + llmProviderService, consumerService, wasmPluginInstanceService); } @Override diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/WasmPluginInstanceService.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/WasmPluginInstanceService.java index f3496968..97cd2943 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/WasmPluginInstanceService.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/WasmPluginInstanceService.java @@ -13,12 +13,15 @@ package com.alibaba.higress.sdk.service; import java.util.List; +import java.util.Map; import com.alibaba.higress.sdk.model.WasmPluginInstance; import com.alibaba.higress.sdk.model.WasmPluginInstanceScope; public interface WasmPluginInstanceService { + WasmPluginInstance createEmptyInstance(String pluginName); + List list(String pluginName); List list(WasmPluginInstanceScope scope, String target); @@ -27,9 +30,15 @@ public interface WasmPluginInstanceService { WasmPluginInstance query(WasmPluginInstanceScope scope, String target, String pluginName, Boolean internal); + WasmPluginInstance query(Map targets, String pluginName, Boolean internal); + WasmPluginInstance addOrUpdate(WasmPluginInstance instance); void delete(WasmPluginInstanceScope scope, String target, String pluginName); + void delete(Map targets, String pluginName); + void deleteAll(WasmPluginInstanceScope scope, String target); + + void deleteAll(Map targets); } diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/WasmPluginInstanceServiceImpl.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/WasmPluginInstanceServiceImpl.java index ddc3185e..c544fa15 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/WasmPluginInstanceServiceImpl.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/WasmPluginInstanceServiceImpl.java @@ -21,6 +21,7 @@ import java.util.Objects; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; import com.alibaba.higress.sdk.exception.BusinessException; @@ -35,6 +36,7 @@ import com.alibaba.higress.sdk.service.kubernetes.KubernetesModelConverter; import com.alibaba.higress.sdk.service.kubernetes.KubernetesUtil; import com.alibaba.higress.sdk.service.kubernetes.crd.wasm.V1alpha1WasmPlugin; +import com.alibaba.higress.sdk.util.MapUtil; import io.kubernetes.client.openapi.ApiException; import io.swagger.v3.core.util.Yaml; @@ -54,6 +56,18 @@ public WasmPluginInstanceServiceImpl(WasmPluginService wasmPluginService, this.kubernetesModelConverter = kubernetesModelConverter; } + @Override + public WasmPluginInstance createEmptyInstance(String pluginName) { + WasmPlugin plugin = wasmPluginService.query(pluginName, null); + if (plugin == null) { + throw new BusinessException("Plugin " + pluginName + " not found"); + } + WasmPluginInstance instance = new WasmPluginInstance(); + instance.setPluginName(plugin.getName()); + instance.setPluginVersion(plugin.getPluginVersion()); + return instance; + } + @Override public List list(String pluginName) { List plugins; @@ -91,6 +105,11 @@ public WasmPluginInstance query(WasmPluginInstanceScope scope, String target, St @Override public WasmPluginInstance query(WasmPluginInstanceScope scope, String target, String pluginName, Boolean internal) { + return query(MapUtil.of(scope, target), pluginName, internal); + } + + @Override + public WasmPluginInstance query(Map targets, String pluginName, Boolean internal) { List plugins; try { plugins = kubernetesClientService.listWasmPlugin(pluginName); @@ -103,7 +122,7 @@ public WasmPluginInstance query(WasmPluginInstanceScope scope, String target, St if (internal != null && internal != KubernetesUtil.isInternalResource(plugin)) { continue; } - instance = kubernetesModelConverter.getWasmPluginInstanceFromCr(plugin, scope, target); + instance = kubernetesModelConverter.getWasmPluginInstanceFromCr(plugin, targets); if (instance != null) { break; } @@ -115,19 +134,27 @@ public WasmPluginInstance query(WasmPluginInstanceScope scope, String target, St @Override @SuppressWarnings("unchecked") public WasmPluginInstance addOrUpdate(WasmPluginInstance instance) { - WasmPluginInstanceScope scope = instance.getScope(); - if (scope == null) { - throw new IllegalArgumentException("instance.scope cannot be null."); + instance.syncDeprecatedFields(); + + Map targets = instance.getTargets(); + if (MapUtils.isEmpty(targets)) { + throw new IllegalArgumentException("instance.targets cannot be empty."); } - String target = instance.getTarget(); - if (scope == WasmPluginInstanceScope.GLOBAL) { + if (targets.containsKey(WasmPluginInstanceScope.GLOBAL)) { + if (targets.size() > 1) { + throw new IllegalArgumentException( + "instance.targets cannot contain GLOBAL and other scopes at the same time."); + } + String target = targets.get(WasmPluginInstanceScope.GLOBAL); if (target != null) { - throw new IllegalArgumentException("instance.target must be null when scope is GLOBAL."); + throw new IllegalArgumentException("instance.target must be empty when scope is GLOBAL."); } } else { - if (StringUtils.isEmpty(target)) { - throw new IllegalArgumentException( - "instance.target must not be null or empty when scope is not GLOBAL."); + for (Map.Entry entry : targets.entrySet()) { + if (StringUtils.isEmpty(entry.getValue())) { + throw new IllegalArgumentException( + "instance.target must not be null or empty when scope is not GLOBAL."); + } } } @@ -190,37 +217,53 @@ public WasmPluginInstance addOrUpdate(WasmPluginInstance instance) { throw new BusinessException( "Error occurs when adding or updating the WasmPlugin CR with name: " + plugin.getName(), e); } - return kubernetesModelConverter.getWasmPluginInstanceFromCr(result, scope, target); + return kubernetesModelConverter.getWasmPluginInstanceFromCr(result, targets); } @Override public void delete(WasmPluginInstanceScope scope, String target, String pluginName) { + delete(MapUtil.of(scope, target), pluginName); + } + + @Override + public void delete(Map targets, String pluginName) { + if (MapUtils.isEmpty(targets)) { + return; + } List existedCrs; try { existedCrs = kubernetesClientService.listWasmPlugin(pluginName); } catch (ApiException e) { throw new BusinessException("Error occurs when getting WasmPlugin.", e); } - deletePluginInstances(existedCrs, scope, target); + deletePluginInstances(existedCrs, targets); } @Override public void deleteAll(WasmPluginInstanceScope scope, String target) { + deleteAll(MapUtil.of(scope, target)); + } + + @Override + public void deleteAll(Map targets) { + if (MapUtils.isEmpty(targets)) { + return; + } List existedCrs; try { existedCrs = kubernetesClientService.listWasmPlugin(); } catch (ApiException e) { throw new BusinessException("Error occurs when getting WasmPlugin.", e); } - deletePluginInstances(existedCrs, scope, target); + deletePluginInstances(existedCrs, targets); } - private void deletePluginInstances(List crs, WasmPluginInstanceScope scope, String target) { + private void deletePluginInstances(List crs, Map targets) { if (CollectionUtils.isEmpty(crs)) { return; } for (V1alpha1WasmPlugin cr : crs) { - boolean needUpdate = kubernetesModelConverter.removeWasmPluginInstanceFromCr(cr, scope, target); + boolean needUpdate = kubernetesModelConverter.removeWasmPluginInstanceFromCr(cr, targets); if (needUpdate) { try { kubernetesClientService.replaceWasmPlugin(cr); diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/AbstractLlmProviderHandler.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/AbstractLlmProviderHandler.java index 05656f37..20500f20 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/AbstractLlmProviderHandler.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/AbstractLlmProviderHandler.java @@ -12,6 +12,18 @@ */ package com.alibaba.higress.sdk.service.ai; +import static com.alibaba.higress.sdk.constant.plugin.config.AiProxyConfig.FAILOVER; +import static com.alibaba.higress.sdk.constant.plugin.config.AiProxyConfig.FAILOVER_ENABLED; +import static com.alibaba.higress.sdk.constant.plugin.config.AiProxyConfig.FAILOVER_FAILURE_THRESHOLD; +import static com.alibaba.higress.sdk.constant.plugin.config.AiProxyConfig.FAILOVER_HEALTH_CHECK_INTERVAL; +import static com.alibaba.higress.sdk.constant.plugin.config.AiProxyConfig.FAILOVER_HEALTH_CHECK_MODEL; +import static com.alibaba.higress.sdk.constant.plugin.config.AiProxyConfig.FAILOVER_HEALTH_CHECK_TIMEOUT; +import static com.alibaba.higress.sdk.constant.plugin.config.AiProxyConfig.FAILOVER_SUCCESS_THRESHOLD; +import static com.alibaba.higress.sdk.constant.plugin.config.AiProxyConfig.PROTOCOL; +import static com.alibaba.higress.sdk.constant.plugin.config.AiProxyConfig.PROVIDER_API_TOKENS; +import static com.alibaba.higress.sdk.constant.plugin.config.AiProxyConfig.PROVIDER_ID; +import static com.alibaba.higress.sdk.constant.plugin.config.AiProxyConfig.PROVIDER_TYPE; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -23,44 +35,28 @@ import com.alibaba.higress.sdk.constant.CommonKey; import com.alibaba.higress.sdk.constant.HigressConstants; +import com.alibaba.higress.sdk.constant.Separators; +import com.alibaba.higress.sdk.model.ServiceSource; import com.alibaba.higress.sdk.model.ai.LlmProvider; import com.alibaba.higress.sdk.model.ai.LlmProviderProtocol; import com.alibaba.higress.sdk.model.ai.TokenFailoverConfig; +import com.alibaba.higress.sdk.model.route.UpstreamService; +import com.alibaba.higress.sdk.service.kubernetes.crd.mcp.V1McpBridge; abstract class AbstractLlmProviderHandler implements LlmProviderHandler { - private static final String PROTOCOL_KEY = "protocol"; - private static final String PROVIDER_API_TOKENS_KEY = "apiTokens"; - private static final String PROVIDER_MODEL_MAPPING_KEY = "modelMapping"; - private static final String FAILOVER_KEY = "failover"; - private static final String FAILOVER_ENABLED_KEY = "enabled"; - private static final String FAILOVER_FAILURE_THRESHOLD_KEY = "failureThreshold"; - private static final String FAILOVER_SUCCESS_THRESHOLD_KEY = "successThreshold"; - private static final String FAILOVER_HEALTH_CHECK_INTERVAL_KEY = "healthCheckInterval"; - private static final String FAILOVER_HEALTH_CHECK_TIMEOUT_KEY = "healthCheckTimeout"; - private static final String FAILOVER_HEALTH_CHECK_MODEL_KEY = "healthCheckModel"; - + @Override public abstract String getType(); + @Override @SuppressWarnings("unchecked") public boolean loadConfig(LlmProvider provider, Map configurations) { - String id = MapUtils.getString(configurations, PROVIDER_ID_KEY); + String id = MapUtils.getString(configurations, PROVIDER_ID); if (StringUtils.isBlank(id)) { return false; } - Object modelMappingObj = configurations.get(PROVIDER_MODEL_MAPPING_KEY); - Map modelMapping = null; - if (modelMappingObj instanceof Map modelMappingMap) { - modelMapping = new HashMap<>(modelMappingMap.size()); - for (Map.Entry entry : modelMappingMap.entrySet()) { - if (entry.getKey() instanceof String key && entry.getValue() instanceof String value) { - modelMapping.put(key, value); - } - } - } - - Object tokensObj = configurations.get(PROVIDER_API_TOKENS_KEY); + Object tokensObj = configurations.get(PROVIDER_API_TOKENS); List tokens = null; if (tokensObj instanceof List tokensList) { tokens = new ArrayList<>(tokensList.size()); @@ -72,13 +68,13 @@ public boolean loadConfig(LlmProvider provider, Map configuratio } TokenFailoverConfig failoverConfig = null; - Object failoverObj = configurations.get(FAILOVER_KEY); + Object failoverObj = configurations.get(FAILOVER); if (failoverObj instanceof Map failoverMap) { failoverConfig = buildTokenFailoverConfig((Map)failoverMap); } LlmProviderProtocol protocol = - LlmProviderProtocol.fromPluginValue(MapUtils.getString(configurations, PROTOCOL_KEY)); + LlmProviderProtocol.fromPluginValue(MapUtils.getString(configurations, PROTOCOL)); if (protocol == null) { protocol = LlmProviderProtocol.DEFAULT; } @@ -86,49 +82,87 @@ public boolean loadConfig(LlmProvider provider, Map configuratio provider.setName(id); provider.setType(getType()); - provider.setModelMapping(modelMapping); provider.setTokens(tokens); provider.setTokenFailoverConfig(failoverConfig); + provider.setRawConfigs(new HashMap<>(configurations)); return true; } + @Override public void saveConfig(LlmProvider provider, Map configurations) { - configurations.put(PROVIDER_ID_KEY, provider.getName()); - configurations.put(PROVIDER_TYPE_KEY, getType()); + configurations.put(PROVIDER_ID, provider.getName()); + configurations.put(PROVIDER_TYPE, getType()); LlmProviderProtocol protocol = LlmProviderProtocol.fromValue(provider.getProtocol()); if (protocol == null) { protocol = LlmProviderProtocol.DEFAULT; - configurations.put(PROTOCOL_KEY, protocol.getPluginValue()); - } - - Map modelMapping = provider.getModelMapping(); - if (MapUtils.isNotEmpty(modelMapping)) { - configurations.put(PROVIDER_MODEL_MAPPING_KEY, new HashMap<>(modelMapping)); - } else { - configurations.remove(PROVIDER_MODEL_MAPPING_KEY); + configurations.put(PROTOCOL, protocol.getPluginValue()); } List tokens = provider.getTokens(); if (CollectionUtils.isNotEmpty(tokens)) { - configurations.put(PROVIDER_API_TOKENS_KEY, tokens); + configurations.put(PROVIDER_API_TOKENS, tokens); } else { - configurations.remove(PROVIDER_API_TOKENS_KEY); + configurations.remove(PROVIDER_API_TOKENS); } TokenFailoverConfig failoverConfig = provider.getTokenFailoverConfig(); if (failoverConfig == null) { - configurations.remove(FAILOVER_KEY); + configurations.remove(FAILOVER); } else { Map failoverMap = new HashMap<>(); saveTokenFailoverConfig(failoverConfig, failoverMap); - configurations.put(FAILOVER_KEY, failoverMap); + configurations.put(FAILOVER, failoverMap); + } + } + + @Override + public void validateConfig(Map configurations) {} + + @Override + public ServiceSource buildServiceSource(String providerName, Map providerConfig) { + ServiceSource serviceSource = new ServiceSource(); + serviceSource.setName(generateServiceProviderName(providerName)); + serviceSource.setType(getServiceRegistryType(providerConfig)); + serviceSource.setProtocol(getServiceProtocol(providerConfig)); + + String domain = getServiceDomain(providerConfig); + int port = getServicePort(providerConfig); + if (V1McpBridge.REGISTRY_TYPE_STATIC.equals(serviceSource.getType())) { + serviceSource.setDomain(domain + Separators.COLON + port); + serviceSource.setPort(V1McpBridge.STATIC_PORT); + } else { + serviceSource.setDomain(domain); + serviceSource.setPort(port); } + + return serviceSource; } + @Override + public UpstreamService buildUpstreamService(String providerName, Map providerConfig) { + UpstreamService service = new UpstreamService(); + String registryType = getServiceRegistryType(providerConfig); + service.setName(generateServiceProviderName(providerName) + Separators.DOT + registryType); + if (V1McpBridge.REGISTRY_TYPE_STATIC.equals(registryType)) { + service.setPort(V1McpBridge.STATIC_PORT); + } else { + service.setPort(getServicePort(providerConfig)); + } + service.setWeight(100); + return service; + } + + protected abstract String getServiceRegistryType(Map providerConfig); + + protected abstract String getServiceDomain(Map providerConfig); + + protected abstract int getServicePort(Map providerConfig); + + protected abstract String getServiceProtocol(Map providerConfig); + protected static String generateServiceProviderName(String llmProviderName) { - return CommonKey.LLM_SERVICE_NAME_PREFIX + llmProviderName - + HigressConstants.INTERNAL_RESOURCE_NAME_SUFFIX; + return CommonKey.LLM_SERVICE_NAME_PREFIX + llmProviderName + HigressConstants.INTERNAL_RESOURCE_NAME_SUFFIX; } private TokenFailoverConfig buildTokenFailoverConfig(Map failoverMap) { @@ -136,21 +170,21 @@ private TokenFailoverConfig buildTokenFailoverConfig(Map failove return null; } TokenFailoverConfig failoverConfig = new TokenFailoverConfig(); - failoverConfig.setEnabled(MapUtils.getBoolean(failoverMap, FAILOVER_ENABLED_KEY, false)); - failoverConfig.setFailureThreshold(MapUtils.getInteger(failoverMap, FAILOVER_FAILURE_THRESHOLD_KEY)); - failoverConfig.setSuccessThreshold(MapUtils.getInteger(failoverMap, FAILOVER_SUCCESS_THRESHOLD_KEY)); - failoverConfig.setHealthCheckInterval(MapUtils.getInteger(failoverMap, FAILOVER_HEALTH_CHECK_INTERVAL_KEY)); - failoverConfig.setHealthCheckTimeout(MapUtils.getInteger(failoverMap, FAILOVER_HEALTH_CHECK_TIMEOUT_KEY)); - failoverConfig.setHealthCheckModel(MapUtils.getString(failoverMap, FAILOVER_HEALTH_CHECK_MODEL_KEY)); + failoverConfig.setEnabled(MapUtils.getBoolean(failoverMap, FAILOVER_ENABLED, false)); + failoverConfig.setFailureThreshold(MapUtils.getInteger(failoverMap, FAILOVER_FAILURE_THRESHOLD)); + failoverConfig.setSuccessThreshold(MapUtils.getInteger(failoverMap, FAILOVER_SUCCESS_THRESHOLD)); + failoverConfig.setHealthCheckInterval(MapUtils.getInteger(failoverMap, FAILOVER_HEALTH_CHECK_INTERVAL)); + failoverConfig.setHealthCheckTimeout(MapUtils.getInteger(failoverMap, FAILOVER_HEALTH_CHECK_TIMEOUT)); + failoverConfig.setHealthCheckModel(MapUtils.getString(failoverMap, FAILOVER_HEALTH_CHECK_MODEL)); return failoverConfig; } private void saveTokenFailoverConfig(TokenFailoverConfig failoverConfig, Map failoverMap) { - failoverMap.put(FAILOVER_ENABLED_KEY, failoverConfig.getEnabled()); - failoverMap.put(FAILOVER_FAILURE_THRESHOLD_KEY, failoverConfig.getFailureThreshold()); - failoverMap.put(FAILOVER_SUCCESS_THRESHOLD_KEY, failoverConfig.getSuccessThreshold()); - failoverMap.put(FAILOVER_HEALTH_CHECK_INTERVAL_KEY, failoverConfig.getHealthCheckInterval()); - failoverMap.put(FAILOVER_HEALTH_CHECK_TIMEOUT_KEY, failoverConfig.getHealthCheckTimeout()); - failoverMap.put(FAILOVER_HEALTH_CHECK_MODEL_KEY, failoverConfig.getHealthCheckModel()); + failoverMap.put(FAILOVER_ENABLED, failoverConfig.getEnabled()); + failoverMap.put(FAILOVER_FAILURE_THRESHOLD, failoverConfig.getFailureThreshold()); + failoverMap.put(FAILOVER_SUCCESS_THRESHOLD, failoverConfig.getSuccessThreshold()); + failoverMap.put(FAILOVER_HEALTH_CHECK_INTERVAL, failoverConfig.getHealthCheckInterval()); + failoverMap.put(FAILOVER_HEALTH_CHECK_TIMEOUT, failoverConfig.getHealthCheckTimeout()); + failoverMap.put(FAILOVER_HEALTH_CHECK_MODEL, failoverConfig.getHealthCheckModel()); } } diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/AiRouteServiceImpl.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/AiRouteServiceImpl.java index 90237837..25887949 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/AiRouteServiceImpl.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/AiRouteServiceImpl.java @@ -25,17 +25,18 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; -import com.alibaba.higress.sdk.constant.BuiltInPluginName; import com.alibaba.higress.sdk.constant.CommonKey; import com.alibaba.higress.sdk.constant.HigressConstants; import com.alibaba.higress.sdk.constant.KubernetesConstants; +import com.alibaba.higress.sdk.constant.plugin.BuiltInPluginName; +import com.alibaba.higress.sdk.constant.plugin.config.ModelMapperConfig; +import com.alibaba.higress.sdk.constant.plugin.config.ModelRouterConfig; import com.alibaba.higress.sdk.exception.BusinessException; import com.alibaba.higress.sdk.exception.ResourceConflictException; import com.alibaba.higress.sdk.http.HttpStatus; import com.alibaba.higress.sdk.model.CommonPageQuery; import com.alibaba.higress.sdk.model.PaginatedResult; import com.alibaba.higress.sdk.model.Route; -import com.alibaba.higress.sdk.model.WasmPlugin; import com.alibaba.higress.sdk.model.WasmPluginInstance; import com.alibaba.higress.sdk.model.WasmPluginInstanceScope; import com.alibaba.higress.sdk.model.ai.AiModelPredicate; @@ -50,12 +51,12 @@ import com.alibaba.higress.sdk.model.route.UpstreamService; import com.alibaba.higress.sdk.service.RouteService; import com.alibaba.higress.sdk.service.WasmPluginInstanceService; -import com.alibaba.higress.sdk.service.WasmPluginService; import com.alibaba.higress.sdk.service.consumer.ConsumerService; import com.alibaba.higress.sdk.service.kubernetes.KubernetesClientService; import com.alibaba.higress.sdk.service.kubernetes.KubernetesModelConverter; import com.alibaba.higress.sdk.service.kubernetes.crd.istio.V1alpha3EnvoyFilter; import com.alibaba.higress.sdk.util.StringUtil; +import com.google.common.annotations.VisibleForTesting; import io.kubernetes.client.openapi.ApiException; import io.kubernetes.client.openapi.models.V1ConfigMap; @@ -69,8 +70,7 @@ public class AiRouteServiceImpl implements AiRouteService { private static final Map AI_ROUTE_LABEL_SELECTORS = Map.of(KubernetesConstants.Label.CONFIG_MAP_TYPE_KEY, KubernetesConstants.Label.CONFIG_MAP_TYPE_VALUE_AI_ROUTE); - private static final String MODEL_ROUTER_ENABLE_KEY = "enable"; - private static final String ADD_HEADER_KEY_KEY = "add_header_key"; + private static final String MODEL_ROUTING_HEADER = "x-higress-llm-model"; private final KubernetesModelConverter kubernetesModelConverter; @@ -82,22 +82,19 @@ public class AiRouteServiceImpl implements AiRouteService { private final ConsumerService consumerService; - private final WasmPluginService wasmPluginService; - private final WasmPluginInstanceService wasmPluginInstanceService; private final String routeFallbackEnvoyFilterConfig; public AiRouteServiceImpl(KubernetesModelConverter kubernetesModelConverter, KubernetesClientService kubernetesClientService, RouteService routeService, - LlmProviderService llmProviderService, ConsumerService consumerService, WasmPluginService wasmPluginService, + LlmProviderService llmProviderService, ConsumerService consumerService, WasmPluginInstanceService wasmPluginInstanceService) { this.kubernetesModelConverter = kubernetesModelConverter; this.kubernetesClientService = kubernetesClientService; this.routeService = routeService; this.llmProviderService = llmProviderService; this.consumerService = consumerService; - this.wasmPluginService = wasmPluginService; this.wasmPluginInstanceService = wasmPluginInstanceService; try { @@ -191,11 +188,6 @@ public AiRoute update(AiRoute route) { private void fillDefaultValues(AiRoute route) { fillDefaultWeights(route.getUpstreams()); - AiModelPredicate modelPredicate = route.getModelPredicate(); - if (modelPredicate != null && Boolean.TRUE.equals(modelPredicate.getEnabled()) - && StringUtils.isEmpty(modelPredicate.getPrefix())) { - modelPredicate.setPrefix(route.getName()); - } AiRouteFallbackConfig fallbackConfig = route.getFallbackConfig(); if (fallbackConfig != null && Boolean.TRUE.equals(fallbackConfig.getEnabled())) { fillDefaultWeights(fallbackConfig.getUpstreams()); @@ -221,7 +213,8 @@ private void writeAiRouteResources(AiRoute aiRoute) { setUpstreams(route, aiRoute.getUpstreams()); saveRoute(route); writeAuthConfigResources(routeName, aiRoute.getAuthConfig()); - writeModelRouteResources(aiRoute.getModelPredicate()); + writeModelRouteResources(aiRoute.getModelPredicates()); + writeModelMappingResources(routeName, aiRoute.getUpstreams()); } private void writeAiRouteFallbackResources(AiRoute aiRoute) { @@ -241,6 +234,8 @@ private void writeAiRouteFallbackResources(AiRoute aiRoute) { fallbackHeaderPredicate.setCaseSensitive(true); if (route.getHeaders() == null) { route.setHeaders(new ArrayList<>()); + } else { + route.setHeaders(new ArrayList<>(route.getHeaders())); } route.getHeaders().add(fallbackHeaderPredicate); String fallbackStrategy = fallbackConfig.getFallbackStrategy(); @@ -277,6 +272,7 @@ private void writeAiRouteFallbackResources(AiRoute aiRoute) { } writeAuthConfigResources(fallbackRouteName, aiRoute.getAuthConfig()); + writeModelMappingResources(fallbackRouteName, fallbackUpStreams); } private void writeAuthConfigResources(String routeName, AiRouteAuthConfig authConfig) { @@ -285,8 +281,8 @@ private void writeAuthConfigResources(String routeName, AiRouteAuthConfig authCo consumerService.updateAllowList(WasmPluginInstanceScope.ROUTE, routeName, allowedConsumers); } - private void writeModelRouteResources(AiModelPredicate modelPredicate) { - if (modelPredicate == null || !Boolean.TRUE.equals(modelPredicate.getEnabled())) { + private void writeModelRouteResources(List modelPredicates) { + if (CollectionUtils.isEmpty(modelPredicates)) { return; } @@ -294,15 +290,9 @@ private void writeModelRouteResources(AiModelPredicate modelPredicate) { WasmPluginInstance instance = wasmPluginInstanceService.query(WasmPluginInstanceScope.GLOBAL, null, pluginName, true); if (instance == null) { - WasmPlugin plugin = wasmPluginService.query(pluginName, null); - if (plugin == null) { - throw new BusinessException("Plugin " + pluginName + " not found"); - } - instance = new WasmPluginInstance(); - instance.setPluginName(plugin.getName()); - instance.setPluginVersion(plugin.getPluginVersion()); + instance = wasmPluginInstanceService.createEmptyInstance(pluginName); instance.setInternal(true); - instance.setScope(WasmPluginInstanceScope.GLOBAL); + instance.setGlobalTarget(); } instance.setEnabled(true); @@ -313,34 +303,101 @@ private void writeModelRouteResources(AiModelPredicate modelPredicate) { instance.setConfigurations(configurations); } - configurations.put(MODEL_ROUTER_ENABLE_KEY, Boolean.TRUE); - configurations.put(ADD_HEADER_KEY_KEY, HigressConstants.MODEL_ROUTER_HEADER); + configurations.put(ModelRouterConfig.MODEL_TO_HEADER, MODEL_ROUTING_HEADER); wasmPluginInstanceService.addOrUpdate(instance); } + private void writeModelMappingResources(String routeName, List upstreams) { + if (CollectionUtils.isEmpty(upstreams)) { + wasmPluginInstanceService.delete(WasmPluginInstanceScope.ROUTE, routeName, BuiltInPluginName.MODEL_MAPPER); + return; + } + + final String pluginName = BuiltInPluginName.MODEL_MAPPER; + for (AiUpstream upstream : upstreams) { + UpstreamService upstreamService = llmProviderService.buildUpstreamService(upstream.getProvider()); + + Map targets = Map.of(WasmPluginInstanceScope.ROUTE, routeName, + WasmPluginInstanceScope.SERVICE, upstreamService.getName()); + + if (MapUtils.isEmpty(upstream.getModelMapping())) { + wasmPluginInstanceService.delete(targets, pluginName); + continue; + } + + WasmPluginInstance instance = wasmPluginInstanceService.query(targets, pluginName, true); + if (instance == null) { + instance = wasmPluginInstanceService.createEmptyInstance(pluginName); + instance.setInternal(true); + instance.setTargets(targets); + } + instance.setEnabled(true); + + Map configurations = instance.getConfigurations(); + if (MapUtils.isEmpty(configurations)) { + // Just in case it is a readonly empty map. + configurations = new HashMap<>(); + instance.setConfigurations(configurations); + } + + configurations.put(ModelMapperConfig.MODEL_MAPPING, new HashMap<>(upstream.getModelMapping())); + + wasmPluginInstanceService.addOrUpdate(instance); + } + } + private Route buildRoute(String routeName, AiRoute aiRoute) { Route route = new Route(); route.setName(routeName); route.setPath(new RoutePredicate(RoutePredicateTypeEnum.PRE.name(), "/", true)); route.setDomains(aiRoute.getDomains()); - AiModelPredicate modelPredicate = aiRoute.getModelPredicate(); - if (modelPredicate != null && Boolean.TRUE.equals(modelPredicate.getEnabled()) - && StringUtils.isNotEmpty(modelPredicate.getPrefix())) { - KeyedRoutePredicate modelHeaderPredicate = new KeyedRoutePredicate(HigressConstants.MODEL_ROUTER_HEADER); - modelHeaderPredicate.setMatchType(RoutePredicateTypeEnum.EQUAL.name()); - modelHeaderPredicate.setMatchValue(modelPredicate.getPrefix()); - modelHeaderPredicate.setCaseSensitive(true); - if (route.getHeaders() == null) { - route.setHeaders(new ArrayList<>()); + List modelPredicates = aiRoute.getModelPredicates(); + if (CollectionUtils.isNotEmpty(modelPredicates)) { + KeyedRoutePredicate headerRoutePredicate = new KeyedRoutePredicate(MODEL_ROUTING_HEADER); + if (modelPredicates.size() == 1) { + AiModelPredicate modelPredicate = modelPredicates.get(0); + headerRoutePredicate.setMatchType(modelPredicate.getMatchType()); + headerRoutePredicate.setMatchValue(modelPredicate.getMatchValue()); + } else { + headerRoutePredicate.setMatchType(RoutePredicateTypeEnum.REGULAR.toString()); + headerRoutePredicate.setMatchValue(buildModelRoutingHeaderRegex(modelPredicates)); } - route.getHeaders().add(modelHeaderPredicate); + route.setHeaders(List.of(headerRoutePredicate)); } return route; } + @VisibleForTesting + String buildModelRoutingHeaderRegex(List modelPredicates) { + StringBuilder regexBuilder = new StringBuilder(); + regexBuilder.append("^("); + for (int i = 0; i < modelPredicates.size(); i++) { + AiModelPredicate modelPredicate = modelPredicates.get(i); + if (i > 0) { + regexBuilder.append("|"); + } + if (modelPredicate.getMatchType().equals(RoutePredicateTypeEnum.REGULAR.toString())) { + // Shouldn't happen as we have checked it in the caller. + throw new IllegalArgumentException( + "Regular expression match is not supported for model routing header."); + } + regexBuilder.append(escapeForRegexMatch(modelPredicate.getMatchValue())); + if (RoutePredicateTypeEnum.PRE == RoutePredicateTypeEnum.fromName(modelPredicate.getMatchType())) { + regexBuilder.append(".*"); + } + } + regexBuilder.append(")"); + return regexBuilder.toString(); + } + + @VisibleForTesting + String escapeForRegexMatch(String value) { + return value.replaceAll("[\\[\\]{}()^$|*+?.\\\\]", "\\\\$0"); + } + private void setUpstreams(Route route, List upstreams) { if (CollectionUtils.isEmpty(upstreams)) { route.setServices(List.of()); diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/AzureLlmProviderHandler.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/AzureLlmProviderHandler.java new file mode 100644 index 00000000..b23bdbde --- /dev/null +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/AzureLlmProviderHandler.java @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2022-2023 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.alibaba.higress.sdk.service.ai; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Map; + +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.StringUtils; + +import com.alibaba.higress.sdk.exception.ValidationException; +import com.alibaba.higress.sdk.model.ai.LlmProviderType; +import com.alibaba.higress.sdk.service.kubernetes.crd.mcp.V1McpBridge; + +public class AzureLlmProviderHandler extends AbstractLlmProviderHandler { + + private static final String SERVICE_URL_KEY = "azureServiceUrl"; + + @Override + public String getType() { + return LlmProviderType.AZURE; + } + + @Override + public void validateConfig(Map configurations) { + if (MapUtils.isEmpty(configurations)){ + throw new ValidationException("Missing Azure specific configurations."); + } + Object serviceUrlObj = configurations.get(SERVICE_URL_KEY); + if (!(serviceUrlObj instanceof String serviceUrl)){ + throw new ValidationException(SERVICE_URL_KEY + " must be a string."); + } + if (StringUtils.isEmpty(serviceUrl)) { + throw new ValidationException(SERVICE_URL_KEY + " cannot be empty."); + } + try { + new URI(serviceUrl); + } catch (URISyntaxException e) { + throw new ValidationException(SERVICE_URL_KEY + " is not a valid URL.", e); + } + } + + @Override + protected String getServiceRegistryType(Map providerConfig) { + return V1McpBridge.REGISTRY_TYPE_DNS; + } + + @Override + protected String getServiceDomain(Map providerConfig) { + if (MapUtils.isEmpty(providerConfig)){ + return ""; + } + String serviceUrl = (String)providerConfig.get(SERVICE_URL_KEY); + if (StringUtils.isEmpty(serviceUrl)) { + return ""; + } + try { + URI uri = new URI(serviceUrl); + return uri.getHost(); + } catch (URISyntaxException e) { + return null; + } + } + + @Override + protected int getServicePort(Map providerConfig) { + return 443; + } + + @Override + protected String getServiceProtocol(Map providerConfig) { + return V1McpBridge.PROTOCOL_HTTPS; + } +} diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/DefaultLlmProviderHandler.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/DefaultLlmProviderHandler.java index 3c4d5a7b..3a7b0678 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/DefaultLlmProviderHandler.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/DefaultLlmProviderHandler.java @@ -12,8 +12,8 @@ */ package com.alibaba.higress.sdk.service.ai; -import com.alibaba.higress.sdk.model.ServiceSource; -import com.alibaba.higress.sdk.model.route.UpstreamService; +import java.util.Map; + import com.alibaba.higress.sdk.service.kubernetes.crd.mcp.V1McpBridge; class DefaultLlmProviderHandler extends AbstractLlmProviderHandler { @@ -36,22 +36,22 @@ public String getType() { } @Override - public ServiceSource buildServiceSource(String providerName) { - ServiceSource serviceSource = new ServiceSource(); - serviceSource.setName(generateServiceProviderName(providerName)); - serviceSource.setType(V1McpBridge.REGISTRY_TYPE_DNS); - serviceSource.setDomain(domain); - serviceSource.setPort(port); - serviceSource.setProtocol(protocol); - return serviceSource; + protected String getServiceRegistryType(Map providerConfig) { + return V1McpBridge.REGISTRY_TYPE_DNS; + } + + @Override + protected String getServiceDomain(Map providerConfig) { + return domain; + } + + @Override + protected int getServicePort(Map providerConfig) { + return port; } @Override - public UpstreamService buildUpstreamService(String providerName) { - UpstreamService service = new UpstreamService(); - service.setName(generateServiceProviderName(providerName) + "." + V1McpBridge.REGISTRY_TYPE_DNS); - service.setPort(port); - service.setWeight(100); - return service; + protected String getServiceProtocol(Map providerConfig) { + return protocol; } } diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/LlmProviderHandler.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/LlmProviderHandler.java index 71f2fb58..976e2d13 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/LlmProviderHandler.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/LlmProviderHandler.java @@ -20,9 +20,6 @@ interface LlmProviderHandler { - String PROVIDER_ID_KEY = "id"; - String PROVIDER_TYPE_KEY = "type"; - String getType(); default LlmProvider createProvider() { @@ -33,7 +30,9 @@ default LlmProvider createProvider() { void saveConfig(LlmProvider provider, Map configurations); - ServiceSource buildServiceSource(String providerName); + void validateConfig(Map configurations); + + ServiceSource buildServiceSource(String providerName, Map providerConfig); - UpstreamService buildUpstreamService(String providerName); + UpstreamService buildUpstreamService(String providerName, Map providerConfig); } diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/LlmProviderServiceImpl.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/LlmProviderServiceImpl.java index e9eabcd0..027b215f 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/LlmProviderServiceImpl.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/LlmProviderServiceImpl.java @@ -12,8 +12,10 @@ */ package com.alibaba.higress.sdk.service.ai; -import static com.alibaba.higress.sdk.service.ai.LlmProviderHandler.PROVIDER_ID_KEY; -import static com.alibaba.higress.sdk.service.ai.LlmProviderHandler.PROVIDER_TYPE_KEY; +import static com.alibaba.higress.sdk.constant.plugin.config.AiProxyConfig.ACTIVE_PROVIDER_ID; +import static com.alibaba.higress.sdk.constant.plugin.config.AiProxyConfig.PROVIDERS; +import static com.alibaba.higress.sdk.constant.plugin.config.AiProxyConfig.PROVIDER_ID; +import static com.alibaba.higress.sdk.constant.plugin.config.AiProxyConfig.PROVIDER_TYPE; import java.util.ArrayList; import java.util.HashMap; @@ -28,13 +30,11 @@ import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; -import com.alibaba.higress.sdk.constant.BuiltInPluginName; -import com.alibaba.higress.sdk.exception.BusinessException; +import com.alibaba.higress.sdk.constant.plugin.BuiltInPluginName; import com.alibaba.higress.sdk.exception.ValidationException; import com.alibaba.higress.sdk.model.CommonPageQuery; import com.alibaba.higress.sdk.model.PaginatedResult; import com.alibaba.higress.sdk.model.ServiceSource; -import com.alibaba.higress.sdk.model.WasmPlugin; import com.alibaba.higress.sdk.model.WasmPluginInstance; import com.alibaba.higress.sdk.model.WasmPluginInstanceScope; import com.alibaba.higress.sdk.model.ai.LlmProvider; @@ -43,15 +43,11 @@ import com.alibaba.higress.sdk.model.route.UpstreamService; import com.alibaba.higress.sdk.service.ServiceSourceService; import com.alibaba.higress.sdk.service.WasmPluginInstanceService; -import com.alibaba.higress.sdk.service.WasmPluginService; import com.alibaba.higress.sdk.service.kubernetes.crd.mcp.V1McpBridge; @SuppressWarnings("unchecked") public class LlmProviderServiceImpl implements LlmProviderService { - private static final String ACTIVE_PROVIDER_ID_KEY = "activeProviderId"; - private static final String PROVIDERS_KEY = "providers"; - private static final Map PROVIDER_HANDLERS; static { @@ -59,18 +55,39 @@ public class LlmProviderServiceImpl implements LlmProviderService { new DefaultLlmProviderHandler(LlmProviderType.OPENAI, "api.openai.com", 443, V1McpBridge.PROTOCOL_HTTPS), new DefaultLlmProviderHandler(LlmProviderType.MOONSHOT, "api.moonshot.cn", 443, V1McpBridge.PROTOCOL_HTTPS), new DefaultLlmProviderHandler(LlmProviderType.QWEN, "dashscope.aliyuncs.com", 443, - V1McpBridge.PROTOCOL_HTTPS)) + V1McpBridge.PROTOCOL_HTTPS), + new AzureLlmProviderHandler(), + new DefaultLlmProviderHandler(LlmProviderType.AI360, "api.360.cn", 443, V1McpBridge.PROTOCOL_HTTPS), + new DefaultLlmProviderHandler(LlmProviderType.GITHUB, "models.inference.ai.azure.com", 443, + V1McpBridge.PROTOCOL_HTTPS), + new DefaultLlmProviderHandler(LlmProviderType.GROQ, "api.groq.com", 443, V1McpBridge.PROTOCOL_HTTPS), + new DefaultLlmProviderHandler(LlmProviderType.BAICHUAN, "api.baichuan-ai.com", 443, + V1McpBridge.PROTOCOL_HTTPS), + new DefaultLlmProviderHandler(LlmProviderType.YI, "api.lingyiwanwu.com", 443, V1McpBridge.PROTOCOL_HTTPS), + new DefaultLlmProviderHandler(LlmProviderType.DEEPSEEK, "api.deepseek.com", 443, + V1McpBridge.PROTOCOL_HTTPS), + new DefaultLlmProviderHandler(LlmProviderType.ZHIPUAI, "open.bigmodel.cn", 443, V1McpBridge.PROTOCOL_HTTPS), + new OllamaLlmProviderHandler(), + new DefaultLlmProviderHandler(LlmProviderType.CLAUDE, "api.minimax.chat", 443, V1McpBridge.PROTOCOL_HTTPS), + new DefaultLlmProviderHandler(LlmProviderType.BAIDU, "aip.baidubce.com", 443, V1McpBridge.PROTOCOL_HTTPS), + new DefaultLlmProviderHandler(LlmProviderType.STEPFUN, "api.stepfun.com", 443, V1McpBridge.PROTOCOL_HTTPS), + new DefaultLlmProviderHandler(LlmProviderType.MINIMAX, "api.minimax.chat", 443, V1McpBridge.PROTOCOL_HTTPS), + new DefaultLlmProviderHandler(LlmProviderType.GEMINI, "generativelanguage.googleapis.com", 443, + V1McpBridge.PROTOCOL_HTTPS), + new DefaultLlmProviderHandler(LlmProviderType.MISTRAL, "api.mistral.ai", 443, V1McpBridge.PROTOCOL_HTTPS), + new DefaultLlmProviderHandler(LlmProviderType.COHERE, "api.cohere.com", 443, V1McpBridge.PROTOCOL_HTTPS), + new DefaultLlmProviderHandler(LlmProviderType.DOUBAO, "ark.cn-beijing.volces.com", 443, + V1McpBridge.PROTOCOL_HTTPS), + new DefaultLlmProviderHandler(LlmProviderType.COZE, "api.coze.cn", 443, V1McpBridge.PROTOCOL_HTTPS)) .collect(Collectors.toMap(LlmProviderHandler::getType, p -> p)); } private final ServiceSourceService serviceSourceService; - private final WasmPluginService wasmPluginService; private final WasmPluginInstanceService wasmPluginInstanceService; - public LlmProviderServiceImpl(ServiceSourceService serviceSourceService, WasmPluginService wasmPluginService, + public LlmProviderServiceImpl(ServiceSourceService serviceSourceService, WasmPluginInstanceService wasmPluginInstanceService) { this.serviceSourceService = serviceSourceService; - this.wasmPluginService = wasmPluginService; this.wasmPluginInstanceService = wasmPluginInstanceService; } @@ -81,22 +98,17 @@ public LlmProvider addOrUpdate(LlmProvider provider) { throw new ValidationException("Provider type " + provider.getType() + " is not supported"); } + handler.validateConfig(provider.getRawConfigs()); + fillDefaultValues(provider); final String pluginName = BuiltInPluginName.AI_PROXY; WasmPluginInstance instance = wasmPluginInstanceService.query(WasmPluginInstanceScope.GLOBAL, null, pluginName, true); if (instance == null) { - WasmPlugin plugin = wasmPluginService.query(pluginName, null); - if (plugin == null) { - throw new BusinessException("Plugin " + pluginName + " not found"); - } - instance = new WasmPluginInstance(); - instance.setPluginName(plugin.getName()); - instance.setPluginVersion(plugin.getPluginVersion()); + instance = wasmPluginInstanceService.createEmptyInstance(pluginName); instance.setInternal(true); - instance.setScope(WasmPluginInstanceScope.GLOBAL); - instance.setConfigurations(new HashMap<>()); + instance.setGlobalTarget(); } instance.setEnabled(true); @@ -107,10 +119,10 @@ public LlmProvider addOrUpdate(LlmProvider provider) { instance.setConfigurations(configurations); } - Object providersObj = configurations.get(PROVIDERS_KEY); + Object providersObj = configurations.get(PROVIDERS); if (!(providersObj instanceof List)) { providersObj = new ArrayList<>(); - configurations.put(PROVIDERS_KEY, providersObj); + configurations.put(PROVIDERS, providersObj); } List providers = (List)providersObj; @@ -120,7 +132,7 @@ public LlmProvider addOrUpdate(LlmProvider provider) { continue; } Map providerMap = (Map)providerObj; - if (provider.getName().equals(providerMap.get(PROVIDER_ID_KEY))) { + if (provider.getName().equals(providerMap.get(PROVIDER_ID))) { providerConfig = providerMap; break; } @@ -129,21 +141,23 @@ public LlmProvider addOrUpdate(LlmProvider provider) { providerConfig = new HashMap<>(); providers.add(providerConfig); } + if (MapUtils.isNotEmpty(provider.getRawConfigs())) { + providerConfig.putAll(provider.getRawConfigs()); + } handler.saveConfig(provider, providerConfig); wasmPluginInstanceService.addOrUpdate(instance); - ServiceSource serviceSource = handler.buildServiceSource(provider.getName()); + ServiceSource serviceSource = handler.buildServiceSource(provider.getName(), providerConfig); serviceSourceService.addOrUpdate(serviceSource); - UpstreamService upstreamService = handler.buildUpstreamService(provider.getName()); + UpstreamService upstreamService = handler.buildUpstreamService(provider.getName(), providerConfig); WasmPluginInstance serviceInstance = new WasmPluginInstance(); serviceInstance.setPluginName(instance.getPluginName()); serviceInstance.setPluginVersion(instance.getPluginVersion()); - serviceInstance.setScope(WasmPluginInstanceScope.SERVICE); - serviceInstance.setTarget(upstreamService.getName()); + serviceInstance.setTarget(WasmPluginInstanceScope.SERVICE, upstreamService.getName()); serviceInstance.setEnabled(true); serviceInstance.setInternal(true); - serviceInstance.setConfigurations(Map.of(ACTIVE_PROVIDER_ID_KEY, provider.getName())); + serviceInstance.setConfigurations(Map.of(ACTIVE_PROVIDER_ID, provider.getName())); wasmPluginInstanceService.addOrUpdate(serviceInstance); return query(provider.getName()); @@ -167,8 +181,8 @@ public void delete(String providerName) { } // Find the global config. - WasmPluginInstance globalInstance = instances.stream() - .filter(instance -> WasmPluginInstanceScope.GLOBAL.equals(instance.getScope())).findFirst().orElse(null); + WasmPluginInstance globalInstance = + instances.stream().filter(i -> i.hasScopedTarget(WasmPluginInstanceScope.GLOBAL)).findFirst().orElse(null); if (globalInstance == null) { return; @@ -179,7 +193,7 @@ public void delete(String providerName) { return; } - Object providersObj = globalConfigurations.get(PROVIDERS_KEY); + Object providersObj = globalConfigurations.get(PROVIDERS); if (!(providersObj instanceof List)) { return; } @@ -193,7 +207,7 @@ public void delete(String providerName) { continue; } Map providerMap = (Map)providerObj; - if (providerName.equals(providerMap.get(PROVIDER_ID_KEY))) { + if (providerName.equals(providerMap.get(PROVIDER_ID))) { providers.remove(i); deletedProvider = providerMap; break; @@ -205,14 +219,14 @@ public void delete(String providerName) { } // Delete other resources related to the deleted provider. - Object type = deletedProvider.get(PROVIDER_TYPE_KEY); + Object type = deletedProvider.get(PROVIDER_TYPE); if (type != null) { LlmProviderHandler handler = PROVIDER_HANDLERS.get((String)type); if (handler != null) { - UpstreamService upstreamService = handler.buildUpstreamService(providerName); + UpstreamService upstreamService = handler.buildUpstreamService(providerName, deletedProvider); wasmPluginInstanceService.delete(WasmPluginInstanceScope.SERVICE, upstreamService.getName(), BuiltInPluginName.AI_PROXY); - ServiceSource serviceSource = handler.buildServiceSource(providerName); + ServiceSource serviceSource = handler.buildServiceSource(providerName, deletedProvider); serviceSourceService.delete(serviceSource.getName()); } } @@ -234,7 +248,7 @@ public UpstreamService buildUpstreamService(String providerName) { "Provider type " + provider.getType() + " of provider " + providerName + " is not supported"); } - return handler.buildUpstreamService(provider.getName()); + return handler.buildUpstreamService(provider.getName(), provider.getRawConfigs()); } private SortedMap getProviders() { @@ -246,7 +260,7 @@ private SortedMap getProviders() { if (MapUtils.isEmpty(instance.getConfigurations())) { return new TreeMap<>(); } - Object providersObj = instance.getConfigurations().get(PROVIDERS_KEY); + Object providersObj = instance.getConfigurations().get(PROVIDERS); if (!(providersObj instanceof List providerList)) { return new TreeMap<>(); } @@ -265,7 +279,7 @@ private SortedMap getProviders() { } private LlmProvider extractProvider(Map configurations) { - String type = MapUtils.getString(configurations, PROVIDER_TYPE_KEY); + String type = MapUtils.getString(configurations, PROVIDER_TYPE); if (StringUtils.isBlank(type)) { return null; } diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/OllamaLlmProviderHandler.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/OllamaLlmProviderHandler.java new file mode 100644 index 00000000..5171252b --- /dev/null +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/ai/OllamaLlmProviderHandler.java @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2022-2023 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.alibaba.higress.sdk.service.ai; + +import java.util.Map; + +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.StringUtils; + +import com.alibaba.higress.sdk.exception.ValidationException; +import com.alibaba.higress.sdk.model.ai.LlmProviderType; +import com.alibaba.higress.sdk.service.kubernetes.crd.mcp.V1McpBridge; +import com.alibaba.higress.sdk.util.ValidateUtil; + +public class OllamaLlmProviderHandler extends AbstractLlmProviderHandler { + + private static final String SERVER_HOST_KEY = "ollamaServerHost"; + private static final String SERVER_PORT_KEY = "ollamaServerPort"; + + private static final int DEFAULT_PORT = 11434; + + @Override + public String getType() { + return LlmProviderType.OLLAMA; + } + + @Override + public void validateConfig(Map configurations) { + if (MapUtils.isEmpty(configurations)) { + throw new ValidationException("Missing Azure specific configurations."); + } + Object serverHostObj = configurations.get(SERVER_HOST_KEY); + if (!(serverHostObj instanceof String serverHost)) { + throw new ValidationException(SERVER_HOST_KEY + " must be a string."); + } + if (StringUtils.isEmpty(serverHost)) { + throw new ValidationException(SERVER_HOST_KEY + " cannot be empty."); + } + Object serverPortObj = configurations.get(SERVER_PORT_KEY); + if (!(serverPortObj instanceof Integer serverPort)) { + throw new ValidationException(SERVER_PORT_KEY + " must be a number."); + } + if (!ValidateUtil.checkPort(serverPort)) { + throw new ValidationException(SERVER_PORT_KEY + " must be a valid port number."); + } + } + + @Override + protected String getServiceRegistryType(Map providerConfig) { + String serviceDomain = getServiceDomain(providerConfig); + if (StringUtils.isEmpty(serviceDomain)) { + return V1McpBridge.REGISTRY_TYPE_DNS; + } + if (ValidateUtil.checkIpAddress(serviceDomain)) { + return V1McpBridge.REGISTRY_TYPE_STATIC; + } + return V1McpBridge.REGISTRY_TYPE_DNS; + } + + @Override + protected String getServiceDomain(Map providerConfig) { + if (MapUtils.isEmpty(providerConfig)) { + return ""; + } + return (String)providerConfig.getOrDefault(SERVER_HOST_KEY, ""); + } + + @Override + protected int getServicePort(Map providerConfig) { + if (MapUtils.isEmpty(providerConfig)) { + return DEFAULT_PORT; + } + Object serverPortObj = providerConfig.get(SERVER_PORT_KEY); + if (!(serverPortObj instanceof Integer serverPort) || !ValidateUtil.checkPort(serverPort)) { + return DEFAULT_PORT; + } + return serverPort; + } + + @Override + protected String getServiceProtocol(Map providerConfig) { + return V1McpBridge.PROTOCOL_HTTP; + } +} diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/consumer/ConsumerServiceImpl.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/consumer/ConsumerServiceImpl.java index 990825f1..bef28b19 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/consumer/ConsumerServiceImpl.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/consumer/ConsumerServiceImpl.java @@ -26,12 +26,10 @@ import com.alibaba.higress.sdk.exception.BusinessException; import com.alibaba.higress.sdk.model.CommonPageQuery; import com.alibaba.higress.sdk.model.PaginatedResult; -import com.alibaba.higress.sdk.model.WasmPlugin; import com.alibaba.higress.sdk.model.WasmPluginInstance; import com.alibaba.higress.sdk.model.WasmPluginInstanceScope; import com.alibaba.higress.sdk.model.consumer.Consumer; import com.alibaba.higress.sdk.service.WasmPluginInstanceService; -import com.alibaba.higress.sdk.service.WasmPluginService; import lombok.extern.slf4j.Slf4j; @@ -45,12 +43,9 @@ public class ConsumerServiceImpl implements ConsumerService { Stream.of(new KeyAuthCredentialHandler()).collect(Collectors.toMap(CredentialHandler::getType, c -> c)); } - private final WasmPluginService wasmPluginService; private final WasmPluginInstanceService wasmPluginInstanceService; - public ConsumerServiceImpl(WasmPluginService wasmPluginService, - WasmPluginInstanceService wasmPluginInstanceService) { - this.wasmPluginService = wasmPluginService; + public ConsumerServiceImpl(WasmPluginInstanceService wasmPluginInstanceService) { this.wasmPluginInstanceService = wasmPluginInstanceService; } @@ -61,11 +56,9 @@ public Consumer addOrUpdate(Consumer consumer) { WasmPluginInstance instance = wasmPluginInstanceService.query(WasmPluginInstanceScope.GLOBAL, null, config.getPluginName(), true); if (instance == null) { - WasmPlugin plugin = wasmPluginService.query(config.getPluginName(), null); - if (plugin == null) { - throw new BusinessException("Plugin " + config.getPluginName() + " not found"); - } - instance = createInstance(WasmPluginInstanceScope.GLOBAL, null, config.getPluginName()); + instance = wasmPluginInstanceService.createEmptyInstance(config.getPluginName()); + instance.setInternal(true); + instance.setGlobalTarget(); } if (config.saveConsumer(instance, consumer)) { instancesToUpdate.add(instance); @@ -100,7 +93,7 @@ public void delete(String consumerName) { for (CredentialHandler config : CREDENTIAL_HANDLERS.values()) { List instances = instancesCache.get(config.getType()); WasmPluginInstance globalInstance = instances.stream() - .filter(i -> WasmPluginInstanceScope.GLOBAL.equals(i.getScope())).findFirst().orElse(null); + .filter(i -> i.hasScopedTarget(WasmPluginInstanceScope.GLOBAL)).findFirst().orElse(null); if (globalInstance == null) { continue; } @@ -122,7 +115,9 @@ public void updateAllowList(WasmPluginInstanceScope scope, String target, List getConsumers() { } return consumers; } - - private WasmPluginInstance createInstance(WasmPluginInstanceScope scope, String target, String pluginName) { - WasmPlugin plugin = wasmPluginService.query(pluginName, null); - if (plugin == null) { - throw new BusinessException("Plugin " + pluginName + " not found"); - } - WasmPluginInstance instance = new WasmPluginInstance(); - instance.setPluginName(plugin.getName()); - instance.setPluginVersion(plugin.getPluginVersion()); - instance.setInternal(true); - instance.setScope(scope); - instance.setTarget(target); - return instance; - } } diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/consumer/KeyAuthCredentialHandler.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/consumer/KeyAuthCredentialHandler.java index 381e56e3..9590a988 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/consumer/KeyAuthCredentialHandler.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/consumer/KeyAuthCredentialHandler.java @@ -12,6 +12,14 @@ */ package com.alibaba.higress.sdk.service.consumer; +import static com.alibaba.higress.sdk.constant.plugin.config.KeyAuthConfig.ALLOW; +import static com.alibaba.higress.sdk.constant.plugin.config.KeyAuthConfig.CONSUMERS; +import static com.alibaba.higress.sdk.constant.plugin.config.KeyAuthConfig.CONSUMER_CREDENTIAL; +import static com.alibaba.higress.sdk.constant.plugin.config.KeyAuthConfig.IN_HEADER; +import static com.alibaba.higress.sdk.constant.plugin.config.KeyAuthConfig.IN_QUERY; +import static com.alibaba.higress.sdk.constant.plugin.config.KeyAuthConfig.KEYS; +import static com.alibaba.higress.sdk.constant.plugin.config.KeyAuthConfig.CONSUMER_NAME; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -21,7 +29,7 @@ import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; -import com.alibaba.higress.sdk.constant.BuiltInPluginName; +import com.alibaba.higress.sdk.constant.plugin.BuiltInPluginName; import com.alibaba.higress.sdk.model.WasmPluginInstance; import com.alibaba.higress.sdk.model.consumer.Consumer; import com.alibaba.higress.sdk.model.consumer.CredentialType; @@ -31,14 +39,6 @@ class KeyAuthCredentialHandler implements CredentialHandler { - private static final String CONSUMERS_KEY = "consumers"; - private static final String NAME_KEY = "name"; - private static final String IN_HEADER_KEY = "in_header"; - private static final String IN_QUERY_KEY = "in_query"; - private static final String KEYS_KEY = "keys"; - private static final String CREDENTIAL_KEY = "credential"; - private static final String ALLOW_KEY = "allow"; - private static final String BEARER_TOKEN_PREFIX = "Bearer "; @Override @@ -61,7 +61,7 @@ public boolean isConsumerInUse(String consumerName, List ins if (MapUtils.isEmpty(configurations)) { return false; } - Object allowObj = configurations.get(ALLOW_KEY); + Object allowObj = configurations.get(ALLOW); if (allowObj instanceof List allowList && allowList.contains(consumerName)) { return true; } @@ -75,7 +75,7 @@ public List extractConsumers(WasmPluginInstance instance) { if (MapUtils.isEmpty(instance.getConfigurations())) { return List.of(); } - Object consumersObj = instance.getConfigurations().get(CONSUMERS_KEY); + Object consumersObj = instance.getConfigurations().get(CONSUMERS); if (!(consumersObj instanceof List consumerList)) { return List.of(); } @@ -87,7 +87,7 @@ public List extractConsumers(WasmPluginInstance instance) { Map consumerMap = (Map)consumerObj; - String name = MapUtils.getString(consumerMap, NAME_KEY); + String name = MapUtils.getString(consumerMap, CONSUMER_NAME); if (StringUtils.isBlank(name)) { continue; } @@ -126,9 +126,9 @@ public boolean saveConsumer(WasmPluginInstance instance, Consumer consumer) { // TODO: Remove this after plugin upgrade. // Add a dummy key to the global keys list because the plugin requires at least one global key. - configurations.put(KEYS_KEY, List.of("x-higress-dummy-key")); + configurations.put(KEYS, List.of("x-higress-dummy-key")); - Object consumersObj = configurations.computeIfAbsent(CONSUMERS_KEY, k -> new ArrayList<>()); + Object consumersObj = configurations.computeIfAbsent(CONSUMERS, k -> new ArrayList<>()); List consumers; if (consumersObj instanceof List) { consumers = new ArrayList<>((List)consumersObj); @@ -141,7 +141,7 @@ public boolean saveConsumer(WasmPluginInstance instance, Consumer consumer) { continue; } Map consumerMap = (Map)consumerObj; - if (consumer.getName().equals(consumerMap.get(NAME_KEY))) { + if (consumer.getName().equals(consumerMap.get(CONSUMER_NAME))) { consumerConfig = consumerMap; break; } @@ -149,7 +149,7 @@ public boolean saveConsumer(WasmPluginInstance instance, Consumer consumer) { if (consumerConfig == null) { consumerConfig = new HashMap<>(); - consumerConfig.put(NAME_KEY, consumer.getName()); + consumerConfig.put(CONSUMER_NAME, consumer.getName()); consumers.add(consumerConfig); } else { keyAuthCredential = mergeExistedConfig(keyAuthCredential, consumerConfig); @@ -166,25 +166,25 @@ public boolean saveConsumer(WasmPluginInstance instance, Consumer consumer) { switch (sourceEnum) { case BEARER: case HEADER: - consumerConfig.put(IN_HEADER_KEY, true); - consumerConfig.put(IN_QUERY_KEY, false); + consumerConfig.put(IN_HEADER, true); + consumerConfig.put(IN_QUERY, false); if (sourceEnum == KeyAuthCredentialSource.BEARER) { key = HttpHeaders.AUTHORIZATION; credential = BEARER_TOKEN_PREFIX + keyAuthCredential.getValue(); } break; case QUERY: - consumerConfig.put(IN_HEADER_KEY, false); - consumerConfig.put(IN_QUERY_KEY, true); + consumerConfig.put(IN_HEADER, false); + consumerConfig.put(IN_QUERY, true); break; default: throw new IllegalArgumentException( "Unsupported key auth credential source: " + keyAuthCredential.getSource()); } - consumerConfig.put(KEYS_KEY, List.of(key)); - consumerConfig.put(CREDENTIAL_KEY, credential); + consumerConfig.put(KEYS, List.of(key)); + consumerConfig.put(CONSUMER_CREDENTIAL, credential); - configurations.put(CONSUMERS_KEY, consumers); + configurations.put(CONSUMERS, consumers); return true; } @@ -195,7 +195,7 @@ public boolean deleteConsumer(WasmPluginInstance globalInstance, String consumer if (MapUtils.isEmpty(globalConfigurations)) { return false; } - Object consumersObj = globalConfigurations.get(CONSUMERS_KEY); + Object consumersObj = globalConfigurations.get(CONSUMERS); if (!(consumersObj instanceof List)) { return false; } @@ -207,7 +207,7 @@ public boolean deleteConsumer(WasmPluginInstance globalInstance, String consumer continue; } Map consumerMap = (Map)consumerObj; - if (consumerName.equals(consumerMap.get(NAME_KEY))) { + if (consumerName.equals(consumerMap.get(CONSUMER_NAME))) { consumers.remove(i); deleted = true; } @@ -224,9 +224,9 @@ public void updateAllowList(WasmPluginInstance instance, List consumerNa } if (CollectionUtils.isEmpty(consumerNames)) { - configurations.remove(ALLOW_KEY); + configurations.remove(ALLOW); } else { - configurations.put(ALLOW_KEY, new ArrayList<>(consumerNames)); + configurations.put(ALLOW, new ArrayList<>(consumerNames)); } } @@ -244,12 +244,12 @@ private KeyAuthCredential mergeExistedConfig(KeyAuthCredential keyAuthCredential } private static KeyAuthCredential parseCredential(Map consumerMap) { - String credential = MapUtils.getString(consumerMap, CREDENTIAL_KEY); + String credential = MapUtils.getString(consumerMap, CONSUMER_CREDENTIAL); if (StringUtils.isBlank(credential)) { return null; } - Object keyObj = MapUtils.getObject(consumerMap, KEYS_KEY); + Object keyObj = MapUtils.getObject(consumerMap, KEYS); if (!(keyObj instanceof List keyList) || keyList.isEmpty()) { return null; } @@ -267,8 +267,8 @@ private static KeyAuthCredential parseCredential(Map consumerMap return null; } - Boolean inHeader = MapUtils.getBoolean(consumerMap, IN_HEADER_KEY); - Boolean inQuery = MapUtils.getBoolean(consumerMap, IN_QUERY_KEY); + Boolean inHeader = MapUtils.getBoolean(consumerMap, IN_HEADER); + Boolean inQuery = MapUtils.getBoolean(consumerMap, IN_QUERY); KeyAuthCredentialSource source; if (Boolean.TRUE.equals(inHeader)) { diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/kubernetes/KubernetesModelConverter.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/kubernetes/KubernetesModelConverter.java index 2a8ac14a..d627dd85 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/kubernetes/KubernetesModelConverter.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/kubernetes/KubernetesModelConverter.java @@ -77,6 +77,7 @@ import com.alibaba.higress.sdk.service.kubernetes.crd.wasm.PluginPhase; import com.alibaba.higress.sdk.service.kubernetes.crd.wasm.V1alpha1WasmPlugin; import com.alibaba.higress.sdk.service.kubernetes.crd.wasm.V1alpha1WasmPluginSpec; +import com.alibaba.higress.sdk.util.MapUtil; import com.alibaba.higress.sdk.util.TypeUtil; import com.google.common.base.Splitter; import com.google.gson.Gson; @@ -503,35 +504,44 @@ public List getWasmPluginInstancesFromCr(V1alpha1WasmPlugin List instances = new ArrayList<>(); if (ObjectUtils.anyNotNull(spec.getDefaultConfigDisable(), spec.getDefaultConfig())) { - WasmPluginInstance instance = WasmPluginInstance.builder().scope(WasmPluginInstanceScope.GLOBAL) - .enabled(Boolean.FALSE.equals(spec.getDefaultConfigDisable())).configurations(spec.getDefaultConfig()) - .build(); + WasmPluginInstance instance = + WasmPluginInstance.builder().targets(MapUtil.of(WasmPluginInstanceScope.GLOBAL, null)) + .enabled(Boolean.FALSE.equals(spec.getDefaultConfigDisable())) + .configurations(spec.getDefaultConfig()).build(); instances.add(instance); } if (CollectionUtils.isNotEmpty(spec.getMatchRules())) { + List matchRules = new ArrayList<>(spec.getMatchRules()); + // Sort here so we can always get the correct order of instances. + sortWasmPluginMatchRules(matchRules); for (MatchRule rule : spec.getMatchRules()) { boolean enabled = Boolean.FALSE.equals(rule.getConfigDisable()); - if (CollectionUtils.isNotEmpty(rule.getDomain())) { - for (String domain : rule.getDomain()) { - WasmPluginInstance instance = WasmPluginInstance.builder().scope(WasmPluginInstanceScope.DOMAIN) - .target(domain).enabled(enabled).configurations(rule.getConfig()).build(); - instances.add(instance); + List> targetsList = new ArrayList<>(); + for (WasmPluginInstanceScope scope : WasmPluginInstanceScope.NON_GLOBAL_SCOPES) { + List targets = getTargetsByScope(rule, scope); + if (CollectionUtils.isEmpty(targets)) { + continue; } - } - if (CollectionUtils.isNotEmpty(rule.getIngress())) { - for (String route : rule.getIngress()) { - WasmPluginInstance instance = WasmPluginInstance.builder().scope(WasmPluginInstanceScope.ROUTE) - .target(route).enabled(enabled).configurations(rule.getConfig()).build(); - instances.add(instance); + if (targetsList.isEmpty()) { + for (String target : targets) { + targetsList.add(MapUtil.of(scope, target)); + } + } else { + List> newTargetsList = new ArrayList<>(); + for (Map existedTargets : targetsList) { + for (String target : targets) { + Map newTargets = new HashMap<>(existedTargets); + newTargets.put(scope, target); + newTargetsList.add(newTargets); + } + } + targetsList = newTargetsList; } } - if (CollectionUtils.isNotEmpty(rule.getService())) { - for (String route : rule.getService()) { - WasmPluginInstance instance = - WasmPluginInstance.builder().scope(WasmPluginInstanceScope.SERVICE).target(route) - .enabled(enabled).configurations(rule.getConfig()).build(); - instances.add(instance); - } + for (Map targets : targetsList) { + WasmPluginInstance instance = WasmPluginInstance.builder().targets(targets).enabled(enabled) + .configurations(rule.getConfig()).build(); + instances.add(instance); } } } @@ -543,19 +553,22 @@ public List getWasmPluginInstancesFromCr(V1alpha1WasmPlugin normalizePluginInstanceConfigurations(i.getConfigurations()); i.setRawConfigurations(generateRawConfigurations(i.getConfigurations())); i.setInternal(KubernetesUtil.isInternalResource(plugin)); - }); - instances.sort((i1, i2) -> { - int ret = i1.getScope().compareTo(i2.getScope()); - if (ret != 0) { - return ret; - } - return ObjectUtils.compare(i1.getTarget(), i2.getTarget()); + i.syncDeprecatedFields(); }); return instances; } public WasmPluginInstance getWasmPluginInstanceFromCr(V1alpha1WasmPlugin plugin, WasmPluginInstanceScope scope, String target) { + return getWasmPluginInstanceFromCr(plugin, MapUtil.of(scope, target)); + } + + public WasmPluginInstance getWasmPluginInstanceFromCr(V1alpha1WasmPlugin plugin, + Map targets) { + if (MapUtils.isEmpty(targets)) { + return null; + } + V1ObjectMeta metadata = plugin.getMetadata(); if (metadata == null || MapUtils.isEmpty(metadata.getLabels())) { return null; @@ -574,45 +587,38 @@ public WasmPluginInstance getWasmPluginInstanceFromCr(V1alpha1WasmPlugin plugin, Boolean enabled = null; Map configurations = null; - switch (scope) { - case GLOBAL: - if (target == null) { - enabled = !Boolean.TRUE.equals(spec.getDefaultConfigDisable()); - configurations = Optional.ofNullable(spec.getDefaultConfig()).orElseGet(HashMap::new); - } - break; - case DOMAIN: - if (StringUtils.isNotEmpty(target) && CollectionUtils.isNotEmpty(spec.getMatchRules())) { - Optional rule = spec.getMatchRules().stream() - .filter(r -> r.getDomain() != null && r.getDomain().contains(target)).findFirst(); - if (rule.isPresent()) { - enabled = !Boolean.TRUE.equals(rule.get().getConfigDisable()); - configurations = rule.get().getConfig(); + if (targets.containsKey(WasmPluginInstanceScope.GLOBAL)) { + if (targets.size() != 1) { + // We don't support query instances by global and another instance scope. + return null; + } + if (targets.get(WasmPluginInstanceScope.GLOBAL) != null) { + return null; + } + enabled = !Boolean.TRUE.equals(spec.getDefaultConfigDisable()); + configurations = Optional.ofNullable(spec.getDefaultConfig()).orElseGet(HashMap::new); + } else if (CollectionUtils.isNotEmpty(spec.getMatchRules())) { + for (MatchRule rule : spec.getMatchRules()) { + boolean matched = true; + for (Map.Entry entry : targets.entrySet()) { + WasmPluginInstanceScope scope = entry.getKey(); + String target = entry.getValue(); + if (StringUtils.isEmpty(target)) { + continue; } - } - break; - case ROUTE: - if (StringUtils.isNotEmpty(target) && CollectionUtils.isNotEmpty(spec.getMatchRules())) { - Optional rule = spec.getMatchRules().stream() - .filter(r -> r.getIngress() != null && r.getIngress().contains(target)).findFirst(); - if (rule.isPresent()) { - enabled = !Boolean.TRUE.equals(rule.get().getConfigDisable()); - configurations = rule.get().getConfig(); + List targetsInMatchRule = getTargetsByScope(rule, scope); + if (targetsInMatchRule == null || targetsInMatchRule.size() != 1 + || !target.equals(targetsInMatchRule.get(0))) { + matched = false; + break; } } - break; - case SERVICE: - if (StringUtils.isNotEmpty(target) && CollectionUtils.isNotEmpty(spec.getMatchRules())) { - Optional rule = spec.getMatchRules().stream() - .filter(r -> r.getService() != null && r.getService().contains(target)).findFirst(); - if (rule.isPresent()) { - enabled = !Boolean.TRUE.equals(rule.get().getConfigDisable()); - configurations = rule.get().getConfig(); - } + if (matched) { + enabled = !Boolean.TRUE.equals(rule.getConfigDisable()); + configurations = rule.getConfig(); + break; } - break; - default: - throw new IllegalArgumentException("Unsupported scope: " + scope); + } } if (enabled == null) { @@ -626,9 +632,33 @@ public WasmPluginInstance getWasmPluginInstanceFromCr(V1alpha1WasmPlugin plugin, normalizePluginInstanceConfigurations(configurations); String rawConfiguration = generateRawConfigurations(configurations); - return WasmPluginInstance.builder().version(metadata.getResourceVersion()).pluginName(name) - .pluginVersion(version).scope(scope).target(target).enabled(enabled).configurations(configurations) - .rawConfigurations(rawConfiguration).internal(KubernetesUtil.isInternalResource(plugin)).build(); + WasmPluginInstance instance = + WasmPluginInstance.builder().version(metadata.getResourceVersion()).pluginName(name).pluginVersion(version) + .targets(new HashMap<>(targets)).enabled(enabled).configurations(configurations) + .rawConfigurations(rawConfiguration).internal(KubernetesUtil.isInternalResource(plugin)).build(); + instance.syncDeprecatedFields(); + return instance; + } + + private List getTargetsByScope(MatchRule rule, WasmPluginInstanceScope scope) { + return switch (scope) { + case GLOBAL -> null; + case DOMAIN -> rule.getDomain(); + case ROUTE -> rule.getIngress(); + case SERVICE -> rule.getService(); + }; + } + + @SuppressWarnings("PMD.SwitchStatementRule") + private void setTargetByScope(MatchRule rule, WasmPluginInstanceScope scope, String target) { + switch (scope) { + case GLOBAL -> { + } + case DOMAIN -> rule.setDomain(List.of(target)); + case ROUTE -> rule.setIngress(List.of(target)); + case SERVICE -> rule.setService(List.of(target)); + default -> throw new IllegalArgumentException("Unsupported scope: " + scope); + } } @SuppressWarnings("unchecked") @@ -669,6 +699,8 @@ private String generateRawConfigurations(Map configurations) { } public void setWasmPluginInstanceToCr(V1alpha1WasmPlugin cr, WasmPluginInstance instance) { + instance.syncDeprecatedFields(); + V1alpha1WasmPluginSpec spec = cr.getSpec(); if (spec == null) { spec = new V1alpha1WasmPluginSpec(); @@ -685,127 +717,93 @@ public void setWasmPluginInstanceToCr(V1alpha1WasmPlugin cr, WasmPluginInstance boolean enabled = instance.getEnabled() == null || instance.getEnabled(); Map configurations = instance.getConfigurations(); - WasmPluginInstanceScope scope = instance.getScope(); - switch (scope) { - case GLOBAL: - if (instance.getTarget() == null) { + Map targets = instance.getTargets(); + if (MapUtils.isNotEmpty(targets)) { + if (targets.containsKey(WasmPluginInstanceScope.GLOBAL)) { + if (targets.size() == 1 && targets.get(WasmPluginInstanceScope.GLOBAL) == null) { spec.setDefaultConfigDisable(!enabled); spec.setDefaultConfig(configurations); } - break; - case DOMAIN: - String domain = instance.getTarget(); - MatchRule domainRule = matchRules.stream() - .filter(r -> r.getDomain() != null && r.getDomain().contains(domain)).findFirst().orElse(null); - if (domainRule == null) { - domainRule = MatchRule.forDomain(domain); - matchRules.add(domainRule); + } else { + MatchRule targetMatchRule = new MatchRule(); + targetMatchRule.setConfigDisable(!enabled); + targetMatchRule.setConfig(configurations); + for (Map.Entry entry : targets.entrySet()) { + setTargetByScope(targetMatchRule, entry.getKey(), entry.getValue()); } - domainRule.setConfigDisable(!enabled); - domainRule.setConfig(configurations); - break; - case ROUTE: - String route = instance.getTarget(); - MatchRule routeRule = matchRules.stream() - .filter(r -> r.getIngress() != null && r.getIngress().contains(route)).findFirst().orElse(null); - if (routeRule == null) { - routeRule = MatchRule.forIngress(route); - matchRules.add(routeRule); + + MatchRule existedMatchRule = null; + for (MatchRule matchRule : matchRules) { + if (matchRule.keyEquals(targetMatchRule)) { + existedMatchRule = matchRule; + break; + } } - routeRule.setConfigDisable(!enabled); - routeRule.setConfig(configurations); - break; - case SERVICE: - String service = instance.getTarget(); - MatchRule serviceRule = matchRules.stream() - .filter(r -> r.getService() != null && r.getService().contains(service)).findFirst().orElse(null); - if (serviceRule == null) { - serviceRule = MatchRule.forService(service); - matchRules.add(serviceRule); + if (existedMatchRule != null) { + matchRules.remove(existedMatchRule); } - serviceRule.setConfigDisable(!enabled); - serviceRule.setConfig(configurations); - break; - default: - throw new IllegalArgumentException("Unsupported scope: " + scope); + matchRules.add(targetMatchRule); + } } sortWasmPluginMatchRules(matchRules); setDefaultValues(spec); } public boolean removeWasmPluginInstanceFromCr(V1alpha1WasmPlugin cr, WasmPluginInstanceScope scope, String target) { + return removeWasmPluginInstanceFromCr(cr, MapUtil.of(scope, target)); + } + + public boolean removeWasmPluginInstanceFromCr(V1alpha1WasmPlugin cr, Map targets) { + if (MapUtils.isEmpty(targets)) { + return false; + } + V1alpha1WasmPluginSpec spec = cr.getSpec(); if (spec == null) { return false; } + if (targets.containsKey(WasmPluginInstanceScope.GLOBAL)) { + if (targets.size() != 1 || targets.get(WasmPluginInstanceScope.GLOBAL) != null) { + return false; + } + spec.setDefaultConfigDisable(true); + spec.setDefaultConfig(null); + return true; + } + + if (CollectionUtils.isEmpty(spec.getMatchRules())) { + return false; + } + boolean changed = false; - switch (scope) { - case GLOBAL: - if (target == null) { - spec.setDefaultConfig(null); - changed = true; - } - break; - case DOMAIN: - if (StringUtils.isEmpty(target) || CollectionUtils.isEmpty(spec.getMatchRules())) { - break; - } - for (Iterator it = spec.getMatchRules().listIterator(); it.hasNext();) { - MatchRule rule = it.next(); - if (CollectionUtils.isEmpty(rule.getDomain())) { - continue; - } - if (!rule.getDomain().contains(target)) { - continue; - } - changed = true; - rule.getDomain().remove(target); - if (CollectionUtils.isEmpty(rule.getDomain()) && CollectionUtils.isEmpty(rule.getIngress())) { - it.remove(); - } - } - break; - case ROUTE: - if (StringUtils.isEmpty(target) || CollectionUtils.isEmpty(spec.getMatchRules())) { - break; - } - for (Iterator it = spec.getMatchRules().listIterator(); it.hasNext();) { - MatchRule rule = it.next(); - if (CollectionUtils.isEmpty(rule.getIngress())) { - continue; - } - if (!rule.getIngress().contains(target)) { - continue; - } - changed = true; - rule.getIngress().remove(target); - if (rule.isEmpty()) { - it.remove(); - } - } - break; - case SERVICE: - if (StringUtils.isEmpty(target) || CollectionUtils.isEmpty(spec.getMatchRules())) { + for (Iterator it = spec.getMatchRules().listIterator(); it.hasNext();) { + MatchRule rule = it.next(); + + boolean matches = true; + + for (Map.Entry entry : targets.entrySet()) { + List targetsInRule = getTargetsByScope(rule, entry.getKey()); + if (targetsInRule == null || !targetsInRule.contains(entry.getValue())) { + matches = false; break; } - for (Iterator it = spec.getMatchRules().listIterator(); it.hasNext();) { - MatchRule rule = it.next(); - if (CollectionUtils.isEmpty(rule.getService())) { - continue; - } - if (!rule.getService().contains(target)) { - continue; - } - changed = true; - rule.getService().remove(target); - if (rule.isEmpty()) { - it.remove(); - } - } - break; - default: - throw new IllegalArgumentException("Unsupported scope: " + scope); + } + + if (!matches) { + continue; + } + + for (Map.Entry entry : targets.entrySet()) { + List targetsInRule = Objects.requireNonNull(getTargetsByScope(rule, entry.getKey())); + targetsInRule.remove(entry.getValue()); + } + + if (rule.hasKey()) { + it.remove(); + } + + changed = true; } return changed; } @@ -874,7 +872,7 @@ private static int compareMatchRules(MatchRule r1, MatchRule r2) { private static void setDefaultValues(V1alpha1WasmPluginSpec spec) { spec.setFailStrategy(FailStrategy.FAIL_OPEN.getName()); - if (spec.getDefaultConfigDisable() == null){ + if (spec.getDefaultConfigDisable() == null) { spec.setDefaultConfigDisable(true); } } diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/kubernetes/crd/mcp/V1McpBridge.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/kubernetes/crd/mcp/V1McpBridge.java index aa5dc694..6c8bf36c 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/kubernetes/crd/mcp/V1McpBridge.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/kubernetes/crd/mcp/V1McpBridge.java @@ -77,6 +77,8 @@ public class V1McpBridge implements io.kubernetes.client.common.KubernetesObject public static final String PROTOCOL_GRPCS = "grpcs"; + public static final int STATIC_PORT = 80; + public static final String SERIALIZED_NAME_API_VERSION = "apiVersion"; @SerializedName(SERIALIZED_NAME_API_VERSION) private String apiVersion = API_GROUP + "/" + VERSION; diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/kubernetes/crd/wasm/MatchRule.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/kubernetes/crd/wasm/MatchRule.java index ff265d1d..9c49c7b4 100644 --- a/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/kubernetes/crd/wasm/MatchRule.java +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/service/kubernetes/crd/wasm/MatchRule.java @@ -22,6 +22,8 @@ import lombok.Data; import lombok.NoArgsConstructor; +import static com.alibaba.higress.sdk.util.ListUtil.equalsUnordered; + @Data @NoArgsConstructor @AllArgsConstructor @@ -50,28 +52,11 @@ public static MatchRule forService(String service) { } public boolean keyEquals(MatchRule rule) { - if ((domain == null) == (rule.domain != null)) { - return false; - } - if ((ingress == null) == (rule.ingress != null)) { - return false; - } - if ((service == null) == (rule.service != null)) { - return false; - } - if (domain != null && (domain.size() != rule.domain.size() || !domain.containsAll(rule.domain))) { - return false; - } - if (ingress != null && (ingress.size() != rule.ingress.size() || !ingress.containsAll(rule.ingress))) { - return false; - } - if (service != null && (service.size() != rule.service.size() || !service.containsAll(rule.service))) { - return false; - } - return true; + return equalsUnordered(domain, rule.domain) && equalsUnordered(ingress, rule.ingress) + && equalsUnordered(service, rule.service); } - public boolean isEmpty() { + public boolean hasKey() { return CollectionUtils.isEmpty(domain) && CollectionUtils.isEmpty(ingress) && CollectionUtils.isEmpty(service); } } diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/util/ListUtil.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/util/ListUtil.java new file mode 100644 index 00000000..2668e4b0 --- /dev/null +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/util/ListUtil.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2022-2024 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.alibaba.higress.sdk.util; + +import java.util.HashSet; +import java.util.List; + +import org.apache.commons.collections4.CollectionUtils; + +public class ListUtil { + + public static boolean equalsUnordered(List list1, List list2) { + if (CollectionUtils.isEmpty(list1) && CollectionUtils.isEmpty(list2)) { + return true; + } + if (list1 == null || list2 == null) { + return false; + } + return list1.size() == list2.size() && new HashSet<>(list1).containsAll(list2); + } +} diff --git a/backend/sdk/src/main/java/com/alibaba/higress/sdk/util/MapUtil.java b/backend/sdk/src/main/java/com/alibaba/higress/sdk/util/MapUtil.java new file mode 100644 index 00000000..6ebcc9bc --- /dev/null +++ b/backend/sdk/src/main/java/com/alibaba/higress/sdk/util/MapUtil.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2022-2024 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.alibaba.higress.sdk.util; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.HashMap; +import java.util.Map; + +public class MapUtil { + + public static Map of(@NotNull K k1, @Nullable V v1) { + Map map = new HashMap<>(); + map.put(k1, v1); + return map; + } +} diff --git a/backend/sdk/src/main/resources/plugins/model-mapper/README.md b/backend/sdk/src/main/resources/plugins/model-mapper/README.md new file mode 100644 index 00000000..1e9fc747 --- /dev/null +++ b/backend/sdk/src/main/resources/plugins/model-mapper/README.md @@ -0,0 +1,63 @@ +## 功能说明 +`model-mapper`插件实现了基于LLM协议中的model参数路由的功能 + +## 配置字段 + +| 名称 | 数据类型 | 填写要求 | 默认值 | 描述 | +| ----------- | --------------- | ----------------------- | ------ | ------------------------------------------- | +| `modelKey` | string | 选填 | model | 请求body中model参数的位置 | +| `modelMapping` | map of string | 选填 | - | AI 模型映射表，用于将请求中的模型名称映射为服务提供商支持模型名称。
1. 支持前缀匹配。例如用 "gpt-3-*" 匹配所有名称以“gpt-3-”开头的模型；
2. 支持使用 "*" 为键来配置通用兜底映射关系；
3. 如果映射的目标名称为空字符串 ""，则表示保留原模型名称。 | +| `enableOnPathSuffix` | array of string | 选填 | ["/v1/chat/completions"] | 只对这些特定路径后缀的请求生效 ## 运行属性 + +插件执行阶段：认证阶段 +插件执行优先级：800 + | +## 效果说明 + +如下配置 + +```yaml +modelMapping: + 'gpt-4-*': "qwen-max" + 'gpt-4o': "qwen-vl-plus" + '*': "qwen-turbo" +``` + +开启后，`gpt-4-` 开头的模型参数会被改写为 `qwen-max`, `gpt-4o` 会被改写为 `qwen-vl-plus`，其他所有模型会被改写为 `qwen-turbo` + +例如原本的请求是： + +```json +{ + "model": "gpt-4o", + "frequency_penalty": 0, + "max_tokens": 800, + "stream": false, + "messages": [{ + "role": "user", + "content": "higress项目主仓库的github地址是什么" + }], + "presence_penalty": 0, + "temperature": 0.7, + "top_p": 0.95 +} +``` + + +经过这个插件后，原始的 LLM 请求体将被改成： + +```json +{ + "model": "qwen-vl-plus", + "frequency_penalty": 0, + "max_tokens": 800, + "stream": false, + "messages": [{ + "role": "user", + "content": "higress项目主仓库的github地址是什么" + }], + "presence_penalty": 0, + "temperature": 0.7, + "top_p": 0.95 +} +``` diff --git a/backend/sdk/src/main/resources/plugins/model-mapper/README_EN.md b/backend/sdk/src/main/resources/plugins/model-mapper/README_EN.md new file mode 100644 index 00000000..f38cc84f --- /dev/null +++ b/backend/sdk/src/main/resources/plugins/model-mapper/README_EN.md @@ -0,0 +1,65 @@ +## Function Description +The `model-mapper` plugin implements the functionality of routing based on the model parameter in the LLM protocol. + +## Configuration Fields + +| Name | Data Type | Filling Requirement | Default Value | Description | +| ----------- | --------------- | ----------------------- | ------ | ------------------------------------------- | +| `modelKey` | string | Optional | model | The location of the model parameter in the request body. | +| `modelMapping` | map of string | Optional | - | AI model mapping table, used to map the model names in the request to the model names supported by the service provider.
1. Supports prefix matching. For example, use "gpt-3-*" to match all models whose names start with “gpt-3-”;
2. Supports using "*" as the key to configure a generic fallback mapping relationship;
3. If the target name in the mapping is an empty string "", it means to keep the original model name. | +| `enableOnPathSuffix` | array of string | Optional | ["/v1/chat/completions"] | Only applies to requests with these specific path suffixes. | + +## Runtime Properties + +Plugin execution phase: Authentication phase +Plugin execution priority: 800 + +## Effect Description + +With the following configuration: + +```yaml +modelMapping: + 'gpt-4-*': "qwen-max" + 'gpt-4o': "qwen-vl-plus" + '*': "qwen-turbo" +``` + +After enabling, model parameters starting with `gpt-4-` will be rewritten to `qwen-max`, `gpt-4o` will be rewritten to `qwen-vl-plus`, and all other models will be rewritten to `qwen-turbo`. + +For example, if the original request was: + +```json +{ + "model": "gpt-4o", + "frequency_penalty": 0, + "max_tokens": 800, + "stream": false, + "messages": [{ + "role": "user", + "content": "What is the GitHub address of the main repository for the higress project?" + }], + "presence_penalty": 0, + "temperature": 0.7, + "top_p": 0.95 +} +``` + + +After processing by this plugin, the original LLM request body will be modified to: + +```json +{ + "model": "qwen-vl-plus", + "frequency_penalty": 0, + "max_tokens": 800, + "stream": false, + "messages": [{ + "role": "user", + "content": "What is the GitHub address of the main repository for the higress project?" + }], + "presence_penalty": 0, + "temperature": 0.7, + "top_p": 0.95 +} +``` diff --git a/backend/sdk/src/main/resources/plugins/model-mapper/spec.yaml b/backend/sdk/src/main/resources/plugins/model-mapper/spec.yaml new file mode 100644 index 00000000..b428d432 --- /dev/null +++ b/backend/sdk/src/main/resources/plugins/model-mapper/spec.yaml @@ -0,0 +1,55 @@ +apiVersion: 1.0.0 +info: + gatewayMinVersion: "2.0.0" + category: ai + name: model-mapper + title: AI Model Mapper + x-title-i18n: + zh-CN: AI 模型映射 + description: Implements the functionality of rule-based model value mapping in the LLM protocol. + x-description-i18n: + zh-CN: 实现了将 LLM 协议中的模型参数取值按照规则进行映射的功能。 + iconUrl: https://img.alicdn.com/imgextra/i1/O1CN018iKKih1iVx287RltL_!!6000000004419-2-tps-42-42.png + version: 1.0.0 + contact: + name: johnlanni +spec: + phase: AUTHN + priority: 800 + configSchema: + openAPIV3Schema: + type: object + properties: + modelKey: + type: string + title: Model Key + x-title-i18n: + zh-CN: Model 字段路径 + description: The location of the model parameter in the request body. + x-description-i18n: + zh-CN: 请求body中model参数的位置 + modelMapping: + type: object + title: Model Mapping + x-title-i18n: + zh-CN: 映射关系 + description: The mapping of model values. You can use "*" as a wildcard for whole or prefix mapping. + x-description-i18n: + zh-CN: 模型名称映射关系。可使用“*”作为通配符进行完全或前缀匹配。 + additionalProperties: + type: string + enableOnPathSuffix: + type: array + title: Enable On Path Suffixes + x-title-i18n: + zh-CN: 生效的请求路径 + description: Only applies to requests with these specific path suffixes. + x-description-i18n: + zh-CN: 只对这些特定路径后缀的请求生效 + items: + type: string + example: + modelMapping: + 'gpt-4-*': "qwen-max" + 'gpt-4o': "qwen-vl-plus" + '*': "qwen-turbo" diff --git a/backend/sdk/src/main/resources/plugins/model-router/README.md b/backend/sdk/src/main/resources/plugins/model-router/README.md index b63be35d..e78988e0 100644 --- a/backend/sdk/src/main/resources/plugins/model-router/README.md +++ b/backend/sdk/src/main/resources/plugins/model-router/README.md @@ -1,33 +1,67 @@ ## 功能说明 `model-router`插件实现了基于LLM协议中的model参数路由的功能 +## 配置字段 + +| 名称 | 数据类型 | 填写要求 | 默认值 | 描述 | +| ----------- | --------------- | ----------------------- | ------ | ------------------------------------------- | +| `modelKey` | string | 选填 | model | 请求body中model参数的位置 | +| `addProviderHeader` | string | 选填 | - | 从model参数中解析出的provider名字放到哪个请求header中 | +| `modelToHeader` | string | 选填 | - | 直接将model参数放到哪个请求header中 | +| `enableOnPathSuffix` | array of string | 选填 | ["/v1/chat/completions"] | 只对这些特定路径后缀的请求生效 | + ## 运行属性 -插件执行阶段：`默认阶段` -插件执行优先级：`260` +插件执行阶段：认证阶段 +插件执行优先级：900 -## 配置字段 +## 效果说明 -| 名称 | 数据类型 | 填写要求 | 默认值 | 描述 | -| ----------- | --------------- | ----------------------- | ------ | ------------------------------------------- | -| `enable` | bool | 选填 | false | 是否开启基于model参数路由 | -| `model_key` | string | 选填 | model | 请求body中model参数的位置 | -| `add_header_key` | string | 选填 | x-higress-llm-provider | 从model参数中解析出的provider名字放到哪个请求header中 | +### 基于 model 参数进行路由 +需要做如下配置： -## 效果说明 +```yaml +modelToHeader: x-higress-llm-model +``` + +插件会将请求中 model 参数提取出来，设置到 x-higress-llm-model 这个请求 header 中，用于后续路由，举例来说，原生的 LLM 请求体是： + +```json +{ + "model": "qwen-long", + "frequency_penalty": 0, + "max_tokens": 800, + "stream": false, + "messages": [{ + "role": "user", + "content": "higress项目主仓库的github地址是什么" + }], + "presence_penalty": 0, + "temperature": 0.7, + "top_p": 0.95 +} +``` + +经过这个插件后，将添加下面这个请求头(可以用于路由匹配)： + +x-higress-llm-model: qwen-long + +### 提取 model 参数中的 provider 字段用于路由 + +> 注意这种模式需要客户端在 model 参数中通过`/`分隔的方式，来指定 provider -如下开启基于model参数路由的功能： +需要做如下配置： ```yaml -enable: true +addProviderHeader: x-higress-llm-provider ``` -开启后，插件将请求中 model 参数的 provider 部分（如果有）提取出来，设置到 x-higress-llm-provider 这个请求 header 中，用于后续路由，并将 model 参数重写为模型名称部分。举例来说，原生的 LLM 请求体是： +插件会将请求中 model 参数的 provider 部分（如果有）提取出来，设置到 x-higress-llm-provider 这个请求 header 中，用于后续路由，并将 model 参数重写为模型名称部分。举例来说，原生的 LLM 请求体是： ```json { - "model": "qwen/qwen-long", + "model": "dashscope/qwen-long", "frequency_penalty": 0, "max_tokens": 800, "stream": false, @@ -43,7 +77,7 @@ enable: true 经过这个插件后，将添加下面这个请求头(可以用于路由匹配)： -x-higress-llm-provider: qwen +x-higress-llm-provider: dashscope 原始的 LLM 请求体将被改成： diff --git a/backend/sdk/src/main/resources/plugins/model-router/README_EN.md b/backend/sdk/src/main/resources/plugins/model-router/README_EN.md index 4d2eaf1f..f5866423 100644 --- a/backend/sdk/src/main/resources/plugins/model-router/README_EN.md +++ b/backend/sdk/src/main/resources/plugins/model-router/README_EN.md @@ -1,38 +1,41 @@ ## Function Description -The `model-router` plugin implements the functionality of routing based on the `model` parameter in the LLM protocol. +The `model-router` plugin implements the function of routing based on the model parameter in the LLM protocol. -## Runtime Properties +## Configuration Fields -Plugin Execution Phase: `Default Phase` -Plugin Execution Priority: `260` +| Name | Data Type | Filling Requirement | Default Value | Description | +| ----------- | --------------- | ----------------------- | ------ | ------------------------------------------- | +| `modelKey` | string | Optional | model | The location of the model parameter in the request body | +| `addProviderHeader` | string | Optional | - | Which request header to place the provider name parsed from the model parameter | +| `modelToHeader` | string | Optional | - | Which request header to directly place the model parameter | +| `enableOnPathSuffix` | array of string | Optional | ["/v1/chat/completions"] | Only effective for requests with these specific path suffixes | -## Configuration Fields +## Runtime Attributes -| Name | Data Type | Filling Requirement | Default Value | Description | -| -------------------- | ------------- | --------------------- | ---------------------- | ----------------------------------------------------- | -| `enable` | bool | Optional | false | Whether to enable routing based on the `model` parameter | -| `model_key` | string | Optional | model | The location of the `model` parameter in the request body | -| `add_header_key` | string | Optional | x-higress-llm-provider | The header where the parsed provider name from the `model` parameter will be placed | +Plugin execution phase: Authentication phase +Plugin execution priority: 900 ## Effect Description -To enable routing based on the `model` parameter, use the following configuration: +### Routing Based on the model Parameter + +The following configuration is required: ```yaml -enable: true +modelToHeader: x-higress-llm-model ``` -After enabling, the plugin extracts the provider part (if any) from the `model` parameter in the request, and sets it in the `x-higress-llm-provider` request header for subsequent routing. It also rewrites the `model` parameter to the model name part. For example, the original LLM request body is: +The plugin will extract the model parameter from the request and set it in the x-higress-llm-model request header, which can be used for subsequent routing. For example, the original LLM request body: ```json { - "model": "openai/gpt-4o", + "model": "qwen-long", "frequency_penalty": 0, "max_tokens": 800, "stream": false, "messages": [{ "role": "user", - "content": "What is the GitHub address for the main repository of the Higress project?" + "content": "What is the GitHub address of the main repository for the higress project" }], "presence_penalty": 0, "temperature": 0.7, @@ -40,24 +43,55 @@ After enabling, the plugin extracts the provider part (if any) from the `model` } ``` -After processing by the plugin, the following request header (which can be used for routing matching) will be added: +After processing by this plugin, the following request header (which can be used for route matching) will be added: + +x-higress-llm-model: qwen-long -`x-higress-llm-provider: openai` +### Extracting the provider Field from the model Parameter for Routing -The original LLM request body will be modified to: +> Note that this mode requires the client to specify the provider using a `/` separator in the model parameter. + +The following configuration is required: + +```yaml +addProviderHeader: x-higress-llm-provider +``` + +The plugin will extract the provider part (if present) from the model parameter in the request and set it in the x-higress-llm-provider request header, which can be used for subsequent routing, and rewrite the model parameter to the model name part. For example, the original LLM request body: ```json { - "model": "gpt-4o", + "model": "dashscope/qwen-long", "frequency_penalty": 0, "max_tokens": 800, "stream": false, "messages": [{ "role": "user", - "content": "What is the GitHub address for the main repository of the Higress project?" + "content": "What is the GitHub address of the main repository for the higress project" }], "presence_penalty": 0, "temperature": 0.7, "top_p": 0.95 } ``` + +After processing by this plugin, the following request header (which can be used for route matching) will be added: + +x-higress-llm-provider: dashscope + +The original LLM request body will be changed to: + +```json +{ + "model": "qwen-long", + "frequency_penalty": 0, + "max_tokens": 800, + "stream": false, + "messages": [{ + "role": "user", + "content": "What is the GitHub address of the main repository for the higress project" + }], + "presence_penalty": 0, + "temperature": 0.7, + "top_p": 0.95 +} diff --git a/backend/sdk/src/main/resources/plugins/model-router/spec.yaml b/backend/sdk/src/main/resources/plugins/model-router/spec.yaml index cf9c30c0..6903531f 100644 --- a/backend/sdk/src/main/resources/plugins/model-router/spec.yaml +++ b/backend/sdk/src/main/resources/plugins/model-router/spec.yaml @@ -14,8 +14,8 @@ info: contact: name: johnlanni spec: - phase: default - priority: 260 + phase: AUTHN + priority: 900 configSchema: openAPIV3Schema: type: object @@ -28,7 +28,7 @@ spec: description: 是否开启基于 model 参数路由 x-description-i18n: zh-CN: 是否开启基于 model 参数路由 - model_key: + modelKey: type: string title: Model Key x-title-i18n: @@ -36,15 +36,32 @@ spec: description: The location of the `model` parameter in the request body x-description-i18n: zh-CN: 请求 body 中 model 参数的位置 - add_header_key: + addProviderHeader: type: string - title: Header Key + title: Provider Header Key x-title-i18n: - zh-CN: Header 名称 + zh-CN: Provider Header 名称 description: The header where the parsed provider name from the `model` parameter will be placed x-description-i18n: zh-CN: 从 model 参数中解析出的 provider 名字放到哪个请求 header 中 + modelToHeader: + type: string + title: Model Header Key + x-title-i18n: + zh-CN: Model Header 名称 + description: The header where the parsed model name from the `model` parameter will be placed + x-description-i18n: + zh-CN: 从 model 参数中解析出的 model 名字放到哪个请求 header 中 + enableOnPathSuffix: + type: array + title: Enable On Path Suffixes + x-title-i18n: + zh-CN: 生效的请求路径 + description: Only applies to requests with these specific path suffixes. + x-description-i18n: + zh-CN: 只对这些特定路径后缀的请求生效 + items: + type: string example: - enable: true - model_key: model - add_header_key: x-higress-llm-provider + addProviderHeader: x-higress-llm-provider + modelToHeader: x-higress-llm-model diff --git a/backend/sdk/src/main/resources/plugins/plugins.properties b/backend/sdk/src/main/resources/plugins/plugins.properties index d6b90b46..80177936 100644 --- a/backend/sdk/src/main/resources/plugins/plugins.properties +++ b/backend/sdk/src/main/resources/plugins/plugins.properties @@ -27,6 +27,7 @@ ai-quota=oci://higress-registry.cn-hangzhou.cr.aliyuncs.com/plugins/ai-quota:lat ai-agent=oci://higress-registry.cn-hangzhou.cr.aliyuncs.com/plugins/ai-agent:latest ai-json-resp=oci://higress-registry.cn-hangzhou.cr.aliyuncs.com/plugins/ai-json-resp:latest model-router=oci://higress-registry.cn-hangzhou.cr.aliyuncs.com/plugins/model-router:latest +model-mapper=oci://higress-registry.cn-hangzhou.cr.aliyuncs.com/plugins/model-mapper:latest # Auth basic-auth=oci://higress-registry.cn-hangzhou.cr.aliyuncs.com/plugins/basic-auth:latest diff --git a/backend/sdk/src/test/java/com/alibaba/higress/sdk/service/WasmPluginInstanceServiceTest.java b/backend/sdk/src/test/java/com/alibaba/higress/sdk/service/WasmPluginInstanceServiceTest.java index 0000d104..c906e9e3 100644 --- a/backend/sdk/src/test/java/com/alibaba/higress/sdk/service/WasmPluginInstanceServiceTest.java +++ b/backend/sdk/src/test/java/com/alibaba/higress/sdk/service/WasmPluginInstanceServiceTest.java @@ -25,7 +25,6 @@ import java.util.List; import java.util.Map; -import com.alibaba.higress.sdk.service.kubernetes.crd.wasm.MatchRule; import org.apache.commons.collections4.CollectionUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; @@ -40,8 +39,10 @@ import com.alibaba.higress.sdk.model.WasmPluginInstanceScope; import com.alibaba.higress.sdk.service.kubernetes.KubernetesClientService; import com.alibaba.higress.sdk.service.kubernetes.KubernetesModelConverter; +import com.alibaba.higress.sdk.service.kubernetes.crd.wasm.MatchRule; import com.alibaba.higress.sdk.service.kubernetes.crd.wasm.PluginPhase; import com.alibaba.higress.sdk.service.kubernetes.crd.wasm.V1alpha1WasmPlugin; +import com.alibaba.higress.sdk.util.MapUtil; public class WasmPluginInstanceServiceTest { @@ -89,16 +90,17 @@ public void queryTest() throws Exception { final Map routeConfig = Map.of("kr", "vr"); V1alpha1WasmPlugin internalCr = buildWasmPluginResource(TEST_BUILT_IN_PLUGIN_NAME, true, true); - kubernetesModelConverter.setWasmPluginInstanceToCr(internalCr, WasmPluginInstance.builder() - .scope(WasmPluginInstanceScope.GLOBAL).enabled(globalEnabled).configurations(globalConfig).build()); kubernetesModelConverter.setWasmPluginInstanceToCr(internalCr, - WasmPluginInstance.builder().scope(WasmPluginInstanceScope.ROUTE).target(route).enabled(routeEnabled) + WasmPluginInstance.builder().targets(MapUtil.of(WasmPluginInstanceScope.GLOBAL, null)) + .enabled(globalEnabled).configurations(globalConfig).build()); + kubernetesModelConverter.setWasmPluginInstanceToCr(internalCr, + WasmPluginInstance.builder().targets(MapUtil.of(WasmPluginInstanceScope.ROUTE, route)).enabled(routeEnabled) .configurations(routeConfig).build()); V1alpha1WasmPlugin userCr = buildWasmPluginResource(TEST_BUILT_IN_PLUGIN_NAME, true, false); kubernetesModelConverter.setWasmPluginInstanceToCr(userCr, - WasmPluginInstance.builder().scope(WasmPluginInstanceScope.DOMAIN).target(domain).enabled(domainEnabled) - .configurations(domainConfig).build()); + WasmPluginInstance.builder().targets(MapUtil.of(WasmPluginInstanceScope.DOMAIN, domain)) + .enabled(domainEnabled).configurations(domainConfig).build()); List crs = List.of(internalCr, userCr); when(kubernetesClientService.listWasmPlugin(eq(TEST_BUILT_IN_PLUGIN_NAME))).thenReturn(crs); @@ -139,7 +141,7 @@ public void queryTest() throws Exception { @Test public void addOrUpdateTestFromEmptyAddUserConfig() throws Exception { WasmPluginInstance instance = WasmPluginInstance.builder().pluginName(TEST_BUILT_IN_PLUGIN_NAME) - .pluginVersion(DEFAULT_VERSION).scope(WasmPluginInstanceScope.GLOBAL).enabled(true) + .pluginVersion(DEFAULT_VERSION).targets(MapUtil.of(WasmPluginInstanceScope.GLOBAL, null)).enabled(true) .configurations(Map.of("k", "v")).internal(false).build(); WasmPluginInstance updatedInstance = service.addOrUpdate(instance); updatedInstance.setRawConfigurations(null); @@ -302,18 +304,18 @@ public void addOrUpdateTestFromInternalAddInternalConfig() throws Exception { public void addOrUpdateTestFromUserAddInternalConfig() throws Exception { V1alpha1WasmPlugin existedCr = buildWasmPluginResource(TEST_BUILT_IN_PLUGIN_NAME, true, false); WasmPluginInstance existedInstance = WasmPluginInstance.builder().pluginName(TEST_BUILT_IN_PLUGIN_NAME) - .pluginVersion(DEFAULT_VERSION).scope(WasmPluginInstanceScope.GLOBAL).enabled(true) - .configurations(Map.of("k", "v")).internal(false).build(); + .pluginVersion(DEFAULT_VERSION).scope(WasmPluginInstanceScope.GLOBAL).enabled(true) + .configurations(Map.of("k", "v")).internal(false).build(); kubernetesModelConverter.setWasmPluginInstanceToCr(existedCr, existedInstance); List crs = List.of(existedCr); when(kubernetesClientService.listWasmPlugin(eq(TEST_BUILT_IN_PLUGIN_NAME))).thenReturn(crs); when(kubernetesClientService.listWasmPlugin(eq(TEST_BUILT_IN_PLUGIN_NAME), anyString())).thenReturn(crs); when(kubernetesClientService.listWasmPlugin(eq(TEST_BUILT_IN_PLUGIN_NAME), anyString(), anyBoolean())) - .thenReturn(crs); + .thenReturn(crs); WasmPluginInstance instance = WasmPluginInstance.builder().pluginName(TEST_BUILT_IN_PLUGIN_NAME) - .pluginVersion(DEFAULT_VERSION).scope(WasmPluginInstanceScope.ROUTE).target("test").enabled(true) - .configurations(Map.of("kd", "vd")).internal(true).build(); + .pluginVersion(DEFAULT_VERSION).scope(WasmPluginInstanceScope.ROUTE).target("test").enabled(true) + .configurations(Map.of("kd", "vd")).internal(true).build(); WasmPluginInstance updatedInstance = service.addOrUpdate(instance); updatedInstance.setRawConfigurations(null); Assertions.assertEquals(instance, updatedInstance); @@ -339,18 +341,18 @@ public void addOrUpdateTestFromUserAddInternalConfig() throws Exception { public void addOrUpdateTestUpdateInternalConfig() throws Exception { V1alpha1WasmPlugin existedCr = buildWasmPluginResource(TEST_BUILT_IN_PLUGIN_NAME, true, true); WasmPluginInstance existedInstance = WasmPluginInstance.builder().pluginName(TEST_BUILT_IN_PLUGIN_NAME) - .pluginVersion(DEFAULT_VERSION).scope(WasmPluginInstanceScope.GLOBAL).enabled(true) - .configurations(Map.of("k", "v")).internal(false).build(); + .pluginVersion(DEFAULT_VERSION).scope(WasmPluginInstanceScope.GLOBAL).enabled(true) + .configurations(Map.of("k", "v")).internal(false).build(); kubernetesModelConverter.setWasmPluginInstanceToCr(existedCr, existedInstance); List crs = List.of(existedCr); when(kubernetesClientService.listWasmPlugin(eq(TEST_BUILT_IN_PLUGIN_NAME))).thenReturn(crs); when(kubernetesClientService.listWasmPlugin(eq(TEST_BUILT_IN_PLUGIN_NAME), anyString())).thenReturn(crs); when(kubernetesClientService.listWasmPlugin(eq(TEST_BUILT_IN_PLUGIN_NAME), anyString(), anyBoolean())) - .thenReturn(crs); + .thenReturn(crs); WasmPluginInstance instance = WasmPluginInstance.builder().pluginName(TEST_BUILT_IN_PLUGIN_NAME) - .pluginVersion(DEFAULT_VERSION).scope(WasmPluginInstanceScope.GLOBAL).enabled(false) - .configurations(Map.of("k2", "v2")).internal(true).build(); + .pluginVersion(DEFAULT_VERSION).scope(WasmPluginInstanceScope.GLOBAL).enabled(false) + .configurations(Map.of("k2", "v2")).internal(true).build(); WasmPluginInstance updatedInstance = service.addOrUpdate(instance); updatedInstance.setRawConfigurations(null); Assertions.assertEquals(instance, updatedInstance); diff --git a/backend/sdk/src/test/java/com/alibaba/higress/sdk/service/consumer/KeyAuthCredentialHandlerTest.java b/backend/sdk/src/test/java/com/alibaba/higress/sdk/service/consumer/KeyAuthCredentialHandlerTest.java index d0a403ff..c384b6e9 100644 --- a/backend/sdk/src/test/java/com/alibaba/higress/sdk/service/consumer/KeyAuthCredentialHandlerTest.java +++ b/backend/sdk/src/test/java/com/alibaba/higress/sdk/service/consumer/KeyAuthCredentialHandlerTest.java @@ -55,30 +55,26 @@ public void isConsumerInUseTestNoInstance() { @Test public void isConsumerInUseTestNotFound() { WasmPluginInstance instance = new WasmPluginInstance(); - instance.setScope(WasmPluginInstanceScope.GLOBAL); + instance.setGlobalTarget(); addConsumer(instance, "zhangsan", true, false, "Authorization", "Bearer sk-123456"); addConsumer(instance, "lisi", true, false, "Authorization", "Bearer sk-567890"); addConsumer(instance, "wangwu", true, false, "Authorization", "Bearer sk-135790"); - WasmPluginInstance domainInstance = createInstance(WasmPluginInstanceScope.DOMAIN); - domainInstance.setTarget("www.example.com"); + WasmPluginInstance domainInstance = createInstance(WasmPluginInstanceScope.DOMAIN, "www.example.com"); addAllow(domainInstance, List.of("lisi")); - WasmPluginInstance routeInstance = createInstance(WasmPluginInstanceScope.ROUTE); - domainInstance.setTarget("test-route"); - addAllow(domainInstance, List.of("lisi", "wangwu")); + WasmPluginInstance routeInstance = createInstance(WasmPluginInstanceScope.ROUTE, "test-route"); + addAllow(routeInstance, List.of("lisi", "wangwu")); Assertions.assertFalse(handler.isConsumerInUse("zhangsan", List.of(instance, domainInstance, routeInstance))); } @Test public void isConsumerInUseTestFoundInOneInstance() { - WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL); + WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL, null); addConsumer(instance, "zhangsan", true, false, "Authorization", "Bearer sk-123456"); addConsumer(instance, "lisi", true, false, "Authorization", "Bearer sk-567890"); - WasmPluginInstance domainInstance = createInstance(WasmPluginInstanceScope.DOMAIN); - domainInstance.setTarget("www.example.com"); + WasmPluginInstance domainInstance = createInstance(WasmPluginInstanceScope.DOMAIN, "www.example.com"); addAllow(domainInstance, List.of("lisi")); - WasmPluginInstance routeInstance = createInstance(WasmPluginInstanceScope.ROUTE); - routeInstance.setTarget("test-route"); + WasmPluginInstance routeInstance = createInstance(WasmPluginInstanceScope.ROUTE, "test-route"); addAllow(routeInstance, List.of("lisi", "zhangsan")); Assertions.assertTrue(handler.isConsumerInUse("zhangsan", List.of(instance, domainInstance, routeInstance))); @@ -86,22 +82,20 @@ public void isConsumerInUseTestFoundInOneInstance() { @Test public void isConsumerInUseTestFoundInMultiInstances() { - WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL); + WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL, null); addConsumer(instance, "zhangsan", true, false, "Authorization", "Bearer sk-123456"); addConsumer(instance, "lisi", true, false, "Authorization", "Bearer sk-567890"); - WasmPluginInstance domainInstance = createInstance(WasmPluginInstanceScope.DOMAIN); - domainInstance.setScope(WasmPluginInstanceScope.DOMAIN); + WasmPluginInstance domainInstance = createInstance(WasmPluginInstanceScope.DOMAIN, "test-domain"); addAllow(domainInstance, List.of("lisi")); - WasmPluginInstance routeInstance = createInstance(WasmPluginInstanceScope.ROUTE); - domainInstance.setTarget("test-route"); - addAllow(domainInstance, List.of("lisi", "zhangsan", "wangwu")); + WasmPluginInstance routeInstance = createInstance(WasmPluginInstanceScope.ROUTE, "test-route"); + addAllow(routeInstance, List.of("lisi", "zhangsan", "wangwu")); Assertions.assertTrue(handler.isConsumerInUse("zhangsan", List.of(instance, domainInstance, routeInstance))); } @Test public void extractConsumersTestEmptyConfiguration() { - WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL); + WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL, null); List consumers = handler.extractConsumers(instance); Assertions.assertNotNull(consumers); @@ -110,7 +104,7 @@ public void extractConsumersTestEmptyConfiguration() { @Test public void extractConsumersTestNoConsumers() { - WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL); + WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL, null); instance.setConfigurations(Map.of("test", "value")); List consumers = handler.extractConsumers(instance); @@ -120,7 +114,7 @@ public void extractConsumersTestNoConsumers() { @Test public void extractConsumersTestConsumersNotList() { - WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL); + WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL, null); instance.setConfigurations(Map.of("consumers", "value")); List consumers = handler.extractConsumers(instance); @@ -130,7 +124,7 @@ public void extractConsumersTestConsumersNotList() { @Test public void extractConsumersTestEmptyConsumerList() { - WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL); + WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL, null); instance.setConfigurations(Map.of("consumers", List.of())); List consumers = handler.extractConsumers(instance); @@ -140,7 +134,7 @@ public void extractConsumersTestEmptyConsumerList() { @Test public void extractConsumersTestGoodConsumerList() { - WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL); + WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL, null); addConsumer(instance, "zhangsan", true, false, "Authorization", "Bearer sk-123456"); addConsumer(instance, "lisi", true, false, "X-API-KEY", "abcd-1234"); addConsumer(instance, "wangwu", false, true, "api_key", "efgh-5678"); @@ -161,7 +155,7 @@ public void extractConsumersTestGoodConsumerList() { @Test public void saveConsumerTestFromNothingBearer() { - WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL); + WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL, null); Consumer consumer = new Consumer("zhangsan", List.of(new KeyAuthCredential(KeyAuthCredentialSource.BEARER.name(), null, "sk-123456"))); Assertions.assertTrue(handler.saveConsumer(instance, consumer)); @@ -179,7 +173,7 @@ public void saveConsumerTestFromNothingBearer() { @Test public void saveConsumerTestBadConsumersHeader() { - WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL); + WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL, null); instance.setConfigurations(new HashMap<>(Map.of("consumers", "value"))); Consumer consumer = new Consumer("zhangsan", @@ -199,7 +193,7 @@ public void saveConsumerTestBadConsumersHeader() { @Test public void saveConsumerTestUpdateQuery() { - WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL); + WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL, null); addConsumer(instance, "zhangsan", true, false, "Authorization", "Bearer sk-123456"); Consumer consumer = new Consumer("zhangsan", @@ -219,7 +213,7 @@ public void saveConsumerTestUpdateQuery() { @Test public void saveConsumerTestNoUpdateEmptyData() { - WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL); + WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL, null); addConsumer(instance, "zhangsan", true, false, "Authorization", "Bearer sk-123456"); Consumer consumer = new Consumer("zhangsan", List.of(new KeyAuthCredential())); @@ -238,11 +232,11 @@ public void saveConsumerTestNoUpdateEmptyData() { @Test public void saveConsumerTestNoUpdatePartialData() { - WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL); + WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL, null); addConsumer(instance, "zhangsan", true, false, "Authorization", "Bearer sk-123456"); - Consumer consumer = - new Consumer("zhangsan", List.of(new KeyAuthCredential(KeyAuthCredentialSource.HEADER.name(), "X-API-KEY", null))); + Consumer consumer = new Consumer("zhangsan", + List.of(new KeyAuthCredential(KeyAuthCredentialSource.HEADER.name(), "X-API-KEY", null))); Assertions.assertTrue(handler.saveConsumer(instance, consumer)); List> consumers = (List>)instance.getConfigurations().get("consumers"); @@ -258,7 +252,7 @@ public void saveConsumerTestNoUpdatePartialData() { @Test public void saveConsumerTestDeleteExistedCredential() { - WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL); + WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL, null); addConsumer(instance, "zhangsan", true, false, "Authorization", "Bearer sk-123456"); Consumer consumer = new Consumer("zhangsan", List.of(new Credential("DUMMY"))); @@ -271,14 +265,14 @@ public void saveConsumerTestDeleteExistedCredential() { @Test public void deleteConsumerTestEmptyConfiguration() { - WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL); + WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL, null); Assertions.assertFalse(handler.deleteConsumer(instance, "zhangsan")); } @Test public void deleteConsumerTestNoConsumers() { - WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL); + WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL, null); instance.setConfigurations(Map.of("test", "value")); Assertions.assertFalse(handler.deleteConsumer(instance, "zhangsan")); @@ -286,7 +280,7 @@ public void deleteConsumerTestNoConsumers() { @Test public void deleteConsumerTestBadConsumers() { - WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL); + WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL, null); instance.setConfigurations(Map.of("consumer", "value")); Assertions.assertFalse(handler.deleteConsumer(instance, "zhangsan")); @@ -294,7 +288,7 @@ public void deleteConsumerTestBadConsumers() { @Test public void deleteConsumerTestNotFound() { - WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL); + WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL, null); addConsumer(instance, "lisi", true, false, "Authorization", "Bearer sk-123456"); Assertions.assertFalse(handler.deleteConsumer(instance, "zhangsan")); @@ -302,7 +296,7 @@ public void deleteConsumerTestNotFound() { @Test public void deleteConsumerTestFoundOnce() { - WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL); + WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL, null); addConsumer(instance, "zhangsan", true, false, "Authorization", "Bearer sk-abcdefg"); addConsumer(instance, "lisi", true, false, "Authorization", "Bearer sk-123456"); @@ -321,7 +315,7 @@ public void deleteConsumerTestFoundOnce() { @Test public void deleteConsumerTestFoundTwice() { - WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL); + WasmPluginInstance instance = createInstance(WasmPluginInstanceScope.GLOBAL, null); addConsumer(instance, "zhangsan", true, false, "Authorization", "Bearer sk-abcdefg"); addConsumer(instance, "lisi", true, false, "Authorization", "Bearer sk-123456"); addConsumer(instance, "zhangsan", false, true, "token", "abcdefg"); @@ -339,11 +333,11 @@ public void deleteConsumerTestFoundTwice() { Assertions.assertEquals("Bearer sk-123456", consumerMap.get("credential")); } - private WasmPluginInstance createInstance(WasmPluginInstanceScope scope) { + private WasmPluginInstance createInstance(WasmPluginInstanceScope scope, String target) { WasmPluginInstance instance = new WasmPluginInstance(); instance.setPluginName(handler.getPluginName()); instance.setPluginVersion("1.0.0"); - instance.setScope(scope); + instance.setTarget(scope, target); instance.setInternal(true); return instance; } diff --git a/backend/sdk/src/test/java/com/alibaba/higress/sdk/service/kubernetes/KubernetesModelConverterTest.java b/backend/sdk/src/test/java/com/alibaba/higress/sdk/service/kubernetes/KubernetesModelConverterTest.java index e5762719..e16fc32c 100644 --- a/backend/sdk/src/test/java/com/alibaba/higress/sdk/service/kubernetes/KubernetesModelConverterTest.java +++ b/backend/sdk/src/test/java/com/alibaba/higress/sdk/service/kubernetes/KubernetesModelConverterTest.java @@ -1160,8 +1160,9 @@ void setWasmPluginInstanceToCrTestRouteScopeConfigured() { @Test void setWasmPluginInstanceToCrTestGlobalScopeShouldSetDefaultConfig() { V1alpha1WasmPlugin cr = new V1alpha1WasmPlugin(); - WasmPluginInstance instance = WasmPluginInstance.builder().scope(WasmPluginInstanceScope.GLOBAL).target(null) - .enabled(true).configurations(Map.of("key", "value")).build(); + WasmPluginInstance instance = + WasmPluginInstance.builder().enabled(true).configurations(Map.of("key", "value")).build(); + instance.setGlobalTarget(); converter.setWasmPluginInstanceToCr(cr, instance); @@ -1174,8 +1175,9 @@ void setWasmPluginInstanceToCrTestGlobalScopeShouldSetDefaultConfig() { @Test void setWasmPluginInstanceToCrTestDomainScopeShouldAddOrUpdateDomainRule() { V1alpha1WasmPlugin cr = new V1alpha1WasmPlugin(); - WasmPluginInstance instance = WasmPluginInstance.builder().scope(WasmPluginInstanceScope.DOMAIN) - .target("higress.cn").enabled(true).configurations(Map.of("key", "value")).build(); + WasmPluginInstance instance = + WasmPluginInstance.builder().enabled(true).configurations(Map.of("key", "value")).build(); + instance.setTarget(WasmPluginInstanceScope.DOMAIN, "higress.cn"); converter.setWasmPluginInstanceToCr(cr, instance); @@ -1198,8 +1200,9 @@ void setWasmPluginInstanceToCrTestDomainScopeExistingRuleShouldUpdateExistingDom List.of(new MatchRule(false, Map.of("key", "original"), List.of("higress.cn"), List.of(), List.of()))); cr.setSpec(spec); - WasmPluginInstance instance = WasmPluginInstance.builder().scope(WasmPluginInstanceScope.DOMAIN) - .target("higress.cn").enabled(true).configurations(Map.of("key", "updated")).build(); + WasmPluginInstance instance = + WasmPluginInstance.builder().enabled(true).configurations(Map.of("key", "updated")).build(); + instance.setTarget(WasmPluginInstanceScope.DOMAIN, "higress.cn"); converter.setWasmPluginInstanceToCr(cr, instance); @@ -1215,8 +1218,9 @@ void setWasmPluginInstanceToCrTestDomainScopeExistingRuleShouldUpdateExistingDom @Test void setWasmPluginInstanceToCrTestRouteScopeShouldAddOrUpdateRouteRule() { V1alpha1WasmPlugin cr = new V1alpha1WasmPlugin(); - WasmPluginInstance instance = WasmPluginInstance.builder().scope(WasmPluginInstanceScope.ROUTE) - .target("route-1").enabled(true).configurations(Map.of("key", "value")).build(); + WasmPluginInstance instance = + WasmPluginInstance.builder().enabled(true).configurations(Map.of("key", "value")).build(); + instance.setTarget(WasmPluginInstanceScope.ROUTE, "route-1"); converter.setWasmPluginInstanceToCr(cr, instance); From 352f108ca69d9de7f998ee846b203274784cd785 Mon Sep 17 00:00:00 2001 From: CH3CHO Date: Fri, 13 Dec 2024 17:58:06 +0800 Subject: [PATCH 2/2] Support a separate dashboard for AI Gateway --- .../console/constant/UserConfigKey.java | 1 + .../controller/DashboardController.java | 35 +- .../console/controller/dto/DashboardType.java | 29 + .../console/service/DashboardService.java | 10 + .../console/service/DashboardServiceImpl.java | 95 +- .../src/main/resources/dashboard/ai.json | 2310 +++++++++++++++++ 6 files changed, 2452 insertions(+), 28 deletions(-) create mode 100644 backend/console/src/main/java/com/alibaba/higress/console/controller/dto/DashboardType.java create mode 100644 backend/console/src/main/resources/dashboard/ai.json diff --git a/backend/console/src/main/java/com/alibaba/higress/console/constant/UserConfigKey.java b/backend/console/src/main/java/com/alibaba/higress/console/constant/UserConfigKey.java index c06393d5..6b9759ac 100644 --- a/backend/console/src/main/java/com/alibaba/higress/console/constant/UserConfigKey.java +++ b/backend/console/src/main/java/com/alibaba/higress/console/constant/UserConfigKey.java @@ -24,6 +24,7 @@ private UserConfigKey() {} public static final String SYSTEM_INITIALIZED = "system.initialized"; public static final String LOGIN_PAGE_PROMPT_KEY = "login.prompt"; public static final String DASHBOARD_URL = "dashboard.url"; + public static final String DASHBOARD_URL_PREFIX = "dashboard.url."; public static final String CHAT_ENABLED = "chat.enabled"; public static final String ADMIN_PASSWORD_CHANGE_DISABLED = "admin.password-change.disabled"; diff --git a/backend/console/src/main/java/com/alibaba/higress/console/controller/DashboardController.java b/backend/console/src/main/java/com/alibaba/higress/console/controller/DashboardController.java index 1692e619..c3cb12c9 100644 --- a/backend/console/src/main/java/com/alibaba/higress/console/controller/DashboardController.java +++ b/backend/console/src/main/java/com/alibaba/higress/console/controller/DashboardController.java @@ -26,9 +26,10 @@ import org.springframework.web.bind.annotation.RestController; import com.alibaba.higress.console.controller.dto.DashboardInfo; +import com.alibaba.higress.console.controller.dto.DashboardType; import com.alibaba.higress.console.controller.dto.Response; -import com.alibaba.higress.sdk.exception.ValidationException; import com.alibaba.higress.console.service.DashboardService; +import com.alibaba.higress.sdk.exception.ValidationException; /** * @author CH3CHO @@ -48,25 +49,41 @@ public void setDashboardService(DashboardService dashboardService) { @GetMapping("/init") public ResponseEntity> init(@RequestParam(required = false) Boolean force) { dashboardService.initializeDashboard(Boolean.TRUE.equals(force)); - return info(); + return info(null); } @GetMapping("/info") - public ResponseEntity> info() { - return ResponseEntity.ok(Response.success(dashboardService.getDashboardInfo())); + public ResponseEntity> info(@RequestParam(required = false) String type) { + DashboardType dashboardType = toDashboardType(type); + return ResponseEntity.ok(Response.success(dashboardService.getDashboardInfo(dashboardType))); } @PutMapping("/info") - public ResponseEntity> setUrl(@RequestBody DashboardInfo dashboardInfo) { + public ResponseEntity> setUrl(@RequestParam(required = false) String type, + @RequestBody DashboardInfo dashboardInfo) { + DashboardType dashboardType = toDashboardType(type); if (StringUtils.isEmpty(dashboardInfo.getUrl())) { throw new ValidationException("Missing required parameter: url"); } - dashboardService.setDashboardUrl(dashboardInfo.getUrl()); - return info(); + dashboardService.setDashboardUrl(dashboardType, dashboardInfo.getUrl()); + return info(dashboardType.toString()); } @GetMapping("/configData") - public ResponseEntity> getConfigData(@RequestParam @NotBlank String dataSourceUid) { - return ResponseEntity.ok(Response.success(dashboardService.buildConfigData(dataSourceUid))); + public ResponseEntity> getConfigData(@RequestParam(required = false) String type, + @RequestParam @NotBlank String dataSourceUid) { + DashboardType dashboardType = toDashboardType(type); + return ResponseEntity.ok(Response.success(dashboardService.buildConfigData(dashboardType, dataSourceUid))); + } + + private static DashboardType toDashboardType(String type) { + if (StringUtils.isEmpty(type)) { + return DashboardType.MAIN; + } + try { + return DashboardType.valueOf(type.toUpperCase()); + } catch (IllegalArgumentException e) { + throw new ValidationException("Unknown dashboard type: " + type); + } } } diff --git a/backend/console/src/main/java/com/alibaba/higress/console/controller/dto/DashboardType.java b/backend/console/src/main/java/com/alibaba/higress/console/controller/dto/DashboardType.java new file mode 100644 index 00000000..83f8e134 --- /dev/null +++ b/backend/console/src/main/java/com/alibaba/higress/console/controller/dto/DashboardType.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2022-2024 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.alibaba.higress.console.controller.dto; + +public enum DashboardType { + + /** + * Main dashboard + */ + MAIN, + /** + * AI Gateway dashboard + */ + AI, + /** + * Access log dashboard + */ + LOG +} diff --git a/backend/console/src/main/java/com/alibaba/higress/console/service/DashboardService.java b/backend/console/src/main/java/com/alibaba/higress/console/service/DashboardService.java index 27361bab..f318fa1c 100644 --- a/backend/console/src/main/java/com/alibaba/higress/console/service/DashboardService.java +++ b/backend/console/src/main/java/com/alibaba/higress/console/service/DashboardService.java @@ -18,19 +18,29 @@ import javax.servlet.http.HttpServletResponse; import com.alibaba.higress.console.controller.dto.DashboardInfo; +import com.alibaba.higress.console.controller.dto.DashboardType; /** * @author CH3CHO */ public interface DashboardService { + @Deprecated DashboardInfo getDashboardInfo(); + DashboardInfo getDashboardInfo(DashboardType type); + void initializeDashboard(boolean overwrite); + @Deprecated void setDashboardUrl(String url); + void setDashboardUrl(DashboardType type, String url); + + @Deprecated String buildConfigData(String dataSourceUid); + String buildConfigData(DashboardType type, String dataSourceUid); + void forwardDashboardRequest(HttpServletRequest request, HttpServletResponse response) throws IOException; } diff --git a/backend/console/src/main/java/com/alibaba/higress/console/service/DashboardServiceImpl.java b/backend/console/src/main/java/com/alibaba/higress/console/service/DashboardServiceImpl.java index 2a005f22..d24a79b4 100644 --- a/backend/console/src/main/java/com/alibaba/higress/console/service/DashboardServiceImpl.java +++ b/backend/console/src/main/java/com/alibaba/higress/console/service/DashboardServiceImpl.java @@ -18,7 +18,10 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Locale; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutorService; @@ -46,6 +49,7 @@ import org.apache.http.impl.client.HttpClients; import org.apache.http.message.BasicHeader; import org.apache.tomcat.util.http.fileupload.util.Streams; +import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -58,6 +62,7 @@ import com.alibaba.higress.console.constant.SystemConfigKey; import com.alibaba.higress.console.constant.UserConfigKey; import com.alibaba.higress.console.controller.dto.DashboardInfo; +import com.alibaba.higress.console.controller.dto.DashboardType; import com.alibaba.higress.sdk.exception.BusinessException; import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.ThreadFactoryBuilder; @@ -79,6 +84,7 @@ public class DashboardServiceImpl implements DashboardService { private static final String DATASOURCE_UID_PLACEHOLDER = "${datasource.id}"; private static final String MAIN_DASHBOARD_DATA_PATH = "/dashboard/main.json"; private static final String LOG_DASHBOARD_DATA_PATH = "/dashboard/logs.json"; + private static final String AI_DASHBOARD_DATA_PATH = "/dashboard/ai.json"; private static final String PROM_DATASOURCE_TYPE = "prometheus"; private static final String LOKI_DATASOURCE_TYPE = "loki"; private static final String DATASOURCE_ACCESS = "proxy"; @@ -129,10 +135,7 @@ public class DashboardServiceImpl implements DashboardService { private CloseableHttpClient realServerClient; private String realServerBaseUrl; - private String mainDashboardConfiguration; - private GrafanaDashboard configuredMainDashboard; - private String logDashboardConfiguration; - private GrafanaDashboard configuredLogDashboard; + private Map dashboardConfigurations; @Resource public void setConfigService(ConfigService configService) { @@ -141,14 +144,18 @@ public void setConfigService(ConfigService configService) { @PostConstruct public void initialize() { + Map dashboardConfigurations = new HashMap<>(); try { - mainDashboardConfiguration = IOUtils.resourceToString(MAIN_DASHBOARD_DATA_PATH, StandardCharsets.UTF_8); - configuredMainDashboard = GrafanaClient.parseDashboardData(mainDashboardConfiguration); - logDashboardConfiguration = IOUtils.resourceToString(LOG_DASHBOARD_DATA_PATH, StandardCharsets.UTF_8); - configuredLogDashboard = GrafanaClient.parseDashboardData(logDashboardConfiguration); + dashboardConfigurations.put(DashboardType.MAIN, + new DashboardConfiguration(DashboardType.MAIN, MAIN_DASHBOARD_DATA_PATH)); + dashboardConfigurations.put(DashboardType.AI, + new DashboardConfiguration(DashboardType.AI, AI_DASHBOARD_DATA_PATH)); + dashboardConfigurations.put(DashboardType.LOG, + new DashboardConfiguration(DashboardType.LOG, LOG_DASHBOARD_DATA_PATH)); } catch (IOException e) { throw new IllegalStateException("Error occurs when loading dashboard configurations from resource.", e); } + this.dashboardConfigurations = Collections.unmodifiableMap(dashboardConfigurations); if (isBuiltIn()) { try { @@ -170,7 +177,12 @@ public void initialize() { @Override public DashboardInfo getDashboardInfo() { - return isBuiltIn() ? getBuiltInDashboardInfo() : getConfiguredDashboardInfo(); + return getDashboardInfo(DashboardType.MAIN); + } + + @Override + public DashboardInfo getDashboardInfo(DashboardType type) { + return isBuiltIn() ? getBuiltInDashboardInfo(type) : getConfiguredDashboardInfo(type); } @Override @@ -194,26 +206,39 @@ public void initializeDashboard(boolean overwrite) { } catch (IOException e) { throw new BusinessException("Error occurs when loading dashboard info from Grafana.", e); } - configureDashboard(results, configuredMainDashboard.getTitle(), mainDashboardConfiguration, promDatasourceUid, - overwrite); - configureDashboard(results, configuredLogDashboard.getTitle(), logDashboardConfiguration, lokiDatasourceUid, - overwrite); + for (DashboardConfiguration configuration : dashboardConfigurations.values()) { + String datasourceId = configuration.getType() == DashboardType.LOG ? lokiDatasourceUid : promDatasourceUid; + configureDashboard(results, configuration.getDashboard().getTitle(), configuration.getRaw(), datasourceId, + overwrite); + } } @Override public void setDashboardUrl(String url) { + setDashboardUrl(DashboardType.MAIN, url); + } + + @Override + public void setDashboardUrl(DashboardType type, String url) { if (StringUtils.isBlank(url)) { throw new IllegalArgumentException("url cannot be null or blank."); } if (isBuiltIn()) { throw new IllegalStateException("Manual dashboard configuration is disabled."); } - configService.setConfig(UserConfigKey.DASHBOARD_URL, url); + DashboardConfiguration configuration = getDashboardConfiguration(type); + configService.setConfig(configuration.getConfigKey(), url); } @Override public String buildConfigData(String datasourceUid) { - return buildConfigData(mainDashboardConfiguration, datasourceUid); + return buildConfigData(DashboardType.MAIN, datasourceUid); + } + + @Override + public String buildConfigData(DashboardType type, String datasourceUid) { + DashboardConfiguration configuration = getDashboardConfiguration(type); + return buildConfigData(configuration.getRaw(), datasourceUid); } @Override @@ -323,7 +348,11 @@ private void configureDashboard(List results, String title, } } - private DashboardInfo getBuiltInDashboardInfo() { + private DashboardInfo getBuiltInDashboardInfo(DashboardType type) { + DashboardConfiguration configuration = dashboardConfigurations.get(type); + if (configuration == null) { + throw new IllegalArgumentException("Invalid dashboard type: " + type); + } List results; try { results = grafanaClient.search(null, SearchType.DB, null, null); @@ -333,7 +362,7 @@ private DashboardInfo getBuiltInDashboardInfo() { if (CollectionUtils.isEmpty(results)) { return new DashboardInfo(true, null, null); } - String expectedTitle = configuredMainDashboard.getTitle(); + String expectedTitle = configuration.getDashboard().getTitle(); if (StringUtils.isEmpty(expectedTitle)) { throw new IllegalStateException("No title is found in the configured dashboard."); } @@ -342,8 +371,9 @@ private DashboardInfo getBuiltInDashboardInfo() { return result.map(r -> new DashboardInfo(true, r.getUid(), r.getUrl())).orElse(null); } - private DashboardInfo getConfiguredDashboardInfo() { - String url = configService.getString(UserConfigKey.DASHBOARD_URL); + private DashboardInfo getConfiguredDashboardInfo(DashboardType type) { + DashboardConfiguration configuration = dashboardConfigurations.get(type); + String url = configService.getString(configuration.getConfigKey()); return new DashboardInfo(false, null, url); } @@ -411,4 +441,31 @@ public void run() { } } } + + private @NotNull DashboardConfiguration getDashboardConfiguration(DashboardType type) { + DashboardConfiguration configuration = dashboardConfigurations.get(type); + if (configuration == null) { + throw new IllegalArgumentException("Invalid dashboard type: " + type); + } + return configuration; + } + + @lombok.Value + private static class DashboardConfiguration { + + DashboardType type; + String configKey; + String resourcePath; + String raw; + GrafanaDashboard dashboard; + + public DashboardConfiguration(DashboardType type, String resourcePath) throws IOException { + this.type = type; + this.configKey = type == DashboardType.MAIN ? UserConfigKey.DASHBOARD_URL + : UserConfigKey.DASHBOARD_URL_PREFIX + type.toString().toLowerCase(Locale.ROOT); + this.resourcePath = resourcePath; + this.raw = IOUtils.resourceToString(resourcePath, StandardCharsets.UTF_8); + this.dashboard = GrafanaClient.parseDashboardData(this.raw); + } + } } diff --git a/backend/console/src/main/resources/dashboard/ai.json b/backend/console/src/main/resources/dashboard/ai.json new file mode 100644 index 00000000..e0b13228 --- /dev/null +++ b/backend/console/src/main/resources/dashboard/ai.json @@ -0,0 +1,2310 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Higress AI Gateway Dashboard", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 17998, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [], + "title": "General", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 4, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "expr": "round(sum(irate(envoy_http_downstream_rq_total{higress=\"$gateway\"}[5m])), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "title": "Downstream Request Volume", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 95 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 99 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 6, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "expr": "sum(irate(envoy_http_downstream_rq{higress=\"$gateway\", response_code_class!=\"5xx\"}[5m])) / sum(irate(envoy_http_downstream_rq_total{higress=\"$gateway\"}[5m]))", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Downstream Success Rate (non-5xx responses)", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "expr": "histogram_quantile(0.50, sum(irate(envoy_http_downstream_rq_time_bucket{higress=\"$gateway\"}[1m])) by (le))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "P50", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "expr": "histogram_quantile(0.90, sum(irate(envoy_http_downstream_rq_time_bucket{higress=\"$gateway\"}[1m])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "P90", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "expr": "histogram_quantile(0.99, sum(irate(envoy_http_downstream_rq_time_bucket{higress=\"$gateway\"}[1m])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "P99", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "editorMode": "code", + "expr": "sum(irate(envoy_http_downstream_rq_time_sum{higress=\"$gateway\"}[1m])) / sum(irate(envoy_http_downstream_rq_time_count{higress=\"$gateway\"}[1m]))", + "hide": false, + "interval": "", + "legendFormat": "Avg", + "range": true, + "refId": "D" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Downstream Request Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:131", + "format": "ms", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:132", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 10, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "expr": "sum(irate(envoy_http_downstream_cx_rx_bytes_total{higress=\"$gateway\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "TCP Received Bytes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 5 + }, + "id": 18, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "expr": "round(sum(irate(envoy_cluster_upstream_rq_total{higress=\"$gateway\"}[5m])), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "title": "Upstream Request Volume", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 95 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 99 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 5 + }, + "id": 19, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "expr": "sum(irate(envoy_cluster_upstream_rq{higress=\"$gateway\", cluster_name!~\"\\\\.internal$\",response_code_class!=\"5xx\"}[5m])) / sum(irate(envoy_cluster_upstream_rq{higress=\"$gateway\", cluster_name!~\"\\\\.internal$\"}[5m]))", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Upstream Success Rate (non-5xx responses)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "dtdurationms" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 5 + }, + "id": 16, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "expr": "histogram_quantile(0.50, sum(irate(envoy_cluster_upstream_rq_time_bucket{higress=\"$gateway\"}[1m])) by (le))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "P50", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "expr": "histogram_quantile(0.90, sum(irate(envoy_cluster_upstream_rq_time_bucket{higress=\"$gateway\"}[1m])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "P90", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "expr": "histogram_quantile(0.99, sum(irate(envoy_cluster_upstream_rq_time_bucket{higress=\"$gateway\"}[1m])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "P99", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "editorMode": "code", + "expr": "sum(irate(envoy_cluster_upstream_rq_time_sum{higress=\"$gateway\"}[1m])) / sum(irate(envoy_cluster_upstream_rq_time_count{higress=\"$gateway\"}[1m]))", + "hide": false, + "legendFormat": "Avg", + "range": true, + "refId": "D" + } + ], + "title": "Upstream Request Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 5 + }, + "id": 12, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "expr": "sum(irate(envoy_http_downstream_cx_tx_bytes_total{higress=\"$gateway\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "TCP Sent Bytes", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 21, + "panels": [], + "title": "Workload", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "fieldConfig": { + "defaults": { + "unit": "percent" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 11, + "x": 0, + "y": 10 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "editorMode": "code", + "expr": "100 * sum(irate(container_cpu_usage_seconds_total{container=\"higress-gateway\", namespace=\"$namespace\"}[1m]))by(pod)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "range": true, + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1997", + "format": "percent", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1998", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 11, + "y": 10 + }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(container_memory_working_set_bytes{container=\"higress-gateway\", namespace=\"$namespace\"}) by (pod)", + "interval": "", + "legendFormat": "{{pod}}-envoy", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "editorMode": "code", + "expr": "max(container_memory_working_set_bytes{container=\"discovery\", namespace=\"$namespace\"}) by (pod)", + "hide": false, + "legendFormat": "{{pod}}-istio", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "editorMode": "code", + "expr": "max(container_memory_working_set_bytes{container=\"higress-core\", namespace=\"$namespace\"}) by (pod)", + "hide": false, + "legendFormat": "{{pod}}-core", + "range": true, + "refId": "C" + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 47, + "panels": [], + "title": "Request", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 11, + "x": 0, + "y": 18 + }, + "id": 45, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "editorMode": "code", + "expr": "sum(irate(envoy_http_downstream_rq{higress=\"$gateway\",http_conn_manager_prefix=~\"outbound_0.0.0.0_.*\"}[1m])) by (response_code_class)", + "legendFormat": "{{response_code_class}}", + "range": true, + "refId": "A" + } + ], + "title": "Downstream QPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 11, + "y": 18 + }, + "id": 49, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "editorMode": "code", + "expr": "sum(irate(envoy_cluster_upstream_rq{higress=\"$gateway\",cluster_name=~\"outbound.*\"}[1m])) by (response_code_class)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Upstream QPS", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 52, + "panels": [], + "title": "WAF", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 27 + }, + "id": 54, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "editorMode": "code", + "expr": "sum(irate(waf_filter_tx_total{higress=\"$gateway\"}[1m]))", + "legendFormat": "qps", + "range": true, + "refId": "A" + } + ], + "title": "WAF Processed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 27 + }, + "id": 55, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "editorMode": "code", + "expr": "sum(irate(waf_filter_tx_interruptions{higress=\"$gateway\"}[1m])) by (phase)", + "legendFormat": "{{phase}}", + "range": true, + "refId": "A" + } + ], + "title": "WAF Denied by Phase", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 27 + }, + "id": 56, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "editorMode": "code", + "expr": "sum(irate(waf_filter_tx_interruptions{higress=\"$gateway\"}[1m])) by (ruleid)", + "legendFormat": "{{ruleid}}", + "range": true, + "refId": "A" + } + ], + "title": "WAF Denied by RuleID", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 26, + "panels": [], + "title": "XDS", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "description": "Shows details about Envoy proxies in the mesh", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 36 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "expr": "sum(irate(envoy_cluster_upstream_cx_total{cluster_name=\"xds-grpc\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "XDS Connections", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "expr": "sum(irate(envoy_cluster_upstream_cx_connect_fail{cluster_name=\"xds-grpc\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "XDS Connection Failures", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "expr": "sum(increase(envoy_server_hot_restart_epoch[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Envoy Restarts", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Envoy Details", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2269", + "format": "ops", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2270", + "format": "ops", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 36 + }, + "hiddenSeries": false, + "id": 30, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "expr": "sum(envoy_cluster_upstream_cx_active{cluster_name=\"xds-grpc\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "XDS Active Connections", + "refId": "C", + "step": 2 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "XDS Active Connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:309", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:310", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "description": "Shows the size of XDS requests and responses", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 36 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "expr": "max(rate(envoy_cluster_upstream_cx_rx_bytes_total{cluster_name=\"xds-grpc\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "XDS Response Bytes Max", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "expr": "quantile(0.5, rate(envoy_cluster_upstream_cx_rx_bytes_total{cluster_name=\"xds-grpc\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "XDS Response Bytes Average", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "expr": "max(rate(envoy_cluster_upstream_cx_tx_bytes_total{cluster_name=\"xds-grpc\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "XDS Request Bytes Max", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "expr": "quantile(.5, rate(envoy_cluster_upstream_cx_tx_bytes_total{cluster_name=\"xds-grpc\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "XDS Request Bytes Average", + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "XDS Requests Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:398", + "format": "Bps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:399", + "format": "ops", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 44 + }, + "id": 38, + "panels": [], + "title": "Service Top", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 45 + }, + "id": 40, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(label_replace(topk(10, sum(irate(envoy_cluster_upstream_rq_total{higress=\"$gateway\", cluster_name!=\"xds-grpc\", cluster_name!=\"prometheus_stats\", cluster_name!=\"agent\", cluster_name!=\"BlackHoleCluster\", cluster_name!=\"sds-grpc\"}[5m])) by (cluster_name)), \"service\", \"$3\", \"cluster_name\", \"outbound_([0-9]+)_(.*)_(.*)$\"), \"port\", \"$1\", \"cluster_name\", \"outbound_([0-9]+)_(.*)_(.*)$\")", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "QPS Top 10", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Value": 2, + "port": 1, + "service": 0 + }, + "renameByName": { + "Value": "QPS" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 45 + }, + "id": 50, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(label_replace(topk(10, sum(irate(envoy_cluster_upstream_rq{response_code_class=~\"(4|5)xx\",higress=\"$gateway\", cluster_name!=\"xds-grpc\", cluster_name!=\"prometheus_stats\", cluster_name!=\"agent\", cluster_name!=\"BlackHoleCluster\", cluster_name!=\"sds-grpc\"}[5m])) by (cluster_name)), \"service\", \"$3\", \"cluster_name\", \"outbound_([0-9]+)_(.*)_(.*)$\"), \"port\", \"$1\", \"cluster_name\", \"outbound_([0-9]+)_(.*)_(.*)$\")", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Failure Requests Top 10", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Value": 2, + "port": 1, + "service": 0 + }, + "renameByName": { + "Value": "QPS" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 53 + }, + "id": 41, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(label_replace(topk(10, sum(envoy_cluster_upstream_cx_active{higress=\"$gateway\", cluster_name!=\"xds-grpc\", cluster_name!=\"prometheus_stats\", cluster_name!=\"agent\", cluster_name!=\"BlackHoleCluster\", cluster_name!=\"sds-grpc\"}) by (cluster_name)), \"service\", \"$3\", \"cluster_name\", \"outbound_([0-9]+)_(.*)_(.*)$\"), \"port\", \"$1\", \"cluster_name\", \"outbound_([0-9]+)_(.*)_(.*)$\")", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Active Connection Top 10", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Value": 2, + "port": 1, + "service": 0 + }, + "renameByName": { + "Value": "Connections", + "cluster_name": "" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 53 + }, + "id": 42, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(label_replace(topk(10, sum(irate(envoy_cluster_upstream_rq_time_sum{higress=\"$gateway\", cluster_name!=\"xds-grpc\", cluster_name!=\"prometheus_stats\", cluster_name!=\"agent\", cluster_name!=\"BlackHoleCluster\", cluster_name!=\"sds-grpc\"}[5m])) by (cluster_name) / sum(irate(envoy_cluster_upstream_rq_time_count{higress=\"$gateway\",cluster_name!=\"xds-grpc\", cluster_name!=\"prometheus_stats\", cluster_name!=\"agent\", cluster_name!=\"BlackHoleCluster\", cluster_name!=\"sds-grpc\"}[5m])) by (cluster_name)), \"service\", \"$3\", \"cluster_name\", \"outbound_([0-9]+)_(.*)_(.*)$\"), \"port\", \"$1\", \"cluster_name\", \"outbound_([0-9]+)_(.*)_(.*)$\")", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "RT Top 10(Slow)", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Value": 2, + "port": 1, + "service": 0 + }, + "renameByName": { + "Value": "RT", + "service": "" + } + } + } + ], + "type": "table" + } + ], + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "higress-system-higress-gateway", + "value": "higress-system-higress-gateway" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "definition": "label_values(envoy_server_live, higress)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "gateway", + "options": [], + "query": { + "query": "label_values(envoy_server_live, higress)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "higress-system", + "value": "higress-system" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource.id}" + }, + "definition": "label_values(envoy_server_live, higress)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(envoy_server_live, higress)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "/(.*)-higress-gateway/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Higress AI Gateway Dashboard", + "weekStart": "" +}