From eea7328f349bab9ec866d276974a18ea400dd404 Mon Sep 17 00:00:00 2001 From: bharathappali Date: Mon, 31 Oct 2022 16:32:20 +0530 Subject: [PATCH 01/35] Adds api's to get list of namespaces and deployments Signed-off-by: bharathappali --- .../services/ListDeploymentsInNamespace.java | 65 +++++++++++++++++++ .../analyzer/services/ListNamespaces.java | 51 +++++++++++++++ .../service/KubernetesServices.java | 3 + .../service/impl/KubernetesServicesImpl.java | 14 ++++ .../com/autotune/utils/AutotuneConstants.java | 4 ++ 5 files changed, 137 insertions(+) create mode 100644 src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java create mode 100644 src/main/java/com/autotune/analyzer/services/ListNamespaces.java diff --git a/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java b/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java new file mode 100644 index 000000000..bdffb2a4f --- /dev/null +++ b/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java @@ -0,0 +1,65 @@ +package com.autotune.analyzer.services; + +import com.autotune.common.target.kubernetes.service.KubernetesServices; +import com.autotune.common.target.kubernetes.service.impl.KubernetesServicesImpl; +import com.autotune.utils.AutotuneConstants; +import org.json.JSONArray; +import org.json.JSONObject; + +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.util.stream.Collectors; + +public class ListDeploymentsInNamespace extends HttpServlet { + @Override + protected void doGet(HttpServletRequest request, HttpServletResponse response) { + KubernetesServices kubernetesServices = null; + try { + String inputData = request.getReader().lines().collect(Collectors.joining()); + JSONObject inputJson = new JSONObject(inputData); + if (!inputJson.has(AutotuneConstants.JSONKeys.NAMESPACE)) { + response.setStatus(HttpServletResponse.SC_BAD_REQUEST); + } else { + if (null == inputJson.getString(AutotuneConstants.JSONKeys.NAMESPACE) + || inputJson.getString(AutotuneConstants.JSONKeys.NAMESPACE).isBlank() + || inputJson.getString(AutotuneConstants.JSONKeys.NAMESPACE).isEmpty()) { + response.setStatus(HttpServletResponse.SC_BAD_REQUEST); + } else { + String namespace = inputJson.getString(AutotuneConstants.JSONKeys.NAMESPACE); + // Initialising the kubernetes service + kubernetesServices = new KubernetesServicesImpl(); + // Set response headers + response.addHeader("Access-Control-Allow-Origin", "*"); + response.addHeader("Access-Control-Allow-Methods", "POST, GET"); + response.addHeader("Access-Control-Allow-Headers", "X-PINGOTHER, Origin, X-Requested-With, Content-Type, Accept"); + response.addHeader("Access-Control-Max-Age", "1728000"); + JSONObject returnJson = new JSONObject(); + JSONObject dataJson = new JSONObject(); + JSONArray deploymentsList = new JSONArray(); + kubernetesServices.getDeploymentsBy(namespace).forEach(deployment -> { + deploymentsList.put(deployment.getMetadata().getName()); + }); + dataJson.put(AutotuneConstants.JSONKeys.DEPLOYMENTS, deploymentsList); + returnJson.put(AutotuneConstants.JSONKeys.DATA, dataJson); + // Set content type + response.setContentType("application/json"); + response.setCharacterEncoding("UTF-8"); + response.setStatus(HttpServletResponse.SC_OK); + response.getWriter().println(returnJson.toString(4)); + } + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (kubernetesServices != null) { + kubernetesServices.shutdownClient(); + } + } + } + + @Override + protected void doPost(HttpServletRequest request, HttpServletResponse response) { + doGet(request, response); + } +} diff --git a/src/main/java/com/autotune/analyzer/services/ListNamespaces.java b/src/main/java/com/autotune/analyzer/services/ListNamespaces.java new file mode 100644 index 000000000..3e8aac89c --- /dev/null +++ b/src/main/java/com/autotune/analyzer/services/ListNamespaces.java @@ -0,0 +1,51 @@ +package com.autotune.analyzer.services; + +import com.autotune.common.target.kubernetes.service.KubernetesServices; +import com.autotune.common.target.kubernetes.service.impl.KubernetesServicesImpl; +import com.autotune.utils.AutotuneConstants; +import org.json.JSONArray; +import org.json.JSONObject; + +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +public class ListNamespaces extends HttpServlet { + @Override + protected void doGet(HttpServletRequest request, HttpServletResponse response) { + KubernetesServices kubernetesServices = null; + try { + // Initialising the kubernetes service + kubernetesServices = new KubernetesServicesImpl(); + // Set response headers + response.addHeader("Access-Control-Allow-Origin", "*"); + response.addHeader("Access-Control-Allow-Methods", "POST, GET"); + response.addHeader("Access-Control-Allow-Headers", "X-PINGOTHER, Origin, X-Requested-With, Content-Type, Accept"); + response.addHeader("Access-Control-Max-Age", "1728000"); + JSONObject returnJson = new JSONObject(); + JSONObject dataJson = new JSONObject(); + JSONArray namespacesList = new JSONArray(); + kubernetesServices.getNamespaces().forEach(namespace -> { + namespacesList.put(namespace.getMetadata().getName()); + }); + dataJson.put(AutotuneConstants.JSONKeys.NAMESPACES, namespacesList); + returnJson.put(AutotuneConstants.JSONKeys.DATA, dataJson); + // Set content type + response.setContentType("application/json"); + response.setCharacterEncoding("UTF-8"); + response.setStatus(HttpServletResponse.SC_OK); + response.getWriter().println(returnJson.toString(4)); + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (kubernetesServices != null) { + kubernetesServices.shutdownClient(); + } + } + } + + @Override + protected void doPost(HttpServletRequest request, HttpServletResponse response) { + doGet(request, response); + } +} diff --git a/src/main/java/com/autotune/common/target/kubernetes/service/KubernetesServices.java b/src/main/java/com/autotune/common/target/kubernetes/service/KubernetesServices.java index cf4f4dcc6..95cacfdb0 100644 --- a/src/main/java/com/autotune/common/target/kubernetes/service/KubernetesServices.java +++ b/src/main/java/com/autotune/common/target/kubernetes/service/KubernetesServices.java @@ -60,6 +60,9 @@ public interface KubernetesServices { //get list of Deployments object based on labelKey, labelValue List getDeploymentsBy(String namespace, String deploymentName, String labelKey, String labelValue); + // get list of deployments in the namespace + List getDeploymentsBy(String namespace); + //Restart deployment. Used by EM to restart deployment during warmup/measurements cycle. boolean restartDeployment(String namespace, String deploymentName); diff --git a/src/main/java/com/autotune/common/target/kubernetes/service/impl/KubernetesServicesImpl.java b/src/main/java/com/autotune/common/target/kubernetes/service/impl/KubernetesServicesImpl.java index d63ace904..2eb032340 100644 --- a/src/main/java/com/autotune/common/target/kubernetes/service/impl/KubernetesServicesImpl.java +++ b/src/main/java/com/autotune/common/target/kubernetes/service/impl/KubernetesServicesImpl.java @@ -272,6 +272,20 @@ public List getDeploymentsBy(String namespace, String deploymentName return deploymentList; } + /** + * Get list of deployments in the namespace + * @param namespace + * @return + */ + @Override + public List getDeploymentsBy(String namespace) { + List deploymentList = null; + if (null != namespace && !namespace.isBlank() && !namespace.isEmpty()) { + deploymentList = kubernetesClient.apps().deployments().inNamespace(namespace).list().getItems(); + } + return deploymentList; + } + /** * Replace with new deployment. * diff --git a/src/main/java/com/autotune/utils/AutotuneConstants.java b/src/main/java/com/autotune/utils/AutotuneConstants.java index 3107c5e5b..cc44147bf 100644 --- a/src/main/java/com/autotune/utils/AutotuneConstants.java +++ b/src/main/java/com/autotune/utils/AutotuneConstants.java @@ -151,6 +151,10 @@ public static final class JSONKeys { public static final String DURATION = "duration"; public static final String PERCENTILE_INFO = "percentile_info"; public static final String UNITS = "units"; + + // UI support JSON keys + public static final String DATA = "data"; + public static final String NAMESPACES = "namespaces"; private JSONKeys() { } } From 9a5752f5288521fc43f52c7f9d8cd6cbfaabbe56 Mon Sep 17 00:00:00 2001 From: bharathappali Date: Mon, 31 Oct 2022 16:37:16 +0530 Subject: [PATCH 02/35] Adds license headers Signed-off-by: bharathappali --- .../services/ListDeploymentsInNamespace.java | 15 +++++++++++++++ .../analyzer/services/ListNamespaces.java | 15 +++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java b/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java index bdffb2a4f..bf1c7ad97 100644 --- a/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java +++ b/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java @@ -1,3 +1,18 @@ +/******************************************************************************* + * Copyright (c) 2022, 2022 Red Hat, IBM Corporation and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ package com.autotune.analyzer.services; import com.autotune.common.target.kubernetes.service.KubernetesServices; diff --git a/src/main/java/com/autotune/analyzer/services/ListNamespaces.java b/src/main/java/com/autotune/analyzer/services/ListNamespaces.java index 3e8aac89c..c8dcf7877 100644 --- a/src/main/java/com/autotune/analyzer/services/ListNamespaces.java +++ b/src/main/java/com/autotune/analyzer/services/ListNamespaces.java @@ -1,3 +1,18 @@ +/******************************************************************************* + * Copyright (c) 2022, 2022 Red Hat, IBM Corporation and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ package com.autotune.analyzer.services; import com.autotune.common.target.kubernetes.service.KubernetesServices; From 27b3d399235b11883e8fcf81ae8562e26e62ae34 Mon Sep 17 00:00:00 2001 From: bharathappali Date: Mon, 31 Oct 2022 16:42:42 +0530 Subject: [PATCH 03/35] Adds api's to server context Signed-off-by: bharathappali --- src/main/java/com/autotune/analyzer/Analyzer.java | 6 +++++- src/main/java/com/autotune/utils/ServerContext.java | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/autotune/analyzer/Analyzer.java b/src/main/java/com/autotune/analyzer/Analyzer.java index 9279e7089..e6c6d352c 100644 --- a/src/main/java/com/autotune/analyzer/Analyzer.java +++ b/src/main/java/com/autotune/analyzer/Analyzer.java @@ -59,5 +59,9 @@ public static void addServlets(ServletContextHandler context) { context.addServlet(UpdateResults.class, ServerContext.UPDATE_RESULTS); context.addServlet(Recommendation.class, ServerContext.RECOMMEND_RESULTS); context.addServlet(PerformanceProfileService.class, ServerContext.CREATE_PERF_PROFILE); - } + + // Adding UI support API's + context.addServlet(ListNamespaces.class, ServerContext.LIST_NAMESPACES); + context.addServlet(ListDeploymentsInNamespace.class, ServerContext.LIST_DEPLOYMENTS); + } } diff --git a/src/main/java/com/autotune/utils/ServerContext.java b/src/main/java/com/autotune/utils/ServerContext.java index 8dbbf73b7..db3412513 100644 --- a/src/main/java/com/autotune/utils/ServerContext.java +++ b/src/main/java/com/autotune/utils/ServerContext.java @@ -56,4 +56,9 @@ public class ServerContext { public static final String EXPERIMENT_MANAGER_LIST_EXPERIMENT_END_POINT = EXPERIMENT_MANAGER_SERVER_PORT + EXPERIMENT_MANAGER_LIST_EXPERIMENT_TRIAL; public static final String EXPERIMENT_MANAGER_LIST_TRIAL_STATUS = ROOT_CONTEXT + "listTrialStatus"; public static final String EXPERIMENT_MANAGER_LIST_TRIAL_STATUS_END_POINT = EXPERIMENT_MANAGER_SERVER_PORT + EXPERIMENT_MANAGER_LIST_TRIAL_STATUS; + + // UI support EndPoints + public static final String QUERY_CONTEXT = ROOT_CONTEXT + "query/"; + public static final String LIST_NAMESPACES = QUERY_CONTEXT + "listNamespaces"; + public static final String LIST_DEPLOYMENTS = QUERY_CONTEXT + "listDeployments"; } From 69403b929711553ed2b0334a53b9a8d7554ca4f1 Mon Sep 17 00:00:00 2001 From: bharathappali Date: Wed, 2 Nov 2022 16:05:40 +0530 Subject: [PATCH 04/35] Check for parameter and then for json body Signed-off-by: bharathappali --- .../services/ListDeploymentsInNamespace.java | 65 ++++++++++--------- 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java b/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java index bf1c7ad97..8af9e9b56 100644 --- a/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java +++ b/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java @@ -31,39 +31,46 @@ public class ListDeploymentsInNamespace extends HttpServlet { protected void doGet(HttpServletRequest request, HttpServletResponse response) { KubernetesServices kubernetesServices = null; try { - String inputData = request.getReader().lines().collect(Collectors.joining()); - JSONObject inputJson = new JSONObject(inputData); - if (!inputJson.has(AutotuneConstants.JSONKeys.NAMESPACE)) { - response.setStatus(HttpServletResponse.SC_BAD_REQUEST); - } else { - if (null == inputJson.getString(AutotuneConstants.JSONKeys.NAMESPACE) - || inputJson.getString(AutotuneConstants.JSONKeys.NAMESPACE).isBlank() - || inputJson.getString(AutotuneConstants.JSONKeys.NAMESPACE).isEmpty()) { + String namespace = request.getParameter(AutotuneConstants.JSONKeys.NAMESPACE); + if (null == namespace) { + String inputData = request.getReader().lines().collect(Collectors.joining()); + JSONObject inputJson = new JSONObject(inputData); + if (!inputJson.has(AutotuneConstants.JSONKeys.NAMESPACE)) { response.setStatus(HttpServletResponse.SC_BAD_REQUEST); + namespace = null; } else { - String namespace = inputJson.getString(AutotuneConstants.JSONKeys.NAMESPACE); - // Initialising the kubernetes service - kubernetesServices = new KubernetesServicesImpl(); - // Set response headers - response.addHeader("Access-Control-Allow-Origin", "*"); - response.addHeader("Access-Control-Allow-Methods", "POST, GET"); - response.addHeader("Access-Control-Allow-Headers", "X-PINGOTHER, Origin, X-Requested-With, Content-Type, Accept"); - response.addHeader("Access-Control-Max-Age", "1728000"); - JSONObject returnJson = new JSONObject(); - JSONObject dataJson = new JSONObject(); - JSONArray deploymentsList = new JSONArray(); - kubernetesServices.getDeploymentsBy(namespace).forEach(deployment -> { - deploymentsList.put(deployment.getMetadata().getName()); - }); - dataJson.put(AutotuneConstants.JSONKeys.DEPLOYMENTS, deploymentsList); - returnJson.put(AutotuneConstants.JSONKeys.DATA, dataJson); - // Set content type - response.setContentType("application/json"); - response.setCharacterEncoding("UTF-8"); - response.setStatus(HttpServletResponse.SC_OK); - response.getWriter().println(returnJson.toString(4)); + if (null == inputJson.getString(AutotuneConstants.JSONKeys.NAMESPACE) + || inputJson.getString(AutotuneConstants.JSONKeys.NAMESPACE).isBlank() + || inputJson.getString(AutotuneConstants.JSONKeys.NAMESPACE).isEmpty()) { + response.setStatus(HttpServletResponse.SC_BAD_REQUEST); + namespace = null; + } else { + namespace = inputJson.getString(AutotuneConstants.JSONKeys.NAMESPACE); + } } } + if (null != namespace) { + // Initialising the kubernetes service + kubernetesServices = new KubernetesServicesImpl(); + // Set response headers + response.addHeader("Access-Control-Allow-Origin", "*"); + response.addHeader("Access-Control-Allow-Methods", "POST, GET"); + response.addHeader("Access-Control-Allow-Headers", "X-PINGOTHER, Origin, X-Requested-With, Content-Type, Accept"); + response.addHeader("Access-Control-Max-Age", "1728000"); + JSONObject returnJson = new JSONObject(); + JSONObject dataJson = new JSONObject(); + JSONArray deploymentsList = new JSONArray(); + kubernetesServices.getDeploymentsBy(namespace).forEach(deployment -> { + deploymentsList.put(deployment.getMetadata().getName()); + }); + dataJson.put(AutotuneConstants.JSONKeys.DEPLOYMENTS, deploymentsList); + returnJson.put(AutotuneConstants.JSONKeys.DATA, dataJson); + // Set content type + response.setContentType("application/json"); + response.setCharacterEncoding("UTF-8"); + response.setStatus(HttpServletResponse.SC_OK); + response.getWriter().println(returnJson.toString(4)); + } } catch (Exception e) { e.printStackTrace(); } finally { From 2567b3dd583f640d94907a1fc7620eab9a94818e Mon Sep 17 00:00:00 2001 From: bharathappali Date: Mon, 6 Feb 2023 15:52:40 +0530 Subject: [PATCH 05/35] Addressed review comments Signed-off-by: bharathappali --- src/main/java/com/autotune/analyzer/Analyzer.java | 6 +++--- .../analyzer/services/ListDeploymentsInNamespace.java | 3 +-- src/main/java/com/autotune/utils/ServerContext.java | 8 ++++---- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/autotune/analyzer/Analyzer.java b/src/main/java/com/autotune/analyzer/Analyzer.java index e6c6d352c..d056b0e3a 100644 --- a/src/main/java/com/autotune/analyzer/Analyzer.java +++ b/src/main/java/com/autotune/analyzer/Analyzer.java @@ -60,8 +60,8 @@ public static void addServlets(ServletContextHandler context) { context.addServlet(Recommendation.class, ServerContext.RECOMMEND_RESULTS); context.addServlet(PerformanceProfileService.class, ServerContext.CREATE_PERF_PROFILE); - // Adding UI support API's - context.addServlet(ListNamespaces.class, ServerContext.LIST_NAMESPACES); - context.addServlet(ListDeploymentsInNamespace.class, ServerContext.LIST_DEPLOYMENTS); + // Adding UI support API's + context.addServlet(ListNamespaces.class, ServerContext.LIST_NAMESPACES); + context.addServlet(ListDeploymentsInNamespace.class, ServerContext.LIST_DEPLOYMENTS); } } diff --git a/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java b/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java index 8af9e9b56..2defe28c9 100644 --- a/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java +++ b/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java @@ -48,8 +48,7 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) { namespace = inputJson.getString(AutotuneConstants.JSONKeys.NAMESPACE); } } - } - if (null != namespace) { + } else { // Initialising the kubernetes service kubernetesServices = new KubernetesServicesImpl(); // Set response headers diff --git a/src/main/java/com/autotune/utils/ServerContext.java b/src/main/java/com/autotune/utils/ServerContext.java index db3412513..2a75babba 100644 --- a/src/main/java/com/autotune/utils/ServerContext.java +++ b/src/main/java/com/autotune/utils/ServerContext.java @@ -57,8 +57,8 @@ public class ServerContext { public static final String EXPERIMENT_MANAGER_LIST_TRIAL_STATUS = ROOT_CONTEXT + "listTrialStatus"; public static final String EXPERIMENT_MANAGER_LIST_TRIAL_STATUS_END_POINT = EXPERIMENT_MANAGER_SERVER_PORT + EXPERIMENT_MANAGER_LIST_TRIAL_STATUS; - // UI support EndPoints - public static final String QUERY_CONTEXT = ROOT_CONTEXT + "query/"; - public static final String LIST_NAMESPACES = QUERY_CONTEXT + "listNamespaces"; - public static final String LIST_DEPLOYMENTS = QUERY_CONTEXT + "listDeployments"; + // UI support EndPoints + public static final String QUERY_CONTEXT = ROOT_CONTEXT + "query/"; + public static final String LIST_NAMESPACES = QUERY_CONTEXT + "listNamespaces"; + public static final String LIST_DEPLOYMENTS = QUERY_CONTEXT + "listDeployments"; } From 87cc7ab7d3f5afdd0f9b4ca88f3e1cce3f5bfcd0 Mon Sep 17 00:00:00 2001 From: bharathappali Date: Mon, 6 Feb 2023 15:55:22 +0530 Subject: [PATCH 06/35] Fixed miss alignment Signed-off-by: bharathappali --- src/main/java/com/autotune/analyzer/Analyzer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/autotune/analyzer/Analyzer.java b/src/main/java/com/autotune/analyzer/Analyzer.java index d056b0e3a..524ba073c 100644 --- a/src/main/java/com/autotune/analyzer/Analyzer.java +++ b/src/main/java/com/autotune/analyzer/Analyzer.java @@ -63,5 +63,5 @@ public static void addServlets(ServletContextHandler context) { // Adding UI support API's context.addServlet(ListNamespaces.class, ServerContext.LIST_NAMESPACES); context.addServlet(ListDeploymentsInNamespace.class, ServerContext.LIST_DEPLOYMENTS); - } + } } From 52e1e1ccd0004fe862b3f07164b43d4c9f62171c Mon Sep 17 00:00:00 2001 From: bharathappali Date: Mon, 6 Feb 2023 16:05:31 +0530 Subject: [PATCH 07/35] Reverting my chnages Signed-off-by: bharathappali --- .../autotune/analyzer/services/ListDeploymentsInNamespace.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java b/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java index 2defe28c9..8af9e9b56 100644 --- a/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java +++ b/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java @@ -48,7 +48,8 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) { namespace = inputJson.getString(AutotuneConstants.JSONKeys.NAMESPACE); } } - } else { + } + if (null != namespace) { // Initialising the kubernetes service kubernetesServices = new KubernetesServicesImpl(); // Set response headers From 1af0d39a43422e535dc51b7c1c76240742882487 Mon Sep 17 00:00:00 2001 From: bharathappali Date: Mon, 6 Feb 2023 16:34:54 +0530 Subject: [PATCH 08/35] Return msg to user on invalid namespace Signed-off-by: bharathappali --- .../services/ListDeploymentsInNamespace.java | 38 ++++++++++++++----- .../com/autotune/utils/AutotuneConstants.java | 13 +++++++ 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java b/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java index 8af9e9b56..869da3ecf 100644 --- a/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java +++ b/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java @@ -31,17 +31,34 @@ public class ListDeploymentsInNamespace extends HttpServlet { protected void doGet(HttpServletRequest request, HttpServletResponse response) { KubernetesServices kubernetesServices = null; try { + // Add headers to avoid CORS + response.addHeader("Access-Control-Allow-Origin", "*"); + response.addHeader("Access-Control-Allow-Methods", "POST, GET"); + response.addHeader("Access-Control-Allow-Headers", "X-PINGOTHER, Origin, X-Requested-With, Content-Type, Accept"); + response.addHeader("Access-Control-Max-Age", "1728000"); + // Set content type + response.setContentType("application/json"); + // Set encoding + response.setCharacterEncoding("UTF-8"); + // Check if the namespace is passed as a URL param String namespace = request.getParameter(AutotuneConstants.JSONKeys.NAMESPACE); + String error = null; if (null == namespace) { + // Check if the request has a JSON body in which namespace is passed String inputData = request.getReader().lines().collect(Collectors.joining()); JSONObject inputJson = new JSONObject(inputData); if (!inputJson.has(AutotuneConstants.JSONKeys.NAMESPACE)) { response.setStatus(HttpServletResponse.SC_BAD_REQUEST); + error = AutotuneConstants.ErrorMsgs.APIErrorMsgs.ListDeploymentsInNamespace.NO_NAMESPACE_SENT; namespace = null; } else { - if (null == inputJson.getString(AutotuneConstants.JSONKeys.NAMESPACE) - || inputJson.getString(AutotuneConstants.JSONKeys.NAMESPACE).isBlank() + if (null == inputJson.getString(AutotuneConstants.JSONKeys.NAMESPACE)) { + error = AutotuneConstants.ErrorMsgs.APIErrorMsgs.ListDeploymentsInNamespace.NO_NAMESPACE_SENT; + response.setStatus(HttpServletResponse.SC_BAD_REQUEST); + namespace = null; + } else if( inputJson.getString(AutotuneConstants.JSONKeys.NAMESPACE).isBlank() || inputJson.getString(AutotuneConstants.JSONKeys.NAMESPACE).isEmpty()) { + error = AutotuneConstants.ErrorMsgs.APIErrorMsgs.ListDeploymentsInNamespace.EMPTY_NAMESPACE; response.setStatus(HttpServletResponse.SC_BAD_REQUEST); namespace = null; } else { @@ -49,14 +66,19 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) { } } } - if (null != namespace) { + // If namespace is not found return error + if (null == namespace) { + // if error is not set, set it to invalid namespace + if (null == error) { + error = AutotuneConstants.ErrorMsgs.APIErrorMsgs.ListDeploymentsInNamespace.INVALID_NAMESPACE; + } + JSONObject returnJson = new JSONObject(); + returnJson.put(AutotuneConstants.JSONKeys.ERROR, error); + response.getWriter().println(returnJson.toString(4)); + } else { // Initialising the kubernetes service kubernetesServices = new KubernetesServicesImpl(); // Set response headers - response.addHeader("Access-Control-Allow-Origin", "*"); - response.addHeader("Access-Control-Allow-Methods", "POST, GET"); - response.addHeader("Access-Control-Allow-Headers", "X-PINGOTHER, Origin, X-Requested-With, Content-Type, Accept"); - response.addHeader("Access-Control-Max-Age", "1728000"); JSONObject returnJson = new JSONObject(); JSONObject dataJson = new JSONObject(); JSONArray deploymentsList = new JSONArray(); @@ -66,8 +88,6 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) { dataJson.put(AutotuneConstants.JSONKeys.DEPLOYMENTS, deploymentsList); returnJson.put(AutotuneConstants.JSONKeys.DATA, dataJson); // Set content type - response.setContentType("application/json"); - response.setCharacterEncoding("UTF-8"); response.setStatus(HttpServletResponse.SC_OK); response.getWriter().println(returnJson.toString(4)); } diff --git a/src/main/java/com/autotune/utils/AutotuneConstants.java b/src/main/java/com/autotune/utils/AutotuneConstants.java index cc44147bf..5cb9b9580 100644 --- a/src/main/java/com/autotune/utils/AutotuneConstants.java +++ b/src/main/java/com/autotune/utils/AutotuneConstants.java @@ -254,4 +254,17 @@ private Memory() { } public static final double MEBIBYTES_TO_GIBIBYTES = INVERSE_BINARY_STANDARD; } } + + public static class ErrorMsgs { + private ErrorMsgs() { } + public static class APIErrorMsgs { + private APIErrorMsgs() { } + public static class ListDeploymentsInNamespace { + private ListDeploymentsInNamespace() { } + public static final String INVALID_NAMESPACE = "Given Namespace is invalid"; + public static final String NO_NAMESPACE_SENT = "Please pass a namespace to get the deployments"; + public static final String EMPTY_NAMESPACE = "Namespace cannot be empty"; + } + } + } } From 2647699b2bfac61f289db90e5605d78597d91515 Mon Sep 17 00:00:00 2001 From: bharathappali Date: Mon, 6 Feb 2023 16:36:48 +0530 Subject: [PATCH 09/35] Change comments Signed-off-by: bharathappali --- .../autotune/analyzer/services/ListDeploymentsInNamespace.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java b/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java index 869da3ecf..cab108b53 100644 --- a/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java +++ b/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java @@ -78,7 +78,7 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) { } else { // Initialising the kubernetes service kubernetesServices = new KubernetesServicesImpl(); - // Set response headers + // Create a return object JSONObject returnJson = new JSONObject(); JSONObject dataJson = new JSONObject(); JSONArray deploymentsList = new JSONArray(); From bfcb7da8399d255223e27cc92961d6fc8968d231 Mon Sep 17 00:00:00 2001 From: bharathappali Date: Mon, 6 Feb 2023 16:37:47 +0530 Subject: [PATCH 10/35] Change comments Signed-off-by: bharathappali --- .../autotune/analyzer/services/ListDeploymentsInNamespace.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java b/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java index cab108b53..dfb8a7e18 100644 --- a/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java +++ b/src/main/java/com/autotune/analyzer/services/ListDeploymentsInNamespace.java @@ -87,7 +87,7 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) { }); dataJson.put(AutotuneConstants.JSONKeys.DEPLOYMENTS, deploymentsList); returnJson.put(AutotuneConstants.JSONKeys.DATA, dataJson); - // Set content type + // Return OK response.setStatus(HttpServletResponse.SC_OK); response.getWriter().println(returnJson.toString(4)); } From 9fadbdee569d0476e500623b0fcf10101e136696 Mon Sep 17 00:00:00 2001 From: msvinaykumar Date: Mon, 6 Feb 2023 19:04:58 +0530 Subject: [PATCH 11/35] General_info renamed to aggregation_info Changed "Long Term" to "Performance", "Short Term" to "Cost" and "Medium Term" to "Balanced" Signed-off-by: msvinaykumar --- design/MonitoringModeAPI.md | 148 +++++++++++++++++++++++------------- 1 file changed, 94 insertions(+), 54 deletions(-) diff --git a/design/MonitoringModeAPI.md b/design/MonitoringModeAPI.md index 108cdc9e6..e0dfab020 100644 --- a/design/MonitoringModeAPI.md +++ b/design/MonitoringModeAPI.md @@ -18,7 +18,7 @@ see [Create Experiment](/design/CreateExperiment.md) "experiment_name": "quarkus-resteasy-autotune-min-http-response-time-db", "namespace": "default", "deployment_name": "tfb-qrh-sample", - "performanceProfile": "resource_optimization", + "performanceProfile": "resource-optimization-openshift", "mode": "monitor", "targetCluster": "remote", "containers": [ @@ -72,7 +72,9 @@ see [Update results](/design/UpdateResults.md) "container_metrics": { "cpuRequest" : { "results": { - "general_info": { + "value" : 1.1, + "units": "cores", + "aggregation_info": { "sum": 4.4, "avg": 1.1, "units": "cores" @@ -81,7 +83,9 @@ see [Update results](/design/UpdateResults.md) }, "cpuLimit": { "results": { - "general_info": { + "value" : 0.5, + "units": "cores", + "aggregation_info": { "sum": 2.0, "avg": 0.5, "units": "cores" @@ -90,7 +94,9 @@ see [Update results](/design/UpdateResults.md) }, "cpuUsage": { "results": { - "general_info": { + "value" : 0.12, + "units": "cores", + "aggregation_info": { "min": 0.14, "max": 0.84, "sum": 0.84, @@ -101,7 +107,9 @@ see [Update results](/design/UpdateResults.md) }, "cpuThrottle": { "results": { - "general_info": { + "value" : 0.045, + "units": "cores", + "aggregation_info": { "sum": 0.19, "max": 0.09, "avg": 0.045, @@ -111,7 +119,9 @@ see [Update results](/design/UpdateResults.md) }, "memoryRequest": { "results": { - "general_info": { + "value" : 50.12, + "units": "MiB", + "aggregation_info": { "sum": 250.85, "avg": 50.21, "units": "MiB" @@ -120,7 +130,9 @@ see [Update results](/design/UpdateResults.md) }, "memoryLimit": { "results": { - "general_info": { + "value" : 100, + "units": "MiB", + "aggregation_info": { "sum": 500, "avg": 100, "units": "MiB" @@ -129,7 +141,9 @@ see [Update results](/design/UpdateResults.md) }, "memoryUsage": { "results": { - "general_info": { + "value" : 40.1, + "units": "MiB", + "aggregation_info": { "min": 50.6, "max": 198.50, "sum": 198.50, @@ -139,8 +153,10 @@ see [Update results](/design/UpdateResults.md) } }, "memoryRSS": { + "value" : 31.91, + "units": "MiB", "results": { - "general_info": { + "aggregation_info": { "min": 50.6, "max": 123.6, "sum": 123.6, @@ -187,75 +203,87 @@ List recommendations output JSON as follows. Some parameters like CPU limit , EN "tfb-server-1": { "name": "tfb-server-1", "recommendation": { - "2023-01-15 17:53:40.498": { - "Short Term": { - "monitoringStartTime": "jan 01, 2023, 5:53:40 PM", - "monitoringEndTime": "Jan 02, 2023, 12:24:04 AM", - "podsCount": 0, + "2022-01-23T18:25:43.511Z": { + "Cost": { + "monitoringStartTime": "2022-01-22T18:25:43.511Z", + "monitoringEndTime": "2022-01-23T18:25:43.511Z", + "podsCount": 4, "confidence_level": 0.0, "config": { "max": { "memory": { - "amount": 0.0 + "amount": 128.8, + "units": "MiB" }, "cpu": { - "amount": 0.0 + "amount": 8.0, + "units": "cores" } }, "capacity": { "memory": { - "amount": 0.0 + "amount": 100.0, + "units": "MiB" }, "cpu": { - "amount": 0.0 + "amount": 4.0, + "units": "cores" } } } }, - "Medium Term": { - "monitoringStartTime": "jan 01, 2023, 5:53:40 PM", - "monitoringEndTime": "jan 07, 2023, 12:24:04 AM", + "Balanced": { + "monitoringStartTime": "2022-01-16T18:25:43.511Z", + "monitoringEndTime": "2022-01-23T18:25:43.511Z", "podsCount": 0, "confidence_level": 0.0, "config": { "max": { "memory": { - "amount": 0.0 + "amount": 128.8, + "units": "MiB" }, "cpu": { - "amount": 0.0 + "amount": 8.8, + "units": "cores" } }, "capacity": { "memory": { - "amount": 0.0 + "amount": 1000, + "units": "MiB" }, "cpu": { - "amount": 0.0 + "amount": 8.8, + "units": "cores" } } } }, - "Long Term": { - "monitoringStartTime": "jan 01, 2023, 5:53:40 PM", - "monitoringEndTime": "jan 15, 2023, 12:24:04 AM", + "Performance": { + "monitoringStartTime": "2022-01-08T18:25:43.511Z", + "monitoringEndTime": "2022-01-23T18:25:43.511Z", "podsCount": 0, "confidence_level": 0.0, "config": { "max": { "memory": { - "amount": 0.0 + "amount": 128.8, + "units": "MiB" }, "cpu": { - "amount": 0.0 + "amount": 8.0, + "units": "cores" } }, "capacity": { "memory": { - "amount": 0.0 + "amount": 1000.0, + "units": "MiB" }, "cpu": { - "amount": 0.0 + "amount": 8.0, + "units": "cores" } } } @@ -266,75 +294,87 @@ List recommendations output JSON as follows. Some parameters like CPU limit , EN "tfb-server-0": { "name": "tfb-server-0", "recommendation": { - "2029-10-26 17:53:40.498": { - "Short Term": { - "monitoringStartTime": "jan 01, 2023, 5:53:40 PM", - "monitoringEndTime": "jan 01, 2023, 12:24:04 AM", + "2022-01-23T18:25:43.511Z": { + "Cost": { + "monitoringStartTime": "2022-01-22T18:25:43.511Z", + "monitoringEndTime": "2022-01-23T18:25:43.511Z", "podsCount": 0, "confidence_level": 0.0, "config": { "max": { "memory": { - "amount": 0.0 + "amount": 128.8, + "units": "MiB" }, "cpu": { - "amount": 0.0 + "amount": 8.8, + "units": "cores" } }, "capacity": { "memory": { - "amount": 0.0 + "amount": 1000.0, + "units": "MiB" }, "cpu": { - "amount": 0.0 + "amount": 8.0, + "units": "cores" } } } }, - "Mid Term": { - "monitoringStartTime": "jan 01, 2023, 5:53:40 PM", - "monitoringEndTime": "jan 07, 2023, 12:24:04 AM", + "Balanced": { + "monitoringStartTime": "2022-01-16T18:25:43.511Z", + "monitoringEndTime": "2022-01-23T18:25:43.511Z", "podsCount": 0, "confidence_level": 0.0, "config": { "max": { "memory": { - "amount": 0.0 + "amount": 1000.0, + "units": "MiB" }, "cpu": { - "amount": 0.0 + "amount": 6.0, + "units": "cores" } }, "capacity": { "memory": { - "amount": 0.0 + "amount": 1000.0, + "units": "MiB" }, "cpu": { - "amount": 0.0 + "amount": 6.0, + "units": "cores" } } } }, - "Long Term": { - "monitoringStartTime": "jan 01, 2023, 5:53:40 PM", - "monitoringEndTime": "jan 30, 2023, 12:24:04 AM", + "Performance": { + "monitoringStartTime": "2022-01-08T18:25:43.511Z", + "monitoringEndTime": "2022-01-23T18:25:43.511Z", "podsCount": 0, "confidence_level": 0.0, "config": { "max": { "memory": { - "amount": 0.0 + "amount": 1000.0, + "units": "MiB" }, "cpu": { - "amount": 0.0 + "amount": 4.0, + "units": "cores" } }, "capacity": { "memory": { - "amount": 0.0 + "amount": 1000.0, + "units": "MiB" }, "cpu": { - "amount": 0.0 + "amount": 4.0, + "units": "cores" } } } From 6959b002970c667231ca322ba857c216cfd7ee70 Mon Sep 17 00:00:00 2001 From: msvinaykumar Date: Mon, 6 Feb 2023 18:42:36 +0530 Subject: [PATCH 12/35] General_info renamed to aggregation_info Changed "Long Term" to "Performance", "Short Term" to "Cost" and "Medium Term" to "Balanced" Signed-off-by: msvinaykumar --- .../updateMetricResultsTemplate.json | 106 +++++++++++++--- .../analyzer/services/ListExperiments.java | 6 +- .../analyzer/services/Recommendation.java | 116 +++++++++++------- .../analyzer/utils/GsonUTCDateAdapter.java | 53 ++++++++ .../common/data/metrics/EMMetricResult.java | 27 ++-- .../data/result/ExperimentResultData.java | 13 +- .../data/input/metrics/EMMetricResult.java | 21 ++-- .../experimentManager/utils/EMUtil.java | 4 +- .../com/autotune/utils/AutotuneConstants.java | 2 +- 9 files changed, 246 insertions(+), 102 deletions(-) create mode 100644 src/main/java/com/autotune/analyzer/utils/GsonUTCDateAdapter.java diff --git a/examples/monitoring-json/updateMetricResultsTemplate.json b/examples/monitoring-json/updateMetricResultsTemplate.json index 7dadee778..5e06b1177 100644 --- a/examples/monitoring-json/updateMetricResultsTemplate.json +++ b/examples/monitoring-json/updateMetricResultsTemplate.json @@ -1,23 +1,89 @@ -[{ - "experiment_name": "quarkus-resteasy-autotune-min-http-response-time-db", - "info": {"trial_info": { - "trial_number": 98, - "trial_timestamp": "yyyymmddhhmmss" - }}, - "deployments": [{ - "deployment_name": "tfb-qrh-sample", - "namespace": "default", - "pod_metrics": [], - "containers": [{ - "image_name": "kruize/tfb-qrh:1.13.2.F_et17", - "container_name": "tfb-server", - "container_metrics": { - "cpuRequest" : { - "results": { - "general_info": { - "sum": 4.4, - "mean": 1.1, - "units": "cores" +[ + { + "experiment_name": "quarkus-resteasy-autotune-min-http-response-time-db", + "trial_timestamp": "yyyymmddhhmmss", + "deployments": [ + { + "deployment_name": "tfb-qrh-sample", + "namespace": "default", + "pod_metrics": [], + "containers": [ + { + "image_name": "kruize/tfb-qrh:1.13.2.F_et17", + "container_name": "tfb-server", + "container_metrics": { + "cpuRequest": { + "results": { + "aggregation_info": { + "sum": 4.4, + "mean": 1.1, + "units": "cores" + } + } + }, + "cpuLimit": { + "results": { + "aggregation_info": { + "sum": 2.0, + "mean": 0.5, + "units": "cores" + } + } + }, + "cpuUsage": { + "results": { + "aggregation_info": { + "max": 0.84, + "mean": 0.12, + "units": "cores" + } + } + }, + "cpuThrottle": { + "results": { + "aggregation_info": { + "max": 0.09, + "mean": 0.045, + "units": "cores" + } + } + }, + "memoryRequest": { + "results": { + "aggregation_info": { + "sum": 250.85, + "mean": 50.21, + "units": "MiB" + } + } + }, + "memoryLimit": { + "results": { + "aggregation_info": { + "sum": 500, + "mean": 100, + "units": "MiB" + } + } + }, + "memoryUsage": { + "results": { + "aggregation_info": { + "max": 198.50, + "mean": 40.1, + "units": "MiB" + } + } + }, + "memoryRSS": { + "results": { + "aggregation_info": { + "max": 123.6, + "mean": 31.91, + "units": "MiB" + } + } + } } } }, diff --git a/src/main/java/com/autotune/analyzer/services/ListExperiments.java b/src/main/java/com/autotune/analyzer/services/ListExperiments.java index 22187b3b9..a30113536 100644 --- a/src/main/java/com/autotune/analyzer/services/ListExperiments.java +++ b/src/main/java/com/autotune/analyzer/services/ListExperiments.java @@ -19,6 +19,7 @@ import com.autotune.analyzer.AutotuneExperiment; import com.autotune.analyzer.RunExperiment; import com.autotune.analyzer.exceptions.InvalidValueException; +import com.autotune.analyzer.utils.GsonUTCDateAdapter; import com.autotune.common.annotations.json.AutotuneJSONExclusionStrategy; import com.autotune.common.experiments.ExperimentTrial; import com.autotune.common.k8sObjects.KruizeObject; @@ -39,6 +40,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.IOException; +import java.util.Date; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -70,6 +72,8 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) t Gson gsonObj = new GsonBuilder() .disableHtmlEscaping() .setPrettyPrinting() + .enableComplexMapKeySerialization() + .registerTypeAdapter(Date.class, new GsonUTCDateAdapter()) .setExclusionStrategies(new AutotuneJSONExclusionStrategy()) .create(); gsonStr = gsonObj.toJson(this.mainKruizeExperimentMap); @@ -136,4 +140,4 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) e.printStackTrace(); } } -} \ No newline at end of file +} diff --git a/src/main/java/com/autotune/analyzer/services/Recommendation.java b/src/main/java/com/autotune/analyzer/services/Recommendation.java index 509b5e23a..79567d22c 100644 --- a/src/main/java/com/autotune/analyzer/services/Recommendation.java +++ b/src/main/java/com/autotune/analyzer/services/Recommendation.java @@ -70,75 +70,87 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) t " \"tfb-server-1\": {\n" + " \"name\": \"tfb-server-1\",\n" + " \"recommendation\": {\n" + - " \"2023-01-15 17:53:40.498\": {\n" + - " \"Short Term\": {\n" + - " \"monitoringStartTime\": \"jan 01, 2023, 5:53:40 PM\",\n" + - " \"monitoringEndTime\": \"Jan 02, 2023, 12:24:04 AM\",\n" + - " \"podsCount\": 0,\n" + + " \"2022-01-23T18:25:43.511Z\": {\n" + + " \"Cost\": {\n" + + " \"monitoringStartTime\": \"2022-01-22T18:25:43.511Z\",\n" + + " \"monitoringEndTime\": \"2022-01-23T18:25:43.511Z\",\n" + + " \"podsCount\": 4,\n" + " \"confidence_level\": 0.0,\n" + " \"config\": {\n" + " \"max\": {\n" + " \"memory\": {\n" + - " \"amount\": 0.0\n" + + " \"amount\": 128.8,\n" + + " \"units\": \"MiB\"\n" + " },\n" + " \"cpu\": {\n" + - " \"amount\": 0.0\n" + + " \"amount\": 8.0,\n" + + " \"units\": \"cores\"\n" + " }\n" + " },\n" + " \"capacity\": {\n" + " \"memory\": {\n" + - " \"amount\": 0.0\n" + + " \"amount\": 100.0,\n" + + " \"units\": \"MiB\"\n" + " },\n" + " \"cpu\": {\n" + - " \"amount\": 0.0\n" + + " \"amount\": 4.0,\n" + + " \"units\": \"cores\"\n" + " }\n" + " }\n" + " }\n" + " },\n" + - " \"Medium Term\": {\n" + - " \"monitoringStartTime\": \"jan 01, 2023, 5:53:40 PM\",\n" + - " \"monitoringEndTime\": \"jan 07, 2023, 12:24:04 AM\",\n" + + " \"Balanced\": {\n" + + " \"monitoringStartTime\": \"2022-01-16T18:25:43.511Z\",\n" + + " \"monitoringEndTime\": \"2022-01-23T18:25:43.511Z\",\n" + " \"podsCount\": 0,\n" + " \"confidence_level\": 0.0,\n" + " \"config\": {\n" + " \"max\": {\n" + " \"memory\": {\n" + - " \"amount\": 0.0\n" + + " \"amount\": 128.8,\n" + + " \"units\": \"MiB\"\n" + " },\n" + " \"cpu\": {\n" + - " \"amount\": 0.0\n" + + " \"amount\": 8.8,\n" + + " \"units\": \"cores\"\n" + " }\n" + " },\n" + " \"capacity\": {\n" + " \"memory\": {\n" + - " \"amount\": 0.0\n" + + " \"amount\": 1000,\n" + + " \"units\": \"MiB\"\n" + " },\n" + " \"cpu\": {\n" + - " \"amount\": 0.0\n" + + " \"amount\": 8.8,\n" + + " \"units\": \"cores\"\n" + " }\n" + " }\n" + " }\n" + " },\n" + - " \"Long Term\": {\n" + - " \"monitoringStartTime\": \"jan 01, 2023, 5:53:40 PM\",\n" + - " \"monitoringEndTime\": \"jan 15, 2023, 12:24:04 AM\",\n" + + " \"Performance\": {\n" + + " \"monitoringStartTime\": \"2022-01-08T18:25:43.511Z\",\n" + + " \"monitoringEndTime\": \"2022-01-23T18:25:43.511Z\",\n" + " \"podsCount\": 0,\n" + " \"confidence_level\": 0.0,\n" + " \"config\": {\n" + " \"max\": {\n" + " \"memory\": {\n" + - " \"amount\": 0.0\n" + + " \"amount\": 128.8,\n" + + " \"units\": \"MiB\"\n" + " },\n" + " \"cpu\": {\n" + - " \"amount\": 0.0\n" + + " \"amount\": 8.0,\n" + + " \"units\": \"cores\"\n" + " }\n" + " },\n" + " \"capacity\": {\n" + " \"memory\": {\n" + - " \"amount\": 0.0\n" + + " \"amount\": 1000.0,\n" + + " \"units\": \"MiB\"\n" + " },\n" + " \"cpu\": {\n" + - " \"amount\": 0.0\n" + + " \"amount\": 8.0,\n" + + " \"units\": \"cores\"\n" + " }\n" + " }\n" + " }\n" + @@ -149,75 +161,87 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) t " \"tfb-server-0\": {\n" + " \"name\": \"tfb-server-0\",\n" + " \"recommendation\": {\n" + - " \"2029-10-26 17:53:40.498\": {\n" + - " \"Short Term\": {\n" + - " \"monitoringStartTime\": \"jan 01, 2023, 5:53:40 PM\",\n" + - " \"monitoringEndTime\": \"jan 01, 2023, 12:24:04 AM\",\n" + + " \"2022-01-23T18:25:43.511Z\": {\n" + + " \"Cost\": {\n" + + " \"monitoringStartTime\": \"2022-01-22T18:25:43.511Z\",\n" + + " \"monitoringEndTime\": \"2022-01-23T18:25:43.511Z\",\n" + " \"podsCount\": 0,\n" + " \"confidence_level\": 0.0,\n" + " \"config\": {\n" + " \"max\": {\n" + " \"memory\": {\n" + - " \"amount\": 0.0\n" + + " \"amount\": 128.8,\n" + + " \"units\": \"MiB\"\n" + " },\n" + " \"cpu\": {\n" + - " \"amount\": 0.0\n" + + " \"amount\": 8.8,\n" + + " \"units\": \"cores\"\n" + " }\n" + " },\n" + " \"capacity\": {\n" + " \"memory\": {\n" + - " \"amount\": 0.0\n" + + " \"amount\": 1000.0,\n" + + " \"units\": \"MiB\"\n" + " },\n" + " \"cpu\": {\n" + - " \"amount\": 0.0\n" + + " \"amount\": 8.0,\n" + + " \"units\": \"cores\"\n" + " }\n" + " }\n" + " }\n" + " },\n" + - " \"Mid Term\": {\n" + - " \"monitoringStartTime\": \"jan 01, 2023, 5:53:40 PM\",\n" + - " \"monitoringEndTime\": \"jan 07, 2023, 12:24:04 AM\",\n" + + " \"Balanced\": {\n" + + " \"monitoringStartTime\": \"2022-01-16T18:25:43.511Z\",\n" + + " \"monitoringEndTime\": \"2022-01-23T18:25:43.511Z\",\n" + " \"podsCount\": 0,\n" + " \"confidence_level\": 0.0,\n" + " \"config\": {\n" + " \"max\": {\n" + " \"memory\": {\n" + - " \"amount\": 0.0\n" + + " \"amount\": 1000.0,\n" + + " \"units\": \"MiB\"\n" + " },\n" + " \"cpu\": {\n" + - " \"amount\": 0.0\n" + + " \"amount\": 6.0,\n" + + " \"units\": \"cores\"\n" + " }\n" + " },\n" + " \"capacity\": {\n" + " \"memory\": {\n" + - " \"amount\": 0.0\n" + + " \"amount\": 1000.0,\n" + + " \"units\": \"MiB\"\n" + " },\n" + " \"cpu\": {\n" + - " \"amount\": 0.0\n" + + " \"amount\": 6.0,\n" + + " \"units\": \"cores\"\n" + " }\n" + " }\n" + " }\n" + " },\n" + - " \"Long Term\": {\n" + - " \"monitoringStartTime\": \"jan 01, 2023, 5:53:40 PM\",\n" + - " \"monitoringEndTime\": \"jan 30, 2023, 12:24:04 AM\",\n" + + " \"Performance\": {\n" + + " \"monitoringStartTime\": \"2022-01-08T18:25:43.511Z\",\n" + + " \"monitoringEndTime\": \"2022-01-23T18:25:43.511Z\",\n" + " \"podsCount\": 0,\n" + " \"confidence_level\": 0.0,\n" + " \"config\": {\n" + " \"max\": {\n" + " \"memory\": {\n" + - " \"amount\": 0.0\n" + + " \"amount\": 1000.0,\n" + + " \"units\": \"MiB\"\n" + " },\n" + " \"cpu\": {\n" + - " \"amount\": 0.0\n" + + " \"amount\": 4.0,\n" + + " \"units\": \"cores\"\n" + " }\n" + " },\n" + " \"capacity\": {\n" + " \"memory\": {\n" + - " \"amount\": 0.0\n" + + " \"amount\": 1000.0,\n" + + " \"units\": \"MiB\"\n" + " },\n" + " \"cpu\": {\n" + - " \"amount\": 0.0\n" + + " \"amount\": 4.0,\n" + + " \"units\": \"cores\"\n" + " }\n" + " }\n" + " }\n" + @@ -226,7 +250,7 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) t " }\n" + " }\n" + " }\n" + - "}\n"; + "}"; out.append("{\"quarkus-resteasy-autotune-min-http-response-time-db\":" + temp + "}"); //TODO add dummy results out.flush(); } diff --git a/src/main/java/com/autotune/analyzer/utils/GsonUTCDateAdapter.java b/src/main/java/com/autotune/analyzer/utils/GsonUTCDateAdapter.java new file mode 100644 index 000000000..de53be1dd --- /dev/null +++ b/src/main/java/com/autotune/analyzer/utils/GsonUTCDateAdapter.java @@ -0,0 +1,53 @@ +/******************************************************************************* + * Copyright (c) 2022 Red Hat, IBM Corporation and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ +package com.autotune.analyzer.utils; + +import com.google.gson.*; + +import java.lang.reflect.Type; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Locale; +import java.util.TimeZone; + +/** + * UTC data handler for GSON + */ +public class GsonUTCDateAdapter implements JsonSerializer, JsonDeserializer { + + private final DateFormat dateFormat; + + public GsonUTCDateAdapter() { + dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US); //This is the format I need + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); //This is the key line which converts the date to UTC which cannot be accessed with the default serializer + } + + @Override + public synchronized JsonElement serialize(Date date, Type type, JsonSerializationContext jsonSerializationContext) { + return new JsonPrimitive(dateFormat.format(date)); + } + + @Override + public synchronized Date deserialize(JsonElement jsonElement, Type type, JsonDeserializationContext jsonDeserializationContext) { + try { + return dateFormat.parse(jsonElement.getAsString()); + } catch (ParseException e) { + throw new JsonParseException(e); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/autotune/common/data/metrics/EMMetricResult.java b/src/main/java/com/autotune/common/data/metrics/EMMetricResult.java index d6dfe3301..3755abd0f 100644 --- a/src/main/java/com/autotune/common/data/metrics/EMMetricResult.java +++ b/src/main/java/com/autotune/common/data/metrics/EMMetricResult.java @@ -7,42 +7,41 @@ import org.json.JSONObject; public class EMMetricResult implements ConvertToJSON { - @SerializedName("general_info") + @SerializedName("aggregation_info") private EMMetricGenericResults emMetricGenericResults; @SerializedName("percentile_info") private EMMetricPercentileResults emMetricPercentileResults; private boolean isPercentileResultsAvailable; - public boolean isPercentileResultsAvailable() { - return isPercentileResultsAvailable; - } - - public void setPercentileResultsAvailable(boolean percentileResultsAvailable) { - isPercentileResultsAvailable = percentileResultsAvailable; - } - - public EMMetricResult () { + public EMMetricResult() { emMetricGenericResults = new EMMetricGenericResults(); emMetricPercentileResults = new EMMetricPercentileResults(); } public EMMetricResult(JSONObject jsonObject) throws IncompatibleInputJSONException { - if (!jsonObject.has(AutotuneConstants.JSONKeys.GENERAL_INFO) && - !jsonObject.has(AutotuneConstants.JSONKeys.PERCENTILE_INFO)) { + if (!jsonObject.has(AutotuneConstants.JSONKeys.AGGREGATION_INFO) && + !jsonObject.has(AutotuneConstants.JSONKeys.PERCENTILE_INFO)) { throw new IncompatibleInputJSONException(); } - emMetricGenericResults = new EMMetricGenericResults(jsonObject.getJSONObject(AutotuneConstants.JSONKeys.GENERAL_INFO)); + emMetricGenericResults = new EMMetricGenericResults(jsonObject.getJSONObject(AutotuneConstants.JSONKeys.AGGREGATION_INFO)); if (jsonObject.has(AutotuneConstants.JSONKeys.PERCENTILE_INFO)) { isPercentileResultsAvailable = true; emMetricPercentileResults = new EMMetricPercentileResults(jsonObject.getJSONObject(AutotuneConstants.JSONKeys.PERCENTILE_INFO)); } } + public boolean isPercentileResultsAvailable() { + return isPercentileResultsAvailable; + } + + public void setPercentileResultsAvailable(boolean percentileResultsAvailable) { + isPercentileResultsAvailable = percentileResultsAvailable; + } @Override public JSONObject toJSON() { JSONObject jsonObject = new JSONObject(); - jsonObject.put(AutotuneConstants.JSONKeys.GENERAL_INFO, emMetricGenericResults.toJSON()); + jsonObject.put(AutotuneConstants.JSONKeys.AGGREGATION_INFO, emMetricGenericResults.toJSON()); if (isPercentileResultsAvailable) { jsonObject.put(AutotuneConstants.JSONKeys.PERCENTILE_INFO, emMetricPercentileResults.toJSON()); } diff --git a/src/main/java/com/autotune/common/data/result/ExperimentResultData.java b/src/main/java/com/autotune/common/data/result/ExperimentResultData.java index ad82a8e99..4aacc9159 100644 --- a/src/main/java/com/autotune/common/data/result/ExperimentResultData.java +++ b/src/main/java/com/autotune/common/data/result/ExperimentResultData.java @@ -19,6 +19,7 @@ import com.autotune.utils.AnalyzerConstants; import com.google.gson.annotations.SerializedName; +import java.sql.Timestamp; import java.util.List; import java.util.Objects; @@ -29,9 +30,9 @@ public class ExperimentResultData { private String experiment_name; private String trialNumber; @SerializedName("start_timestamp") - private String starttimestamp; + private Timestamp starttimestamp; @SerializedName("end_timestamp") - private String endtimestamp; + private Timestamp endtimestamp; private List deployments; private AnalyzerConstants.ExperimentStatus status; private ValidationResultData validationResultData; @@ -77,19 +78,19 @@ public void setValidationResultData(ValidationResultData validationResultData) { this.validationResultData = validationResultData; } - public String getStarttimestamp() { + public Timestamp getStarttimestamp() { return starttimestamp; } - public void setStarttimestamp(String starttimestamp) { + public void setStarttimestamp(Timestamp starttimestamp) { this.starttimestamp = starttimestamp; } - public String getEndtimestamp() { + public Timestamp getEndtimestamp() { return endtimestamp; } - public void setEndtimestamp(String endtimestamp) { + public void setEndtimestamp(Timestamp endtimestamp) { this.endtimestamp = endtimestamp; } diff --git a/src/main/java/com/autotune/experimentManager/data/input/metrics/EMMetricResult.java b/src/main/java/com/autotune/experimentManager/data/input/metrics/EMMetricResult.java index bcbc5e09c..62a5794a7 100644 --- a/src/main/java/com/autotune/experimentManager/data/input/metrics/EMMetricResult.java +++ b/src/main/java/com/autotune/experimentManager/data/input/metrics/EMMetricResult.java @@ -11,21 +11,21 @@ public class EMMetricResult implements ConvertToJSON { private EMMetricPercentileResults emMetricPercentileResults; private boolean isPercentileResultsAvailable; - public EMMetricResult () { + public EMMetricResult() { emMetricGenericResults = new EMMetricGenericResults(); emMetricPercentileResults = new EMMetricPercentileResults(); } public EMMetricResult(JSONObject jsonObject) throws IncompatibleInputJSONException { - if (!(jsonObject.has(AutotuneConstants.JSONKeys.GENERAL_INFO) || - jsonObject.has(AutotuneConstants.JSONKeys.PERCENTILE_INFO))) { + if (!(jsonObject.has(AutotuneConstants.JSONKeys.AGGREGATION_INFO) || + jsonObject.has(AutotuneConstants.JSONKeys.PERCENTILE_INFO))) { throw new IncompatibleInputJSONException(); } if (jsonObject.has(AutotuneConstants.JSONKeys.PERCENTILE_INFO)) { isPercentileResultsAvailable = true; } - emMetricGenericResults = new EMMetricGenericResults(jsonObject.getJSONObject(AutotuneConstants.JSONKeys.GENERAL_INFO)); - if (isPercentileResultsAvailable){ + emMetricGenericResults = new EMMetricGenericResults(jsonObject.getJSONObject(AutotuneConstants.JSONKeys.AGGREGATION_INFO)); + if (isPercentileResultsAvailable) { emMetricPercentileResults = new EMMetricPercentileResults(jsonObject.getJSONObject(AutotuneConstants.JSONKeys.PERCENTILE_INFO)); } } @@ -34,21 +34,18 @@ public EMMetricGenericResults getEmMetricGenericResults() { return emMetricGenericResults; } + public EMMetricPercentileResults getEmMetricPercentileResults() { return emMetricPercentileResults; } + + public boolean isPercentileResultsAvailable() { return isPercentileResultsAvailable; } - public void setEmMetricGenericResults(EMMetricGenericResults emMetricGenericResults) { - this.emMetricGenericResults = emMetricGenericResults; - } - public void setEmMetricPercentileResults(EMMetricPercentileResults emMetricPercentileResults) { - this.emMetricPercentileResults = emMetricPercentileResults; - } public void setPercentileResultsAvailable(boolean percentileResultsAvailable) { isPercentileResultsAvailable = percentileResultsAvailable; @@ -57,7 +54,7 @@ public void setPercentileResultsAvailable(boolean percentileResultsAvailable) { @Override public JSONObject toJSON() { JSONObject jsonObject = new JSONObject(); - jsonObject.put(AutotuneConstants.JSONKeys.GENERAL_INFO, emMetricGenericResults.toJSON()); + jsonObject.put(AutotuneConstants.JSONKeys.AGGREGATION_INFO, emMetricGenericResults.toJSON()); if (isPercentileResultsAvailable) { jsonObject.put(AutotuneConstants.JSONKeys.PERCENTILE_INFO, emMetricPercentileResults.toJSON()); } diff --git a/src/main/java/com/autotune/experimentManager/utils/EMUtil.java b/src/main/java/com/autotune/experimentManager/utils/EMUtil.java index 12999352b..dd2bb5400 100644 --- a/src/main/java/com/autotune/experimentManager/utils/EMUtil.java +++ b/src/main/java/com/autotune/experimentManager/utils/EMUtil.java @@ -183,8 +183,8 @@ public static ExperimentResultData getRealMetricsJSON(ExperimentTrial experiment deploymentResultData.setContainers(containerResultDataList); ExperimentResultData experimentResultData = new ExperimentResultData(); experimentResultData.setExperiment_name(experimentTrial.getExperimentName()); - experimentResultData.setEndtimestamp(new Timestamp(System.currentTimeMillis()).toString()); - experimentResultData.setStarttimestamp(new Timestamp(System.currentTimeMillis()).toString()); + experimentResultData.setEndtimestamp(new Timestamp(System.currentTimeMillis())); + experimentResultData.setStarttimestamp(new Timestamp(System.currentTimeMillis())); experimentResultData.setTrialNumber(triaLNumber); List deploymentResultDataList = new ArrayList<>(); deploymentResultDataList.add(deploymentResultData); diff --git a/src/main/java/com/autotune/utils/AutotuneConstants.java b/src/main/java/com/autotune/utils/AutotuneConstants.java index 5cb9b9580..d98e7724b 100644 --- a/src/main/java/com/autotune/utils/AutotuneConstants.java +++ b/src/main/java/com/autotune/utils/AutotuneConstants.java @@ -129,7 +129,7 @@ public static final class JSONKeys { public static final String MEASUREMENT_RESULTS = "measurement_results"; public static final String SUMMARY_RESULTS = "summary_results"; public static final String ITERATION_RESULT = "iteration_result"; - public static final String GENERAL_INFO = "general_info"; + public static final String AGGREGATION_INFO = "aggregation_info"; public static final String RESULTS = "results"; public static final String SCORE = "score"; public static final String ERROR = "error"; From 312fbf40d010565e0d79a749613aa9363e6f933c Mon Sep 17 00:00:00 2001 From: msvinaykumar Date: Tue, 7 Feb 2023 14:50:01 +0530 Subject: [PATCH 13/35] Added Value attributs for Metrics. Signed-off-by: msvinaykumar --- .../analyzer/services/UpdateResults.java | 1 + ...ntainerResultData.java => Containers.java} | 10 +- .../data/result/DeploymentResultData.java | 6 +- .../autotune/common/data/result/Results.java | 59 ++++++++++ .../RemoteMonitoringOpenShiftImpl.java | 107 +++++++++--------- .../experimentManager/utils/EMUtil.java | 28 ++--- .../com/autotune/utils/AnalyzerConstants.java | 25 ++++ 7 files changed, 163 insertions(+), 73 deletions(-) rename src/main/java/com/autotune/common/data/result/{ContainerResultData.java => Containers.java} (81%) create mode 100644 src/main/java/com/autotune/common/data/result/Results.java diff --git a/src/main/java/com/autotune/analyzer/services/UpdateResults.java b/src/main/java/com/autotune/analyzer/services/UpdateResults.java index 15fdedc46..65ba58ff0 100644 --- a/src/main/java/com/autotune/analyzer/services/UpdateResults.java +++ b/src/main/java/com/autotune/analyzer/services/UpdateResults.java @@ -62,6 +62,7 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) try { String inputData = request.getReader().lines().collect(Collectors.joining()); List experimentResultDataList = Arrays.asList(new Gson().fromJson(inputData, ExperimentResultData[].class)); + LOGGER.debug(experimentResultDataList.toString()); new ExperimentInitiator().validateAndUpdateResults(mainKruizeExperimentMap, experimentResultDataList); ExperimentResultData invalidKExperimentResultData = experimentResultDataList.stream().filter((rData) -> (!rData.getValidationResultData().isSuccess())).findAny().orElse(null); if (null == invalidKExperimentResultData) { diff --git a/src/main/java/com/autotune/common/data/result/ContainerResultData.java b/src/main/java/com/autotune/common/data/result/Containers.java similarity index 81% rename from src/main/java/com/autotune/common/data/result/ContainerResultData.java rename to src/main/java/com/autotune/common/data/result/Containers.java index fcb8182a5..879fbd0d7 100644 --- a/src/main/java/com/autotune/common/data/result/ContainerResultData.java +++ b/src/main/java/com/autotune/common/data/result/Containers.java @@ -15,15 +15,17 @@ *******************************************************************************/ package com.autotune.common.data.result; +import com.autotune.utils.AnalyzerConstants; + import java.util.HashMap; /** * Experiments results storage object which is related to container metrics. */ -public class ContainerResultData { +public class Containers { private String image_name; private String container_name; - private HashMap>> container_metrics; + private HashMap> container_metrics; public String getImage_name() { return image_name; @@ -41,11 +43,11 @@ public void setContainer_name(String container_name) { this.container_name = container_name; } - public HashMap>> getContainer_metrics() { + public HashMap> getContainer_metrics() { return container_metrics; } - public void setContainer_metrics(HashMap>> container_metrics) { + public void setContainer_metrics(HashMap> container_metrics) { this.container_metrics = container_metrics; } diff --git a/src/main/java/com/autotune/common/data/result/DeploymentResultData.java b/src/main/java/com/autotune/common/data/result/DeploymentResultData.java index 66d718795..723bc7ca2 100644 --- a/src/main/java/com/autotune/common/data/result/DeploymentResultData.java +++ b/src/main/java/com/autotune/common/data/result/DeploymentResultData.java @@ -24,7 +24,7 @@ public class DeploymentResultData { private String deployment_name; private String namespace; - private List containers; + private List containers; private List pod_metrics; public String getDeployment_name() { @@ -43,11 +43,11 @@ public void setNamespace(String namespace) { this.namespace = namespace; } - public List getContainers() { + public List getContainers() { return containers; } - public void setContainers(List containers) { + public void setContainers(List containers) { this.containers = containers; } diff --git a/src/main/java/com/autotune/common/data/result/Results.java b/src/main/java/com/autotune/common/data/result/Results.java new file mode 100644 index 000000000..dda632116 --- /dev/null +++ b/src/main/java/com/autotune/common/data/result/Results.java @@ -0,0 +1,59 @@ +/******************************************************************************* + * Copyright (c) 2022 Red Hat, IBM Corporation and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ +package com.autotune.common.data.result; + + +import com.google.gson.annotations.SerializedName; + +public class Results { + @SerializedName("aggregation_info") + private AggregationInfoResult aggregation_info; + private Double value; + private String units; + + public AggregationInfoResult getAggregation_info() { + return aggregation_info; + } + + public void setAggregation_info(AggregationInfoResult aggregation_info) { + this.aggregation_info = aggregation_info; + } + + public Double getValue() { + return value; + } + + public void setValue(Double value) { + this.value = value; + } + + public String getUnits() { + return units; + } + + public void setUnits(String units) { + this.units = units; + } + + @Override + public String toString() { + return "ContainersResultData{" + + "aggregation_info=" + aggregation_info + + ", value=" + value + + ", units='" + units + '\'' + + '}'; + } +} diff --git a/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/RemoteMonitoringOpenShiftImpl.java b/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/RemoteMonitoringOpenShiftImpl.java index acaf595e2..7a3f8a2d9 100644 --- a/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/RemoteMonitoringOpenShiftImpl.java +++ b/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/RemoteMonitoringOpenShiftImpl.java @@ -15,13 +15,15 @@ *******************************************************************************/ package com.autotune.common.performanceProfiles.PerformanceProfileInterface; -import com.autotune.common.data.result.ContainerResultData; +import com.autotune.common.data.result.Containers; import com.autotune.common.data.result.DeploymentResultData; import com.autotune.common.data.result.ExperimentResultData; -import com.autotune.common.data.result.AggregationInfoResult; -import com.autotune.common.k8sObjects.*; +import com.autotune.common.data.result.Results; +import com.autotune.common.k8sObjects.KruizeObject; +import com.autotune.common.k8sObjects.Metric; import com.autotune.common.performanceProfiles.PerformanceProfile; import com.autotune.common.performanceProfiles.PerformanceProfilesDeployment; +import com.autotune.utils.AnalyzerConstants; import com.autotune.utils.AnalyzerErrorConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,6 +39,29 @@ public class RemoteMonitoringOpenShiftImpl implements PerfProfileInterface { private static final Logger LOGGER = LoggerFactory.getLogger(RemoteMonitoringOpenShiftImpl.class); + /** + * Converts the generalInfo class into Map to extract values for validation + * + * @param obj + * @return + * @throws IllegalAccessException + * @throws IllegalArgumentException + * @throws InvocationTargetException + */ + public static Map convertObjectToMap(Object obj) throws IllegalAccessException, + IllegalArgumentException, InvocationTargetException { + Method[] methods = obj.getClass().getMethods(); + Map map = new HashMap<>(); + for (Method m : methods) { + if (m.getName().startsWith("get") && !m.getName().startsWith("getClass")) { + Object value = m.invoke(obj); + if (value instanceof Double) + map.put(m.getName().substring(3).toLowerCase(), value); + } + } + return map; + } + @Override public String validate(KruizeObject kruizeObject, ExperimentResultData experimentResultData) { @@ -46,7 +71,7 @@ public String validate(KruizeObject kruizeObject, ExperimentResultData experimen .get(kruizeObject.getPerformanceProfile()); List aggrFunctionsObjects = new ArrayList<>(); List perfProfileFunctionVariablesList = new ArrayList<>(); - for (Metric metric:performanceProfile.getSloInfo().getFunctionVariables()) { + for (Metric metric : performanceProfile.getSloInfo().getFunctionVariables()) { perfProfileFunctionVariablesList.add(metric.getName()); metric.getAggregationFunctions().forEach(aggregationFunctions -> aggrFunctionsObjects.add(aggregationFunctions.getFunction())); @@ -56,36 +81,32 @@ public String validate(KruizeObject kruizeObject, ExperimentResultData experimen // Get the metrics data from the Kruize Object for (DeploymentResultData deploymentResultData : experimentResultData.getDeployments()) { - for (ContainerResultData containerResultData : deploymentResultData.getContainers()) { - HashMap>> containerMetricsMap = - containerResultData.getContainer_metrics(); - List kruizeFunctionVariablesList = containerMetricsMap.keySet().stream().toList(); + for (Containers containers : deploymentResultData.getContainers()) { + HashMap> containerMetricsMap = + containers.getContainer_metrics(); + List kruizeFunctionVariablesList = containerMetricsMap.keySet().stream().toList().stream().map(Enum::name).toList(); if (!(perfProfileFunctionVariablesList.size() == kruizeFunctionVariablesList.size() && - new HashSet<>(perfProfileFunctionVariablesList).containsAll(kruizeFunctionVariablesList) && - new HashSet<>(kruizeFunctionVariablesList).containsAll(perfProfileFunctionVariablesList))) { - LOGGER.debug("perfProfileFunctionVariablesList: {}",perfProfileFunctionVariablesList); - LOGGER.debug("kruizeFunctionVariablesList: {}",kruizeFunctionVariablesList); + new HashSet<>(perfProfileFunctionVariablesList).containsAll(kruizeFunctionVariablesList) && + new HashSet<>(kruizeFunctionVariablesList).containsAll(perfProfileFunctionVariablesList))) { + LOGGER.debug("perfProfileFunctionVariablesList: {}", perfProfileFunctionVariablesList); + LOGGER.debug("kruizeFunctionVariablesList: {}", kruizeFunctionVariablesList); perfProfileFunctionVariablesList.removeAll(kruizeFunctionVariablesList); errorMsg = errorMsg.concat(String.format("Following Performance Profile parameters are missing for experiment - %s : %s", experimentResultData.getExperiment_name(), perfProfileFunctionVariablesList)); break; - } else { - for(HashMap> funcVar:containerMetricsMap.values()){ - for(HashMap genInfo:funcVar.values()){ - Map genInfoClassAsMap; - for(AggregationInfoResult genInfoObj:genInfo.values()){ - try { - genInfoClassAsMap = RemoteMonitoringOpenShiftImpl.convertObjectToMap(genInfoObj); - errorMsg = validateAggFunction(genInfoClassAsMap.keySet(), aggrFunctionsObjects); - if (!errorMsg.isBlank()) { - errorMsg = errorMsg.concat(String.format("for the experiment : %s" - ,experimentResultData.getExperiment_name())); - return errorMsg; - } - } catch (IllegalAccessException | InvocationTargetException e) { - throw new RuntimeException(e); - } - } - } + } else { + for (HashMap funcVar : containerMetricsMap.values()) { + Map genInfoClassAsMap; + try { + genInfoClassAsMap = RemoteMonitoringOpenShiftImpl.convertObjectToMap(funcVar.get("results").getAggregation_info()); + errorMsg = validateAggFunction(genInfoClassAsMap.keySet(), aggrFunctionsObjects); + if (!errorMsg.isBlank()) { + errorMsg = errorMsg.concat(String.format("for the experiment : %s" + , experimentResultData.getExperiment_name())); + return errorMsg; + } + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } } } } @@ -96,6 +117,7 @@ public String validate(KruizeObject kruizeObject, ExperimentResultData experimen /** * Validates the aggregation function objects against the generalInfo metrics + * * @param keySet * @param aggrFunctionsObjects * @return @@ -122,32 +144,11 @@ private String validateAggFunction(Set keySet, List aggrFunction } return errorMsg; } + @Override public void recommend() { - //TODO: Will be updated once algo is completed + //TODO: Will be updated once algo is completed } - - /** - * Converts the generalInfo class into Map to extract values for validation - * @param obj - * @return - * @throws IllegalAccessException - * @throws IllegalArgumentException - * @throws InvocationTargetException - */ - public static Map convertObjectToMap(Object obj) throws IllegalAccessException, - IllegalArgumentException, InvocationTargetException { - Method[] methods = obj.getClass().getMethods(); - Map map = new HashMap<>(); - for (Method m : methods) { - if (m.getName().startsWith("get") && !m.getName().startsWith("getClass")) { - Object value = m.invoke(obj); - if (value instanceof Double) - map.put(m.getName().substring(3).toLowerCase(), value); - } - } - return map; - } } diff --git a/src/main/java/com/autotune/experimentManager/utils/EMUtil.java b/src/main/java/com/autotune/experimentManager/utils/EMUtil.java index dd2bb5400..6938963ab 100644 --- a/src/main/java/com/autotune/experimentManager/utils/EMUtil.java +++ b/src/main/java/com/autotune/experimentManager/utils/EMUtil.java @@ -25,6 +25,7 @@ import com.autotune.common.target.kubernetes.service.impl.KubernetesServicesImpl; import com.autotune.experimentManager.data.ExperimentTrialData; import com.autotune.experimentManager.data.input.EMMetricInput; +import com.autotune.utils.AnalyzerConstants; import com.autotune.utils.AutotuneConstants; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -157,30 +158,31 @@ public static ExperimentResultData getRealMetricsJSON(ExperimentTrial experiment podResultDataList.add(podResultData); } } - List containerResultDataList = new ArrayList<>(); + List containersList = new ArrayList<>(); for (Map.Entry> containerMapEntry : containersMap.entrySet()) { - ContainerResultData containerResultData = new ContainerResultData(); - containerResultData.setContainer_name(containerMapEntry.getKey()); - containerResultData.setImage_name(null); - HashMap>> containerMetrics = new HashMap<>(); + Containers containers = new Containers(); + containers.setContainer_name(containerMapEntry.getKey()); + containers.setImage_name(null); + + HashMap> containerMetrics = new HashMap<>(); for (Map.Entry containerMetricEntry : containerMapEntry.getValue().entrySet()) { Metric containerMetric = containerMetricEntry.getValue(); if (null != containerMetric.getEmMetricResult() && Float.MIN_VALUE != containerMetric.getEmMetricResult().getEmMetricGenericResults().getMean()) { - HashMap> resultMap = new HashMap<>(); + Results results = new Results(); AggregationInfoResult aggregationInfoResult = new AggregationInfoResult(); aggregationInfoResult.setAvg(containerMetric.getEmMetricResult().getEmMetricGenericResults().getMean()); aggregationInfoResult.setUnits(containerMetric.getEmMetricResult().getEmMetricGenericResults().getUnits()); - HashMap generalInfoResultHashMap = new HashMap<>(); - generalInfoResultHashMap.put("general_info", aggregationInfoResult); - resultMap.put("results", generalInfoResultHashMap); - containerMetrics.put(containerMetric.getName(), resultMap); + results.setAggregation_info(aggregationInfoResult); + HashMap resultsHashMap = new HashMap<>(); + resultsHashMap.put("results", results); + containerMetrics.put(AnalyzerConstants.MetricName.valueOf(containerMetric.getName()), resultsHashMap); } } - containerResultData.setContainer_metrics(containerMetrics); - containerResultDataList.add(containerResultData); + containers.setContainer_metrics(containerMetrics); + containersList.add(containers); } deploymentResultData.setPod_metrics(podResultDataList); - deploymentResultData.setContainers(containerResultDataList); + deploymentResultData.setContainers(containersList); ExperimentResultData experimentResultData = new ExperimentResultData(); experimentResultData.setExperiment_name(experimentTrial.getExperimentName()); experimentResultData.setEndtimestamp(new Timestamp(System.currentTimeMillis())); diff --git a/src/main/java/com/autotune/utils/AnalyzerConstants.java b/src/main/java/com/autotune/utils/AnalyzerConstants.java index de58c8300..bff330e97 100644 --- a/src/main/java/com/autotune/utils/AnalyzerConstants.java +++ b/src/main/java/com/autotune/utils/AnalyzerConstants.java @@ -80,6 +80,7 @@ public class AnalyzerConstants { public static final String DEPLOYMENT_NAME = "deployment_name"; public static final String SELECTOR = "selectorInfo"; public static final String NULL = "null"; + private AnalyzerConstants() { } @@ -94,6 +95,7 @@ public enum TargetType { REMOTE; } + public enum ExperimentStatus { QUEUED, IN_PROGRESS, @@ -105,6 +107,18 @@ public enum ExperimentStatus { FAILED; } + + public enum MetricName { + cpuRequest, + cpuLimit, + cpuUsage, + cpuThrottle, + memoryRequest, + memoryLimit, + memoryUsage, + memoryRSS + } + /** * Used to parse the Autotune kind resource */ @@ -146,6 +160,7 @@ public static final class AutotuneObjectConstants { private AutotuneObjectConstants() { } } + /** * Used to parse the AutotuneConfig resource */ @@ -194,10 +209,12 @@ public static final class AutotuneConfigConstants { public static final String LAYER_QUARKUS = "quarkus"; public static final String LAYER_OPENJ9 = "openj9"; public static final String LAYER_NODEJS = "nodejs"; + private AutotuneConfigConstants() { } } + /** * Contains Strings used in REST services */ @@ -232,9 +249,11 @@ public static final class ServiceConstants { public static final String EXPERIMENT_TRIALS = "experiment_trials"; public static final String NA = "NA"; public static final String SECONDS = " seconds"; + private ServiceConstants() { } } + /** * Contains Strings used in the HOTSPOT Layer */ @@ -275,10 +294,12 @@ public static final class HotspotConstants { public static final String USE_STRING_DEDUPLICATION = "UseStringDeduplication"; public static final String USE_SUPER_WORD = "UseSuperWord"; public static final String USE_TYPE_SPECULATION = "UseTypeSpeculation"; + private HotspotConstants() { } } + /** * Contains Strings used in the QUARKUS Layer */ @@ -286,10 +307,12 @@ public static final class QuarkusConstants { public static final String QUARKUS = "quarkus"; public static final String DOPTION = " -D"; + private QuarkusConstants() { } } + /** * Contains Strings used in the Container Layer */ @@ -297,10 +320,12 @@ public static final class ContainerConstants { public static final String CPU_REQUEST = "cpuRequest"; public static final String MEM_REQUEST = "memoryRequest"; + private ContainerConstants() { } } + public static class createExperimentParallelEngineConfigs { /** * MAX Queue size to stack experiments From 259ff493cc8c59acb3f452dd81216bc377776c10 Mon Sep 17 00:00:00 2001 From: msvinaykumar Date: Tue, 7 Feb 2023 14:59:14 +0530 Subject: [PATCH 14/35] Format issue fixed. Signed-off-by: msvinaykumar --- .../updateMetricResultsTemplate.json | 114 ++++++------------ 1 file changed, 38 insertions(+), 76 deletions(-) diff --git a/examples/monitoring-json/updateMetricResultsTemplate.json b/examples/monitoring-json/updateMetricResultsTemplate.json index 5e06b1177..17a61ab38 100644 --- a/examples/monitoring-json/updateMetricResultsTemplate.json +++ b/examples/monitoring-json/updateMetricResultsTemplate.json @@ -1,7 +1,8 @@ [ { "experiment_name": "quarkus-resteasy-autotune-min-http-response-time-db", - "trial_timestamp": "yyyymmddhhmmss", + "start_timestamp": "2022-01-23T18:25:43.511Z", + "end_timestamp": "2022-01-23T18:25:43.511Z", "deployments": [ { "deployment_name": "tfb-qrh-sample", @@ -14,143 +15,104 @@ "container_metrics": { "cpuRequest": { "results": { + "value": 1.1, + "units": "cores", "aggregation_info": { "sum": 4.4, - "mean": 1.1, + "avg": 1.1, "units": "cores" } } }, "cpuLimit": { "results": { + "value": 0.5, + "units": "cores", "aggregation_info": { "sum": 2.0, - "mean": 0.5, + "avg": 0.5, "units": "cores" } } }, "cpuUsage": { "results": { + "value": 0.12, + "units": "cores", "aggregation_info": { + "min": 0.14, "max": 0.84, - "mean": 0.12, + "sum": 0.84, + "avg": 0.12, "units": "cores" } } }, "cpuThrottle": { "results": { + "value": 0.045, + "units": "cores", "aggregation_info": { + "sum": 0.19, "max": 0.09, - "mean": 0.045, + "avg": 0.045, "units": "cores" } } }, "memoryRequest": { "results": { + "value": 50.12, + "units": "MiB", "aggregation_info": { "sum": 250.85, - "mean": 50.21, + "avg": 50.21, "units": "MiB" } } }, "memoryLimit": { "results": { + "value": 100, + "units": "MiB", "aggregation_info": { "sum": 500, - "mean": 100, + "avg": 100, "units": "MiB" } } }, "memoryUsage": { "results": { + "value": 40.1, + "units": "MiB", "aggregation_info": { + "min": 50.6, "max": 198.50, - "mean": 40.1, + "sum": 198.50, + "avg": 40.1, "units": "MiB" } } }, "memoryRSS": { + "value": 31.91, + "units": "MiB", "results": { "aggregation_info": { + "min": 50.6, "max": 123.6, - "mean": 31.91, + "sum": 123.6, + "avg": 31.91, "units": "MiB" } } } } } - }, - "cpuLimit": { - "results": { - "general_info": { - "sum": 2.0, - "mean": 0.5, - "units": "cores" - } - } - }, - "cpuUsage": { - "results": { - "general_info": { - "max": 0.84, - "mean": 0.12, - "units": "cores" - } - } - }, - "cpuThrottle": { - "results": { - "general_info": { - "max": 0.09, - "mean": 0.045, - "units": "cores" - } - } - }, - "memoryRequest": { - "results": { - "general_info": { - "sum": 250.85, - "mean": 50.21, - "units": "MiB" - } - } - }, - "memoryLimit": { - "results": { - "general_info": { - "sum": 500, - "mean": 100, - "units": "MiB" - } - } - }, - "memoryUsage": { - "results": { - "general_info": { - "max": 198.50, - "mean": 40.1, - "units": "MiB" - } - } - }, - "memoryRSS": { - "results": { - "general_info": { - "max": 123.6, - "mean": 31.91, - "units": "MiB" - } - } - } + ] } - }] - }] -}] + ] + } +] + From 61e29227682c969f7352688f48888a7c2ae5b187 Mon Sep 17 00:00:00 2001 From: msvinaykumar Date: Wed, 8 Feb 2023 15:07:41 +0530 Subject: [PATCH 15/35] EOF added. Signed-off-by: msvinaykumar --- .../com/autotune/analyzer/utils/GsonUTCDateAdapter.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/autotune/analyzer/utils/GsonUTCDateAdapter.java b/src/main/java/com/autotune/analyzer/utils/GsonUTCDateAdapter.java index de53be1dd..fddbcde80 100644 --- a/src/main/java/com/autotune/analyzer/utils/GsonUTCDateAdapter.java +++ b/src/main/java/com/autotune/analyzer/utils/GsonUTCDateAdapter.java @@ -33,8 +33,8 @@ public class GsonUTCDateAdapter implements JsonSerializer, JsonDeserialize private final DateFormat dateFormat; public GsonUTCDateAdapter() { - dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US); //This is the format I need - dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); //This is the key line which converts the date to UTC which cannot be accessed with the default serializer + dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US); + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); } @Override @@ -50,4 +50,4 @@ public synchronized Date deserialize(JsonElement jsonElement, Type type, JsonDes throw new JsonParseException(e); } } -} \ No newline at end of file +} From a148f96185ffdfd04da50e0c99a949f6038881b5 Mon Sep 17 00:00:00 2001 From: msvinaykumar Date: Wed, 8 Feb 2023 15:21:31 +0530 Subject: [PATCH 16/35] Changed Locale to root. Signed-off-by: msvinaykumar --- .../java/com/autotune/analyzer/utils/GsonUTCDateAdapter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/autotune/analyzer/utils/GsonUTCDateAdapter.java b/src/main/java/com/autotune/analyzer/utils/GsonUTCDateAdapter.java index fddbcde80..f8aea6bcd 100644 --- a/src/main/java/com/autotune/analyzer/utils/GsonUTCDateAdapter.java +++ b/src/main/java/com/autotune/analyzer/utils/GsonUTCDateAdapter.java @@ -33,7 +33,7 @@ public class GsonUTCDateAdapter implements JsonSerializer, JsonDeserialize private final DateFormat dateFormat; public GsonUTCDateAdapter() { - dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US); + dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ROOT); dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); } From 87e5453eca29915e2b0a028a398a1f9a22557be2 Mon Sep 17 00:00:00 2001 From: saakhan Date: Thu, 9 Feb 2023 10:58:09 +0530 Subject: [PATCH 17/35] add default profile and corresponding class, update interface data and related files Signed-off-by: saakhan --- .../default_openshift.yaml | 38 ++++ .../analyzer/services/UpdateResults.java | 7 +- .../analyzer/utils/AlgebraicParser.java | 2 +- .../autotune/analyzer/utils/EvalExParser.java | 6 +- .../analyzer/utils/ExperimentInitiator.java | 10 +- .../utils/ExperimentResultValidation.java | 66 +++++-- .../DefaultOpenshiftImpl.java | 183 ++++++++++++++++++ .../PerfProfileInterface.java | 6 +- ...=> ResourceOptimizationOpenshiftImpl.java} | 14 +- .../com/autotune/utils/AnalyzerConstants.java | 1 + .../analyzer/utils/TestEvalExParser.java | 2 +- 11 files changed, 303 insertions(+), 32 deletions(-) create mode 100644 manifests/autotune/performance-profiles/default_openshift.yaml create mode 100644 src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/DefaultOpenshiftImpl.java rename src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/{RemoteMonitoringOpenShiftImpl.java => ResourceOptimizationOpenshiftImpl.java} (92%) diff --git a/manifests/autotune/performance-profiles/default_openshift.yaml b/manifests/autotune/performance-profiles/default_openshift.yaml new file mode 100644 index 000000000..5230382d8 --- /dev/null +++ b/manifests/autotune/performance-profiles/default_openshift.yaml @@ -0,0 +1,38 @@ +apiVersion: "recommender.com/v1" +kind: "KruizePerformanceProfile" +metadata: + name: "default-openshift" +profile_version: 1.0 +k8s_type: openshift + +slo: + slo_class: "resource_usage" + direction: "minimize" + + # Refer to src/.../performanceProfiles/PerformanceProfileInterface/DefaultImpl.java + objective_function: + function_type: expression + expression: "(80*min(memoryRSS))/(85*avg(cpuRequest))" + + function_variables: + # CPU Request + # Show cpu requests in cores for a container in a deployment + - name: cpuRequest + datasource: prometheus + value_type: "double" + kubernetes_object: "container" + + aggregation_functions: + - function: 'avg' + query: 'avg(kube_pod_container_resource_requests{pod=~"$DEPLOYMENT_NAME$-[^-]*-[^-]*$", container="$CONTAINER_NAME$", namespace="$NAMESPACE", resource="cpu", unit="core"})' + + # 2.4 Memory RSS + - name: memoryRSS + datasource: prometheus + value_type: "double" + kubernetes_object: "container" + + aggregation_functions: + # Approx minimum memory RSS per container in a deployment + - function: min + query: 'min(min_over_time(container_memory_rss{pod=~"$DEPLOYMENT_NAME$-[^-]*-[^-]*$", namespace=$NAMESPACE$, container="$CONTAINER_NAME$"}[15m]))' diff --git a/src/main/java/com/autotune/analyzer/services/UpdateResults.java b/src/main/java/com/autotune/analyzer/services/UpdateResults.java index 15fdedc46..c60714270 100644 --- a/src/main/java/com/autotune/analyzer/services/UpdateResults.java +++ b/src/main/java/com/autotune/analyzer/services/UpdateResults.java @@ -20,6 +20,7 @@ import com.autotune.analyzer.utils.ExperimentInitiator; import com.autotune.common.data.result.ExperimentResultData; import com.autotune.common.k8sObjects.KruizeObject; +import com.autotune.common.performanceProfiles.PerformanceProfile; import com.autotune.utils.AnalyzerConstants; import com.google.gson.Gson; import org.slf4j.Logger; @@ -34,6 +35,7 @@ import java.io.IOException; import java.io.PrintWriter; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -50,11 +52,14 @@ public class UpdateResults extends HttpServlet { private static final long serialVersionUID = 1L; private static final Logger LOGGER = LoggerFactory.getLogger(UpdateResults.class); Map mainKruizeExperimentMap; + Map performanceProfilesMap; @Override public void init(ServletConfig config) throws ServletException { super.init(config); this.mainKruizeExperimentMap = (ConcurrentHashMap) getServletContext().getAttribute(AnalyzerConstants.EXPERIMENT_MAP); + this.performanceProfilesMap = (HashMap) getServletContext() + .getAttribute(AnalyzerConstants.PerformanceProfileConstants.PERF_PROFILE_MAP); } @Override @@ -62,7 +67,7 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) try { String inputData = request.getReader().lines().collect(Collectors.joining()); List experimentResultDataList = Arrays.asList(new Gson().fromJson(inputData, ExperimentResultData[].class)); - new ExperimentInitiator().validateAndUpdateResults(mainKruizeExperimentMap, experimentResultDataList); + new ExperimentInitiator().validateAndUpdateResults(mainKruizeExperimentMap, experimentResultDataList, performanceProfilesMap); ExperimentResultData invalidKExperimentResultData = experimentResultDataList.stream().filter((rData) -> (!rData.getValidationResultData().isSuccess())).findAny().orElse(null); if (null == invalidKExperimentResultData) { sendSuccessResponse(response, "Results added successfully! View saved results at /listExperiments."); diff --git a/src/main/java/com/autotune/analyzer/utils/AlgebraicParser.java b/src/main/java/com/autotune/analyzer/utils/AlgebraicParser.java index 5edc42f57..698805446 100644 --- a/src/main/java/com/autotune/analyzer/utils/AlgebraicParser.java +++ b/src/main/java/com/autotune/analyzer/utils/AlgebraicParser.java @@ -22,7 +22,7 @@ public interface AlgebraicParser { - String parse(String objFunction, String valueType, Map objFunctionMap); + String parse(String objFunction, Map objFunctionMap); Boolean validate(String objFunction, ArrayList functionVariables); diff --git a/src/main/java/com/autotune/analyzer/utils/EvalExParser.java b/src/main/java/com/autotune/analyzer/utils/EvalExParser.java index 6d8deadc3..e8c3aaed2 100644 --- a/src/main/java/com/autotune/analyzer/utils/EvalExParser.java +++ b/src/main/java/com/autotune/analyzer/utils/EvalExParser.java @@ -42,14 +42,14 @@ public class EvalExParser implements AlgebraicParser { private static final Logger LOGGER = LoggerFactory.getLogger(EvalExParser.class); /** - * parse the objective function and return the result based on the valueType received + * parse the objective function and return the result based on the valueType received + * * @param objFunction - * @param valueType * @param objFunctionMap * @return */ @Override - public String parse(String objFunction, String valueType, Map objFunctionMap) { + public String parse(String objFunction, Map objFunctionMap) { BigDecimal result; diff --git a/src/main/java/com/autotune/analyzer/utils/ExperimentInitiator.java b/src/main/java/com/autotune/analyzer/utils/ExperimentInitiator.java index 71168e058..d3f4a6e89 100644 --- a/src/main/java/com/autotune/analyzer/utils/ExperimentInitiator.java +++ b/src/main/java/com/autotune/analyzer/utils/ExperimentInitiator.java @@ -20,6 +20,7 @@ import com.autotune.common.data.ValidationResultData; import com.autotune.common.data.result.ExperimentResultData; import com.autotune.common.k8sObjects.KruizeObject; +import com.autotune.common.performanceProfiles.PerformanceProfile; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -70,16 +71,17 @@ public ValidationResultData validateAndAddNewExperiments( /** * @param mainKruizeExperimentMap * @param experimentResultDataList + * @param performanceProfilesMap * @return */ public ValidationResultData validateAndUpdateResults( Map mainKruizeExperimentMap, - List experimentResultDataList - ) { + List experimentResultDataList, + Map performanceProfilesMap) { ValidationResultData validationResultData = new ValidationResultData(false, null); try { - ExperimentResultValidation experimentResultValidation = new ExperimentResultValidation(mainKruizeExperimentMap); - experimentResultValidation.validate(experimentResultDataList); + ExperimentResultValidation experimentResultValidation = new ExperimentResultValidation(mainKruizeExperimentMap, performanceProfilesMap); + experimentResultValidation.validate(experimentResultDataList, performanceProfilesMap); if (experimentResultValidation.isSuccess()) { ExperimentInterface experimentInterface = new ExperimentInterfaceImpl(); experimentInterface.addResultsToLocalStorage(mainKruizeExperimentMap, experimentResultDataList); diff --git a/src/main/java/com/autotune/analyzer/utils/ExperimentResultValidation.java b/src/main/java/com/autotune/analyzer/utils/ExperimentResultValidation.java index a550c2f8c..fd18620c7 100644 --- a/src/main/java/com/autotune/analyzer/utils/ExperimentResultValidation.java +++ b/src/main/java/com/autotune/analyzer/utils/ExperimentResultValidation.java @@ -18,10 +18,15 @@ import com.autotune.common.data.ValidationResultData; import com.autotune.common.data.result.ExperimentResultData; import com.autotune.common.k8sObjects.KruizeObject; -import com.autotune.common.performanceProfiles.PerformanceProfileInterface.RemoteMonitoringOpenShiftImpl; +import com.autotune.common.performanceProfiles.PerformanceProfile; +import com.autotune.common.performanceProfiles.PerformanceProfileInterface.ResourceOptimizationOpenshiftImpl; +import com.autotune.utils.AnalyzerConstants; +import com.autotune.utils.AnalyzerErrorConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.List; import java.util.Map; @@ -33,12 +38,14 @@ public class ExperimentResultValidation { private boolean success; private String errorMessage; private Map mainKruizeExperimentMAP; + private Map performanceProfileMap; - public ExperimentResultValidation(Map mainKruizeExperimentMAP) { + public ExperimentResultValidation(Map mainKruizeExperimentMAP,Map performanceProfileMap) { this.mainKruizeExperimentMAP = mainKruizeExperimentMAP; + this.performanceProfileMap = performanceProfileMap; } - public void validate(List experimentResultDataList) { + public void validate(List experimentResultDataList, Map performanceProfilesMap) { try { boolean proceed = false; String errorMsg = ""; @@ -55,18 +62,43 @@ public void validate(List experimentResultDataList) { resultData.setValidationResultData(new ValidationResultData(false, errorMsg)); break; } - // Validate Performance Profile data - RemoteMonitoringOpenShiftImpl remoteMonitoringOpenShiftImpl = new RemoteMonitoringOpenShiftImpl(); - errorMsg = remoteMonitoringOpenShiftImpl.validate(kruizeObject,resultData); - if (errorMsg.isEmpty() || errorMsg.isBlank()) { - // call recommend() method here - remoteMonitoringOpenShiftImpl.recommend(); - proceed = true; - } else { + /* + Fetch the performance profile from the Map corresponding to the name in the kruize object, + and then validate the Performance Profile data + */ + try { + PerformanceProfile performanceProfile = performanceProfilesMap.get(kruizeObject.getPerformanceProfile()); + // Get the corresponding class for validating the performance profile + String validationClassName = AnalyzerConstants.PerformanceProfileConstants.PERFORMANCE_PROFILE_PKG + .concat(getValidationClass(performanceProfile.getName())); + Class validationClass = Class.forName(validationClassName); + + Object object = validationClass.getDeclaredConstructor().newInstance(); + Class[] parameterTypes = new Class[] { PerformanceProfile.class, ExperimentResultData.class }; + Method method = validationClass.getMethod("validate",parameterTypes); + + errorMsg = (String) method.invoke(object, performanceProfile, resultData); + + if (errorMsg.isEmpty() || errorMsg.isBlank()) { + // call recommend() method here + method = validationClass.getMethod("recommend",parameterTypes); + errorMsg = (String) method.invoke(object, performanceProfile, resultData); + + proceed = true; + } else { + proceed = false; + resultData.setValidationResultData(new ValidationResultData(false, errorMsg)); + break; + } + } catch (NullPointerException | ClassNotFoundException | NoSuchMethodException | + IllegalAccessException | InvocationTargetException e) { + LOGGER.error("Caught Exception: {}",e); + errorMsg = "Validation failed due to : " + e.getMessage(); proceed = false; resultData.setValidationResultData(new ValidationResultData(false, errorMsg)); break; } + } else { proceed = false; errorMsg = errorMsg.concat(String.format("Experiment name : %s not found", resultData.getExperiment_name())); @@ -91,6 +123,18 @@ public void validate(List experimentResultDataList) { } } + private String getValidationClass(String name) { + String[] words = name.split("-"); + StringBuilder output = new StringBuilder(); + for (String word : words) { + output.append(word.substring(0,1).toUpperCase() + word.substring(1)); + } + output.append("Impl"); + LOGGER.debug("ClassName = {}",output); + + return output.toString(); + } + public void markFailed(String message) { setSuccess(false); setErrorMessage(message); diff --git a/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/DefaultOpenshiftImpl.java b/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/DefaultOpenshiftImpl.java new file mode 100644 index 000000000..1642918de --- /dev/null +++ b/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/DefaultOpenshiftImpl.java @@ -0,0 +1,183 @@ +/******************************************************************************* + * Copyright (c) 2022 Red Hat, IBM Corporation and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ +package com.autotune.common.performanceProfiles.PerformanceProfileInterface; + +import com.autotune.common.data.result.AggregationInfoResult; +import com.autotune.common.data.result.ContainerResultData; +import com.autotune.common.data.result.DeploymentResultData; +import com.autotune.common.data.result.ExperimentResultData; +import com.autotune.common.k8sObjects.Metric; +import com.autotune.common.performanceProfiles.PerformanceProfile; +import com.autotune.utils.AnalyzerErrorConstants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.*; + +/** + * Util class to validate the performance profile metrics with the experiment results metrics. + */ +public class DefaultOpenshiftImpl implements PerfProfileInterface { + + private static final Logger LOGGER = LoggerFactory.getLogger(DefaultOpenshiftImpl.class); + List perfProfileFunctionVariablesList = new ArrayList<>(); + + @Override + public String validate(PerformanceProfile performanceProfile, ExperimentResultData experimentResultData) { + + String errorMsg = ""; + // Get the metrics data from the Performance Profile + List aggrFunctionsObjects = new ArrayList<>(); + for (Metric metric:performanceProfile.getSloInfo().getFunctionVariables()) { + perfProfileFunctionVariablesList.add(metric.getName()); + metric.getAggregationFunctions().forEach(aggregationFunctions -> + aggrFunctionsObjects.add(aggregationFunctions.getFunction())); + } + LOGGER.debug(String.format("List of functionVariables: %s", perfProfileFunctionVariablesList)); + LOGGER.debug(String.format("List of agg func objects: %s", aggrFunctionsObjects)); + + // Get the metrics data from the Kruize Object + for (DeploymentResultData deploymentResultData : experimentResultData.getDeployments()) { + for (ContainerResultData containerResultData : deploymentResultData.getContainers()) { + HashMap>> containerMetricsMap = + containerResultData.getContainer_metrics(); + List kruizeFunctionVariablesList = containerMetricsMap.keySet().stream().toList(); + if (!(perfProfileFunctionVariablesList.size() == kruizeFunctionVariablesList.size() && + new HashSet<>(perfProfileFunctionVariablesList).containsAll(kruizeFunctionVariablesList) && + new HashSet<>(kruizeFunctionVariablesList).containsAll(perfProfileFunctionVariablesList))) { + LOGGER.debug("perfProfileFunctionVariablesList: {}",perfProfileFunctionVariablesList); + LOGGER.debug("kruizeFunctionVariablesList: {}",kruizeFunctionVariablesList); + perfProfileFunctionVariablesList.removeAll(kruizeFunctionVariablesList); + errorMsg = errorMsg.concat(String.format("Following Performance Profile parameters are missing for experiment - %s : %s", experimentResultData.getExperiment_name(), perfProfileFunctionVariablesList)); + break; + } else { + for(HashMap> funcVar:containerMetricsMap.values()){ + for(HashMap genInfo:funcVar.values()){ + Map genInfoClassAsMap; + for(AggregationInfoResult genInfoObj:genInfo.values()){ + try { + genInfoClassAsMap = DefaultOpenshiftImpl.convertObjectToMap(genInfoObj); + errorMsg = validateAggFunction(genInfoClassAsMap.keySet(), aggrFunctionsObjects); + if (!errorMsg.isBlank()) { + errorMsg = errorMsg.concat(String.format("for the experiment : %s" + ,experimentResultData.getExperiment_name())); + return errorMsg; + } + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } + } + } + } + } + } + } + + return errorMsg; + } + + /** + * Calculates the objective function by calling the algebraic parser library. The result is then sent to HPO. + * @param performanceProfile + * @param experimentResultData + * @return + */ + @Override + public String recommend(PerformanceProfile performanceProfile, ExperimentResultData experimentResultData) { + + String objectiveFunction = performanceProfile.getSloInfo().getObjectiveFunction().getExpression(); + Map objFunctionMap = new HashMap<>(); + String errorMsg = ""; + + // Get the metrics data from the Kruize Object + for (DeploymentResultData deploymentResultData : experimentResultData.getDeployments()) { + for (ContainerResultData containers : deploymentResultData.getContainers()) { + HashMap>> containerMetricsMap = + containers.getContainer_metrics(); + List kruizeFunctionVariablesList = containerMetricsMap.keySet().stream().toList(); + for(HashMap> funcVar:containerMetricsMap.values()){ + for(HashMap aggregationInfoResultMap:funcVar.values()){ + Map aggrInfoClassAsMap; + for(AggregationInfoResult aggregationInfoResult:aggregationInfoResultMap.values()){ + try { + // TODO: Need to update the below code + aggrInfoClassAsMap = DefaultOpenshiftImpl.convertObjectToMap(aggregationInfoResult); + LOGGER.info("aggrInfoClassAsMap: {}", aggrInfoClassAsMap); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } + } + } + } + + } + } + + return objectiveFunction; + } + /** + * Validates the aggregation function objects against the generalInfo metrics + * @param keySet + * @param aggrFunctionsObjects + * @return + */ + private String validateAggFunction(Set keySet, List aggrFunctionsObjects) { + + List genInfoObjects = keySet.stream().toList(); + List missingAggFunction = new ArrayList<>(); + String errorMsg = ""; + // check if none of the aggrfunctions are present in the genInfo List + if (genInfoObjects.stream().noneMatch(aggrFunctionsObjects::contains)) { + LOGGER.error(AnalyzerErrorConstants.AutotuneObjectErrors.MISSING_AGG_FUNCTION); + errorMsg = errorMsg.concat(AnalyzerErrorConstants.AutotuneObjectErrors.MISSING_AGG_FUNCTION); + } else { + // check if some or all the values are present or not and respond accordingly + for (String aggFuncObj : aggrFunctionsObjects) { + if (!genInfoObjects.contains(aggFuncObj)) { + missingAggFunction.add(aggFuncObj); + } + } + if (!missingAggFunction.isEmpty()) { + LOGGER.warn("Missing Aggregation Functions: {}", missingAggFunction); + } + } + return errorMsg; + } + + /** + * Converts the generalInfo class into Map to extract values for validation + * @param obj + * @return + * @throws IllegalAccessException + * @throws IllegalArgumentException + * @throws InvocationTargetException + */ + public static Map convertObjectToMap(Object obj) throws IllegalAccessException, + IllegalArgumentException, InvocationTargetException { + Method[] methods = obj.getClass().getMethods(); + Map map = new HashMap<>(); + for (Method m : methods) { + if (m.getName().startsWith("get") && !m.getName().startsWith("getClass")) { + Object value = m.invoke(obj); + if (value instanceof Double) + map.put(m.getName().substring(3).toLowerCase(), value); + } + } + return map; + } +} diff --git a/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/PerfProfileInterface.java b/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/PerfProfileInterface.java index 50dacd83d..de3383084 100644 --- a/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/PerfProfileInterface.java +++ b/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/PerfProfileInterface.java @@ -17,7 +17,7 @@ package com.autotune.common.performanceProfiles.PerformanceProfileInterface; import com.autotune.common.data.result.ExperimentResultData; -import com.autotune.common.k8sObjects.KruizeObject; +import com.autotune.common.performanceProfiles.PerformanceProfile; /** * Abstraction layer containing validate and recommend methods for the @@ -25,7 +25,7 @@ */ public interface PerfProfileInterface { - String validate(KruizeObject kruizeObject, ExperimentResultData experimentResultData); + String validate(PerformanceProfile performanceProfile, ExperimentResultData experimentResultData); - void recommend(); + String recommend(PerformanceProfile performanceProfile, ExperimentResultData experimentResultData); } diff --git a/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/RemoteMonitoringOpenShiftImpl.java b/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/ResourceOptimizationOpenshiftImpl.java similarity index 92% rename from src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/RemoteMonitoringOpenShiftImpl.java rename to src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/ResourceOptimizationOpenshiftImpl.java index acaf595e2..29761d716 100644 --- a/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/RemoteMonitoringOpenShiftImpl.java +++ b/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/ResourceOptimizationOpenshiftImpl.java @@ -21,7 +21,6 @@ import com.autotune.common.data.result.AggregationInfoResult; import com.autotune.common.k8sObjects.*; import com.autotune.common.performanceProfiles.PerformanceProfile; -import com.autotune.common.performanceProfiles.PerformanceProfilesDeployment; import com.autotune.utils.AnalyzerErrorConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,17 +32,15 @@ /** * Util class to validate the performance profile metrics with the experiment results metrics. */ -public class RemoteMonitoringOpenShiftImpl implements PerfProfileInterface { +public class ResourceOptimizationOpenshiftImpl implements PerfProfileInterface { - private static final Logger LOGGER = LoggerFactory.getLogger(RemoteMonitoringOpenShiftImpl.class); + private static final Logger LOGGER = LoggerFactory.getLogger(ResourceOptimizationOpenshiftImpl.class); @Override - public String validate(KruizeObject kruizeObject, ExperimentResultData experimentResultData) { + public String validate(PerformanceProfile performanceProfile, ExperimentResultData experimentResultData) { String errorMsg = ""; // Get the metrics data from the Performance Profile - PerformanceProfile performanceProfile = PerformanceProfilesDeployment.performanceProfilesMap - .get(kruizeObject.getPerformanceProfile()); List aggrFunctionsObjects = new ArrayList<>(); List perfProfileFunctionVariablesList = new ArrayList<>(); for (Metric metric:performanceProfile.getSloInfo().getFunctionVariables()) { @@ -74,7 +71,7 @@ public String validate(KruizeObject kruizeObject, ExperimentResultData experimen Map genInfoClassAsMap; for(AggregationInfoResult genInfoObj:genInfo.values()){ try { - genInfoClassAsMap = RemoteMonitoringOpenShiftImpl.convertObjectToMap(genInfoObj); + genInfoClassAsMap = ResourceOptimizationOpenshiftImpl.convertObjectToMap(genInfoObj); errorMsg = validateAggFunction(genInfoClassAsMap.keySet(), aggrFunctionsObjects); if (!errorMsg.isBlank()) { errorMsg = errorMsg.concat(String.format("for the experiment : %s" @@ -123,10 +120,11 @@ private String validateAggFunction(Set keySet, List aggrFunction return errorMsg; } @Override - public void recommend() { + public String recommend(PerformanceProfile performanceProfile, ExperimentResultData experimentResultData) { //TODO: Will be updated once algo is completed + return null; } /** diff --git a/src/main/java/com/autotune/utils/AnalyzerConstants.java b/src/main/java/com/autotune/utils/AnalyzerConstants.java index de58c8300..4d739a870 100644 --- a/src/main/java/com/autotune/utils/AnalyzerConstants.java +++ b/src/main/java/com/autotune/utils/AnalyzerConstants.java @@ -365,5 +365,6 @@ public static final class PerformanceProfileConstants { public static final String FUNCTION_VARIABLES = "functionVariables"; public static final String VALUE_TYPE = "valueType"; public static final String SOURCE = "source"; + public static final String PERFORMANCE_PROFILE_PKG = "com.autotune.common.performanceProfiles.PerformanceProfileInterface."; } } diff --git a/src/test/java/com/autotune/analyzer/utils/TestEvalExParser.java b/src/test/java/com/autotune/analyzer/utils/TestEvalExParser.java index a63c6bf2f..9d36606c6 100644 --- a/src/test/java/com/autotune/analyzer/utils/TestEvalExParser.java +++ b/src/test/java/com/autotune/analyzer/utils/TestEvalExParser.java @@ -102,7 +102,7 @@ public void testParse() { for(String objFunction : objFunctionsList) { Map objFunctionMap = new HashMap<>(); - assertEquals(AnalyzerErrorConstants.AutotuneObjectErrors.OBJECTIVE_FUNCTION_MAP_MISSING, new EvalExParser().parse(objFunction, "String", objFunctionMap)); + assertEquals(AnalyzerErrorConstants.AutotuneObjectErrors.OBJECTIVE_FUNCTION_MAP_MISSING, new EvalExParser().parse(objFunction, objFunctionMap)); } } From 20d327e7ed58fc47fe73a87ce54a33242f59dfde Mon Sep 17 00:00:00 2001 From: msvinaykumar Date: Thu, 9 Feb 2023 13:52:20 +0530 Subject: [PATCH 18/35] MD files updated about timestamp format Signed-off-by: msvinaykumar --- design/ExperimentModeAPI.md | 38 +++++++++++++++++++++++++++++-------- design/MonitoringModeAPI.md | 3 +++ 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/design/ExperimentModeAPI.md b/design/ExperimentModeAPI.md index 7d03455df..8b0a62765 100644 --- a/design/ExperimentModeAPI.md +++ b/design/ExperimentModeAPI.md @@ -1,7 +1,13 @@ # Experiment Monitoring Mode -This article describes how to quickly get started with the Local cluster Experimental Mode use case REST API using curl command. -## listStacks +This article describes how to quickly get started with the Local cluster Experimental Mode use case REST API using curl +command. + +**Note :** The ISO 8601 standard underpins all timestamp formats. An example of a valid timestamp in this format is +2022-01-23T18:25:43.511Z, which represents January 23, 2022, at 18:25:43.511 UTC. + +## listStacks + Get the list of application stacks monitored by autotune. **Request** @@ -12,6 +18,7 @@ Get the list of application stacks monitored by autotune. `curl -H 'Accept: application/json' http://:/listStacks?experiment_name=` **Response** + ``` [ { @@ -42,6 +49,7 @@ Get the list of application stacks monitored by autotune. ``` ## listStackLayers + Returns the list of application stacks monitored by autotune along with layers detected in the stacks. **Request** @@ -52,6 +60,7 @@ Returns the list of application stacks monitored by autotune along with layers d `curl -H 'Accept: application/json' http://:/listStackLayers?experiment_name=` **Response** + ``` [ { @@ -113,18 +122,22 @@ Returns the list of application stacks monitored by autotune along with layers d ] ``` -## listStackTunables +## listStackTunables + Returns the list of application stacks monitored by autotune along with their tunables. **Request** `GET /listStackTunables` gives the tunables and layer information for all the application stacks monitored by autotune. -`GET /listStackTunables?experiment_name=` for getting the tunables information of a specific application. +`GET /listStackTunables?experiment_name=` for getting the tunables information of a specific +application. -`GET /listStackTunables?experiment_name=&layer_name=` for getting tunables of a specific layer for the application. +`GET /listStackTunables?experiment_name=&layer_name=` for getting tunables of a specific layer +for the application. `curl -H 'Accept: application/json' http://:/listStackTunables?experiment_name=` **Response** + ``` [ { @@ -174,7 +187,9 @@ Returns the list of application stacks monitored by autotune along with their tu } ] ``` -## listAutotuneTunables + +## listAutotuneTunables + Get the tunables supported by autotune for the SLO. **Request** @@ -187,6 +202,7 @@ Get the tunables supported by autotune for the SLO. `curl -H 'Accept: application/json' http://:/listAutotuneTunables?slo_class=` **Response** + ``` [ { @@ -261,7 +277,9 @@ Get the tunables supported by autotune for the SLO. } ] ``` -## SearchSpace + +## SearchSpace + Generates the search space used for the analysis. **Request** @@ -331,13 +349,16 @@ Generates the search space used for the analysis. } ] ``` -## CreateExperimentTrial + +## CreateExperimentTrial + Create experiment trials using input JSON provided by Analyser module. **Request** `POST /createExperimentTrial` `curl -H 'Accept: application/json' -X POST --data 'copy paste below JSON' http://:/createExperimentTrial` + ``` [{ "settings": { @@ -458,6 +479,7 @@ Create experiment trials using input JSON provided by Analyser module. ``` **Response** + ``` { "message": "Experiment trial registered successfully with Autotune. View registered experiments at /listExperiments", diff --git a/design/MonitoringModeAPI.md b/design/MonitoringModeAPI.md index e0dfab020..cfbf24dcc 100644 --- a/design/MonitoringModeAPI.md +++ b/design/MonitoringModeAPI.md @@ -3,6 +3,9 @@ This article describes how to quickly get started with the Remote Monitoring Mode use case REST API using curl command. Documentation still in progress stay tuned. +**Note :** The ISO 8601 standard underpins all timestamp formats. An example of a valid timestamp in this format is +2022-01-23T18:25:43.511Z, which represents January 23, 2022, at 18:25:43.511 UTC. + ## CreateExperiment This is quick guide instructions to create experiments using input JSON as follows. For a more detailed guide, From 99d8be907a3058f41e532fe034250e8adb4bc674 Mon Sep 17 00:00:00 2001 From: saakhan Date: Thu, 9 Feb 2023 16:39:41 +0530 Subject: [PATCH 19/35] Remove default profile yaml and add code to attach default profile Signed-off-by: saakhan --- .../default_openshift.yaml | 38 --------------- .../utils/ExperimentResultValidation.java | 48 ++++++++++++------- .../com/autotune/utils/AnalyzerConstants.java | 1 + 3 files changed, 32 insertions(+), 55 deletions(-) delete mode 100644 manifests/autotune/performance-profiles/default_openshift.yaml diff --git a/manifests/autotune/performance-profiles/default_openshift.yaml b/manifests/autotune/performance-profiles/default_openshift.yaml deleted file mode 100644 index 5230382d8..000000000 --- a/manifests/autotune/performance-profiles/default_openshift.yaml +++ /dev/null @@ -1,38 +0,0 @@ -apiVersion: "recommender.com/v1" -kind: "KruizePerformanceProfile" -metadata: - name: "default-openshift" -profile_version: 1.0 -k8s_type: openshift - -slo: - slo_class: "resource_usage" - direction: "minimize" - - # Refer to src/.../performanceProfiles/PerformanceProfileInterface/DefaultImpl.java - objective_function: - function_type: expression - expression: "(80*min(memoryRSS))/(85*avg(cpuRequest))" - - function_variables: - # CPU Request - # Show cpu requests in cores for a container in a deployment - - name: cpuRequest - datasource: prometheus - value_type: "double" - kubernetes_object: "container" - - aggregation_functions: - - function: 'avg' - query: 'avg(kube_pod_container_resource_requests{pod=~"$DEPLOYMENT_NAME$-[^-]*-[^-]*$", container="$CONTAINER_NAME$", namespace="$NAMESPACE", resource="cpu", unit="core"})' - - # 2.4 Memory RSS - - name: memoryRSS - datasource: prometheus - value_type: "double" - kubernetes_object: "container" - - aggregation_functions: - # Approx minimum memory RSS per container in a deployment - - function: min - query: 'min(min_over_time(container_memory_rss{pod=~"$DEPLOYMENT_NAME$-[^-]*-[^-]*$", namespace=$NAMESPACE$, container="$CONTAINER_NAME$"}[15m]))' diff --git a/src/main/java/com/autotune/analyzer/utils/ExperimentResultValidation.java b/src/main/java/com/autotune/analyzer/utils/ExperimentResultValidation.java index fd18620c7..e196b30b9 100644 --- a/src/main/java/com/autotune/analyzer/utils/ExperimentResultValidation.java +++ b/src/main/java/com/autotune/analyzer/utils/ExperimentResultValidation.java @@ -15,13 +15,14 @@ *******************************************************************************/ package com.autotune.analyzer.utils; +import com.autotune.analyzer.exceptions.InvalidValueException; import com.autotune.common.data.ValidationResultData; import com.autotune.common.data.result.ExperimentResultData; import com.autotune.common.k8sObjects.KruizeObject; +import com.autotune.common.k8sObjects.SloInfo; import com.autotune.common.performanceProfiles.PerformanceProfile; -import com.autotune.common.performanceProfiles.PerformanceProfileInterface.ResourceOptimizationOpenshiftImpl; +import com.autotune.common.performanceProfiles.PerformanceProfileInterface.DefaultOpenshiftImpl; import com.autotune.utils.AnalyzerConstants; -import com.autotune.utils.AnalyzerErrorConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,21 +69,34 @@ public void validate(List experimentResultDataList, Map validationClass = Class.forName(validationClassName); - - Object object = validationClass.getDeclaredConstructor().newInstance(); - Class[] parameterTypes = new Class[] { PerformanceProfile.class, ExperimentResultData.class }; - Method method = validationClass.getMethod("validate",parameterTypes); - - errorMsg = (String) method.invoke(object, performanceProfile, resultData); - - if (errorMsg.isEmpty() || errorMsg.isBlank()) { - // call recommend() method here - method = validationClass.getMethod("recommend",parameterTypes); + Class validationClass = null; + Class[] parameterTypes = null; + Method method = null; + Object object = null; + // check if performance profile is absent and then attach a default one + if (null == performanceProfile) { + SloInfo sloInfo = kruizeObject.getSloInfo(); + performanceProfile = new PerformanceProfile(AnalyzerConstants.PerformanceProfileConstants.DEFAULT_PROFILE,AnalyzerConstants.DEFAULT_PROFILE_VERSION, AnalyzerConstants.DEFAULT_K8S_TYPE, sloInfo); + kruizeObject.setPerformanceProfile(performanceProfile.getName()); + errorMsg = new DefaultOpenshiftImpl().validate(performanceProfile, resultData); + } else { + // Get the corresponding class for validating the performance profile + String validationClassName = AnalyzerConstants.PerformanceProfileConstants.PERFORMANCE_PROFILE_PKG + .concat(getValidationClass(performanceProfile.getName())); + validationClass = Class.forName(validationClassName); + + object = validationClass.getDeclaredConstructor().newInstance(); + parameterTypes = new Class[] { PerformanceProfile.class, ExperimentResultData.class }; + method = validationClass.getMethod("validate",parameterTypes); errorMsg = (String) method.invoke(object, performanceProfile, resultData); + } + if (errorMsg.isEmpty()) { + // call recommend() method here + if ( null != method) { + method = validationClass.getMethod("recommend",parameterTypes); + errorMsg = (String) method.invoke(object, performanceProfile, resultData); + } else + errorMsg = new DefaultOpenshiftImpl().recommend(performanceProfile, resultData); proceed = true; } else { @@ -91,7 +105,7 @@ public void validate(List experimentResultDataList, Map Date: Fri, 10 Feb 2023 15:40:12 +0530 Subject: [PATCH 20/35] add API to list performance profiles Signed-off-by: saakhan --- .../java/com/autotune/analyzer/Analyzer.java | 1 + .../services/PerformanceProfileService.java | 24 ++++++++++++++++--- .../utils/AnalyzerErrorConstants.java | 1 + .../com/autotune/utils/ServerContext.java | 2 ++ 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/autotune/analyzer/Analyzer.java b/src/main/java/com/autotune/analyzer/Analyzer.java index 524ba073c..2551f61b4 100644 --- a/src/main/java/com/autotune/analyzer/Analyzer.java +++ b/src/main/java/com/autotune/analyzer/Analyzer.java @@ -59,6 +59,7 @@ public static void addServlets(ServletContextHandler context) { context.addServlet(UpdateResults.class, ServerContext.UPDATE_RESULTS); context.addServlet(Recommendation.class, ServerContext.RECOMMEND_RESULTS); context.addServlet(PerformanceProfileService.class, ServerContext.CREATE_PERF_PROFILE); + context.addServlet(PerformanceProfileService.class, ServerContext.LIST_PERF_PROFILE); // Adding UI support API's context.addServlet(ListNamespaces.class, ServerContext.LIST_NAMESPACES); diff --git a/src/main/java/com/autotune/analyzer/services/PerformanceProfileService.java b/src/main/java/com/autotune/analyzer/services/PerformanceProfileService.java index 810ee56b0..7a724cf01 100644 --- a/src/main/java/com/autotune/analyzer/services/PerformanceProfileService.java +++ b/src/main/java/com/autotune/analyzer/services/PerformanceProfileService.java @@ -17,11 +17,15 @@ package com.autotune.analyzer.services; import com.autotune.analyzer.exceptions.PerformanceProfileResponse; +import com.autotune.analyzer.utils.GsonUTCDateAdapter; import com.autotune.analyzer.utils.PerformanceProfileValidation; +import com.autotune.common.annotations.json.AutotuneJSONExclusionStrategy; import com.autotune.common.data.ValidationResultData; import com.autotune.common.performanceProfiles.PerformanceProfile; import com.autotune.utils.AnalyzerConstants; +import com.autotune.utils.AnalyzerErrorConstants; import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,6 +38,7 @@ import java.io.IOException; import java.io.PrintWriter; import java.io.Serial; +import java.util.Date; import java.util.HashMap; import java.util.Map; import java.util.stream.Collectors; @@ -85,7 +90,7 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) } } - /**TODO: Need to implement + /** * Get List of Performance Profiles * @param req * @param response @@ -97,8 +102,21 @@ protected void doGet(HttpServletRequest req, HttpServletResponse response) throw response.setContentType(JSON_CONTENT_TYPE); response.setCharacterEncoding(CHARACTER_ENCODING); response.setStatus(HttpServletResponse.SC_OK); - // TODO: Will be updated later - + String gsonStr = "[]"; + if (this.performanceProfilesMap.size() > 0) { + Gson gsonObj = new GsonBuilder() + .disableHtmlEscaping() + .setPrettyPrinting() + .enableComplexMapKeySerialization() + .registerTypeAdapter(Date.class, new GsonUTCDateAdapter()) + .setExclusionStrategies(new AutotuneJSONExclusionStrategy()) + .create(); + gsonStr = gsonObj.toJson(this.performanceProfilesMap); + } else { + gsonStr = AnalyzerErrorConstants.AutotuneObjectErrors.NO_PERF_PROFILE; + } + response.getWriter().println(gsonStr); + response.getWriter().close(); } /**TODO: Need to implement diff --git a/src/main/java/com/autotune/utils/AnalyzerErrorConstants.java b/src/main/java/com/autotune/utils/AnalyzerErrorConstants.java index be2c40d39..75410b19c 100644 --- a/src/main/java/com/autotune/utils/AnalyzerErrorConstants.java +++ b/src/main/java/com/autotune/utils/AnalyzerErrorConstants.java @@ -66,6 +66,7 @@ private AutotuneObjectErrors() { public static final String MISSING_EXPRESSION = "Expression value is missing or null!\n"; public static final String MISPLACED_EXPRESSION = "Expression is not allowed when the type is source\n"; public static final String INVALID_TYPE = "Objective function type can only be either expression or source\n"; + public static final String NO_PERF_PROFILE = "No performance profiles present!"; } diff --git a/src/main/java/com/autotune/utils/ServerContext.java b/src/main/java/com/autotune/utils/ServerContext.java index 2a75babba..f0146d600 100644 --- a/src/main/java/com/autotune/utils/ServerContext.java +++ b/src/main/java/com/autotune/utils/ServerContext.java @@ -37,6 +37,8 @@ public class ServerContext { public static final String UPDATE_RESULTS = ROOT_CONTEXT + "updateResults"; public static final String RECOMMEND_RESULTS = ROOT_CONTEXT + "listRecommendations"; public static final String CREATE_PERF_PROFILE = ROOT_CONTEXT + "createPerformanceProfile"; + public static final String LIST_PERF_PROFILE = ROOT_CONTEXT + "listPerformanceProfile"; + public static final String AUTOTUNE_SERVER_PORT = "http://localhost:" + AUTOTUNE_PORT; From 2eb3f08b0e48887fc0c5a152223e6a4b158577af Mon Sep 17 00:00:00 2001 From: Suraj Patil Date: Fri, 10 Feb 2023 17:43:46 +0530 Subject: [PATCH 21/35] Set AUTOTUNE_PORT based on environment variable --- src/main/java/com/autotune/utils/ServerContext.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/autotune/utils/ServerContext.java b/src/main/java/com/autotune/utils/ServerContext.java index 2a75babba..465d8b14e 100644 --- a/src/main/java/com/autotune/utils/ServerContext.java +++ b/src/main/java/com/autotune/utils/ServerContext.java @@ -19,7 +19,7 @@ * Holds the server context of the dependency analyzer. */ public class ServerContext { - public static final int AUTOTUNE_PORT = 8080; + public static final int AUTOTUNE_PORT = Integer.parseInt(System.getenv().getOrDefault("AUTOTUNE_PORT", "8080")); public static final int HPO_PORT = 8085; // AnalyzerConstants end points From 1a043b4313d4f25e64dfefd77e4e119f81af0557 Mon Sep 17 00:00:00 2001 From: saakhan Date: Fri, 10 Feb 2023 18:33:12 +0530 Subject: [PATCH 22/35] update design docs and bug fixes Signed-off-by: saakhan --- design/CreatePerformanceProfile.md | 58 +- design/PerformanceProfileAPI.md | 247 --------- design/PerformanceProfiles.md | 501 ++++++++++++++++++ .../com/autotune/utils/ServerContext.java | 2 +- 4 files changed, 537 insertions(+), 271 deletions(-) delete mode 100644 design/PerformanceProfileAPI.md create mode 100644 design/PerformanceProfiles.md diff --git a/design/CreatePerformanceProfile.md b/design/CreatePerformanceProfile.md index d72981f1e..cdfcce3c0 100644 --- a/design/CreatePerformanceProfile.md +++ b/design/CreatePerformanceProfile.md @@ -5,30 +5,42 @@ tuned. # Attributes -* name \ +- **name** \ A unique string name for identifying each performance profile. -* profile_version \ +- **profile_version** \ a double value specifying the current version of the profile. -* slo \ - Service Level Objective containing the direction, objective_function and function_variables - * direction \ - to be updated - * objective_function \ - to be updated - * function_variables \ - to be updated - * name \ - to be updated - * value_type \ - to be updated - * query \ - to be updated - * aggregation_functions \ - to be updated - * function \ - to be updated - * query \ - to be updated +- **slo** \ + Service Level Objective containing the _direction_, _objective_function_ and _function_variables_ + - **slo_class** \ + a standard slo "bucket" defined by Kruize. Can be "_resource_usage_", "_throughput_" or "_response_time_" + - **direction** \ + based on the slo_class, it can be '_maximize_' or '_minimize_' + - **objective_function** \ + Define the performance objective here. + - **function_type** \ + can be specified as '_source_' (a java file) or as an '_expression_'(algebraic). If it's an expression, it needs to defined below. + - **expression** \ + an algebraic expression that details the calculation using function variables. Only valid if the "_function_type_" is "expression" + - **function_variables** \ + Define the variables used in the _objective_function_ + - **name** \ + name of the variable + - **datasource** \ + datasource of the query + - **value_type** \ + can be double or integer + - **query** \ + one of the query or _aggregation_functions_ is mandatory. Both can be present. + - **kubernetes_object** \ + k8s object that this query is tied to: "_deployment_", "_pod_" or "_container_" + - **aggregation_functions** \ + aggregate functions associated with this variable + - **function** \ + can be '_avg_', '_sum_', '_min_', '_max_' + - **query** \ + corresponding query + - **versions** \ + Any specific versions that this query is tied to @@ -63,4 +75,4 @@ tuned. "documentationLink": "", "status": "FAILED" } - ``` \ No newline at end of file + ``` diff --git a/design/PerformanceProfileAPI.md b/design/PerformanceProfileAPI.md deleted file mode 100644 index 91c5ae5db..000000000 --- a/design/PerformanceProfileAPI.md +++ /dev/null @@ -1,247 +0,0 @@ -# Performance Profile - -This article describes how to add/update Performance Profiles with REST APIs using curl command. -Documentation still in progress stay tuned. - -## CreatePerformanceProfile - -This is quick guide instructions to create performance profile using input JSON as follows. For a more detailed guide, -see [Create Performance Profile](/design/CreatePerformanceProfile.md) - -**Request** -`POST /createPerformanceProfile` - -`curl -H 'Accept: application/json' -X POST --data 'copy paste below JSON' http://:/createPerformanceProfile` - -``` -{ - "name": "resource-optimization-openshift", - "profile_version": 1, - "slo": { - "direction": "minimize", - "objective_function": { - "function_type": "expression", - "expression": "request_sum/request_count" - }, - "function_variables": [ - { - "name": "cpuRequest", - "datasource": "prometheus", - "value_type": "double", - "query": "kube_pod_container_resource_requests{pod=~'$DEPLOYMENT_NAME$-[^-]*-[^-]*$', container='$CONTAINER_NAME$', namespace='$NAMESPACE', resource='cpu', unit='core'}", - "aggregation_functions": [ - { - "function": "avg", - "query": "avg(kube_pod_container_resource_requests{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=\"$CONTAINER_NAME$\", namespace=\"$NAMESPACE\", resource=\"cpu\", unit=\"core\"})" - } - ] - } - ] - } -} -``` - -**Response** - -``` -{ - "message": "Performance Profile : created successfully. View all performance profiles at /listPerformanceProfiles", - "httpcode": 201, - "documentationLink": "", - "status": "SUCCESS" -} -``` -#### Note: One of query or aggregation_functions is mandatory. Both can be present together. - -## Update Performance Profile - -Update Performance Profile data using input JSON as follows. - -**Request** -`POST /updatePerformanceProfiles` - -`curl -H 'Accept: application/json' -X POST --data 'copy paste below JSON' http://:/updatePerformanceProfiles` - -``` -{ - "name": "resource-optimization-openshift", - "profile_version": 1, - "k8s_type": "openshift", - "slo": { - "slo_class": "resource_usage", - "direction": "minimize", - "objective_function": { - "function_type": "expression", - "expression": "request_sum/request_count" - }, - "function_variables": [ - { - "name": "cpuRequest", - "datasource": "prometheus", - "value_type": "double", - "kubernetes_object": "container", - "aggregation_functions": [ - { - "function": "avg", - "query": "avg(kube_pod_container_resource_requests{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=\"$CONTAINER_NAME$\", namespace=\"$NAMESPACE\", resource=\"cpu\", unit=\"core\"})" - }, - { - "function": "sum", - "query": "sum(kube_pod_container_resource_requests{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=\"$CONTAINER_NAME$\", namespace=\"$NAMESPACE\", resource=\"cpu\", unit=\"core\"})" - } - ] - }, - { - "name": "cpuLimit", - "datasource": "prometheus", - "value_type": "double", - "kubernetes_object": "container", - "aggregation_functions": [ - { - "function": "avg", - "query": "avg(kube_pod_container_resource_limits{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=\"$CONTAINER_NAME$\", namespace=\"$NAMESPACE\", resource=\"cpu\", unit=\"core\"})" - }, - { - "function": "sum", - "query": "sum(kube_pod_container_resource_limits{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=\"$CONTAINER_NAME$\", namespace=\"$NAMESPACE$\", resource=\"cpu\", unit=\"core\"})" - } - ] - }, - { - "name": "cpuUsage", - "datasource": "prometheus", - "value_type": "double", - "kubernetes_object": "container", - "aggregation_functions": [ - { - "function": "avg", - "query": "avg(avg_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=”$CONTAINER_NAME$”}[15m]))", - "versions": "<=4.8" - }, - { - "function": "avg", - "query": "avg(avg_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=”$CONTAINER_NAME$”}[15m]))", - "versions": ">4.9" - }, - { - "function": "min", - "query": "min(min_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=\"$CONTAINER_NAME$\"}[15m]))", - "versions": "<=4.8" - }, - { - "function": "min", - "query": "min(min_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=\"$CONTAINER_NAME$\"}[15m]))", - "versions": ">4.9" - }, - { - "function": "max", - "query": "max(max_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=\"$CONTAINER_NAME$\"}[15m]))", - "versions": "<=4.8" - }, - { - "function": "max", - "query": "max(max_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=\"$CONTAINER_NAME$\"}[15m]))", - "versions": ">4.9" - } - ] - }, - { - "name": "cpuThrottle", - "datasource": "prometheus", - "value_type": "double", - "kubernetes_object": "container", - "aggregation_functions": [ - { - "function": "avg", - "query": "rate(container_cpu_cfs_throttled_seconds_total{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=”$CONTAINER_NAME$”}[15m])" - } - ] - }, - { - "name": "memoryRequest", - "datasource": "prometheus", - "value_type": "double", - "kubernetes_object": "container", - "aggregation_functions": [ - { - "function": "avg", - "query": "avg(kube_pod_container_resource_requests{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=”$CONTAINER_NAME$”, namespace=”$NAMESPACE”, resource=\"memory\", unit=\"byte\"})" - }, - { - "function": "sum", - "query": "sum(kube_pod_container_resource_requests{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=”$CONTAINER_NAME$”, namespace=”$NAMESPACE”, resource=\"memory\", unit=\"byte\"})" - } - ] - }, - { - "name": "memoryLimit", - "datasource": "prometheus", - "value_type": "double", - "kubernetes_object": "container", - "aggregation_functions": [ - { - "function": "avg", - "query": "avg(kube_pod_container_resource_limits{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=\"$CONTAINER_NAME$\", namespace=\"$NAMESPACE\", resource=\"memory\", unit=\"byte\"})" - }, - { - "function": "sum", - "query": "sum(kube_pod_container_resource_limits{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=”$CONTAINER_NAME$”, namespace=”$NAMESPACE”, resource=\"memory\", unit=\"byte\"})" - } - ] - }, - { - "name": "memoryUsage", - "datasource": "prometheus", - "value_type": "double", - "kubernetes_object": "container", - "aggregation_functions": [ - { - "function": "avg", - "query": "avg(avg_over_time(container_memory_working_set_bytes{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=$NAMESPACE$, container=”$CONTAINER_NAME$”}[15m]))" - }, - { - "function": "min", - "query": "min(min_over_time(container_memory_working_set_bytes{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=$NAMESPACE$, container=\"$CONTAINER_NAME$\"}[15m]))" - }, - { - "function": "max", - "query": "max(max_over_time(container_memory_working_set_bytes{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=$NAMESPACE$, container=\"$CONTAINER_NAME$\"}[15m]))" - } - ] - }, - { - "name": "memoryRSS", - "datasource": "prometheus", - "value_type": "double", - "kubernetes_object": "container", - "aggregation_functions": [ - { - "function": "avg", - "query": "avg(avg_over_time(container_memory_rss{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=$NAMESPACE$, container=”$CONTAINER_NAME$”}[15m]))" - }, - { - "function": "min", - "query": "min(min_over_time(container_memory_rss{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=$NAMESPACE$, container=\"$CONTAINER_NAME$\"}[15m]))" - }, - { - "function": "max", - "query": "max(max_over_time(container_memory_rss{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=$NAMESPACE$, container=\"$CONTAINER_NAME$\"}[15m]))" - } - ] - } - ] - } -} - -``` - -**Response** - -``` -{ - "message": "Updated performance profile successfully. View updated profile at /listPerformanceProfiles", - "httpcode": 201, - "documentationLink": "", - "status": "SUCCESS" -} -``` diff --git a/design/PerformanceProfiles.md b/design/PerformanceProfiles.md new file mode 100644 index 000000000..1334750b1 --- /dev/null +++ b/design/PerformanceProfiles.md @@ -0,0 +1,501 @@ +# Performance Profile + +This article describes how to add/update Performance Profiles with REST APIs using curl command. +Documentation still in progress stay tuned. + +## CreatePerformanceProfile + +This is quick guide instructions to create performance profile using input JSON as follows. For a more detailed guide, +see [Create Performance Profile](/design/CreatePerformanceProfile.md) + +**Request** +`POST /createPerformanceProfile` + +`curl -H 'Accept: application/json' -X POST --data 'copy paste below JSON' http://:/createPerformanceProfile` + +``` +{ + "name": "resource-optimization-openshift", + "profile_version": 1, + "slo": { + "direction": "minimize", + "objective_function": { + "function_type": "expression", + "expression": "request_sum/request_count" + }, + "function_variables": [ + { + "name": "cpuRequest", + "datasource": "prometheus", + "value_type": "double", + "query": "kube_pod_container_resource_requests{pod=~'$DEPLOYMENT_NAME$-[^-]*-[^-]*$', container='$CONTAINER_NAME$', namespace='$NAMESPACE', resource='cpu', unit='core'}", + "aggregation_functions": [ + { + "function": "avg", + "query": "avg(kube_pod_container_resource_requests{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=\"$CONTAINER_NAME$\", namespace=\"$NAMESPACE\", resource=\"cpu\", unit=\"core\"})" + } + ] + } + ] + } +} +``` + +**Response** + +``` +{ + "message": "Performance Profile : created successfully. View all performance profiles at /listPerformanceProfiles", + "httpcode": 201, + "documentationLink": "", + "status": "SUCCESS" +} +``` +#### Note: One of query or aggregation_functions is mandatory. Both can be present together. + +## Update Performance Profile + +Update Performance Profile data using input JSON as follows. + +**Request** +`POST /updatePerformanceProfiles` + +`curl -H 'Accept: application/json' -X POST --data 'copy paste below JSON' http://:/updatePerformanceProfiles` + +``` +{ + "name": "resource-optimization-openshift", + "profile_version": 1, + "k8s_type": "openshift", + "slo": { + "slo_class": "resource_usage", + "direction": "minimize", + "objective_function": { + "function_type": "expression", + "expression": "request_sum/request_count" + }, + "function_variables": [ + { + "name": "cpuRequest", + "datasource": "prometheus", + "value_type": "double", + "kubernetes_object": "container", + "aggregation_functions": [ + { + "function": "avg", + "query": "avg(kube_pod_container_resource_requests{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=\"$CONTAINER_NAME$\", namespace=\"$NAMESPACE\", resource=\"cpu\", unit=\"core\"})" + }, + { + "function": "sum", + "query": "sum(kube_pod_container_resource_requests{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=\"$CONTAINER_NAME$\", namespace=\"$NAMESPACE\", resource=\"cpu\", unit=\"core\"})" + } + ] + }, + { + "name": "cpuLimit", + "datasource": "prometheus", + "value_type": "double", + "kubernetes_object": "container", + "aggregation_functions": [ + { + "function": "avg", + "query": "avg(kube_pod_container_resource_limits{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=\"$CONTAINER_NAME$\", namespace=\"$NAMESPACE\", resource=\"cpu\", unit=\"core\"})" + }, + { + "function": "sum", + "query": "sum(kube_pod_container_resource_limits{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=\"$CONTAINER_NAME$\", namespace=\"$NAMESPACE$\", resource=\"cpu\", unit=\"core\"})" + } + ] + }, + { + "name": "cpuUsage", + "datasource": "prometheus", + "value_type": "double", + "kubernetes_object": "container", + "aggregation_functions": [ + { + "function": "avg", + "query": "avg(avg_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=”$CONTAINER_NAME$”}[15m]))", + "versions": "<=4.8" + }, + { + "function": "avg", + "query": "avg(avg_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=”$CONTAINER_NAME$”}[15m]))", + "versions": ">4.9" + }, + { + "function": "min", + "query": "min(min_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=\"$CONTAINER_NAME$\"}[15m]))", + "versions": "<=4.8" + }, + { + "function": "min", + "query": "min(min_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=\"$CONTAINER_NAME$\"}[15m]))", + "versions": ">4.9" + }, + { + "function": "max", + "query": "max(max_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=\"$CONTAINER_NAME$\"}[15m]))", + "versions": "<=4.8" + }, + { + "function": "max", + "query": "max(max_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=\"$CONTAINER_NAME$\"}[15m]))", + "versions": ">4.9" + } + ] + }, + { + "name": "cpuThrottle", + "datasource": "prometheus", + "value_type": "double", + "kubernetes_object": "container", + "aggregation_functions": [ + { + "function": "avg", + "query": "rate(container_cpu_cfs_throttled_seconds_total{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=”$CONTAINER_NAME$”}[15m])" + } + ] + }, + { + "name": "memoryRequest", + "datasource": "prometheus", + "value_type": "double", + "kubernetes_object": "container", + "aggregation_functions": [ + { + "function": "avg", + "query": "avg(kube_pod_container_resource_requests{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=”$CONTAINER_NAME$”, namespace=”$NAMESPACE”, resource=\"memory\", unit=\"byte\"})" + }, + { + "function": "sum", + "query": "sum(kube_pod_container_resource_requests{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=”$CONTAINER_NAME$”, namespace=”$NAMESPACE”, resource=\"memory\", unit=\"byte\"})" + } + ] + }, + { + "name": "memoryLimit", + "datasource": "prometheus", + "value_type": "double", + "kubernetes_object": "container", + "aggregation_functions": [ + { + "function": "avg", + "query": "avg(kube_pod_container_resource_limits{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=\"$CONTAINER_NAME$\", namespace=\"$NAMESPACE\", resource=\"memory\", unit=\"byte\"})" + }, + { + "function": "sum", + "query": "sum(kube_pod_container_resource_limits{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=”$CONTAINER_NAME$”, namespace=”$NAMESPACE”, resource=\"memory\", unit=\"byte\"})" + } + ] + }, + { + "name": "memoryUsage", + "datasource": "prometheus", + "value_type": "double", + "kubernetes_object": "container", + "aggregation_functions": [ + { + "function": "avg", + "query": "avg(avg_over_time(container_memory_working_set_bytes{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=$NAMESPACE$, container=”$CONTAINER_NAME$”}[15m]))" + }, + { + "function": "min", + "query": "min(min_over_time(container_memory_working_set_bytes{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=$NAMESPACE$, container=\"$CONTAINER_NAME$\"}[15m]))" + }, + { + "function": "max", + "query": "max(max_over_time(container_memory_working_set_bytes{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=$NAMESPACE$, container=\"$CONTAINER_NAME$\"}[15m]))" + } + ] + }, + { + "name": "memoryRSS", + "datasource": "prometheus", + "value_type": "double", + "kubernetes_object": "container", + "aggregation_functions": [ + { + "function": "avg", + "query": "avg(avg_over_time(container_memory_rss{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=$NAMESPACE$, container=”$CONTAINER_NAME$”}[15m]))" + }, + { + "function": "min", + "query": "min(min_over_time(container_memory_rss{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=$NAMESPACE$, container=\"$CONTAINER_NAME$\"}[15m]))" + }, + { + "function": "max", + "query": "max(max_over_time(container_memory_rss{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=$NAMESPACE$, container=\"$CONTAINER_NAME$\"}[15m]))" + } + ] + } + ] + } +} + +``` + +**Response** + +``` +{ + "message": "Updated performance profile successfully. View updated profile at /listPerformanceProfiles", + "httpcode": 201, + "documentationLink": "", + "status": "SUCCESS" +} +``` + +## List Performance Profiles + +List performance profiles output JSON as follows. + +**Request** +`GET /listPerformanceProfiles` + +`curl -H 'Accept: application/json' http://:/listPerformanceProfiles` + +**Response** + +``` +{ + "resource-optimization-openshift": { + "name": "resource-optimization-openshift", + "profile_version": 1.0, + "k8s_type": "openshift", + "slo": { + "sloClass": "resource_usage", + "objective_function": { + "function_type": "source" + }, + "direction": "minimize", + "hpoAlgoImpl": "optuna_tpe", + "function_variables": [ + { + "name": "cpuRequest", + "query": "", + "datasource": "prometheus", + "value_type": "double", + "kubernetes_object": "container", + "trialSummaryResult": {}, + "cycleDataMap": {}, + "aggregation_functions": [ + { + "function": "avg", + "query": "avg(kube_pod_container_resource_requests{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=\"$CONTAINER_NAME$\", namespace=\"$NAMESPACE\", resource=\"cpu\", unit=\"core\"})", + "version": "" + }, + { + "function": "sum", + "query": "sum(kube_pod_container_resource_requests{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=\"$CONTAINER_NAME$\", namespace=\"$NAMESPACE\", resource=\"cpu\", unit=\"core\"})", + "version": "" + } + ] + }, + { + "name": "cpuLimit", + "query": "", + "datasource": "prometheus", + "value_type": "double", + "kubernetes_object": "container", + "trialSummaryResult": {}, + "cycleDataMap": {}, + "aggregation_functions": [ + { + "function": "avg", + "query": "avg(kube_pod_container_resource_limits{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=\"$CONTAINER_NAME$\", namespace=\"$NAMESPACE\", resource=\"cpu\", unit=\"core\"})", + "version": "" + }, + { + "function": "sum", + "query": "sum(kube_pod_container_resource_limits{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=\"$CONTAINER_NAME$\", namespace=\"$NAMESPACE$\", resource=\"cpu\", unit=\"core\"})", + "version": "" + } + ] + }, + { + "name": "cpuUsage", + "query": "", + "datasource": "prometheus", + "value_type": "double", + "kubernetes_object": "container", + "trialSummaryResult": {}, + "cycleDataMap": {}, + "aggregation_functions": [ + { + "function": "avg", + "query": "avg(avg_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=”$CONTAINER_NAME$”}[15m]))", + "version": "<=4.8" + }, + { + "function": "avg", + "query": "avg(avg_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=”$CONTAINER_NAME$”}[15m]))", + "version": ">4.9" + }, + { + "function": "min", + "query": "min(min_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=\"$CONTAINER_NAME$\"}[15m]))", + "version": "<=4.8" + }, + { + "function": "min", + "query": "min(min_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=\"$CONTAINER_NAME$\"}[15m]))", + "version": ">4.9" + }, + { + "function": "max", + "query": "max(max_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=\"$CONTAINER_NAME$\"}[15m]))", + "version": "<=4.8" + }, + { + "function": "max", + "query": "max(max_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=\"$CONTAINER_NAME$\"}[15m]))", + "version": ">4.9" + }, + { + "function": "sum", + "query": "sum(avg_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=\"$CONTAINER_NAME$\"}[15m]))", + "version": "<=4.8" + }, + { + "function": "sum", + "query": "sum(avg_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=\"$CONTAINER_NAME$\"}[15m]))", + "version": ">4.9" + } + ] + }, + { + "name": "cpuThrottle", + "query": "", + "datasource": "prometheus", + "value_type": "double", + "kubernetes_object": "container", + "trialSummaryResult": {}, + "cycleDataMap": {}, + "aggregation_functions": [ + { + "function": "avg", + "query": "avg(rate(container_cpu_cfs_throttled_seconds_total{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=”$CONTAINER_NAME$”}[15m]))", + "version": "" + }, + { + "function": "max", + "query": "max(rate(container_cpu_cfs_throttled_seconds_total{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=”$CONTAINER_NAME$”}[15m]))", + "version": "" + }, + { + "function": "sum", + "query": "sum(rate(container_cpu_cfs_throttled_seconds_total{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=”$CONTAINER_NAME$”}[15m]))", + "version": "" + } + ] + }, + { + "name": "memoryRequest", + "query": "", + "datasource": "prometheus", + "value_type": "double", + "kubernetes_object": "container", + "trialSummaryResult": {}, + "cycleDataMap": {}, + "aggregation_functions": [ + { + "function": "avg", + "query": "avg(kube_pod_container_resource_requests{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=”$CONTAINER_NAME$”, namespace=”$NAMESPACE”, resource=\"memory\", unit=\"byte\"})", + "version": "" + }, + { + "function": "sum", + "query": "sum(kube_pod_container_resource_requests{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=”$CONTAINER_NAME$”, namespace=”$NAMESPACE”, resource=\"memory\", unit=\"byte\"})", + "version": "" + } + ] + }, + { + "name": "memoryLimit", + "query": "", + "datasource": "prometheus", + "value_type": "double", + "kubernetes_object": "container", + "trialSummaryResult": {}, + "cycleDataMap": {}, + "aggregation_functions": [ + { + "function": "avg", + "query": "avg(kube_pod_container_resource_limits{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=\"$CONTAINER_NAME$\", namespace=\"$NAMESPACE\", resource=\"memory\", unit=\"byte\"})", + "version": "" + }, + { + "function": "sum", + "query": "sum(kube_pod_container_resource_limits{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=”$CONTAINER_NAME$”, namespace=”$NAMESPACE”, resource=\"memory\", unit=\"byte\"})", + "version": "" + } + ] + }, + { + "name": "memoryUsage", + "query": "", + "datasource": "prometheus", + "value_type": "double", + "kubernetes_object": "container", + "trialSummaryResult": {}, + "cycleDataMap": {}, + "aggregation_functions": [ + { + "function": "avg", + "query": "avg(avg_over_time(container_memory_working_set_bytes{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=$NAMESPACE$, container=”$CONTAINER_NAME$”}[15m]))", + "version": "" + }, + { + "function": "min", + "query": "min(min_over_time(container_memory_working_set_bytes{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=$NAMESPACE$, container=\"$CONTAINER_NAME$\"}[15m]))", + "version": "" + }, + { + "function": "max", + "query": "max(max_over_time(container_memory_working_set_bytes{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=$NAMESPACE$, container=\"$CONTAINER_NAME$\"}[15m]))", + "version": "" + }, + { + "function": "sum", + "query": "sum(avg_over_time(container_memory_working_set_bytes{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=$NAMESPACE$, container=\"$CONTAINER_NAME$\"}[15m]))", + "version": "" + } + ] + }, + { + "name": "memoryRSS", + "query": "", + "datasource": "prometheus", + "value_type": "double", + "kubernetes_object": "container", + "trialSummaryResult": {}, + "cycleDataMap": {}, + "aggregation_functions": [ + { + "function": "avg", + "query": "avg(avg_over_time(container_memory_rss{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=$NAMESPACE$, container=”$CONTAINER_NAME$”}[15m]))", + "version": "" + }, + { + "function": "min", + "query": "min(min_over_time(container_memory_rss{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=$NAMESPACE$, container=\"$CONTAINER_NAME$\"}[15m]))", + "version": "" + }, + { + "function": "max", + "query": "max(max_over_time(container_memory_rss{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=$NAMESPACE$, container=\"$CONTAINER_NAME$\"}[15m]))", + "version": "" + }, + { + "function": "sum", + "query": "sum(avg_over_time(container_memory_rss{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=$NAMESPACE$, container=”$CONTAINER_NAME$”}[15m]))", + "version": "" + } + ] + } + ] + } + } +} +``` diff --git a/src/main/java/com/autotune/utils/ServerContext.java b/src/main/java/com/autotune/utils/ServerContext.java index f0146d600..e9c773ee2 100644 --- a/src/main/java/com/autotune/utils/ServerContext.java +++ b/src/main/java/com/autotune/utils/ServerContext.java @@ -37,7 +37,7 @@ public class ServerContext { public static final String UPDATE_RESULTS = ROOT_CONTEXT + "updateResults"; public static final String RECOMMEND_RESULTS = ROOT_CONTEXT + "listRecommendations"; public static final String CREATE_PERF_PROFILE = ROOT_CONTEXT + "createPerformanceProfile"; - public static final String LIST_PERF_PROFILE = ROOT_CONTEXT + "listPerformanceProfile"; + public static final String LIST_PERF_PROFILE = ROOT_CONTEXT + "listPerformanceProfiles"; From b52dbae2c5fc67956dd816ce966d3afade962ee7 Mon Sep 17 00:00:00 2001 From: saakhan Date: Tue, 14 Feb 2023 10:00:15 +0530 Subject: [PATCH 23/35] remove unnecessary fields from PerfProfile JSON, update response to blank array, update design docs and related file changes Signed-off-by: saakhan --- ...rmanceProfile.md => PerformanceProfile.md} | 1 + ...ceProfiles.md => PerformanceProfileAPI.md} | 211 +----------------- .../java/com/autotune/analyzer/Analyzer.java | 2 +- .../services/PerformanceProfileService.java | 20 +- .../com/autotune/utils/ServerContext.java | 2 +- 5 files changed, 21 insertions(+), 215 deletions(-) rename design/{CreatePerformanceProfile.md => PerformanceProfile.md} (96%) rename design/{PerformanceProfiles.md => PerformanceProfileAPI.md} (62%) diff --git a/design/CreatePerformanceProfile.md b/design/PerformanceProfile.md similarity index 96% rename from design/CreatePerformanceProfile.md rename to design/PerformanceProfile.md index cdfcce3c0..6c0c093e3 100644 --- a/design/CreatePerformanceProfile.md +++ b/design/PerformanceProfile.md @@ -76,3 +76,4 @@ tuned. "status": "FAILED" } ``` +* ##### You can get the API details [here](/design/PerformanceProfileAPI.md) diff --git a/design/PerformanceProfiles.md b/design/PerformanceProfileAPI.md similarity index 62% rename from design/PerformanceProfiles.md rename to design/PerformanceProfileAPI.md index 1334750b1..cd8261024 100644 --- a/design/PerformanceProfiles.md +++ b/design/PerformanceProfileAPI.md @@ -6,7 +6,7 @@ Documentation still in progress stay tuned. ## CreatePerformanceProfile This is quick guide instructions to create performance profile using input JSON as follows. For a more detailed guide, -see [Create Performance Profile](/design/CreatePerformanceProfile.md) +see [Create Performance Profile](/design/PerformanceProfile.md) **Request** `POST /createPerformanceProfile` @@ -53,199 +53,6 @@ see [Create Performance Profile](/design/CreatePerformanceProfile.md) ``` #### Note: One of query or aggregation_functions is mandatory. Both can be present together. -## Update Performance Profile - -Update Performance Profile data using input JSON as follows. - -**Request** -`POST /updatePerformanceProfiles` - -`curl -H 'Accept: application/json' -X POST --data 'copy paste below JSON' http://:/updatePerformanceProfiles` - -``` -{ - "name": "resource-optimization-openshift", - "profile_version": 1, - "k8s_type": "openshift", - "slo": { - "slo_class": "resource_usage", - "direction": "minimize", - "objective_function": { - "function_type": "expression", - "expression": "request_sum/request_count" - }, - "function_variables": [ - { - "name": "cpuRequest", - "datasource": "prometheus", - "value_type": "double", - "kubernetes_object": "container", - "aggregation_functions": [ - { - "function": "avg", - "query": "avg(kube_pod_container_resource_requests{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=\"$CONTAINER_NAME$\", namespace=\"$NAMESPACE\", resource=\"cpu\", unit=\"core\"})" - }, - { - "function": "sum", - "query": "sum(kube_pod_container_resource_requests{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=\"$CONTAINER_NAME$\", namespace=\"$NAMESPACE\", resource=\"cpu\", unit=\"core\"})" - } - ] - }, - { - "name": "cpuLimit", - "datasource": "prometheus", - "value_type": "double", - "kubernetes_object": "container", - "aggregation_functions": [ - { - "function": "avg", - "query": "avg(kube_pod_container_resource_limits{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=\"$CONTAINER_NAME$\", namespace=\"$NAMESPACE\", resource=\"cpu\", unit=\"core\"})" - }, - { - "function": "sum", - "query": "sum(kube_pod_container_resource_limits{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=\"$CONTAINER_NAME$\", namespace=\"$NAMESPACE$\", resource=\"cpu\", unit=\"core\"})" - } - ] - }, - { - "name": "cpuUsage", - "datasource": "prometheus", - "value_type": "double", - "kubernetes_object": "container", - "aggregation_functions": [ - { - "function": "avg", - "query": "avg(avg_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=”$CONTAINER_NAME$”}[15m]))", - "versions": "<=4.8" - }, - { - "function": "avg", - "query": "avg(avg_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=”$CONTAINER_NAME$”}[15m]))", - "versions": ">4.9" - }, - { - "function": "min", - "query": "min(min_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=\"$CONTAINER_NAME$\"}[15m]))", - "versions": "<=4.8" - }, - { - "function": "min", - "query": "min(min_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=\"$CONTAINER_NAME$\"}[15m]))", - "versions": ">4.9" - }, - { - "function": "max", - "query": "max(max_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=\"$CONTAINER_NAME$\"}[15m]))", - "versions": "<=4.8" - }, - { - "function": "max", - "query": "max(max_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=\"$CONTAINER_NAME$\"}[15m]))", - "versions": ">4.9" - } - ] - }, - { - "name": "cpuThrottle", - "datasource": "prometheus", - "value_type": "double", - "kubernetes_object": "container", - "aggregation_functions": [ - { - "function": "avg", - "query": "rate(container_cpu_cfs_throttled_seconds_total{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=\"$NAMESPACE$\", container=”$CONTAINER_NAME$”}[15m])" - } - ] - }, - { - "name": "memoryRequest", - "datasource": "prometheus", - "value_type": "double", - "kubernetes_object": "container", - "aggregation_functions": [ - { - "function": "avg", - "query": "avg(kube_pod_container_resource_requests{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=”$CONTAINER_NAME$”, namespace=”$NAMESPACE”, resource=\"memory\", unit=\"byte\"})" - }, - { - "function": "sum", - "query": "sum(kube_pod_container_resource_requests{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=”$CONTAINER_NAME$”, namespace=”$NAMESPACE”, resource=\"memory\", unit=\"byte\"})" - } - ] - }, - { - "name": "memoryLimit", - "datasource": "prometheus", - "value_type": "double", - "kubernetes_object": "container", - "aggregation_functions": [ - { - "function": "avg", - "query": "avg(kube_pod_container_resource_limits{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=\"$CONTAINER_NAME$\", namespace=\"$NAMESPACE\", resource=\"memory\", unit=\"byte\"})" - }, - { - "function": "sum", - "query": "sum(kube_pod_container_resource_limits{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", container=”$CONTAINER_NAME$”, namespace=”$NAMESPACE”, resource=\"memory\", unit=\"byte\"})" - } - ] - }, - { - "name": "memoryUsage", - "datasource": "prometheus", - "value_type": "double", - "kubernetes_object": "container", - "aggregation_functions": [ - { - "function": "avg", - "query": "avg(avg_over_time(container_memory_working_set_bytes{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=$NAMESPACE$, container=”$CONTAINER_NAME$”}[15m]))" - }, - { - "function": "min", - "query": "min(min_over_time(container_memory_working_set_bytes{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=$NAMESPACE$, container=\"$CONTAINER_NAME$\"}[15m]))" - }, - { - "function": "max", - "query": "max(max_over_time(container_memory_working_set_bytes{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=$NAMESPACE$, container=\"$CONTAINER_NAME$\"}[15m]))" - } - ] - }, - { - "name": "memoryRSS", - "datasource": "prometheus", - "value_type": "double", - "kubernetes_object": "container", - "aggregation_functions": [ - { - "function": "avg", - "query": "avg(avg_over_time(container_memory_rss{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=$NAMESPACE$, container=”$CONTAINER_NAME$”}[15m]))" - }, - { - "function": "min", - "query": "min(min_over_time(container_memory_rss{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=$NAMESPACE$, container=\"$CONTAINER_NAME$\"}[15m]))" - }, - { - "function": "max", - "query": "max(max_over_time(container_memory_rss{pod=~\"$DEPLOYMENT_NAME$-[^-]*-[^-]*$\", namespace=$NAMESPACE$, container=\"$CONTAINER_NAME$\"}[15m]))" - } - ] - } - ] - } -} - -``` - -**Response** - -``` -{ - "message": "Updated performance profile successfully. View updated profile at /listPerformanceProfiles", - "httpcode": 201, - "documentationLink": "", - "status": "SUCCESS" -} -``` - ## List Performance Profiles List performance profiles output JSON as follows. @@ -277,8 +84,6 @@ List performance profiles output JSON as follows. "datasource": "prometheus", "value_type": "double", "kubernetes_object": "container", - "trialSummaryResult": {}, - "cycleDataMap": {}, "aggregation_functions": [ { "function": "avg", @@ -298,8 +103,6 @@ List performance profiles output JSON as follows. "datasource": "prometheus", "value_type": "double", "kubernetes_object": "container", - "trialSummaryResult": {}, - "cycleDataMap": {}, "aggregation_functions": [ { "function": "avg", @@ -319,8 +122,6 @@ List performance profiles output JSON as follows. "datasource": "prometheus", "value_type": "double", "kubernetes_object": "container", - "trialSummaryResult": {}, - "cycleDataMap": {}, "aggregation_functions": [ { "function": "avg", @@ -370,8 +171,6 @@ List performance profiles output JSON as follows. "datasource": "prometheus", "value_type": "double", "kubernetes_object": "container", - "trialSummaryResult": {}, - "cycleDataMap": {}, "aggregation_functions": [ { "function": "avg", @@ -396,8 +195,6 @@ List performance profiles output JSON as follows. "datasource": "prometheus", "value_type": "double", "kubernetes_object": "container", - "trialSummaryResult": {}, - "cycleDataMap": {}, "aggregation_functions": [ { "function": "avg", @@ -417,8 +214,6 @@ List performance profiles output JSON as follows. "datasource": "prometheus", "value_type": "double", "kubernetes_object": "container", - "trialSummaryResult": {}, - "cycleDataMap": {}, "aggregation_functions": [ { "function": "avg", @@ -438,8 +233,6 @@ List performance profiles output JSON as follows. "datasource": "prometheus", "value_type": "double", "kubernetes_object": "container", - "trialSummaryResult": {}, - "cycleDataMap": {}, "aggregation_functions": [ { "function": "avg", @@ -469,8 +262,6 @@ List performance profiles output JSON as follows. "datasource": "prometheus", "value_type": "double", "kubernetes_object": "container", - "trialSummaryResult": {}, - "cycleDataMap": {}, "aggregation_functions": [ { "function": "avg", diff --git a/src/main/java/com/autotune/analyzer/Analyzer.java b/src/main/java/com/autotune/analyzer/Analyzer.java index 2551f61b4..5d1102d66 100644 --- a/src/main/java/com/autotune/analyzer/Analyzer.java +++ b/src/main/java/com/autotune/analyzer/Analyzer.java @@ -59,7 +59,7 @@ public static void addServlets(ServletContextHandler context) { context.addServlet(UpdateResults.class, ServerContext.UPDATE_RESULTS); context.addServlet(Recommendation.class, ServerContext.RECOMMEND_RESULTS); context.addServlet(PerformanceProfileService.class, ServerContext.CREATE_PERF_PROFILE); - context.addServlet(PerformanceProfileService.class, ServerContext.LIST_PERF_PROFILE); + context.addServlet(PerformanceProfileService.class, ServerContext.LIST_PERF_PROFILES); // Adding UI support API's context.addServlet(ListNamespaces.class, ServerContext.LIST_NAMESPACES); diff --git a/src/main/java/com/autotune/analyzer/services/PerformanceProfileService.java b/src/main/java/com/autotune/analyzer/services/PerformanceProfileService.java index 7a724cf01..9e24aaac7 100644 --- a/src/main/java/com/autotune/analyzer/services/PerformanceProfileService.java +++ b/src/main/java/com/autotune/analyzer/services/PerformanceProfileService.java @@ -19,11 +19,13 @@ import com.autotune.analyzer.exceptions.PerformanceProfileResponse; import com.autotune.analyzer.utils.GsonUTCDateAdapter; import com.autotune.analyzer.utils.PerformanceProfileValidation; -import com.autotune.common.annotations.json.AutotuneJSONExclusionStrategy; import com.autotune.common.data.ValidationResultData; +import com.autotune.common.k8sObjects.Metric; import com.autotune.common.performanceProfiles.PerformanceProfile; import com.autotune.utils.AnalyzerConstants; import com.autotune.utils.AnalyzerErrorConstants; +import com.google.gson.ExclusionStrategy; +import com.google.gson.FieldAttributes; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import org.slf4j.Logger; @@ -109,11 +111,23 @@ protected void doGet(HttpServletRequest req, HttpServletResponse response) throw .setPrettyPrinting() .enableComplexMapKeySerialization() .registerTypeAdapter(Date.class, new GsonUTCDateAdapter()) - .setExclusionStrategies(new AutotuneJSONExclusionStrategy()) + .setExclusionStrategies(new ExclusionStrategy() { + @Override + public boolean shouldSkipField(FieldAttributes f) { + return f.getDeclaringClass() == Metric.class && ( + f.getName().equals("trialSummaryResult") + || f.getName().equals("cycleDataMap") + ); + } + @Override + public boolean shouldSkipClass(Class aClass) { + return false; + } + }) .create(); gsonStr = gsonObj.toJson(this.performanceProfilesMap); } else { - gsonStr = AnalyzerErrorConstants.AutotuneObjectErrors.NO_PERF_PROFILE; + LOGGER.debug(AnalyzerErrorConstants.AutotuneObjectErrors.NO_PERF_PROFILE); } response.getWriter().println(gsonStr); response.getWriter().close(); diff --git a/src/main/java/com/autotune/utils/ServerContext.java b/src/main/java/com/autotune/utils/ServerContext.java index 94b20eaeb..bc431e45c 100644 --- a/src/main/java/com/autotune/utils/ServerContext.java +++ b/src/main/java/com/autotune/utils/ServerContext.java @@ -37,7 +37,7 @@ public class ServerContext { public static final String UPDATE_RESULTS = ROOT_CONTEXT + "updateResults"; public static final String RECOMMEND_RESULTS = ROOT_CONTEXT + "listRecommendations"; public static final String CREATE_PERF_PROFILE = ROOT_CONTEXT + "createPerformanceProfile"; - public static final String LIST_PERF_PROFILE = ROOT_CONTEXT + "listPerformanceProfiles"; + public static final String LIST_PERF_PROFILES = ROOT_CONTEXT + "listPerformanceProfiles"; From 18d24e71e13b73ce845382456865f7211b3daa33 Mon Sep 17 00:00:00 2001 From: saakhan Date: Tue, 14 Feb 2023 14:30:44 +0530 Subject: [PATCH 24/35] update API response to correct array format Signed-off-by: saakhan --- .../autotune/analyzer/services/PerformanceProfileService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/autotune/analyzer/services/PerformanceProfileService.java b/src/main/java/com/autotune/analyzer/services/PerformanceProfileService.java index 9e24aaac7..234929505 100644 --- a/src/main/java/com/autotune/analyzer/services/PerformanceProfileService.java +++ b/src/main/java/com/autotune/analyzer/services/PerformanceProfileService.java @@ -40,6 +40,7 @@ import java.io.IOException; import java.io.PrintWriter; import java.io.Serial; +import java.util.Collection; import java.util.Date; import java.util.HashMap; import java.util.Map; @@ -106,6 +107,7 @@ protected void doGet(HttpServletRequest req, HttpServletResponse response) throw response.setStatus(HttpServletResponse.SC_OK); String gsonStr = "[]"; if (this.performanceProfilesMap.size() > 0) { + Collection values = performanceProfilesMap.values(); Gson gsonObj = new GsonBuilder() .disableHtmlEscaping() .setPrettyPrinting() @@ -125,7 +127,7 @@ public boolean shouldSkipClass(Class aClass) { } }) .create(); - gsonStr = gsonObj.toJson(this.performanceProfilesMap); + gsonStr = gsonObj.toJson(values); } else { LOGGER.debug(AnalyzerErrorConstants.AutotuneObjectErrors.NO_PERF_PROFILE); } From 754d1bd0fd3eb02ffdee318472da7b0e09248575 Mon Sep 17 00:00:00 2001 From: saakhan Date: Tue, 14 Feb 2023 17:37:57 +0530 Subject: [PATCH 25/35] update API response in the design doc Signed-off-by: saakhan --- design/PerformanceProfileAPI.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/design/PerformanceProfileAPI.md b/design/PerformanceProfileAPI.md index cd8261024..8941f5228 100644 --- a/design/PerformanceProfileAPI.md +++ b/design/PerformanceProfileAPI.md @@ -65,8 +65,8 @@ List performance profiles output JSON as follows. **Response** ``` -{ - "resource-optimization-openshift": { +[ + { "name": "resource-optimization-openshift", "profile_version": 1.0, "k8s_type": "openshift", @@ -288,5 +288,5 @@ List performance profiles output JSON as follows. ] } } -} +] ``` From 79e5be90edc4e1b0c66d2fe6333df7c8d5377323 Mon Sep 17 00:00:00 2001 From: Dinakar Guniguntala Date: Tue, 14 Feb 2023 18:05:16 +0530 Subject: [PATCH 26/35] Change ENV_VAR from AUTOTUNE_PORT to AUTOTUNE_SERVER_PORT Signed-off-by: Dinakar Guniguntala --- src/main/java/com/autotune/Autotune.java | 5 +--- .../com/autotune/utils/ServerContext.java | 24 +++++++++---------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/autotune/Autotune.java b/src/main/java/com/autotune/Autotune.java index 2515ba5a3..cfe5d0aaf 100644 --- a/src/main/java/com/autotune/Autotune.java +++ b/src/main/java/com/autotune/Autotune.java @@ -25,7 +25,6 @@ import io.prometheus.client.exporter.MetricsServlet; import io.prometheus.client.hotspot.DefaultExports; import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.servlet.ErrorPageErrorHandler; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; import org.slf4j.Logger; @@ -33,8 +32,6 @@ import static com.autotune.utils.ServerContext.*; -import javax.servlet.http.HttpServletResponse; - public class Autotune { private static final Logger LOGGER = LoggerFactory.getLogger(Autotune.class); @@ -45,7 +42,7 @@ public static void main(String[] args) { disableServerLogging(); - Server server = new Server(AUTOTUNE_PORT); + Server server = new Server(AUTOTUNE_SERVER_PORT); context = new ServletContextHandler(); context.setContextPath(ServerContext.ROOT_CONTEXT); context.setErrorHandler(new AutotuneErrorHandler()); diff --git a/src/main/java/com/autotune/utils/ServerContext.java b/src/main/java/com/autotune/utils/ServerContext.java index 465d8b14e..be3aa2f94 100644 --- a/src/main/java/com/autotune/utils/ServerContext.java +++ b/src/main/java/com/autotune/utils/ServerContext.java @@ -19,8 +19,8 @@ * Holds the server context of the dependency analyzer. */ public class ServerContext { - public static final int AUTOTUNE_PORT = Integer.parseInt(System.getenv().getOrDefault("AUTOTUNE_PORT", "8080")); - public static final int HPO_PORT = 8085; + public static final int AUTOTUNE_SERVER_PORT = Integer.parseInt(System.getenv().getOrDefault("AUTOTUNE_SERVER_PORT", "8080")); + public static final int HPO_SERVER_PORT = 8085; // AnalyzerConstants end points public static final String ROOT_CONTEXT = "/"; @@ -39,23 +39,23 @@ public class ServerContext { public static final String CREATE_PERF_PROFILE = ROOT_CONTEXT + "createPerformanceProfile"; - public static final String AUTOTUNE_SERVER_PORT = "http://localhost:" + AUTOTUNE_PORT; - public static final String SEARCH_SPACE_END_POINT = AUTOTUNE_SERVER_PORT + SEARCH_SPACE; - public static final String LIST_EXPERIMENTS_END_POINT = AUTOTUNE_SERVER_PORT + LIST_EXPERIMENTS; - public static final String UPDATE_RESULTS_END_POINT = AUTOTUNE_SERVER_PORT + UPDATE_RESULTS; + public static final String AUTOTUNE_SERVER_URL = "http://localhost:" + AUTOTUNE_SERVER_PORT; + public static final String SEARCH_SPACE_END_POINT = AUTOTUNE_SERVER_URL + SEARCH_SPACE; + public static final String LIST_EXPERIMENTS_END_POINT = AUTOTUNE_SERVER_URL + LIST_EXPERIMENTS; + public static final String UPDATE_RESULTS_END_POINT = AUTOTUNE_SERVER_URL + UPDATE_RESULTS; // HPO End Points - public static final String HPO_SERVER_PORT = "http://localhost:" + HPO_PORT; + public static final String HPO_SERVER_URL = "http://localhost:" + HPO_SERVER_PORT; public static final String HPO_TRIALS = ROOT_CONTEXT + "experiment_trials"; - public static final String HPO_TRIALS_END_POINT = HPO_SERVER_PORT + HPO_TRIALS; + public static final String HPO_TRIALS_END_POINT = HPO_SERVER_URL + HPO_TRIALS; - public static final String EXPERIMENT_MANAGER_SERVER_PORT = "http://localhost:" + AUTOTUNE_PORT; + public static final String EXPERIMENT_MANAGER_SERVER_URL = "http://localhost:" + AUTOTUNE_SERVER_PORT; public static final String EXPERIMENT_MANAGER_CREATE_TRIAL = ROOT_CONTEXT + "createExperimentTrial"; - public static final String EXPERIMENT_MANAGER_CREATE_TRIAL_END_POINT = EXPERIMENT_MANAGER_SERVER_PORT + EXPERIMENT_MANAGER_CREATE_TRIAL; + public static final String EXPERIMENT_MANAGER_CREATE_TRIAL_END_POINT = EXPERIMENT_MANAGER_SERVER_URL + EXPERIMENT_MANAGER_CREATE_TRIAL; public static final String EXPERIMENT_MANAGER_LIST_EXPERIMENT_TRIAL = ROOT_CONTEXT + "listExperimentTrial"; - public static final String EXPERIMENT_MANAGER_LIST_EXPERIMENT_END_POINT = EXPERIMENT_MANAGER_SERVER_PORT + EXPERIMENT_MANAGER_LIST_EXPERIMENT_TRIAL; + public static final String EXPERIMENT_MANAGER_LIST_EXPERIMENT_END_POINT = EXPERIMENT_MANAGER_SERVER_URL + EXPERIMENT_MANAGER_LIST_EXPERIMENT_TRIAL; public static final String EXPERIMENT_MANAGER_LIST_TRIAL_STATUS = ROOT_CONTEXT + "listTrialStatus"; - public static final String EXPERIMENT_MANAGER_LIST_TRIAL_STATUS_END_POINT = EXPERIMENT_MANAGER_SERVER_PORT + EXPERIMENT_MANAGER_LIST_TRIAL_STATUS; + public static final String EXPERIMENT_MANAGER_LIST_TRIAL_STATUS_END_POINT = EXPERIMENT_MANAGER_SERVER_URL + EXPERIMENT_MANAGER_LIST_TRIAL_STATUS; // UI support EndPoints public static final String QUERY_CONTEXT = ROOT_CONTEXT + "query/"; From b7e8f200071bc65a85155cef19c16b8951473174 Mon Sep 17 00:00:00 2001 From: bharathappali Date: Wed, 15 Feb 2023 11:01:38 +0530 Subject: [PATCH 27/35] Adds cluster name in kruize object Signed-off-by: bharathappali --- .../common/k8sObjects/KruizeObject.java | 19 ++++++++++++++++++- .../k8sObjects/ValidateAutotuneObject.java | 8 ++++++++ .../com/autotune/utils/AnalyzerConstants.java | 1 + 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/autotune/common/k8sObjects/KruizeObject.java b/src/main/java/com/autotune/common/k8sObjects/KruizeObject.java index a2dbddda4..43334db57 100644 --- a/src/main/java/com/autotune/common/k8sObjects/KruizeObject.java +++ b/src/main/java/com/autotune/common/k8sObjects/KruizeObject.java @@ -54,6 +54,7 @@ public final class KruizeObject { private ExperimentUseCaseType experimentUseCaseType; private Set resultData; private ValidationResultData validationData; + private String clusterName; public KruizeObject(String experimentName, String namespace, @@ -61,7 +62,10 @@ public KruizeObject(String experimentName, String targetCluster, SloInfo sloInfo, SelectorInfo selectorInfo, - ObjectReference objectReference) throws InvalidValueException { + ObjectReference objectReference, + String... clusterNameContent // Adding an optional param to not break existing implementation + // Please add any required param above/before the `clusterName` variable + ) throws InvalidValueException { HashMap map = new HashMap<>(); map.put(AnalyzerConstants.AutotuneObjectConstants.NAME, experimentName); @@ -70,6 +74,8 @@ public KruizeObject(String experimentName, map.put(AnalyzerConstants.AutotuneObjectConstants.TARGET_CLUSTER, targetCluster); map.put(AnalyzerConstants.AutotuneObjectConstants.SLO, sloInfo); map.put(AnalyzerConstants.AutotuneObjectConstants.SELECTOR, selectorInfo); + if (clusterNameContent.length > 0) + map.put(AnalyzerConstants.AutotuneObjectConstants.CLUSTER_NAME, clusterNameContent[0].trim()); StringBuilder error = ValidateAutotuneObject.validate(map); if (error.toString().isEmpty()) { @@ -81,6 +87,7 @@ public KruizeObject(String experimentName, this.selectorInfo = selectorInfo; this.experimentId = Utils.generateID(toString()); this.objectReference = objectReference; + this.clusterName = clusterNameContent[0].trim(); } else { throw new InvalidValueException(error.toString()); } @@ -226,8 +233,17 @@ public void setValidationData(ValidationResultData validationData) { this.validationData = validationData; } + public String getClusterName() { + return clusterName; + } + @Override public String toString() { + // Creating a temparory cluster name as we allow null for cluster name now + // Please change it to use `clusterName` variable itself if there is a null check already in place for that + String tmpClusterName = ""; + if (clusterName != null) + tmpClusterName = clusterName; return "KruizeObject{" + "experimentId='" + experimentId + '\'' + ", experimentName='" + experimentName + '\'' + @@ -246,6 +262,7 @@ public String toString() { ", experimentUseCaseType=" + experimentUseCaseType + ", resultData=" + resultData + ", validationData=" + validationData + + ", clusterName=" + tmpClusterName + '}'; } } diff --git a/src/main/java/com/autotune/common/k8sObjects/ValidateAutotuneObject.java b/src/main/java/com/autotune/common/k8sObjects/ValidateAutotuneObject.java index 8da01ca4f..3696ba544 100644 --- a/src/main/java/com/autotune/common/k8sObjects/ValidateAutotuneObject.java +++ b/src/main/java/com/autotune/common/k8sObjects/ValidateAutotuneObject.java @@ -156,6 +156,14 @@ public static StringBuilder validate(HashMap map) { } } + // Placeholder for cluster name validation + if (map.containsKey(AnalyzerConstants.AutotuneObjectConstants.CLUSTER_NAME)) { + // if you don't want cluster name to be `null` update the code block below + if (map.get(AnalyzerConstants.AutotuneObjectConstants.CLUSTER_NAME) == null || ((String)map.get(AnalyzerConstants.AutotuneObjectConstants.CLUSTER_NAME)).isEmpty()) { + // Add your logic of appending the error string + } + } + return errorString; } } diff --git a/src/main/java/com/autotune/utils/AnalyzerConstants.java b/src/main/java/com/autotune/utils/AnalyzerConstants.java index bff330e97..8a410338d 100644 --- a/src/main/java/com/autotune/utils/AnalyzerConstants.java +++ b/src/main/java/com/autotune/utils/AnalyzerConstants.java @@ -156,6 +156,7 @@ public static final class AutotuneObjectConstants { public static final String HPO_ALGO_IMPL = "hpo_algo_impl"; public static final String DEFAULT_HPO_ALGO_IMPL = "optuna_tpe"; public static final String FUNCTION_VARIABLE = "function_variable: "; + public static final String CLUSTER_NAME = "cluster_name"; private AutotuneObjectConstants() { } From 03da727452cb15232c94102bcb11b1528d2575b1 Mon Sep 17 00:00:00 2001 From: bharathappali Date: Wed, 15 Feb 2023 11:15:35 +0530 Subject: [PATCH 28/35] Check if cluster name available before assigning Signed-off-by: bharathappali --- src/main/java/com/autotune/common/k8sObjects/KruizeObject.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/autotune/common/k8sObjects/KruizeObject.java b/src/main/java/com/autotune/common/k8sObjects/KruizeObject.java index 43334db57..e8809979f 100644 --- a/src/main/java/com/autotune/common/k8sObjects/KruizeObject.java +++ b/src/main/java/com/autotune/common/k8sObjects/KruizeObject.java @@ -87,7 +87,8 @@ public KruizeObject(String experimentName, this.selectorInfo = selectorInfo; this.experimentId = Utils.generateID(toString()); this.objectReference = objectReference; - this.clusterName = clusterNameContent[0].trim(); + if (clusterNameContent.length > 0) + this.clusterName = clusterNameContent[0].trim(); } else { throw new InvalidValueException(error.toString()); } From bc86f37e50bc53ba74da1f00203df6cbcd157ae4 Mon Sep 17 00:00:00 2001 From: bharathappali Date: Wed, 15 Feb 2023 11:21:24 +0530 Subject: [PATCH 29/35] Add serialised name for cluster name Signed-off-by: bharathappali --- src/main/java/com/autotune/common/k8sObjects/KruizeObject.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/autotune/common/k8sObjects/KruizeObject.java b/src/main/java/com/autotune/common/k8sObjects/KruizeObject.java index e8809979f..1fa61462e 100644 --- a/src/main/java/com/autotune/common/k8sObjects/KruizeObject.java +++ b/src/main/java/com/autotune/common/k8sObjects/KruizeObject.java @@ -54,6 +54,7 @@ public final class KruizeObject { private ExperimentUseCaseType experimentUseCaseType; private Set resultData; private ValidationResultData validationData; + @SerializedName("cluster_name") private String clusterName; public KruizeObject(String experimentName, From 2d0f661f5939d61c2d95dd43ea1cf21a0737f69c Mon Sep 17 00:00:00 2001 From: bharathappali Date: Wed, 15 Feb 2023 16:47:59 +0530 Subject: [PATCH 30/35] Adds CRD changes Signed-off-by: bharathappali --- manifests/autotune/autotune-operator-crd.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/manifests/autotune/autotune-operator-crd.yaml b/manifests/autotune/autotune-operator-crd.yaml index cdfbb5dc8..b4b4cbf0e 100644 --- a/manifests/autotune/autotune-operator-crd.yaml +++ b/manifests/autotune/autotune-operator-crd.yaml @@ -87,6 +87,10 @@ spec: description: 'Target cluster where the experiments are performed. Supported targets: local (default), remote' type: string default: 'local' + cluster_name: + description: 'Name of the cluster in which the autotune instance is deployed' + type: string + default: 'local' selector: description: 'test' type: object From 51f3f365a5c04bc38ba5c1a999d0f7b3b4213c15 Mon Sep 17 00:00:00 2001 From: saakhan Date: Thu, 16 Feb 2023 16:45:28 +0530 Subject: [PATCH 31/35] move SLO from Kruize to Performance Profile, move hpoAlgo to Kruize, add map to dynamically fetch the PerfProfile class and other related files changes Signed-off-by: saakhan --- .../java/com/autotune/analyzer/Analyzer.java | 5 +- .../autotune/analyzer/AutotuneExperiment.java | 6 +- .../com/autotune/analyzer/Experimentator.java | 10 +- .../analyzer/deployment/KruizeDeployment.java | 116 ++++++----- .../services/PerformanceProfileService.java | 4 +- .../utils/ExperimentResultValidation.java | 50 ++--- .../analyzer/utils/ExperimentValidation.java | 57 +++-- .../utils/PerformanceProfileValidation.java | 174 ++++++++++++++-- .../analyzer/utils/ServiceHelpers.java | 16 +- .../common/k8sObjects/KruizeObject.java | 21 +- .../autotune/common/k8sObjects/SloInfo.java | 29 +-- .../k8sObjects/ValidateAutotuneObject.java | 91 -------- .../ValidatePerformanceProfileObject.java | 68 ++++-- .../PerformanceProfile.java | 21 +- .../AdvancedClusterMgmtImpl.java | 39 ++++ .../DefaultImpl.java | 74 +++++++ .../DefaultOpenshiftImpl.java | 183 ---------------- .../PerfProfileImpl.java | 196 ++++++++++++++++++ .../PerfProfileInterface.java | 14 +- .../ResourceOptimizationOpenshiftImpl.java | 122 +---------- .../PerformanceProfilesDeployment.java | 87 ++------ .../com/autotune/utils/AnalyzerConstants.java | 6 + .../utils/AnalyzerErrorConstants.java | 6 +- .../java/com/autotune/utils/TrialHelpers.java | 6 +- 24 files changed, 701 insertions(+), 700 deletions(-) create mode 100644 src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/AdvancedClusterMgmtImpl.java create mode 100644 src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/DefaultImpl.java delete mode 100644 src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/DefaultOpenshiftImpl.java create mode 100644 src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/PerfProfileImpl.java diff --git a/src/main/java/com/autotune/analyzer/Analyzer.java b/src/main/java/com/autotune/analyzer/Analyzer.java index 5d1102d66..69903380a 100644 --- a/src/main/java/com/autotune/analyzer/Analyzer.java +++ b/src/main/java/com/autotune/analyzer/Analyzer.java @@ -21,6 +21,7 @@ import com.autotune.analyzer.exceptions.MonitoringAgentNotFoundException; import com.autotune.analyzer.exceptions.MonitoringAgentNotSupportedException; import com.autotune.analyzer.services.*; +import com.autotune.common.performanceProfiles.PerformanceProfile; import com.autotune.common.performanceProfiles.PerformanceProfilesDeployment; import com.autotune.utils.ServerContext; import org.eclipse.jetty.servlet.ServletContextHandler; @@ -40,8 +41,8 @@ public static void start(ServletContextHandler contextHandler) { try { addServlets(contextHandler); - KruizeDeployment.getAutotuneObjects(kruizeDeployment); - PerformanceProfilesDeployment.getPerformanceProfiles(); + PerformanceProfilesDeployment.getPerformanceProfiles(); // Performance profile should be called first + KruizeDeployment.getKruizeObjects(kruizeDeployment); } catch (Exception e) { e.printStackTrace(); } diff --git a/src/main/java/com/autotune/analyzer/AutotuneExperiment.java b/src/main/java/com/autotune/analyzer/AutotuneExperiment.java index 0391d11ca..21baba59d 100644 --- a/src/main/java/com/autotune/analyzer/AutotuneExperiment.java +++ b/src/main/java/com/autotune/analyzer/AutotuneExperiment.java @@ -6,6 +6,8 @@ import com.autotune.common.experiments.ExperimentSummary; import com.autotune.common.experiments.ExperimentTrial; import com.autotune.common.experiments.TrialDetails; +import com.autotune.common.performanceProfiles.PerformanceProfile; +import com.autotune.common.performanceProfiles.PerformanceProfilesDeployment; import java.util.TreeMap; @@ -138,7 +140,9 @@ public void summarizeTrial(TrialDetails trialDetails) { ExperimentTrial bestExperimentTrial = getExperimentTrials().get(bestTrial); TrialDetails bestTrialDetails = bestExperimentTrial.getTrialDetails().get(TRAINING); double bestResult = Double.parseDouble(bestTrialDetails.getResult()); - String direction = kruizeObject.getSloInfo().getDirection(); + PerformanceProfile performanceProfile = PerformanceProfilesDeployment.performanceProfilesMap + .get(kruizeObject.getPerformanceProfile()); + String direction = performanceProfile.getSloInfo().getDirection(); if ((direction.equals(MINIMIZE) && currentResult < bestResult) || (direction.equals(MAXIMIZE) && currentResult > bestResult)) { experimentSummary.setBestTrial(currentTrial); diff --git a/src/main/java/com/autotune/analyzer/Experimentator.java b/src/main/java/com/autotune/analyzer/Experimentator.java index d67dc9726..749420af8 100644 --- a/src/main/java/com/autotune/analyzer/Experimentator.java +++ b/src/main/java/com/autotune/analyzer/Experimentator.java @@ -9,6 +9,8 @@ import com.autotune.common.k8sObjects.AutotuneConfig; import com.autotune.common.k8sObjects.KruizeObject; import com.autotune.common.k8sObjects.ObjectiveFunction; +import com.autotune.common.performanceProfiles.PerformanceProfile; +import com.autotune.common.performanceProfiles.PerformanceProfilesDeployment; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -103,9 +105,11 @@ private static ApplicationSearchSpace updateSearchSpace(AutotuneExperiment autot KruizeObject kruizeObject = autotuneExperiment.getAutotuneObject(); String experimentName = autotuneExperiment.getExperimentName(); String experimentId = kruizeObject.getExperimentId(); - ObjectiveFunction objectiveFunction = kruizeObject.getSloInfo().getObjectiveFunction(); - String hpoAlgoImpl = kruizeObject.getSloInfo().getHpoAlgoImpl(); - String direction = kruizeObject.getSloInfo().getDirection(); + PerformanceProfile performanceProfile = PerformanceProfilesDeployment.performanceProfilesMap + .get(kruizeObject.getPerformanceProfile()); + ObjectiveFunction objectiveFunction = performanceProfile.getSloInfo().getObjectiveFunction(); + String hpoAlgoImpl = kruizeObject.getHpoAlgoImpl(); + String direction = performanceProfile.getSloInfo().getDirection(); // TODO: Need to add valueType to the ObjectiveFunction! String valueType = "double"; diff --git a/src/main/java/com/autotune/analyzer/deployment/KruizeDeployment.java b/src/main/java/com/autotune/analyzer/deployment/KruizeDeployment.java index c38940102..29317d339 100644 --- a/src/main/java/com/autotune/analyzer/deployment/KruizeDeployment.java +++ b/src/main/java/com/autotune/analyzer/deployment/KruizeDeployment.java @@ -18,17 +18,16 @@ import com.autotune.analyzer.application.ApplicationDeployment; import com.autotune.analyzer.application.ApplicationServiceStack; import com.autotune.analyzer.application.Tunable; -import com.autotune.analyzer.exceptions.InvalidBoundsException; -import com.autotune.analyzer.exceptions.InvalidValueException; -import com.autotune.analyzer.exceptions.MonitoringAgentNotFoundException; -import com.autotune.analyzer.exceptions.MonitoringAgentNotSupportedException; +import com.autotune.analyzer.exceptions.*; import com.autotune.analyzer.utils.ExperimentInitiator; import com.autotune.analyzer.variables.Variables; import com.autotune.common.data.ValidationResultData; import com.autotune.common.data.datasource.DataSource; import com.autotune.common.data.datasource.DataSourceFactory; -import com.autotune.common.data.result.ExperimentResultData; import com.autotune.common.k8sObjects.*; +import com.autotune.common.performanceProfiles.PerformanceProfile; +import com.autotune.common.performanceProfiles.PerformanceProfileInterface.PerfProfileImpl; +import com.autotune.common.performanceProfiles.PerformanceProfilesDeployment; import com.autotune.common.target.kubernetes.service.KubernetesServices; import com.autotune.common.target.kubernetes.service.impl.KubernetesServicesImpl; import com.autotune.utils.AnalyzerConstants; @@ -57,6 +56,7 @@ import java.time.Clock; import java.util.*; import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Stream; import static com.autotune.utils.AnalyzerConstants.POD_TEMPLATE_HASH; @@ -88,7 +88,7 @@ public class KruizeDeployment { * @param kruizeDeployment * @throws IOException if unable to get Kubernetes config */ - public static void getAutotuneObjects(final KruizeDeployment kruizeDeployment) throws IOException { + public static void getKruizeObjects(final KruizeDeployment kruizeDeployment) throws IOException { /* Watch for events (additions, modifications or deletions) of autotune objects */ Watcher autotuneObjectWatcher = new Watcher<>() { @Override @@ -98,7 +98,7 @@ public void eventReceived(Action action, String resource) { switch (action.toString().toUpperCase()) { case "MODIFIED": case "ADDED": //TO DO consider MODIFIED after discussing PATCH request on already created experiments. - kruizeObject = getAutotuneObject(resource); + kruizeObject = getKruizeObject(resource); processKruizeObject(kruizeObject); break; case "DELETED": @@ -314,66 +314,48 @@ public static void matchPodsToAutotuneObject(KruizeObject kruizeObject) { } /** - * Add Autotune object to map of monitored objects. + * Add Kruize object to map of monitored objects. * * @param autotuneObjectJsonStr JSON string of the autotune object */ - private static KruizeObject getAutotuneObject(String autotuneObjectJsonStr) { + private static KruizeObject getKruizeObject(String autotuneObjectJsonStr) { try { JSONObject autotuneObjectJson = new JSONObject(autotuneObjectJsonStr); JSONObject metadataJson = autotuneObjectJson.getJSONObject(AnalyzerConstants.AutotuneObjectConstants.METADATA); - String name; + String perfProfileName = null; String mode; String targetCluster; - SloInfo sloInfo; - ObjectiveFunction objectiveFunction = null; String namespace; SelectorInfo selectorInfo; + JSONObject sloJson = null; + String hpoAlgoImpl = null; JSONObject specJson = autotuneObjectJson.optJSONObject(AnalyzerConstants.AutotuneObjectConstants.SPEC); - JSONObject sloJson = null; - String slo_class = null; - String direction = null; - JSONObject objectiveFunctionJson = null; - String hpoAlgoImpl = null; + mode = specJson.optString(AnalyzerConstants.AutotuneObjectConstants.MODE, + AnalyzerConstants.AutotuneObjectConstants.DEFAULT_MODE); + targetCluster = specJson.optString(AnalyzerConstants.AutotuneObjectConstants.TARGET_CLUSTER, + AnalyzerConstants.AutotuneObjectConstants.DEFAULT_TARGET_CLUSTER); + name = metadataJson.optString(AnalyzerConstants.AutotuneObjectConstants.NAME); + namespace = metadataJson.optString(AnalyzerConstants.AutotuneObjectConstants.NAMESPACE); + hpoAlgoImpl = specJson.optString(AnalyzerConstants.AutotuneObjectConstants.HPO_ALGO_IMPL, + AnalyzerConstants.AutotuneObjectConstants.DEFAULT_HPO_ALGO_IMPL); + if (specJson != null) { sloJson = specJson.optJSONObject(AnalyzerConstants.AutotuneObjectConstants.SLO); - slo_class = sloJson.optString(AnalyzerConstants.AutotuneObjectConstants.SLO_CLASS); - direction = sloJson.optString(AnalyzerConstants.AutotuneObjectConstants.DIRECTION); - hpoAlgoImpl = sloJson.optString(AnalyzerConstants.AutotuneObjectConstants.HPO_ALGO_IMPL, - AnalyzerConstants.AutotuneObjectConstants.DEFAULT_HPO_ALGO_IMPL); - objectiveFunctionJson = sloJson.optJSONObject(AnalyzerConstants.AutotuneObjectConstants.OBJECTIVE_FUNCTION); - objectiveFunction = new Gson().fromJson(String.valueOf(objectiveFunctionJson), ObjectiveFunction.class); - LOGGER.debug("Objective_Function = {}",objectiveFunction.toString()); - } + perfProfileName = specJson.optString(AnalyzerConstants.PerformanceProfileConstants.PERF_PROFILE); - JSONArray functionVariables = new JSONArray(); - if (sloJson != null) { - functionVariables = sloJson.getJSONArray(AnalyzerConstants.AutotuneObjectConstants.FUNCTION_VARIABLES); - } - ArrayList metricArrayList = new ArrayList<>(); - - for (Object functionVariableObj : functionVariables) { - JSONObject functionVariableJson = (JSONObject) functionVariableObj; - String variableName = functionVariableJson.optString(AnalyzerConstants.AutotuneObjectConstants.NAME); - String query = functionVariableJson.optString(AnalyzerConstants.AutotuneObjectConstants.QUERY); - String datasource = functionVariableJson.optString(AnalyzerConstants.AutotuneObjectConstants.DATASOURCE); - String valueType = functionVariableJson.optString(AnalyzerConstants.AutotuneObjectConstants.VALUE_TYPE); - - Metric metric = new Metric(variableName, - query, - datasource, - valueType); + if (Stream.of(sloJson, perfProfileName).allMatch(Objects::isNull)) { + throw new SloClassNotSupportedException(AnalyzerErrorConstants.AutotuneObjectErrors.MISSING_SLO_DATA); + } + if (Stream.of(sloJson, perfProfileName).noneMatch(Objects::isNull)) { + throw new SloClassNotSupportedException(AnalyzerErrorConstants.AutotuneObjectErrors.SLO_REDUNDANCY_ERROR); + } - metricArrayList.add(metric); + SloInfo sloInfo = new Gson().fromJson(String.valueOf(sloJson), SloInfo.class); + perfProfileName = setDefaultPerformanceProfile(sloInfo, mode, targetCluster); } - sloInfo = new SloInfo(slo_class, - objectiveFunction, - direction, - hpoAlgoImpl, - metricArrayList); JSONObject selectorJson = null; if (specJson != null) { @@ -393,13 +375,6 @@ private static KruizeObject getAutotuneObject(String autotuneObjectJsonStr) { matchURI, matchService); - mode = specJson.optString(AnalyzerConstants.AutotuneObjectConstants.MODE, - AnalyzerConstants.AutotuneObjectConstants.DEFAULT_MODE); - targetCluster = specJson.optString(AnalyzerConstants.AutotuneObjectConstants.TARGET_CLUSTER, - AnalyzerConstants.AutotuneObjectConstants.DEFAULT_TARGET_CLUSTER); - name = metadataJson.optString(AnalyzerConstants.AutotuneObjectConstants.NAME); - namespace = metadataJson.optString(AnalyzerConstants.AutotuneObjectConstants.NAMESPACE); - String resourceVersion = metadataJson.optString(AnalyzerConstants.RESOURCE_VERSION); String uid = metadataJson.optString(AnalyzerConstants.UID); String apiVersion = autotuneObjectJson.optString(AnalyzerConstants.API_VERSION); @@ -417,17 +392,44 @@ private static KruizeObject getAutotuneObject(String autotuneObjectJsonStr) { namespace, mode, targetCluster, - sloInfo, + hpoAlgoImpl, selectorInfo, + perfProfileName, objectReference ); - } catch (InvalidValueException | NullPointerException | JSONException e) { + } catch (InvalidValueException | NullPointerException | JSONException | SloClassNotSupportedException e) { LOGGER.error(e.getMessage()); new KubeEventLogger(Clock.systemUTC()).log("Failed", e.getMessage(), EventLogger.Type.Warning, null, null,null, null); return null; } } + public static String setDefaultPerformanceProfile(SloInfo sloInfo, String mode, String targetCluster) { + PerformanceProfile performanceProfile = null; + try { + String name = AnalyzerConstants.PerformanceProfileConstants.DEFAULT_PROFILE; + double profile_version = AnalyzerConstants.DEFAULT_PROFILE_VERSION; + String k8s_type = AnalyzerConstants.DEFAULT_K8S_TYPE; + performanceProfile = new PerformanceProfile(name, profile_version, k8s_type, sloInfo); + + if ( null != performanceProfile) { + ValidationResultData validationResultData = new PerfProfileImpl().validateAndAddProfile(PerformanceProfilesDeployment.performanceProfilesMap, performanceProfile); + if (validationResultData.isSuccess()) { + LOGGER.info("Added Performance Profile : {} into the map with version: {}", + performanceProfile.getName(), performanceProfile.getProfile_version()); + } else { + new KubeEventLogger(Clock.systemUTC()).log("Failed", validationResultData.getMessage(), EventLogger.Type.Warning, null, null, null, null); + } + } else { + new KubeEventLogger(Clock.systemUTC()).log("Failed", "Unable to create performance profile ", EventLogger.Type.Warning, null, null, null, null); + } + } catch (Exception e) { + LOGGER.debug("Exception while adding PP with message: {} ",e.getMessage()); + new KubeEventLogger(Clock.systemUTC()).log("Failed", e.getMessage(), EventLogger.Type.Warning, null, null, null, null); + return null; + } + return performanceProfile.getName(); + } /** * Parse AutotuneConfig JSON and create matching AutotuneConfig object diff --git a/src/main/java/com/autotune/analyzer/services/PerformanceProfileService.java b/src/main/java/com/autotune/analyzer/services/PerformanceProfileService.java index 234929505..e988bce69 100644 --- a/src/main/java/com/autotune/analyzer/services/PerformanceProfileService.java +++ b/src/main/java/com/autotune/analyzer/services/PerformanceProfileService.java @@ -18,10 +18,10 @@ import com.autotune.analyzer.exceptions.PerformanceProfileResponse; import com.autotune.analyzer.utils.GsonUTCDateAdapter; -import com.autotune.analyzer.utils.PerformanceProfileValidation; import com.autotune.common.data.ValidationResultData; import com.autotune.common.k8sObjects.Metric; import com.autotune.common.performanceProfiles.PerformanceProfile; +import com.autotune.common.performanceProfiles.PerformanceProfileInterface.PerfProfileImpl; import com.autotune.utils.AnalyzerConstants; import com.autotune.utils.AnalyzerErrorConstants; import com.google.gson.ExclusionStrategy; @@ -80,7 +80,7 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) try { String inputData = request.getReader().lines().collect(Collectors.joining()); PerformanceProfile performanceProfile = new Gson().fromJson(inputData, PerformanceProfile.class); - ValidationResultData validationResultData = new PerformanceProfileValidation(performanceProfilesMap).validate(performanceProfilesMap,performanceProfile); + ValidationResultData validationResultData = new PerfProfileImpl().validateAndAddProfile(performanceProfilesMap, performanceProfile); if (validationResultData.isSuccess()) { LOGGER.debug("Added Performance Profile : {} into the map with version: {}", performanceProfile.getName(), performanceProfile.getProfile_version()); diff --git a/src/main/java/com/autotune/analyzer/utils/ExperimentResultValidation.java b/src/main/java/com/autotune/analyzer/utils/ExperimentResultValidation.java index e196b30b9..bdf39b4d4 100644 --- a/src/main/java/com/autotune/analyzer/utils/ExperimentResultValidation.java +++ b/src/main/java/com/autotune/analyzer/utils/ExperimentResultValidation.java @@ -15,13 +15,13 @@ *******************************************************************************/ package com.autotune.analyzer.utils; -import com.autotune.analyzer.exceptions.InvalidValueException; import com.autotune.common.data.ValidationResultData; import com.autotune.common.data.result.ExperimentResultData; import com.autotune.common.k8sObjects.KruizeObject; import com.autotune.common.k8sObjects.SloInfo; import com.autotune.common.performanceProfiles.PerformanceProfile; -import com.autotune.common.performanceProfiles.PerformanceProfileInterface.DefaultOpenshiftImpl; +import com.autotune.common.performanceProfiles.PerformanceProfileInterface.DefaultImpl; +import com.autotune.common.performanceProfiles.PerformanceProfileInterface.PerfProfileImpl; import com.autotune.utils.AnalyzerConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,44 +68,32 @@ public void validate(List experimentResultDataList, Map validationClass = null; - Class[] parameterTypes = null; - Method method = null; - Object object = null; - // check if performance profile is absent and then attach a default one - if (null == performanceProfile) { - SloInfo sloInfo = kruizeObject.getSloInfo(); - performanceProfile = new PerformanceProfile(AnalyzerConstants.PerformanceProfileConstants.DEFAULT_PROFILE,AnalyzerConstants.DEFAULT_PROFILE_VERSION, AnalyzerConstants.DEFAULT_K8S_TYPE, sloInfo); - kruizeObject.setPerformanceProfile(performanceProfile.getName()); - errorMsg = new DefaultOpenshiftImpl().validate(performanceProfile, resultData); - } else { - // Get the corresponding class for validating the performance profile - String validationClassName = AnalyzerConstants.PerformanceProfileConstants.PERFORMANCE_PROFILE_PKG - .concat(getValidationClass(performanceProfile.getName())); - validationClass = Class.forName(validationClassName); - - object = validationClass.getDeclaredConstructor().newInstance(); - parameterTypes = new Class[] { PerformanceProfile.class, ExperimentResultData.class }; - method = validationClass.getMethod("validate",parameterTypes); - errorMsg = (String) method.invoke(object, performanceProfile, resultData); - } + // validate the 'resultdata' with the performance profile + errorMsg = new PerfProfileImpl().validateResults(performanceProfile,resultData); if (errorMsg.isEmpty()) { - // call recommend() method here - if ( null != method) { - method = validationClass.getMethod("recommend",parameterTypes); + if (performanceProfile.getName().equalsIgnoreCase(AnalyzerConstants.PerformanceProfileConstants.DEFAULT_PROFILE)) { + errorMsg = new DefaultImpl().recommend(performanceProfile, resultData); + } else { + // check the performance profile and instantiate corresponding class for parsing + String validationClassName = AnalyzerConstants.PerformanceProfileConstants + .PERFORMANCE_PROFILE_PKG.concat(new PerfProfileImpl().getName(performanceProfile)); + Class validationClass = Class.forName(validationClassName); + Object object = validationClass.getDeclaredConstructor().newInstance(); + Class[] parameterTypes = new Class[] { PerformanceProfile.class, ExperimentResultData.class }; + Method method = validationClass.getMethod("recommend",parameterTypes); errorMsg = (String) method.invoke(object, performanceProfile, resultData); - } else - errorMsg = new DefaultOpenshiftImpl().recommend(performanceProfile, resultData); - - proceed = true; + } + if (errorMsg.isEmpty()) + proceed = true; } else { proceed = false; resultData.setValidationResultData(new ValidationResultData(false, errorMsg)); break; } } catch (NullPointerException | ClassNotFoundException | NoSuchMethodException | - IllegalAccessException | InvocationTargetException | InvalidValueException e) { + IllegalAccessException | InvocationTargetException e) { LOGGER.error("Caught Exception: {}",e); errorMsg = "Validation failed due to : " + e.getMessage(); proceed = false; diff --git a/src/main/java/com/autotune/analyzer/utils/ExperimentValidation.java b/src/main/java/com/autotune/analyzer/utils/ExperimentValidation.java index 47a2b4c58..129383ace 100644 --- a/src/main/java/com/autotune/analyzer/utils/ExperimentValidation.java +++ b/src/main/java/com/autotune/analyzer/utils/ExperimentValidation.java @@ -15,19 +15,21 @@ *******************************************************************************/ package com.autotune.analyzer.utils; +import com.autotune.analyzer.deployment.KruizeDeployment; +import com.autotune.analyzer.exceptions.SloClassNotSupportedException; import com.autotune.common.data.ValidationResultData; import com.autotune.common.k8sObjects.KruizeObject; +import com.autotune.common.k8sObjects.SloInfo; import com.autotune.common.performanceProfiles.PerformanceProfilesDeployment; import com.autotune.utils.AnalyzerConstants; +import com.autotune.utils.AnalyzerErrorConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.stream.Stream; /** * create Experiment input validation @@ -47,10 +49,6 @@ public class ExperimentValidation { private List mandatoryFieldsForLocalRemoteMonitoring = new ArrayList<>(( Arrays.asList(AnalyzerConstants.RECOMMENDATION_SETTINGS) )); - private List mandatorySLOPerf = new ArrayList<>(Arrays.asList( - AnalyzerConstants.SLO, - AnalyzerConstants.PerformanceProfileConstants.PERF_PROFILE - )); private List mandatoryDeploymentSelector = new ArrayList<>(Arrays.asList( AnalyzerConstants.DEPLOYMENT_NAME, AnalyzerConstants.SELECTOR @@ -86,13 +84,22 @@ public void validate(List kruizeExptList) { if (null != ao.getDeployment_name()) { String nsDepName = ao.getNamespace().toLowerCase() + ":" + ao.getDeployment_name().toLowerCase(); if (!namespaceDeploymentNameList.contains(nsDepName)) { - if (null != PerformanceProfilesDeployment.performanceProfilesMap.get(ao.getPerformanceProfile())) - proceed = true; - else { - errorMsg = errorMsg.concat(String.format("Performance Profile : %s does not exist!", ao.getPerformanceProfile())); + if (null != ao.getPerformanceProfile()) { + if (null != PerformanceProfilesDeployment.performanceProfilesMap.get(ao.getPerformanceProfile())) + proceed = true; + else { + errorMsg = errorMsg.concat(String.format("Performance Profile : %s does not exist!", ao.getPerformanceProfile())); + } + } else { + if (null == ao.getSloInfo()) + errorMsg = AnalyzerErrorConstants.AutotuneObjectErrors.MISSING_SLO_DATA; + else { + String perfProfileName = KruizeDeployment.setDefaultPerformanceProfile(ao.getSloInfo(), mode, target_cluster); + ao.setPerformanceProfile(perfProfileName); + proceed = true; + } } - } - else { + } else { if (!ao.getExperimentUseCaseType().isRemoteMonitoring()) errorMsg = errorMsg.concat(String.format("Experiment name : %s with Deployment name : %s is duplicate", expName, nsDepName)); else @@ -186,17 +193,6 @@ public ValidationResultData validateMandatoryFields(KruizeObject expObj) { } ); } - for (String mField : mandatorySLOPerf) { - String methodName = "get" + mField.substring(0, 1).toUpperCase() + mField.substring(1); - try { - Method getNameMethod = KruizeObject.class.getMethod(methodName); - if (getNameMethod.invoke(expObj) != null) { - missingSLOPerf = false; - } - } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { - //LOGGER.warn("Methode name for {} not exsist and error is {}", mField, e.getMessage()); - } - } for (String mField : mandatoryDeploymentSelector) { String methodName = "get" + mField.substring(0, 1).toUpperCase() + mField.substring(1); try { @@ -208,13 +204,10 @@ public ValidationResultData validateMandatoryFields(KruizeObject expObj) { //LOGGER.warn("Methode name for {} not exist and error is {}", mField, e.getMessage()); } } - if (missingSLOPerf || missingDeploySelector) { - if (missingSLOPerf) { - errorMsg = errorMsg.concat(String.format("Either one of the parameter should present %s ", mandatorySLOPerf)); - } - if (missingDeploySelector) { - errorMsg = errorMsg.concat(String.format("Either one of the parameter should present %s ", mandatoryDeploymentSelector)); - } + + if (missingDeploySelector) { + errorMsg = errorMsg.concat(String.format("Either one of the parameter should present %s ", mandatoryDeploymentSelector)); + validationResultData.setSuccess(false); validationResultData.setMessage(errorMsg); } else { diff --git a/src/main/java/com/autotune/analyzer/utils/PerformanceProfileValidation.java b/src/main/java/com/autotune/analyzer/utils/PerformanceProfileValidation.java index aff54e2ea..c1ebc61f6 100644 --- a/src/main/java/com/autotune/analyzer/utils/PerformanceProfileValidation.java +++ b/src/main/java/com/autotune/analyzer/utils/PerformanceProfileValidation.java @@ -15,9 +15,16 @@ *******************************************************************************/ package com.autotune.analyzer.utils; +import com.autotune.analyzer.exceptions.InvalidValueException; import com.autotune.common.data.ValidationResultData; +import com.autotune.common.k8sObjects.Metric; +import com.autotune.common.k8sObjects.SloInfo; import com.autotune.common.performanceProfiles.PerformanceProfile; +import com.autotune.experimentManager.utils.EMConstants; import com.autotune.utils.AnalyzerConstants; +import com.autotune.utils.AnalyzerErrorConstants; +import com.autotune.utils.AutotuneConstants; +import com.autotune.utils.AutotuneSupportedTypes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,28 +70,18 @@ public PerformanceProfileValidation(Map performanceP /** * Validates function variables * - * @param performanceProfilesMap Map which contains the profiles and in which new profiles will be added post validation. * @param performanceProfile Performance Profile Object to be validated * @return Returns the ValidationResultData containing the response based on the validation */ - public ValidationResultData validate(Map performanceProfilesMap, PerformanceProfile performanceProfile) { - - ValidationResultData validationResultData = validateMandatoryFields(performanceProfile); - String errorMsg = ""; - if (validationResultData.isSuccess()) { - String perfProfileName = performanceProfile.getName(); - if (null == performanceProfilesMap.get(perfProfileName)) { - LOGGER.debug("Performance Profile {} doesn't exist, Proceeding to create a new one..",perfProfileName); - addPerformanceProfile(performanceProfilesMap, performanceProfile); - setSuccess(true); - }else { - errorMsg = errorMsg.concat(String.format("Performance Profile name : %s is duplicate", perfProfileName)); - validationResultData.setMessage(errorMsg); - validationResultData.setSuccess(false); - } - } - else + public ValidationResultData validate(PerformanceProfile performanceProfile) { + + ValidationResultData validationResultData = validateMandatoryFieldsAndData(performanceProfile); + if (validationResultData.isSuccess()) + setSuccess(true); + else { markFailed(validationResultData.getMessage()); + LOGGER.debug("Performance profile validation failed due to : {}",validationResultData.getMessage()); + } return validationResultData; } @@ -96,10 +93,13 @@ public void setSuccess(boolean success) { this.success = success; } + public String getErrorMessage() { + return errorMessage; + } + public void setErrorMessage(String errorMessage) { this.errorMessage = errorMessage; } - public void markFailed(String message) { setSuccess(false); setErrorMessage(message); @@ -111,7 +111,7 @@ public void markFailed(String message) { * @param perfObj Mandatory fields of this Performance Profile Object will be validated * @return ValidationResultData object containing status of the validations */ - public ValidationResultData validateMandatoryFields(PerformanceProfile perfObj) { + public ValidationResultData validateMandatoryFieldsAndData(PerformanceProfile perfObj) { List missingMandatoryFields = new ArrayList<>(); ValidationResultData validationResultData = new ValidationResultData(false, null); String errorMsg; @@ -186,12 +186,140 @@ public ValidationResultData validateMandatoryFields(PerformanceProfile perfObj) LOGGER.debug("Validation error message :{}", errorMsg); } LOGGER.debug("{}", validationResultData); + + if (validationResultData.isSuccess()) + validationResultData = validatePerformanceProfileData(perfObj); + return validationResultData; } - public static void addPerformanceProfile(Map performanceProfileMap, PerformanceProfile performanceProfile) { - performanceProfileMap.put(performanceProfile.getName(), performanceProfile); - LOGGER.debug("Added PerformanceProfile {}: ",performanceProfile.getName()); + /** + * Validates the data present in the performance profile object before adding it to the map + * @param performanceProfile + * @return + */ + private ValidationResultData validatePerformanceProfileData(PerformanceProfile performanceProfile) { + ValidationResultData validationResultData = new ValidationResultData(false, null); + StringBuilder errorString = new StringBuilder(); + // check if the performance profile is duplicate + if (performanceProfilesMap.get(performanceProfile.getName()) != null) { + errorString.append(AnalyzerErrorConstants.AutotuneObjectErrors.DUPLICATE_PERF_PROFILE); + validationResultData.setMessage(errorString.toString()); + return validationResultData; + } + // Check if k8s type is supported + String k8sType = performanceProfile.getK8S_TYPE(); + if (!AutotuneSupportedTypes.K8S_TYPES_SUPPORTED.contains(k8sType)) { + errorString.append(AnalyzerConstants.PerformanceProfileConstants.K8S_TYPE).append(k8sType) + .append(AnalyzerErrorConstants.AutotuneObjectErrors.UNSUPPORTED); + } + + SloInfo sloInfo = performanceProfile.getSloInfo(); + // Check if direction is supported + if (!AutotuneSupportedTypes.DIRECTIONS_SUPPORTED.contains(sloInfo.getDirection())) + errorString.append(AnalyzerErrorConstants.AutotuneObjectErrors.DIRECTION_NOT_SUPPORTED); + // if slo_class is present, do further validations + if (sloInfo.getSloClass() != null) { + // Check if slo_class is supported + if (!AutotuneSupportedTypes.SLO_CLASSES_SUPPORTED.contains(sloInfo.getSloClass())) + errorString.append(AnalyzerErrorConstants.AutotuneObjectErrors.SLO_CLASS_NOT_SUPPORTED); + + //check if slo_class is 'response_time' and direction is 'minimize' + if (sloInfo.getSloClass().equalsIgnoreCase(EMConstants.StandardDefaults.RESPONSE_TIME) && !sloInfo.getDirection() + .equalsIgnoreCase(AnalyzerConstants.AutotuneObjectConstants.MINIMIZE)) { + errorString.append(AnalyzerErrorConstants.AutotuneObjectErrors.INVALID_DIRECTION_FOR_SLO_CLASS); + } + + //check if slo_class is 'throughput' and direction is 'maximize' + if (sloInfo.getSloClass().equalsIgnoreCase(EMConstants.StandardDefaults.THROUGHPUT) && !sloInfo.getDirection() + .equalsIgnoreCase(AnalyzerConstants.AutotuneObjectConstants.MAXIMIZE)) { + errorString.append(AnalyzerErrorConstants.AutotuneObjectErrors.INVALID_DIRECTION_FOR_SLO_CLASS); + } + } + // Check if function_variables is empty + if (sloInfo.getFunctionVariables().isEmpty()) + errorString.append(AnalyzerErrorConstants.AutotuneObjectErrors.FUNCTION_VARIABLES_EMPTY); + + // Check if objective_function and it's type exists + if (sloInfo.getObjectiveFunction() == null) + errorString.append(AnalyzerErrorConstants.AutotuneObjectErrors.OBJECTIVE_FUNCTION_MISSING); + + // Get the objective_function type + String objFunctionType = sloInfo.getObjectiveFunction().getFunction_type(); + String expression = null; + for (Metric functionVariable : sloInfo.getFunctionVariables()) { + // Check if datasource is supported + if (!AutotuneSupportedTypes.MONITORING_AGENTS_SUPPORTED.contains(functionVariable.getDatasource().toLowerCase())) { + errorString.append(AnalyzerConstants.AutotuneObjectConstants.FUNCTION_VARIABLE) + .append(functionVariable.getName()) + .append(AnalyzerErrorConstants.AutotuneObjectErrors.DATASOURCE_NOT_SUPPORTED); + } + + // Check if value_type is supported + if (!AutotuneSupportedTypes.VALUE_TYPES_SUPPORTED.contains(functionVariable.getValueType().toLowerCase())) { + errorString.append(AnalyzerConstants.AutotuneObjectConstants.FUNCTION_VARIABLE) + .append(functionVariable.getName()) + .append(AnalyzerErrorConstants.AutotuneObjectErrors.VALUE_TYPE_NOT_SUPPORTED); + } + + // Check if kubernetes_object type is supported, set default to 'container' if it's absent. + String kubernetes_object = functionVariable.getKubernetesObject(); + if (null == kubernetes_object) + functionVariable.setKubernetesObject(AutotuneConstants.JSONKeys.CONTAINER); + else { + if (!AutotuneSupportedTypes.KUBERNETES_OBJECTS_SUPPORTED.contains(kubernetes_object.toLowerCase())) + errorString.append(AnalyzerConstants.KUBERNETES_OBJECTS).append(kubernetes_object) + .append(AnalyzerErrorConstants.AutotuneObjectErrors.UNSUPPORTED); + } + + // Validate Objective Function + try { + if (objFunctionType.equals(AnalyzerConstants.AutotuneObjectConstants.EXPRESSION)) { + + expression = sloInfo.getObjectiveFunction().getExpression(); + if (null == expression || expression.equals(AnalyzerConstants.NULL)) { + throw new NullPointerException(AnalyzerErrorConstants.AutotuneObjectErrors.MISSING_EXPRESSION); + } + + } else if (objFunctionType.equals(AnalyzerConstants.PerformanceProfileConstants.SOURCE)) { + if (null != sloInfo.getObjectiveFunction().getExpression()) { + errorString.append(AnalyzerErrorConstants.AutotuneObjectErrors.MISPLACED_EXPRESSION); + throw new InvalidValueException(errorString.toString()); + } + } else { + errorString.append(AnalyzerErrorConstants.AutotuneObjectErrors.INVALID_TYPE); + throw new InvalidValueException(errorString.toString()); + } + } catch (NullPointerException | InvalidValueException npe) { + errorString.append(npe.getMessage()); + validationResultData.setSuccess(false); + validationResultData.setMessage(errorString.toString()); + } + + // Check if function_variable is part of objective_function + if (objFunctionType.equals(AnalyzerConstants.AutotuneObjectConstants.EXPRESSION)) { + if (!expression.contains(functionVariable.getName())) { + errorString.append(AnalyzerConstants.AutotuneObjectConstants.FUNCTION_VARIABLE) + .append(functionVariable.getName()).append(" ") + .append(AnalyzerErrorConstants.AutotuneObjectErrors.FUNCTION_VARIABLE_ERROR); + } + } + } + + // Check if objective_function is correctly formatted + if (objFunctionType.equals(AnalyzerConstants.AutotuneObjectConstants.EXPRESSION)) { + if (expression.equals(AnalyzerConstants.NULL) || !new EvalExParser().validate(sloInfo.getObjectiveFunction().getExpression(), sloInfo.getFunctionVariables())) { + errorString.append(AnalyzerErrorConstants.AutotuneObjectErrors.INVALID_OBJECTIVE_FUNCTION); + } + } + + if (!errorString.isEmpty()) { + validationResultData.setSuccess(false); + validationResultData.setMessage(errorString.toString()); + } else + validationResultData.setSuccess(true); + + return validationResultData; } @Override diff --git a/src/main/java/com/autotune/analyzer/utils/ServiceHelpers.java b/src/main/java/com/autotune/analyzer/utils/ServiceHelpers.java index 2e3e08b66..e6ec536ab 100644 --- a/src/main/java/com/autotune/analyzer/utils/ServiceHelpers.java +++ b/src/main/java/com/autotune/analyzer/utils/ServiceHelpers.java @@ -23,6 +23,8 @@ import com.autotune.common.k8sObjects.AutotuneConfig; import com.autotune.common.k8sObjects.KruizeObject; import com.autotune.common.k8sObjects.Metric; +import com.autotune.common.performanceProfiles.PerformanceProfile; +import com.autotune.common.performanceProfiles.PerformanceProfilesDeployment; import com.autotune.utils.AnalyzerConstants; import org.json.JSONArray; import org.json.JSONObject; @@ -44,12 +46,14 @@ private ServiceHelpers() { * @param kruizeObject */ public static void addExperimentDetails(JSONObject experimentJson, KruizeObject kruizeObject) { + PerformanceProfile performanceProfile = PerformanceProfilesDeployment.performanceProfilesMap + .get(kruizeObject.getPerformanceProfile()); experimentJson.put(AnalyzerConstants.ServiceConstants.EXPERIMENT_NAME, kruizeObject.getExperimentName()); - experimentJson.put(AnalyzerConstants.AutotuneObjectConstants.DIRECTION, kruizeObject.getSloInfo().getDirection()); - experimentJson.put(AnalyzerConstants.AutotuneObjectConstants.OBJECTIVE_FUNCTION, kruizeObject.getSloInfo().getObjectiveFunction()); - experimentJson.put(AnalyzerConstants.AutotuneObjectConstants.SLO_CLASS, kruizeObject.getSloInfo().getSloClass()); + experimentJson.put(AnalyzerConstants.AutotuneObjectConstants.DIRECTION, performanceProfile.getSloInfo().getDirection()); + experimentJson.put(AnalyzerConstants.AutotuneObjectConstants.OBJECTIVE_FUNCTION, performanceProfile.getSloInfo().getObjectiveFunction()); + experimentJson.put(AnalyzerConstants.AutotuneObjectConstants.SLO_CLASS, performanceProfile.getSloInfo().getSloClass()); experimentJson.put(AnalyzerConstants.AutotuneObjectConstants.EXPERIMENT_ID, kruizeObject.getExperimentId()); - experimentJson.put(AnalyzerConstants.AutotuneObjectConstants.HPO_ALGO_IMPL, kruizeObject.getSloInfo().getHpoAlgoImpl()); + experimentJson.put(AnalyzerConstants.AutotuneObjectConstants.HPO_ALGO_IMPL, kruizeObject.getHpoAlgoImpl()); experimentJson.put(AnalyzerConstants.AutotuneObjectConstants.NAMESPACE, kruizeObject.getNamespace()); } @@ -153,7 +157,9 @@ public static void addLayerTunableDetails(JSONArray tunablesArray, AutotuneConfi public static void addFunctionVariablesDetails(JSONObject funcVarJson, KruizeObject kruizeObject) { // Add function_variables info JSONArray functionVariablesArray = new JSONArray(); - for (Metric functionVariable : kruizeObject.getSloInfo().getFunctionVariables()) { + PerformanceProfile performanceProfile = PerformanceProfilesDeployment.performanceProfilesMap + .get(kruizeObject.getPerformanceProfile()); + for (Metric functionVariable : performanceProfile.getSloInfo().getFunctionVariables()) { JSONObject functionVariableJson = new JSONObject(); functionVariableJson.put(AnalyzerConstants.AutotuneObjectConstants.NAME, functionVariable.getName()); functionVariableJson.put(AnalyzerConstants.AutotuneObjectConstants.VALUE_TYPE, functionVariable.getValueType()); diff --git a/src/main/java/com/autotune/common/k8sObjects/KruizeObject.java b/src/main/java/com/autotune/common/k8sObjects/KruizeObject.java index a2dbddda4..d13cbb407 100644 --- a/src/main/java/com/autotune/common/k8sObjects/KruizeObject.java +++ b/src/main/java/com/autotune/common/k8sObjects/KruizeObject.java @@ -20,6 +20,7 @@ import com.autotune.common.data.ValidationResultData; import com.autotune.common.data.result.ExperimentResultData; import com.autotune.utils.AnalyzerConstants; +import com.autotune.utils.AutotuneSupportedTypes; import com.autotune.utils.Utils; import com.google.gson.annotations.SerializedName; import io.fabric8.kubernetes.api.model.ObjectReference; @@ -42,6 +43,7 @@ public final class KruizeObject { private String targetCluster; //Todo convert into Enum @SerializedName("slo") private SloInfo sloInfo; + private String hpoAlgoImpl; @SerializedName("selector") private SelectorInfo selectorInfo; private ObjectReference objectReference; @@ -59,8 +61,9 @@ public KruizeObject(String experimentName, String namespace, String mode, String targetCluster, - SloInfo sloInfo, + String hpoAlgoImpl, SelectorInfo selectorInfo, + String performanceProfile, ObjectReference objectReference) throws InvalidValueException { HashMap map = new HashMap<>(); @@ -68,7 +71,6 @@ public KruizeObject(String experimentName, map.put(AnalyzerConstants.AutotuneObjectConstants.NAMESPACE, namespace); map.put(AnalyzerConstants.AutotuneObjectConstants.MODE, mode); map.put(AnalyzerConstants.AutotuneObjectConstants.TARGET_CLUSTER, targetCluster); - map.put(AnalyzerConstants.AutotuneObjectConstants.SLO, sloInfo); map.put(AnalyzerConstants.AutotuneObjectConstants.SELECTOR, selectorInfo); StringBuilder error = ValidateAutotuneObject.validate(map); @@ -77,13 +79,18 @@ public KruizeObject(String experimentName, this.namespace = namespace; this.mode = mode; this.targetCluster = targetCluster; - this.sloInfo = sloInfo; this.selectorInfo = selectorInfo; this.experimentId = Utils.generateID(toString()); this.objectReference = objectReference; } else { throw new InvalidValueException(error.toString()); } + this.performanceProfile = performanceProfile; + if (AutotuneSupportedTypes.HPO_ALGOS_SUPPORTED.contains(hpoAlgoImpl)) + this.hpoAlgoImpl = hpoAlgoImpl; + else + throw new InvalidValueException("Hyperparameter Optimization Algorithm " + hpoAlgoImpl + " not supported"); + } public KruizeObject() { @@ -99,7 +106,7 @@ public void setExperimentName(String experimentName) { } public SloInfo getSloInfo() { - return new SloInfo(sloInfo); + return sloInfo; } public void setSloInfo(SloInfo sloInfo) { @@ -226,6 +233,10 @@ public void setValidationData(ValidationResultData validationData) { this.validationData = validationData; } + public String getHpoAlgoImpl() { + return hpoAlgoImpl; + } + @Override public String toString() { return "KruizeObject{" + @@ -234,7 +245,7 @@ public String toString() { ", namespace='" + namespace + '\'' + ", mode='" + mode + '\'' + ", targetCluster='" + targetCluster + '\'' + - ", sloInfo=" + sloInfo + + ", hpoAlgoImpl=" + hpoAlgoImpl + ", selectorInfo=" + selectorInfo + ", objectReference=" + objectReference + ", status=" + status + diff --git a/src/main/java/com/autotune/common/k8sObjects/SloInfo.java b/src/main/java/com/autotune/common/k8sObjects/SloInfo.java index f10b29e67..83f6abb0a 100644 --- a/src/main/java/com/autotune/common/k8sObjects/SloInfo.java +++ b/src/main/java/com/autotune/common/k8sObjects/SloInfo.java @@ -16,7 +16,6 @@ package com.autotune.common.k8sObjects; import com.autotune.analyzer.exceptions.InvalidValueException; -import com.autotune.utils.AutotuneSupportedTypes; import com.google.gson.annotations.SerializedName; import java.util.ArrayList; @@ -31,7 +30,6 @@ * expression: "transaction_response_time" * slo_class: "response_time" * direction: "minimize" - * hpo_algo_impl: "optuna_tpe" * function_variables: * - name: "transaction_response_time" * query: "application_org_acme_microprofile_metrics_PrimeNumberChecker_checksTimer_mean_seconds" @@ -47,31 +45,16 @@ public final class SloInfo { @SerializedName("objective_function") private final ObjectiveFunction objectiveFunction; private final String direction; - private final String hpoAlgoImpl; @SerializedName("function_variables") private final ArrayList metrics; public SloInfo(String sloClass, ObjectiveFunction objectiveFunction, String direction, - String hpoAlgoImpl, ArrayList metrics) throws InvalidValueException { - if (AutotuneSupportedTypes.SLO_CLASSES_SUPPORTED.contains(sloClass)) - this.sloClass = sloClass; - else - throw new InvalidValueException("slo_class: " + sloClass + " not supported"); + this.sloClass = sloClass; this.objectiveFunction = objectiveFunction; - - if (AutotuneSupportedTypes.DIRECTIONS_SUPPORTED.contains(direction)) - this.direction = direction; - else - throw new InvalidValueException("Invalid direction for autotune kind"); - - if (AutotuneSupportedTypes.HPO_ALGOS_SUPPORTED.contains(hpoAlgoImpl)) - this.hpoAlgoImpl = hpoAlgoImpl; - else - throw new InvalidValueException("Hyperparameter Optimization Algorithm " + hpoAlgoImpl + " not supported"); - + this.direction = direction; this.metrics = new ArrayList<>(metrics); } @@ -79,10 +62,7 @@ public SloInfo(SloInfo copy) { this.sloClass = copy.getSloClass(); this.objectiveFunction = copy.getObjectiveFunction(); this.direction = copy.getDirection(); - this.hpoAlgoImpl = copy.getHpoAlgoImpl(); - this.metrics = new ArrayList<>(copy.getFunctionVariables()); - } public String getSloClass() { @@ -101,17 +81,12 @@ public ArrayList getFunctionVariables() { return new ArrayList<>(metrics); } - public String getHpoAlgoImpl() { - return hpoAlgoImpl; - } - @Override public String toString() { return "SloInfo{" + "sloClass='" + sloClass + '\'' + ", objectiveFunction='" + objectiveFunction + '\'' + ", direction='" + direction + '\'' + - ", hpoAlgoImpl='" + this.hpoAlgoImpl + '\'' + ", functionVariables=" + metrics + '}'; } diff --git a/src/main/java/com/autotune/common/k8sObjects/ValidateAutotuneObject.java b/src/main/java/com/autotune/common/k8sObjects/ValidateAutotuneObject.java index 8da01ca4f..d2c4a1da3 100644 --- a/src/main/java/com/autotune/common/k8sObjects/ValidateAutotuneObject.java +++ b/src/main/java/com/autotune/common/k8sObjects/ValidateAutotuneObject.java @@ -65,97 +65,6 @@ public static StringBuilder validate(HashMap map) { if (selectorInfo.getMatchLabelValue() == null || selectorInfo.getMatchLabelValue().trim().isEmpty()) { errorString.append(AnalyzerErrorConstants.AutotuneObjectErrors.INVALID_MATCHLABEL_VALUE); } - - // Check if slo_class is supported - SloInfo sloInfo = (SloInfo) map.get(AnalyzerConstants.AutotuneObjectConstants.SLO); - if (!AutotuneSupportedTypes.SLO_CLASSES_SUPPORTED.contains(sloInfo.getSloClass())) { - errorString.append(AnalyzerErrorConstants.AutotuneObjectErrors.SLO_CLASS_NOT_SUPPORTED); - } - - // Check if direction is supported - if (!AutotuneSupportedTypes.DIRECTIONS_SUPPORTED.contains(sloInfo.getDirection())) { - errorString.append(AnalyzerErrorConstants.AutotuneObjectErrors.DIRECTION_NOT_SUPPORTED); - } - - //check if slo_class is 'response_time' and direction is minimize - if (sloInfo.getSloClass().equalsIgnoreCase(EMConstants.StandardDefaults.RESPONSE_TIME) && !sloInfo.getDirection().equalsIgnoreCase(AnalyzerConstants.AutotuneObjectConstants.MINIMIZE)) { - errorString.append(AnalyzerErrorConstants.AutotuneObjectErrors.INVALID_DIRECTION_FOR_SLO_CLASS); - } - - //check if slo_class is 'throughput' and direction is maximize - if (sloInfo.getSloClass().equalsIgnoreCase(EMConstants.StandardDefaults.THROUGHPUT) && !sloInfo.getDirection().equalsIgnoreCase(AnalyzerConstants.AutotuneObjectConstants.MAXIMIZE)) { - errorString.append(AnalyzerErrorConstants.AutotuneObjectErrors.INVALID_DIRECTION_FOR_SLO_CLASS); - } - - - // Check if hpo_algo_impl is supported - if (!AutotuneSupportedTypes.HPO_ALGOS_SUPPORTED.contains(sloInfo.getHpoAlgoImpl())) { - errorString.append(AnalyzerErrorConstants.AutotuneObjectErrors.HPO_ALGO_NOT_SUPPORTED); - } - - // Check if objective_function and it's type exists - if (sloInfo.getObjectiveFunction() == null) { - errorString.append(AnalyzerErrorConstants.AutotuneObjectErrors.OBJECTIVE_FUNCTION_MISSING); - } - - // Check if function_variables is empty - if (sloInfo.getFunctionVariables().isEmpty()) { - errorString.append(AnalyzerErrorConstants.AutotuneObjectErrors.FUNCTION_VARIABLES_EMPTY); - } - // Get the objective_function type - String objFunctionType = sloInfo.getObjectiveFunction().getFunction_type(); - String expression = null; - for (Metric functionVariable : sloInfo.getFunctionVariables()) { - // Check if datasource is supported - if (!AutotuneSupportedTypes.MONITORING_AGENTS_SUPPORTED.contains(functionVariable.getDatasource().toLowerCase())) - errorString.append(AnalyzerConstants.AutotuneObjectConstants.FUNCTION_VARIABLE).append(functionVariable.getName()) - .append(AnalyzerErrorConstants.AutotuneObjectErrors.DATASOURCE_NOT_SUPPORTED); - - // Check if value_type is supported - if (!AutotuneSupportedTypes.VALUE_TYPES_SUPPORTED.contains(functionVariable.getValueType().toLowerCase())) - errorString.append(AnalyzerConstants.AutotuneObjectConstants.FUNCTION_VARIABLE).append(functionVariable.getName()) - .append(AnalyzerErrorConstants.AutotuneObjectErrors.VALUE_TYPE_NOT_SUPPORTED); - - // Validate Objective Function - try { - if (objFunctionType.equals(AnalyzerConstants.AutotuneObjectConstants.EXPRESSION)) { - - expression = sloInfo.getObjectiveFunction().getExpression(); - System.out.println("****** Exprssion = "+expression); - if (null == expression || expression.equals(AnalyzerConstants.NULL)) - throw new NullPointerException(AnalyzerErrorConstants.AutotuneObjectErrors.MISSING_EXPRESSION); - - } else if (objFunctionType.equals(AnalyzerConstants.PerformanceProfileConstants.SOURCE)) { - if (null != sloInfo.getObjectiveFunction().getExpression()) { - errorString.append(AnalyzerErrorConstants.AutotuneObjectErrors.MISPLACED_EXPRESSION); - throw new InvalidValueException(errorString.toString()); - } - } else { - errorString.append(AnalyzerErrorConstants.AutotuneObjectErrors.INVALID_TYPE); - throw new InvalidValueException(errorString.toString()); - } - } catch (NullPointerException | InvalidValueException npe) { - errorString.append(npe.getMessage()); - return errorString; - } - - // Check if function_variable is part of objective_function - if (objFunctionType.equals(AnalyzerConstants.AutotuneObjectConstants.EXPRESSION)) { - if (!expression.contains(functionVariable.getName())) - errorString.append(AnalyzerConstants.AutotuneObjectConstants.FUNCTION_VARIABLE) - .append(functionVariable.getName()).append(" ") - .append(AnalyzerErrorConstants.AutotuneObjectErrors.FUNCTION_VARIABLE_ERROR); - return errorString; - } - } - - // Check if objective_function is correctly formatted - if (objFunctionType.equals(AnalyzerConstants.AutotuneObjectConstants.EXPRESSION)) { - if (expression.equals(AnalyzerConstants.NULL) || !new EvalExParser().validate(sloInfo.getObjectiveFunction().getExpression(), sloInfo.getFunctionVariables())) { - errorString.append(AnalyzerErrorConstants.AutotuneObjectErrors.INVALID_OBJECTIVE_FUNCTION); - } - } - return errorString; } } diff --git a/src/main/java/com/autotune/common/k8sObjects/ValidatePerformanceProfileObject.java b/src/main/java/com/autotune/common/k8sObjects/ValidatePerformanceProfileObject.java index 0cf30c9a5..92a349980 100644 --- a/src/main/java/com/autotune/common/k8sObjects/ValidatePerformanceProfileObject.java +++ b/src/main/java/com/autotune/common/k8sObjects/ValidatePerformanceProfileObject.java @@ -16,13 +16,14 @@ package com.autotune.common.k8sObjects; import com.autotune.analyzer.exceptions.InvalidValueException; +import com.autotune.analyzer.utils.EvalExParser; import com.autotune.experimentManager.utils.EMConstants; import com.autotune.utils.AnalyzerConstants; import com.autotune.utils.AnalyzerErrorConstants; +import com.autotune.utils.AutotuneConstants; import com.autotune.utils.AutotuneSupportedTypes; import java.util.HashMap; -import java.util.List; /** * Check if the Performance Profile object in the kubernetes cluster is valid @@ -41,8 +42,9 @@ public static StringBuilder validate(HashMap map) { // Check if k8s type is supported String k8sType = (String) map.get(AnalyzerConstants.K8S_TYPE); - if (!AutotuneSupportedTypes.K8S_TYPES_SUPPORTED.contains(k8sType)) + if (!AutotuneSupportedTypes.K8S_TYPES_SUPPORTED.contains(k8sType)) { errorString.append(AnalyzerConstants.PerformanceProfileConstants.K8S_TYPE).append(k8sType).append(AnalyzerErrorConstants.AutotuneObjectErrors.UNSUPPORTED); + } // Check if slo_class is supported SloInfo sloInfo = (SloInfo) map.get(AnalyzerConstants.AutotuneObjectConstants.SLO); @@ -55,50 +57,62 @@ public static StringBuilder validate(HashMap map) { errorString.append(AnalyzerErrorConstants.AutotuneObjectErrors.DIRECTION_NOT_SUPPORTED); } - //check if slo_class is 'response_time' and direction is minimize + //check if slo_class is 'response_time' and direction is 'minimize' if (sloInfo.getSloClass().equalsIgnoreCase(EMConstants.StandardDefaults.RESPONSE_TIME) && !sloInfo.getDirection() .equalsIgnoreCase(AnalyzerConstants.AutotuneObjectConstants.MINIMIZE)) { errorString.append(AnalyzerErrorConstants.AutotuneObjectErrors.INVALID_DIRECTION_FOR_SLO_CLASS); } - //check if slo_class is 'throughput' and direction is maximize + //check if slo_class is 'throughput' and direction is 'maximize' if (sloInfo.getSloClass().equalsIgnoreCase(EMConstants.StandardDefaults.THROUGHPUT) && !sloInfo.getDirection() .equalsIgnoreCase(AnalyzerConstants.AutotuneObjectConstants.MAXIMIZE)) { errorString.append(AnalyzerErrorConstants.AutotuneObjectErrors.INVALID_DIRECTION_FOR_SLO_CLASS); } // Check if function_variables is empty - if (sloInfo.getFunctionVariables().isEmpty()) { + if (sloInfo.getFunctionVariables().isEmpty()) errorString.append(AnalyzerErrorConstants.AutotuneObjectErrors.FUNCTION_VARIABLES_EMPTY); - } + + // Check if objective_function and it's type exists + if (sloInfo.getObjectiveFunction() == null) + errorString.append(AnalyzerErrorConstants.AutotuneObjectErrors.OBJECTIVE_FUNCTION_MISSING); // Get the objective_function type String objFunctionType = sloInfo.getObjectiveFunction().getFunction_type(); - String expression; + String expression = null; for (Metric functionVariable : sloInfo.getFunctionVariables()) { // Check if datasource is supported - if (!AutotuneSupportedTypes.MONITORING_AGENTS_SUPPORTED.contains(functionVariable.getDatasource().toLowerCase())) + if (!AutotuneSupportedTypes.MONITORING_AGENTS_SUPPORTED.contains(functionVariable.getDatasource().toLowerCase())) { errorString.append(AnalyzerConstants.AutotuneObjectConstants.FUNCTION_VARIABLE) - .append(functionVariable.getName()).append(AnalyzerErrorConstants.AutotuneObjectErrors.DATASOURCE_NOT_SUPPORTED); + .append(functionVariable.getName()) + .append(AnalyzerErrorConstants.AutotuneObjectErrors.DATASOURCE_NOT_SUPPORTED); + } // Check if value_type is supported - if (!AutotuneSupportedTypes.VALUE_TYPES_SUPPORTED.contains(functionVariable.getValueType().toLowerCase())) + if (!AutotuneSupportedTypes.VALUE_TYPES_SUPPORTED.contains(functionVariable.getValueType().toLowerCase())) { errorString.append(AnalyzerConstants.AutotuneObjectConstants.FUNCTION_VARIABLE) - .append(functionVariable.getName()).append(AnalyzerErrorConstants.AutotuneObjectErrors.VALUE_TYPE_NOT_SUPPORTED); + .append(functionVariable.getName()) + .append(AnalyzerErrorConstants.AutotuneObjectErrors.VALUE_TYPE_NOT_SUPPORTED); + } - // Check if kubernetes_object type is supported - String kubernetes_object = functionVariable.getKubernetesObject().toLowerCase(); - if (!AutotuneSupportedTypes.KUBERNETES_OBJECTS_SUPPORTED.contains(functionVariable.getKubernetesObject().toLowerCase())) - errorString.append(AnalyzerConstants.KUBERNETES_OBJECTS).append(kubernetes_object).append(AnalyzerErrorConstants.AutotuneObjectErrors.UNSUPPORTED); + // Check if kubernetes_object type is supported, set default to 'container' if it's absent. + String kubernetes_object = functionVariable.getKubernetesObject(); + if (null == kubernetes_object) + functionVariable.setKubernetesObject(AutotuneConstants.JSONKeys.CONTAINER); + else { + if (!AutotuneSupportedTypes.KUBERNETES_OBJECTS_SUPPORTED.contains(kubernetes_object.toLowerCase())) + errorString.append(AnalyzerConstants.KUBERNETES_OBJECTS).append(kubernetes_object) + .append(AnalyzerErrorConstants.AutotuneObjectErrors.UNSUPPORTED); + } // Validate Objective Function try { if (objFunctionType.equals(AnalyzerConstants.AutotuneObjectConstants.EXPRESSION)) { expression = sloInfo.getObjectiveFunction().getExpression(); - System.out.println("****** Exprssion = "+expression); - if (null == expression || expression.equals(AnalyzerConstants.NULL)) + if (null == expression || expression.equals(AnalyzerConstants.NULL)) { throw new NullPointerException(AnalyzerErrorConstants.AutotuneObjectErrors.MISSING_EXPRESSION); + } } else if (objFunctionType.equals(AnalyzerConstants.PerformanceProfileConstants.SOURCE)) { if (null != sloInfo.getObjectiveFunction().getExpression()) { @@ -113,12 +127,22 @@ public static StringBuilder validate(HashMap map) { errorString.append(npe.getMessage()); return errorString; } - // Check if one of query or aggregation_functions is present - String query = (String) map.get(AnalyzerConstants.AutotuneObjectConstants.QUERY); - List aggregationFunctionsList = functionVariable.getAggregationFunctions(); - if (query == null && aggregationFunctionsList.isEmpty()) { - errorString.append(AnalyzerErrorConstants.AutotuneObjectErrors.AGG_FUNCTION_ERROR); + // Check if function_variable is part of objective_function + if (objFunctionType.equals(AnalyzerConstants.AutotuneObjectConstants.EXPRESSION)) { + if (!expression.contains(functionVariable.getName())) { + errorString.append(AnalyzerConstants.AutotuneObjectConstants.FUNCTION_VARIABLE) + .append(functionVariable.getName()).append(" ") + .append(AnalyzerErrorConstants.AutotuneObjectErrors.FUNCTION_VARIABLE_ERROR); + } + return errorString; + } + } + + // Check if objective_function is correctly formatted + if (objFunctionType.equals(AnalyzerConstants.AutotuneObjectConstants.EXPRESSION)) { + if (expression.equals(AnalyzerConstants.NULL) || !new EvalExParser().validate(sloInfo.getObjectiveFunction().getExpression(), sloInfo.getFunctionVariables())) { + errorString.append(AnalyzerErrorConstants.AutotuneObjectErrors.INVALID_OBJECTIVE_FUNCTION); } } return errorString; diff --git a/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfile.java b/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfile.java index 08a363206..b3b68a845 100644 --- a/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfile.java +++ b/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfile.java @@ -41,22 +41,11 @@ public class PerformanceProfile { @SerializedName("slo") private final SloInfo sloInfo; - public PerformanceProfile(String name, double profile_version, String k8s_type, SloInfo sloInfo) throws InvalidValueException { - HashMap map = new HashMap<>(); - map.put(AnalyzerConstants.PerformanceProfileConstants.PERF_PROFILE_NAME, name); - map.put(AnalyzerConstants.PROFILE_VERSION, profile_version); - map.put(AnalyzerConstants.PerformanceProfileConstants.K8S_TYPE, k8s_type); - map.put(AnalyzerConstants.AutotuneObjectConstants.SLO, sloInfo); - - StringBuilder error = ValidatePerformanceProfileObject.validate(map); - if (error.toString().isEmpty()) { - this.name = name; - this.profile_version = profile_version; - this.k8s_type = k8s_type; - this.sloInfo = sloInfo; - } else { - throw new InvalidValueException(error.toString()); - } + public PerformanceProfile(String name, double profile_version, String k8s_type, SloInfo sloInfo) { + this.name = name; + this.profile_version = profile_version; + this.k8s_type = k8s_type; + this.sloInfo = sloInfo; } public String getName() { diff --git a/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/AdvancedClusterMgmtImpl.java b/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/AdvancedClusterMgmtImpl.java new file mode 100644 index 000000000..a342c8f2c --- /dev/null +++ b/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/AdvancedClusterMgmtImpl.java @@ -0,0 +1,39 @@ +/******************************************************************************* + * Copyright (c) 2022 Red Hat, IBM Corporation and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ +package com.autotune.common.performanceProfiles.PerformanceProfileInterface; + +import com.autotune.common.data.result.ExperimentResultData; +import com.autotune.common.performanceProfiles.PerformanceProfile; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Util class to validate the performance profile metrics with the experiment results metrics. + */ +public class AdvancedClusterMgmtImpl extends PerfProfileImpl { + + private static final Logger LOGGER = LoggerFactory.getLogger(AdvancedClusterMgmtImpl.class); + + @Override + public String recommend(PerformanceProfile performanceProfile, ExperimentResultData experimentResultData) { + + //TODO: Will be updated once algo is completed + + return null; + } + + +} diff --git a/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/DefaultImpl.java b/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/DefaultImpl.java new file mode 100644 index 000000000..f3dcff316 --- /dev/null +++ b/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/DefaultImpl.java @@ -0,0 +1,74 @@ +/******************************************************************************* + * Copyright (c) 2022 Red Hat, IBM Corporation and others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ +package com.autotune.common.performanceProfiles.PerformanceProfileInterface; + +import com.autotune.common.data.result.*; +import com.autotune.common.performanceProfiles.PerformanceProfile; +import com.autotune.utils.AnalyzerConstants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.InvocationTargetException; +import java.util.*; + +/** + * Util class to validate the performance profile metrics with the experiment results metrics. + */ +public class DefaultImpl extends PerfProfileImpl { + + private static final Logger LOGGER = LoggerFactory.getLogger(DefaultImpl.class); + + private String validateValues(HashMap funcVar, List funcVarValueTypes) { + LOGGER.info("Func variables : {}", funcVar); + LOGGER.info("Func variable value types : {}", funcVarValueTypes); + return ""; + } + + /** + * Calculates the objective function by calling the algebraic parser library. The result is then sent to HPO. + * @param performanceProfile + * @param experimentResultData + * @return + */ + @Override + public String recommend(PerformanceProfile performanceProfile, ExperimentResultData experimentResultData) { + + String objectiveFunction = performanceProfile.getSloInfo().getObjectiveFunction().getExpression(); + Map objFunctionMap = new HashMap<>(); + String errorMsg = ""; + + // Get the metrics data from the Kruize Object + for (DeploymentResultData deploymentResultData : experimentResultData.getDeployments()) { + for (Containers containers : deploymentResultData.getContainers()) { + HashMap> containerMetricsMap = + containers.getContainer_metrics(); + List kruizeFunctionVariablesList = containerMetricsMap.keySet().stream().toList().stream().map(Enum::name).toList(); + for (HashMap funcVar : containerMetricsMap.values()) { + Map aggrInfoClassAsMap; + try { + // TODO: Need to update the below code + aggrInfoClassAsMap = DefaultImpl.convertObjectToMap(funcVar.get("results").getAggregation_info()); + LOGGER.info("aggrInfoClassAsMap: {}", aggrInfoClassAsMap); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } + } + } + } + + return ""; + } +} diff --git a/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/DefaultOpenshiftImpl.java b/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/DefaultOpenshiftImpl.java deleted file mode 100644 index 1642918de..000000000 --- a/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/DefaultOpenshiftImpl.java +++ /dev/null @@ -1,183 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2022 Red Hat, IBM Corporation and others. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -package com.autotune.common.performanceProfiles.PerformanceProfileInterface; - -import com.autotune.common.data.result.AggregationInfoResult; -import com.autotune.common.data.result.ContainerResultData; -import com.autotune.common.data.result.DeploymentResultData; -import com.autotune.common.data.result.ExperimentResultData; -import com.autotune.common.k8sObjects.Metric; -import com.autotune.common.performanceProfiles.PerformanceProfile; -import com.autotune.utils.AnalyzerErrorConstants; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.*; - -/** - * Util class to validate the performance profile metrics with the experiment results metrics. - */ -public class DefaultOpenshiftImpl implements PerfProfileInterface { - - private static final Logger LOGGER = LoggerFactory.getLogger(DefaultOpenshiftImpl.class); - List perfProfileFunctionVariablesList = new ArrayList<>(); - - @Override - public String validate(PerformanceProfile performanceProfile, ExperimentResultData experimentResultData) { - - String errorMsg = ""; - // Get the metrics data from the Performance Profile - List aggrFunctionsObjects = new ArrayList<>(); - for (Metric metric:performanceProfile.getSloInfo().getFunctionVariables()) { - perfProfileFunctionVariablesList.add(metric.getName()); - metric.getAggregationFunctions().forEach(aggregationFunctions -> - aggrFunctionsObjects.add(aggregationFunctions.getFunction())); - } - LOGGER.debug(String.format("List of functionVariables: %s", perfProfileFunctionVariablesList)); - LOGGER.debug(String.format("List of agg func objects: %s", aggrFunctionsObjects)); - - // Get the metrics data from the Kruize Object - for (DeploymentResultData deploymentResultData : experimentResultData.getDeployments()) { - for (ContainerResultData containerResultData : deploymentResultData.getContainers()) { - HashMap>> containerMetricsMap = - containerResultData.getContainer_metrics(); - List kruizeFunctionVariablesList = containerMetricsMap.keySet().stream().toList(); - if (!(perfProfileFunctionVariablesList.size() == kruizeFunctionVariablesList.size() && - new HashSet<>(perfProfileFunctionVariablesList).containsAll(kruizeFunctionVariablesList) && - new HashSet<>(kruizeFunctionVariablesList).containsAll(perfProfileFunctionVariablesList))) { - LOGGER.debug("perfProfileFunctionVariablesList: {}",perfProfileFunctionVariablesList); - LOGGER.debug("kruizeFunctionVariablesList: {}",kruizeFunctionVariablesList); - perfProfileFunctionVariablesList.removeAll(kruizeFunctionVariablesList); - errorMsg = errorMsg.concat(String.format("Following Performance Profile parameters are missing for experiment - %s : %s", experimentResultData.getExperiment_name(), perfProfileFunctionVariablesList)); - break; - } else { - for(HashMap> funcVar:containerMetricsMap.values()){ - for(HashMap genInfo:funcVar.values()){ - Map genInfoClassAsMap; - for(AggregationInfoResult genInfoObj:genInfo.values()){ - try { - genInfoClassAsMap = DefaultOpenshiftImpl.convertObjectToMap(genInfoObj); - errorMsg = validateAggFunction(genInfoClassAsMap.keySet(), aggrFunctionsObjects); - if (!errorMsg.isBlank()) { - errorMsg = errorMsg.concat(String.format("for the experiment : %s" - ,experimentResultData.getExperiment_name())); - return errorMsg; - } - } catch (IllegalAccessException | InvocationTargetException e) { - throw new RuntimeException(e); - } - } - } - } - } - } - } - - return errorMsg; - } - - /** - * Calculates the objective function by calling the algebraic parser library. The result is then sent to HPO. - * @param performanceProfile - * @param experimentResultData - * @return - */ - @Override - public String recommend(PerformanceProfile performanceProfile, ExperimentResultData experimentResultData) { - - String objectiveFunction = performanceProfile.getSloInfo().getObjectiveFunction().getExpression(); - Map objFunctionMap = new HashMap<>(); - String errorMsg = ""; - - // Get the metrics data from the Kruize Object - for (DeploymentResultData deploymentResultData : experimentResultData.getDeployments()) { - for (ContainerResultData containers : deploymentResultData.getContainers()) { - HashMap>> containerMetricsMap = - containers.getContainer_metrics(); - List kruizeFunctionVariablesList = containerMetricsMap.keySet().stream().toList(); - for(HashMap> funcVar:containerMetricsMap.values()){ - for(HashMap aggregationInfoResultMap:funcVar.values()){ - Map aggrInfoClassAsMap; - for(AggregationInfoResult aggregationInfoResult:aggregationInfoResultMap.values()){ - try { - // TODO: Need to update the below code - aggrInfoClassAsMap = DefaultOpenshiftImpl.convertObjectToMap(aggregationInfoResult); - LOGGER.info("aggrInfoClassAsMap: {}", aggrInfoClassAsMap); - } catch (IllegalAccessException | InvocationTargetException e) { - throw new RuntimeException(e); - } - } - } - } - - } - } - - return objectiveFunction; - } - /** - * Validates the aggregation function objects against the generalInfo metrics - * @param keySet - * @param aggrFunctionsObjects - * @return - */ - private String validateAggFunction(Set keySet, List aggrFunctionsObjects) { - - List genInfoObjects = keySet.stream().toList(); - List missingAggFunction = new ArrayList<>(); - String errorMsg = ""; - // check if none of the aggrfunctions are present in the genInfo List - if (genInfoObjects.stream().noneMatch(aggrFunctionsObjects::contains)) { - LOGGER.error(AnalyzerErrorConstants.AutotuneObjectErrors.MISSING_AGG_FUNCTION); - errorMsg = errorMsg.concat(AnalyzerErrorConstants.AutotuneObjectErrors.MISSING_AGG_FUNCTION); - } else { - // check if some or all the values are present or not and respond accordingly - for (String aggFuncObj : aggrFunctionsObjects) { - if (!genInfoObjects.contains(aggFuncObj)) { - missingAggFunction.add(aggFuncObj); - } - } - if (!missingAggFunction.isEmpty()) { - LOGGER.warn("Missing Aggregation Functions: {}", missingAggFunction); - } - } - return errorMsg; - } - - /** - * Converts the generalInfo class into Map to extract values for validation - * @param obj - * @return - * @throws IllegalAccessException - * @throws IllegalArgumentException - * @throws InvocationTargetException - */ - public static Map convertObjectToMap(Object obj) throws IllegalAccessException, - IllegalArgumentException, InvocationTargetException { - Method[] methods = obj.getClass().getMethods(); - Map map = new HashMap<>(); - for (Method m : methods) { - if (m.getName().startsWith("get") && !m.getName().startsWith("getClass")) { - Object value = m.invoke(obj); - if (value instanceof Double) - map.put(m.getName().substring(3).toLowerCase(), value); - } - } - return map; - } -} diff --git a/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/PerfProfileImpl.java b/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/PerfProfileImpl.java new file mode 100644 index 000000000..ca64f23d4 --- /dev/null +++ b/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/PerfProfileImpl.java @@ -0,0 +1,196 @@ +package com.autotune.common.performanceProfiles.PerformanceProfileInterface; + +import com.autotune.analyzer.utils.PerformanceProfileValidation; +import com.autotune.common.data.ValidationResultData; +import com.autotune.common.data.result.Containers; +import com.autotune.common.data.result.DeploymentResultData; +import com.autotune.common.data.result.ExperimentResultData; +import com.autotune.common.data.result.Results; +import com.autotune.common.k8sObjects.Metric; +import com.autotune.common.performanceProfiles.PerformanceProfile; +import com.autotune.utils.AnalyzerConstants; +import com.autotune.utils.AnalyzerErrorConstants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.*; + +public class PerfProfileImpl implements PerfProfileInterface { + + private static final Logger LOGGER = LoggerFactory.getLogger(PerfProfileImpl.class); + + @Override + public String getName(PerformanceProfile performanceProfile) { + String name = AnalyzerConstants.PerformanceProfileConstants.PerfProfileNames.get(performanceProfile.getName()); + return name; + } + + + /** + * validates the performance profile fields and the data and then adds it to the map + * @param performanceProfile + * @return + */ + @Override + public ValidationResultData validateAndAddProfile(Map performanceProfilesMap, PerformanceProfile performanceProfile) { + ValidationResultData validationResultData = new ValidationResultData(false, null); + try { + PerformanceProfileValidation performanceProfileValidation = new PerformanceProfileValidation(performanceProfilesMap); + performanceProfileValidation.validate(performanceProfile); + if (performanceProfileValidation.isSuccess()) { + addPerformanceProfile(performanceProfilesMap, performanceProfile); + validationResultData.setSuccess(true); + } else { + validationResultData.setSuccess(false); + validationResultData.setMessage("Validation failed due to " + performanceProfileValidation.getErrorMessage()); + } + } catch (Exception e) { + LOGGER.error("Validate and add profile falied due to : " + e.getMessage()); + validationResultData.setSuccess(false); + validationResultData.setMessage("Validation failed due to " + e.getMessage()); + } + return validationResultData; + } + + /** + * @param performanceProfile + * @param experimentResultData + * @return + */ + @Override + public String validateResults(PerformanceProfile performanceProfile, ExperimentResultData experimentResultData) { + String errorMsg = ""; + // Get the metrics data from the Performance Profile + List aggrFunctionsObjects = new ArrayList<>(); + List queryList = new ArrayList<>(); + List perfProfileFunctionVariablesList = new ArrayList<>(); + for (Metric metric : performanceProfile.getSloInfo().getFunctionVariables()) { + perfProfileFunctionVariablesList.add(metric.getName()); + if (null != metric.getAggregationFunctions()) { + metric.getAggregationFunctions().forEach(aggregationFunctions -> + aggrFunctionsObjects.add(aggregationFunctions.getFunction())); + } + if (null != metric.getQuery()) + queryList.add(metric.getQuery()); + } + LOGGER.debug(String.format("List of functionVariables: %s", perfProfileFunctionVariablesList)); + LOGGER.debug(String.format("List of agg func objects: %s", aggrFunctionsObjects)); + + // Get the metrics data from the Kruize Object + for (DeploymentResultData deploymentResultData : experimentResultData.getDeployments()) { + for (Containers containers : deploymentResultData.getContainers()) { + HashMap> containerMetricsMap = + containers.getContainer_metrics(); + List kruizeFunctionVariablesList = containerMetricsMap.keySet().stream().toList().stream().map(Enum::name).toList(); + if (!(perfProfileFunctionVariablesList.size() == kruizeFunctionVariablesList.size() && + new HashSet<>(perfProfileFunctionVariablesList).containsAll(kruizeFunctionVariablesList) && + new HashSet<>(kruizeFunctionVariablesList).containsAll(perfProfileFunctionVariablesList))) { + LOGGER.debug("perfProfileFunctionVariablesList: {}", perfProfileFunctionVariablesList); + LOGGER.debug("kruizeFunctionVariablesList: {}", kruizeFunctionVariablesList); + perfProfileFunctionVariablesList.removeAll(kruizeFunctionVariablesList); + errorMsg = errorMsg.concat(String.format("Following Performance Profile parameters are missing for experiment - %s : %s", experimentResultData.getExperiment_name(), perfProfileFunctionVariablesList)); + break; + } else { + for (HashMap funcVar : containerMetricsMap.values()) { + Map aggrInfoClassAsMap; + if (!aggrFunctionsObjects.isEmpty()) { + try { + aggrInfoClassAsMap = convertObjectToMap(funcVar.get("results").getAggregation_info()); + errorMsg = validateAggFunction(aggrInfoClassAsMap.keySet(), aggrFunctionsObjects); + if (!errorMsg.isBlank()) { + errorMsg = errorMsg.concat(String.format("for the experiment : %s" + , experimentResultData.getExperiment_name())); + break; + } + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } + } else { + // TODO: check for query and validate against value in kruize object + if (queryList.isEmpty()) { + errorMsg = AnalyzerErrorConstants.AutotuneObjectErrors.QUERY_FUNCTION_MISSING; + break; + } else if (null == funcVar.get("results").getValue()) { + errorMsg = AnalyzerErrorConstants.AutotuneObjectErrors.MISSING_VALUE; + break; + } + } + + + } + } + } + } + return errorMsg; + } + + /** + * @param performanceProfile + * @param experimentResultData + * @return + */ + @Override + public String recommend(PerformanceProfile performanceProfile, ExperimentResultData experimentResultData) { + return null; + } + + public static void addPerformanceProfile(Map performanceProfileMap, PerformanceProfile performanceProfile) { + performanceProfileMap.put(performanceProfile.getName(), performanceProfile); + LOGGER.info("Added PerformanceProfile: {} ",performanceProfile.getName()); + } + + /** + * Validates the aggregation function objects against the aggregationInfoResult metrics + * + * @param keySet + * @param aggrFunctionsObjects + * @return + */ + private String validateAggFunction(Set keySet, List aggrFunctionsObjects) { + + List aggrInfoObjects = keySet.stream().toList(); + List missingAggFunction = new ArrayList<>(); + String errorMsg = ""; + // check if none of the aggrfunctions are present in the aggrInfoObjects List + if (aggrInfoObjects.stream().noneMatch(aggrFunctionsObjects::contains)) { + LOGGER.error(AnalyzerErrorConstants.AutotuneObjectErrors.MISSING_AGG_FUNCTION); + errorMsg = errorMsg.concat(AnalyzerErrorConstants.AutotuneObjectErrors.MISSING_AGG_FUNCTION); + } else { + // check if some or all the values are present or not and respond accordingly + for (String aggFuncObj : aggrFunctionsObjects) { + if (!aggrInfoObjects.contains(aggFuncObj)) { + missingAggFunction.add(aggFuncObj); + } + } + if (!missingAggFunction.isEmpty()) { + LOGGER.warn("Missing Aggregation Functions: {}", missingAggFunction); + } + } + return errorMsg; + } + + /** + * Converts the aggregationInfoResult class into Map to extract values for validation + * + * @param obj + * @return + * @throws IllegalAccessException + * @throws IllegalArgumentException + * @throws InvocationTargetException + */ + public static Map convertObjectToMap(Object obj) throws IllegalAccessException, + IllegalArgumentException, InvocationTargetException { + Method[] methods = obj.getClass().getMethods(); + Map map = new HashMap<>(); + for (Method m : methods) { + if (m.getName().startsWith("get") && !m.getName().startsWith("getClass")) { + Object value = m.invoke(obj); + if (value instanceof Double) + map.put(m.getName().substring(3).toLowerCase(), value); + } + } + return map; + } +} diff --git a/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/PerfProfileInterface.java b/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/PerfProfileInterface.java index de3383084..12d6ac4dc 100644 --- a/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/PerfProfileInterface.java +++ b/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/PerfProfileInterface.java @@ -16,16 +16,20 @@ package com.autotune.common.performanceProfiles.PerformanceProfileInterface; +import com.autotune.common.data.ValidationResultData; import com.autotune.common.data.result.ExperimentResultData; import com.autotune.common.performanceProfiles.PerformanceProfile; +import java.util.Map; + /** - * Abstraction layer containing validate and recommend methods for the - * validation of the Performance Profiles with respect to the updated experiment. + * Abstraction layer containing methods for the validation of the Performance Profiles with respect to the updated experiment. + * and to parse the objective function data. */ public interface PerfProfileInterface { - - String validate(PerformanceProfile performanceProfile, ExperimentResultData experimentResultData); - +// name, validateResults, validateProfile, recommend + String getName(PerformanceProfile profile); + ValidationResultData validateAndAddProfile(Map performanceProfilesMap, PerformanceProfile performanceProfile); + String validateResults(PerformanceProfile performanceProfile, ExperimentResultData experimentResultData); String recommend(PerformanceProfile performanceProfile, ExperimentResultData experimentResultData); } diff --git a/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/ResourceOptimizationOpenshiftImpl.java b/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/ResourceOptimizationOpenshiftImpl.java index 879f61426..a4a6d88e0 100644 --- a/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/ResourceOptimizationOpenshiftImpl.java +++ b/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfileInterface/ResourceOptimizationOpenshiftImpl.java @@ -15,138 +15,22 @@ *******************************************************************************/ package com.autotune.common.performanceProfiles.PerformanceProfileInterface; -import com.autotune.common.data.result.Containers; -import com.autotune.common.data.result.DeploymentResultData; import com.autotune.common.data.result.ExperimentResultData; -import com.autotune.common.data.result.Results; -import com.autotune.common.k8sObjects.KruizeObject; -import com.autotune.common.k8sObjects.Metric; import com.autotune.common.performanceProfiles.PerformanceProfile; -import com.autotune.common.performanceProfiles.PerformanceProfilesDeployment; -import com.autotune.utils.AnalyzerConstants; -import com.autotune.utils.AnalyzerErrorConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.*; - /** * Util class to validate the performance profile metrics with the experiment results metrics. */ -public class ResourceOptimizationOpenshiftImpl implements PerfProfileInterface { +public class ResourceOptimizationOpenshiftImpl extends PerfProfileImpl { private static final Logger LOGGER = LoggerFactory.getLogger(ResourceOptimizationOpenshiftImpl.class); - - /** - * Converts the generalInfo class into Map to extract values for validation - * - * @param obj - * @return - * @throws IllegalAccessException - * @throws IllegalArgumentException - * @throws InvocationTargetException - */ - public static Map convertObjectToMap(Object obj) throws IllegalAccessException, - IllegalArgumentException, InvocationTargetException { - Method[] methods = obj.getClass().getMethods(); - Map map = new HashMap<>(); - for (Method m : methods) { - if (m.getName().startsWith("get") && !m.getName().startsWith("getClass")) { - Object value = m.invoke(obj); - if (value instanceof Double) - map.put(m.getName().substring(3).toLowerCase(), value); - } - } - return map; - } - - @Override - public String validate(PerformanceProfile performanceProfile, ExperimentResultData experimentResultData) { - - String errorMsg = ""; - // Get the metrics data from the Performance Profile - List aggrFunctionsObjects = new ArrayList<>(); - List perfProfileFunctionVariablesList = new ArrayList<>(); - for (Metric metric : performanceProfile.getSloInfo().getFunctionVariables()) { - perfProfileFunctionVariablesList.add(metric.getName()); - metric.getAggregationFunctions().forEach(aggregationFunctions -> - aggrFunctionsObjects.add(aggregationFunctions.getFunction())); - } - LOGGER.debug(String.format("List of functionVariables: %s", perfProfileFunctionVariablesList)); - LOGGER.debug(String.format("List of agg func objects: %s", aggrFunctionsObjects)); - - // Get the metrics data from the Kruize Object - for (DeploymentResultData deploymentResultData : experimentResultData.getDeployments()) { - for (Containers containers : deploymentResultData.getContainers()) { - HashMap> containerMetricsMap = - containers.getContainer_metrics(); - List kruizeFunctionVariablesList = containerMetricsMap.keySet().stream().toList().stream().map(Enum::name).toList(); - if (!(perfProfileFunctionVariablesList.size() == kruizeFunctionVariablesList.size() && - new HashSet<>(perfProfileFunctionVariablesList).containsAll(kruizeFunctionVariablesList) && - new HashSet<>(kruizeFunctionVariablesList).containsAll(perfProfileFunctionVariablesList))) { - LOGGER.debug("perfProfileFunctionVariablesList: {}", perfProfileFunctionVariablesList); - LOGGER.debug("kruizeFunctionVariablesList: {}", kruizeFunctionVariablesList); - perfProfileFunctionVariablesList.removeAll(kruizeFunctionVariablesList); - errorMsg = errorMsg.concat(String.format("Following Performance Profile parameters are missing for experiment - %s : %s", experimentResultData.getExperiment_name(), perfProfileFunctionVariablesList)); - break; - } else { - for (HashMap funcVar : containerMetricsMap.values()) { - Map genInfoClassAsMap; - try { - genInfoClassAsMap = ResourceOptimizationOpenshiftImpl.convertObjectToMap(funcVar.get("results").getAggregation_info()); - errorMsg = validateAggFunction(genInfoClassAsMap.keySet(), aggrFunctionsObjects); - if (!errorMsg.isBlank()) { - errorMsg = errorMsg.concat(String.format("for the experiment : %s" - , experimentResultData.getExperiment_name())); - return errorMsg; - } - } catch (IllegalAccessException | InvocationTargetException e) { - throw new RuntimeException(e); - } - } - } - } - } - - return errorMsg; - } - - /** - * Validates the aggregation function objects against the generalInfo metrics - * - * @param keySet - * @param aggrFunctionsObjects - * @return - */ - private String validateAggFunction(Set keySet, List aggrFunctionsObjects) { - - List genInfoObjects = keySet.stream().toList(); - List missingAggFunction = new ArrayList<>(); - String errorMsg = ""; - // check if none of the aggrfunctions are present in the genInfo List - if (genInfoObjects.stream().noneMatch(aggrFunctionsObjects::contains)) { - LOGGER.error(AnalyzerErrorConstants.AutotuneObjectErrors.MISSING_AGG_FUNCTION); - errorMsg = errorMsg.concat(AnalyzerErrorConstants.AutotuneObjectErrors.MISSING_AGG_FUNCTION); - } else { - // check if some or all the values are present or not and respond accordingly - for (String aggFuncObj : aggrFunctionsObjects) { - if (!genInfoObjects.contains(aggFuncObj)) { - missingAggFunction.add(aggFuncObj); - } - } - if (!missingAggFunction.isEmpty()) { - LOGGER.warn("Missing Aggregation Functions: {}", missingAggFunction); - } - } - return errorMsg; - } - @Override public String recommend(PerformanceProfile performanceProfile, ExperimentResultData experimentResultData) { //TODO: Will be updated once algo is completed - + return null; } + } diff --git a/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfilesDeployment.java b/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfilesDeployment.java index f87ca1e6a..1e6a4cbd8 100644 --- a/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfilesDeployment.java +++ b/src/main/java/com/autotune/common/performanceProfiles/PerformanceProfilesDeployment.java @@ -4,6 +4,7 @@ import com.autotune.analyzer.utils.PerformanceProfileValidation; import com.autotune.common.data.ValidationResultData; import com.autotune.common.k8sObjects.*; +import com.autotune.common.performanceProfiles.PerformanceProfileInterface.PerfProfileImpl; import com.autotune.common.target.kubernetes.service.KubernetesServices; import com.autotune.common.target.kubernetes.service.impl.KubernetesServicesImpl; import com.autotune.utils.AnalyzerConstants; @@ -18,6 +19,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.time.Clock; import java.util.ArrayList; import java.util.HashMap; @@ -40,13 +42,13 @@ public static void getPerformanceProfiles() { Watcher performanceProfileObjectWatcher = new Watcher<>() { @Override public void eventReceived(Action action, String resource) { - PerformanceProfile performanceProfile; + PerformanceProfile performanceProfile = null; switch (action.toString().toUpperCase()) { case "ADDED": performanceProfile = getPerformanceProfile(resource); if ( validatePerformanceProfile(performanceProfile)) - PerformanceProfileValidation.addPerformanceProfile(performanceProfilesMap, performanceProfile); + new PerfProfileImpl().addPerformanceProfile(performanceProfilesMap, performanceProfile); break; case "MODIFIED": performanceProfile = getPerformanceProfile(resource); @@ -56,7 +58,7 @@ public void eventReceived(Action action, String resource) { .equals(performanceProfile)) { if (validatePerformanceProfile(performanceProfile)) { deleteExistingPerformanceProfile(resource); - PerformanceProfileValidation.addPerformanceProfile(performanceProfilesMap, performanceProfile); + new PerfProfileImpl().addPerformanceProfile(performanceProfilesMap, performanceProfile); } } } @@ -80,7 +82,7 @@ private static boolean validatePerformanceProfile(PerformanceProfile performance boolean validationStatus = false; try { if (null != performanceProfile) { - ValidationResultData validationResultData = new PerformanceProfileValidation(performanceProfilesMap).validate(performanceProfilesMap, performanceProfile); + ValidationResultData validationResultData = new PerformanceProfileValidation(performanceProfilesMap).validate(performanceProfile); if (validationResultData.isSuccess()) validationStatus = true; else @@ -98,79 +100,16 @@ public static PerformanceProfile getPerformanceProfile(String performanceProfile JSONObject performanceProfileObjectJson = new JSONObject(performanceProfileObjectJsonStr); JSONObject metadataJson = performanceProfileObjectJson .getJSONObject(AnalyzerConstants.AutotuneObjectConstants.METADATA); + performanceProfileObjectJson.remove("apiversion"); + performanceProfileObjectJson.remove("kind"); + performanceProfileObjectJson.remove("metadata"); - String name; - String k8s_type; - double profile_version; - SloInfo sloInfo; - ObjectiveFunction objectiveFunction = null; + performanceProfileObjectJson.put("name",metadataJson.optString(AnalyzerConstants.PerformanceProfileConstants.PERF_PROFILE_NAME)); + PerformanceProfile performanceProfile = new Gson().fromJson(performanceProfileObjectJson.toString(), PerformanceProfile.class); - JSONObject sloJson; - String slo_class = null; - String direction = null; - JSONObject objectiveFunctionJson; + return performanceProfile; - name = metadataJson.optString(AnalyzerConstants.PerformanceProfileConstants.PERF_PROFILE_NAME); - profile_version = Double.parseDouble(performanceProfileObjectJson.optString(AnalyzerConstants.PROFILE_VERSION, - String.valueOf(AnalyzerConstants.DEFAULT_PROFILE_VERSION))); - k8s_type = performanceProfileObjectJson.optString(AnalyzerConstants.K8S_TYPE,AnalyzerConstants.DEFAULT_K8S_TYPE); - sloJson = performanceProfileObjectJson.optJSONObject(AnalyzerConstants.AutotuneObjectConstants.SLO); - - if (sloJson != null) { - slo_class = sloJson.optString(AnalyzerConstants.AutotuneObjectConstants.SLO_CLASS); - direction = sloJson.optString(AnalyzerConstants.AutotuneObjectConstants.DIRECTION); - objectiveFunctionJson = sloJson.optJSONObject(AnalyzerConstants.AutotuneObjectConstants.OBJECTIVE_FUNCTION); - objectiveFunction = new Gson().fromJson(String.valueOf(objectiveFunctionJson), ObjectiveFunction.class); - } - - JSONArray functionVariables = new JSONArray(); - JSONArray aggregationFunctionsArr; - if (sloJson != null) { - functionVariables = sloJson.getJSONArray(AnalyzerConstants.AutotuneObjectConstants.FUNCTION_VARIABLES); - } - ArrayList metricArrayList = new ArrayList<>(); - - for (Object functionVariableObj : functionVariables) { - JSONObject functionVariableJson = (JSONObject) functionVariableObj; - - String variableName = functionVariableJson.optString(AnalyzerConstants.AutotuneObjectConstants.NAME); - String query = functionVariableJson.optString(AnalyzerConstants.AutotuneObjectConstants.QUERY); - String datasource = functionVariableJson.optString(AnalyzerConstants.AutotuneObjectConstants.DATASOURCE); - String valueType = functionVariableJson.optString(AnalyzerConstants.AutotuneObjectConstants.VALUE_TYPE); - String kubernetes_object = functionVariableJson.optString(AnalyzerConstants.KUBERNETES_OBJECTS); - - aggregationFunctionsArr = ((JSONObject) functionVariableObj).getJSONArray(AnalyzerConstants.AGGREGATION_FUNCTIONS); - - List aggregationFunctionsList = new ArrayList<>(); - for (Object aggregationFunctionsObj : aggregationFunctionsArr) { - JSONObject aggregationFunctionsJson = (JSONObject) aggregationFunctionsObj; - String function = aggregationFunctionsJson.optString(AnalyzerConstants.FUNCTION); - String aggregationFunctionSQuery = aggregationFunctionsJson.optString( - AnalyzerConstants.AutotuneObjectConstants.QUERY); - String versions = aggregationFunctionsJson.optString(AnalyzerConstants.VERSIONS); - - AggregationFunctions aggregationFunctions = new AggregationFunctions(function, - aggregationFunctionSQuery, versions); - aggregationFunctionsList.add(aggregationFunctions); - } - - Metric metric = new Metric(variableName, - query, - datasource, - valueType); - metric.setAggregationFunctions(aggregationFunctionsList); - metric.setKubernetesObject(kubernetes_object); - - metricArrayList.add(metric); - } - sloInfo = new SloInfo(slo_class, - objectiveFunction, - direction, - AnalyzerConstants.AutotuneObjectConstants.DEFAULT_HPO_ALGO_IMPL, - metricArrayList); - return new PerformanceProfile(name, profile_version, k8s_type, sloInfo); - - } catch (InvalidValueException | NullPointerException | JSONException e) { + } catch (NullPointerException | JSONException e) { LOGGER.error("Exception occurred while parsing the data: {}",e.getMessage()); return null; } diff --git a/src/main/java/com/autotune/utils/AnalyzerConstants.java b/src/main/java/com/autotune/utils/AnalyzerConstants.java index badda8bc9..05fa8cf8c 100644 --- a/src/main/java/com/autotune/utils/AnalyzerConstants.java +++ b/src/main/java/com/autotune/utils/AnalyzerConstants.java @@ -15,6 +15,7 @@ *******************************************************************************/ package com.autotune.utils; +import java.util.Map; import java.util.regex.Pattern; /** @@ -392,5 +393,10 @@ public static final class PerformanceProfileConstants { public static final String SOURCE = "source"; public static final String PERFORMANCE_PROFILE_PKG = "com.autotune.common.performanceProfiles.PerformanceProfileInterface."; public static final String DEFAULT_PROFILE = "default"; + + public static final Map PerfProfileNames = Map.of( + "resource-optimization-openshift", "ResourceOptimizationOpenshiftImpl", + "advance-cluster-management", "AdvancedClusterMgmtImpl" + ); } } diff --git a/src/main/java/com/autotune/utils/AnalyzerErrorConstants.java b/src/main/java/com/autotune/utils/AnalyzerErrorConstants.java index 75410b19c..18244a707 100644 --- a/src/main/java/com/autotune/utils/AnalyzerErrorConstants.java +++ b/src/main/java/com/autotune/utils/AnalyzerErrorConstants.java @@ -60,13 +60,17 @@ private AutotuneObjectErrors() { public static final String INVALID_DIRECTION_FOR_SLO_CLASS = "Invalid direction for slo_class\n"; public static final String DATASOURCE_NOT_SUPPORTED = " datasource " + UNSUPPORTED; public static final String VALUE_TYPE_NOT_SUPPORTED = " value_type " + UNSUPPORTED; - public static final String AGG_FUNCTION_ERROR = "One of query or aggregation_functions is mandatory. Both cannot be null!"; + public static final String QUERY_FUNCTION_MISSING = "One of query or aggregation_functions is mandatory. Both cannot be null!"; public static final String MISSING_AGG_FUNCTION = "At least one aggregation function value needs to be present "; public static final String FUNCTION_VARIABLE_ERROR = "missing in objective_function\n"; public static final String MISSING_EXPRESSION = "Expression value is missing or null!\n"; public static final String MISPLACED_EXPRESSION = "Expression is not allowed when the type is source\n"; public static final String INVALID_TYPE = "Objective function type can only be either expression or source\n"; public static final String NO_PERF_PROFILE = "No performance profiles present!"; + public static final String MISSING_VALUE = "Missing 'value' in the results "; + public static final String MISSING_SLO_DATA = "No Performance Profile or SLO data is Present!"; + public static final String SLO_REDUNDANCY_ERROR = "SLO Data and Performance Profile cannot exist simultaneously!"; + public static final String DUPLICATE_PERF_PROFILE = "Performance Profile already exists!"; } diff --git a/src/main/java/com/autotune/utils/TrialHelpers.java b/src/main/java/com/autotune/utils/TrialHelpers.java index adf5fff18..46b1b462d 100644 --- a/src/main/java/com/autotune/utils/TrialHelpers.java +++ b/src/main/java/com/autotune/utils/TrialHelpers.java @@ -28,6 +28,8 @@ import com.autotune.common.k8sObjects.KruizeObject; import com.autotune.common.k8sObjects.Metric; import com.autotune.common.k8sObjects.SloInfo; +import com.autotune.common.performanceProfiles.PerformanceProfile; +import com.autotune.common.performanceProfiles.PerformanceProfilesDeployment; import com.autotune.experimentManager.exceptions.IncompatibleInputJSONException; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -184,7 +186,9 @@ public static ExperimentTrial createDefaultExperimentTrial(int trialNumber, ContainerConfigData configData = new ContainerConfigData(); HashMap podMetricsHashMap = new HashMap<>(); HashMap> containerMetricsHashMap = new HashMap<>(); - SloInfo sloInfo = kruizeObject.getSloInfo(); + PerformanceProfile performanceProfile = PerformanceProfilesDeployment.performanceProfilesMap + .get(kruizeObject.getPerformanceProfile()); + SloInfo sloInfo = performanceProfile.getSloInfo(); for (Metric metric : sloInfo.getFunctionVariables()) { podMetricsHashMap.put(metric.getName(), metric); } From 64f8683441b048d52ece83ea42b89f767b330656 Mon Sep 17 00:00:00 2001 From: bharathappali Date: Fri, 17 Feb 2023 10:15:18 +0530 Subject: [PATCH 32/35] Adding cluster name as param to constructor Signed-off-by: bharathappali --- .../autotune/analyzer/deployment/KruizeDeployment.java | 3 ++- .../com/autotune/common/k8sObjects/KruizeObject.java | 9 +++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/autotune/analyzer/deployment/KruizeDeployment.java b/src/main/java/com/autotune/analyzer/deployment/KruizeDeployment.java index c38940102..c1f711bdc 100644 --- a/src/main/java/com/autotune/analyzer/deployment/KruizeDeployment.java +++ b/src/main/java/com/autotune/analyzer/deployment/KruizeDeployment.java @@ -419,7 +419,8 @@ private static KruizeObject getAutotuneObject(String autotuneObjectJsonStr) { targetCluster, sloInfo, selectorInfo, - objectReference + objectReference, + null ); } catch (InvalidValueException | NullPointerException | JSONException e) { diff --git a/src/main/java/com/autotune/common/k8sObjects/KruizeObject.java b/src/main/java/com/autotune/common/k8sObjects/KruizeObject.java index 1fa61462e..18d9c6e23 100644 --- a/src/main/java/com/autotune/common/k8sObjects/KruizeObject.java +++ b/src/main/java/com/autotune/common/k8sObjects/KruizeObject.java @@ -64,8 +64,7 @@ public KruizeObject(String experimentName, SloInfo sloInfo, SelectorInfo selectorInfo, ObjectReference objectReference, - String... clusterNameContent // Adding an optional param to not break existing implementation - // Please add any required param above/before the `clusterName` variable + String clusterNameContent ) throws InvalidValueException { HashMap map = new HashMap<>(); @@ -75,8 +74,7 @@ public KruizeObject(String experimentName, map.put(AnalyzerConstants.AutotuneObjectConstants.TARGET_CLUSTER, targetCluster); map.put(AnalyzerConstants.AutotuneObjectConstants.SLO, sloInfo); map.put(AnalyzerConstants.AutotuneObjectConstants.SELECTOR, selectorInfo); - if (clusterNameContent.length > 0) - map.put(AnalyzerConstants.AutotuneObjectConstants.CLUSTER_NAME, clusterNameContent[0].trim()); + map.put(AnalyzerConstants.AutotuneObjectConstants.CLUSTER_NAME, clusterNameContent); StringBuilder error = ValidateAutotuneObject.validate(map); if (error.toString().isEmpty()) { @@ -88,8 +86,7 @@ public KruizeObject(String experimentName, this.selectorInfo = selectorInfo; this.experimentId = Utils.generateID(toString()); this.objectReference = objectReference; - if (clusterNameContent.length > 0) - this.clusterName = clusterNameContent[0].trim(); + this.clusterName = clusterNameContent; } else { throw new InvalidValueException(error.toString()); } From 52f8a9d526c6d5e5575ccb3b081a1243ae8c776c Mon Sep 17 00:00:00 2001 From: bharathappali Date: Mon, 20 Feb 2023 10:22:20 +0530 Subject: [PATCH 33/35] Addressed review comments Signed-off-by: bharathappali --- .../autotune/analyzer/deployment/KruizeDeployment.java | 6 ++++-- .../java/com/autotune/common/k8sObjects/KruizeObject.java | 8 ++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/autotune/analyzer/deployment/KruizeDeployment.java b/src/main/java/com/autotune/analyzer/deployment/KruizeDeployment.java index c1f711bdc..afe970cd9 100644 --- a/src/main/java/com/autotune/analyzer/deployment/KruizeDeployment.java +++ b/src/main/java/com/autotune/analyzer/deployment/KruizeDeployment.java @@ -326,6 +326,7 @@ private static KruizeObject getAutotuneObject(String autotuneObjectJsonStr) { String name; String mode; String targetCluster; + String clusterName; SloInfo sloInfo; ObjectiveFunction objectiveFunction = null; String namespace; @@ -397,6 +398,7 @@ private static KruizeObject getAutotuneObject(String autotuneObjectJsonStr) { AnalyzerConstants.AutotuneObjectConstants.DEFAULT_MODE); targetCluster = specJson.optString(AnalyzerConstants.AutotuneObjectConstants.TARGET_CLUSTER, AnalyzerConstants.AutotuneObjectConstants.DEFAULT_TARGET_CLUSTER); + clusterName = specJson.optString(AnalyzerConstants.AutotuneObjectConstants.CLUSTER_NAME); name = metadataJson.optString(AnalyzerConstants.AutotuneObjectConstants.NAME); namespace = metadataJson.optString(AnalyzerConstants.AutotuneObjectConstants.NAMESPACE); @@ -417,10 +419,10 @@ private static KruizeObject getAutotuneObject(String autotuneObjectJsonStr) { namespace, mode, targetCluster, + clusterName, sloInfo, selectorInfo, - objectReference, - null + objectReference ); } catch (InvalidValueException | NullPointerException | JSONException e) { diff --git a/src/main/java/com/autotune/common/k8sObjects/KruizeObject.java b/src/main/java/com/autotune/common/k8sObjects/KruizeObject.java index 18d9c6e23..ea28fcc87 100644 --- a/src/main/java/com/autotune/common/k8sObjects/KruizeObject.java +++ b/src/main/java/com/autotune/common/k8sObjects/KruizeObject.java @@ -61,10 +61,10 @@ public KruizeObject(String experimentName, String namespace, String mode, String targetCluster, + String clusterName, SloInfo sloInfo, SelectorInfo selectorInfo, - ObjectReference objectReference, - String clusterNameContent + ObjectReference objectReference ) throws InvalidValueException { HashMap map = new HashMap<>(); @@ -74,7 +74,7 @@ public KruizeObject(String experimentName, map.put(AnalyzerConstants.AutotuneObjectConstants.TARGET_CLUSTER, targetCluster); map.put(AnalyzerConstants.AutotuneObjectConstants.SLO, sloInfo); map.put(AnalyzerConstants.AutotuneObjectConstants.SELECTOR, selectorInfo); - map.put(AnalyzerConstants.AutotuneObjectConstants.CLUSTER_NAME, clusterNameContent); + map.put(AnalyzerConstants.AutotuneObjectConstants.CLUSTER_NAME, clusterName); StringBuilder error = ValidateAutotuneObject.validate(map); if (error.toString().isEmpty()) { @@ -86,7 +86,7 @@ public KruizeObject(String experimentName, this.selectorInfo = selectorInfo; this.experimentId = Utils.generateID(toString()); this.objectReference = objectReference; - this.clusterName = clusterNameContent; + this.clusterName = clusterName; } else { throw new InvalidValueException(error.toString()); } From 15d9c459677c7ef3c665cf8892fc9e771ca001de Mon Sep 17 00:00:00 2001 From: bharathappali Date: Mon, 20 Feb 2023 10:24:53 +0530 Subject: [PATCH 34/35] Repositioning cluster name Signed-off-by: bharathappali --- .../java/com/autotune/analyzer/deployment/KruizeDeployment.java | 2 +- src/main/java/com/autotune/common/k8sObjects/KruizeObject.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/autotune/analyzer/deployment/KruizeDeployment.java b/src/main/java/com/autotune/analyzer/deployment/KruizeDeployment.java index afe970cd9..929dfa320 100644 --- a/src/main/java/com/autotune/analyzer/deployment/KruizeDeployment.java +++ b/src/main/java/com/autotune/analyzer/deployment/KruizeDeployment.java @@ -416,10 +416,10 @@ private static KruizeObject getAutotuneObject(String autotuneObjectJsonStr) { uid); return new KruizeObject(name, + clusterName, namespace, mode, targetCluster, - clusterName, sloInfo, selectorInfo, objectReference diff --git a/src/main/java/com/autotune/common/k8sObjects/KruizeObject.java b/src/main/java/com/autotune/common/k8sObjects/KruizeObject.java index ea28fcc87..7e337385f 100644 --- a/src/main/java/com/autotune/common/k8sObjects/KruizeObject.java +++ b/src/main/java/com/autotune/common/k8sObjects/KruizeObject.java @@ -58,10 +58,10 @@ public final class KruizeObject { private String clusterName; public KruizeObject(String experimentName, + String clusterName, String namespace, String mode, String targetCluster, - String clusterName, SloInfo sloInfo, SelectorInfo selectorInfo, ObjectReference objectReference From 461056e628a8232a143516068fdd21f925bb0d25 Mon Sep 17 00:00:00 2001 From: saakhan Date: Mon, 20 Feb 2023 11:16:35 +0530 Subject: [PATCH 35/35] minor slo condition fix, cmd to remove per-profile from deploy script on termination Signed-off-by: saakhan --- scripts/minikube-helpers.sh | 4 ++++ .../autotune/analyzer/deployment/KruizeDeployment.java | 10 +++------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/minikube-helpers.sh b/scripts/minikube-helpers.sh index 8c6f21f4c..7aab8a13a 100755 --- a/scripts/minikube-helpers.sh +++ b/scripts/minikube-helpers.sh @@ -130,6 +130,10 @@ function minikube_terminate() { kubectl_cmd="kubectl -n ${autotune_ns}" + echo + echo "Removing Performance Profile" + ${kubectl_cmd} delete -f ${AUTOTUNE_PERF_PROFILE_CRD} 2>/dev/null + echo echo "Removing autotune" ${kubectl_cmd} delete -f ${AUTOTUNE_DEPLOY_MANIFEST} 2>/dev/null diff --git a/src/main/java/com/autotune/analyzer/deployment/KruizeDeployment.java b/src/main/java/com/autotune/analyzer/deployment/KruizeDeployment.java index 29317d339..acd999e30 100644 --- a/src/main/java/com/autotune/analyzer/deployment/KruizeDeployment.java +++ b/src/main/java/com/autotune/analyzer/deployment/KruizeDeployment.java @@ -342,26 +342,22 @@ private static KruizeObject getKruizeObject(String autotuneObjectJsonStr) { hpoAlgoImpl = specJson.optString(AnalyzerConstants.AutotuneObjectConstants.HPO_ALGO_IMPL, AnalyzerConstants.AutotuneObjectConstants.DEFAULT_HPO_ALGO_IMPL); + JSONObject selectorJson = null; if (specJson != null) { sloJson = specJson.optJSONObject(AnalyzerConstants.AutotuneObjectConstants.SLO); perfProfileName = specJson.optString(AnalyzerConstants.PerformanceProfileConstants.PERF_PROFILE); - if (Stream.of(sloJson, perfProfileName).allMatch(Objects::isNull)) { + if (sloJson.isEmpty() && perfProfileName.isEmpty()) { throw new SloClassNotSupportedException(AnalyzerErrorConstants.AutotuneObjectErrors.MISSING_SLO_DATA); } - if (Stream.of(sloJson, perfProfileName).noneMatch(Objects::isNull)) { + if (!sloJson.isEmpty() && !perfProfileName.isEmpty()) { throw new SloClassNotSupportedException(AnalyzerErrorConstants.AutotuneObjectErrors.SLO_REDUNDANCY_ERROR); } SloInfo sloInfo = new Gson().fromJson(String.valueOf(sloJson), SloInfo.class); perfProfileName = setDefaultPerformanceProfile(sloInfo, mode, targetCluster); - } - - JSONObject selectorJson = null; - if (specJson != null) { selectorJson = specJson.getJSONObject(AnalyzerConstants.AutotuneObjectConstants.SELECTOR); } - assert selectorJson != null; String matchLabel = selectorJson.optString(AnalyzerConstants.AutotuneObjectConstants.MATCH_LABEL); String matchLabelValue = selectorJson.optString(AnalyzerConstants.AutotuneObjectConstants.MATCH_LABEL_VALUE);