feat: Add E2E Endpoint Checks (#864)

**Reason for Change**: Add checks for checking two endpoints `validateCompletionsEndpoint` and `validateModelsEndpoint` --------- Signed-off-by: jerryzhuang <[email protected]> Co-authored-by: jerryzhuang <[email protected]>
kaito-project · Feb 12, 2025 · 2a9b5c5 · 2a9b5c5
1 parent d6cd2f2
commit 2a9b5c5
Show file tree

Hide file tree

Showing 4 changed files with 109 additions and 3 deletions.
diff --git a/test/e2e/inference_with_adapters_test.go b/test/e2e/inference_with_adapters_test.go
@@ -160,7 +160,8 @@ func validateAdapterAdded(workspaceObj *kaitov1alpha1.Workspace, deploymentName
 	})
 }
 
-func validateAdapterLoadedInVLLM(workspaceObj *kaitov1alpha1.Workspace, deploymentName string, adapterName string) {
+func validateAdapterLoadedInVLLM(workspaceObj *kaitov1alpha1.Workspace, adapterName string) {
+	deploymentName := workspaceObj.Name
 	execOption := corev1.PodExecOptions{
 		Command:   []string{"bash", "-c", "apt-get update && apt-get install curl -y; curl -s 127.0.0.1:5000/v1/models | grep " + adapterName},
 		Container: deploymentName,

diff --git a/test/e2e/preset_test.go b/test/e2e/preset_test.go
@@ -4,6 +4,7 @@
 package e2e
 
 import (
+	"context"
 	"fmt"
 	"log"
 	"math/rand"
@@ -17,6 +18,7 @@ import (
 	"github.com/samber/lo"
 	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -608,6 +610,93 @@ func validateWorkspaceReadiness(workspaceObj *kaitov1alpha1.Workspace) {
 	})
 }
 
+func validateModelsEndpoint(workspaceObj *kaitov1alpha1.Workspace) {
+	deploymentName := workspaceObj.Name
+	modelName := workspaceObj.Inference.Preset.Name
+	expectedModelID := fmt.Sprintf(`"id":"%s"`, modelName)
+	execOption := corev1.PodExecOptions{
+		Command:   []string{"bash", "-c", fmt.Sprintf(`apt-get update && apt-get install curl -y; curl -s -X GET http://%s.%s.svc.cluster.local:80/v1/models | grep -e '%s'`, workspaceObj.Name, workspaceObj.Namespace, expectedModelID)},
+		Container: deploymentName,
+		Stdout:    true,
+		Stderr:    true,
+	}
+
+	By("Validating the /v1/models endpoint", func() {
+		Eventually(func() bool {
+			coreClient, err := utils.GetK8sClientset()
+			if err != nil {
+				GinkgoWriter.Printf("Failed to create core client: %v\n", err)
+				return false
+			}
+
+			namespace := workspaceObj.Namespace
+			podName, err := utils.GetPodNameForDeployment(coreClient, namespace, deploymentName)
+			if err != nil {
+				GinkgoWriter.Printf("Failed to get pod name for deployment %s: %v\n", deploymentName, err)
+				return false
+			}
+
+			k8sConfig, err := utils.GetK8sConfig()
+			if err != nil {
+				GinkgoWriter.Printf("Failed to get k8s config: %v\n", err)
+				return false
+			}
+
+			ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
+			defer cancel()
+			_, err = utils.ExecSync(ctx, k8sConfig, coreClient, namespace, podName, execOption)
+			if err != nil {
+				GinkgoWriter.Printf("validate command fails: %v\n", err)
+				return false
+			}
+			return true
+		}, 5*time.Minute, utils.PollInterval).Should(BeTrue(), "Failed to wait for /v1/models endpoint to be ready")
+	})
+}
+
+func validateCompletionsEndpoint(workspaceObj *kaitov1alpha1.Workspace) {
+	deploymentName := workspaceObj.Name
+	expectedCompletion := `"object":"text_completion"`
+	execOption := corev1.PodExecOptions{
+		Command:   []string{"bash", "-c", fmt.Sprintf(`apt-get update && apt-get install curl -y; curl -s -X POST -H "Content-Type: application/json" -d '{"model":"%s","prompt":"What is Kubernetes?","max_tokens":7,"temperature":0}' http://%s.%s.svc.cluster.local:80/v1/completions | grep -e '%s'`, workspaceObj.Inference.Preset.Name, workspaceObj.Name, workspaceObj.Namespace, expectedCompletion)},
+		Container: deploymentName,
+		Stdout:    true,
+		Stderr:    true,
+	}
+
+	By("Validating the /v1/completions endpoint", func() {
+		Eventually(func() bool {
+			coreClient, err := utils.GetK8sClientset()
+			if err != nil {
+				GinkgoWriter.Printf("Failed to create core client: %v\n", err)
+				return false
+			}
+
+			namespace := workspaceObj.Namespace
+			podName, err := utils.GetPodNameForDeployment(coreClient, namespace, deploymentName)
+			if err != nil {
+				GinkgoWriter.Printf("Failed to get pod name for deployment %s: %v\n", deploymentName, err)
+				return false
+			}
+
+			k8sConfig, err := utils.GetK8sConfig()
+			if err != nil {
+				GinkgoWriter.Printf("Failed to get k8s config: %v\n", err)
+				return false
+			}
+
+			ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
+			defer cancel()
+			_, err = utils.ExecSync(ctx, k8sConfig, coreClient, namespace, podName, execOption)
+			if err != nil {
+				GinkgoWriter.Printf("validate command fails: %v\n", err)
+				return false
+			}
+			return true
+		}, 5*time.Minute, utils.PollInterval).Should(BeTrue(), "Failed to wait for /v1/completions endpoint to be ready")
+	})
+}
+
 func cleanupResources(workspaceObj *kaitov1alpha1.Workspace) {
 	By("Cleaning up resources", func() {
 		if !CurrentSpecReport().Failed() {

diff --git a/test/e2e/preset_vllm_test.go b/test/e2e/preset_vllm_test.go
@@ -45,6 +45,8 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 		validateInferenceResource(workspaceObj, int32(numOfNode), false)
 
 		validateWorkspaceReadiness(workspaceObj)
+		validateModelsEndpoint(workspaceObj)
+		validateCompletionsEndpoint(workspaceObj)
 	})
 
 	It("should create a deepseek-distilled-qwen-14b workspace with preset public mode successfully", func() {
@@ -68,6 +70,8 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 		validateInferenceResource(workspaceObj, int32(numOfNode), false)
 
 		validateWorkspaceReadiness(workspaceObj)
+		validateModelsEndpoint(workspaceObj)
+		validateCompletionsEndpoint(workspaceObj)
 	})
 
 	It("should create a falcon workspace with preset public mode successfully", func() {
@@ -88,6 +92,8 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 		validateInferenceResource(workspaceObj, int32(numOfNode), false)
 
 		validateWorkspaceReadiness(workspaceObj)
+		validateModelsEndpoint(workspaceObj)
+		validateCompletionsEndpoint(workspaceObj)
 	})
 
 	It("should create a mistral workspace with preset public mode successfully", func() {
@@ -108,6 +114,8 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 		validateInferenceResource(workspaceObj, int32(numOfNode), false)
 
 		validateWorkspaceReadiness(workspaceObj)
+		validateModelsEndpoint(workspaceObj)
+		validateCompletionsEndpoint(workspaceObj)
 	})
 
 	It("should create a Phi-2 workspace with preset public mode successfully", func() {
@@ -128,6 +136,8 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 		validateInferenceResource(workspaceObj, int32(numOfNode), false)
 
 		validateWorkspaceReadiness(workspaceObj)
+		validateModelsEndpoint(workspaceObj)
+		validateCompletionsEndpoint(workspaceObj)
 	})
 
 	It("should create a Phi-3-mini-128k-instruct workspace with preset public mode successfully", func() {
@@ -148,6 +158,8 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 		validateInferenceResource(workspaceObj, int32(numOfNode), false)
 
 		validateWorkspaceReadiness(workspaceObj)
+		validateModelsEndpoint(workspaceObj)
+		validateCompletionsEndpoint(workspaceObj)
 	})
 
 	It("should create a qwen2.5 coder workspace with preset public mode and 2 gpu successfully", func() {
@@ -169,6 +181,8 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 		validateInferenceResource(workspaceObj, int32(numOfNode), false)
 
 		validateWorkspaceReadiness(workspaceObj)
+		validateModelsEndpoint(workspaceObj)
+		validateCompletionsEndpoint(workspaceObj)
 	})
 
 	It("should create a falcon workspace with adapter successfully", func() {
@@ -189,10 +203,12 @@ var _ = Describe("Workspace Preset on vllm runtime", func() {
 		validateInferenceResource(workspaceObj, int32(numOfNode), false)
 
 		validateWorkspaceReadiness(workspaceObj)
+		validateModelsEndpoint(workspaceObj)
+		validateCompletionsEndpoint(workspaceObj)
 
 		validateInitContainers(workspaceObj, expectedInitContainers2)
 
-		validateAdapterLoadedInVLLM(workspaceObj, workspaceObj.Name, imageName2)
+		validateAdapterLoadedInVLLM(workspaceObj, imageName2)
 	})
 })
 

diff --git a/test/e2e/utils/utils.go b/test/e2e/utils/utils.go
@@ -180,7 +180,7 @@ func ExecSync(ctx context.Context, config *rest.Config, coreClient *kubernetes.C
 		Stderr: &stderr,
 	})
 	if err != nil {
-		return "", fmt.Errorf("failed to execute command: %w, stderr: %q", err, stderr.String())
+		return "", fmt.Errorf("failed to execute command %v: %w, stderr: %q", options.Command, err, stderr.String())
 	}
 
 	if stderr.Len() > 0 {