test: Retina e2e scale test

Signed-off-by: Alex Castilio dos Santos <[email protected]>

test/e2e/framework/helpers/helpers.go

@@ -0,0 +1,22 @@
+package helpers
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+// Context returns a context with a deadline set to the test deadline - 1 min to ensure cleanup.
+// If the test deadline is not set, a deadline is set to Now + 24h to prevent the test from running indefinitely
+func Context(t *testing.T) (context.Context, context.CancelFunc) {
+	deadline, ok := t.Deadline()
+	if !ok {
+		t.Log("Test deadline disabled, deadline set to Now + 24h to prevent test from running indefinitely")
+		deadline = time.Now().Add(24 * time.Hour)
+	}
+
+	// Subtract a minute from the deadline to ensure we have time to cleanup
+	deadline = deadline.Add(-time.Minute)
+
+	return context.WithDeadline(context.Background(), deadline)
+}

test/e2e/framework/types/background_test.go

@@ -6,12 +6,17 @@ import (
 	"sync"
 	"testing"
 	"time"
+
+	"github.com/microsoft/retina/test/e2e/framework/helpers"
 )
 
 func TestFramework(t *testing.T) {
+	ctx, cancel := helpers.Context(t)
+	defer cancel()
+
 	job := NewJob("Validate that drop metrics are present in the prometheus endpoint")
 	runner := NewRunner(t, job)
-	defer runner.Run()
+	defer runner.Run(ctx)
 
 	job.AddStep(&TestBackground{
 		CounterName: "Example Counter",

test/e2e/framework/types/runner.go

@@ -1,6 +1,7 @@
 package types
 
 import (
+	"context"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -20,9 +21,21 @@ func NewRunner(t *testing.T, job *Job) *Runner {
 	}
 }
 
-func (r *Runner) Run() {
+func (r *Runner) Run(ctx context.Context) {
 	if r.t.Failed() {
 		return
 	}
-	require.NoError(r.t, r.Job.Run())
+	runComplete := make(chan struct{})
+	defer close(runComplete)
+
+	go func() {
+		require.NoError(r.t, r.Job.Run())
+		runComplete <- struct{}{}
+	}()
+	select {
+	case <-ctx.Done():
+		r.t.Fatal("Test deadline exceeded. If more time is needed, set -timeout flag to a higher value")
+	case <-runComplete:
+		return
+	}
 }

test/e2e/framework/types/scenarios_test.go

@@ -3,15 +3,20 @@ package types
 import (
 	"fmt"
 	"testing"
+
+	"github.com/microsoft/retina/test/e2e/framework/helpers"
 )
 
 // Test against a BYO cluster with Cilium and Hubble enabled,
 // create a pod with a deny all network policy and validate
 // that the drop metrics are present in the prometheus endpoint
 func TestScenarioValues(t *testing.T) {
+	ctx, cancel := helpers.Context(t)
+	defer cancel()
+
 	job := NewJob("Validate that drop metrics are present in the prometheus endpoint")
 	runner := NewRunner(t, job)
-	defer runner.Run()
+	defer runner.Run(ctx)
 
 	// Add top level step
 	job.AddStep(&DummyStep{
@@ -30,9 +35,12 @@ func TestScenarioValues(t *testing.T) {
 // create a pod with a deny all network policy and validate
 // that the drop metrics are present in the prometheus endpoint
 func TestScenarioValuesWithSkip(t *testing.T) {
+	ctx, cancel := helpers.Context(t)
+	defer cancel()
+
 	job := NewJob("Validate that drop metrics are present in the prometheus endpoint")
 	runner := NewRunner(t, job)
-	defer runner.Run()
+	defer runner.Run(ctx)
 
 	// Add top level step
 	job.AddStep(&DummyStep{
@@ -53,9 +61,12 @@ func TestScenarioValuesWithSkip(t *testing.T) {
 }
 
 func TestScenarioValuesWithScenarioSkip(t *testing.T) {
+	ctx, cancel := helpers.Context(t)
+	defer cancel()
+
 	job := NewJob("Validate that drop metrics are present in the prometheus endpoint")
 	runner := NewRunner(t, job)
-	defer runner.Run()
+	defer runner.Run(ctx)
 
 	// Add top level step
 	job.AddStep(&DummyStep{

test/e2e/jobs/jobs.go

@@ -62,6 +62,20 @@ func DeleteTestInfra(subID, clusterName, location string) *types.Job {
 	return job
 }
 
+func InstallRetina(kubeConfigFilePath, chartPath string) *types.Job {
+	job := types.NewJob("Install and test Retina with basic metrics")
+
+	job.AddStep(&kubernetes.InstallHelmChart{
+		Namespace:          "kube-system",
+		ReleaseName:        "retina",
+		KubeConfigFilePath: kubeConfigFilePath,
+		ChartPath:          chartPath,
+		TagEnv:             generic.DefaultTagEnv,
+	}, nil)
+
+	return job
+}
+
 func InstallAndTestRetinaBasicMetrics(kubeConfigFilePath, chartPath string) *types.Job {
 	job := types.NewJob("Install and test Retina with basic metrics")
 

test/e2e/retina_e2e_test.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/microsoft/retina/test/e2e/common"
+	"github.com/microsoft/retina/test/e2e/framework/helpers"
 	"github.com/microsoft/retina/test/e2e/framework/types"
 	jobs "github.com/microsoft/retina/test/e2e/jobs"
 	"github.com/stretchr/testify/require"
@@ -20,6 +21,9 @@ var locations = []string{"eastus2", "centralus", "southcentralus", "uksouth", "c
 
 // TestE2ERetina tests all e2e scenarios for retina
 func TestE2ERetina(t *testing.T) {
+	ctx, cancel := helpers.Context(t)
+	defer cancel()
+
 	curuser, err := user.Current()
 	require.NoError(t, err)
 
@@ -50,21 +54,17 @@ func TestE2ERetina(t *testing.T) {
 
 	// CreateTestInfra
 	createTestInfra := types.NewRunner(t, jobs.CreateTestInfra(subID, clusterName, location, kubeConfigFilePath))
-	createTestInfra.Run()
+	createTestInfra.Run(ctx)
 
-	// Hacky way to ensure that the test infra is deleted even if the test panics
-	defer func() {
-		if r := recover(); r != nil {
-			t.Logf("Recovered in TestE2ERetina, %v", r)
-		}
+	t.Cleanup(func() {
 		_ = jobs.DeleteTestInfra(subID, clusterName, location).Run()
-	}()
+	})
 
 	// Install and test Retina basic metrics
 	basicMetricsE2E := types.NewRunner(t, jobs.InstallAndTestRetinaBasicMetrics(kubeConfigFilePath, chartPath))
-	basicMetricsE2E.Run()
+	basicMetricsE2E.Run(ctx)
 
 	// Upgrade and test Retina with advanced metrics
 	advanceMetricsE2E := types.NewRunner(t, jobs.UpgradeAndTestRetinaAdvancedMetrics(kubeConfigFilePath, chartPath, profilePath))
-	advanceMetricsE2E.Run()
+	advanceMetricsE2E.Run(ctx)
 }

test/e2e/scale_test.go

@@ -0,0 +1,65 @@
+package retina
+
+import (
+	"crypto/rand"
+	"math/big"
+	"os"
+	"os/user"
+	"path/filepath"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/microsoft/retina/test/e2e/common"
+	"github.com/microsoft/retina/test/e2e/framework/helpers"
+	"github.com/microsoft/retina/test/e2e/framework/types"
+	jobs "github.com/microsoft/retina/test/e2e/jobs"
+	"github.com/stretchr/testify/require"
+)
+
+func TestE2ERetina_Scale(t *testing.T) {
+	ctx, cancel := helpers.Context(t)
+	defer cancel()
+
+	curuser, err := user.Current()
+	require.NoError(t, err)
+
+	clusterName := curuser.Username + common.NetObsRGtag + strconv.FormatInt(time.Now().Unix(), 10)
+
+	subID := os.Getenv("AZURE_SUBSCRIPTION_ID")
+	require.NotEmpty(t, subID)
+
+	location := os.Getenv("AZURE_LOCATION")
+	if location == "" {
+		var nBig *big.Int
+		nBig, err = rand.Int(rand.Reader, big.NewInt(int64(len(locations))))
+		if err != nil {
+			t.Fatalf("Failed to generate a secure random index: %v", err)
+		}
+		location = locations[nBig.Int64()]
+	}
+	cwd, err := os.Getwd()
+	require.NoError(t, err)
+
+	// Get to root of the repo by going up two directories
+	rootDir := filepath.Dir(filepath.Dir(cwd))
+
+	chartPath := filepath.Join(rootDir, "deploy", "legacy", "manifests", "controller", "helm", "retina")
+	kubeConfigFilePath := filepath.Join(rootDir, "test", "e2e", "test.pem")
+
+	// CreateTestInfra
+	createTestInfra := types.NewRunner(t, jobs.CreateTestInfra(subID, clusterName, location, kubeConfigFilePath))
+	createTestInfra.Run(ctx)
+
+	t.Cleanup(func() {
+		_ = jobs.DeleteTestInfra(subID, clusterName, location).Run()
+	})
+
+	// Install Retina
+	installRetina := types.NewRunner(t, jobs.InstallRetina(kubeConfigFilePath, chartPath))
+	installRetina.Run(ctx)
+
+	// TODO: Scale Cluster
+
+	// TODO: equivalent of github.com/Azure/azure-container-networking/test/scale/test-scale.sh
+}