test(cluster): split tests between parallel and sequential packages (#…

…5390)

internal/daemon/controller/testing.go

@@ -529,7 +529,6 @@ func NewTestController(t testing.TB, opts *TestControllerOpts) *TestController {
 	var err error
 	tc.c, err = New(ctx, conf)
 	if err != nil {
-		tc.Shutdown()
 		t.Fatal(err)
 	}
 
@@ -552,7 +551,6 @@ func NewTestController(t testing.TB, opts *TestControllerOpts) *TestController {
 
 	if !opts.DisableAutoStart {
 		if err := tc.c.Start(); err != nil {
-			tc.Shutdown()
 			t.Fatal(err)
 		}
 	}

internal/daemon/worker/testing.go

@@ -72,6 +72,9 @@ func (tw *TestWorker) Name() string {
 func (tw *TestWorker) UpstreamAddrs() []string {
 	var addrs []string
 	lastStatus := tw.w.LastStatusSuccess()
+	if lastStatus == nil {
+		return addrs
+	}
 	for _, v := range lastStatus.GetCalculatedUpstreams() {
 		addrs = append(addrs, v.Address)
 	}
@@ -360,7 +363,6 @@ func NewTestWorker(t testing.TB, opts *TestWorkerOpts) *TestWorker {
 
 	tw.w, err = New(ctx, conf)
 	if err != nil {
-		tw.Shutdown()
 		t.Fatal(err)
 	}
 
@@ -387,7 +389,6 @@ func NewTestWorker(t testing.TB, opts *TestWorkerOpts) *TestWorker {
 
 	if !opts.DisableAutoStart {
 		if err := tw.w.Start(); err != nil {
-			tw.Shutdown()
 			t.Fatal(err)
 		}
 	}

internal/tests/cluster/anon_listing_test.go → internal/tests/cluster/parallel/anon_listing_test.go

@@ -1,7 +1,7 @@
 // Copyright (c) HashiCorp, Inc.
 // SPDX-License-Identifier: BUSL-1.1
 
-package cluster
+package parallel
 
 import (
 	"testing"
@@ -15,6 +15,8 @@ import (
 )
 
 func TestAnonListing(t *testing.T) {
+	t.Parallel()
+
 	require := require.New(t)
 	logger := hclog.New(&hclog.LoggerOptions{
 		Level: hclog.Trace,
@@ -28,7 +30,6 @@ func TestAnonListing(t *testing.T) {
 		InitialResourcesSuffix: "1234567890",
 		Logger:                 logger,
 	})
-	defer c1.Shutdown()
 
 	// Anon user has list and read permissions on scopes by default,
 	// verify that list scopes returns expected scope without setting token

internal/tests/cluster/parallel/doc.go

@@ -0,0 +1,13 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+/*
+This package includes a set of tests that run in parallel.
+A test should only be added to this package if it can be
+completely isolated from other tests that currently exist
+in this package. If a test is consistently failing due to
+not having an isolated environment, it should be moved to
+the adjacent "sequential" package.
+*/
+
+package parallel

internal/tests/cluster/ipv6_listener_test.go → internal/tests/cluster/parallel/ipv6_listener_test.go

@@ -1,7 +1,7 @@
 // Copyright (c) HashiCorp, Inc.
 // SPDX-License-Identifier: BUSL-1.1
 
-package cluster
+package parallel
 
 import (
 	"context"
@@ -20,6 +20,7 @@ import (
 
 func TestIPv6Listener(t *testing.T) {
 	t.Parallel()
+
 	require := require.New(t)
 	logger := hclog.New(&hclog.LoggerOptions{
 		Level: hclog.Trace,
@@ -32,13 +33,11 @@ func TestIPv6Listener(t *testing.T) {
 		Config: conf,
 		Logger: logger.Named("c1"),
 	})
-	defer c1.Shutdown()
 
 	c2 := c1.AddClusterControllerMember(t, &controller.TestControllerOpts{
 		Config: conf,
 		Logger: c1.Config().Logger.ResetNamed("c2"),
 	})
-	defer c2.Shutdown()
 
 	wg := new(sync.WaitGroup)
 	wg.Add(2)
@@ -61,7 +60,6 @@ func TestIPv6Listener(t *testing.T) {
 		InitialUpstreams: append(c1.ClusterAddrs(), c2.ClusterAddrs()...),
 		Logger:           logger.Named("w1"),
 	})
-	defer w1.Shutdown()
 
 	wg.Add(2)
 	go func() {

internal/tests/cluster/multi_controller_worker_test.go → internal/tests/cluster/parallel/multi_controller_worker_test.go

@@ -1,10 +1,10 @@
 // Copyright (c) HashiCorp, Inc.
 // SPDX-License-Identifier: BUSL-1.1
 
-package cluster
+package parallel
 
 import (
-	"context"
+	"slices"
 	"sync"
 	"testing"
 	"time"
@@ -14,12 +14,12 @@ import (
 	"github.com/hashicorp/boundary/internal/daemon/worker"
 	"github.com/hashicorp/boundary/internal/tests/helper"
 	"github.com/hashicorp/go-hclog"
-	"github.com/hashicorp/go-secure-stdlib/strutil"
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestMultiControllerMultiWorkerConnections(t *testing.T) {
+	t.Parallel()
+
 	require := require.New(t)
 	logger := hclog.New(&hclog.LoggerOptions{
 		Level: hclog.Trace,
@@ -32,12 +32,10 @@ func TestMultiControllerMultiWorkerConnections(t *testing.T) {
 		Config: conf,
 		Logger: logger.Named("c1"),
 	})
-	defer c1.Shutdown()
 
 	c2 := c1.AddClusterControllerMember(t, &controller.TestControllerOpts{
 		Logger: c1.Config().Logger.ResetNamed("c2"),
 	})
-	defer c2.Shutdown()
 
 	wg := new(sync.WaitGroup)
 	wg.Add(2)
@@ -56,7 +54,6 @@ func TestMultiControllerMultiWorkerConnections(t *testing.T) {
 		InitialUpstreams: append(c1.ClusterAddrs(), c2.ClusterAddrs()...),
 		Logger:           logger.Named("w1"),
 	})
-	defer w1.Shutdown()
 
 	wg.Add(2)
 	go func() {
@@ -72,7 +69,6 @@ func TestMultiControllerMultiWorkerConnections(t *testing.T) {
 	w2 := w1.AddClusterWorkerMember(t, &worker.TestWorkerOpts{
 		Logger: logger.Named("w2"),
 	})
-	defer w2.Shutdown()
 
 	wg.Add(2)
 	go func() {
@@ -98,53 +94,47 @@ func TestMultiControllerMultiWorkerConnections(t *testing.T) {
 	}()
 	wg.Wait()
 
-	w1 = worker.NewTestWorker(t, &worker.TestWorkerOpts{
+	w3 := worker.NewTestWorker(t, &worker.TestWorkerOpts{
 		WorkerAuthKms:    c1.Config().WorkerAuthKms,
 		InitialUpstreams: c1.ClusterAddrs(),
-		Logger:           logger.Named("w1"),
+		Logger:           logger.Named("w3"),
 	})
-	defer w1.Shutdown()
 
 	wg.Add(2)
 	go func() {
 		defer wg.Done()
-		helper.ExpectWorkers(t, c1, w1, w2)
+		helper.ExpectWorkers(t, c1, w2, w3)
 	}()
 	go func() {
 		defer wg.Done()
-		helper.ExpectWorkers(t, c2, w1, w2)
+		helper.ExpectWorkers(t, c2, w2, w3)
 	}()
 	wg.Wait()
 
 	require.NoError(c2.Controller().Shutdown())
 
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		helper.ExpectWorkers(t, c1, w1, w2)
-	}()
-	wg.Wait()
+	helper.ExpectWorkers(t, c1, w2, w3)
 
-	c2 = c1.AddClusterControllerMember(t, &controller.TestControllerOpts{
-		Logger: c1.Config().Logger.ResetNamed("c2"),
+	c3 := c1.AddClusterControllerMember(t, &controller.TestControllerOpts{
+		Logger: c1.Config().Logger.ResetNamed("c3"),
 	})
-	defer c2.Shutdown()
 
 	wg.Add(2)
 	go func() {
 		defer wg.Done()
-		helper.ExpectWorkers(t, c1, w1, w2)
+		helper.ExpectWorkers(t, c1, w2, w3)
 	}()
 	go func() {
 		defer wg.Done()
-		helper.ExpectWorkers(t, c2, w1, w2)
+		helper.ExpectWorkers(t, c3, w2, w3)
 	}()
 	wg.Wait()
 }
 
 func TestWorkerAppendInitialUpstreams(t *testing.T) {
-	ctx := context.Background()
-	require, assert := require.New(t), assert.New(t)
+	t.Parallel()
+
+	require := require.New(t)
 	logger := hclog.New(&hclog.LoggerOptions{
 		Level: hclog.Trace,
 	})
@@ -156,7 +146,6 @@ func TestWorkerAppendInitialUpstreams(t *testing.T) {
 		Config: conf,
 		Logger: logger.Named("c1"),
 	})
-	defer c1.Shutdown()
 
 	helper.ExpectWorkers(t, c1)
 
@@ -167,31 +156,29 @@ func TestWorkerAppendInitialUpstreams(t *testing.T) {
 		Logger:                              logger.Named("w1"),
 		SuccessfulStatusGracePeriodDuration: 1 * time.Second,
 	})
-	defer w1.Shutdown()
 
 	// Wait for worker to send status
-	cancelCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
-	t.Cleanup(cancel)
-	for {
-		select {
-		case <-time.After(500 * time.Millisecond):
-		case <-cancelCtx.Done():
-			require.FailNow("No worker found after 10 seconds")
-		}
-		successSent := w1.Worker().LastStatusSuccess()
-		if successSent != nil {
-			break
-		}
-	}
 	helper.ExpectWorkers(t, c1, w1)
 
 	// Upstreams should be equivalent to the controller cluster addr after status updates
-	assert.Equal(c1.ClusterAddrs(), w1.Worker().LastStatusSuccess().LastCalculatedUpstreams)
+	require.Eventually(func() bool {
+		if w1.Worker().LastStatusSuccess() == nil {
+			return false
+		}
+		return slices.Equal(c1.ClusterAddrs(), w1.Worker().LastStatusSuccess().LastCalculatedUpstreams)
+	}, 4*time.Second, 250*time.Millisecond)
 
 	// Bring down the controller
-	c1.Shutdown()
-	time.Sleep(3 * time.Second) // Wait a little longer than the grace period
+	wg := new(sync.WaitGroup)
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		c1.Shutdown()
+	}()
 
 	// Upstreams should now match initial upstreams
-	assert.True(strutil.EquivalentSlices(initialUpstreams, w1.Worker().LastStatusSuccess().LastCalculatedUpstreams))
+	require.Eventually(func() bool {
+		return slices.Equal(initialUpstreams, w1.Worker().LastStatusSuccess().LastCalculatedUpstreams)
+	}, 4*time.Second, 250*time.Millisecond)
+	wg.Wait()
 }

internal/tests/cluster/package_registry_test.go → internal/tests/cluster/parallel/package_registry_test.go

@@ -1,7 +1,7 @@
 // Copyright (c) HashiCorp, Inc.
 // SPDX-License-Identifier: BUSL-1.1
 
-package cluster
+package parallel
 
 import (
 	// Enable tcp target support.

internal/tests/cluster/recursive_anon_listing_test.go → internal/tests/cluster/parallel/recursive_anon_listing_test.go

@@ -1,7 +1,7 @@
 // Copyright (c) HashiCorp, Inc.
 // SPDX-License-Identifier: BUSL-1.1
 
-package cluster
+package parallel
 
 import (
 	"strings"
@@ -16,9 +16,10 @@ import (
 
 // This test validates the fix for ICU-2301
 func TestListAnonymousRecursing(t *testing.T) {
+	t.Parallel()
+
 	require := require.New(t)
 	tc := controller.NewTestController(t, nil)
-	defer tc.Shutdown()
 
 	client := tc.Client()
 	token := tc.Token()