Add breakdown of tests output for integration testing framework

magefile.go

@@ -32,6 +32,8 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/elastic/elastic-agent/pkg/testing/supported"
+
 	"github.com/jedib0t/go-pretty/v6/table"
 	"github.com/otiai10/copy"
 
@@ -2552,19 +2554,19 @@ func (Integration) TestOnRemote(ctx context.Context) error {
 	return nil
 }
 
-func (Integration) Buildkite() error {
+func getSupportedBatches(matrix bool) ([]tcommon.OSBatch, tcommon.Config, error) {
 	goTestFlags := os.Getenv("GOTEST_FLAGS")
 	batches, err := define.DetermineBatches("testing/integration", goTestFlags, "integration")
 	if err != nil {
-		return fmt.Errorf("failed to determine batches: %w", err)
+		return nil, tcommon.Config{}, fmt.Errorf("failed to determine batches: %w", err)
 	}
 	agentVersion, agentStackVersion, err := getTestRunnerVersions()
 	if err != nil {
-		return fmt.Errorf("failed to get agent versions: %w", err)
+		return nil, tcommon.Config{}, fmt.Errorf("failed to get agent versions: %w", err)
 	}
 	goVersion, err := mage.DefaultBeatBuildVariableSources.GetGoVersion()
 	if err != nil {
-		return fmt.Errorf("failed to get go versions: %w", err)
+		return nil, tcommon.Config{}, fmt.Errorf("failed to get go versions: %w", err)
 	}
 
 	cfg := tcommon.Config{
@@ -2574,11 +2576,112 @@ func (Integration) Buildkite() error {
 		Platforms:    testPlatforms(),
 		Packages:     testPackages(),
 		Groups:       testGroups(),
-		Matrix:       false,
+		Matrix:       matrix,
 		VerboseMode:  mg.Verbose(),
 		TestFlags:    goTestFlags,
 	}
 
+	platforms, err := cfg.GetPlatforms()
+	if err != nil {
+		return nil, cfg, err
+	}
+	osBatches, err := supported.CreateBatches(batches, platforms, cfg.Groups, cfg.Matrix, cfg.SingleTest)
+	if err != nil {
+		return nil, cfg, err
+	}
+
+	return osBatches, cfg, nil
+}
+
+type groupSummary struct {
+	Name        string                `yaml:"name"`
+	OS          []tcommon.SupportedOS `yaml:"os"`
+	RequireSudo bool                  `yaml:"require_sudo"`
+	Tests       []string              `yaml:"tests"`
+}
+
+func writeBreakdown(matrix bool) error {
+	batches, _, err := getSupportedBatches(matrix)
+	if err != nil {
+		return err
+	}
+
+	groupsByName := make(map[string]groupSummary)
+	for _, batch := range batches {
+		if len(batch.Batch.SudoTests) > 0 {
+			groupName := fmt.Sprintf("%s-sudo", batch.Batch.Group)
+			group, _ := groupsByName[groupName]
+			group.Name = groupName
+			group.RequireSudo = true
+			group.OS = append(group.OS, batch.OS)
+			for _, pack := range batch.Batch.SudoTests {
+				for _, packTest := range pack.Tests {
+					if !slices.Contains(group.Tests, packTest.Name) {
+						group.Tests = append(group.Tests, packTest.Name)
+					}
+				}
+			}
+			groupsByName[groupName] = group
+		}
+		if len(batch.Batch.Tests) > 0 {
+			groupName := batch.Batch.Group
+			group, _ := groupsByName[groupName]
+			group.Name = groupName
+			group.OS = append(group.OS, batch.OS)
+			for _, pack := range batch.Batch.SudoTests {
+				for _, packTest := range pack.Tests {
+					if !slices.Contains(group.Tests, packTest.Name) {
+						group.Tests = append(group.Tests, packTest.Name)
+					}
+				}
+			}
+			groupsByName[groupName] = group
+		}
+	}
+
+	groups := make([]groupSummary, 0, len(groupsByName))
+	for _, group := range groupsByName {
+		groups = append(groups, group)
+	}
+
+	data, err := yaml.Marshal(groups)
+	if err != nil {
+		return err
+	}
+
+	// write output to tests.yaml
+	cwd, err := os.Getwd()
+	if err != nil {
+		return fmt.Errorf("error getting current working directory: %w", err)
+	}
+	ymlFilePath := filepath.Join(cwd, "tests.yml")
+	file, err := os.Create(ymlFilePath)
+	if err != nil {
+		return fmt.Errorf("error creating file: %w", err)
+	}
+	defer file.Close()
+	if _, err := file.Write(data); err != nil {
+		return fmt.Errorf("error writing to file: %w", err)
+	}
+
+	fmt.Printf(">>> Generated breakdown steps written to: %s\n", ymlFilePath)
+	return nil
+}
+
+func (Integration) Breakdown() error {
+	return writeBreakdown(false)
+}
+
+func (Integration) BreakdownMatrix() error {
+	return writeBreakdown(true)
+}
+
+func writeBuildkite(matrix bool) error {
+	batches, cfg, err := getSupportedBatches(matrix)
+	if err != nil {
+		return err
+	}
+
 	steps, err := buildkite.GenerateSteps(cfg, batches...)
 	if err != nil {
 		return fmt.Errorf("error generating buildkite steps: %w", err)
@@ -2603,6 +2706,14 @@ func (Integration) Buildkite() error {
 	return nil
 }
 
+func (Integration) Buildkite() error {
+	return writeBuildkite(false)
+}
+
+func (Integration) BuildkiteMatrix() error {
+	return writeBuildkite(true)
+}
+
 func integRunner(ctx context.Context, matrix bool, singleTest string) error {
 	if _, ok := ctx.Deadline(); !ok {
 		// If the context doesn't have a timeout (usually via the mage -t option), give it one.

pkg/testing/buildkite/buildkite.go

@@ -13,7 +13,6 @@ import (
 
 	"github.com/elastic/elastic-agent/pkg/testing/common"
 	"github.com/elastic/elastic-agent/pkg/testing/define"
-	"github.com/elastic/elastic-agent/pkg/testing/supported"
 )
 
 const (
@@ -201,23 +200,13 @@ func shouldSkip(os common.SupportedOS) bool {
 }
 
 // GenerateSteps returns a computed set of steps to run the integration tests on buildkite.
-func GenerateSteps(cfg common.Config, batches ...define.Batch) (string, error) {
+func GenerateSteps(cfg common.Config, batches ...common.OSBatch) (string, error) {
 	stackSteps := map[string]Step{}
 	stackTeardown := map[string][]string{}
 	var steps []Step
 
-	// create the supported batches first
-	platforms, err := cfg.GetPlatforms()
-	if err != nil {
-		return "", err
-	}
-	osBatches, err := supported.CreateBatches(batches, platforms, cfg.Groups, cfg.Matrix, cfg.SingleTest)
-	if err != nil {
-		return "", err
-	}
-
 	// create the stack steps first
-	for _, lb := range osBatches {
+	for _, lb := range batches {
 		if !lb.Skip && lb.Batch.Stack != nil {
 			if lb.Batch.Stack.Version == "" {
 				// no version defined on the stack; set it to the defined stack version
@@ -241,7 +230,7 @@ func GenerateSteps(cfg common.Config, batches ...define.Batch) (string, error) {
 	}
 
 	// generate the steps for the tests
-	for _, lb := range osBatches {
+	for _, lb := range batches {
 		if lb.Skip {
 			continue
 		}

pkg/testing/common/batch.go

@@ -11,11 +11,11 @@ import (
 // OSBatch defines the mapping between a SupportedOS and a define.Batch.
 type OSBatch struct {
 	// ID is the unique ID for the batch.
-	ID string
+	ID string `yaml:"id"`
 	// LayoutOS provides all the OS information to create an instance.
-	OS SupportedOS
+	OS SupportedOS `yaml:"os"`
 	// Batch defines the batch of tests to run on this layout.
-	Batch define.Batch
+	Batch define.Batch `yaml:"batch"`
 	// Skip defines if this batch will be skipped because no supported layout exists yet.
-	Skip bool
+	Skip bool `yaml:"skip,omitempty"`
 }

pkg/testing/common/supported.go

@@ -8,8 +8,8 @@ import "github.com/elastic/elastic-agent/pkg/testing/define"
 
 // SupportedOS maps a OS definition to a OSRunner.
 type SupportedOS struct {
-	define.OS
+	define.OS `yaml:",inline"`
 
 	// Runner is the runner to use for the OS.
-	Runner OSRunner
+	Runner OSRunner `yaml:"-"`
 }

pkg/testing/define/batch.go

@@ -43,37 +43,37 @@ var defaultOS = []OS{
 type Batch struct {
 	// Group must be set on each test to define which group the tests belongs.
 	// Tests that are in the same group are executed on the same runner.
-	Group string `json:"group"`
+	Group string `json:"group" yaml:"group"`
 
 	// OS defines the operating systems this test batch needs.
-	OS OS `json:"os"`
+	OS OS `json:"os,omitempty" yaml:"os,omitempty"`
 
 	// Stack defines the stack required for this batch.
-	Stack *Stack `json:"stack,omitempty"`
+	Stack *Stack `json:"stack,omitempty" yaml:"stack,omitempty"`
 
 	// Tests define the set of packages and tests that do not require sudo
 	// privileges to be performed.
-	Tests []BatchPackageTests `json:"tests"`
+	Tests []BatchPackageTests `json:"tests,omitempty" yaml:"tests,omitempty"`
 
 	// SudoTests define the set of packages and tests that do require sudo
 	// privileges to be performed.
-	SudoTests []BatchPackageTests `json:"sudo_tests"`
+	SudoTests []BatchPackageTests `json:"sudo_tests,omitempty" yaml:"sudo_tests,omitempty"`
 }
 
 // BatchPackageTests is a package and its tests that belong to a batch.
 type BatchPackageTests struct {
 	// Name is the package name.
-	Name string `json:"name"`
+	Name string `json:"name" yaml:"name"`
 	// Tests is the set of tests in the package.
-	Tests []BatchPackageTest `json:"tests"`
+	Tests []BatchPackageTest `json:"tests" yaml:"tests"`
 }
 
 // BatchPackageTest is a specific test in a package.
 type BatchPackageTest struct {
 	// Name of the test.
-	Name string `json:"name"`
+	Name string `json:"name" yaml:"name"`
 	// Stack needed for test.
-	Stack bool `json:"stack"`
+	Stack bool `json:"stack" yaml:"stack"`
 }
 
 // DetermineBatches parses the package directory with the possible extra build

pkg/testing/define/requirements.go

@@ -40,24 +40,24 @@ type OS struct {
 	//
 	// This is always required to be defined on the OS structure.
 	// If it is not defined the test runner will error.
-	Type string `json:"type"`
+	Type string `json:"type,omitempty" yaml:"type,omitempty"`
 	// Arch is the architecture type (amd64 or arm64).
 	//
 	// In the case that it's not provided the test will run on every
 	// architecture that is supported.
-	Arch string `json:"arch"`
+	Arch string `json:"arch,omitempty" yaml:"arch,omitempty"`
 	// Version is a specific version of the OS type to run this test on
 	//
 	// When defined the test runs on this specific version only. When not
 	// defined the test is run on a selected version for this operating system.
-	Version string `json:"version"`
+	Version string `json:"version,omitempty" yaml:"version,omitempty"`
 	// Distro allows in the Linux case for a specific distribution to be
 	// selected for running on. Example would be "ubuntu". In the Kubernetes case
 	// for a specific distribution of kubernetes. Example would be "kind".
-	Distro string `json:"distro"`
+	Distro string `json:"distro,omitempty" yaml:"distro,omitempty"`
 	// DockerVariant allows in the Kubernetes case for a specific variant to
 	// be selected for running with. Example would be "wolfi".
-	DockerVariant string `json:"docker_variant"`
+	DockerVariant string `json:"docker_variant,omitempty" yaml:"docker_variant,omitempty"`
 }
 
 // Validate returns an error if not valid.
@@ -91,7 +91,7 @@ type Stack struct {
 	//
 	// In the case that no version is provided the same version being used for
 	// the current test execution is used.
-	Version string `json:"version"`
+	Version string `json:"version" yaml:"version"`
 }
 
 // Requirements defines the testing requirements for the test to run.