diff --git a/magefile.go b/magefile.go index 4ab9db4743a..91dd3603aaa 100644 --- a/magefile.go +++ b/magefile.go @@ -32,6 +32,8 @@ import ( "sync/atomic" "time" + "github.com/elastic/elastic-agent/pkg/testing/supported" + "github.com/jedib0t/go-pretty/v6/table" "github.com/otiai10/copy" @@ -2552,19 +2554,19 @@ func (Integration) TestOnRemote(ctx context.Context) error { return nil } -func (Integration) Buildkite() error { +func getSupportedBatches(matrix bool) ([]tcommon.OSBatch, tcommon.Config, error) { goTestFlags := os.Getenv("GOTEST_FLAGS") batches, err := define.DetermineBatches("testing/integration", goTestFlags, "integration") if err != nil { - return fmt.Errorf("failed to determine batches: %w", err) + return nil, tcommon.Config{}, fmt.Errorf("failed to determine batches: %w", err) } agentVersion, agentStackVersion, err := getTestRunnerVersions() if err != nil { - return fmt.Errorf("failed to get agent versions: %w", err) + return nil, tcommon.Config{}, fmt.Errorf("failed to get agent versions: %w", err) } goVersion, err := mage.DefaultBeatBuildVariableSources.GetGoVersion() if err != nil { - return fmt.Errorf("failed to get go versions: %w", err) + return nil, tcommon.Config{}, fmt.Errorf("failed to get go versions: %w", err) } cfg := tcommon.Config{ @@ -2574,11 +2576,112 @@ func (Integration) Buildkite() error { Platforms: testPlatforms(), Packages: testPackages(), Groups: testGroups(), - Matrix: false, + Matrix: matrix, VerboseMode: mg.Verbose(), TestFlags: goTestFlags, } + platforms, err := cfg.GetPlatforms() + if err != nil { + return nil, cfg, err + } + osBatches, err := supported.CreateBatches(batches, platforms, cfg.Groups, cfg.Matrix, cfg.SingleTest) + if err != nil { + return nil, cfg, err + } + + return osBatches, cfg, nil +} + +type groupSummary struct { + Name string `yaml:"name"` + OS []tcommon.SupportedOS `yaml:"os"` + RequireSudo bool `yaml:"require_sudo"` + Tests []string `yaml:"tests"` +} + +func writeBreakdown(matrix bool) error { + batches, _, err := getSupportedBatches(matrix) + if err != nil { + return err + } + + groupsByName := make(map[string]groupSummary) + for _, batch := range batches { + if len(batch.Batch.SudoTests) > 0 { + groupName := fmt.Sprintf("%s-sudo", batch.Batch.Group) + group, _ := groupsByName[groupName] + group.Name = groupName + group.RequireSudo = true + group.OS = append(group.OS, batch.OS) + for _, pack := range batch.Batch.SudoTests { + for _, packTest := range pack.Tests { + if !slices.Contains(group.Tests, packTest.Name) { + group.Tests = append(group.Tests, packTest.Name) + } + } + } + groupsByName[groupName] = group + } + if len(batch.Batch.Tests) > 0 { + groupName := batch.Batch.Group + group, _ := groupsByName[groupName] + group.Name = groupName + group.OS = append(group.OS, batch.OS) + for _, pack := range batch.Batch.SudoTests { + for _, packTest := range pack.Tests { + if !slices.Contains(group.Tests, packTest.Name) { + group.Tests = append(group.Tests, packTest.Name) + } + } + } + groupsByName[groupName] = group + } + } + + groups := make([]groupSummary, 0, len(groupsByName)) + for _, group := range groupsByName { + groups = append(groups, group) + } + + data, err := yaml.Marshal(groups) + if err != nil { + return err + } + + // write output to tests.yaml + cwd, err := os.Getwd() + if err != nil { + return fmt.Errorf("error getting current working directory: %w", err) + } + ymlFilePath := filepath.Join(cwd, "tests.yml") + file, err := os.Create(ymlFilePath) + if err != nil { + return fmt.Errorf("error creating file: %w", err) + } + defer file.Close() + if _, err := file.Write(data); err != nil { + return fmt.Errorf("error writing to file: %w", err) + } + + fmt.Printf(">>> Generated breakdown steps written to: %s\n", ymlFilePath) + return nil +} + +func (Integration) Breakdown() error { + return writeBreakdown(false) +} + +func (Integration) BreakdownMatrix() error { + return writeBreakdown(true) +} + +func writeBuildkite(matrix bool) error { + batches, cfg, err := getSupportedBatches(matrix) + if err != nil { + return err + } + steps, err := buildkite.GenerateSteps(cfg, batches...) if err != nil { return fmt.Errorf("error generating buildkite steps: %w", err) @@ -2603,6 +2706,14 @@ func (Integration) Buildkite() error { return nil } +func (Integration) Buildkite() error { + return writeBuildkite(false) +} + +func (Integration) BuildkiteMatrix() error { + return writeBuildkite(true) +} + func integRunner(ctx context.Context, matrix bool, singleTest string) error { if _, ok := ctx.Deadline(); !ok { // If the context doesn't have a timeout (usually via the mage -t option), give it one. diff --git a/pkg/testing/buildkite/buildkite.go b/pkg/testing/buildkite/buildkite.go index 7b0bac34bc1..692e6d7830b 100644 --- a/pkg/testing/buildkite/buildkite.go +++ b/pkg/testing/buildkite/buildkite.go @@ -13,7 +13,6 @@ import ( "github.com/elastic/elastic-agent/pkg/testing/common" "github.com/elastic/elastic-agent/pkg/testing/define" - "github.com/elastic/elastic-agent/pkg/testing/supported" ) const ( @@ -201,23 +200,13 @@ func shouldSkip(os common.SupportedOS) bool { } // GenerateSteps returns a computed set of steps to run the integration tests on buildkite. -func GenerateSteps(cfg common.Config, batches ...define.Batch) (string, error) { +func GenerateSteps(cfg common.Config, batches ...common.OSBatch) (string, error) { stackSteps := map[string]Step{} stackTeardown := map[string][]string{} var steps []Step - // create the supported batches first - platforms, err := cfg.GetPlatforms() - if err != nil { - return "", err - } - osBatches, err := supported.CreateBatches(batches, platforms, cfg.Groups, cfg.Matrix, cfg.SingleTest) - if err != nil { - return "", err - } - // create the stack steps first - for _, lb := range osBatches { + for _, lb := range batches { if !lb.Skip && lb.Batch.Stack != nil { if lb.Batch.Stack.Version == "" { // no version defined on the stack; set it to the defined stack version @@ -241,7 +230,7 @@ func GenerateSteps(cfg common.Config, batches ...define.Batch) (string, error) { } // generate the steps for the tests - for _, lb := range osBatches { + for _, lb := range batches { if lb.Skip { continue } diff --git a/pkg/testing/common/batch.go b/pkg/testing/common/batch.go index ade24e98826..ebe46588638 100644 --- a/pkg/testing/common/batch.go +++ b/pkg/testing/common/batch.go @@ -11,11 +11,11 @@ import ( // OSBatch defines the mapping between a SupportedOS and a define.Batch. type OSBatch struct { // ID is the unique ID for the batch. - ID string + ID string `yaml:"id"` // LayoutOS provides all the OS information to create an instance. - OS SupportedOS + OS SupportedOS `yaml:"os"` // Batch defines the batch of tests to run on this layout. - Batch define.Batch + Batch define.Batch `yaml:"batch"` // Skip defines if this batch will be skipped because no supported layout exists yet. - Skip bool + Skip bool `yaml:"skip,omitempty"` } diff --git a/pkg/testing/common/supported.go b/pkg/testing/common/supported.go index 94e17ed56cf..92f5dfae5bf 100644 --- a/pkg/testing/common/supported.go +++ b/pkg/testing/common/supported.go @@ -8,8 +8,8 @@ import "github.com/elastic/elastic-agent/pkg/testing/define" // SupportedOS maps a OS definition to a OSRunner. type SupportedOS struct { - define.OS + define.OS `yaml:",inline"` // Runner is the runner to use for the OS. - Runner OSRunner + Runner OSRunner `yaml:"-"` } diff --git a/pkg/testing/define/batch.go b/pkg/testing/define/batch.go index be254dec6eb..98a2aa74045 100644 --- a/pkg/testing/define/batch.go +++ b/pkg/testing/define/batch.go @@ -43,37 +43,37 @@ var defaultOS = []OS{ type Batch struct { // Group must be set on each test to define which group the tests belongs. // Tests that are in the same group are executed on the same runner. - Group string `json:"group"` + Group string `json:"group" yaml:"group"` // OS defines the operating systems this test batch needs. - OS OS `json:"os"` + OS OS `json:"os,omitempty" yaml:"os,omitempty"` // Stack defines the stack required for this batch. - Stack *Stack `json:"stack,omitempty"` + Stack *Stack `json:"stack,omitempty" yaml:"stack,omitempty"` // Tests define the set of packages and tests that do not require sudo // privileges to be performed. - Tests []BatchPackageTests `json:"tests"` + Tests []BatchPackageTests `json:"tests,omitempty" yaml:"tests,omitempty"` // SudoTests define the set of packages and tests that do require sudo // privileges to be performed. - SudoTests []BatchPackageTests `json:"sudo_tests"` + SudoTests []BatchPackageTests `json:"sudo_tests,omitempty" yaml:"sudo_tests,omitempty"` } // BatchPackageTests is a package and its tests that belong to a batch. type BatchPackageTests struct { // Name is the package name. - Name string `json:"name"` + Name string `json:"name" yaml:"name"` // Tests is the set of tests in the package. - Tests []BatchPackageTest `json:"tests"` + Tests []BatchPackageTest `json:"tests" yaml:"tests"` } // BatchPackageTest is a specific test in a package. type BatchPackageTest struct { // Name of the test. - Name string `json:"name"` + Name string `json:"name" yaml:"name"` // Stack needed for test. - Stack bool `json:"stack"` + Stack bool `json:"stack" yaml:"stack"` } // DetermineBatches parses the package directory with the possible extra build diff --git a/pkg/testing/define/requirements.go b/pkg/testing/define/requirements.go index d75693c5756..1550fc9f15f 100644 --- a/pkg/testing/define/requirements.go +++ b/pkg/testing/define/requirements.go @@ -40,24 +40,24 @@ type OS struct { // // This is always required to be defined on the OS structure. // If it is not defined the test runner will error. - Type string `json:"type"` + Type string `json:"type,omitempty" yaml:"type,omitempty"` // Arch is the architecture type (amd64 or arm64). // // In the case that it's not provided the test will run on every // architecture that is supported. - Arch string `json:"arch"` + Arch string `json:"arch,omitempty" yaml:"arch,omitempty"` // Version is a specific version of the OS type to run this test on // // When defined the test runs on this specific version only. When not // defined the test is run on a selected version for this operating system. - Version string `json:"version"` + Version string `json:"version,omitempty" yaml:"version,omitempty"` // Distro allows in the Linux case for a specific distribution to be // selected for running on. Example would be "ubuntu". In the Kubernetes case // for a specific distribution of kubernetes. Example would be "kind". - Distro string `json:"distro"` + Distro string `json:"distro,omitempty" yaml:"distro,omitempty"` // DockerVariant allows in the Kubernetes case for a specific variant to // be selected for running with. Example would be "wolfi". - DockerVariant string `json:"docker_variant"` + DockerVariant string `json:"docker_variant,omitempty" yaml:"docker_variant,omitempty"` } // Validate returns an error if not valid. @@ -91,7 +91,7 @@ type Stack struct { // // In the case that no version is provided the same version being used for // the current test execution is used. - Version string `json:"version"` + Version string `json:"version" yaml:"version"` } // Requirements defines the testing requirements for the test to run.