From ce5eff05905a12dd2224ba6653bba0147a4e1123 Mon Sep 17 00:00:00 2001 From: Davide Petilli Date: Wed, 17 Aug 2022 18:11:58 +0200 Subject: [PATCH] global: enable parallel execution (#127) * workdir: add Dealer cache * mutator: refactor apply and rollback * mutator: enable mutations to be applied in parallel * mutator: basic parallel implementation * mutator: refactor mutator job * docs: update for parallel execution * global: fix linting errors * cmd: add int flag test --- cmd/internal/flags/flags.go | 10 + cmd/internal/flags/flags_test.go | 19 +- cmd/unleash.go | 24 +- cmd/unleash_test.go | 15 + configuration/configuration.go | 3 + .../commands/{unleash.md => unleash/index.md} | 63 ++- docs/docs/usage/commands/unleash/workers.md | 36 ++ docs/docs/usage/configuration.md | 13 +- docs/mkdocs.yml | 4 +- pkg/mutator/executor.go | 270 ++++++++++ pkg/mutator/executor_test.go | 472 ++++++++++++++++++ pkg/mutator/internal/tokenmutant.go | 137 +++-- pkg/mutator/internal/tokenmutant_test.go | 3 +- pkg/mutator/internal/workerpool/workerpool.go | 134 +++++ .../internal/workerpool/workerpool_test.go | 201 ++++++++ pkg/mutator/mutator.go | 219 ++------ pkg/mutator/mutator_test.go | 365 ++------------ pkg/mutator/stubs_test.go | 182 +++++++ pkg/mutator/workdir/workdir.go | 100 ++-- pkg/mutator/workdir/workdir_test.go | 124 ++++- 20 files changed, 1779 insertions(+), 615 deletions(-) rename docs/docs/usage/commands/{unleash.md => unleash/index.md} (70%) create mode 100644 docs/docs/usage/commands/unleash/workers.md create mode 100644 pkg/mutator/executor.go create mode 100644 pkg/mutator/executor_test.go create mode 100644 pkg/mutator/internal/workerpool/workerpool.go create mode 100644 pkg/mutator/internal/workerpool/workerpool_test.go create mode 100644 pkg/mutator/stubs_test.go diff --git a/cmd/internal/flags/flags.go b/cmd/internal/flags/flags.go index 04a048f0..4d7fdecd 100644 --- a/cmd/internal/flags/flags.go +++ b/cmd/internal/flags/flags.go @@ -55,6 +55,8 @@ func setFlags(flag *Flag, fs *pflag.FlagSet) error { setBool(flag, fs, dv) case string: setString(flag, fs, dv) + case int: + setInt(flag, fs, dv) case float64: setFloat64(flag, fs, dv) } @@ -66,6 +68,14 @@ func setFlags(flag *Flag, fs *pflag.FlagSet) error { return nil } +func setInt(flag *Flag, flags *pflag.FlagSet, dv int) { + if flag.Shorthand != "" { + flags.IntP(flag.Name, flag.Shorthand, dv, flag.Usage) + } else { + flags.Int(flag.Name, dv, flag.Usage) + } +} + func setFloat64(flag *Flag, flags *pflag.FlagSet, dv float64) { if flag.Shorthand != "" { flags.Float64P(flag.Name, flag.Shorthand, dv, flag.Usage) diff --git a/cmd/internal/flags/flags_test.go b/cmd/internal/flags/flags_test.go index 8c583af7..97ebee8a 100644 --- a/cmd/internal/flags/flags_test.go +++ b/cmd/internal/flags/flags_test.go @@ -67,7 +67,24 @@ func TestSet(t *testing.T) { Usage: "test usage", }, }, - + { + flag: Flag{ + Name: "int-flag-no-sh", + CfgKey: "test.cfg", + Shorthand: "", + DefaultV: 0, + Usage: "test usage", + }, + }, + { + flag: Flag{ + Name: "int-flag-sh", + CfgKey: "test.cfg", + Shorthand: "t", + DefaultV: 0, + Usage: "test usage", + }, + }, { flag: Flag{ Name: "float64-flag-no-sh", diff --git a/cmd/unleash.go b/cmd/unleash.go index 809dad1e..17569c66 100644 --- a/cmd/unleash.go +++ b/cmd/unleash.go @@ -44,10 +44,13 @@ type unleashCmd struct { const ( commandName = "unleash" - paramBuildTags = "tags" - paramDryRun = "dry-run" - paramOutput = "output" - paramIntegrationMode = "integration" + paramBuildTags = "tags" + paramDryRun = "dry-run" + paramOutput = "output" + paramIntegrationMode = "integration" + paramTestCPU = "test-cpu" + paramWorkers = "workers" + paramTimeoutCoefficient = "timeout-coefficient" // Thresholds. paramThresholdEfficacy = "threshold-efficacy" @@ -142,17 +145,19 @@ func cleanUp(wd string) { } func run(ctx context.Context, mod gomodule.GoModule, workDir string) (report.Results, error) { - c := coverage.New(workDir, mod) - p, err := c.Run() + cProfile, err := c.Run() if err != nil { return report.Results{}, fmt.Errorf("failed to gather coverage: %w", err) } - d := workdir.NewDealer(workDir, mod.Root) + wdDealer := workdir.NewCachedDealer(workDir, mod.Root) + defer wdDealer.Clean() + + jDealer := mutator.NewExecutorDealer(mod, wdDealer, cProfile.Elapsed) - mut := mutator.New(mod, p, d) + mut := mutator.New(mod, cProfile, jDealer) results := mut.Run(ctx) return results, nil @@ -177,6 +182,9 @@ func setFlagsOnCmd(cmd *cobra.Command) error { {Name: paramIntegrationMode, CfgKey: configuration.UnleashIntegrationMode, Shorthand: "i", DefaultV: false, Usage: "makes Gremlins run the complete test suite for each mutation"}, {Name: paramThresholdEfficacy, CfgKey: configuration.UnleashThresholdEfficacyKey, DefaultV: float64(0), Usage: "threshold for code-efficacy percent"}, {Name: paramThresholdMCoverage, CfgKey: configuration.UnleashThresholdMCoverageKey, DefaultV: float64(0), Usage: "threshold for mutant-coverage percent"}, + {Name: paramWorkers, CfgKey: configuration.UnleashWorkersKey, DefaultV: 0, Usage: "the number of workers to use in mutation testing"}, + {Name: paramTestCPU, CfgKey: configuration.UnleashTestCPUKey, DefaultV: 0, Usage: "the number of CPUs to allow each test run to use"}, + {Name: paramTimeoutCoefficient, CfgKey: configuration.UnleashTimeoutCoefficientKey, DefaultV: 0, Usage: "the coefficient by which the timeout is increased"}, } for _, f := range fls { diff --git a/cmd/unleash_test.go b/cmd/unleash_test.go index 26f7cfa7..f35fd2c6 100644 --- a/cmd/unleash_test.go +++ b/cmd/unleash_test.go @@ -79,6 +79,21 @@ func TestUnleash(t *testing.T) { flagType: "bool", defValue: "false", }, + { + name: "test-cpu", + flagType: "int", + defValue: "0", + }, + { + name: "workers", + flagType: "int", + defValue: "0", + }, + { + name: "timeout-coefficient", + flagType: "int", + defValue: "0", + }, } for _, tc := range testCases { diff --git a/configuration/configuration.go b/configuration/configuration.go index 96bfaad8..844cacb6 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -36,6 +36,9 @@ const ( UnleashDryRunKey = "unleash.dry-run" UnleashOutputKey = "unleash.output" UnleashTagsKey = "unleash.tags" + UnleashWorkersKey = "unleash.workers" + UnleashTestCPUKey = "unleash.test-cpu" + UnleashTimeoutCoefficientKey = "unleash.timeout-coefficient" UnleashIntegrationMode = "unleash.integration" UnleashThresholdEfficacyKey = "unleash.threshold.efficacy" UnleashThresholdMCoverageKey = "unleash.threshold.mutant-coverage" diff --git a/docs/docs/usage/commands/unleash.md b/docs/docs/usage/commands/unleash/index.md similarity index 70% rename from docs/docs/usage/commands/unleash.md rename to docs/docs/usage/commands/unleash/index.md index 21a46cf3..1fedbfe3 100644 --- a/docs/docs/usage/commands/unleash.md +++ b/docs/docs/usage/commands/unleash/index.md @@ -143,7 +143,7 @@ gremlins unleash --threshold-mcover 80 :material-flag: `--arithmetic-base` · :material-sign-direction: Default: `true` -Enables/disables the [ARITHMETIC BASE](../mutations/arithmetic_base.md) mutant type. +Enables/disables the [ARITHMETIC BASE](../../mutations/arithmetic_base.md) mutant type. ```shell gremlins unleash --arithmetic-base=false @@ -153,7 +153,7 @@ gremlins unleash --arithmetic-base=false :material-flag: `--conditionals-boundary` · :material-sign-direction: Default: `true` -Enables/disables the [CONDITIONALS BOUNDARY](../mutations/conditionals_boundary.md) mutant type. +Enables/disables the [CONDITIONALS BOUNDARY](../../mutations/conditionals_boundary.md) mutant type. ```shell gremlins unleash --conditionals_boundary=false @@ -163,7 +163,7 @@ gremlins unleash --conditionals_boundary=false :material-flag: `--conditionals-negation` · :material-sign-direction: Default: `true` -Enables/disables the [CONDITIONALS NEGATION](../mutations/conditionals_negation.md) mutant type. +Enables/disables the [CONDITIONALS NEGATION](../../mutations/conditionals_negation.md) mutant type. ```shell gremlins unleash --conditionals_negation=false @@ -173,7 +173,7 @@ gremlins unleash --conditionals_negation=false :material-flag: `--increment-decrement` · :material-sign-direction: Default: `true` -Enables/disables the [INCREMENT DECREMENT](../mutations/increment_decrement.md) mutant type. +Enables/disables the [INCREMENT DECREMENT](../../mutations/increment_decrement.md) mutant type. ```shell gremlins unleash --increment-decrement=false @@ -183,8 +183,61 @@ gremlins unleash --increment-decrement=false :material-flag: `--invert-negatives` · :material-sign-direction: Default: `true` -Enables/disables the [INVERT NEGATIVES](../mutations/invert_negatives.md) mutant type. +Enables/disables the [INVERT NEGATIVES](../../mutations/invert_negatives.md) mutant type. ```shell gremlins unleash --invert_negatives=false ``` + +### Workers + +:material-flag: `--workers` · :material-sign-direction: Default: `0` + +[//]: # (@formatter:off) +!!! tip + To understand better the use of these flag, check [workers](workers.md) +[//]: # (@formatter:on) + +Gremlins runs in parallel mode, which means that more than one test at a time will be performed, based on the number of +CPU cores available. + +By default, Gremlins will use all the available CPU cores of, and , in _integration mode_, it will use half of the +available CPU cores. + +The `--workers` flag allows to override the number of CPUs to use (`0` means use the default). + +```shell +gremlins unleash --workers=4 +``` + +### Test CPU + +:material-flag: `--test-cpu` · :material-sign-direction: Default: `0` + +[//]: # (@formatter:off) +!!! tip + To understand better the use of these flag, check [workers](workers.md) +[//]: # (@formatter:on) + +This flag overrides the number of CPUs the Go test tool will utilize. By default, Gremlins doesn't set this value. + +```shell +gremlins unleash --test-cpu=1 +``` + +### Timeout coefficient + +:material-flag: `--timeout-coefficient` · :material-sign-direction: Default: `0` + +[//]: # (@formatter:off) +!!! tip + To understand better the use of these flag, check [workers](workers.md) +[//]: # (@formatter:on) + +Gremlins determines the timeout for each Go test run by multiplying by a coefficient the time it took to perform the +coverage run. +It is possible to override this coefficient (`0` means use the default). + +```shell +gremlins unleash --timeout-coefficient=3 +``` \ No newline at end of file diff --git a/docs/docs/usage/commands/unleash/workers.md b/docs/docs/usage/commands/unleash/workers.md new file mode 100644 index 00000000..6e62bff8 --- /dev/null +++ b/docs/docs/usage/commands/unleash/workers.md @@ -0,0 +1,36 @@ +# Workers + +Gremlins works in parallel mode. It uses some sensible defaults, but it may be necessary to tweak them in your specific +use case. Finding the correct settings mandates a little trial and error, and we are still learning how to get the most +of it. + +The first setting you should be aware of is the number of _workers_ (`--workers`). By default, Gremlins uses the number +of available CPU cores. This value is correct most of the time, but if you notice an excessive number of mutations going +into `TIMED OUT`, you may try to decrease this value. + +If you decrease this value, you may also try to increase the number of CPU cores available to each test +run (`--test-cpu`). This is equivalent to the `-cpu` flag of the Go test tool, but for each mutation test. Gremlins +doesn't enforce this by default. + +A rule of thumb may be setting it so that the sum of _workers_ and _test CPU_ is equal to the total number of cores of +of the machine. + +The symptom of a run excessively stressed is the number of mutants going into `TIMED OUT`. You should tweak the two +values above until your runs stabilize on a low and constant number of `TIMED OUT` mutants. To understand what could be +your correct value, you can run Gremlins with a single worker and see the results. + +## Timeout coefficient + +Another setting you may want to tweak is the _timeout coefficient_. This is the multiplier used to increase the +estimated time it takes to do a run of the tests. The default value should be ok, but if you see too much tests timing +out, then you may try to play a little with this value. Don't increase it too much though, or the run might become +excessively slow. At the moment, it defaults to 3. + +If your test suite takes a lot of time to run, you may want to tweak this setting to _decrease_ the coefficient. We are +thinking of a dynamic way to set this, but it is not clear yet the correct algorithm to use. + +## Integration mode + +_Integration mode_ is quite heavy on the CPU in parallel mode. For this reason, Gremlins halves the values for workers +and _test CPU_ if it is running in _integration mode_. So, if you set, for example, 4 workers, it will run effectively +with 2. And same goes for _test CPU_. diff --git a/docs/docs/usage/configuration.md b/docs/docs/usage/configuration.md index 661a8b41..ea3b5401 100644 --- a/docs/docs/usage/configuration.md +++ b/docs/docs/usage/configuration.md @@ -50,7 +50,10 @@ unleash: dry-run: false tags: "" output: "" - threshold: #(1) + workers: 0 #(1) + test-cpu: 0 #(2) + timeout-coefficient: 0 #(3) + threshold: #(4) efficacy: 0 mutant-coverage: 0 @@ -68,8 +71,12 @@ mutants: ``` -1. Thresholds are set by default to `0`, which means they are not enforced. For further information check the specific - documentation. +1. By default `0`, which means that Gremlins will use the system CPUs number. +2. By default `0`, which means that no test process CPU will be enforced. +3. By default `0`, which means a default coefficient will be enforced. +4. Thresholds are set by default to `0`, which means they are not enforced. + +For further information check the specific command documentation. [//]: # (@formatter:off) !!! tip diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 7e349fef..5ec33ace 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -45,7 +45,9 @@ nav: - Usage: - Commands: - usage/commands/index.md - - usage/commands/unleash.md + - Unleash: + - usage/commands/unleash/index.md + - usage/commands/unleash/workers.md - usage/configuration.md - Mutations: - usage/mutations/index.md diff --git a/pkg/mutator/executor.go b/pkg/mutator/executor.go new file mode 100644 index 00000000..b38ab395 --- /dev/null +++ b/pkg/mutator/executor.go @@ -0,0 +1,270 @@ +/* + * Copyright 2022 The Gremlins Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mutator + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "sync" + "time" + + "github.com/go-gremlins/gremlins/configuration" + "github.com/go-gremlins/gremlins/internal/gomodule" + "github.com/go-gremlins/gremlins/pkg/log" + "github.com/go-gremlins/gremlins/pkg/mutant" + "github.com/go-gremlins/gremlins/pkg/mutator/internal/workerpool" + "github.com/go-gremlins/gremlins/pkg/mutator/workdir" + "github.com/go-gremlins/gremlins/pkg/report" +) + +// DefaultTimeoutCoefficient is the default multiplier for the timeout length +// of each test run. +const DefaultTimeoutCoefficient = 3 + +// ExecutorDealer is the initializer for new workerpool.Executor. +type ExecutorDealer interface { + NewExecutor(mut mutant.Mutant, outCh chan<- mutant.Mutant, wg *sync.WaitGroup) workerpool.Executor +} + +// MutantExecutorDealer is a ExecutorDealer for the initialisation of a mutantExecutor. +// +// By default, it sets uses exec.Command to perform the tests on the source +// code. This can be overridden, for example in tests. +// +// The apply and rollback functions are wrappers around the TokenMutant apply and +// rollback. These can be overridden with nop functions in tests. Not an +// ideal setup. In the future we can think of a better way to handle this. +type MutantExecutorDealer struct { + wdDealer workdir.Dealer + execContext execContext + mod gomodule.GoModule + buildTags string + testExecutionTime time.Duration + dryRun bool + integrationMode bool + testCPU int +} + +// ExecutorDealerOption is the defining option for the initialisation of a ExecutorDealer. +type ExecutorDealerOption func(j MutantExecutorDealer) MutantExecutorDealer + +// WithExecContext overrides the default exec.Command with a custom executor. +func WithExecContext(c execContext) ExecutorDealerOption { + return func(m MutantExecutorDealer) MutantExecutorDealer { + m.execContext = c + + return m + } +} + +// NewExecutorDealer initialises a MutantExecutorDealer. +func NewExecutorDealer(mod gomodule.GoModule, wdd workdir.Dealer, elapsed time.Duration, opts ...ExecutorDealerOption) *MutantExecutorDealer { + buildTags := configuration.Get[string](configuration.UnleashTagsKey) + dryRun := configuration.Get[bool](configuration.UnleashDryRunKey) + integrationMode := configuration.Get[bool](configuration.UnleashIntegrationMode) + testCPU := configuration.Get[int](configuration.UnleashTestCPUKey) + tCoefficient := configuration.Get[int](configuration.UnleashTimeoutCoefficientKey) + + coefficient := DefaultTimeoutCoefficient + if tCoefficient != 0 { + coefficient = tCoefficient + } + + if testCPU != 0 && integrationMode { + testCPU /= testCPU + } + + jd := MutantExecutorDealer{ + mod: mod, + wdDealer: wdd, + buildTags: buildTags, + dryRun: dryRun, + integrationMode: integrationMode, + testCPU: testCPU, + testExecutionTime: elapsed * time.Duration(coefficient), + execContext: exec.CommandContext, + } + + for _, opt := range opts { + jd = opt(jd) + } + + return &jd +} + +// NewExecutor returns a new workerpool.Executor for the given mutant.Mutant. +// It gets an output channel of mutant.Mutant and a sync.WaitGroup. The channel +// will stream the results of the executor, and the wait group will be done when the +// executor is complete. +func (m MutantExecutorDealer) NewExecutor(mut mutant.Mutant, outCh chan<- mutant.Mutant, wg *sync.WaitGroup) workerpool.Executor { + mj := mutantExecutor{ + mutant: mut, + outCh: outCh, + wg: wg, + wdDealer: m.wdDealer, + module: m.mod, + dryRun: m.dryRun, + integrationMode: m.integrationMode, + buildTags: m.buildTags, + execContext: m.execContext, + testCPU: m.testCPU, + testExecutionTime: m.testExecutionTime, + } + + return &mj +} + +type execContext = func(ctx context.Context, name string, args ...string) *exec.Cmd + +type mutantExecutor struct { + mutant mutant.Mutant + wdDealer workdir.Dealer + outCh chan<- mutant.Mutant + wg *sync.WaitGroup + execContext execContext + module gomodule.GoModule + buildTags string + testExecutionTime time.Duration + dryRun bool + integrationMode bool + testCPU int +} + +// Start is the implementation of the workerpool.Executor definition and is the +// method responsible for performing the actual mutation testing. +// The executor runs on its mutant.Mutant. +// If it is RUNNABLE, and it is not in dry-run mode, it will apply the mutation, +// run the tests and mark the TokenMutant as either KILLED or LIVED depending +// on the result. If the tests pass, it means the TokenMutant survived, so it +// will be LIVED, if the tests fail, the TokenMutant will be KILLED. +// The timeout of the test is managed outside the run of the test, using +// a context with timeout. This is done because the Go test command doesn't +// make it easy to distinguish failures from timeouts. +func (m *mutantExecutor) Start(w *workerpool.Worker) { + defer m.wg.Done() + workerName := fmt.Sprintf("%s-%d", w.Name, w.ID) + currDir, _ := os.Getwd() + rootDir, err := m.wdDealer.Get(workerName) + if err != nil { + panic("error, this is temporary") + } + defer func(d string) { + _ = os.Chdir(d) + }(currDir) + _ = os.Chdir(rootDir) + + workingDir := filepath.Join(rootDir, m.module.CallingDir) + m.mutant.SetWorkdir(workingDir) + + if m.mutant.Status() == mutant.NotCovered || m.dryRun { + m.outCh <- m.mutant + report.Mutant(m.mutant) + + return + } + + if err := m.mutant.Apply(); err != nil { + log.Errorf("failed to apply mutation at %s - %s\n\t%v", m.mutant.Position(), m.mutant.Status(), err) + + return + } + + m.mutant.SetStatus(m.runTests(m.mutant.Pkg())) + + if err := m.mutant.Rollback(); err != nil { + // What should we do now? + log.Errorf("failed to restore mutation at %s - %s\n\t%v", m.mutant.Position(), m.mutant.Status(), err) + } + + m.outCh <- m.mutant + report.Mutant(m.mutant) +} + +func (m *mutantExecutor) runTests(pkg string) mutant.Status { + ctx, cancel := context.WithTimeout(context.Background(), m.testExecutionTime) + defer cancel() + cmd := m.execContext(ctx, "go", m.getTestArgs(pkg)...) + + rel, err := run(cmd) + defer rel() + + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + return mutant.TimedOut + } + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return getTestFailedStatus(exitErr.ExitCode()) + } + + return mutant.Lived +} + +func (m *mutantExecutor) getTestArgs(pkg string) []string { + args := []string{"test"} + if m.buildTags != "" { + args = append(args, "-tags", m.buildTags) + } + // Here we add some seconds to the timeout to be sure it's gremlins that catches the test + // timeout and not the test itself. The timeout on the test prevents the test.* processes + // from hanging forever. + args = append(args, "-timeout", (2*time.Second + m.testExecutionTime).String()) + args = append(args, "-failfast") + + if m.testCPU != 0 { + args = append(args, fmt.Sprintf("-cpu %d", m.testCPU)) + } + + path := pkg + if m.integrationMode { + path = "./..." + if m.module.CallingDir != "." { + path = fmt.Sprintf("./%s/...", m.module.CallingDir) + } + } + args = append(args, path) + + return args +} + +func run(cmd *exec.Cmd) (func(), error) { + if err := cmd.Run(); err != nil { + + return func() {}, err + } + + return func() { + err := cmd.Process.Release() + if err != nil { + _ = cmd.Process.Kill() + } + }, nil +} + +func getTestFailedStatus(exitCode int) mutant.Status { + switch exitCode { + case 1: + return mutant.Killed + case 2: + return mutant.NotViable + default: + return mutant.Lived + } +} diff --git a/pkg/mutator/executor_test.go b/pkg/mutator/executor_test.go new file mode 100644 index 00000000..9911bdf7 --- /dev/null +++ b/pkg/mutator/executor_test.go @@ -0,0 +1,472 @@ +/* + * Copyright 2022 The Gremlins Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mutator_test + +import ( + "context" + "fmt" + "os" + "os/exec" + "strings" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + + "github.com/go-gremlins/gremlins/configuration" + "github.com/go-gremlins/gremlins/internal/gomodule" + "github.com/go-gremlins/gremlins/pkg/mutant" + "github.com/go-gremlins/gremlins/pkg/mutator" + "github.com/go-gremlins/gremlins/pkg/mutator/internal/workerpool" +) + +func TestApplyAndRollback(t *testing.T) { + t.Run("applies and rolls back", func(t *testing.T) { + wdDealer := newWdDealerStub(t) + tmpDir, _ := wdDealer.Get("") + mod := gomodule.GoModule{ + Name: "example.com", + Root: tmpDir, + CallingDir: ".", + } + mjd := mutator.NewExecutorDealer(mod, wdDealer, expectedTimeout, mutator.WithExecContext(fakeExecCommandSuccess)) + mut := &mutantStub{ + status: mutant.Runnable, + mutType: mutant.ConditionalsBoundary, + pkg: "example.com", + } + outCh := make(chan mutant.Mutant) + wg := sync.WaitGroup{} + wg.Add(1) + executor := mjd.NewExecutor(mut, outCh, &wg) + w := &workerpool.Worker{ + Name: "test", + ID: 1, + } + go func() { + <-outCh + close(outCh) + }() + + executor.Start(w) + + wg.Wait() + + if !mut.applyCalled { + t.Errorf("expected apply to be called") + } + + if !mut.rollbackCalled { + t.Errorf("expected rollback to be called") + } + }) + + t.Run("does nothing if apply goes to error", func(t *testing.T) { + wdDealer := newWdDealerStub(t) + tmpDir, _ := wdDealer.Get("") + mod := gomodule.GoModule{ + Name: "example.com", + Root: tmpDir, + CallingDir: ".", + } + mjd := mutator.NewExecutorDealer(mod, wdDealer, expectedTimeout, mutator.WithExecContext(fakeExecCommandSuccess)) + mut := &mutantStub{ + status: mutant.Runnable, + mutType: mutant.ConditionalsBoundary, + pkg: "example.com", + hasApplyError: true, + } + outCh := make(chan mutant.Mutant) + wg := sync.WaitGroup{} + wg.Add(1) + executor := mjd.NewExecutor(mut, outCh, &wg) + w := &workerpool.Worker{ + Name: "test", + ID: 1, + } + go func() { + <-outCh + close(outCh) + }() + + executor.Start(w) + + wg.Wait() + + if !mut.applyCalled { + t.Errorf("expected apply to be called") + } + + if mut.rollbackCalled { + t.Errorf("expected rollback not to be called") + } + }) +} + +type execContext = func(ctx context.Context, name string, args ...string) *exec.Cmd + +func TestMutatorTestExecution(t *testing.T) { + testCases := []struct { + testResult execContext + name string + mutantStatus mutant.Status + wantMutStatus mutant.Status + }{ + { + name: "it skips NOT_COVERED", + testResult: fakeExecCommandSuccess, + mutantStatus: mutant.NotCovered, + wantMutStatus: mutant.NotCovered, + }, + { + name: "if tests pass then mutation is LIVED", + testResult: fakeExecCommandSuccess, + mutantStatus: mutant.Runnable, + wantMutStatus: mutant.Lived, + }, + { + name: "if tests fails then mutation is KILLED", + testResult: fakeExecCommandTestsFailure, + mutantStatus: mutant.Runnable, + wantMutStatus: mutant.Killed, + }, + { + name: "if build fails then mutation is BUILD FAILED", + testResult: fakeExecCommandBuildFailure, + mutantStatus: mutant.Runnable, + wantMutStatus: mutant.NotViable, + }, + } + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + viperSet(map[string]any{configuration.UnleashDryRunKey: false}) + defer viperReset() + wdDealer := newWdDealerStub(t) + mod := gomodule.GoModule{ + Name: "example.com", + Root: ".", + CallingDir: ".", + } + mjd := mutator.NewExecutorDealer(mod, wdDealer, expectedTimeout, mutator.WithExecContext(tc.testResult)) + mut := &mutantStub{ + status: tc.mutantStatus, + mutType: mutant.ConditionalsBoundary, + pkg: "example.com", + } + outCh := make(chan mutant.Mutant) + wg := sync.WaitGroup{} + wg.Add(1) + executor := mjd.NewExecutor(mut, outCh, &wg) + w := &workerpool.Worker{ + Name: "test", + ID: 1, + } + + var got mutant.Mutant + mutex := sync.RWMutex{} + go func() { + mutex.Lock() + defer mutex.Unlock() + got = <-outCh + close(outCh) + }() + executor.Start(w) + wg.Wait() + + mutex.RLock() + defer mutex.RUnlock() + if got.Status() != tc.wantMutStatus { + t.Errorf("expected mutation to be %v, but got: %v", tc.wantMutStatus, got.Status()) + } + }) + } +} + +const expectedTimeout = 10 * time.Second + +type commandHolder struct { + command string + args []string + timeout time.Duration + m sync.Mutex +} + +func TestMutatorRun(t *testing.T) { + testCases := []struct { + name string + pkg string + callDir string + tags string + wantPath string + timeoutCoefficient int + intMode bool + }{ + { + name: "normal mode", + intMode: false, + pkg: "example.com/my/package", + callDir: "test/dir", + tags: "tag1,t1g2", + wantPath: "example.com/my/package", + }, + { + name: "integration mode", + intMode: true, + pkg: "example.com/my/package", + callDir: "test/dir", + tags: "tag1,t1g2", + wantPath: "./test/dir/...", + }, + { + name: "it can override timeout coefficient", + timeoutCoefficient: 4, + pkg: "example.com/my/package", + callDir: "test/dir", + tags: "tag1,t1g2", + wantPath: "example.com/my/package", + }, + } + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + settings := map[string]any{ + configuration.UnleashIntegrationMode: tc.intMode, + configuration.UnleashTagsKey: tc.tags, + } + if tc.timeoutCoefficient != 0 { + settings[configuration.UnleashTimeoutCoefficientKey] = tc.timeoutCoefficient + } + viperSet(settings) + defer viperReset() + + mod := gomodule.GoModule{ + Name: "example.com", + Root: ".", + CallingDir: tc.callDir, + } + wdDealer := newWdDealerStub(t) + holder := &commandHolder{} + mjd := mutator.NewExecutorDealer(mod, wdDealer, expectedTimeout, + mutator.WithExecContext(fakeExecCommandSuccessWithHolder(holder))) + mut := &mutantStub{ + status: mutant.Runnable, + mutType: mutant.ConditionalsBoundary, + pkg: tc.pkg, + } + outCh := make(chan mutant.Mutant) + wg := sync.WaitGroup{} + wg.Add(1) + executor := mjd.NewExecutor(mut, outCh, &wg) + w := &workerpool.Worker{ + Name: "test", + ID: 1, + } + go func() { + <-outCh + close(outCh) + }() + executor.Start(w) + wg.Wait() + + wantTimeout := 2*time.Second + expectedTimeout*mutator.DefaultTimeoutCoefficient + if tc.timeoutCoefficient != 0 { + wantTimeout = 2*time.Second + expectedTimeout*time.Duration(tc.timeoutCoefficient) + } + want := fmt.Sprintf("go test -tags %s -timeout %s -failfast %s", tc.tags, wantTimeout, tc.wantPath) + got := fmt.Sprintf("go %v", strings.Join(holder.args, " ")) + + if !cmp.Equal(got, want) { + t.Errorf(fmt.Sprintf("\n+ %s\n- %s\n", got, want)) + } + + timeoutDifference := absTimeDiff(holder.timeout, expectedTimeout*2) + diffThreshold := 100 * time.Second + if timeoutDifference > diffThreshold { + t.Errorf("expected timeout to be within %s from the set timeout, got %s", diffThreshold, timeoutDifference) + } + }) + } +} + +func TestCPU(t *testing.T) { + testCases := []struct { + name string + testCPU int + wantTestCPU int + intMode bool + cpuPresent bool + }{ + { + name: "default normal mode doesn't set CPU", + cpuPresent: false, + }, + { + name: "default integration mode doesn't set CPU", + intMode: true, + cpuPresent: false, + }, + { + name: "normal mode can override CPU", + testCPU: 1, + wantTestCPU: 1, + cpuPresent: true, + }, + { + name: "integration mode overrides CPU to half", + intMode: true, + testCPU: 2, + wantTestCPU: 1, + cpuPresent: true, + }, + } + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + viperSet(map[string]any{ + configuration.UnleashIntegrationMode: tc.intMode, + configuration.UnleashTestCPUKey: tc.testCPU, + }) + defer viperReset() + + mod := gomodule.GoModule{ + Name: "example.com", + Root: ".", + CallingDir: ".", + } + wdDealer := newWdDealerStub(t) + holder := &commandHolder{} + mjd := mutator.NewExecutorDealer(mod, wdDealer, expectedTimeout, + mutator.WithExecContext(fakeExecCommandSuccessWithHolder(holder))) + mut := &mutantStub{ + status: mutant.Runnable, + mutType: mutant.ConditionalsBoundary, + pkg: "test", + } + outCh := make(chan mutant.Mutant) + wg := sync.WaitGroup{} + wg.Add(1) + executor := mjd.NewExecutor(mut, outCh, &wg) + w := &workerpool.Worker{ + Name: "test", + ID: 1, + } + go func() { + <-outCh + close(outCh) + }() + executor.Start(w) + wg.Wait() + + for _, arg := range holder.args { + if !tc.cpuPresent && strings.Contains(arg, "-cpu") { + t.Fatalf("didn't expect to have -cpu flag") + } + if !tc.cpuPresent { + return + } + got := fmt.Sprintf("go %v", strings.Join(holder.args, " ")) + cpuFlag := fmt.Sprintf("-cpu %d", tc.wantTestCPU) + if strings.Contains(got, cpuFlag) { + // PASS + return + } + t.Fatalf("want flag %q, got args %s", cpuFlag, holder.args) + } + + }) + } +} + +func absTimeDiff(a, b time.Duration) time.Duration { + if a > b { + return a - b + } + + return b - a +} + +func TestCoverageProcessSuccess(_ *testing.T) { + if os.Getenv("GO_TEST_PROCESS") != "1" { + return + } + os.Exit(0) +} + +func TestProcessTestsFailure(_ *testing.T) { + if os.Getenv("GO_TEST_PROCESS") != "1" { + return + } + os.Exit(1) +} + +func TestProcessBuildFailure(_ *testing.T) { + if os.Getenv("GO_TEST_PROCESS") != "1" { + return + } + os.Exit(2) +} + +func fakeExecCommandSuccess(ctx context.Context, command string, args ...string) *exec.Cmd { + cs := []string{"-test.run=TestCoverageProcessSuccess", "--", command} + cs = append(cs, args...) + // #nosec G204 - We are in tests, we don't care + cmd := exec.CommandContext(ctx, os.Args[0], cs...) + cmd.Env = []string{"GO_TEST_PROCESS=1"} + + return cmd +} + +func fakeExecCommandSuccessWithHolder(got *commandHolder) execContext { + return func(ctx context.Context, command string, args ...string) *exec.Cmd { + dl, _ := ctx.Deadline() + got.m.Lock() + defer got.m.Unlock() + if got != nil { + got.command = command + got.args = args + got.timeout = time.Until(dl) + } + cs := []string{"-test.run=TestCoverageProcessSuccess", "--", command} + cs = append(cs, args...) + + return getCmd(ctx, cs) + } +} + +func fakeExecCommandTestsFailure(ctx context.Context, command string, args ...string) *exec.Cmd { + cs := []string{"-test.run=TestProcessTestsFailure", "--", command} + cs = append(cs, args...) + + return getCmd(ctx, cs) +} + +func fakeExecCommandBuildFailure(ctx context.Context, command string, args ...string) *exec.Cmd { + cs := []string{"-test.run=TestProcessBuildFailure", "--", command} + cs = append(cs, args...) + + return getCmd(ctx, cs) +} + +func getCmd(ctx context.Context, cs []string) *exec.Cmd { + // #nosec G204 - We are in tests, we don't care + cmd := exec.CommandContext(ctx, os.Args[0], cs...) + cmd.Env = []string{"GO_TEST_PROCESS=1"} + + return cmd +} diff --git a/pkg/mutator/internal/tokenmutant.go b/pkg/mutator/internal/tokenmutant.go index 74b027a1..e64dc19f 100644 --- a/pkg/mutator/internal/tokenmutant.go +++ b/pkg/mutator/internal/tokenmutant.go @@ -17,22 +17,34 @@ package internal import ( + "bytes" "go/ast" "go/printer" "go/token" "os" + "path/filepath" + "sync" - "github.com/go-gremlins/gremlins/pkg/log" "github.com/go-gremlins/gremlins/pkg/mutant" ) // TokenMutant is a mutant.Mutant of a token.Token. +// +// Since the AST is shared among mutants, it is important to avoid that more +// than one mutation is applied to the same file before writing it. For this +// reason, TokenMutant contains a cache of locks, one for each file. +// Every time a mutation is about to being applied, a lock is acquired for +// the file it is operating on. Once the file is written and the token is +// rolled back, the lock is released. +// Keeping a lock per file instead of a lock per TokenMutant allows to apply +// mutations on different files in parallel. type TokenMutant struct { pkg string fs *token.FileSet file *ast.File tokenNode *NodeToken workDir string + origFile []byte status mutant.Status mutantType mutant.Type actualToken token.Token @@ -85,36 +97,60 @@ func (m *TokenMutant) Pkg() string { // Apply saves the original token.Token of the mutant.Mutant and sets the // current token from the tokenMutations table. +// Apply overwrites the source code file with the mutated one. It also +// stores the original file in the TokenMutant in order to allow +// Rollback to put it back later. // // To apply the modification, it first removes the source code file which // contains the mutant position, then it writes it back with the mutation // applied. +// The removal of the file is necessary because it might be a hard link +// to the original file, and, if it was modified in place, it would modify +// the original. Removing the link and re-writing the file preserves the +// original to be modified. +// +// Apply also puts back the original Token after the mutated file write. +// This is done in order to facilitate the atomicity of the operation, +// avoiding locking in a method and unlocking in another. func (m *TokenMutant) Apply() error { + fileLock(m.Position().Filename).Lock() + defer fileLock(m.Position().Filename).Unlock() + + filename := filepath.Join(m.workDir, m.Position().Filename) + var err error + m.origFile, err = os.ReadFile(filename) + if err != nil { + return err + } + m.actualToken = m.tokenNode.Tok() m.tokenNode.SetTok(tokenMutations[m.Type()][m.tokenNode.Tok()]) - return m.writeOnFile() -} + if err = m.writeMutatedFile(filename); err != nil { + return err + } -// Rollback puts back the original token of the TokenMutant, then it writes -// it on the actual file which contains the position of the mutant.Mutant. -func (m *TokenMutant) Rollback() error { + // Rollback here to facilitate the atomicity of the operation. m.tokenNode.SetTok(m.actualToken) - return m.writeOnFile() + return nil } -func (m *TokenMutant) writeOnFile() error { - err := removeFile(m.fs, m.tokenNode.TokPos, m.workDir) +func (m *TokenMutant) writeMutatedFile(filename string) error { + w := &bytes.Buffer{} + err := printer.Fprint(w, m.fs, m.file) if err != nil { return err } - f, err := openFile(m.fs, m.tokenNode.TokPos, m.workDir) + + // We need to remove the file before writing because it can be + // a hard link to the original file. + err = os.RemoveAll(filename) if err != nil { return err } - defer closeFile(f) - err = printer.Fprint(f, m.fs, m.file) + + err = os.WriteFile(filename, w.Bytes(), 0600) if err != nil { return err } @@ -122,6 +158,49 @@ func (m *TokenMutant) writeOnFile() error { return nil } +var locks = make(map[string]*sync.Mutex) +var mutex sync.RWMutex + +func fileLock(filename string) *sync.Mutex { + lock, ok := cachedLock(filename) + if !ok { + mutex.Lock() + defer mutex.Unlock() + lock, ok = locks[filename] + if !ok { + lock = &sync.Mutex{} + locks[filename] = lock + + return lock + } + + return lock + } + + return lock +} + +func cachedLock(filename string) (*sync.Mutex, bool) { + mutex.RLock() + defer mutex.RUnlock() + lock, ok := locks[filename] + + return lock, ok +} + +// Rollback puts back the original file after the test and cleans up the +// TokenMutant to free memory. +// +// It isn't necessary to remove the file before writing as it is done in +// Apply, because in this case, we can be sure the file is not a hard link, +// since Apply already made it a concrete one. +func (m *TokenMutant) Rollback() error { + defer m.resetOrigFile() + filename := filepath.Join(m.workDir, m.Position().Filename) + + return os.WriteFile(filename, m.origFile, 0600) +} + // SetWorkdir sets the base path on which to Apply and Rollback operations. // // By default, TokenMutant will operate on the same source on which the analysis @@ -131,35 +210,7 @@ func (m *TokenMutant) SetWorkdir(path string) { m.workDir = path } -func removeFile(fs *token.FileSet, tokPos token.Pos, workdir string) error { - file := fs.File(tokPos) - file.Name() - if workdir != "" { - workdir += "/" - } - err := os.RemoveAll(workdir + file.Name()) - if err != nil { - return err - } - - return nil -} - -func openFile(fs *token.FileSet, tokPos token.Pos, workdir string) (*os.File, error) { - file := fs.File(tokPos) - if workdir != "" { - workdir += "/" - } - f, err := os.OpenFile(workdir+file.Name(), os.O_CREATE|os.O_WRONLY, 0600) //nolint:nosnakecase - if err != nil { - return nil, err - } - - return f, err -} - -func closeFile(f *os.File) { - if err := f.Close(); err != nil { - log.Errorln("an error occurred while closing the mutated file") - } +func (m *TokenMutant) resetOrigFile() { + var zeroByte []byte + m.origFile = zeroByte } diff --git a/pkg/mutator/internal/tokenmutant_test.go b/pkg/mutator/internal/tokenmutant_test.go index da2dbfa5..169c8108 100644 --- a/pkg/mutator/internal/tokenmutant_test.go +++ b/pkg/mutator/internal/tokenmutant_test.go @@ -21,6 +21,7 @@ import ( "go/parser" "go/token" "os" + "path/filepath" "testing" "github.com/google/go-cmp/cmp" @@ -38,7 +39,7 @@ func TestMutantApplyAndRollback(t *testing.T) { workdir := t.TempDir() filePath := "sourceFile.go" - fileFullPath := workdir + "/" + filePath + fileFullPath := filepath.Join(workdir, filePath) err := os.WriteFile(fileFullPath, []byte(rollbackWant), os.ModePerm) if err != nil { diff --git a/pkg/mutator/internal/workerpool/workerpool.go b/pkg/mutator/internal/workerpool/workerpool.go new file mode 100644 index 00000000..91c5f7af --- /dev/null +++ b/pkg/mutator/internal/workerpool/workerpool.go @@ -0,0 +1,134 @@ +/* + * Copyright 2022 The Gremlins Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package workerpool + +import ( + "runtime" + "sync" + + "github.com/go-gremlins/gremlins/configuration" +) + +// Executor is the unit of work that executes a task. +type Executor interface { + Start(worker *Worker) +} + +// Worker takes an executor and starts the actual executor. +type Worker struct { + stopCh chan struct{} + Name string + ID int +} + +// NewWorker instantiates a new worker with an ID and name. +func NewWorker(id int, name string) *Worker { + return &Worker{ + Name: name, + ID: id, + } +} + +// Start gets an executor queue and starts working on it. +func (w *Worker) Start(executorQueue <-chan Executor) { + w.stopCh = make(chan struct{}) + go func() { + for { + executor, ok := <-executorQueue + if !ok { + w.stopCh <- struct{}{} + + break + } + executor.Start(w) + } + }() +} + +func (w *Worker) stop() { + <-w.stopCh +} + +// Pool manages and limits the number of concurrent Worker. +type Pool struct { + queue chan Executor + name string + workers []*Worker + size int +} + +// Initialize creates a new Pool with a name and the number of parallel +// workers it will use. +func Initialize(name string) *Pool { + wNum := configuration.Get[int](configuration.UnleashWorkersKey) + intMode := configuration.Get[bool](configuration.UnleashIntegrationMode) + + p := &Pool{ + size: size(wNum, intMode), + name: name, + } + p.workers = []*Worker{} + for i := 0; i < p.size; i++ { + w := NewWorker(i, p.name) + p.workers = append(p.workers, w) + } + p.queue = make(chan Executor, 1) + + return p +} + +func size(wNum int, intMode bool) int { + if wNum == 0 { + wNum = runtime.NumCPU() + } + if intMode && wNum > 1 { + wNum /= 2 + } + + return wNum +} + +// AppendExecutor adds a new Executor to the queue of Executor to be processed. +func (p *Pool) AppendExecutor(executor Executor) { + p.queue <- executor +} + +// Start the Pool. +func (p *Pool) Start() { + for _, w := range p.workers { + w.Start(p.queue) + } +} + +// Stop the Pool and wait for all the pending Worker to complete. +func (p *Pool) Stop() { + close(p.queue) + var wg sync.WaitGroup + for _, worker := range p.workers { + wg.Add(1) + go func(w *Worker) { + defer wg.Done() + w.stop() + }(worker) + } + wg.Wait() +} + +// ActiveWorkers gives the number of active workers on the Pool. +func (p *Pool) ActiveWorkers() int { + return len(p.workers) +} diff --git a/pkg/mutator/internal/workerpool/workerpool_test.go b/pkg/mutator/internal/workerpool/workerpool_test.go new file mode 100644 index 00000000..9ff44d58 --- /dev/null +++ b/pkg/mutator/internal/workerpool/workerpool_test.go @@ -0,0 +1,201 @@ +/* + * Copyright 2022 The Gremlins Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package workerpool_test + +import ( + "go/token" + "runtime" + "testing" + + "github.com/go-gremlins/gremlins/configuration" + "github.com/go-gremlins/gremlins/pkg/mutant" + "github.com/go-gremlins/gremlins/pkg/mutator/internal/workerpool" +) + +type ExecutorMock struct { + mutant mutant.Mutant + outCh chan<- mutant.Mutant +} + +func (tj *ExecutorMock) Start(w *workerpool.Worker) { + fm := fakeMutant{ + name: w.Name, + id: w.ID, + } + tj.outCh <- fm +} + +func TestWorker(t *testing.T) { + executorQueue := make(chan workerpool.Executor) + outCh := make(chan mutant.Mutant) + + worker := workerpool.NewWorker(1, "test") + worker.Start(executorQueue) + + tj := &ExecutorMock{ + mutant: &fakeMutant{}, + outCh: outCh, + } + + executorQueue <- tj + close(executorQueue) + + m := <-outCh + got, ok := m.(fakeMutant) + if !ok { + t.Fatal("it should be a fakeMutant") + } + + if got.name != "test" { + t.Errorf("want %q, got %q", "test", got.name) + } + if got.id != 1 { + t.Errorf("want %d, got %d", 1, got.id) + } +} + +func TestPool(t *testing.T) { + t.Run("test executes work", func(t *testing.T) { + configuration.Set(configuration.UnleashWorkersKey, 1) + defer configuration.Reset() + + outCh := make(chan mutant.Mutant) + + pool := workerpool.Initialize("test") + pool.Start() + defer pool.Stop() + + tj := &ExecutorMock{ + mutant: &fakeMutant{}, + outCh: outCh, + } + + pool.AppendExecutor(tj) + + m := <-outCh + got, ok := m.(fakeMutant) + if !ok { + t.Fatal("it should be a fakeMutant") + } + + if got.name != "test" { + t.Errorf("want %q, got %q", "test", got.name) + } + if got.id != 0 { + t.Errorf("want %d, got %d", 0, got.id) + } + }) + + t.Run("default uses runtime CPUs as number of workers", func(t *testing.T) { + configuration.Set(configuration.UnleashWorkersKey, 0) + defer configuration.Reset() + + pool := workerpool.Initialize("test") + pool.Start() + defer pool.Stop() + + if pool.ActiveWorkers() != runtime.NumCPU() { + t.Errorf("want %d, got %d", runtime.NumCPU(), pool.ActiveWorkers()) + } + }) + + t.Run("default uses half of runtime CPUs as number of workers in integration mode", func(t *testing.T) { + configuration.Set(configuration.UnleashWorkersKey, 0) + configuration.Set(configuration.UnleashIntegrationMode, true) + defer configuration.Reset() + + pool := workerpool.Initialize("test") + pool.Start() + defer pool.Stop() + + if pool.ActiveWorkers() != runtime.NumCPU()/2 { + t.Errorf("want %d, got %d", runtime.NumCPU()/2, pool.ActiveWorkers()) + } + }) + + t.Run("can override CPUs", func(t *testing.T) { + configuration.Set(configuration.UnleashWorkersKey, 3) + defer configuration.Reset() + + pool := workerpool.Initialize("test") + pool.Start() + defer pool.Stop() + + if pool.ActiveWorkers() != 3 { + t.Errorf("want %d, got %d", 3, pool.ActiveWorkers()) + } + }) + + t.Run("in integration mode, overrides CPUs by half", func(t *testing.T) { + configuration.Set(configuration.UnleashWorkersKey, 2) + configuration.Set(configuration.UnleashIntegrationMode, true) + defer configuration.Reset() + + pool := workerpool.Initialize("test") + pool.Start() + defer pool.Stop() + + if pool.ActiveWorkers() != 1 { + t.Errorf("want %d, got %d", 1, pool.ActiveWorkers()) + } + }) +} + +type fakeMutant struct { + name string + id int +} + +func (fakeMutant) Type() mutant.Type { + panic("not used in test") +} + +func (fakeMutant) SetType(_ mutant.Type) { + panic("not used in test") +} + +func (fakeMutant) Status() mutant.Status { + panic("not used in test") +} + +func (fakeMutant) SetStatus(_ mutant.Status) { + panic("not used in test") +} + +func (fakeMutant) Position() token.Position { + panic("not used in test") +} + +func (fakeMutant) Pos() token.Pos { + panic("not used in test") +} + +func (fakeMutant) Pkg() string { + panic("not used in test") +} + +func (fakeMutant) SetWorkdir(_ string) { + panic("not used in test") +} + +func (fakeMutant) Apply() error { + panic("not used in test") +} + +func (fakeMutant) Rollback() error { + panic("not used in test") +} diff --git a/pkg/mutator/mutator.go b/pkg/mutator/mutator.go index 597ff8d8..c14334b4 100644 --- a/pkg/mutator/mutator.go +++ b/pkg/mutator/mutator.go @@ -18,25 +18,23 @@ package mutator import ( "context" - "errors" "fmt" "go/ast" "go/parser" "go/token" "io/fs" "os" - "os/exec" "path/filepath" "strings" + "sync" "time" "github.com/go-gremlins/gremlins/configuration" "github.com/go-gremlins/gremlins/internal/gomodule" "github.com/go-gremlins/gremlins/pkg/coverage" - "github.com/go-gremlins/gremlins/pkg/log" "github.com/go-gremlins/gremlins/pkg/mutant" "github.com/go-gremlins/gremlins/pkg/mutator/internal" - "github.com/go-gremlins/gremlins/pkg/mutator/workdir" + "github.com/go-gremlins/gremlins/pkg/mutator/internal/workerpool" "github.com/go-gremlins/gremlins/pkg/report" ) @@ -45,24 +43,13 @@ import ( // It traverses the AST of the project, finds which TokenMutant can be applied and // performs the actual mutation testing. type Mutator struct { - module gomodule.GoModule - fs fs.FS - wdManager workdir.Dealer - covProfile coverage.Profile - execContext execContext - apply func(m mutant.Mutant) error - rollback func(m mutant.Mutant) error - mutantStream chan mutant.Mutant - buildTags string - testExecutionTime time.Duration - dryRun bool - integrationMode bool + fs fs.FS + jDealer ExecutorDealer + covProfile coverage.Profile + mutantStream chan mutant.Mutant + module gomodule.GoModule } -const timeoutCoefficient = 2 - -type execContext = func(ctx context.Context, name string, args ...string) *exec.Cmd - // Option for the Mutator initialization. type Option func(m Mutator) Mutator @@ -70,36 +57,13 @@ type Option func(m Mutator) Mutator // // It gets a fs.FS on which to perform the analysis, a coverage.Profile to // check if the mutants are covered and a sets of Option. -// -// By default, it sets uses exec.Command to perform the tests on the source -// code. This can be overridden, for example in tests. -// -// The apply and rollback functions are wrappers around the TokenMutant apply and -// rollback. These can be overridden with nop functions in tests. Not an -// ideal setup. In the future we can think of a better way to handle this. -func New(mod gomodule.GoModule, r coverage.Result, manager workdir.Dealer, opts ...Option) Mutator { +func New(mod gomodule.GoModule, r coverage.Result, jDealer ExecutorDealer, opts ...Option) Mutator { dirFS := os.DirFS(filepath.Join(mod.Root, mod.CallingDir)) - buildTags := configuration.Get[string](configuration.UnleashTagsKey) - dryRun := configuration.Get[bool](configuration.UnleashDryRunKey) - integrationMode := configuration.Get[bool](configuration.UnleashIntegrationMode) - mut := Mutator{ - module: mod, - wdManager: manager, - covProfile: r.Profile, - testExecutionTime: r.Elapsed * timeoutCoefficient, - fs: dirFS, - execContext: exec.CommandContext, - apply: func(m mutant.Mutant) error { - return m.Apply() - }, - rollback: func(m mutant.Mutant) error { - return m.Rollback() - }, - - buildTags: buildTags, - dryRun: dryRun, - integrationMode: integrationMode, + module: mod, + jDealer: jDealer, + covProfile: r.Profile, + fs: dirFS, } for _, opt := range opts { mut = opt(mut) @@ -108,25 +72,6 @@ func New(mod gomodule.GoModule, r coverage.Result, manager workdir.Dealer, opts return mut } -// WithExecContext overrides the default exec.Command with a custom executor. -func WithExecContext(c execContext) Option { - return func(m Mutator) Mutator { - m.execContext = c - - return m - } -} - -// WithApplyAndRollback overrides the apply and rollback functions. -func WithApplyAndRollback(a, r func(m mutant.Mutant) error) Option { - return func(m Mutator) Mutator { - m.apply = a - m.rollback = r - - return m - } -} - // WithDirFs overrides the fs.FS of the module (mainly used for testing purposes). func WithDirFs(dirFS fs.FS) Option { return func(m Mutator) Mutator { @@ -140,11 +85,6 @@ func WithDirFs(dirFS fs.FS) Option { // // It walks the fs.FS provided and checks every .go file which is not a test. // For each file it will scan for tokenMutations and gather all the mutants found. -// For each TokenMutant found, if it is RUNNABLE, and it is not in dry-run mode, -// it will apply the mutation, run the tests and mark the TokenMutant as either -// KILLED or LIVED depending on the result. If the tests pass, it means the -// TokenMutant survived, so it will be LIVED, if the tests fail, the TokenMutant will -// be KILLED. func (mu *Mutator) Run(ctx context.Context) report.Results { mu.mutantStream = make(chan mutant.Mutant) go func() { @@ -241,53 +181,40 @@ func (mu *Mutator) mutationStatus(pos token.Position) mutant.Status { } func (mu *Mutator) executeTests(ctx context.Context) report.Results { - var mutants []mutant.Mutant - if mu.dryRun { - log.Infoln("Running in 'dry-run' mode...") - } else { - log.Infoln("Executing mutation testing on covered mutants...") - } - currDir, _ := os.Getwd() - rootDir, cl, err := mu.wdManager.Get() - if err != nil { - panic("error, this is temporary") - } - defer func(d string) { - _ = os.Chdir(d) - cl() - }(currDir) - wrkDir := filepath.Join(rootDir, mu.module.CallingDir) - - _ = os.Chdir(rootDir) - - for mut := range mu.mutantStream { - ok := checkDone(ctx) - if !ok { - return results(mutants) - } - mut.SetWorkdir(wrkDir) - if mut.Status() == mutant.NotCovered || mu.dryRun { - mutants = append(mutants, mut) - report.Mutant(mut) - - continue - } + // TODO: add config for CPU + // - if integration mode, use half cpu + // - if cpu not set, use numCPU + // - make timeout coefficient configurable + // - make test cpu configurable + // - set sensible defaults + pool := workerpool.Initialize("mutator") + pool.Start() - if err := mu.apply(mut); err != nil { - log.Errorf("failed to apply mutation at %s - %s\n\t%v", mut.Position(), mut.Status(), err) + var mutants []mutant.Mutant + outCh := make(chan mutant.Mutant) + wg := &sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + for mut := range mu.mutantStream { + ok := checkDone(ctx) + if !ok { + pool.Stop() - continue + break + } + wg.Add(1) + pool.AppendExecutor(mu.jDealer.NewExecutor(mut, outCh, wg)) } + }() - mut.SetStatus(mu.runTests(mut.Pkg())) - - if err := mu.rollback(mut); err != nil { - // What should we do now? - log.Errorf("failed to restore mutation at %s - %s\n\t%v", mut.Position(), mut.Status(), err) - } + go func() { + wg.Wait() + close(outCh) + }() - report.Mutant(mut) - mutants = append(mutants, mut) + for m := range outCh { + mutants = append(mutants, m) } return results(mutants) @@ -305,67 +232,3 @@ func checkDone(ctx context.Context) bool { func results(m []mutant.Mutant) report.Results { return report.Results{Mutants: m} } - -func (mu *Mutator) runTests(pkg string) mutant.Status { - ctx, cancel := context.WithTimeout(context.Background(), mu.testExecutionTime) - defer cancel() - cmd := mu.execContext(ctx, "go", mu.getTestArgs(pkg)...) - - rel, err := run(cmd) - defer rel() - - if errors.Is(ctx.Err(), context.DeadlineExceeded) { - return mutant.TimedOut - } - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { - return getTestFailedStatus(exitErr.ExitCode()) - } - - return mutant.Lived -} - -func run(cmd *exec.Cmd) (func(), error) { - if err := cmd.Run(); err != nil { - - return func() {}, err - } - - return func() { - err := cmd.Process.Release() - if err != nil { - _ = cmd.Process.Kill() - } - }, nil -} - -func (mu *Mutator) getTestArgs(pkg string) []string { - args := []string{"test"} - if mu.buildTags != "" { - args = append(args, "-tags", mu.buildTags) - } - args = append(args, "-timeout", (1*time.Second + mu.testExecutionTime).String()) - args = append(args, "-failfast") - - path := pkg - if mu.integrationMode { - path = "./..." - if mu.module.CallingDir != "." { - path = fmt.Sprintf("./%s/...", mu.module.CallingDir) - } - } - args = append(args, path) - - return args -} - -func getTestFailedStatus(exitCode int) mutant.Status { - switch exitCode { - case 1: - return mutant.Killed - case 2: - return mutant.NotViable - default: - return mutant.Lived - } -} diff --git a/pkg/mutator/mutator_test.go b/pkg/mutator/mutator_test.go index cf9e95e0..229f8f79 100644 --- a/pkg/mutator/mutator_test.go +++ b/pkg/mutator/mutator_test.go @@ -18,19 +18,11 @@ package mutator_test import ( "context" - "errors" - "fmt" "go/token" "io" "os" - "os/exec" - "strings" - "sync" "testing" "testing/fstest" - "time" - - "github.com/google/go-cmp/cmp" "github.com/go-gremlins/gremlins/configuration" "github.com/go-gremlins/gremlins/internal/gomodule" @@ -39,58 +31,27 @@ import ( "github.com/go-gremlins/gremlins/pkg/mutator" ) -var viperMutex sync.RWMutex - -func init() { - viperMutex.Lock() - viperReset() -} - -const defaultFixture = "testdata/fixtures/gtr_go" - -func viperSet(set map[string]any) { - viperMutex.Lock() - for k, v := range set { - configuration.Set(k, v) - } -} - -func viperReset() { - configuration.Reset() - for _, mt := range mutant.MutantTypes { - configuration.Set(configuration.MutantTypeEnabledKey(mt), true) - } - viperMutex.Unlock() -} - -const expectedTimeout = 10 * time.Second -const expectedModule = "example.com" +const ( + defaultFixture = "testdata/fixtures/gtr_go" + expectedModule = "example.com" +) func coveredPosition(fixture string) coverage.Result { fn := filenameFromFixture(fixture) p := coverage.Profile{fn: {{StartLine: 6, EndLine: 7, StartCol: 8, EndCol: 9}}} - return coverage.Result{Profile: p, Elapsed: expectedTimeout} + return coverage.Result{Profile: p, Elapsed: 10} } func notCoveredPosition(fixture string) coverage.Result { fn := filenameFromFixture(fixture) p := coverage.Profile{fn: {{StartLine: 9, EndLine: 9, StartCol: 8, EndCol: 9}}} - return coverage.Result{Profile: p, Elapsed: expectedTimeout} -} - -type dealerStub struct { - t *testing.T -} - -func (d dealerStub) Get() (string, func(), error) { - return d.t.TempDir(), func() {}, nil + return coverage.Result{Profile: p, Elapsed: 10} } func TestMutations(t *testing.T) { t.Parallel() - testCases := []struct { name string fixture string @@ -275,7 +236,7 @@ func TestMutations(t *testing.T) { mapFS, mod, c := loadFixture(tc.fixture, ".") defer c() - mut := mutator.New(mod, tc.covResult, dealerStub{t: t}, mutator.WithDirFs(mapFS)) + mut := mutator.New(mod, tc.covResult, newJobDealerStub(t), mutator.WithDirFs(mapFS)) res := mut.Run(context.Background()) got := res.Mutants @@ -307,6 +268,7 @@ func TestMutations(t *testing.T) { func TestMutantSkipDisabled(t *testing.T) { t.Parallel() for _, mt := range mutant.MutantTypes { + mt := mt t.Run(mt.String(), func(t *testing.T) { t.Parallel() mapFS, mod, c := loadFixture(defaultFixture, ".") @@ -314,12 +276,11 @@ func TestMutantSkipDisabled(t *testing.T) { viperSet(map[string]any{ configuration.UnleashDryRunKey: true, - configuration.MutantTypeEnabledKey(mt): false}, - ) + configuration.MutantTypeEnabledKey(mt): false, + }) defer viperReset() - mut := mutator.New(mod, coveredPosition(defaultFixture), dealerStub{t: t}, - mutator.WithExecContext(fakeExecCommandSuccess), mutator.WithDirFs(mapFS)) + mut := mutator.New(mod, coveredPosition(defaultFixture), newJobDealerStub(t), mutator.WithDirFs(mapFS)) res := mut.Run(context.Background()) got := res.Mutants @@ -348,7 +309,7 @@ func TestSkipTestAndNonGoFiles(t *testing.T) { } viperSet(map[string]any{configuration.UnleashDryRunKey: true}) defer viperReset() - mut := mutator.New(mod, coverage.Result{}, dealerStub{t: t}, mutator.WithDirFs(sys)) + mut := mutator.New(mod, coverage.Result{}, newJobDealerStub(t), mutator.WithDirFs(sys)) res := mut.Run(context.Background()) if got := res.Mutants; len(got) != 0 { @@ -356,15 +317,23 @@ func TestSkipTestAndNonGoFiles(t *testing.T) { } } -type commandHolder struct { - command string - args []string - timeout time.Duration -} +func TestStopsOnCancel(t *testing.T) { + mapFS, mod, c := loadFixture(defaultFixture, ".") + defer c() -type execContext = func(ctx context.Context, name string, args ...string) *exec.Cmd + mut := mutator.New(mod, coveredPosition(defaultFixture), newJobDealerStub(t), + mutator.WithDirFs(mapFS)) -func TestMutatorRun(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + res := mut.Run(ctx) + + if len(res.Mutants) > 0 { + t.Errorf("expected to receive no mutants, got %d", len(res.Mutants)) + } +} + +func TestPackageDiscovery(t *testing.T) { testCases := []struct { name string fromPkg string @@ -383,21 +352,9 @@ func TestMutatorRun(t *testing.T) { intMode: false, wantPath: "example.com/testdata/main", }, - { - name: "from root, integration mode", - fromPkg: ".", - intMode: true, - wantPath: "./...", - }, - { - name: "from subpackage, integration mode", - fromPkg: "testdata/fixture", - intMode: true, - wantPath: "./testdata/fixture/...", - }, } for _, tc := range testCases { - + tc := tc t.Run(tc.name, func(t *testing.T) { viperSet(map[string]any{ configuration.UnleashIntegrationMode: tc.intMode, @@ -407,273 +364,17 @@ func TestMutatorRun(t *testing.T) { mapFS, mod, c := loadFixture(defaultFixture, tc.fromPkg) defer c() - holder := &commandHolder{} - mut := mutator.New(mod, coveredPosition(defaultFixture), dealerStub{t: t}, - mutator.WithDirFs(mapFS), - mutator.WithExecContext(fakeExecCommandSuccessWithHolder(holder)), - mutator.WithApplyAndRollback( - func(m mutant.Mutant) error { - return nil - }, - func(m mutant.Mutant) error { - return nil - })) + jds := newJobDealerStub(t) + mut := mutator.New(mod, coveredPosition(defaultFixture), jds, mutator.WithDirFs(mapFS)) _ = mut.Run(context.Background()) - want := "go test -tags tag1 tag2 -timeout 21s -failfast " + tc.wantPath - got := fmt.Sprintf("go %v", strings.Join(holder.args, " ")) - - if !cmp.Equal(got, want) { - t.Errorf(cmp.Diff(got, want)) - } - - timeoutDifference := absTimeDiff(holder.timeout, expectedTimeout*2) - diffThreshold := 70 * time.Microsecond - if timeoutDifference > diffThreshold { - t.Errorf("expected timeout to be within %s from the set timeout, got %s", diffThreshold, timeoutDifference) - } - }) - } -} - -func absTimeDiff(a, b time.Duration) time.Duration { - if a > b { - return a - b - } - - return b - a -} - -func TestMutatorTestExecution(t *testing.T) { - testCases := []struct { - name string - fixture string - testResult execContext - covResult coverage.Result - wantMutStatus mutant.Status - }{ - { - name: "it skips NOT_COVERED", - fixture: "testdata/fixtures/gtr_go", - testResult: fakeExecCommandSuccess, - covResult: notCoveredPosition("testdata/fixtures/gtr_go"), - wantMutStatus: mutant.NotCovered, - }, - { - name: "if tests pass then mutation is LIVED", - fixture: "testdata/fixtures/gtr_go", - testResult: fakeExecCommandSuccess, - covResult: coveredPosition("testdata/fixtures/gtr_go"), - wantMutStatus: mutant.Lived, - }, - { - name: "if tests fails then mutation is KILLED", - fixture: "testdata/fixtures/gtr_go", - testResult: fakeExecCommandTestsFailure, - covResult: coveredPosition("testdata/fixtures/gtr_go"), - wantMutStatus: mutant.Killed, - }, - { - name: "if build fails then mutation is BUILD FAILED", - fixture: "testdata/fixtures/gtr_go", - testResult: fakeExecCommandBuildFailure, - covResult: coveredPosition("testdata/fixtures/gtr_go"), - wantMutStatus: mutant.NotViable, - }, - } - for _, tc := range testCases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - viperSet(map[string]any{configuration.UnleashDryRunKey: false}) - defer viperReset() - mapFS, mod, c := loadFixture(tc.fixture, ".") - defer c() + got := jds.gotMutants[0].Pkg() - mut := mutator.New(mod, tc.covResult, dealerStub{t: t}, - mutator.WithDirFs(mapFS), - mutator.WithExecContext(tc.testResult), - mutator.WithApplyAndRollback( - func(m mutant.Mutant) error { - return nil - }, - func(m mutant.Mutant) error { - return nil - })) - res := mut.Run(context.Background()) - got := res.Mutants + if got != tc.wantPath { + t.Errorf("want %q, got %q", tc.wantPath, got) - if len(got) < 1 { - t.Fatal("no mutants received") - } - if got[0].Status() != tc.wantMutStatus { - t.Errorf("expected mutation to be %v, but got: %v", tc.wantMutStatus, got[0].Status()) - } - if tc.wantMutStatus != mutant.NotCovered && res.Elapsed <= 0 { - t.Errorf("expected elapsed time to be greater than zero, got %s", res.Elapsed) } }) } } - -func TestApplyAndRollbackError(t *testing.T) { - t.Run("apply fails", func(t *testing.T) { - mapFS, mod, c := loadFixture(defaultFixture, ".") - defer c() - - mut := mutator.New(mod, coveredPosition(defaultFixture), dealerStub{t: t}, - mutator.WithDirFs(mapFS), - mutator.WithExecContext(fakeExecCommandSuccess), - mutator.WithApplyAndRollback( - func(m mutant.Mutant) error { - return errors.New("test error") - }, - func(m mutant.Mutant) error { - return nil - })) - res := mut.Run(context.Background()) - got := res.Mutants - - if len(got) != 0 { - t.Fatal("expected no mutants") - } - }) - - t.Run("rollback fails", func(t *testing.T) { - mapFS, mod, c := loadFixture(defaultFixture, ".") - defer c() - - mut := mutator.New(mod, coveredPosition(defaultFixture), dealerStub{t: t}, - mutator.WithDirFs(mapFS), - mutator.WithExecContext(fakeExecCommandSuccess), - mutator.WithApplyAndRollback( - func(m mutant.Mutant) error { - return nil - }, - func(m mutant.Mutant) error { - return errors.New("test error") - })) - res := mut.Run(context.Background()) - got := res.Mutants - - if len(got) < 1 { // For now, in case of rollback failure, we expect the mutations still to be reported. - t.Fatal("expected no mutants") - } - }) -} - -func TestStopsOnCancel(t *testing.T) { - mapFS, mod, c := loadFixture(defaultFixture, ".") - defer c() - - mut := mutator.New(mod, coveredPosition(defaultFixture), dealerStub{t: t}, - mutator.WithDirFs(mapFS), - mutator.WithExecContext(fakeExecCommandSuccess), - mutator.WithApplyAndRollback( - func(m mutant.Mutant) error { - return nil - }, - func(m mutant.Mutant) error { - return nil - })) - - ctx, cancel := context.WithCancel(context.Background()) - cancel() - res := mut.Run(ctx) - - if len(res.Mutants) > 0 { - t.Errorf("expected to receive no mutants, got %d", len(res.Mutants)) - } -} - -// loadFixture loads a fixture into a mapFS and returns also the GoModule. -// -// fromPackage parameters can be path/pkgName. -func loadFixture(fixture, fromPackage string) (fstest.MapFS, gomodule.GoModule, func()) { - f, _ := os.Open(fixture) - src, _ := io.ReadAll(f) - filename := filenameFromFixture(fixture) - mapFS := fstest.MapFS{ - filename: {Data: src}, - } - - return mapFS, gomodule.GoModule{ - Name: "example.com", - Root: ".", - CallingDir: fromPackage, - }, func() { - _ = f.Close() - } -} - -func TestCoverageProcessSuccess(_ *testing.T) { - if os.Getenv("GO_TEST_PROCESS") != "1" { - return - } - os.Exit(0) -} - -func TestProcessTestsFailure(_ *testing.T) { - if os.Getenv("GO_TEST_PROCESS") != "1" { - return - } - os.Exit(1) -} - -func TestProcessBuildFailure(_ *testing.T) { - if os.Getenv("GO_TEST_PROCESS") != "1" { - return - } - os.Exit(2) -} - -func fakeExecCommandSuccess(ctx context.Context, command string, args ...string) *exec.Cmd { - cs := []string{"-test.run=TestCoverageProcessSuccess", "--", command} - cs = append(cs, args...) - // #nosec G204 - We are in tests, we don't care - cmd := exec.CommandContext(ctx, os.Args[0], cs...) - cmd.Env = []string{"GO_TEST_PROCESS=1"} - - return cmd -} - -func fakeExecCommandSuccessWithHolder(got *commandHolder) execContext { - return func(ctx context.Context, command string, args ...string) *exec.Cmd { - dl, _ := ctx.Deadline() - if got != nil { - got.command = command - got.args = args - got.timeout = time.Until(dl) - } - cs := []string{"-test.run=TestCoverageProcessSuccess", "--", command} - cs = append(cs, args...) - - return getCmd(ctx, cs) - } -} - -func fakeExecCommandTestsFailure(ctx context.Context, command string, args ...string) *exec.Cmd { - cs := []string{"-test.run=TestProcessTestsFailure", "--", command} - cs = append(cs, args...) - - return getCmd(ctx, cs) -} - -func fakeExecCommandBuildFailure(ctx context.Context, command string, args ...string) *exec.Cmd { - cs := []string{"-test.run=TestProcessBuildFailure", "--", command} - cs = append(cs, args...) - - return getCmd(ctx, cs) -} - -func getCmd(ctx context.Context, cs []string) *exec.Cmd { - // #nosec G204 - We are in tests, we don't care - cmd := exec.CommandContext(ctx, os.Args[0], cs...) - cmd.Env = []string{"GO_TEST_PROCESS=1"} - - return cmd -} - -func filenameFromFixture(fix string) string { - return strings.ReplaceAll(fix, "_go", ".go") -} diff --git a/pkg/mutator/stubs_test.go b/pkg/mutator/stubs_test.go new file mode 100644 index 00000000..630a09d2 --- /dev/null +++ b/pkg/mutator/stubs_test.go @@ -0,0 +1,182 @@ +/* + * Copyright 2022 The Gremlins Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mutator_test + +import ( + "errors" + "go/token" + "io" + "os" + "strings" + "sync" + "testing" + "testing/fstest" + + "github.com/go-gremlins/gremlins/configuration" + "github.com/go-gremlins/gremlins/internal/gomodule" + "github.com/go-gremlins/gremlins/pkg/mutant" + "github.com/go-gremlins/gremlins/pkg/mutator/internal/workerpool" +) + +var viperMutex sync.RWMutex + +func init() { + viperMutex.Lock() + viperReset() +} + +func viperSet(set map[string]any) { + viperMutex.Lock() + for k, v := range set { + configuration.Set(k, v) + } +} + +func viperReset() { + configuration.Reset() + for _, mt := range mutant.MutantTypes { + configuration.Set(configuration.MutantTypeEnabledKey(mt), true) + } + viperMutex.Unlock() +} + +func loadFixture(fixture, fromPackage string) (fstest.MapFS, gomodule.GoModule, func()) { + f, _ := os.Open(fixture) + src, _ := io.ReadAll(f) + filename := filenameFromFixture(fixture) + mapFS := fstest.MapFS{ + filename: {Data: src}, + } + + return mapFS, gomodule.GoModule{ + Name: "example.com", + Root: ".", + CallingDir: fromPackage, + }, func() { + _ = f.Close() + } +} + +func filenameFromFixture(fix string) string { + return strings.ReplaceAll(fix, "_go", ".go") +} + +type dealerStub struct { + t *testing.T +} + +func newWdDealerStub(t *testing.T) dealerStub { + t.Helper() + + return dealerStub{t: t} +} + +func (d dealerStub) Get(_ string) (string, error) { + return d.t.TempDir(), nil +} + +func (dealerStub) Clean() {} + +type executorDealerStub struct { + gotMutants []mutant.Mutant +} + +func newJobDealerStub(t *testing.T) *executorDealerStub { + t.Helper() + + return &executorDealerStub{} +} + +func (j *executorDealerStub) NewExecutor(mut mutant.Mutant, outCh chan<- mutant.Mutant, wg *sync.WaitGroup) workerpool.Executor { + j.gotMutants = append(j.gotMutants, mut) + + return &executorStub{ + mut: mut, + outCh: outCh, + wg: wg, + } +} + +type executorStub struct { + mut mutant.Mutant + outCh chan<- mutant.Mutant + wg *sync.WaitGroup +} + +func (j *executorStub) Start(_ *workerpool.Worker) { + j.outCh <- j.mut + j.wg.Done() +} + +type mutantStub struct { + worDir string + pkg string + position token.Position + status mutant.Status + mutType mutant.Type + applyCalled bool + rollbackCalled bool + + hasApplyError bool +} + +func (m *mutantStub) Type() mutant.Type { + return m.mutType +} + +func (m *mutantStub) SetType(mt mutant.Type) { + m.mutType = mt +} + +func (m *mutantStub) Status() mutant.Status { + return m.status +} + +func (m *mutantStub) SetStatus(s mutant.Status) { + m.status = s +} + +func (m *mutantStub) Position() token.Position { + return m.position +} + +func (*mutantStub) Pos() token.Pos { + panic("not used in test") +} + +func (m *mutantStub) Pkg() string { + return m.pkg +} + +func (m *mutantStub) SetWorkdir(w string) { + m.worDir = w +} + +func (m *mutantStub) Apply() error { + m.applyCalled = true + if m.hasApplyError { + return errors.New("test error") + } + + return nil +} + +func (m *mutantStub) Rollback() error { + m.rollbackCalled = true + + return nil +} diff --git a/pkg/mutator/workdir/workdir.go b/pkg/mutator/workdir/workdir.go index 2e1a5fb5..29f15788 100644 --- a/pkg/mutator/workdir/workdir.go +++ b/pkg/mutator/workdir/workdir.go @@ -22,6 +22,7 @@ import ( "os" "path/filepath" "strings" + "sync" "github.com/go-gremlins/gremlins/pkg/log" ) @@ -29,27 +30,41 @@ import ( // Dealer is the responsible for creating and returning the reference // to a workdir to use during mutation testing instead of the actual // source code. +// +// It has two methods: +// +// Get that returns a folder name that will be used by Gremlins as workdir. +// Clean that must be called to remove all the created folders. type Dealer interface { - Get() (string, func(), error) + Get(idf string) (string, error) + Clean() } -// CDealer is the implementation of the Dealer interface, responsible +// CachedDealer is the implementation of the Dealer interface, responsible // for creating a working directory of a source directory. It allows // Gremlins not to work in the actual source directory messing up // with the source code files. -type CDealer struct { +type CachedDealer struct { + mutex *sync.RWMutex + cache map[string]string workDir string srcDir string dockerRootFolder string withinDocker bool } -// Option for the CDealer initialization. -type Option func(d CDealer) CDealer - -// NewDealer instantiates a new CDealer evaluating whether it operates within a docker container. -func NewDealer(workDir, srcDir string, opts ...Option) CDealer { - dealer := CDealer{ +// Option for the CachedDealer initialization. +type Option func(d *CachedDealer) *CachedDealer + +// NewCachedDealer instantiates a new Dealer that keeps a cache of the +// instantiated folders. Every time a new working directory is requested +// with the same identifier, the same folder reference is returned. +// It also verifies whether it is running inside a Docker container or not, +// and makes copies instead of hard links if it is. +func NewCachedDealer(workDir, srcDir string, opts ...Option) *CachedDealer { + dealer := &CachedDealer{ + mutex: &sync.RWMutex{}, + cache: make(map[string]string), workDir: workDir, srcDir: srcDir, dockerRootFolder: "/", @@ -70,7 +85,7 @@ func NewDealer(workDir, srcDir string, opts ...Option) CDealer { // WithDockerRootFolder overrides the default root folder where to look for .dockerenv file. func WithDockerRootFolder(rootFolder string) Option { - return func(d CDealer) CDealer { + return func(d *CachedDealer) *CachedDealer { d.dockerRootFolder = rootFolder return d @@ -78,35 +93,62 @@ func WithDockerRootFolder(rootFolder string) Option { } // Get provides a working directory where all the files are hard links -// to the original files in the source directory. It also returns a -// closer function that cleans up the directory. -// -// The idea is to make this a sort of workdir pool when Gremlins will -// support parallel execution. -func (fm CDealer) Get() (string, func(), error) { - dstDir, err := os.MkdirTemp(fm.workDir, "wd-*") +// to the original files in the source directory. It makes full copies +// in case Gremlins is running inside a Docker container. +func (cd *CachedDealer) Get(idf string) (string, error) { + dstDir, ok := cd.getFromCache(idf) + if ok { + return dstDir, nil + } + + dstDir, err := os.MkdirTemp(cd.workDir, "wd-*") if err != nil { - return "", nil, err + return "", err } - err = filepath.Walk(fm.srcDir, fm.copyTo(dstDir)) + err = filepath.Walk(cd.srcDir, cd.copyTo(dstDir)) if err != nil { - return "", nil, err + return "", err } - return dstDir, func() { - err := os.RemoveAll(dstDir) + cd.setCache(idf, dstDir) + + return dstDir, nil +} + +// Clean frees all the cached folders and removes all of them from disk. +func (cd *CachedDealer) Clean() { + for _, v := range cd.cache { + err := os.RemoveAll(v) if err != nil { - log.Errorln("impossible to remove temporary folder") + log.Errorf("impossible to remove temporary folder %s: %s\n", v, err) } - }, nil + } + cd.cache = make(map[string]string) +} + +func (cd *CachedDealer) getFromCache(idf string) (string, bool) { + cd.mutex.RLock() + defer cd.mutex.RUnlock() + dstDir, ok := cd.cache[idf] + if ok { + return dstDir, true + } + + return "", false +} + +func (cd *CachedDealer) setCache(idf, folder string) { + cd.mutex.Lock() + defer cd.mutex.Unlock() + cd.cache[idf] = folder } -func (fm CDealer) copyTo(dstDir string) func(srcPath string, info fs.FileInfo, err error) error { +func (cd *CachedDealer) copyTo(dstDir string) func(srcPath string, info fs.FileInfo, err error) error { return func(srcPath string, info fs.FileInfo, err error) error { if err != nil { return err } - relPath, err := filepath.Rel(fm.srcDir, srcPath) + relPath, err := filepath.Rel(cd.srcDir, srcPath) if err != nil { return err } @@ -115,18 +157,18 @@ func (fm CDealer) copyTo(dstDir string) func(srcPath string, info fs.FileInfo, e } dstPath := filepath.Join(dstDir, relPath) - return fm.copyPath(srcPath, dstPath, info) + return cd.copyPath(srcPath, dstPath, info) } } -func (fm CDealer) copyPath(srcPath, dstPath string, info fs.FileInfo) error { +func (cd *CachedDealer) copyPath(srcPath, dstPath string, info fs.FileInfo) error { switch mode := info.Mode(); { case mode.IsDir(): if err := os.Mkdir(dstPath, mode); err != nil && !os.IsExist(err) { return err } case mode.IsRegular(): - if fm.withinDocker { + if cd.withinDocker { // When gremlins is running within a docker container, hard link doesn't work, so we do a copy of the file if err := doCopy(srcPath, dstPath, mode); err != nil { return err diff --git a/pkg/mutator/workdir/workdir_test.go b/pkg/mutator/workdir/workdir_test.go index 379d6171..bf7de4bd 100644 --- a/pkg/mutator/workdir/workdir_test.go +++ b/pkg/mutator/workdir/workdir_test.go @@ -22,6 +22,7 @@ import ( "os" "path/filepath" "runtime" + "sync" "testing" "github.com/google/go-cmp/cmp" @@ -35,13 +36,13 @@ func TestLinkFolder(t *testing.T) { populateSrcDir(t, srcDir, 3) dstDir := t.TempDir() - mngr := workdir.NewDealer(dstDir, srcDir, workdir.WithDockerRootFolder(dstDir)) + dealer := workdir.NewCachedDealer(dstDir, srcDir, workdir.WithDockerRootFolder(dstDir)) - dstDir, cl, err := mngr.Get() + dstDir, err := dealer.Get("test") if err != nil { t.Fatal(err) } - defer cl() + defer dealer.Clean() err = filepath.Walk(srcDir, func(path string, srcFileInfo fs.FileInfo, err error) error { if err != nil { @@ -83,7 +84,7 @@ func TestLinkFolder(t *testing.T) { func TestCopyFolder(t *testing.T) { srcDir := t.TempDir() populateSrcDir(t, srcDir, 3) - dstDir := t.TempDir() + wdDir := t.TempDir() dockerRootDir := t.TempDir() dockerEnv := filepath.Join(dockerRootDir, ".dockerenv") @@ -92,13 +93,13 @@ func TestCopyFolder(t *testing.T) { t.Fatal(err) } - mngr := workdir.NewDealer(dstDir, srcDir, workdir.WithDockerRootFolder(dockerRootDir)) + dealer := workdir.NewCachedDealer(wdDir, srcDir, workdir.WithDockerRootFolder(dockerRootDir)) + defer dealer.Clean() - dstDir, cl, err := mngr.Get() + dstDir, err := dealer.Get("test") if err != nil { t.Fatal(err) } - defer cl() err = filepath.Walk(srcDir, func(path string, srcFileInfo fs.FileInfo, err error) error { if err != nil { @@ -136,14 +137,109 @@ func TestCopyFolder(t *testing.T) { } } -func TestCDealerErrors(t *testing.T) { +func TestCachesFolder(t *testing.T) { + t.Run("caches copy folders", func(t *testing.T) { + srcDir := t.TempDir() + populateSrcDir(t, srcDir, 0) + dstDir := t.TempDir() + + mngr := workdir.NewCachedDealer(dstDir, srcDir, workdir.WithDockerRootFolder(dstDir)) + defer mngr.Clean() + + firstDir, err := mngr.Get("worker-1") + if err != nil { + t.Fatal(err) + } + + secondDir, err := mngr.Get("worker-1") + if err != nil { + t.Fatal(err) + } + + thirdDir, err := mngr.Get("worker-2") + if err != nil { + t.Fatal(err) + } + + if firstDir != secondDir { + t.Errorf("expected dirs to be cached, got %s", cmp.Diff(firstDir, secondDir)) + } + if firstDir == thirdDir { + t.Errorf("expected a new dir to be instanciated") + } + }) + + t.Run("cleans up all the folders", func(t *testing.T) { + srcDir := t.TempDir() + populateSrcDir(t, srcDir, 0) + dstDir := t.TempDir() + + dealer := workdir.NewCachedDealer(dstDir, srcDir, workdir.WithDockerRootFolder(dstDir)) + + firstDir, err := dealer.Get("worker-1") + if err != nil { + t.Fatal(err) + } + + dealer.Clean() + + secondDir, err := dealer.Get("worker-1") + if err != nil { + t.Fatal(err) + } + + if firstDir == secondDir { + t.Errorf("expected manager to be cleaned up") + } + }) + + t.Run("it works in parallel", func(t *testing.T) { + srcDir := t.TempDir() + populateSrcDir(t, srcDir, 0) + dstDir := t.TempDir() + + dealer := workdir.NewCachedDealer(dstDir, srcDir, workdir.WithDockerRootFolder(dstDir)) + defer dealer.Clean() + + foldersLock := sync.Mutex{} + var folders []string + + wg := sync.WaitGroup{} + wg.Add(10) + for i := 0; i < 10; i++ { + i := i + go func() { + defer wg.Done() + f, err := dealer.Get(fmt.Sprintf("test-%d", i)) + if err != nil { + t.Errorf("unexpected error: %s", err) + } + foldersLock.Lock() + defer foldersLock.Unlock() + folders = append(folders, f) + }() + } + + wg.Wait() + + occurred := make(map[string]bool) + for _, v := range folders { + if occurred[v] == true { + t.Fatal("expected values to be unique") + } + occurred[v] = true + } + }) +} + +func TestErrors(t *testing.T) { t.Run("dstDir is not a path", func(t *testing.T) { srcDir := "not a dir" dstDir := t.TempDir() - mngr := workdir.NewDealer(dstDir, srcDir) + dealer := workdir.NewCachedDealer(dstDir, srcDir) - _, _, err := mngr.Get() + _, err := dealer.Get("test") if err == nil { t.Errorf("expected an error") } @@ -166,9 +262,9 @@ func TestCDealerErrors(t *testing.T) { dstDir := t.TempDir() - mngr := workdir.NewDealer(dstDir, srcDir) + mngr := workdir.NewCachedDealer(dstDir, srcDir) - _, _, err = mngr.Get() + _, err = mngr.Get("test") if err == nil { t.Errorf("expected an error") } @@ -190,9 +286,9 @@ func TestCDealerErrors(t *testing.T) { _ = clean(d, 0700) }(dstDir) - mngr := workdir.NewDealer(dstDir, srcDir) + dealer := workdir.NewCachedDealer(dstDir, srcDir) - _, _, err = mngr.Get() + _, err = dealer.Get("test") if err == nil { t.Errorf("expected an error") }