global: enable parallel execution (#127)

* workdir: add Dealer cache

* mutator: refactor apply and rollback

* mutator: enable mutations to be applied in parallel

* mutator: basic parallel implementation

* mutator: refactor mutator job

* docs: update for parallel execution

* global: fix linting errors

* cmd: add int flag test

cmd/internal/flags/flags.go

@@ -55,6 +55,8 @@ func setFlags(flag *Flag, fs *pflag.FlagSet) error {
 		setBool(flag, fs, dv)
 	case string:
 		setString(flag, fs, dv)
+	case int:
+		setInt(flag, fs, dv)
 	case float64:
 		setFloat64(flag, fs, dv)
 	}
@@ -66,6 +68,14 @@ func setFlags(flag *Flag, fs *pflag.FlagSet) error {
 	return nil
 }
 
+func setInt(flag *Flag, flags *pflag.FlagSet, dv int) {
+	if flag.Shorthand != "" {
+		flags.IntP(flag.Name, flag.Shorthand, dv, flag.Usage)
+	} else {
+		flags.Int(flag.Name, dv, flag.Usage)
+	}
+}
+
 func setFloat64(flag *Flag, flags *pflag.FlagSet, dv float64) {
 	if flag.Shorthand != "" {
 		flags.Float64P(flag.Name, flag.Shorthand, dv, flag.Usage)

cmd/internal/flags/flags_test.go

@@ -67,7 +67,24 @@ func TestSet(t *testing.T) {
 				Usage:     "test usage",
 			},
 		},
-
+		{
+			flag: Flag{
+				Name:      "int-flag-no-sh",
+				CfgKey:    "test.cfg",
+				Shorthand: "",
+				DefaultV:  0,
+				Usage:     "test usage",
+			},
+		},
+		{
+			flag: Flag{
+				Name:      "int-flag-sh",
+				CfgKey:    "test.cfg",
+				Shorthand: "t",
+				DefaultV:  0,
+				Usage:     "test usage",
+			},
+		},
 		{
 			flag: Flag{
 				Name:      "float64-flag-no-sh",

cmd/unleash.go

@@ -44,10 +44,13 @@ type unleashCmd struct {
 const (
 	commandName = "unleash"
 
-	paramBuildTags       = "tags"
-	paramDryRun          = "dry-run"
-	paramOutput          = "output"
-	paramIntegrationMode = "integration"
+	paramBuildTags          = "tags"
+	paramDryRun             = "dry-run"
+	paramOutput             = "output"
+	paramIntegrationMode    = "integration"
+	paramTestCPU            = "test-cpu"
+	paramWorkers            = "workers"
+	paramTimeoutCoefficient = "timeout-coefficient"
 
 	// Thresholds.
 	paramThresholdEfficacy  = "threshold-efficacy"
@@ -142,17 +145,19 @@ func cleanUp(wd string) {
 }
 
 func run(ctx context.Context, mod gomodule.GoModule, workDir string) (report.Results, error) {
-
 	c := coverage.New(workDir, mod)
 
-	p, err := c.Run()
+	cProfile, err := c.Run()
 	if err != nil {
 		return report.Results{}, fmt.Errorf("failed to gather coverage: %w", err)
 	}
 
-	d := workdir.NewDealer(workDir, mod.Root)
+	wdDealer := workdir.NewCachedDealer(workDir, mod.Root)
+	defer wdDealer.Clean()
+
+	jDealer := mutator.NewExecutorDealer(mod, wdDealer, cProfile.Elapsed)
 
-	mut := mutator.New(mod, p, d)
+	mut := mutator.New(mod, cProfile, jDealer)
 	results := mut.Run(ctx)
 
 	return results, nil
@@ -177,6 +182,9 @@ func setFlagsOnCmd(cmd *cobra.Command) error {
 		{Name: paramIntegrationMode, CfgKey: configuration.UnleashIntegrationMode, Shorthand: "i", DefaultV: false, Usage: "makes Gremlins run the complete test suite for each mutation"},
 		{Name: paramThresholdEfficacy, CfgKey: configuration.UnleashThresholdEfficacyKey, DefaultV: float64(0), Usage: "threshold for code-efficacy percent"},
 		{Name: paramThresholdMCoverage, CfgKey: configuration.UnleashThresholdMCoverageKey, DefaultV: float64(0), Usage: "threshold for mutant-coverage percent"},
+		{Name: paramWorkers, CfgKey: configuration.UnleashWorkersKey, DefaultV: 0, Usage: "the number of workers to use in mutation testing"},
+		{Name: paramTestCPU, CfgKey: configuration.UnleashTestCPUKey, DefaultV: 0, Usage: "the number of CPUs to allow each test run to use"},
+		{Name: paramTimeoutCoefficient, CfgKey: configuration.UnleashTimeoutCoefficientKey, DefaultV: 0, Usage: "the coefficient by which the timeout is increased"},
 	}
 
 	for _, f := range fls {

cmd/unleash_test.go

@@ -79,6 +79,21 @@ func TestUnleash(t *testing.T) {
 			flagType:  "bool",
 			defValue:  "false",
 		},
+		{
+			name:     "test-cpu",
+			flagType: "int",
+			defValue: "0",
+		},
+		{
+			name:     "workers",
+			flagType: "int",
+			defValue: "0",
+		},
+		{
+			name:     "timeout-coefficient",
+			flagType: "int",
+			defValue: "0",
+		},
 	}
 
 	for _, tc := range testCases {

configuration/configuration.go

@@ -36,6 +36,9 @@ const (
 	UnleashDryRunKey             = "unleash.dry-run"
 	UnleashOutputKey             = "unleash.output"
 	UnleashTagsKey               = "unleash.tags"
+	UnleashWorkersKey            = "unleash.workers"
+	UnleashTestCPUKey            = "unleash.test-cpu"
+	UnleashTimeoutCoefficientKey = "unleash.timeout-coefficient"
 	UnleashIntegrationMode       = "unleash.integration"
 	UnleashThresholdEfficacyKey  = "unleash.threshold.efficacy"
 	UnleashThresholdMCoverageKey = "unleash.threshold.mutant-coverage"

docs/docs/usage/commands/unleash.md → docs/docs/usage/commands/unleash/index.md

@@ -143,7 +143,7 @@ gremlins unleash --threshold-mcover 80
 
 :material-flag: `--arithmetic-base` · :material-sign-direction: Default: `true`
 
-Enables/disables the [ARITHMETIC BASE](../mutations/arithmetic_base.md) mutant type.
+Enables/disables the [ARITHMETIC BASE](../../mutations/arithmetic_base.md) mutant type.
 
 ```shell
 gremlins unleash --arithmetic-base=false
@@ -153,7 +153,7 @@ gremlins unleash --arithmetic-base=false
 
 :material-flag: `--conditionals-boundary` · :material-sign-direction: Default: `true`
 
-Enables/disables the [CONDITIONALS BOUNDARY](../mutations/conditionals_boundary.md) mutant type.
+Enables/disables the [CONDITIONALS BOUNDARY](../../mutations/conditionals_boundary.md) mutant type.
 
 ```shell
 gremlins unleash --conditionals_boundary=false
@@ -163,7 +163,7 @@ gremlins unleash --conditionals_boundary=false
 
 :material-flag: `--conditionals-negation` · :material-sign-direction: Default: `true`
 
-Enables/disables the [CONDITIONALS NEGATION](../mutations/conditionals_negation.md) mutant type.
+Enables/disables the [CONDITIONALS NEGATION](../../mutations/conditionals_negation.md) mutant type.
 
 ```shell
 gremlins unleash --conditionals_negation=false
@@ -173,7 +173,7 @@ gremlins unleash --conditionals_negation=false
 
 :material-flag: `--increment-decrement` · :material-sign-direction: Default: `true`
 
-Enables/disables the [INCREMENT DECREMENT](../mutations/increment_decrement.md) mutant type.
+Enables/disables the [INCREMENT DECREMENT](../../mutations/increment_decrement.md) mutant type.
 
 ```shell
 gremlins unleash --increment-decrement=false
@@ -183,8 +183,61 @@ gremlins unleash --increment-decrement=false
 
 :material-flag: `--invert-negatives` · :material-sign-direction: Default: `true`
 
-Enables/disables the [INVERT NEGATIVES](../mutations/invert_negatives.md) mutant type.
+Enables/disables the [INVERT NEGATIVES](../../mutations/invert_negatives.md) mutant type.
 
 ```shell
 gremlins unleash --invert_negatives=false
 ```
+
+### Workers
+
+:material-flag: `--workers` · :material-sign-direction: Default: `0`
+
+[//]: # (@formatter:off)
+!!! tip
+    To understand better the use of these flag, check [workers](workers.md)
+[//]: # (@formatter:on)
+
+Gremlins runs in parallel mode, which means that more than one test at a time will be performed, based on the number of
+CPU cores available.
+
+By default, Gremlins will use all the available CPU cores of, and , in _integration mode_, it will use half of the
+available CPU cores.
+
+The `--workers` flag allows to override the number of CPUs to use (`0` means use the default).
+
+```shell
+gremlins unleash --workers=4
+```
+
+### Test CPU
+
+:material-flag: `--test-cpu` · :material-sign-direction: Default: `0`
+
+[//]: # (@formatter:off)
+!!! tip
+    To understand better the use of these flag, check [workers](workers.md)
+[//]: # (@formatter:on)
+
+This flag overrides the number of CPUs the Go test tool will utilize. By default, Gremlins doesn't set this value.
+
+```shell
+gremlins unleash --test-cpu=1
+```
+
+### Timeout coefficient
+
+:material-flag: `--timeout-coefficient` · :material-sign-direction: Default: `0`
+
+[//]: # (@formatter:off)
+!!! tip
+    To understand better the use of these flag, check [workers](workers.md)
+[//]: # (@formatter:on)
+
+Gremlins determines the timeout for each Go test run by multiplying by a coefficient the time it took to perform the
+coverage run.
+It is possible to override this coefficient (`0` means use the default).
+
+```shell
+gremlins unleash --timeout-coefficient=3
+```

docs/docs/usage/commands/unleash/workers.md

@@ -0,0 +1,36 @@
+# Workers
+
+Gremlins works in parallel mode. It uses some sensible defaults, but it may be necessary to tweak them in your specific
+use case. Finding the correct settings mandates a little trial and error, and we are still learning how to get the most
+of it.
+
+The first setting you should be aware of is the number of _workers_ (`--workers`). By default, Gremlins uses the number
+of available CPU cores. This value is correct most of the time, but if you notice an excessive number of mutations going
+into `TIMED OUT`, you may try to decrease this value.
+
+If you decrease this value, you may also try to increase the number of CPU cores available to each test
+run (`--test-cpu`). This is equivalent to the `-cpu` flag of the Go test tool, but for each mutation test. Gremlins
+doesn't enforce this by default.
+
+A rule of thumb may be setting it so that the sum of _workers_ and _test CPU_ is equal to the total number of cores of
+of the machine.
+
+The symptom of a run excessively stressed is the number of mutants going into `TIMED OUT`. You should tweak the two
+values above until your runs stabilize on a low and constant number of `TIMED OUT` mutants. To understand what could be
+your correct value, you can run Gremlins with a single worker and see the results.
+
+## Timeout coefficient
+
+Another setting you may want to tweak is the _timeout coefficient_. This is the multiplier used to increase the
+estimated time it takes to do a run of the tests. The default value should be ok, but if you see too much tests timing
+out, then you may try to play a little with this value. Don't increase it too much though, or the run might become
+excessively slow. At the moment, it defaults to 3.
+
+If your test suite takes a lot of time to run, you may want to tweak this setting to _decrease_ the coefficient. We are
+thinking of a dynamic way to set this, but it is not clear yet the correct algorithm to use.
+
+## Integration mode
+
+_Integration mode_ is quite heavy on the CPU in parallel mode. For this reason, Gremlins halves the values for workers
+and _test CPU_ if it is running in _integration mode_. So, if you set, for example, 4 workers, it will run effectively
+with 2. And same goes for _test CPU_.

docs/docs/usage/configuration.md

@@ -50,7 +50,10 @@ unleash:
   dry-run: false
   tags: ""
   output: ""
-  threshold: #(1)
+  workers: 0 #(1)
+  test-cpu: 0 #(2)
+  timeout-coefficient: 0 #(3)
+  threshold: #(4)
     efficacy: 0
     mutant-coverage: 0
 
@@ -68,8 +71,12 @@ mutants:
 
 ```
 
-1. Thresholds are set by default to `0`, which means they are not enforced. For further information check the specific
-   documentation.
+1. By default `0`, which means that Gremlins will use the system CPUs number.
+2. By default `0`, which means that no test process CPU will be enforced.
+3. By default `0`, which means a default coefficient will be enforced.
+4. Thresholds are set by default to `0`, which means they are not enforced.
+
+For further information check the specific command documentation.
 
 [//]: # (@formatter:off)
 !!! tip

docs/mkdocs.yml

@@ -45,7 +45,9 @@ nav:
   - Usage:
       - Commands:
           - usage/commands/index.md
-          - usage/commands/unleash.md
+          - Unleash:
+            - usage/commands/unleash/index.md
+            - usage/commands/unleash/workers.md
       - usage/configuration.md
       - Mutations:
           - usage/mutations/index.md