Add TIMED OUT and NOT VIABLE (#40) (#49)

.deepsource.toml

@@ -5,4 +5,6 @@ name = "go"
 enabled = true
 
 [analyzers.meta]
-import_root = "github.com/k3rn31/gremlins"
+import_root = "github.com/k3rn31/gremlins"
+cgo_enabled = false
+dependencies_vendored = false

.github/workflows/release.yml

@@ -29,3 +29,4 @@ jobs:
           args: release --rm-dist
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          WORKFLOW_PAT: ${{ secrets.WORKFLOW_PAT }}

.goreleaser.yaml

@@ -33,6 +33,9 @@ brews:
   - tap:
       owner: k3rn31
       name: gremlins-tap
+      branch: main
+      token: "{{ .Env.WORKFLOW_PAT }}"
+    folder: Formula
     homepage: https://github.com/k3rn31/gremlins
     description: A mutation testing tool for Go.
     license: Apache 2.0 License

README.md

@@ -8,8 +8,8 @@
 [![codecov](https://codecov.io/gh/k3rn31/gremlins/branch/main/graph/badge.svg?token=MICF9A6U3J)](https://codecov.io/gh/k3rn31/gremlins)
 [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fk3rn31%2Fgremlins.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2Fk3rn31%2Fgremlins?ref=badge_shield)
 
-**WARNING: Gremlins is in its early stages of development, and it can be unstable, poorly performant and not really
-polished.**
+**WARNING1: Gremlins is in its early stages of development, and it can be unstable and/or poorly performant.**
+**WARNING2: Gremlins isn't currently supported on Windows.**
 
 Gremlins is a mutation testing tool for Go.
 
@@ -27,6 +27,7 @@ Gremlins is a mutation testing tool for Go.
 - [What Inspired Gremlins](#what-inspired-gremlins)
 - [Other Mutation Testing Tools for Go](#other-mutation-testing-tools-for-go)
 - [Contributing](#contributing)
+- [License](#license)
 
 ## What is Mutation Testing
 
@@ -102,6 +103,16 @@ To perform the analysis without actually running the tests:
 $ gremlins unleash --dry-run
 ```
 
+Gremlins will report each mutation as:
+
+- `RUNNABLE`: In _dry-run_ mode, a mutation that can be tested.
+- `NOT COVERED`: A mutation not covered by tests; it will not be tested.
+- `KILLED`: The mutation has been caught by the test suite.
+- `LIVED`: The mutation hasn't been caught by the test suite.
+- `TIMED OUT`: The tests timed out while testing the mutation: the mutation actually made the tests fail, but not
+  explicitly.
+- `NOT VIABLE`: The mutation makes the build fail.
+
 ### Supported mutations
 
 #### Conditionals Boundaries

mutant/mutant.go

@@ -36,6 +36,8 @@ const (
 	Runnable
 	Lived
 	Killed
+	NotViable
+	TimedOut
 )
 
 func (ms Status) String() string {
@@ -48,6 +50,10 @@ func (ms Status) String() string {
 		return "LIVED"
 	case Killed:
 		return "KILLED"
+	case NotViable:
+		return "NOT VIABLE"
+	case TimedOut:
+		return "TIMED OUT"
 	default:
 		panic("this should not happen")
 	}

mutant/mutant_test.go

@@ -48,6 +48,16 @@ func TestStatusString(t *testing.T) {
 			mutant.Killed,
 			"KILLED",
 		},
+		{
+			"NotViable",
+			mutant.NotViable,
+			"NOT VIABLE",
+		},
+		{
+			"TimedOut",
+			mutant.TimedOut,
+			"TIMED OUT",
+		},
 	}
 	for _, tc := range testCases {
 		tc := tc

mutator/mutator.go

@@ -17,6 +17,7 @@
 package mutator
 
 import (
+	"context"
 	"github.com/k3rn31/gremlins/coverage"
 	"github.com/k3rn31/gremlins/log"
 	"github.com/k3rn31/gremlins/mutant"
@@ -55,7 +56,7 @@ type Mutator struct {
 
 const timeoutCoefficient = 2
 
-type execContext = func(name string, args ...string) *exec.Cmd
+type execContext = func(ctx context.Context, name string, args ...string) *exec.Cmd
 
 // Option for the Mutator initialization.
 type Option func(m Mutator) Mutator
@@ -77,7 +78,7 @@ func New(fs fs.FS, r coverage.Result, manager workdir.Dealer, opts ...Option) Mu
 		covProfile:        r.Profile,
 		testExecutionTime: r.Elapsed * timeoutCoefficient,
 		fs:                fs,
-		execContext:       exec.Command,
+		execContext:       exec.CommandContext,
 		apply: func(m mutant.Mutant) error {
 			return m.Apply()
 		},
@@ -212,24 +213,19 @@ func (mu *Mutator) executeTests() report.Results {
 			report.Mutant(m)
 			continue
 		}
+
 		if err := mu.apply(m); err != nil {
 			log.Errorf("failed to apply mutation at %s - %s\n\t%v", m.Position(), m.Status(), err)
 			continue
 		}
-		m.SetStatus(mutant.Lived)
-		args := []string{"test", "-timeout", mu.testExecutionTime.String()}
-		if mu.buildTags != "" {
-			args = append(args, "-tags", mu.buildTags)
-		}
-		args = append(args, "./...")
-		cmd := mu.execContext("go", args...)
-		if err := cmd.Run(); err != nil {
-			m.SetStatus(mutant.Killed)
-		}
+
+		m.SetStatus(mu.runTests())
+
 		if err := mu.rollback(m); err != nil {
-			log.Errorf("failed to restore mutation at %s - %s\n\t%v", m.Position(), m.Status(), err)
 			// What should we do now?
+			log.Errorf("failed to restore mutation at %s - %s\n\t%v", m.Position(), m.Status(), err)
 		}
+
 		report.Mutant(m)
 		mutants = append(mutants, m)
 	}
@@ -239,3 +235,41 @@ func (mu *Mutator) executeTests() report.Results {
 	}
 	return results
 }
+
+func (mu *Mutator) runTests() mutant.Status {
+	ctx, cancel := context.WithTimeout(context.Background(), mu.testExecutionTime)
+	defer cancel()
+	cmd := mu.execContext(ctx, "go", mu.getTestArgs()...)
+
+	err := cmd.Run()
+	if ctx.Err() == context.DeadlineExceeded {
+		return mutant.TimedOut
+	}
+	if err != nil {
+		err, ok := err.(*exec.ExitError)
+		if ok {
+			return getTestFailedStatus(err.ExitCode())
+		}
+	}
+	return mutant.Lived
+}
+
+func (mu *Mutator) getTestArgs() []string {
+	args := []string{"test"}
+	if mu.buildTags != "" {
+		args = append(args, "-tags", mu.buildTags)
+	}
+	args = append(args, "./...")
+	return args
+}
+
+func getTestFailedStatus(exitCode int) mutant.Status {
+	switch exitCode {
+	case 1:
+		return mutant.Killed
+	case 2:
+		return mutant.NotViable
+	default:
+		return mutant.Lived
+	}
+}

mutator/mutator_test.go

@@ -17,6 +17,7 @@
 package mutator_test
 
 import (
+	"context"
 	"fmt"
 	"github.com/google/go-cmp/cmp"
 	"github.com/k3rn31/gremlins/coverage"
@@ -32,16 +33,18 @@ import (
 	"time"
 )
 
+const expectedTimeout = 10 * time.Second
+
 func coveredPosition(fixture string) coverage.Result {
 	fn := filenameFromFixture(fixture)
 	p := coverage.Profile{fn: {{StartLine: 6, EndLine: 7, StartCol: 8, EndCol: 9}}}
-	return coverage.Result{Profile: p, Elapsed: 1 * time.Second}
+	return coverage.Result{Profile: p, Elapsed: expectedTimeout}
 }
 
 func notCoveredPosition(fixture string) coverage.Result {
 	fn := filenameFromFixture(fixture)
 	p := coverage.Profile{fn: {{StartLine: 9, EndLine: 9, StartCol: 8, EndCol: 9}}}
-	return coverage.Result{Profile: p, Elapsed: 1 * time.Second}
+	return coverage.Result{Profile: p, Elapsed: expectedTimeout}
 }
 
 type dealerStub struct{}
@@ -280,9 +283,10 @@ func TestSkipTestAndNonGoFiles(t *testing.T) {
 type commandHolder struct {
 	command string
 	args    []string
+	timeout time.Duration
 }
 
-type execContext = func(name string, args ...string) *exec.Cmd
+type execContext = func(ctx context.Context, name string, args ...string) *exec.Cmd
 
 func TestMutatorRun(t *testing.T) {
 	t.Parallel()
@@ -308,12 +312,25 @@ func TestMutatorRun(t *testing.T) {
 
 	_ = mut.Run()
 
-	want := "go test -timeout 2s -tags tag1 tag2 ./..."
+	want := "go test -tags tag1 tag2 ./..."
 	got := fmt.Sprintf("go %v", strings.Join(holder.args, " "))
 
 	if !cmp.Equal(got, want) {
 		t.Errorf(cmp.Diff(got, want))
 	}
+
+	timeoutDifference := absTimeDiff(holder.timeout, expectedTimeout*2)
+	diffThreshold := 50 * time.Microsecond
+	if timeoutDifference > diffThreshold {
+		t.Errorf("expected timeout to be within %s from the set timeout, got %s", diffThreshold, timeoutDifference)
+	}
+}
+
+func absTimeDiff(a, b time.Duration) time.Duration {
+	if a > b {
+		return a - b
+	}
+	return b - a
 }
 
 func TestMutatorTestExecution(t *testing.T) {
@@ -341,10 +358,17 @@ func TestMutatorTestExecution(t *testing.T) {
 		{
 			name:          "if tests fails then mutation is KILLED",
 			fixture:       "testdata/fixtures/gtr_go",
-			testResult:    fakeExecCommandFailure,
+			testResult:    fakeExecCommandTestsFailure,
 			covResult:     coveredPosition("testdata/fixtures/gtr_go"),
 			wantMutStatus: mutant.Killed,
 		},
+		{
+			name:          "if build fails then mutation is BUILD FAILED",
+			fixture:       "testdata/fixtures/gtr_go",
+			testResult:    fakeExecCommandBuildFailure,
+			covResult:     coveredPosition("testdata/fixtures/gtr_go"),
+			wantMutStatus: mutant.NotViable,
+		},
 	}
 	for _, tc := range testCases {
 		tc := tc
@@ -390,43 +414,58 @@ func TestCoverageProcessSuccess(_ *testing.T) {
 	os.Exit(0)
 }
 
-func TestCoverageProcessFailure(_ *testing.T) {
+func TestProcessTestsFailure(_ *testing.T) {
 	if os.Getenv("GO_TEST_PROCESS") != "1" {
 		return
 	}
 	os.Exit(1)
 }
 
-func fakeExecCommandSuccess(command string, args ...string) *exec.Cmd {
+func TestProcessBuildFailure(_ *testing.T) {
+	if os.Getenv("GO_TEST_PROCESS") != "1" {
+		return
+	}
+	os.Exit(2)
+}
+
+func fakeExecCommandSuccess(ctx context.Context, command string, args ...string) *exec.Cmd {
 	cs := []string{"-test.run=TestCoverageProcessSuccess", "--", command}
 	cs = append(cs, args...)
 	// #nosec G204 - We are in tests, we don't care
-	cmd := exec.Command(os.Args[0], cs...)
+	cmd := exec.CommandContext(ctx, os.Args[0], cs...)
 	cmd.Env = []string{"GO_TEST_PROCESS=1"}
 	return cmd
 }
 
 func fakeExecCommandSuccessWithHolder(got *commandHolder) execContext {
-	return func(command string, args ...string) *exec.Cmd {
+	return func(ctx context.Context, command string, args ...string) *exec.Cmd {
+		dl, _ := ctx.Deadline()
 		if got != nil {
 			got.command = command
 			got.args = args
+			got.timeout = time.Until(dl)
 		}
 		cs := []string{"-test.run=TestCoverageProcessSuccess", "--", command}
 		cs = append(cs, args...)
-		// #nosec G204 - We are in tests, we don't care
-		cmd := exec.Command(os.Args[0], cs...)
-		cmd.Env = []string{"GO_TEST_PROCESS=1"}
-
-		return cmd
+		return getCmd(ctx, cs)
 	}
 }
 
-func fakeExecCommandFailure(command string, args ...string) *exec.Cmd {
-	cs := []string{"-test.run=TestCoverageProcessFailure", "--", command}
+func fakeExecCommandTestsFailure(ctx context.Context, command string, args ...string) *exec.Cmd {
+	cs := []string{"-test.run=TestProcessTestsFailure", "--", command}
 	cs = append(cs, args...)
+	return getCmd(ctx, cs)
+}
+
+func fakeExecCommandBuildFailure(ctx context.Context, command string, args ...string) *exec.Cmd {
+	cs := []string{"-test.run=TestProcessBuildFailure", "--", command}
+	cs = append(cs, args...)
+	return getCmd(ctx, cs)
+}
+
+func getCmd(ctx context.Context, cs []string) *exec.Cmd {
 	// #nosec G204 - We are in tests, we don't care
-	cmd := exec.Command(os.Args[0], cs...)
+	cmd := exec.CommandContext(ctx, os.Args[0], cs...)
 	cmd.Env = []string{"GO_TEST_PROCESS=1"}
 	return cmd
 }