Merge pull request #1089 from cloudflare/tests

Simplify tests

cmd/pint/main_test.go

@@ -25,33 +25,17 @@ import (
 	"github.com/rogpeppe/go-internal/testscript"
 )
 
-// mock command that fails tests if error is returned.
-func mockMainShouldSucceed() int {
-	app := newApp()
-	err := app.Run(os.Args)
-	if err != nil {
-		slog.Error("Fatal error", slog.Any("err", err))
-		return 1
-	}
-	return 0
-}
-
-// mock command that fails tests if no error is returned.
-func mockMainShouldFail() int {
-	app := newApp()
-	err := app.Run(os.Args)
-	if err != nil {
-		slog.Error("Fatal error", slog.Any("err", err))
-		return 0
-	}
-	fmt.Fprintf(os.Stderr, "expected an error but none was returned\n")
-	return 1
-}
-
 func TestMain(m *testing.M) {
 	os.Exit(testscript.RunMain(m, map[string]func() int{
-		"pint.ok":    mockMainShouldSucceed,
-		"pint.error": mockMainShouldFail,
+		"pint": func() int {
+			app := newApp()
+			err := app.Run(os.Args)
+			if err != nil {
+				slog.Error("Fatal error", slog.Any("err", err))
+				return 1
+			}
+			return 0
+		},
 	}))
 }
 

cmd/pint/tests/0001_match_path.txt

@@ -1,4 +1,4 @@
-pint.error --no-color lint rules
+! exec pint --no-color lint rules
 ! stdout .
 cmp stderr stderr.txt
 

cmd/pint/tests/0002_nothing_to_lint.txt

@@ -1,5 +1,5 @@
 mkdir rules
-pint.error  --no-color lint rules
+! exec pint  --no-color lint rules
 ! stdout .
 cmp stderr stderr.txt
 

cmd/pint/tests/0003_lint_workdir.txt

@@ -1,5 +1,5 @@
 env NO_COLOR=1
-pint.error --no-color lint --min-severity=info rules
+! exec pint --no-color lint --min-severity=info rules
 ! stdout .
 cmp stderr stderr.txt
 

cmd/pint/tests/0004_fail_invalid_yaml.txt

@@ -1,4 +1,4 @@
-pint.error --no-color lint rules
+! exec pint --no-color lint rules
 ! stdout .
 cmp stderr stderr.txt
 

cmd/pint/tests/0005_false_positive.txt

@@ -1,4 +1,4 @@
-pint.ok --no-color lint rules
+exec pint --no-color lint rules
 ! stdout .
 cmp stderr stderr.txt
 

cmd/pint/tests/0006_rr_labels.txt

@@ -1,4 +1,4 @@
-pint.error --no-color lint rules
+! exec pint --no-color lint rules
 ! stdout .
 cmp stderr stderr.txt
 

cmd/pint/tests/0007_alerts.txt

@@ -1,4 +1,4 @@
-pint.error --no-color lint rules
+! exec pint --no-color lint rules
 ! stdout .
 cmp stderr stderr.txt
 

cmd/pint/tests/0008_recording_rule_prometheus.txt

@@ -1,4 +1,4 @@
-pint.error --no-color lint rules
+! exec pint --no-color lint rules
 ! stdout .
 cmp stderr stderr.txt
 

cmd/pint/tests/0009_alerting_rule_prometheus.txt

@@ -1,4 +1,4 @@
-pint.error --no-color lint rules
+! exec pint --no-color lint rules
 ! stdout .
 cmp stderr stderr.txt
 

cmd/pint/tests/0010_syntax_check.txt

@@ -1,4 +1,4 @@
-pint.error --no-color lint rules
+! exec pint --no-color lint rules
 ! stdout .
 cmp stderr stderr.txt
 

cmd/pint/tests/0011_ignore_rules.txt

@@ -1,4 +1,4 @@
-pint.error --no-color lint rules
+! exec pint --no-color lint rules
 ! stdout .
 cmp stderr stderr.txt
 

cmd/pint/tests/0012_issue_20.txt

@@ -1,4 +1,4 @@
-pint.ok --no-color lint rules
+exec pint --no-color lint rules
 ! stdout .
 cmp stderr stderr.txt
 

cmd/pint/tests/0013_issue49_1.txt

@@ -1,4 +1,4 @@
-pint.error --no-color --config not_existed_config.hcl lint rules
+! exec pint --no-color --config not_existed_config.hcl lint rules
 ! stdout .
 cmp stderr stderr.txt
 

cmd/pint/tests/0014_issue49_2.txt

@@ -1,4 +1,4 @@
-pint.ok --no-color lint rules
+exec pint --no-color lint rules
 ! stdout .
 cmp stderr stderr.txt
 

cmd/pint/tests/0015_parse_1.txt

@@ -1,4 +1,4 @@
-pint.ok parse 'sum(rate(foo[5m])) without(job) + (sum(rate(bar[5m])) without(job) > 0)'
+exec pint parse 'sum(rate(foo[5m])) without(job) + (sum(rate(bar[5m])) without(job) > 0)'
 cmp stdout stdout.txt
 ! stderr .
 

cmd/pint/tests/0016_parse_2.txt

@@ -1,4 +1,4 @@
-pint.ok parse 'sum(rate(foo[5m:1m])) without(job) / on(instance) group_left() sum(rate(bar[5m])) without(job)'
+exec pint parse 'sum(rate(foo[5m:1m])) without(job) / on(instance) group_left() sum(rate(bar[5m])) without(job)'
 cmp stdout stdout.txt
 ! stderr .
 

cmd/pint/tests/0017_issue69.txt

@@ -1,6 +1,6 @@
 exec sh ./copy.sh
 exec sh ./ulimit.sh
-pint.error --no-color -l error lint rules
+! exec pint --no-color -l error lint rules
 ! stdout .
 
 cmp stderr stderr.txt

cmd/pint/tests/0018_match_alerting.txt

@@ -1,4 +1,4 @@
-pint.ok --no-color -l debug lint rules
+exec pint --no-color -l debug lint rules
 ! stdout .
 cmp stderr stderr.txt
 

cmd/pint/tests/0019_match_recording.txt

@@ -1,4 +1,4 @@
-pint.ok --no-color -l debug lint rules
+exec pint --no-color -l debug lint rules
 ! stdout .
 cmp stderr stderr.txt
 

cmd/pint/tests/0020_ignore_kind.txt

@@ -1,4 +1,4 @@
-pint.ok --no-color -l debug lint rules
+exec pint --no-color -l debug lint rules
 ! stdout .
 cmp stderr stderr.txt
 

cmd/pint/tests/0021_ignore_all.txt

@@ -1,4 +1,4 @@
-pint.error --no-color -l debug lint rules
+! exec pint --no-color -l debug lint rules
 ! stdout .
 cmp stderr stderr.txt
 

cmd/pint/tests/0022_ignore_multi.txt

@@ -1,4 +1,4 @@
-pint.ok --no-color lint rules
+exec pint --no-color lint rules
 ! stdout .
 cmp stderr stderr.txt
 

cmd/pint/tests/0023_enabled_checks.txt

@@ -1,4 +1,4 @@
-pint.error -l debug --no-color lint rules
+! exec pint -l debug --no-color lint rules
 ! stdout .
 stderr 'level=DEBUG msg="Configured checks for rule" enabled=\["promql/syntax","alerts/for","alerts/comparison","alerts/template","promql/fragile","promql/regexp","promql/rate\(prom\)","promql/series\(prom\)","promql/vector_matching\(prom\)"\,"promql/range_query\(prom\)","rule/duplicate\(prom\)","labels/conflict\(prom\)","alerts/external_labels\(prom\)","promql/counter\(prom\)","alerts/absent\(prom\)"] path=rules/1.yaml rule=one'
 stderr 'level=DEBUG msg="Configured checks for rule" enabled=\["promql/syntax","alerts/for","alerts/comparison","alerts/template","promql/fragile","promql/regexp","promql/rate\(prom\)","promql/series\(prom\)","promql/vector_matching\(prom\)"\,"promql/range_query\(prom\)","rule/duplicate\(prom\)","labels/conflict\(prom\)","alerts/external_labels\(prom\)","promql/counter\(prom\)","alerts/absent\(prom\)"] path=rules/1.yaml rule=two'

cmd/pint/tests/0024_color_output.txt

@@ -1,5 +1,5 @@
 env NO_COLOR=0
-pint.error lint rules
+! exec pint lint rules
 ! stdout .
 cmp stderr stderr.txt
 

cmd/pint/tests/0025_config.txt

@@ -1,6 +1,6 @@
 cert $WORK prom
 
-pint.ok --no-color config
+exec pint --no-color config
 ! stdout .
 cmp stderr stderr.txt
 

cmd/pint/tests/0026_aggregate_empty_name.txt

@@ -1,4 +1,4 @@
-pint.error --no-color config
+! exec pint --no-color config
 ! stdout .
 cmp stderr stderr.txt
 

cmd/pint/tests/0027_ci_branch.txt

@@ -15,7 +15,7 @@ exec git checkout -b v2
 cp ../src/v2.yml rules.yml
 exec git commit -am 'v2'
 
-pint.error --no-color ci
+! exec pint --no-color ci
 ! stdout .
 cmp stderr ../stderr.txt
 

cmd/pint/tests/0028_ci_git_error.txt

@@ -15,7 +15,7 @@ exec git checkout -b v2
 cp ../src/v2.yml rules.yml
 exec git commit -am 'v2'
 
-pint.error -l debug --no-color ci
+! exec pint -l debug --no-color ci
 ! stdout .
 cmp stderr ../stderr.txt
 

cmd/pint/tests/0029_ci_too_many_commits.txt

@@ -22,7 +22,7 @@ exec git commit -am 'recert to v1'
 cp ../src/v2.yml rules.yml
 exec git commit -am 'v2'
 
-pint.error --no-color ci
+! exec pint --no-color ci
 ! stdout .
 cmp stderr ../stderr.txt
 

cmd/pint/tests/0030_parse_string.txt

@@ -1,4 +1,4 @@
-pint.ok parse '"foo"'
+exec pint parse '"foo"'
 cmp stdout stdout.txt
 ! stderr .
 

cmd/pint/tests/0031_ci_bitbucket.txt

@@ -20,7 +20,7 @@ cp ../src/v2.yml rules.yml
 exec git commit -am 'v2'
 
 env BITBUCKET_AUTH_TOKEN="12345"
-pint.ok -l debug --no-color ci
+exec pint -l debug --no-color ci
 ! stdout .
 stderr 'msg="Sending a request to BitBucket" method=PUT'
 stderr 'msg="BitBucket request completed" status=200'

cmd/pint/tests/0032_ci_github.txt

@@ -24,7 +24,7 @@ exec git commit -am 'v2'
 
 env GITHUB_AUTH_TOKEN=12345
 env GITHUB_PULL_REQUEST_NUMBER=1
-pint.ok -l debug --offline --no-color ci
+exec pint -l debug --offline --no-color ci
 ! stdout .
 stderr 'level=INFO msg="Pull request review created" status="200 OK"'
 

cmd/pint/tests/0033_ci_github_multi.txt

@@ -24,7 +24,7 @@ exec git commit -am 'v2'
 
 env GITHUB_AUTH_TOKEN=12345
 env GITHUB_PULL_REQUEST_NUMBER=1
-pint.error -l debug --no-color ci
+! exec pint -l debug --no-color ci
 ! stdout .
 stderr 'level=INFO msg="Pull request review created" status="200 OK"'
 

cmd/pint/tests/0034_version.txt

@@ -1,4 +1,4 @@
-pint.ok version
+exec pint version
 cmp stdout stdout.txt
 ! stderr .
 

cmd/pint/tests/0035_bad_loglevel.txt

@@ -1,3 +1,3 @@
-pint.error -l invalid --no-color lint rules
+! exec pint -l invalid --no-color lint rules
 ! stdout .
 stderr 'ERROR Fatal error err="failed to set log level: ''invalid'' is not a valid log level"'

cmd/pint/tests/0036_ci_basebranch.txt

@@ -11,7 +11,7 @@ env [email protected]
 exec git add .
 exec git commit -am 'import rules and config'
 
-pint.ok --no-color ci
+exec pint --no-color ci
 ! stdout .
 cmp stderr ../stderr.txt
 

cmd/pint/tests/0037_disable_checks.txt

@@ -1,4 +1,4 @@
-pint.ok -l debug --no-color -d promql/counter -d alerts/for -d alerts/comparison -d promql/rate(prom) -d promql/series(prom) -d promql/aggregate(prom) -d promql/range_query lint rules
+exec pint -l debug --no-color -d promql/counter -d alerts/for -d alerts/comparison -d promql/rate(prom) -d promql/series(prom) -d promql/aggregate(prom) -d promql/range_query lint rules
 ! stdout .
 cmp stderr stderr.txt
 

cmd/pint/tests/0038_disable_checks_regex.txt

@@ -1,4 +1,4 @@
-pint.ok --no-color -d 'alerts/.*' -d 'promql/c.+' lint rules
+exec pint --no-color -d 'alerts/.*' -d 'promql/c.+' lint rules
 ! stdout .
 cmp stderr stderr.txt
 

cmd/pint/tests/0039_prom_selected_path.txt

@@ -1,4 +1,4 @@
-pint.ok -l debug --no-color lint rules
+exec pint -l debug --no-color lint rules
 ! stdout .
 cmp stderr stderr.txt
 

cmd/pint/tests/0040_rule_match_label.txt

@@ -1,4 +1,4 @@
-pint.ok --no-color -l debug lint rules
+exec pint --no-color -l debug lint rules
 ! stdout .
 cmp stderr stderr.txt
 

cmd/pint/tests/0041_watch.txt

@@ -1,6 +1,6 @@
 exec bash -x ./test.sh &
 
-pint.ok --no-color -l debug watch --interval=5s --listen=127.0.0.1:6041 --pidfile=pint.pid glob rules
+exec pint --no-color -l debug watch --interval=5s --listen=127.0.0.1:6041 --pidfile=pint.pid glob rules
 ! stdout .
 
 stderr 'level=INFO msg="Pidfile created" path=pint.pid'

cmd/pint/tests/0042_watch_metrics.txt

@@ -1,6 +1,6 @@
 exec bash -x ./test.sh &
 
-pint.ok watch --listen=127.0.0.1:6042 --pidfile=pint.pid glob rules
+exec pint watch --listen=127.0.0.1:6042 --pidfile=pint.pid glob rules
 cmp curl.txt metrics.txt
 
 -- test.sh --

cmd/pint/tests/0043_watch_cancel.txt

@@ -3,7 +3,7 @@ http start github 127.0.0.1:7043
 
 exec bash -x ./test.sh &
 
-pint.ok --no-color watch --interval=1h --listen=127.0.0.1:6043 --pidfile=pint.pid glob rules
+exec pint --no-color watch --interval=1h --listen=127.0.0.1:6043 --pidfile=pint.pid glob rules
 ! stdout .
 stderr 'level=INFO msg="Shutting down"'
 stderr 'level=INFO msg="Waiting for all background tasks to finish"'

cmd/pint/tests/0044_parse_error.txt

@@ -1,4 +1,4 @@
-pint.error parse 'sum(foo) by('
+! exec pint parse 'sum(foo) by('
 ! stdout .
 cmp stderr stderr.txt
 

cmd/pint/tests/0045_parse_no_query.txt

@@ -1,4 +1,4 @@
-pint.error --no-color parse
+! exec pint --no-color parse
 ! stdout .
 cmp stderr stderr.txt
 

cmd/pint/tests/0046_parse_3.txt

@@ -1,4 +1,4 @@
-pint.ok parse 'sum(http_requests_total{method="GET"} @ 1609746000)'
+exec pint parse 'sum(http_requests_total{method="GET"} @ 1609746000)'
 cmp stdout stdout.txt
 ! stderr .
 

cmd/pint/tests/0047_parse_4.txt

@@ -1,4 +1,4 @@
-pint.ok parse 'rate(http_requests_total[5m] offset -1w)'
+exec pint parse 'rate(http_requests_total[5m] offset -1w)'
 cmp stdout stdout.txt
 ! stderr .
 

cmd/pint/tests/0048_watch_limit.txt

@@ -1,6 +1,6 @@
 exec bash -x ./test.sh &
 
-pint.ok watch --listen=127.0.0.1:6048 --max-problems=2 --pidfile=pint.pid glob rules
+exec pint watch --listen=127.0.0.1:6048 --max-problems=2 --pidfile=pint.pid glob rules
 cmp curl.txt metrics.txt
 
 -- test.sh --

cmd/pint/tests/0049_watch_severity_warning.txt

@@ -1,6 +1,6 @@
 exec bash -x ./test.sh &
 
-pint.ok watch --listen=127.0.0.1:6049 --min-severity=warning --pidfile=pint.pid glob rules
+exec pint watch --listen=127.0.0.1:6049 --min-severity=warning --pidfile=pint.pid glob rules
 cmp curl.txt metrics.txt
 
 -- test.sh --

cmd/pint/tests/0050_watch_severity_fatal.txt

@@ -1,6 +1,6 @@
 exec bash -x ./test.sh &
 
-pint.ok watch --listen=127.0.0.1:6050 --min-severity=fatal --pidfile=pint.pid glob rules
+exec pint watch --listen=127.0.0.1:6050 --min-severity=fatal --pidfile=pint.pid glob rules
 cmp curl.txt metrics.txt
 
 -- test.sh --

cmd/pint/tests/0051_watch_severity_invalid.txt

@@ -1,4 +1,4 @@
-pint.error --no-color watch --min-severity=foo glob bar
+! exec pint --no-color watch --min-severity=foo glob bar
 ! stdout .
 cmp stderr stderr.txt
 

cmd/pint/tests/0052_match_multiple.txt

@@ -1,4 +1,4 @@
-pint.ok --no-color -l debug lint rules
+exec pint --no-color -l debug lint rules
 ! stdout .
 cmp stderr stderr.txt