Merge pull request #29 from jhrozek/fixes

Fixes to catch changes to versioned rules as well as rules going NOT-APPLICABLE

go.mod

@@ -9,16 +9,17 @@ require (
 	github.com/googleapis/gnostic v0.5.5 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/json-iterator/go v1.1.11 // indirect
+	github.com/onsi/gomega v1.19.0 // indirect
 	github.com/openshift/cluster-authentication-operator v0.0.3-0.20210603131321-6b9c13549b48
-	github.com/openshift/compliance-operator v0.1.34
+	github.com/openshift/compliance-operator v0.1.46
 	github.com/openshift/library-go v0.0.0-20210611143017-0d0ef669a361 // indirect
 	github.com/openshift/machine-config-operator v0.0.1-0.20200913004441-7eba765c69c9
-	golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a // indirect
-	golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect
+	github.com/stretchr/testify v1.7.2 // indirect
+	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
 	golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c // indirect
-	golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect
-	golang.org/x/term v0.0.0-20210503060354-a79de5458b56 // indirect
+	golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c // indirect
 	golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 // indirect
+	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0

helpers.go

@@ -394,6 +394,7 @@ func (ctx *e2econtext) ensureTestSettings(t *testing.T) {
 	}
 	autoApplySettings.AutoApplyRemediations = true
 	autoApplySettings.Debug = true
+	autoApplySettings.ShowNotApplicable = true // so that we can test if a setting goes from PASS/FAIL to N/A
 	err = backoff.RetryNotify(func() error {
 		found := &cmpv1alpha1.ScanSetting{}
 		if err := ctx.dynclient.Get(goctx.TODO(), key, found); err != nil {
@@ -883,10 +884,17 @@ func (ctx *e2econtext) verifyRule(
 
 // getTestDefinition attempts to use a versioned test (<version>.yml)
 // definition, if it fails it'll try to use the standard test
-// definition (e2e.yml).
+// definition (e2e.yml). If that does not exist either, the function checks
+// if other files (presumably versioned tests) exist in that file and if
+// they do, it would fail. This is better than just silently ignoring the
+// files because:
+//  1) we catch rules that have versioned results but no result for the
+//	   current version more easily
+//  2) with each version, this forces us to think if we can already retire
+//     certain rules
 func (ctx *e2econtext) getTestDefinition(rulePath string) ([]byte, error) {
 	versionedManifest := fmt.Sprintf("%s.yml", ctx.version)
-	versionedRuleTestFilePath := path.Join(ruleTestDir, versionedManifest)
+	versionedRuleTestFilePath := path.Join(rulePath, ruleTestDir, versionedManifest)
 	vbuf, verr := ioutil.ReadFile(versionedRuleTestFilePath)
 
 	if verr == nil {
@@ -897,8 +905,23 @@ func (ctx *e2econtext) getTestDefinition(rulePath string) ([]byte, error) {
 		return nil, verr
 	}
 
+	// the error is now os.IsNotExist, let's try the global file
 	testFilePath := path.Join(rulePath, ruleTestFilePath)
-	return ioutil.ReadFile(testFilePath)
+	gbuf, gerr := ioutil.ReadFile(testFilePath)
+	if os.IsNotExist(gerr) {
+		// let's check for other files and fail if they don't exist
+		files, err := os.ReadDir(ruleTestDir)
+		if err != nil {
+			return nil, err
+		}
+		if len(files) > 0 {
+			return nil, fmt.Errorf("E2E-FAILURE: the rule directory %s contains versioned files, but none for %s", ruleTestDir, ctx.version)
+		}
+	} else if gerr != nil {
+		return nil, gerr
+	}
+
+	return gbuf, nil
 }
 
 // getManualRemediationPath attempts to get a versioned remediation