From dc1552f9cf8dbd893c965f20a253ea59539e0623 Mon Sep 17 00:00:00 2001 From: "U-ASIAPACIFIC\\nanjundl" Date: Tue, 25 Apr 2023 17:52:01 +0530 Subject: [PATCH] Added changes to validate nexus-repositories.yaml against schema files --- .../manifest_data_validate.go | 68 ++++++++++++++----- src/api/models/iuf/mutils/utils.go | 2 +- .../schemas/nr-hosted-repo-schema.yaml | 22 +++--- tests/manifest_validate_test.go | 4 +- tests/utils_test.go | 20 ++++++ 5 files changed, 88 insertions(+), 28 deletions(-) diff --git a/src/api/models/iuf/manifestDataValidation/manifest_data_validate.go b/src/api/models/iuf/manifestDataValidation/manifest_data_validate.go index 2425e068..fb797105 100755 --- a/src/api/models/iuf/manifestDataValidation/manifest_data_validate.go +++ b/src/api/models/iuf/manifestDataValidation/manifest_data_validate.go @@ -1,15 +1,12 @@ package manifestDataValidation import ( - "bytes" - "errors" "fmt" - "io" mutils "github.com/Cray-HPE/cray-nls/src/api/models/iuf/mutils" - //v2yaml "gopkg.in/yaml.v2" - goyaml "github.com/go-yaml/yaml" + sv "github.com/Cray-HPE/cray-nls/src/api/models/iuf/schemaValidator" + "sigs.k8s.io/yaml" ) // const for yaml keys @@ -36,9 +33,10 @@ var FileReader = mutils.ReadYamFile // Struct with list of validators type validators struct { - content map[string]interface{} - nexusRepoFileName string - hostedRepoNames []string + content map[string]interface{} + nexusRepoFileName string + hostedRepoNames []string + nexusSchemaFileNames []string } // Method to process s3 content, returns error in case of issues @@ -83,6 +81,33 @@ func (vs *validators) processNexusRepo() error { return nil } +// Method to process nexus repo content, returns repo file path and error(in case of issues), +func (vs *validators) processNexusRepoSchemaFile(nc interface{}) error { + + // Nexus repo possible schemas + vs.nexusSchemaFileNames = append(vs.nexusSchemaFileNames, "schemas/nr-hosted-repo-schema.yaml") + vs.nexusSchemaFileNames = append(vs.nexusSchemaFileNames, "schemas/nr-group-repo-schema.yaml") + + // nexus repo schema validation + + validated := false + + for _, sf := range vs.nexusSchemaFileNames { + err := sv.Validate(nc, sf) + if err != nil { + fmt.Println("failed to validate ", sf, " schema: ", err) + } else { + validated = true + break + } + } + + if !validated { + return fmt.Errorf("failed to validate against all schemas") + } + return nil +} + // Method to process nexus repo file and get hosted repo names func (vs *validators) processNexusRepoFile() error { @@ -91,29 +116,38 @@ func (vs *validators) processNexusRepoFile() error { } nexusFile_contents, err := FileReader(vs.nexusRepoFileName) + var temp_repo_names []string if err != nil { return fmt.Errorf("failed to open Nexus Repository file: %v", err) } - dec := goyaml.NewDecoder(bytes.NewReader(nexusFile_contents)) + docs := mutils.SplitMultiYamlFile(nexusFile_contents) skipFormats := []string{"docker", "helm"} - for { - var nexusContent map[string]interface{} + for _, d := range docs { - err := dec.Decode(&nexusContent) - if errors.Is(err, io.EOF) { - break + var nc interface{} + err = yaml.Unmarshal(d, &nc) + if err != nil { + return fmt.Errorf("failed to parse Nexus Repositorty as YAML: %v", err) } + err = vs.processNexusRepoSchemaFile(nc) + if err != nil { + return fmt.Errorf("failed to validate Nexus Repository yaml file against available schemas: %v", err) + } + + nexusContent := nc.(map[string]interface{}) + format := nexusContent[FORMAT].(string) isFormatToBeSkipped, _ := mutils.StringFoundInArray(skipFormats, format) if isFormatToBeSkipped { + fmt.Println("Format skipped") continue //Skip doc which has format that does not require validataion } @@ -123,7 +157,7 @@ func (vs *validators) processNexusRepoFile() error { temp_repo_names = append(temp_repo_names, nexusContent["name"].(string)) } else if nexusContent[REPO_TYPE] == "group" { - group_map := nexusContent["group"].(map[interface{}]interface{}) + group_map := nexusContent["group"].(map[string]interface{}) for _, v := range group_map { memNames_array := v.([]interface{}) @@ -136,7 +170,7 @@ func (vs *validators) processNexusRepoFile() error { temp_repo_names, err = mutils.Delete(temp_repo_names, index) if err != nil { - fmt.Println(err) + fmt.Println("Repo defined in host repo is not listed in group repo") } } else { @@ -145,7 +179,9 @@ func (vs *validators) processNexusRepoFile() error { } } } + } + if len(temp_repo_names) > 0 { return fmt.Errorf("Repo defined in host repo is not listed in group repo") } diff --git a/src/api/models/iuf/mutils/utils.go b/src/api/models/iuf/mutils/utils.go index af5638f6..929662e9 100755 --- a/src/api/models/iuf/mutils/utils.go +++ b/src/api/models/iuf/mutils/utils.go @@ -71,7 +71,7 @@ func ReadYamFile(filePath string) ([]byte, error) { func SplitMultiYamlFile(fileData []byte) [][]byte { var yamlDataBytes [][]byte for _, yamlData := range strings.Split(string(fileData), yamlFileDelimiter) { - if yamlData == "\n" { // skipping new line characters + if yamlData == "\n" || yamlData == "" { // skipping new line characters continue } yamlDataBytes = append(yamlDataBytes, []byte(yamlData)) diff --git a/src/api/models/iuf/schemaValidator/schemas/nr-hosted-repo-schema.yaml b/src/api/models/iuf/schemaValidator/schemas/nr-hosted-repo-schema.yaml index b01b4eef..05532957 100755 --- a/src/api/models/iuf/schemaValidator/schemas/nr-hosted-repo-schema.yaml +++ b/src/api/models/iuf/schemaValidator/schemas/nr-hosted-repo-schema.yaml @@ -47,12 +47,14 @@ required: properties: cleanup: - description: > - Should the repo be cleaned. - type: string - default: none - examples: - - null + type: [ object, "null" ] + properties: + policyNames: + type: array + example: weekly-cleanup + description: Components that match any of the applied policies will be deleted + items: + type: string format: description: > @@ -98,12 +100,12 @@ properties: repository format writePolicy: type: string - example: allow_once + example: ALLOW_ONCE description: Controls if deployments of and updates to assets are allowed enum: - - allow - - allow_once - - deny + - ALLOW + - ALLOW_ONCE + - DENY type: description: > diff --git a/tests/manifest_validate_test.go b/tests/manifest_validate_test.go index 2706bc88..5a8281a3 100755 --- a/tests/manifest_validate_test.go +++ b/tests/manifest_validate_test.go @@ -387,7 +387,9 @@ storage: strictContentTypeValidation: false type: group --- -cleanup: null +cleanup: + policyNames: + - weekly-cleanup format: raw name: cos-2.5.97-net-sle-15sp4-shs-2.0 online: true diff --git a/tests/utils_test.go b/tests/utils_test.go index ba1fd235..a5ed2b30 100644 --- a/tests/utils_test.go +++ b/tests/utils_test.go @@ -107,3 +107,23 @@ iuf_version: ^0.5.0 } } } + +func TestMultiSchemaYamlDocSingle_3(t *testing.T) { + data := []byte(`--- +iuf_version: ^0.5.0 +`) + + expected := [][]byte{ + []byte(` +iuf_version: ^0.5.0 +`)} + + response := mutils.SplitMultiYamlFile(data) + + for i, b := range expected { + + if string(b) != string(response[i]) { + t.Fatal("Spilt operations is not working properly, expected:", string(b), "response got:", string(response[i])) + } + } +}