From 086d0303c7b8400f61f53b4a732b9b467c9722e6 Mon Sep 17 00:00:00 2001 From: "U-ASIAPACIFIC\\nanjundl" Date: Wed, 12 Apr 2023 16:35:31 +0530 Subject: [PATCH] Added validation code for iuf-product-manifest.yaml --- go.mod | 1 + go.sum | 2 + .../manifest_data_validate.go | 271 ++++++++ src/api/models/iuf/manifest_validate.go | 7 +- src/api/models/iuf/mutils/utils.go | 65 ++ .../data/manifests/iuf-product-manifest.yaml | 61 ++ src/tests/data/nexus-blobstores.yaml | 4 + src/tests/data/nexus-repositories.yaml_bk | 4 + .../data/np/nexus-repositories-dummy.yaml | 0 src/tests/data/np/nexus-repositories.yaml | 84 +++ src/tests/data/np/nexus-repositories_cos.yaml | 84 +++ src/tests/data/np/nexus-repositories_csm.yaml | 151 +++++ src/tests/data/rpms/rpm_dummy_1/1 | 0 src/tests/data/rpms/rpm_dummy_2/2 | 0 src/tests/data/rpms/rpm_dummy_3/3 | 0 src/tests/data/s3/dummy_upload_1.txt | 0 src/tests/data/s3/dummy_upload_2.txt | 0 src/tests/data/vcs/dummy_vcs.txt | 0 src/tests/manifest_validate_test.go | 620 ++++++++++++++++++ 19 files changed, 1353 insertions(+), 1 deletion(-) create mode 100755 src/api/models/iuf/manifestDataValidation/manifest_data_validate.go create mode 100755 src/api/models/iuf/mutils/utils.go create mode 100755 src/tests/data/manifests/iuf-product-manifest.yaml create mode 100755 src/tests/data/nexus-blobstores.yaml create mode 100755 src/tests/data/nexus-repositories.yaml_bk create mode 100755 src/tests/data/np/nexus-repositories-dummy.yaml create mode 100755 src/tests/data/np/nexus-repositories.yaml create mode 100755 src/tests/data/np/nexus-repositories_cos.yaml create mode 100755 src/tests/data/np/nexus-repositories_csm.yaml create mode 100755 src/tests/data/rpms/rpm_dummy_1/1 create mode 100755 src/tests/data/rpms/rpm_dummy_2/2 create mode 100755 src/tests/data/rpms/rpm_dummy_3/3 create mode 100755 src/tests/data/s3/dummy_upload_1.txt create mode 100755 src/tests/data/s3/dummy_upload_2.txt create mode 100755 src/tests/data/vcs/dummy_vcs.txt create mode 100755 src/tests/manifest_validate_test.go diff --git a/go.mod b/go.mod index 4c262603..b43a3f43 100644 --- a/go.mod +++ b/go.mod @@ -81,6 +81,7 @@ require ( github.com/go-jose/go-jose/v3 v3.0.0 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-sql-driver/mysql v1.6.0 // indirect + github.com/go-yaml/yaml v2.1.0+incompatible // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/gofuzz v1.2.0 // indirect diff --git a/go.sum b/go.sum index fd551cd2..f9e45b16 100644 --- a/go.sum +++ b/go.sum @@ -211,6 +211,8 @@ github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o= +github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= diff --git a/src/api/models/iuf/manifestDataValidation/manifest_data_validate.go b/src/api/models/iuf/manifestDataValidation/manifest_data_validate.go new file mode 100755 index 00000000..2b5a9bc1 --- /dev/null +++ b/src/api/models/iuf/manifestDataValidation/manifest_data_validate.go @@ -0,0 +1,271 @@ +package manifestDataValidation + +import ( + "bytes" + "errors" + "fmt" + "io" + mutils "iuf-internal/src/api/models/iuf/mutils" + + //v2yaml "gopkg.in/yaml.v2" + goyaml "github.com/go-yaml/yaml" +) + +// const for yaml keys +const ( + CONTENT_KEY string = "content" + S3_KEY string = "s3" + S3_PATH_KEY string = "path" + NEXUS_REPO_KEY string = "nexus_repositories" + NEXUS_REPO_PATH_KEY string = "yaml_path" + NEXUS_BLOB_KEY string = "nexus_blob_stores" + NEXUS_BLOB_PATH_KEY string = "yaml_path" + VCS_KEY string = "vcs" + VCS_PATH_KEY string = "path" + RPM_KEY string = "rpms" + RPM_PATH_KEY string = "path" + REPO_NAME string = "repository_name" + HOSTED_REPO_NAME string = "name" + REPO_TYPE string = "type" + FORMAT string = "format" +) + +// Getting the file reader +var FileReader = mutils.ReadYamFile + +// Struct with list of validators +type validators struct { + content map[string]interface{} + nexusRepoFileName string + hostedRepoNames []string +} + +// Method to process s3 content, returns error in case of issues +func (vs *validators) processS3() error { + + s3, s3_present := vs.content[S3_KEY] // extracting s3 key + + if !s3_present { + return nil // s3 key missing + } + + s3_array := s3.([]interface{}) // assuming array, is it validated in schema?? + for _, s3 := range s3_array { + s3_element := s3.(map[string]interface{}) + file_path := s3_element[S3_PATH_KEY].(string) + + exist := mutils.IsPathExist(file_path) // do we need error details?? + if !exist { // if path is invalid + return fmt.Errorf("error in processing s3 file %v", file_path) + } + } + return nil +} + +// Method to process nexus repo content, returns repo file path and error(in case of issues), +func (vs *validators) processNexusRepo() error { + nr, nr_present := vs.content[NEXUS_REPO_KEY] // extracting nexus repo key + + if !nr_present { + return nil // nexus repo key missing + } + + nr_map := nr.(map[string]interface{}) // assuming map, is it validated in schema?? + file_path := nr_map[NEXUS_REPO_PATH_KEY].(string) + vs.nexusRepoFileName = file_path + + exist := mutils.IsPathExist(file_path) // do we need error details?? + if !exist { // if path is invalid + return fmt.Errorf("error in processing nexus repo file %v", file_path) + } + + return nil +} + +// Method to process nexus repo file and get hosted repo names +func (vs *validators) processNexusRepoFile() error { + + if vs.nexusRepoFileName == "" { + return nil // skip processing of nexus repo file + } + + nexusFile_contents, err := FileReader(vs.nexusRepoFileName) + var temp_repo_names []string + + if err != nil { + return fmt.Errorf("failed to open Nexus Repository file: %v", err) + } + + dec := goyaml.NewDecoder(bytes.NewReader(nexusFile_contents)) + + skipFormats := []string{"docker", "helm"} + + for { + var nexusContent map[string]interface{} + + err := dec.Decode(&nexusContent) + if errors.Is(err, io.EOF) { + break + } + + format := nexusContent[FORMAT].(string) + + isFormatToBeSkipped, _ := mutils.StringFoundInArray(skipFormats, format) + + if isFormatToBeSkipped { + continue //Skip doc which has format that does not require validataion + } + + if nexusContent[REPO_TYPE] == "hosted" { + + vs.hostedRepoNames = append(vs.hostedRepoNames, nexusContent["name"].(string)) + temp_repo_names = append(temp_repo_names, nexusContent["name"].(string)) + + } else if nexusContent[REPO_TYPE] == "group" { + group_map := nexusContent["group"].(map[interface{}]interface{}) + + for _, v := range group_map { + memNames_array := v.([]interface{}) + for _, m := range memNames_array { + + memberRepo := m.(string) + + isHostedRepo, index := mutils.StringFoundInArray(temp_repo_names, memberRepo) + if isHostedRepo { + temp_repo_names, err = mutils.Delete(temp_repo_names, index) + + if err != nil { + fmt.Println(err) + } + + } else { + return fmt.Errorf("Repo referenced in group does not match hosted repo or Hosted Repos are not listed before group repos") + } + } + } + } + } + if len(temp_repo_names) > 0 { + return fmt.Errorf("Repo defined in host repo is not listed in group repo") + } + + return nil +} + +// Method to process nexus blob content, returns error in case of issues +func (vs *validators) processNexusBlob() error { + nb, nb_present := vs.content[NEXUS_BLOB_KEY] // extracting nexus blob key + + if !nb_present { + return nil // nexus blob key missing + } + + nb_map := nb.(map[string]interface{}) // assuming map, is it validated in schema?? + file_path := nb_map[NEXUS_BLOB_PATH_KEY].(string) + + exist := mutils.IsPathExist(file_path) // do we need error details?? + if !exist { // if path is invalid + return fmt.Errorf("error in processing nexus repo file %v", file_path) + } + return nil +} + +// Method to process vcs content, return error in case of issues +func (vs *validators) processVcs() error { + + vcs, vcs_present := vs.content[VCS_KEY] // extracting nexus repo key + + if !vcs_present { + return nil // vcs key missing + } + vcs_map := vcs.(map[string]interface{}) + + dir_path := vcs_map[VCS_PATH_KEY].(string) + empty := mutils.IsEmptyDirectory(dir_path) // do we need error details?? + if empty { // if path is invalid + return fmt.Errorf("error in processing vcs directory %v", dir_path) + } + return nil +} + +// Method to process rpm content, return error in case of issues +func (vs *validators) processRpm() error { + + rpm, rpm_present := vs.content[RPM_KEY] // extracting rpm key + + if !rpm_present { + return nil // rpm key missing + } + + rpm_array := rpm.([]interface{}) + + for _, rpm := range rpm_array { + rpm_map := rpm.(map[string]interface{}) + + dir_path := rpm_map[RPM_PATH_KEY].(string) + empty := mutils.IsEmptyDirectory(dir_path) // do we need error details?? + if empty { // if path is invalid + return fmt.Errorf("error in processing rpm directory %v", dir_path) + } + repoName := rpm_map[REPO_NAME].(string) + + found := false + + found, _ = mutils.StringFoundInArray(vs.hostedRepoNames, repoName) + if !found { + return fmt.Errorf("Repo referenced in rpms section is not a hosted repo") + } + } + + return nil +} + +// Function to extract content data +// Assumtion: Schema validation is done be the data comes to this function +func getManifestContentMap(manifest interface{}) map[string]interface{} { + manifest_map := manifest.(map[string]interface{}) + content_map := manifest_map[CONTENT_KEY].(map[string]interface{}) + return content_map +} + +// Function to validate manifest data post schema validation +func Validate(manifest interface{}) error { + + var pipeline *validators = &validators{} + pipeline.content = getManifestContentMap(manifest) + + // var hosted_repo_names []string + // content := getManifestContentMap(manifest) + + // content.s3 checks + if err := pipeline.processS3(); err != nil { + return fmt.Errorf("issue in processing s3 content, details %v", err) + } + + // content.nexus_repositories checks + if err := pipeline.processNexusRepo(); err != nil { + return fmt.Errorf("issue in processing nexus repo file content, details %v", err) + } + + // Validate content of nexus_repositories.yaml + if err := pipeline.processNexusRepoFile(); err != nil { + return fmt.Errorf("issue in processing nexus repo file content, details %v", err) + } + + // content.nexus_blob_stores checks + if err := pipeline.processNexusBlob(); err != nil { + return fmt.Errorf("issue in processing nexus repo file content, details %v", err) + } + + // contents.vcs checks + if err := pipeline.processVcs(); err != nil { + return fmt.Errorf("issue in processing vcs directory content, details %v", err) + } + + // contents.rpms checks + if err := pipeline.processRpm(); err != nil { + return fmt.Errorf("issue in processing rpm directory content, details %v", err) + } + + return nil +} diff --git a/src/api/models/iuf/manifest_validate.go b/src/api/models/iuf/manifest_validate.go index ce937b9d..cfaf9dc3 100644 --- a/src/api/models/iuf/manifest_validate.go +++ b/src/api/models/iuf/manifest_validate.go @@ -2,7 +2,7 @@ * * MIT License * - * (C) Copyright 2022 Hewlett Packard Enterprise Development LP + * (C) Copyright 2022-2023 Hewlett Packard Enterprise Development LP * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -28,6 +28,7 @@ package iuf import ( "embed" "fmt" + mdv "iuf-internal/src/api/models/iuf/manifestDataValidation" "os" "github.com/santhosh-tekuri/jsonschema/v5" @@ -84,5 +85,9 @@ func Validate(file_contents []byte) error { return fmt.Errorf("failed to validate IUF Product Manifest schema: %#v", err) } + err = mdv.Validate(manifest) + if err != nil { + return fmt.Errorf("failed to validate IUF Product Manifest data: %#v", err) + } return nil } diff --git a/src/api/models/iuf/mutils/utils.go b/src/api/models/iuf/mutils/utils.go new file mode 100755 index 00000000..bc5846c6 --- /dev/null +++ b/src/api/models/iuf/mutils/utils.go @@ -0,0 +1,65 @@ +package mutils + +import ( + "errors" + "io" + "os" +) + +// Function to check a path exist or not +func IsPathExist(path string) bool { + _, err := os.Stat(path) + if err != nil { + return false + } else { + return true + } +} + +// Function to check a directory is empty or not +func IsEmptyDirectory(directoryPath string) bool { + f, err := os.Open(directoryPath) // reading the path given + + if err != nil { + return true + } + + defer f.Close() // close the file handler post function exist + + _, err = f.Readdir(1) // checking for atleast single file in the directory + + if err == io.EOF { // if empty + return true + } else { + return false + } +} + +// Function for string search operations +func StringFoundInArray(searchArray []string, searchString string) (found bool, index int) { + found = false + for i, x := range searchArray { + if x == searchString { + found = true + index = i + + break + } + } + return found, index +} + +func Delete(orig []string, index int) ([]string, error) { + if index < 0 || index >= len(orig) { + return nil, errors.New("Index cannot be less than 0") + } + + orig = append(orig[:index], orig[index+1:]...) + + return orig, nil +} + +// File to manage +func ReadYamFile(filePath string) ([]byte, error) { + return os.ReadFile(filePath) +} diff --git a/src/tests/data/manifests/iuf-product-manifest.yaml b/src/tests/data/manifests/iuf-product-manifest.yaml new file mode 100755 index 00000000..b5566d04 --- /dev/null +++ b/src/tests/data/manifests/iuf-product-manifest.yaml @@ -0,0 +1,61 @@ +--- +iuf_version: ^0.5.0 +name: cos +description: > + The Cray Operating System (COS). +version: 2.5.97 + +hooks: + deliver_product: + post: + script_path: hooks/deliver_product-posthook.sh + post_install_service_check: + pre: + script_path: hooks/post_install_service_check-prehook.sh + +content: + docker: + - path: docker + + s3: + - path: data/s3/dummy_upload_1.txt + bucket: dummy-bucket + key: dummy-key + + - path: data/s3/dummy_upload_2.txt + bucket: dummy-bucket + key: dummy-key + + helm: + - path: helm + + loftsman: + - path: manifests/cos-services.yaml + use_manifestgen: true + deploy: true + + nexus_blob_stores: + yaml_path: 'data/nexus-blobstores.yaml' + + nexus_repositories: + yaml_path: 'data/np/nexus-repositories.yaml' + + rpms: + - path: data/rpms/rpm_dummy_1 + repository_name: cos-2.5.97-sle-15sp4 + repository_type: raw + + - path: data/rpms/rpm_dummy_2 + repository_name: cos-2.5.97-net-sle-15sp4-shs-2.0 + repository_type: raw + + - path: data/rpms/rpm_dummy_3 + repository_name: cos-2.5.97-sle-15sp4-compute + repository_type: raw + + vcs: + path: data/vcs + + ims: + content_dirs: + - ims/recipes/x86_64 diff --git a/src/tests/data/nexus-blobstores.yaml b/src/tests/data/nexus-blobstores.yaml new file mode 100755 index 00000000..14fc0615 --- /dev/null +++ b/src/tests/data/nexus-blobstores.yaml @@ -0,0 +1,4 @@ +--- +name: cos +path: /nexus-data/blobs/cos +softQuota: null diff --git a/src/tests/data/nexus-repositories.yaml_bk b/src/tests/data/nexus-repositories.yaml_bk new file mode 100755 index 00000000..14fc0615 --- /dev/null +++ b/src/tests/data/nexus-repositories.yaml_bk @@ -0,0 +1,4 @@ +--- +name: cos +path: /nexus-data/blobs/cos +softQuota: null diff --git a/src/tests/data/np/nexus-repositories-dummy.yaml b/src/tests/data/np/nexus-repositories-dummy.yaml new file mode 100755 index 00000000..e69de29b diff --git a/src/tests/data/np/nexus-repositories.yaml b/src/tests/data/np/nexus-repositories.yaml new file mode 100755 index 00000000..bf54c4ea --- /dev/null +++ b/src/tests/data/np/nexus-repositories.yaml @@ -0,0 +1,84 @@ +--- +cleanup: null +format: raw +name: cos-2.5.97-sle-15sp4 +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false + writePolicy: ALLOW_ONCE +type: hosted +--- +format: raw +group: + memberNames: + - cos-2.5.97-sle-15sp4 +name: cos-2.5-sle-15sp4 +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false +type: group +--- +cleanup: null +format: raw +name: cos-2.5.97-net-sle-15sp4-shs-2.0 +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false + writePolicy: ALLOW_ONCE +type: hosted +--- +format: raw +group: + memberNames: + - cos-2.5.97-net-sle-15sp4-shs-2.0 +name: cos-2.5-net-sle-15sp4-shs-2.0 +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false +type: group +--- +cleanup: null +format: raw +name: cos-2.5.97-sle-15sp4-compute +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false + writePolicy: ALLOW_ONCE +type: hosted +--- +format: raw +group: + memberNames: + - cos-2.5.97-sle-15sp4-compute +name: cos-2.5-sle-15sp4-compute +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false +type: group +--- +cleanup: null +format: raw +name: cos-2.5.97-net-sle-15sp4-compute-shs-2.0 +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false + writePolicy: ALLOW_ONCE +type: hosted +--- +format: raw +group: + memberNames: + - cos-2.5.97-net-sle-15sp4-compute-shs-2.0 +name: cos-2.5-net-sle-15sp4-compute-shs-2.0 +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false +type: group diff --git a/src/tests/data/np/nexus-repositories_cos.yaml b/src/tests/data/np/nexus-repositories_cos.yaml new file mode 100755 index 00000000..bf54c4ea --- /dev/null +++ b/src/tests/data/np/nexus-repositories_cos.yaml @@ -0,0 +1,84 @@ +--- +cleanup: null +format: raw +name: cos-2.5.97-sle-15sp4 +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false + writePolicy: ALLOW_ONCE +type: hosted +--- +format: raw +group: + memberNames: + - cos-2.5.97-sle-15sp4 +name: cos-2.5-sle-15sp4 +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false +type: group +--- +cleanup: null +format: raw +name: cos-2.5.97-net-sle-15sp4-shs-2.0 +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false + writePolicy: ALLOW_ONCE +type: hosted +--- +format: raw +group: + memberNames: + - cos-2.5.97-net-sle-15sp4-shs-2.0 +name: cos-2.5-net-sle-15sp4-shs-2.0 +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false +type: group +--- +cleanup: null +format: raw +name: cos-2.5.97-sle-15sp4-compute +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false + writePolicy: ALLOW_ONCE +type: hosted +--- +format: raw +group: + memberNames: + - cos-2.5.97-sle-15sp4-compute +name: cos-2.5-sle-15sp4-compute +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false +type: group +--- +cleanup: null +format: raw +name: cos-2.5.97-net-sle-15sp4-compute-shs-2.0 +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false + writePolicy: ALLOW_ONCE +type: hosted +--- +format: raw +group: + memberNames: + - cos-2.5.97-net-sle-15sp4-compute-shs-2.0 +name: cos-2.5-net-sle-15sp4-compute-shs-2.0 +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false +type: group diff --git a/src/tests/data/np/nexus-repositories_csm.yaml b/src/tests/data/np/nexus-repositories_csm.yaml new file mode 100755 index 00000000..bccf3a0c --- /dev/null +++ b/src/tests/data/np/nexus-repositories_csm.yaml @@ -0,0 +1,151 @@ +--- +cleanup: null +docker: + forceBasicAuth: true + httpPort: 5003 + httpsPort: null + v1Enabled: false +format: docker +name: registry +online: true +storage: + blobStoreName: csm + strictContentTypeValidation: false + writePolicy: ALLOW +type: hosted +--- +cleanup: null +format: helm +name: charts +online: true +storage: + blobStoreName: csm + strictContentTypeValidation: false + writePolicy: ALLOW_ONCE +type: hosted +--- +cleanup: null +format: raw +name: csm-1.4.0-beta.43-sle-15sp2 +online: true +storage: + blobStoreName: csm + strictContentTypeValidation: false + writePolicy: ALLOW_ONCE +type: hosted +--- +format: raw +group: + memberNames: + - csm-1.4.0-beta.43-sle-15sp2 +name: csm-sle-15sp2 +online: true +storage: + blobStoreName: csm + strictContentTypeValidation: false +type: group +--- +cleanup: null +format: raw +name: csm-1.4.0-beta.43-sle-15sp2-compute +online: true +storage: + blobStoreName: csm + strictContentTypeValidation: false + writePolicy: ALLOW_ONCE +type: hosted +--- +format: raw +group: + memberNames: + - csm-1.4.0-beta.43-sle-15sp2-compute +name: csm-sle-15sp2-compute +online: true +storage: + blobStoreName: csm + strictContentTypeValidation: false +type: group +--- +cleanup: null +format: raw +name: csm-1.4.0-beta.43-sle-15sp3 +online: true +storage: + blobStoreName: csm + strictContentTypeValidation: false + writePolicy: ALLOW_ONCE +type: hosted +--- +format: raw +group: + memberNames: + - csm-1.4.0-beta.43-sle-15sp3 +name: csm-sle-15sp3 +online: true +storage: + blobStoreName: csm + strictContentTypeValidation: false +type: group +--- +cleanup: null +format: raw +name: csm-1.4.0-beta.43-sle-15sp3-compute +online: true +storage: + blobStoreName: csm + strictContentTypeValidation: false + writePolicy: ALLOW_ONCE +type: hosted +--- +format: raw +group: + memberNames: + - csm-1.4.0-beta.43-sle-15sp3-compute +name: csm-sle-15sp3-compute +online: true +storage: + blobStoreName: csm + strictContentTypeValidation: false +type: group +--- +cleanup: null +format: raw +name: csm-1.4.0-beta.43-sle-15sp4 +online: true +storage: + blobStoreName: csm + strictContentTypeValidation: false + writePolicy: ALLOW_ONCE +type: hosted +--- +format: raw +group: + memberNames: + - csm-1.4.0-beta.43-sle-15sp4 +name: csm-sle-15sp4 +online: true +storage: + blobStoreName: csm + strictContentTypeValidation: false +type: group +--- +cleanup: null +format: raw +name: csm-1.4.0-beta.43-sle-15sp4-compute +online: true +storage: + blobStoreName: csm + strictContentTypeValidation: false + writePolicy: ALLOW_ONCE +type: hosted +--- +format: raw +group: + memberNames: + - csm-1.4.0-beta.43-sle-15sp4-compute +name: csm-sle-15sp4-compute +online: true +storage: + blobStoreName: csm + strictContentTypeValidation: false +type: group diff --git a/src/tests/data/rpms/rpm_dummy_1/1 b/src/tests/data/rpms/rpm_dummy_1/1 new file mode 100755 index 00000000..e69de29b diff --git a/src/tests/data/rpms/rpm_dummy_2/2 b/src/tests/data/rpms/rpm_dummy_2/2 new file mode 100755 index 00000000..e69de29b diff --git a/src/tests/data/rpms/rpm_dummy_3/3 b/src/tests/data/rpms/rpm_dummy_3/3 new file mode 100755 index 00000000..e69de29b diff --git a/src/tests/data/s3/dummy_upload_1.txt b/src/tests/data/s3/dummy_upload_1.txt new file mode 100755 index 00000000..e69de29b diff --git a/src/tests/data/s3/dummy_upload_2.txt b/src/tests/data/s3/dummy_upload_2.txt new file mode 100755 index 00000000..e69de29b diff --git a/src/tests/data/vcs/dummy_vcs.txt b/src/tests/data/vcs/dummy_vcs.txt new file mode 100755 index 00000000..e69de29b diff --git a/src/tests/manifest_validate_test.go b/src/tests/manifest_validate_test.go new file mode 100755 index 00000000..36aaf9c6 --- /dev/null +++ b/src/tests/manifest_validate_test.go @@ -0,0 +1,620 @@ +package tests + +import ( + iuf "iuf-internal/src/api/models/iuf" + mdv "iuf-internal/src/api/models/iuf/manifestDataValidation" + "testing" + + "sigs.k8s.io/yaml" +) + +// Unit testcase for Basic sanity of manifest validate +func TestValidateManifestSaninty(t *testing.T) { + file := "data/manifests/iuf-product-manifest.yaml" + err := iuf.ValidateFile(file) + if err != nil { + t.Fatal("Issue seen in manifest file:", file, "error:", err) + } +} + +// Unit testcase for checking invalid 3s path +func TestValidateManifestInvalidS3(t *testing.T) { + + data := []byte(` +content: + docker: + - path: docker + s3: + - path: data/s3/dummy_upload_1.txt + bucket: dummy-bucket + key: dummy-key + - path: not/exists/path/dummy_upload_2.txt + bucket: dummy-bucket + key: dummy-key +`) + var dataObject map[string]interface{} + inputErr := yaml.Unmarshal(data, &dataObject) + + if inputErr != nil { + t.Fatal("Test setup has issues, error details:", inputErr) + } + + err := mdv.Validate(dataObject) + + if err == nil { + t.Fatal("s3 path validation logic is not working properly") + } +} + +// Unit testcase for checking invalid nexus blob path +func TestValidateManifestInvalidNexusBlob(t *testing.T) { + + data := []byte(` +content: + nexus_blob_stores: + yaml_path: 'do/not/exist/nexus-blobstores.yaml' +`) + var dataObject map[string]interface{} + inputErr := yaml.Unmarshal(data, &dataObject) + + if inputErr != nil { + t.Fatal("Test setup has issues, error details:", inputErr) + } + + err := mdv.Validate(dataObject) + + if err == nil { + t.Fatal("nexus blob validation logic is not working properly") + } +} + +// Unit testcase for checking empty vcs path +func TestValidateManifestEmptyVcs(t *testing.T) { + + data := []byte(` +content: + vcs: + path: 'data/vcs_empty' +`) + var dataObject map[string]interface{} + inputErr := yaml.Unmarshal(data, &dataObject) + + if inputErr != nil { + t.Fatal("Test setup has issues, error details:", inputErr) + } + + err := mdv.Validate(dataObject) + + if err == nil { + t.Fatal("vcs validation logic is not working properly") + } +} + +// Unit testcase for checking invalid vcs path +func TestValidateManifestInvalidVcs(t *testing.T) { + data := []byte(` +content: + vcs: + path: 'data/do/not/exist' +`) + var dataObject map[string]interface{} + inputErr := yaml.Unmarshal(data, &dataObject) + + if inputErr != nil { + t.Fatal("Test setup has issues, error details:", inputErr) + } + + err := mdv.Validate(dataObject) + + if err == nil { + t.Fatal("vcs validation logic is not working properly") + } +} + +/* + Test cases for rpm validations +*/ +// checking empty rpm path +func TestValidateManifestEmptyRpm(t *testing.T) { + data := []byte(` +content: + nexus_repositories: + yaml_path: 'data/np/nexus-repositories.yaml' + rpms: + - path: data/rpms/rpm_dummy_1 + repository_name: cos-2.5.97-sle-15sp4 + repository_type: raw + + - path: data/rpms/rpm_dummy_2 + repository_name: cos-2.5.97-net-sle-15sp4-shs-2.0 + repository_type: raw + + - path: data/rpms/rpm_dummy_3 + repository_name: cos-2.5.97-sle-15sp4-compute + repository_type: raw + + - path: data/rpms/empty_rpm + repository_name: cos-2.5.97-sle-15sp4-compute + repository_type: raw +`) + var dataObject map[string]interface{} + inputErr := yaml.Unmarshal(data, &dataObject) + + if inputErr != nil { + t.Fatal("Test setup has issues, error details:", inputErr) + } + + err := mdv.Validate(dataObject) + + if err == nil { + t.Fatal("rpm validation logic is not working properly") + } +} + +// checking invalid rpm path +func TestValidateManifestInvalidRpm(t *testing.T) { + data := []byte(` +content: + nexus_repositories: + yaml_path: 'data/np/nexus-repositories.yaml' + rpms: + - path: data/rpms/rpm_dummy_1 + repository_name: cos-2.5.97-sle-15sp4 + repository_type: raw + + - path: data/rpms/rpm_dummy_2 + repository_name: cos-2.5.97-net-sle-15sp4-shs-2.0 + repository_type: raw + + - path: data/rpms/rpm_dummy_3 + repository_name: cos-2.5.97-sle-15sp4-compute + repository_type: raw + + - path: data/rpms/wrong/path + repository_name: cos-2.5.97-sle-15sp4-compute + repository_type: raw +`) + var dataObject map[string]interface{} + inputErr := yaml.Unmarshal(data, &dataObject) + + if inputErr != nil { + t.Fatal("Test setup has issues, error details:", inputErr) + } + + err := mdv.Validate(dataObject) + + if err == nil { + t.Fatal("rpm validation logic is not working properly") + } +} + +// Handling of rpm not present in nexus repo +func TestValidateHostedLogic(t *testing.T) { + data := []byte(` +content: + nexus_repositories: + yaml_path: 'data/np/nexus-repositories.yaml' + rpms: + - path: data/rpms/rpm_dummy_1 + repository_name: cos-2.5.97-sle-15sp4 + repository_type: raw + + - path: data/rpms/rpm_dummy_2 + repository_name: repo_non_existent + repository_type: raw + + - path: data/rpms/rpm_dummy_3 + repository_name: cos-2.5.97-sle-15sp4-compute + repository_type: raw +`) + var dataObject map[string]interface{} + inputErr := yaml.Unmarshal(data, &dataObject) + + if inputErr != nil { + t.Fatal("Test setup has issues, error details:", inputErr) + } + + err := mdv.Validate(dataObject) + + if err == nil { + t.Fatal("rpm validation logic is not working properly") + } +} + +// Handling of group rpm in manifest file +func TestValidateGroupRpmExpectionLogic(t *testing.T) { + data := []byte(` +content: + nexus_repositories: + yaml_path: 'data/np/nexus-repositories.yaml' + rpms: + - path: data/rpms/rpm_dummy_1 + repository_name: cos-2.5.97-sle-15sp4 + repository_type: raw + + - path: data/rpms/rpm_dummy_2 + repository_name: cos-2.5.97-net-sle-15sp4-shs-2.0 + repository_type: raw + + - path: data/rpms/rpm_dummy_3 + repository_name: cos-2.5-net-sle-15sp4-shs-2.0 + repository_type: raw +`) // last path is a group repo, defined in nexus-repositories.yaml + + var dataObject map[string]interface{} + inputErr := yaml.Unmarshal(data, &dataObject) + + if inputErr != nil { + t.Fatal("Test setup has issues, error details:", inputErr) + } + + err := mdv.Validate(dataObject) + + if err == nil { + t.Fatal("rpm validation logic is not working properly", err) + } +} + +/* + Test cases for Nexus repo validations +*/ +// checking invalid nexus repo path +func TestValidateManifestInvalidNexusRepo(t *testing.T) { + data := []byte(` +content: + nexus_repositories: + yaml_path: 'data/np/dummy/filenotpresent.yaml' +`) + var dataObject map[string]interface{} + inputErr := yaml.Unmarshal(data, &dataObject) + + if inputErr != nil { + t.Fatal("Test setup has issues, error details:", inputErr) + } + + err := mdv.Validate(dataObject) + + if err == nil { + t.Fatal("nexus repo validation logic is not working properly") + } +} + +// Handling of missing hosted +func TestValidateNexusRepoMissingHosted(t *testing.T) { + data := []byte(` +content: + nexus_repositories: + yaml_path: 'data/np/nexus-repositories-dummy.yaml' +`) + var dataObject map[string]interface{} + inputErr := yaml.Unmarshal(data, &dataObject) + + if inputErr != nil { + t.Fatal("Test setup has issues, error details:", inputErr) + } + + nexusCotent := []byte(` +--- +cleanup: null +format: raw +name: cos-2.5.97-sle-15sp4 +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false + writePolicy: ALLOW_ONCE +type: hosted +--- +format: raw +group: + memberNames: + - cos-2.5.97-sle-15sp4 +name: cos-2.5-sle-15sp4 +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false +type: group +--- +format: raw +group: + memberNames: + - cos-2.5.97-net-sle-15sp4-shs-2.0 +name: cos-2.5-net-sle-15sp4-shs-2.0 +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false +type: group +`) + //mocking the file IO + mdv.FileReader = func(filePath string) ([]byte, error) { + return nexusCotent, nil + } + err := mdv.Validate(dataObject) + + if err == nil { + t.Fatal("nexus repo validation logic is not working properly", err) + } +} + +// Handling of hosted defined below group +func TestValidateNexusRepoMisplacedHosted(t *testing.T) { + data := []byte(` +content: + nexus_repositories: + yaml_path: 'data/np/nexus-repositories-dummy.yaml' +`) + var dataObject map[string]interface{} + inputErr := yaml.Unmarshal(data, &dataObject) + + if inputErr != nil { + t.Fatal("Test setup has issues, error details:", inputErr) + } + + nexusCotent := []byte(` +--- +cleanup: null +format: raw +name: cos-2.5.97-sle-15sp4 +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false + writePolicy: ALLOW_ONCE +type: hosted +--- +format: raw +group: + memberNames: + - cos-2.5.97-sle-15sp4 +name: cos-2.5-sle-15sp4 +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false +type: group +--- +format: raw +group: + memberNames: + - cos-2.5.97-net-sle-15sp4-shs-2.0 +name: cos-2.5-net-sle-15sp4-shs-2.0 +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false +type: group +--- +cleanup: null +format: raw +name: cos-2.5.97-net-sle-15sp4-shs-2.0 +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false + writePolicy: ALLOW_ONCE +type: hosted +`) + //mocking the file IO + mdv.FileReader = func(filePath string) ([]byte, error) { + return nexusCotent, nil + } + err := mdv.Validate(dataObject) + + if err == nil { + t.Fatal("nexus repo validation logic is not working properly") + } +} + +// Handling of format skipping logic +func TestValidateNexusRepoSkipFormat(t *testing.T) { + data := []byte(` +content: + nexus_repositories: + yaml_path: 'data/np/nexus-repositories-dummy.yaml' +`) + var dataObject map[string]interface{} + inputErr := yaml.Unmarshal(data, &dataObject) + + if inputErr != nil { + t.Fatal("Test setup has issues, error details:", inputErr) + } + + nexusCotent := []byte(` +--- +cleanup: null +format: raw +name: cos-2.5.97-sle-15sp4 +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false + writePolicy: ALLOW_ONCE +type: hosted +--- +format: raw +group: + memberNames: + - cos-2.5.97-sle-15sp4 +name: cos-2.5-sle-15sp4 +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false +type: group +--- +cleanup: null +format: helm +name: charts +online: true +storage: + blobStoreName: csm + strictContentTypeValidation: false + writePolicy: ALLOW_ONCE +type: hosted +--- +cleanup: null +format: raw +name: cos-2.5.97-net-sle-15sp4-shs-2.0 +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false + writePolicy: ALLOW_ONCE +type: hosted +--- +cleanup: null +docker: + forceBasicAuth: true + httpPort: 5003 + httpsPort: null + v1Enabled: false +format: docker +name: registry +online: true +storage: + blobStoreName: csm + strictContentTypeValidation: false + writePolicy: ALLOW +type: hosted +--- +format: raw +group: + memberNames: + - cos-2.5.97-net-sle-15sp4-shs-2.0 +name: cos-2.5-net-sle-15sp4-shs-2.0 +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false +type: group +`) + //mocking the file IO + mdv.FileReader = func(filePath string) ([]byte, error) { + return nexusCotent, nil + } + err := mdv.Validate(dataObject) + + if err != nil { + t.Fatal("nexus repo validation logic is not working properly") + } +} + +// Handling of multiple repo member logic logic -> + ve test case +func TestValidateNexusRepoMultiMemberPos(t *testing.T) { + data := []byte(` +content: + nexus_repositories: + yaml_path: 'data/np/nexus-repositories-dummy.yaml' +`) + var dataObject map[string]interface{} + inputErr := yaml.Unmarshal(data, &dataObject) + + if inputErr != nil { + t.Fatal("Test setup has issues, error details:", inputErr) + } + + nexusCotent := []byte(` +--- +cleanup: null +format: raw +name: cos-2.5.98-sle-15sp4 +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false + writePolicy: ALLOW_ONCE +type: hosted +--- +cleanup: null +format: raw +name: cos-2.5.97-sle-15sp4 +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false + writePolicy: ALLOW_ONCE +type: hosted +--- +format: raw +group: + memberNames: + - cos-2.5.97-sle-15sp4 + - cos-2.5.98-sle-15sp4 +name: cos-2.5-sle-15sp4 +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false +type: group +`) + //mocking the file IO + mdv.FileReader = func(filePath string) ([]byte, error) { + return nexusCotent, nil + } + err := mdv.Validate(dataObject) + + if err != nil { + t.Fatal("nexus repo validation logic is not working properly") + } +} + +// Handling of multiple repo member logic logic -> - ve test case +func TestValidateNexusRepoMultiMemberNeg(t *testing.T) { + data := []byte(` +content: + nexus_repositories: + yaml_path: 'data/np/nexus-repositories-dummy.yaml' +`) + var dataObject map[string]interface{} + inputErr := yaml.Unmarshal(data, &dataObject) + + if inputErr != nil { + t.Fatal("Test setup has issues, error details:", inputErr) + } + + nexusCotent := []byte(` +--- +cleanup: null +format: raw +name: cos-2.5.98-sle-15sp4 +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false + writePolicy: ALLOW_ONCE +type: hosted +--- +format: raw +group: + memberNames: + - cos-2.5.97-sle-15sp4 + - cos-2.5.98-sle-15sp4 +name: cos-2.5-sle-15sp4 +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false +type: group +--- +cleanup: null +format: raw +name: cos-2.5.97-sle-15sp4 +online: true +storage: + blobStoreName: cos + strictContentTypeValidation: false + writePolicy: ALLOW_ONCE +type: hosted +`) + //mocking the file IO + mdv.FileReader = func(filePath string) ([]byte, error) { + return nexusCotent, nil + } + err := mdv.Validate(dataObject) + + if err == nil { + t.Fatal("nexus repo validation logic is not working properly") + } +}