diff --git a/Dockerfile b/Dockerfile index bed351f..3634abd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -180,8 +180,16 @@ RUN go install github.com/onsi/ginkgo/v2/ginkgo@v2.20.2 && mv /go/bin/ginkgo /go go install k8s.io/code-generator/cmd/defaulter-gen@${K8S_LIBS_VERSION} && \ go install k8s.io/code-generator/cmd/informer-gen@${K8S_LIBS_VERSION} && \ go install k8s.io/code-generator/cmd/lister-gen@${K8S_LIBS_VERSION} && \ - go install k8s.io/code-generator/cmd/openapi-gen@${K8S_LIBS_VERSION} && \ - go clean -modcache && go clean -cache + go install k8s.io/code-generator/cmd/openapi-gen@${K8S_LIBS_VERSION} + +# Build and install semvalidator +COPY semvalidator/go.mod semvalidator/go.sum semvalidator/main.go /tmp/semvalidator/ + +RUN cd /tmp/semvalidator && CGO_ENABLED=0 go build -o /usr/local/bin/semvalidator -v -buildvcs=false -ldflags "-s -w" main.go \ + && rm -fr /tmp/semvalidator + +# Cleanup module cache after we have built and installed all Go utilities +RUN go clean -modcache && go clean -cache # Ensure that everything under the GOPATH is writable by everyone RUN chmod -R 777 $GOPATH diff --git a/semvalidator/README.md b/semvalidator/README.md new file mode 100644 index 0000000..d180c93 --- /dev/null +++ b/semvalidator/README.md @@ -0,0 +1,62 @@ +# semvalidator + +This allows running validations on semaphore pipeline files. + +## Usage + +The help give all the required options. + +```sh +$ docker run --rm calico/go-build:${GOBUILD_VERSION} semvalidator --help +Usage of semvalidator: + -debug + enable debug logging + -dirs string + comma separated list of directories to search for Semaphore pipeline files + -files string + comma separated list of Semaphore pipeline files + -org string + Semaphore organization + -org-url string + Semaphore organization URL + -skip-dirs string + comma separated list of directories to skip when searching for Semaphore pipeline files + -token string + Semaphore API token +``` + +You can specify dirs that contain semaphore pipeline files (using `-dirs`) +and/or files that are semphore pipeline files (using `-files`). +If using `-dirs`, this tool assumes all YAML files in the folder recursively are Semaphore pipeline files. +To skip specific folders in the directories specified, use `-skip-dirs` + +Set the organization using either `-org` or `-org-url` as it is needed to determine +where to send the validation requests. + +The token needs to be a valid [Semaphore API token](https://docs.semaphoreci.com/reference/api-v1alpha/#authentication). +It will try to use the `SEMAPHORE_API_TOKEN` environment variable if flag is empty. + +### Examples + +Using `latest` as `${GOBUILD_VERSION}` + +1. Give a project `` with semaphore files in `/.semaphore` directory, + below is how to validate the files in that directory. + + ```sh + docker run --rm -v ::ro calico/go-build:latest semvalidator -dirs /.semaphore -org -token + ``` + +1. Give a project `` with semaphore files in `/.semaphore` directory, + below is how to validate the files in that directory using `-org-url` flag with `$SEMAPHORE_ORGANIZATION_URL` environment variable. + + ```sh + docker run --rm -v ::ro calico/go-build:latest semvalidator -dirs /.semaphore -org-url ${SEMAPHORE_ORGANIZATION_URL} -token + ``` + +1. Give a project `` with semaphore file in `/.semaphore/semaphore.yml` directory, + below is how to validate the files in that directory. + + ```sh + docker run --rm -v ::ro calico/go-build:latest semvalidator -files /.semaphore/semaphore.yml -org -token + ``` diff --git a/semvalidator/go.mod b/semvalidator/go.mod new file mode 100644 index 0000000..1456b70 --- /dev/null +++ b/semvalidator/go.mod @@ -0,0 +1,7 @@ +module github.com/projectcalico/semvalidator + +go 1.23.2 + +require github.com/sirupsen/logrus v1.9.3 + +require golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect diff --git a/semvalidator/go.sum b/semvalidator/go.sum new file mode 100644 index 0000000..21f9bfb --- /dev/null +++ b/semvalidator/go.sum @@ -0,0 +1,15 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/semvalidator/main.go b/semvalidator/main.go new file mode 100644 index 0000000..daff3f3 --- /dev/null +++ b/semvalidator/main.go @@ -0,0 +1,181 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/sirupsen/logrus" +) + +var ( + dir string + skipDir string + file string + org string + orgURL string + token string + debug bool +) + +func init() { + flag.StringVar(&dir, "dirs", "", "comma separated list of directories to search for Semaphore pipeline files") + flag.StringVar(&skipDir, "skip-dirs", "", "comma separated list of directories to skip when searching for Semaphore pipeline files") + flag.StringVar(&file, "files", "", "comma separated list of Semaphore pipeline files") + flag.StringVar(&org, "org", "", "Semaphore organization") + flag.StringVar(&orgURL, "org-url", "", "Semaphore organization URL") + flag.StringVar(&token, "token", "", "Semaphore API token") + flag.BoolVar(&debug, "debug", false, "enable debug logging") +} + +func inSkipDirs(path string, skipDirs []string) bool { + if len(skipDirs) == 0 { + return false + } + for _, skipDir := range skipDirs { + if strings.HasSuffix(path, skipDir) { + return true + } + } + return false +} + +func getPipelineYAMLFiles(dir string, skipDirs []string) ([]string, error) { + var files []string + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + // Skip the YAML .semaphore/semaphore.yml.d directory + // as it contains building blocks which are not full pipeline definitions + // The resulting pipeline will be validated as part of semaphore.yml and semaphore-scheduled-builds.yml + if info.IsDir() && !inSkipDirs(path, skipDirs) { + return filepath.SkipDir + } + if !info.IsDir() && (filepath.Ext(path) == ".yml" || filepath.Ext(path) == ".yaml") { + files = append(files, path) + } + return nil + }) + return files, err +} + +func validateYAML(file, baseURL, token string) error { + logrus.WithField("file", file).Info("validating YAML") + content, err := os.ReadFile(file) + if err != nil { + logrus.WithError(err).Error("failed to read file") + return err + } + payload := map[string]string{ + "yaml_definition": fmt.Sprintf("%v", string(content)), + } + data, err := json.Marshal(payload) + if err != nil { + logrus.WithError(err).Error("failed to marshal payload for yaml validation") + return err + } + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1alpha/yaml", baseURL), bytes.NewBuffer(data)) + if err != nil { + logrus.WithError(err).Error("failed to create request for yaml validation") + return err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + resp, err := http.DefaultClient.Do(req) + if err != nil { + logrus.WithError(err).Error("failed to make request for yaml validation") + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to validate YAML: %s", resp.Status) + } + result := map[string]interface{}{} + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + logrus.WithError(err).Error("failed to decode response for yaml validation") + return err + } + logrus.Debug(result["message"].(string)) + return nil +} + +func main() { + flag.Parse() + if debug { + logrus.SetLevel(logrus.DebugLevel) + } + // Validate flags + if orgURL == "" && org == "" { + logrus.Fatal("Either Semaphore organization URL or organization name is required, use the -org-url or -org flag to specify the organization") + } else if orgURL != "" && org != "" { + logrus.Fatal("Only one of Semaphore organization URL or organization name is required, use either the -org-url or -org flag to specify the organization") + } + if token == "" { + if os.Getenv("SEMAPHORE_API_TOKEN") == "" { + logrus.Fatal("Semaphore API token is required, use the -token flag to specify the token or set as environment variable SEMAPHORE_API_TOKEN") + } else { + token = os.Getenv("SEMAPHORE_API_TOKEN") + } + } + + // Get YAML files + var yamlFiles []string + if file != "" { + yamlFiles = strings.Split(file, ",") + } + if dir != "" { + semaphoreDirs := strings.Split(dir, ",") + logrus.WithField("semaphoreDirs", semaphoreDirs).Debug("looking for pipeline YAML files") + for _, semaphoreDir := range semaphoreDirs { + files, err := getPipelineYAMLFiles(semaphoreDir, strings.Split(skipDir, ",")) + if err != nil { + logrus.WithError(err).Errorf("failed to get YAML files in %s", semaphoreDir) + continue + } + yamlFiles = append(yamlFiles, files...) + } + } + if len(yamlFiles) == 0 { + logrus.Fatal("no YAML files found, use either -dirs or -files to specify the location of Semaphore pipeline files") + } + logrus.Debugf("will validate %d YAML pipeline file(s)", len(yamlFiles)) + var failedFiles []string + + // Send YAML files for validation + baseURL := orgURL + if org != "" { + baseURL = fmt.Sprintf("https://%s.semaphoreci.com", org) + } + for _, file := range yamlFiles { + err := validateYAML(file, baseURL, token) + if err != nil { + logrus.WithError(err).Error("invalid YAML definition") + failedFiles = append(failedFiles, file) + } + } + if len(failedFiles) > 0 { + logrus.Fatalf("failed to validate %d files", len(failedFiles)) + } else { + logrus.Info("all pipeline YAML files are valid") + } +}