From 2a2f5073076bc84546d151481d3bd44595f8fab3 Mon Sep 17 00:00:00 2001 From: tuti Date: Wed, 2 Oct 2024 14:27:26 -0700 Subject: [PATCH 1/2] add semaphore pipeline validator --- Dockerfile | 17 +++++ semvalidator/README.md | 51 +++++++++++++ semvalidator/go.mod | 7 ++ semvalidator/go.sum | 15 ++++ semvalidator/main.go | 164 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 254 insertions(+) create mode 100644 semvalidator/README.md create mode 100644 semvalidator/go.mod create mode 100644 semvalidator/go.sum create mode 100644 semvalidator/main.go diff --git a/Dockerfile b/Dockerfile index bed351f..55a4791 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,20 @@ FROM calico/bpftool:v7.4.0 as bpftool FROM --platform=amd64 calico/qemu-user-static:latest as qemu +FROM --platform=amd64 golang:latest AS sembuilder + +ARG TARGETARCH + +WORKDIR /semhome + +COPY semvalidator/go.mod semvalidator/go.sum ./ + +RUN go mod download + +COPY semvalidator/main.go . + +RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -o semvalidator . + FROM registry.access.redhat.com/ubi8/ubi:latest as ubi ARG TARGETARCH @@ -195,6 +209,9 @@ COPY ssh_known_hosts /etc/ssh/ssh_known_hosts # Add bpftool for Felix UT/FV. COPY --from=bpftool /bpftool /usr/bin +# Add semvalidator +COPY --from=sembuilder /semhome/semvalidator /usr/bin/ + COPY entrypoint.sh /usr/local/bin/entrypoint.sh # Squash into a single layer diff --git a/semvalidator/README.md b/semvalidator/README.md new file mode 100644 index 0000000..0994206 --- /dev/null +++ b/semvalidator/README.md @@ -0,0 +1,51 @@ +# semvalidator + +This allows running validations on semaphore pipeline files. + +## Usage + +The help give all the required options. + +```sh +$ docker run --rm calico/go-build:${GOBUILD_VERSION} semvalidator --help +Usage of semvalidator: + -debug + enable debug logging + -dirs string + comma separated list of directories to search for Semaphore pipeline files + -files string + comma separated list of Semaphore pipeline files + -org string + Semaphore organization + -skip-dirs string + comma separated list of directories to skip when searching for Semaphore pipeline files + -token string + Semaphore API token +``` + +You can specify dirs that contain semaphore pipeline files (using `-dirs`) +and/or files that are semphore pipeline files (using `-files`). + +The organization is need to determine the Semaphore instance. +It will also use the value of `SEMAPHORE_ORGANIZATION` environment variable if flag is empty. + +The token needs to be a valid [Semaphore API token](https://docs.semaphoreci.com/reference/api-v1alpha/#authentication). +It will try to use the `SEMAPHORE_API_TOKEN` environment variable if flag is empty. + +### Examples + +Using `latest` as `${GOBUILD_VERSION}` + +1. Give a project `` with semaphore files in `/.semaphore` directory, + below is how to validate the files in that directory. + + ```sh + docker run --rm -v ::ro calico/go-build:latest semvalidator -dirs /.semaphore -org -token + ``` + +1. Give a project `` with semaphore file in `/.semaphore/semaphore.yml` directory, + below is how to validate the files in that directory. + + ```sh + docker run --rm -v ::ro calico/go-build:latest semvalidator -files /.semaphore/semaphore.yml -org -token + ``` diff --git a/semvalidator/go.mod b/semvalidator/go.mod new file mode 100644 index 0000000..1456b70 --- /dev/null +++ b/semvalidator/go.mod @@ -0,0 +1,7 @@ +module github.com/projectcalico/semvalidator + +go 1.23.2 + +require github.com/sirupsen/logrus v1.9.3 + +require golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect diff --git a/semvalidator/go.sum b/semvalidator/go.sum new file mode 100644 index 0000000..21f9bfb --- /dev/null +++ b/semvalidator/go.sum @@ -0,0 +1,15 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/semvalidator/main.go b/semvalidator/main.go new file mode 100644 index 0000000..b118a96 --- /dev/null +++ b/semvalidator/main.go @@ -0,0 +1,164 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/sirupsen/logrus" +) + +var ( + dir string + skipDir string + file string + org string + token string + debug bool +) + +func init() { + flag.StringVar(&dir, "dirs", "", "comma separated list of directories to search for Semaphore pipeline files") + flag.StringVar(&skipDir, "skip-dirs", "", "comma separated list of directories to skip when searching for Semaphore pipeline files") + flag.StringVar(&file, "files", "", "comma separated list of Semaphore pipeline files") + flag.StringVar(&org, "org", os.Getenv("SEMAPHORE_ORGANIZATION"), "Semaphore organization") + flag.StringVar(&token, "token", "", "Semaphore API token") + flag.BoolVar(&debug, "debug", false, "enable debug logging") +} + +func inSkipDirs(path string, skipDirs []string) bool { + for _, skipDir := range skipDirs { + if strings.HasSuffix(path, skipDir) { + return true + } + } + return false +} + +func getPipelineYAMLFiles(dir string, skipDirs []string) ([]string, error) { + var files []string + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + // Skip the YAML .semaphore/semaphore.yml.d directory + // as it contains building blocks which are not full pipeline definitions + // The resulting pipeline will be validated as part of semaphore.yml and semaphore-scheduled-builds.yml + if info.IsDir() && !inSkipDirs(path, skipDirs) { + return filepath.SkipDir + } + if !info.IsDir() && (filepath.Ext(path) == ".yml" || filepath.Ext(path) == ".yaml") { + files = append(files, path) + } + return nil + }) + return files, err +} + +func validateYAML(file, org, token string) error { + logrus.WithField("file", file).Info("validating YAML") + content, err := os.ReadFile(file) + if err != nil { + logrus.WithError(err).Error("failed to read file") + return err + } + payload := map[string]string{ + "yaml_definition": fmt.Sprintf("%v", string(content)), + } + data, err := json.Marshal(payload) + if err != nil { + logrus.WithError(err).Error("failed to marshal payload for yaml validation") + return err + } + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("https://%s.semaphoreci.com/api/v1alpha/yaml", org), bytes.NewBuffer(data)) + if err != nil { + logrus.WithError(err).Error("failed to create request for yaml validation") + return err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + resp, err := http.DefaultClient.Do(req) + if err != nil { + logrus.WithError(err).Error("failed to make request for yaml validation") + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to validate YAML: %s", resp.Status) + } + result := map[string]interface{}{} + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + logrus.WithError(err).Error("failed to decode response for yaml validation") + return err + } + logrus.Debug(result["message"].(string)) + return nil +} + +func main() { + flag.Parse() + if debug { + logrus.SetLevel(logrus.DebugLevel) + } + if org == "" { + logrus.Fatal("Semaphore organization is required, set the SEMAPHORE_ORGANIZATION environment variable or use the -org flag") + } + if token == "" { + if os.Getenv("SEMAPHORE_API_TOKEN") == "" { + logrus.Fatal("Semaphore API token is required, set the SEMAPHORE_API_TOKEN environment variable or use the -token flag") + } else { + token = os.Getenv("SEMAPHORE_API_TOKEN") + } + } + var yamlFiles []string + if file != "" { + yamlFiles = strings.Split(file, ",") + } + if dir != "" { + semaphoreDirs := strings.Split(dir, ",") + logrus.WithField("semaphoreDirs", semaphoreDirs).Debug("looking for pipeline YAML files") + for _, semaphoreDir := range semaphoreDirs { + files, err := getPipelineYAMLFiles(semaphoreDir, strings.Split(skipDir, ",")) + if err != nil { + logrus.WithError(err).Errorf("failed to get YAML files in %s", semaphoreDir) + continue + } + yamlFiles = append(yamlFiles, files...) + } + } + if len(yamlFiles) == 0 { + logrus.Error("no YAML files found") + return + } + logrus.Infof("will validate %d YAML pipeline file(s)", len(yamlFiles)) + var failedFiles []string + for _, file := range yamlFiles { + err := validateYAML(file, org, token) + if err != nil { + logrus.WithError(err).Error("invalid YAML definition") + failedFiles = append(failedFiles, file) + } + } + if len(failedFiles) > 0 { + logrus.Fatalf("failed to validate %d files", len(failedFiles)) + } +} From 8758469b9eebcffe54c0983ca5a37745118699b0 Mon Sep 17 00:00:00 2001 From: tuti Date: Wed, 2 Oct 2024 16:56:55 -0700 Subject: [PATCH 2/2] update validator - build semvalidator in existing golang dev environment - modify tool to allow specifying org URL - update tool documentation --- Dockerfile | 29 ++++++++++------------------- semvalidator/README.md | 15 +++++++++++++-- semvalidator/main.go | 37 +++++++++++++++++++++++++++---------- 3 files changed, 50 insertions(+), 31 deletions(-) diff --git a/Dockerfile b/Dockerfile index 55a4791..3634abd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,20 +4,6 @@ FROM calico/bpftool:v7.4.0 as bpftool FROM --platform=amd64 calico/qemu-user-static:latest as qemu -FROM --platform=amd64 golang:latest AS sembuilder - -ARG TARGETARCH - -WORKDIR /semhome - -COPY semvalidator/go.mod semvalidator/go.sum ./ - -RUN go mod download - -COPY semvalidator/main.go . - -RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -o semvalidator . - FROM registry.access.redhat.com/ubi8/ubi:latest as ubi ARG TARGETARCH @@ -194,8 +180,16 @@ RUN go install github.com/onsi/ginkgo/v2/ginkgo@v2.20.2 && mv /go/bin/ginkgo /go go install k8s.io/code-generator/cmd/defaulter-gen@${K8S_LIBS_VERSION} && \ go install k8s.io/code-generator/cmd/informer-gen@${K8S_LIBS_VERSION} && \ go install k8s.io/code-generator/cmd/lister-gen@${K8S_LIBS_VERSION} && \ - go install k8s.io/code-generator/cmd/openapi-gen@${K8S_LIBS_VERSION} && \ - go clean -modcache && go clean -cache + go install k8s.io/code-generator/cmd/openapi-gen@${K8S_LIBS_VERSION} + +# Build and install semvalidator +COPY semvalidator/go.mod semvalidator/go.sum semvalidator/main.go /tmp/semvalidator/ + +RUN cd /tmp/semvalidator && CGO_ENABLED=0 go build -o /usr/local/bin/semvalidator -v -buildvcs=false -ldflags "-s -w" main.go \ + && rm -fr /tmp/semvalidator + +# Cleanup module cache after we have built and installed all Go utilities +RUN go clean -modcache && go clean -cache # Ensure that everything under the GOPATH is writable by everyone RUN chmod -R 777 $GOPATH @@ -209,9 +203,6 @@ COPY ssh_known_hosts /etc/ssh/ssh_known_hosts # Add bpftool for Felix UT/FV. COPY --from=bpftool /bpftool /usr/bin -# Add semvalidator -COPY --from=sembuilder /semhome/semvalidator /usr/bin/ - COPY entrypoint.sh /usr/local/bin/entrypoint.sh # Squash into a single layer diff --git a/semvalidator/README.md b/semvalidator/README.md index 0994206..d180c93 100644 --- a/semvalidator/README.md +++ b/semvalidator/README.md @@ -17,6 +17,8 @@ Usage of semvalidator: comma separated list of Semaphore pipeline files -org string Semaphore organization + -org-url string + Semaphore organization URL -skip-dirs string comma separated list of directories to skip when searching for Semaphore pipeline files -token string @@ -25,9 +27,11 @@ Usage of semvalidator: You can specify dirs that contain semaphore pipeline files (using `-dirs`) and/or files that are semphore pipeline files (using `-files`). +If using `-dirs`, this tool assumes all YAML files in the folder recursively are Semaphore pipeline files. +To skip specific folders in the directories specified, use `-skip-dirs` -The organization is need to determine the Semaphore instance. -It will also use the value of `SEMAPHORE_ORGANIZATION` environment variable if flag is empty. +Set the organization using either `-org` or `-org-url` as it is needed to determine +where to send the validation requests. The token needs to be a valid [Semaphore API token](https://docs.semaphoreci.com/reference/api-v1alpha/#authentication). It will try to use the `SEMAPHORE_API_TOKEN` environment variable if flag is empty. @@ -43,6 +47,13 @@ Using `latest` as `${GOBUILD_VERSION}` docker run --rm -v ::ro calico/go-build:latest semvalidator -dirs /.semaphore -org -token ``` +1. Give a project `` with semaphore files in `/.semaphore` directory, + below is how to validate the files in that directory using `-org-url` flag with `$SEMAPHORE_ORGANIZATION_URL` environment variable. + + ```sh + docker run --rm -v ::ro calico/go-build:latest semvalidator -dirs /.semaphore -org-url ${SEMAPHORE_ORGANIZATION_URL} -token + ``` + 1. Give a project `` with semaphore file in `/.semaphore/semaphore.yml` directory, below is how to validate the files in that directory. diff --git a/semvalidator/main.go b/semvalidator/main.go index b118a96..daff3f3 100644 --- a/semvalidator/main.go +++ b/semvalidator/main.go @@ -32,6 +32,7 @@ var ( skipDir string file string org string + orgURL string token string debug bool ) @@ -40,12 +41,16 @@ func init() { flag.StringVar(&dir, "dirs", "", "comma separated list of directories to search for Semaphore pipeline files") flag.StringVar(&skipDir, "skip-dirs", "", "comma separated list of directories to skip when searching for Semaphore pipeline files") flag.StringVar(&file, "files", "", "comma separated list of Semaphore pipeline files") - flag.StringVar(&org, "org", os.Getenv("SEMAPHORE_ORGANIZATION"), "Semaphore organization") + flag.StringVar(&org, "org", "", "Semaphore organization") + flag.StringVar(&orgURL, "org-url", "", "Semaphore organization URL") flag.StringVar(&token, "token", "", "Semaphore API token") flag.BoolVar(&debug, "debug", false, "enable debug logging") } func inSkipDirs(path string, skipDirs []string) bool { + if len(skipDirs) == 0 { + return false + } for _, skipDir := range skipDirs { if strings.HasSuffix(path, skipDir) { return true @@ -74,7 +79,7 @@ func getPipelineYAMLFiles(dir string, skipDirs []string) ([]string, error) { return files, err } -func validateYAML(file, org, token string) error { +func validateYAML(file, baseURL, token string) error { logrus.WithField("file", file).Info("validating YAML") content, err := os.ReadFile(file) if err != nil { @@ -89,7 +94,7 @@ func validateYAML(file, org, token string) error { logrus.WithError(err).Error("failed to marshal payload for yaml validation") return err } - req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("https://%s.semaphoreci.com/api/v1alpha/yaml", org), bytes.NewBuffer(data)) + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1alpha/yaml", baseURL), bytes.NewBuffer(data)) if err != nil { logrus.WithError(err).Error("failed to create request for yaml validation") return err @@ -119,16 +124,21 @@ func main() { if debug { logrus.SetLevel(logrus.DebugLevel) } - if org == "" { - logrus.Fatal("Semaphore organization is required, set the SEMAPHORE_ORGANIZATION environment variable or use the -org flag") + // Validate flags + if orgURL == "" && org == "" { + logrus.Fatal("Either Semaphore organization URL or organization name is required, use the -org-url or -org flag to specify the organization") + } else if orgURL != "" && org != "" { + logrus.Fatal("Only one of Semaphore organization URL or organization name is required, use either the -org-url or -org flag to specify the organization") } if token == "" { if os.Getenv("SEMAPHORE_API_TOKEN") == "" { - logrus.Fatal("Semaphore API token is required, set the SEMAPHORE_API_TOKEN environment variable or use the -token flag") + logrus.Fatal("Semaphore API token is required, use the -token flag to specify the token or set as environment variable SEMAPHORE_API_TOKEN") } else { token = os.Getenv("SEMAPHORE_API_TOKEN") } } + + // Get YAML files var yamlFiles []string if file != "" { yamlFiles = strings.Split(file, ",") @@ -146,13 +156,18 @@ func main() { } } if len(yamlFiles) == 0 { - logrus.Error("no YAML files found") - return + logrus.Fatal("no YAML files found, use either -dirs or -files to specify the location of Semaphore pipeline files") } - logrus.Infof("will validate %d YAML pipeline file(s)", len(yamlFiles)) + logrus.Debugf("will validate %d YAML pipeline file(s)", len(yamlFiles)) var failedFiles []string + + // Send YAML files for validation + baseURL := orgURL + if org != "" { + baseURL = fmt.Sprintf("https://%s.semaphoreci.com", org) + } for _, file := range yamlFiles { - err := validateYAML(file, org, token) + err := validateYAML(file, baseURL, token) if err != nil { logrus.WithError(err).Error("invalid YAML definition") failedFiles = append(failedFiles, file) @@ -160,5 +175,7 @@ func main() { } if len(failedFiles) > 0 { logrus.Fatalf("failed to validate %d files", len(failedFiles)) + } else { + logrus.Info("all pipeline YAML files are valid") } }