From ebb8068bbdcd2c3404f2e466e17e8fa7f6d9e12e Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Fri, 13 Oct 2023 18:18:46 -0400 Subject: [PATCH] enhance(exec): templates and env support for exec pipeline (#492) * enhance(exec): templates and env support for exec pipeline * adjust validate local test and verify local templates are included in pipeline --- action/pipeline/exec.go | 12 +-- action/pipeline/validate.go | 16 ++++ action/pipeline/validate_test.go | 3 +- command/pipeline/exec.go | 155 +++++++++++++++++++++++++------ go.mod | 5 +- go.sum | 10 +- 6 files changed, 159 insertions(+), 42 deletions(-) diff --git a/action/pipeline/exec.go b/action/pipeline/exec.go index e36c5c99..bec8e921 100644 --- a/action/pipeline/exec.go +++ b/action/pipeline/exec.go @@ -78,19 +78,17 @@ func (c *Config) Exec(client compiler.Engine) error { WithComment(c.Comment). WithLocal(true). WithRepo(r). + WithLocalTemplates(c.TemplateFiles). Compile(path) if err != nil { return err } - // check if the local configuration is enabled - if c.Local { - // create current directory path for local mount - mount := fmt.Sprintf("%s:%s:rw", base, constants.WorkspaceDefault) + // create current directory path for local mount + mount := fmt.Sprintf("%s:%s:rw", base, constants.WorkspaceDefault) - // add the current directory path to volume mounts - c.Volumes = append(c.Volumes, mount) - } + // add the current directory path to volume mounts + c.Volumes = append(c.Volumes, mount) logrus.Tracef("creating runtime engine %s", constants.DriverDocker) diff --git a/action/pipeline/validate.go b/action/pipeline/validate.go index 7d2c4d6b..e2c6b8ce 100644 --- a/action/pipeline/validate.go +++ b/action/pipeline/validate.go @@ -110,6 +110,22 @@ func (c *Config) ValidateLocal(client compiler.Engine) error { return err } + // check to see if locally provided templates were included in compilation + for _, tmpl := range c.TemplateFiles { + parts := strings.Split(tmpl, ":") + included := false + + for _, pTmpl := range p.Templates { + if strings.EqualFold(parts[0], pTmpl.Name) { + included = true + } + } + + if !included { + return fmt.Errorf("local template with name %s not included in pipeline templates", parts[0]) + } + } + // output the message in stderr format // // https://pkg.go.dev/github.com/go-vela/cli/internal/output?tab=doc#Stderr diff --git a/action/pipeline/validate_test.go b/action/pipeline/validate_test.go index 0b363cdf..bbcbc484 100644 --- a/action/pipeline/validate_test.go +++ b/action/pipeline/validate_test.go @@ -253,8 +253,7 @@ func TestPipeline_Config_ValidateLocal(t *testing.T) { }, }, { - name: "pipeline with multiple template (local overrides) template mismatch", - failure: true, + name: "pipeline with multiple template (local overrides) only one template specified", config: &Config{ Action: "validate", File: "default_multi_template.yml", diff --git a/command/pipeline/exec.go b/command/pipeline/exec.go index 5f0b4650..aa2f2b1c 100644 --- a/command/pipeline/exec.go +++ b/command/pipeline/exec.go @@ -4,6 +4,8 @@ package pipeline import ( "fmt" + "os" + "strings" "github.com/go-vela/cli/action" "github.com/go-vela/cli/action/pipeline" @@ -11,7 +13,9 @@ import ( "github.com/go-vela/server/compiler/native" "github.com/go-vela/server/util" "github.com/go-vela/types/constants" + "github.com/sirupsen/logrus" + "github.com/joho/godotenv" "github.com/urfave/cli/v2" ) @@ -26,31 +30,31 @@ var CommandExec = &cli.Command{ // Build Flags &cli.StringFlag{ - EnvVars: []string{"VELA_BRANCH", "PIPELINE_BRANCH"}, + EnvVars: []string{"VELA_BRANCH", "PIPELINE_BRANCH", "VELA_BUILD_BRANCH"}, Name: "branch", Aliases: []string{"b"}, Usage: "provide the build branch for the pipeline", }, &cli.StringFlag{ - EnvVars: []string{"VELA_COMMENT", "PIPELINE_COMMENT"}, + EnvVars: []string{"VELA_COMMENT", "PIPELINE_COMMENT", "VELA_BUILD_COMMENT"}, Name: "comment", Aliases: []string{"c"}, Usage: "provide the build comment for the pipeline", }, &cli.StringFlag{ - EnvVars: []string{"VELA_EVENT", "PIPELINE_EVENT"}, + EnvVars: []string{"VELA_EVENT", "PIPELINE_EVENT", "VELA_BUILD_EVENT"}, Name: "event", Aliases: []string{"e"}, Usage: "provide the build event for the pipeline", }, &cli.StringFlag{ - EnvVars: []string{"VELA_TAG", "PIPELINE_TAG"}, + EnvVars: []string{"VELA_TAG", "PIPELINE_TAG", "VELA_BUILD_TAG"}, Name: "tag", Aliases: []string{"t"}, Usage: "provide the build tag for the pipeline", }, &cli.StringFlag{ - EnvVars: []string{"VELA_TARGET", "PIPELINE_TARGET"}, + EnvVars: []string{"VELA_TARGET", "PIPELINE_TARGET", "VELA_BUILD_TARGET"}, Name: "target", Usage: "provide the build target for the pipeline", }, @@ -93,12 +97,6 @@ var CommandExec = &cli.Command{ Aliases: []string{"v"}, Usage: "provide list of local volumes to mount", }, - &cli.IntFlag{ - EnvVars: []string{"VELA_MAX_TEMPLATE_DEPTH", "MAX_TEMPLATE_DEPTH"}, - Name: "max-template-depth", - Usage: "set the maximum depth for nested templates", - Value: 3, - }, // Repo Flags @@ -121,6 +119,60 @@ var CommandExec = &cli.Command{ Usage: "type of pipeline for the compiler to render", Value: constants.PipelineTypeYAML, }, + + // Compiler Template Flags + + &cli.StringFlag{ + EnvVars: []string{"VELA_COMPILER_GITHUB_TOKEN", "COMPILER_GITHUB_TOKEN"}, + Name: internal.FlagCompilerGitHubToken, + Aliases: []string{"ct"}, + Usage: "github compiler token", + }, + &cli.StringFlag{ + EnvVars: []string{"VELA_COMPILER_GITHUB_URL", "COMPILER_GITHUB_URL"}, + Name: internal.FlagCompilerGitHubURL, + Aliases: []string{"cgu"}, + Usage: "github url, used by compiler, for pulling registry templates", + }, + &cli.StringSliceFlag{ + EnvVars: []string{"VELA_TEMPLATE_FILE", "PIPELINE_TEMPLATE_FILE"}, + Name: "template-file", + Aliases: []string{"tf, tfs, template-files"}, + Usage: "enables using a local template file for expansion in the form :", + }, + &cli.IntFlag{ + EnvVars: []string{"VELA_MAX_TEMPLATE_DEPTH", "MAX_TEMPLATE_DEPTH"}, + Name: "max-template-depth", + Usage: "set the maximum depth for nested templates", + Value: 3, + }, + + // Environment Flags + &cli.BoolFlag{ + EnvVars: []string{"VELA_ENV_FILE", "ENV_FILE"}, + Name: "env-file", + Aliases: []string{"ef"}, + Usage: "load environment variables from a .env file", + Value: false, + }, + &cli.StringFlag{ + EnvVars: []string{"VELA_ENV_FILE_PATH", "ENV_FILE_PATH"}, + Name: "env-file-path", + Aliases: []string{"efp"}, + Usage: "provide the path to the file for the environment", + }, + &cli.BoolFlag{ + EnvVars: []string{"ONBOARD_LOCAL_ENV", "LOCAL_ENV"}, + Name: "local-env", + Usage: "load environment variables from local environment", + Value: false, + }, + &cli.StringSliceFlag{ + EnvVars: []string{"VELA_ENV_VARS"}, + Name: "env-vars", + Aliases: []string{"env"}, + Usage: "load a set of environment variables in the form of KEY1=VAL1,KEY2=VAL2", + }, }, CustomHelpTemplate: fmt.Sprintf(`%s EXAMPLES: @@ -138,6 +190,18 @@ EXAMPLES: $ {{.HelpName}} --volume /tmp/bar.txt:/tmp/bar.txt:rw 7. Execute a local Vela pipeline with type of go $ {{.HelpName}} --pipeline-type go + 8. Execute a local Vela pipeline with local templates + $ {{.HelpName}} --template-file : + 9. Execute a local Vela pipeline with specific environment variables + $ {{.HelpName}} --env KEY1=VAL1,KEY2=VAL2 + 10. Execute a local Vela pipeline with your existing local environment loaded into pipeline + $ {{.HelpName}} --local-env + 11. Execute a local Vela pipeline with an environment file loaded in + $ {{.HelpName}} --env-file (uses .env by default) + OR + $ {{.HelpName}} --env-file-path + 12. Execute a local Vela pipeline using remote templates + $ {{.HelpName}} --compiler.github.token --compiler.github.url DOCUMENTATION: @@ -154,23 +218,52 @@ func exec(c *cli.Context) error { return err } + // clear local environment unless told otherwise + if !c.Bool("local-env") { + os.Clearenv() + } + + // iterate through command-based env variables and set them in environment + for _, envSet := range c.StringSlice("env-vars") { + parts := strings.SplitN(envSet, "=", 2) + + os.Setenv(parts[0], parts[1]) + } + + // load env file if provided + if c.Bool("env-file") || len(c.String("env-file-path")) > 0 { + switch len(c.String("env-file-path")) { + case 0: + err := godotenv.Load() + if err != nil { + logrus.Fatal("Error loading env file") + } + default: + err := godotenv.Load(c.String("env-file-path")) + if err != nil { + logrus.Fatal("Error loading env file") + } + } + } + // create the pipeline configuration // // https://pkg.go.dev/github.com/go-vela/cli/action/pipeline?tab=doc#Config p := &pipeline.Config{ - Action: internal.ActionExec, - Branch: c.String("branch"), - Comment: c.String("comment"), - Event: c.String("event"), - Tag: c.String("tag"), - Target: c.String("target"), - Org: c.String(internal.FlagOrg), - Repo: c.String(internal.FlagRepo), - File: c.String("file"), - Local: c.Bool("local"), - Path: c.String("path"), - Volumes: c.StringSlice("volume"), - PipelineType: c.String("pipeline-type"), + Action: internal.ActionExec, + Branch: c.String("branch"), + Comment: c.String("comment"), + Event: c.String("event"), + Tag: c.String("tag"), + Target: c.String("target"), + Org: c.String(internal.FlagOrg), + Repo: c.String(internal.FlagRepo), + File: c.String("file"), + TemplateFiles: c.StringSlice("template-file"), + Local: c.Bool("local"), + Path: c.String("path"), + Volumes: c.StringSlice("volume"), + PipelineType: c.String("pipeline-type"), } // validate pipeline configuration @@ -189,11 +282,19 @@ func exec(c *cli.Context) error { return err } - // set the max template depth using provided configuration (max of 5) - client.TemplateDepth = util.MinInt(c.Int("max-template-depth"), 5) + // set when user is sourcing templates from local machine + if len(p.TemplateFiles) != 0 { + client.WithLocalTemplates(p.TemplateFiles) + client.TemplateDepth = util.MinInt(c.Int("max-template-depth"), 10) + } else { + // set max template depth to minimum of 5 and provided value if local templates are not provided. + // This prevents users from spamming SCM + client.TemplateDepth = util.MinInt(c.Int("max-template-depth"), 5) + logrus.Debugf("no local template files provided, setting max template depth to %d", client.TemplateDepth) + } // execute the exec call for the pipeline configuration // // https://pkg.go.dev/github.com/go-vela/cli/action/pipeline?tab=doc#Config.Exec - return p.Exec(client) + return p.Exec(client.WithPrivateGitHub(c.String(internal.FlagCompilerGitHubURL), c.String(internal.FlagCompilerGitHubToken))) } diff --git a/go.mod b/go.mod index 4c29591d..56fe90b1 100644 --- a/go.mod +++ b/go.mod @@ -11,11 +11,12 @@ require ( github.com/gin-gonic/gin v1.9.1 github.com/go-git/go-git/v5 v5.9.0 github.com/go-vela/sdk-go v0.21.0 - github.com/go-vela/server v0.21.0 + github.com/go-vela/server v0.21.1-0.20231006151542-174ab52b94ff github.com/go-vela/types v0.21.0 - github.com/go-vela/worker v0.21.0 + github.com/go-vela/worker v0.21.1-0.20231009163044-26628ba836f7 github.com/golang-jwt/jwt/v5 v5.0.0 github.com/gosuri/uitable v0.0.4 + github.com/joho/godotenv v1.5.1 github.com/manifoldco/promptui v0.9.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/afero v1.10.0 diff --git a/go.sum b/go.sum index e959644c..5fe09e7b 100644 --- a/go.sum +++ b/go.sum @@ -168,12 +168,12 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEe github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-vela/sdk-go v0.21.0 h1:Hedak1Yk9rGn3ZBOvLvxLrMcyvBf3+RB6/wMgHNxyxw= github.com/go-vela/sdk-go v0.21.0/go.mod h1:fNMQxSqBCXQH6bK3Ej0aCj/iugEDZNEIWW3Xj/m22AQ= -github.com/go-vela/server v0.21.0 h1:tBSUMp1rni2i1wOzP2uxh7o6uTkzjrYfRe0fKJBOcNA= -github.com/go-vela/server v0.21.0/go.mod h1:3pp/hg5NUZ6VbbDC6JO97H7Ry7vv/qKA8GpWsaUpZ1M= +github.com/go-vela/server v0.21.1-0.20231006151542-174ab52b94ff h1:tPaHz1ZWqmG1qak0uU0smGgt6u7Py2xJb68FANGxmfA= +github.com/go-vela/server v0.21.1-0.20231006151542-174ab52b94ff/go.mod h1:3pp/hg5NUZ6VbbDC6JO97H7Ry7vv/qKA8GpWsaUpZ1M= github.com/go-vela/types v0.21.0 h1:yZrVUw4jKO0JHaUBkOIZZdniDGyDOpTMbKriemdm1jg= github.com/go-vela/types v0.21.0/go.mod h1:Jn8K28uj7mACc55fkFgaIzL0q45iXydOFGEeoSeHUtQ= -github.com/go-vela/worker v0.21.0 h1:TEkW0tZSn0JyrJUSdoKjJ4DaxHkKtS9wwL5r7/ugsZ8= -github.com/go-vela/worker v0.21.0/go.mod h1:cSdF+nduPovzqiOjLb9mXZLY1Jd8ZGttHub8S3Y6oRo= +github.com/go-vela/worker v0.21.1-0.20231009163044-26628ba836f7 h1:yRELmZuHEgr7d/WVx8W+jNNXPB3Zk4N0ISXOcCxXli4= +github.com/go-vela/worker v0.21.1-0.20231009163044-26628ba836f7/go.mod h1:cSdF+nduPovzqiOjLb9mXZLY1Jd8ZGttHub8S3Y6oRo= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -282,6 +282,8 @@ github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=