enhance(exec): templates and env support for exec pipeline (#492)

* enhance(exec): templates and env support for exec pipeline

* adjust validate local test and verify local templates are included in pipeline

action/pipeline/exec.go

@@ -78,19 +78,17 @@ func (c *Config) Exec(client compiler.Engine) error {
 		WithComment(c.Comment).
 		WithLocal(true).
 		WithRepo(r).
+		WithLocalTemplates(c.TemplateFiles).
 		Compile(path)
 	if err != nil {
 		return err
 	}
 
-	// check if the local configuration is enabled
-	if c.Local {
-		// create current directory path for local mount
-		mount := fmt.Sprintf("%s:%s:rw", base, constants.WorkspaceDefault)
+	// create current directory path for local mount
+	mount := fmt.Sprintf("%s:%s:rw", base, constants.WorkspaceDefault)
 
-		// add the current directory path to volume mounts
-		c.Volumes = append(c.Volumes, mount)
-	}
+	// add the current directory path to volume mounts
+	c.Volumes = append(c.Volumes, mount)
 
 	logrus.Tracef("creating runtime engine %s", constants.DriverDocker)
 

action/pipeline/validate.go

@@ -110,6 +110,22 @@ func (c *Config) ValidateLocal(client compiler.Engine) error {
 		return err
 	}
 
+	// check to see if locally provided templates were included in compilation
+	for _, tmpl := range c.TemplateFiles {
+		parts := strings.Split(tmpl, ":")
+		included := false
+
+		for _, pTmpl := range p.Templates {
+			if strings.EqualFold(parts[0], pTmpl.Name) {
+				included = true
+			}
+		}
+
+		if !included {
+			return fmt.Errorf("local template with name %s not included in pipeline templates", parts[0])
+		}
+	}
+
 	// output the message in stderr format
 	//
 	// https://pkg.go.dev/github.com/go-vela/cli/internal/output?tab=doc#Stderr

action/pipeline/validate_test.go

@@ -253,8 +253,7 @@ func TestPipeline_Config_ValidateLocal(t *testing.T) {
 			},
 		},
 		{
-			name:    "pipeline with multiple template (local overrides) template mismatch",
-			failure: true,
+			name: "pipeline with multiple template (local overrides) only one template specified",
 			config: &Config{
 				Action:        "validate",
 				File:          "default_multi_template.yml",

command/pipeline/exec.go

@@ -4,14 +4,18 @@ package pipeline
 
 import (
 	"fmt"
+	"os"
+	"strings"
 
 	"github.com/go-vela/cli/action"
 	"github.com/go-vela/cli/action/pipeline"
 	"github.com/go-vela/cli/internal"
 	"github.com/go-vela/server/compiler/native"
 	"github.com/go-vela/server/util"
 	"github.com/go-vela/types/constants"
+	"github.com/sirupsen/logrus"
 
+	"github.com/joho/godotenv"
 	"github.com/urfave/cli/v2"
 )
 
@@ -26,31 +30,31 @@ var CommandExec = &cli.Command{
 		// Build Flags
 
 		&cli.StringFlag{
-			EnvVars: []string{"VELA_BRANCH", "PIPELINE_BRANCH"},
+			EnvVars: []string{"VELA_BRANCH", "PIPELINE_BRANCH", "VELA_BUILD_BRANCH"},
 			Name:    "branch",
 			Aliases: []string{"b"},
 			Usage:   "provide the build branch for the pipeline",
 		},
 		&cli.StringFlag{
-			EnvVars: []string{"VELA_COMMENT", "PIPELINE_COMMENT"},
+			EnvVars: []string{"VELA_COMMENT", "PIPELINE_COMMENT", "VELA_BUILD_COMMENT"},
 			Name:    "comment",
 			Aliases: []string{"c"},
 			Usage:   "provide the build comment for the pipeline",
 		},
 		&cli.StringFlag{
-			EnvVars: []string{"VELA_EVENT", "PIPELINE_EVENT"},
+			EnvVars: []string{"VELA_EVENT", "PIPELINE_EVENT", "VELA_BUILD_EVENT"},
 			Name:    "event",
 			Aliases: []string{"e"},
 			Usage:   "provide the build event for the pipeline",
 		},
 		&cli.StringFlag{
-			EnvVars: []string{"VELA_TAG", "PIPELINE_TAG"},
+			EnvVars: []string{"VELA_TAG", "PIPELINE_TAG", "VELA_BUILD_TAG"},
 			Name:    "tag",
 			Aliases: []string{"t"},
 			Usage:   "provide the build tag for the pipeline",
 		},
 		&cli.StringFlag{
-			EnvVars: []string{"VELA_TARGET", "PIPELINE_TARGET"},
+			EnvVars: []string{"VELA_TARGET", "PIPELINE_TARGET", "VELA_BUILD_TARGET"},
 			Name:    "target",
 			Usage:   "provide the build target for the pipeline",
 		},
@@ -93,12 +97,6 @@ var CommandExec = &cli.Command{
 			Aliases: []string{"v"},
 			Usage:   "provide list of local volumes to mount",
 		},
-		&cli.IntFlag{
-			EnvVars: []string{"VELA_MAX_TEMPLATE_DEPTH", "MAX_TEMPLATE_DEPTH"},
-			Name:    "max-template-depth",
-			Usage:   "set the maximum depth for nested templates",
-			Value:   3,
-		},
 
 		// Repo Flags
 
@@ -121,6 +119,60 @@ var CommandExec = &cli.Command{
 			Usage:   "type of pipeline for the compiler to render",
 			Value:   constants.PipelineTypeYAML,
 		},
+
+		// Compiler Template Flags
+
+		&cli.StringFlag{
+			EnvVars: []string{"VELA_COMPILER_GITHUB_TOKEN", "COMPILER_GITHUB_TOKEN"},
+			Name:    internal.FlagCompilerGitHubToken,
+			Aliases: []string{"ct"},
+			Usage:   "github compiler token",
+		},
+		&cli.StringFlag{
+			EnvVars: []string{"VELA_COMPILER_GITHUB_URL", "COMPILER_GITHUB_URL"},
+			Name:    internal.FlagCompilerGitHubURL,
+			Aliases: []string{"cgu"},
+			Usage:   "github url, used by compiler, for pulling registry templates",
+		},
+		&cli.StringSliceFlag{
+			EnvVars: []string{"VELA_TEMPLATE_FILE", "PIPELINE_TEMPLATE_FILE"},
+			Name:    "template-file",
+			Aliases: []string{"tf, tfs, template-files"},
+			Usage:   "enables using a local template file for expansion in the form <name>:<path>",
+		},
+		&cli.IntFlag{
+			EnvVars: []string{"VELA_MAX_TEMPLATE_DEPTH", "MAX_TEMPLATE_DEPTH"},
+			Name:    "max-template-depth",
+			Usage:   "set the maximum depth for nested templates",
+			Value:   3,
+		},
+
+		// Environment Flags
+		&cli.BoolFlag{
+			EnvVars: []string{"VELA_ENV_FILE", "ENV_FILE"},
+			Name:    "env-file",
+			Aliases: []string{"ef"},
+			Usage:   "load environment variables from a .env file",
+			Value:   false,
+		},
+		&cli.StringFlag{
+			EnvVars: []string{"VELA_ENV_FILE_PATH", "ENV_FILE_PATH"},
+			Name:    "env-file-path",
+			Aliases: []string{"efp"},
+			Usage:   "provide the path to the file for the environment",
+		},
+		&cli.BoolFlag{
+			EnvVars: []string{"ONBOARD_LOCAL_ENV", "LOCAL_ENV"},
+			Name:    "local-env",
+			Usage:   "load environment variables from local environment",
+			Value:   false,
+		},
+		&cli.StringSliceFlag{
+			EnvVars: []string{"VELA_ENV_VARS"},
+			Name:    "env-vars",
+			Aliases: []string{"env"},
+			Usage:   "load a set of environment variables in the form of KEY1=VAL1,KEY2=VAL2",
+		},
 	},
 	CustomHelpTemplate: fmt.Sprintf(`%s
 EXAMPLES:
@@ -138,6 +190,18 @@ EXAMPLES:
     $ {{.HelpName}} --volume /tmp/bar.txt:/tmp/bar.txt:rw
   7. Execute a local Vela pipeline with type of go
     $ {{.HelpName}} --pipeline-type go
+  8. Execute a local Vela pipeline with local templates
+    $ {{.HelpName}} --template-file <template_name>:<path_to_template>
+  9. Execute a local Vela pipeline with specific environment variables
+    $ {{.HelpName}} --env KEY1=VAL1,KEY2=VAL2
+  10. Execute a local Vela pipeline with your existing local environment loaded into pipeline
+    $ {{.HelpName}} --local-env
+  11. Execute a local Vela pipeline with an environment file loaded in
+    $ {{.HelpName}} --env-file (uses .env by default)
+      OR
+    $ {{.HelpName}} --env-file-path <path_to_file>
+  12. Execute a local Vela pipeline using remote templates
+    $ {{.HelpName}} --compiler.github.token <GITHUB_PAT> --compiler.github.url <GITHUB_URL>
 
 DOCUMENTATION:
 
@@ -154,23 +218,52 @@ func exec(c *cli.Context) error {
 		return err
 	}
 
+	// clear local environment unless told otherwise
+	if !c.Bool("local-env") {
+		os.Clearenv()
+	}
+
+	// iterate through command-based env variables and set them in environment
+	for _, envSet := range c.StringSlice("env-vars") {
+		parts := strings.SplitN(envSet, "=", 2)
+
+		os.Setenv(parts[0], parts[1])
+	}
+
+	// load env file if provided
+	if c.Bool("env-file") || len(c.String("env-file-path")) > 0 {
+		switch len(c.String("env-file-path")) {
+		case 0:
+			err := godotenv.Load()
+			if err != nil {
+				logrus.Fatal("Error loading env file")
+			}
+		default:
+			err := godotenv.Load(c.String("env-file-path"))
+			if err != nil {
+				logrus.Fatal("Error loading env file")
+			}
+		}
+	}
+
 	// create the pipeline configuration
 	//
 	// https://pkg.go.dev/github.com/go-vela/cli/action/pipeline?tab=doc#Config
 	p := &pipeline.Config{
-		Action:       internal.ActionExec,
-		Branch:       c.String("branch"),
-		Comment:      c.String("comment"),
-		Event:        c.String("event"),
-		Tag:          c.String("tag"),
-		Target:       c.String("target"),
-		Org:          c.String(internal.FlagOrg),
-		Repo:         c.String(internal.FlagRepo),
-		File:         c.String("file"),
-		Local:        c.Bool("local"),
-		Path:         c.String("path"),
-		Volumes:      c.StringSlice("volume"),
-		PipelineType: c.String("pipeline-type"),
+		Action:        internal.ActionExec,
+		Branch:        c.String("branch"),
+		Comment:       c.String("comment"),
+		Event:         c.String("event"),
+		Tag:           c.String("tag"),
+		Target:        c.String("target"),
+		Org:           c.String(internal.FlagOrg),
+		Repo:          c.String(internal.FlagRepo),
+		File:          c.String("file"),
+		TemplateFiles: c.StringSlice("template-file"),
+		Local:         c.Bool("local"),
+		Path:          c.String("path"),
+		Volumes:       c.StringSlice("volume"),
+		PipelineType:  c.String("pipeline-type"),
 	}
 
 	// validate pipeline configuration
@@ -189,11 +282,19 @@ func exec(c *cli.Context) error {
 		return err
 	}
 
-	// set the max template depth using provided configuration (max of 5)
-	client.TemplateDepth = util.MinInt(c.Int("max-template-depth"), 5)
+	// set when user is sourcing templates from local machine
+	if len(p.TemplateFiles) != 0 {
+		client.WithLocalTemplates(p.TemplateFiles)
+		client.TemplateDepth = util.MinInt(c.Int("max-template-depth"), 10)
+	} else {
+		// set max template depth to minimum of 5 and provided value if local templates are not provided.
+		// This prevents users from spamming SCM
+		client.TemplateDepth = util.MinInt(c.Int("max-template-depth"), 5)
+		logrus.Debugf("no local template files provided, setting max template depth to %d", client.TemplateDepth)
+	}
 
 	// execute the exec call for the pipeline configuration
 	//
 	// https://pkg.go.dev/github.com/go-vela/cli/action/pipeline?tab=doc#Config.Exec
-	return p.Exec(client)
+	return p.Exec(client.WithPrivateGitHub(c.String(internal.FlagCompilerGitHubURL), c.String(internal.FlagCompilerGitHubToken)))
 }

go.mod

@@ -11,11 +11,12 @@ require (
 	github.com/gin-gonic/gin v1.9.1
 	github.com/go-git/go-git/v5 v5.9.0
 	github.com/go-vela/sdk-go v0.21.0
-	github.com/go-vela/server v0.21.0
+	github.com/go-vela/server v0.21.1-0.20231006151542-174ab52b94ff
 	github.com/go-vela/types v0.21.0
-	github.com/go-vela/worker v0.21.0
+	github.com/go-vela/worker v0.21.1-0.20231009163044-26628ba836f7
 	github.com/golang-jwt/jwt/v5 v5.0.0
 	github.com/gosuri/uitable v0.0.4
+	github.com/joho/godotenv v1.5.1
 	github.com/manifoldco/promptui v0.9.0
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/afero v1.10.0

go.sum

@@ -168,12 +168,12 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEe
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
 github.com/go-vela/sdk-go v0.21.0 h1:Hedak1Yk9rGn3ZBOvLvxLrMcyvBf3+RB6/wMgHNxyxw=
 github.com/go-vela/sdk-go v0.21.0/go.mod h1:fNMQxSqBCXQH6bK3Ej0aCj/iugEDZNEIWW3Xj/m22AQ=
-github.com/go-vela/server v0.21.0 h1:tBSUMp1rni2i1wOzP2uxh7o6uTkzjrYfRe0fKJBOcNA=
-github.com/go-vela/server v0.21.0/go.mod h1:3pp/hg5NUZ6VbbDC6JO97H7Ry7vv/qKA8GpWsaUpZ1M=
+github.com/go-vela/server v0.21.1-0.20231006151542-174ab52b94ff h1:tPaHz1ZWqmG1qak0uU0smGgt6u7Py2xJb68FANGxmfA=
+github.com/go-vela/server v0.21.1-0.20231006151542-174ab52b94ff/go.mod h1:3pp/hg5NUZ6VbbDC6JO97H7Ry7vv/qKA8GpWsaUpZ1M=
 github.com/go-vela/types v0.21.0 h1:yZrVUw4jKO0JHaUBkOIZZdniDGyDOpTMbKriemdm1jg=
 github.com/go-vela/types v0.21.0/go.mod h1:Jn8K28uj7mACc55fkFgaIzL0q45iXydOFGEeoSeHUtQ=
-github.com/go-vela/worker v0.21.0 h1:TEkW0tZSn0JyrJUSdoKjJ4DaxHkKtS9wwL5r7/ugsZ8=
-github.com/go-vela/worker v0.21.0/go.mod h1:cSdF+nduPovzqiOjLb9mXZLY1Jd8ZGttHub8S3Y6oRo=
+github.com/go-vela/worker v0.21.1-0.20231009163044-26628ba836f7 h1:yRELmZuHEgr7d/WVx8W+jNNXPB3Zk4N0ISXOcCxXli4=
+github.com/go-vela/worker v0.21.1-0.20231009163044-26628ba836f7/go.mod h1:cSdF+nduPovzqiOjLb9mXZLY1Jd8ZGttHub8S3Y6oRo=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@@ -282,6 +282,8 @@ github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM=
 github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=