V0.82.1

bucket/bucket.go

@@ -2,6 +2,7 @@ package bucket
 
 import (
 	"bytes"
+	"io"
 	"net/url"
 	"strings"
 	"time"
@@ -46,12 +47,12 @@ func S3Read(URL string, sess *session.Session) (string, error) {
 }
 
 // S3write - Writes a file to s3 and returns the presigned url
-func S3write(bucket string, key string, body string, sess *session.Session) (string, error) {
+func S3write(bucket string, key string, body io.Reader, sess *session.Session) (string, error) {
 	svc := s3.New(sess)
 	params := &s3.PutObjectInput{
 		Bucket: &bucket,
 		Key:    &key,
-		Body:   bytes.NewReader([]byte(body)),
+		Body:   aws.ReadSeekCloser(body),
 		Metadata: map[string]*string{
 			"created_by": aws.String("qaz"),
 		},

commands/commands.go

@@ -166,7 +166,7 @@ var (
 			output, err := yaml.Marshal(values)
 			utils.HandleError(err)
 
-			reg, err := regexp.Compile(OutputRegex)
+			reg, err := regexp.Compile(stacks.OutputRegex)
 			utils.HandleError(err)
 
 			resp := reg.ReplaceAllStringFunc(string(output), func(s string) string {

commands/config.go

@@ -7,6 +7,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/daidokoro/qaz/log"
 	"github.com/daidokoro/qaz/stacks"
+
 	yaml "gopkg.in/yaml.v2"
 
 	"github.com/daidokoro/hcl"
@@ -36,10 +37,17 @@ func Configure(confSource string, conf string) (stks stacks.Map, err error) {
 		return
 	}
 
+	// decrypt config secrets
+	if err = config.SopsDecrypt(); err != nil {
+		err = fmt.Errorf("failed to decrypt secrets in config: %s", err)
+		return
+	}
+
 	log.Debug("checking Config for HCL format...")
 	if err = hcl.Unmarshal([]byte(config.String), &config); err != nil {
 		// fmt.Println(err)
 		log.Debug("failed to parse hcl... moving to JSON/YAML... error: %v", err)
+
 		if err = yaml.Unmarshal([]byte(config.String), &config); err != nil {
 			return
 		}
@@ -69,6 +77,7 @@ func Configure(confSource string, conf string) (stks stacks.Map, err error) {
 			Project:          &config.Project,
 			Timeout:          v.Timeout,
 			NotificationARNs: v.NotificationARNs,
+			CFRoleARN:        v.CFRoleArn,
 		})
 
 		stks.MustGet(s).SetStackName()
@@ -101,8 +110,12 @@ func Configure(confSource string, conf string) (stks stacks.Map, err error) {
 			Parameters(stks.MustGet(s)).
 			Tags(stks.MustGet(s))
 
-		// update DeployTimeFunctions
-		stks.AddMapFuncs(DeployTimeFunctions)
+		// add stack based functions stackoutput
+		stks.AddFuncs(DeployTimeFunctions)
+	}
+
+	if len(config.S3Package) > 0 {
+		err = config.PackageToS3(run.executePackage)
 	}
 
 	return

commands/functions.go

@@ -82,7 +82,6 @@ var (
 			opts.Profile = profile[0]
 			return
 		})
-
 		utils.HandleError(err)
 
 		resp, err := bucket.S3Read(url, sess)

commands/generate.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/daidokoro/qaz/log"
+	"github.com/daidokoro/qaz/stacks"
 	"github.com/daidokoro/qaz/utils"
 	"github.com/spf13/cobra"
 )
@@ -50,7 +51,7 @@ var generateCmd = &cobra.Command{
 		log.Debug("generating a template for %s", name)
 		utils.HandleError(stks.MustGet(s).GenTimeParser())
 
-		resp := regexp.MustCompile(OutputRegex).
+		resp := regexp.MustCompile(stacks.OutputRegex).
 			ReplaceAllStringFunc(string(stks.MustGet(s).Template), func(s string) string {
 				return log.ColorString(s, log.CYAN)
 			})

commands/init.go

@@ -15,6 +15,7 @@ func init() {
 	deployCmd.Flags().StringArrayVarP(&run.tplSources, "template", "t", []string{}, "path to template file(s) Or stack::url")
 	deployCmd.Flags().BoolVarP(&run.rollback, "disable-rollback", "", false, "Set Stack to rollback on deployment failures")
 	deployCmd.Flags().BoolVarP(&run.all, "all", "A", false, "deploy all stacks with defined Sources in config")
+	deployCmd.Flags().BoolVarP(&run.executePackage, "package", "P", false, "execute s3 packaging actions from config")
 
 	// Define Git Deploy Flags
 	gitDeployCmd.Flags().BoolVarP(&run.rollback, "disable-rollback", "", false, "Set Stack to rollback on deployment failures")
@@ -57,6 +58,7 @@ func init() {
 
 	// Define Update Command
 	updateCmd.Flags().BoolVarP(&run.interactive, "interactive", "i", false, "preview change-set and ask before executing it")
+	updateCmd.Flags().BoolVarP(&run.executePackage, "package", "P", false, "execute s3 packaging actions from config")
 
 	// Add Config --config common flag
 	for _, cmd := range []interface{}{

commands/lint.go

@@ -58,12 +58,13 @@ var lintCmd = &cobra.Command{
 		filename := fmt.Sprintf(".%s.qaz", s)
 		writeErr := ioutil.WriteFile(filename, content, 0644)
 		utils.HandleError(writeErr)
+		defer os.Remove(filename)
 
 		// run cfn-lint against temporary file
 		_, lookErr := exec.LookPath("cfn-lint")
 		if lookErr != nil {
 			utils.HandleError(
-				fmt.Errorf("cfn-lint executable not found! Please consider https://pypi.org/project/cfn-lint/ for help."),
+				fmt.Errorf("cfn-lint executable not found! Please consider https://pypi.org/project/cfn-lint/ for help"),
 			)
 		}
 		execCmd := exec.Command("cfn-lint", filename)

commands/outputs.go

@@ -55,7 +55,7 @@ var (
 							log.Error(err.Error())
 						}
 
-						resp := regexp.MustCompile(OutputRegex).
+						resp := regexp.MustCompile(stacks.OutputRegex).
 							ReplaceAllStringFunc(string(m), func(s string) string {
 								return log.ColorString(s, log.CYAN)
 							})

commands/shell.go

@@ -123,7 +123,7 @@ func initShell(p string, stks *stacks.Map, s *ishell.Shell) {
 								log.Error(err.Error())
 							}
 
-							reg, err := regexp.Compile(OutputRegex)
+							reg, err := regexp.Compile(stacks.OutputRegex)
 							utils.HandleError(err)
 
 							resp := reg.ReplaceAllStringFunc(string(m), func(s string) string {
@@ -299,7 +299,7 @@ func initShell(p string, stks *stacks.Map, s *ishell.Shell) {
 					return
 				}
 
-				reg, err := regexp.Compile(OutputRegex)
+				reg, err := regexp.Compile(stacks.OutputRegex)
 				utils.HandleError(err)
 
 				resp := reg.ReplaceAllStringFunc(string(stks.MustGet(s).Template), func(s string) string {

commands/vars.go

@@ -18,31 +18,32 @@ const (
 	configENV = "QAZ_CONFIG"
 
 	// OutputRegex for printing yaml/json output
-	OutputRegex = `(?m)^[ ]*([^\r\n:]+?)\s*:`
+	// OutputRegex = `(?m)^[ ]*([^\r\n:]+?)\s*:`
 )
 
 // run.var used as a central point for command data from flags
 var run = struct {
-	cfgSource   string
-	tplSource   string
-	profile     string
-	region      string
-	tplSources  []string
-	stacks      map[string]string
-	all         bool
-	version     bool
-	request     string
-	debug       bool
-	funcEvent   string
-	lambdAsync  bool
-	changeName  string
-	stackName   string
-	rollback    bool
-	colors      bool
-	cfgRaw      string
-	gituser     string
-	gitpass     string
-	gitrsa      string
-	protectOff  bool
-	interactive bool
+	cfgSource      string
+	tplSource      string
+	profile        string
+	region         string
+	tplSources     []string
+	stacks         map[string]string
+	all            bool
+	version        bool
+	request        string
+	debug          bool
+	funcEvent      string
+	lambdAsync     bool
+	changeName     string
+	stackName      string
+	rollback       bool
+	colors         bool
+	cfgRaw         string
+	gituser        string
+	gitpass        string
+	gitrsa         string
+	protectOff     bool
+	interactive    bool
+	executePackage bool
 }{}

commands/version.go

@@ -1,4 +1,4 @@
 package commands
 
 // Version
-const version = "v0.82.0-beta"
+const version = "v0.82.1-beta"

examples/package_to_s3/code/main.py

@@ -0,0 +1,3 @@
+def handle(event, ctx):
+    print('I got this event: %s' % event)
+    return event

examples/package_to_s3/config.yaml

@@ -0,0 +1,22 @@
+project: package-test
+
+# list of sources and destinations to package to s3
+# note that the credentials used to run qaz will need access
+# to any/all S3 destinations and sources specified.
+# 
+# qaz needs to be executed with the --package/-p flag
+# to trigger package uploads - for eg. qaz deploy --all --package
+s3_package:
+  myfunc:
+    source: ./code
+    destination: s3://daidokoro-lambda/code_test.zip
+
+
+stacks:
+  func:
+    source: template.yaml
+    cf:
+      handle: main.handle
+      name: code_test_function
+      code_uri: s3://daidokoro-lambda/code_test.zip
+

examples/package_to_s3/template.yaml

@@ -0,0 +1,21 @@
+AWSTemplateFormatVersion: '2010-09-09'
+Transform: AWS::Serverless-2016-10-31
+Resources:
+  Function:
+    Type: AWS::Serverless::Function
+    Properties:
+      Handler: {{ .stack.handle }}
+      Runtime: python3.8
+      FunctionName: {{ .stack.name }}
+      CodeUri: {{ .stack.code_uri }}
+      Events:
+        ApiRoot:
+          Type: Api
+          Properties:
+            Path: /
+            Method: ANY
+        ApiGreedy:
+          Type: Api
+          Properties:
+            Path: /{proxy+}
+            Method: ANY

go.mod

@@ -1,23 +1,24 @@
 module github.com/daidokoro/qaz
 
-go 1.12
+go 1.15
 
 require (
 	github.com/CrowdSurge/banner v0.0.0-20140923200336-8c0e79dc5ff7
 	github.com/abiosoft/ishell v2.0.0+incompatible // indirect
 	github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db // indirect
-	github.com/aws/aws-sdk-go v1.29.34
+	github.com/aws/aws-sdk-go v1.37.18
 	github.com/chzyer/logex v1.1.10 // indirect
 	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
 	github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 // indirect
 	github.com/daidokoro/hcl v0.0.0-20170509225359-392dba7d905e
 	github.com/daidokoro/ishell v0.0.0-20170626201312-73d87bbaf310
 	github.com/fatih/color v1.9.0
 	github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
-	github.com/spf13/cobra v0.0.7
-	github.com/stretchr/testify v1.5.1
-	golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59
+	github.com/spf13/cobra v1.0.0
+	github.com/stretchr/testify v1.6.1
+	go.mozilla.org/sops/v3 v3.7.1
+	golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83
 	gopkg.in/src-d/go-billy.v4 v4.3.2
 	gopkg.in/src-d/go-git.v4 v4.13.1
-	gopkg.in/yaml.v2 v2.2.8
+	gopkg.in/yaml.v2 v2.3.0
 )