From a14920ba0136dfc15aa0ea87bf839706f4170651 Mon Sep 17 00:00:00 2001
From: Steve Ramage <49958178+steve-r-west@users.noreply.github.com>
Date: Sun, 7 Jan 2024 10:45:40 -0800
Subject: [PATCH] Resolves #431 - Add support for dynamic runbook steps (#434)

* Resolves #431 - Add support for dynamic steps

* Resolves #431 - Add support for dynamic runbook steps
---
 .github/workflows/test.yml                    |  1 +
 README.md                                     |  6 --
 cmd/root.go                                   |  2 +-
 cmd/runbooks.go                               | 49 +++++++++++--
 docs/runbook-development.md                   | 68 +++++++++++++++++++
 external/runbooks/account-management.epcc.yml | 41 +++++++++--
 external/runbooks/hello-world.epcc.yml        | 24 +++++++
 external/runbooks/run-all-runbooks.sh         |  3 +
 external/runbooks/runbook_validation.go       | 21 +++++-
 9 files changed, 195 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index e90933a..0f50bfd 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -18,6 +18,7 @@ jobs:
     name: Build
     runs-on: ubuntu-latest
     timeout-minutes: 15
+    concurrency: github
     steps:
       - name: Set up Go
         uses: actions/setup-go@v5
diff --git a/README.md b/README.md
index f97d9b7..3c8d1f5 100644
--- a/README.md
+++ b/README.md
@@ -242,9 +242,3 @@ go fmt "./..."
 echo "Adding changed files back to git"
 git diff --cached --name-only --diff-filter=ACM | grep -E "\.(go)$" | xargs  git add
 ```
-
-### Generating Go Routine Dumps
-
-You can send a `SIGQUIT` to `epcc` to get it to generate a go routine dump, this can be useful for debugging deadlocks and other performance issues.
-
-On Linux and maybe other OS, you can use CTRL+\ to send a `SIGQUIT` to the process.
diff --git a/cmd/root.go b/cmd/root.go
index 9162403..4148672 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -299,7 +299,7 @@ func DumpTraces() {
 		for {
 			<-sigs
 			stacklen := runtime.Stack(buf, true)
-			fmt.Printf("=== received SIGQUIT ===\n*** goroutine dump...\n%s\n*** end\n", buf[:stacklen])
+			log.Printf("=== received SIGQUIT ===\n*** goroutine dump...\n%s\n*** end\n", buf[:stacklen])
 		}
 	}()
 }
diff --git a/cmd/runbooks.go b/cmd/runbooks.go
index 586bd3e..ceff758 100644
--- a/cmd/runbooks.go
+++ b/cmd/runbooks.go
@@ -14,6 +14,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"golang.org/x/sync/semaphore"
+	"gopkg.in/yaml.v3"
 	"strconv"
 	"strings"
 	"sync/atomic"
@@ -120,13 +121,31 @@ func initRunbookShowCommands() *cobra.Command {
 				Short: runbookAction.Description.Short,
 				RunE: func(cmd *cobra.Command, args []string) error {
 
-					for stepIdx, cmd := range runbookAction.RawCommands {
+					cmds := runbookAction.RawCommands
+					for stepIdx := 0; stepIdx < len(cmds); stepIdx++ {
+						cmd := cmds[stepIdx]
 						templateName := fmt.Sprintf("Runbook: %s Action: %s Step: %d", runbook.Name, runbookAction.Name, stepIdx)
-
 						rawCmdLines, err := runbooks.RenderTemplates(templateName, cmd, runbookStringArguments, runbookAction.Variables)
+
 						if err != nil {
 							return err
 						}
+
+						joinedString := strings.Join(rawCmdLines, "\n")
+						renderedCmd := []string{}
+						err = yaml.Unmarshal([]byte(joinedString), &renderedCmd)
+
+						if err == nil {
+							log.Tracef("Line %d is a Yaml array %s, inserting into stack", stepIdx, joinedString)
+							newCmds := make([]string, 0, len(cmds)+len(renderedCmd)-1)
+							newCmds = append(newCmds, cmds[0:stepIdx]...)
+							newCmds = append(newCmds, renderedCmd...)
+							newCmds = append(newCmds, cmds[stepIdx+1:]...)
+							cmds = newCmds
+							stepIdx--
+							continue
+						}
+
 						for _, line := range rawCmdLines {
 							if len(strings.Trim(line, " \n")) > 0 {
 								println(line)
@@ -224,13 +243,15 @@ func initRunbookRunCommands() *cobra.Command {
 						MaxTotal: *maxConcurrency,
 						MaxIdle:  *maxConcurrency,
 					})
-					for stepIdx, rawCmd := range runbookAction.RawCommands {
 
+					rawCmds := runbookAction.RawCommands
+					for stepIdx := 0; stepIdx < len(rawCmds); stepIdx++ {
+
+						origIndex := &stepIdx
 						// Create a copy of loop variables
 						stepIdx := stepIdx
-						rawCmd := rawCmd
+						rawCmd := rawCmds[stepIdx]
 
-						log.Infof("Executing> %s", rawCmd)
 						templateName := fmt.Sprintf("Runbook: %s Action: %s Step: %d", runbook.Name, runbookAction.Name, stepIdx)
 						rawCmdLines, err := runbooks.RenderTemplates(templateName, rawCmd, runbookStringArguments, runbookAction.Variables)
 
@@ -238,6 +259,24 @@ func initRunbookRunCommands() *cobra.Command {
 							cancelFunc()
 							return err
 						}
+
+						joinedString := strings.Join(rawCmdLines, "\n")
+						renderedCmd := []string{}
+
+						err = yaml.Unmarshal([]byte(joinedString), &renderedCmd)
+
+						if err == nil {
+							log.Tracef("Line %d is a Yaml array %s, inserting into stack", stepIdx, joinedString)
+							newCmds := make([]string, 0, len(rawCmds)+len(renderedCmd)-1)
+							newCmds = append(newCmds, rawCmds[0:stepIdx]...)
+							newCmds = append(newCmds, renderedCmd...)
+							newCmds = append(newCmds, rawCmds[stepIdx+1:]...)
+							rawCmds = newCmds
+							*origIndex--
+							continue
+						}
+
+						log.Infof("Executing> %s", rawCmd)
 						resultChan := make(chan *commandResult, *maxConcurrency*2)
 						funcs := make([]func(), 0, len(rawCmdLines))
 
diff --git a/docs/runbook-development.md b/docs/runbook-development.md
index 5b53840..021e887 100644
--- a/docs/runbook-development.md
+++ b/docs/runbook-development.md
@@ -4,6 +4,8 @@
 
 This document outlines the syntax and capabilities of runbooks. 
 
+
+
 ## Prerequisites
 
 Users developing runbooks should have familiarity with:
@@ -158,6 +160,72 @@ epcc create customer-address  "Hello World" name "address_0" first_name "John" l
 epcc create customer-address  "Hello World" name "address_1" first_name "John" last_name "Smith" line_1 "1234 Main Street" county "XX" "postcode" "H0H 0H0" country "DE"
 ```
 
+### Dynamic Steps
+
+Templates are only renderable for a particular step, but you can instead of generating a command, actually generate a Yaml array, that will be interpreted as steps.
+This can give you more control over control flow, where-as otherwise you are stuck with all rendered commands being in the same sequence.
+
+```yaml
+name: hello-world
+description:
+  short: "A hello world runbook"
+actions:
+   sequential-sleeps:
+   variables:
+      count:
+         type: INT
+         default: 2
+         description:
+            short: "The number of sleeps"
+   commands:
+      - |2
+        {{- range untilStep 0 .count 1}}
+          - sleep 1
+        {{- end -}}
+   concurrent-sleeps:
+      variables:
+         count:
+            type: INT
+            default: 2
+            description:
+               short: "The number of sleeps"
+      commands:
+         - |
+            {{- range untilStep 0 .count 1}}
+            sleep 1
+            {{- end -}}
+```
+
+It's important for all commands to be indended the same amount, and start with a `-` this will cause the entire string to be valid as a Yaml array.
+
+You can see the results of executing these runbooks below, without the `-` the commands all execute concurrently. With the `-` the execute one after another.
+
+```bash
+$time epcc runbooks run hello-world concurrent-sleeps  --count 4
+INFO[0000] Executing> {{- range untilStep 0 .count 1}}
+sleep 1
+{{- end -}} 
+INFO[0000] Sleeping for 1 seconds                       
+INFO[0000] Sleeping for 1 seconds                       
+INFO[0000] Sleeping for 1 seconds                       
+INFO[0000] Sleeping for 1 seconds                       
+real    0m1.302s
+user    0m0.441s
+sys     0m0.059s
+$time epcc runbooks run hello-world sequential-sleeps --count 4
+INFO[0000] Executing> sleep 1                           
+INFO[0000] Sleeping for 1 seconds                       
+INFO[0001] Executing> sleep 1                           
+INFO[0001] Sleeping for 1 seconds                       
+INFO[0002] Executing> sleep 1                           
+INFO[0002] Sleeping for 1 seconds                       
+INFO[0003] Executing> sleep 1                           
+INFO[0003] Sleeping for 1 seconds                       
+real    0m4.289s
+user    0m0.405s
+sys     0m0.050s
+```
+
 ### Error Handling
 
 By default, an error in a command will stop execution, when operating concurrently all commands will finish in that block, and then abort. In some cases, it may be the case that errors are unavoidable, in which case the **ignore_errors** block can be used. In the future more granular error handling could be implemented based on need.
diff --git a/external/runbooks/account-management.epcc.yml b/external/runbooks/account-management.epcc.yml
index 6de7aed..1c29ec2 100644
--- a/external/runbooks/account-management.epcc.yml
+++ b/external/runbooks/account-management.epcc.yml
@@ -10,6 +10,37 @@ actions:
       # Initialize alias for Authentication Realm
       - epcc get account-authentication-settings
       - epcc create password-profile related_authentication_realm_for_account_authentication_settings_last_read=entity name "Username and Password Authentication"
+  create-deep-hierarchy:
+    description:
+      short: "Create a hierarchy"
+    variables:
+      depth:
+        type: INT
+        default: 4
+        description:
+          short: "Depth of the hierarchy"
+      width:
+        type: INT
+        default: 2
+        description:
+          short: "Width of the hierarchy"
+    commands:
+      # language=YAML
+      - |2
+        {{- range untilStep 0 $.depth  1 -}}
+          {{- $d := . -}}
+          {{- $total := 1 -}}
+          {{- range untilStep 0 $d  1 -}}
+              {{- $total = mul $total $.width | int }}
+          {{ end }}
+          -
+          {{ range untilStep 0 $total 1 -}}
+          {{- $x := . }}
+            epcc create -s account name "Account_{{ $d }}_{{ $x }}" legal_name "Account at {{ $d }} {{ $x }}" {{ if ne $d 0 }} parent_id name=Account_{{ sub $d 1 }}_{{ div $x $.width }} {{ end }}
+          {{ end }}
+        {{ end }}
+
+
   create-singleton-account-member:
     description:
       short: "Create an account member with an account"
@@ -115,7 +146,7 @@ actions:
          epcc create pcm-catalog-release name=Account_1_Catalog --save-as-alias account-management-catalog-rule-account-1-catalog
          epcc create pcm-catalog-release name=Account_2_Catalog --save-as-alias account-management-catalog-rule-account-2-catalog
       # Wait for Catalogs to be Published
-      - | 
+      - |
         epcc get pcm-catalog-release --retry-while-jq '.data.meta.release_status != "PUBLISHED"' name=Account_1_Catalog account-management-catalog-rule-account-1-catalog
         epcc get pcm-catalog-release --retry-while-jq '.data.meta.release_status != "PUBLISHED"' name=Account_2_Catalog account-management-catalog-rule-account-2-catalog
       - epcc get pcm-catalog-releases name=Account_1_Catalog
@@ -127,7 +158,7 @@ actions:
       short: "Reset the store configuration"
     ignore_errors: true
     commands:
-      - | 
+      - |
         epcc delete accounts name=Account_1
         epcc delete accounts name=Account_2
         epcc delete pcm-catalog-rule name=Account_1_Rule
@@ -135,7 +166,7 @@ actions:
       - |
         epcc delete pcm-catalog-release name=Account_1_Catalog name=Account_1_Catalog
         epcc delete pcm-catalog-release name=Account_2_Catalog name=Account_2_Catalog
-      - | 
+      - |
         epcc delete pcm-catalog name=Account_1_Catalog
         epcc delete pcm-catalog name=Account_2_Catalog
       - |
@@ -148,7 +179,3 @@ actions:
         epcc delete pcm-pricebook name=Account_1_Pricebook
         epcc delete pcm-pricebook name=Account_2_Pricebook
         epcc delete pcm-pricebook name=Account_2_Pricebook
-      
-        
-      
-
diff --git a/external/runbooks/hello-world.epcc.yml b/external/runbooks/hello-world.epcc.yml
index bdb0a24..7d86f7a 100644
--- a/external/runbooks/hello-world.epcc.yml
+++ b/external/runbooks/hello-world.epcc.yml
@@ -48,6 +48,30 @@ actions:
         {{- range untilStep 0 .number_of_addresses 1 }}
         epcc create customer-address "{{$.customer_id }}" name "address_{{.}}" first_name "John" last_name "Smith" line_1 "1234 Main Street" county "XX" "postcode" "H0H 0H0" country "{{$.country}}"
         {{- end -}}
+  sequential-sleeps:
+    variables:
+      count:
+        type: INT
+        default: 2
+        description:
+          short: "The number of sleeps"
+    commands:
+      - |2
+        {{- range untilStep 0 .count 1}}
+          - sleep 1
+        {{- end -}}
+  concurrent-sleeps:
+    variables:
+      count:
+        type: INT
+        default: 2
+        description:
+          short: "The number of sleeps"
+    commands:
+      - |
+        {{- range untilStep 0 .count 1}}
+        sleep 1
+        {{- end -}}
   reset:
     ignore_errors: true
     commands:
diff --git a/external/runbooks/run-all-runbooks.sh b/external/runbooks/run-all-runbooks.sh
index 25cc8d1..88fec3f 100755
--- a/external/runbooks/run-all-runbooks.sh
+++ b/external/runbooks/run-all-runbooks.sh
@@ -38,6 +38,9 @@ epcc runbooks run hello-world create-10-customers
 epcc create customer --auto-fill
 epcc runbooks run hello-world create-some-customer-addresses --customer_id last_read=entity
 
+epcc runbooks run hello-world concurrent-sleeps  --count 2
+epcc runbooks run hello-world sequential-sleeps  --count 2
+
 epcc runbooks run hello-world reset
 
 echo "Starting Extend Customer Resources"
diff --git a/external/runbooks/runbook_validation.go b/external/runbooks/runbook_validation.go
index 5506483..78bdfe2 100644
--- a/external/runbooks/runbook_validation.go
+++ b/external/runbooks/runbook_validation.go
@@ -3,6 +3,8 @@ package runbooks
 import (
 	"fmt"
 	"github.com/buildkite/shellwords"
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v3"
 	"strconv"
 	"strings"
 )
@@ -24,7 +26,9 @@ func ValidateRunbook(runbook *Runbook) error {
 
 		argumentsWithDefaults := CreateMapForRunbookArgumentPointers(runbookAction)
 
-		for stepIdx, rawCmd := range runbookAction.RawCommands {
+		cmds := runbookAction.RawCommands
+		for stepIdx := 0; stepIdx < len(cmds); stepIdx++ {
+			rawCmd := cmds[stepIdx]
 
 			templateName := fmt.Sprintf("Runbook: %s Action: %s Step: %d", runbook.Name, runbookAction.Name, stepIdx+1)
 			rawCmdLines, err := RenderTemplates(templateName, rawCmd, argumentsWithDefaults, runbookAction.Variables)
@@ -33,6 +37,21 @@ func ValidateRunbook(runbook *Runbook) error {
 				return fmt.Errorf("error rendering template: %w", err)
 			}
 
+			joinedString := strings.Join(rawCmdLines, "\n")
+			renderedCmd := []string{}
+			err = yaml.Unmarshal([]byte(joinedString), &renderedCmd)
+
+			if err == nil {
+				log.Tracef("Line %d is a Yaml array %s, inserting into stack", stepIdx, joinedString)
+				newCmds := make([]string, 0, len(cmds)+len(renderedCmd)-1)
+				newCmds = append(newCmds, cmds[0:stepIdx]...)
+				newCmds = append(newCmds, renderedCmd...)
+				newCmds = append(newCmds, cmds[stepIdx+1:]...)
+				cmds = newCmds
+				stepIdx--
+				continue
+			}
+
 			for commandIdx, rawCmdLine := range rawCmdLines {
 				rawCmdLine := strings.Trim(rawCmdLine, " \n")