From a14920ba0136dfc15aa0ea87bf839706f4170651 Mon Sep 17 00:00:00 2001 From: Steve Ramage <49958178+steve-r-west@users.noreply.github.com> Date: Sun, 7 Jan 2024 10:45:40 -0800 Subject: [PATCH] Resolves #431 - Add support for dynamic runbook steps (#434) * Resolves #431 - Add support for dynamic steps * Resolves #431 - Add support for dynamic runbook steps --- .github/workflows/test.yml | 1 + README.md | 6 -- cmd/root.go | 2 +- cmd/runbooks.go | 49 +++++++++++-- docs/runbook-development.md | 68 +++++++++++++++++++ external/runbooks/account-management.epcc.yml | 41 +++++++++-- external/runbooks/hello-world.epcc.yml | 24 +++++++ external/runbooks/run-all-runbooks.sh | 3 + external/runbooks/runbook_validation.go | 21 +++++- 9 files changed, 195 insertions(+), 20 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e90933a..0f50bfd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,6 +18,7 @@ jobs: name: Build runs-on: ubuntu-latest timeout-minutes: 15 + concurrency: github steps: - name: Set up Go uses: actions/setup-go@v5 diff --git a/README.md b/README.md index f97d9b7..3c8d1f5 100644 --- a/README.md +++ b/README.md @@ -242,9 +242,3 @@ go fmt "./..." echo "Adding changed files back to git" git diff --cached --name-only --diff-filter=ACM | grep -E "\.(go)$" | xargs git add ``` - -### Generating Go Routine Dumps - -You can send a `SIGQUIT` to `epcc` to get it to generate a go routine dump, this can be useful for debugging deadlocks and other performance issues. - -On Linux and maybe other OS, you can use CTRL+\ to send a `SIGQUIT` to the process. diff --git a/cmd/root.go b/cmd/root.go index 9162403..4148672 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -299,7 +299,7 @@ func DumpTraces() { for { <-sigs stacklen := runtime.Stack(buf, true) - fmt.Printf("=== received SIGQUIT ===\n*** goroutine dump...\n%s\n*** end\n", buf[:stacklen]) + log.Printf("=== received SIGQUIT ===\n*** goroutine dump...\n%s\n*** end\n", buf[:stacklen]) } }() } diff --git a/cmd/runbooks.go b/cmd/runbooks.go index 586bd3e..ceff758 100644 --- a/cmd/runbooks.go +++ b/cmd/runbooks.go @@ -14,6 +14,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "golang.org/x/sync/semaphore" + "gopkg.in/yaml.v3" "strconv" "strings" "sync/atomic" @@ -120,13 +121,31 @@ func initRunbookShowCommands() *cobra.Command { Short: runbookAction.Description.Short, RunE: func(cmd *cobra.Command, args []string) error { - for stepIdx, cmd := range runbookAction.RawCommands { + cmds := runbookAction.RawCommands + for stepIdx := 0; stepIdx < len(cmds); stepIdx++ { + cmd := cmds[stepIdx] templateName := fmt.Sprintf("Runbook: %s Action: %s Step: %d", runbook.Name, runbookAction.Name, stepIdx) - rawCmdLines, err := runbooks.RenderTemplates(templateName, cmd, runbookStringArguments, runbookAction.Variables) + if err != nil { return err } + + joinedString := strings.Join(rawCmdLines, "\n") + renderedCmd := []string{} + err = yaml.Unmarshal([]byte(joinedString), &renderedCmd) + + if err == nil { + log.Tracef("Line %d is a Yaml array %s, inserting into stack", stepIdx, joinedString) + newCmds := make([]string, 0, len(cmds)+len(renderedCmd)-1) + newCmds = append(newCmds, cmds[0:stepIdx]...) + newCmds = append(newCmds, renderedCmd...) + newCmds = append(newCmds, cmds[stepIdx+1:]...) + cmds = newCmds + stepIdx-- + continue + } + for _, line := range rawCmdLines { if len(strings.Trim(line, " \n")) > 0 { println(line) @@ -224,13 +243,15 @@ func initRunbookRunCommands() *cobra.Command { MaxTotal: *maxConcurrency, MaxIdle: *maxConcurrency, }) - for stepIdx, rawCmd := range runbookAction.RawCommands { + rawCmds := runbookAction.RawCommands + for stepIdx := 0; stepIdx < len(rawCmds); stepIdx++ { + + origIndex := &stepIdx // Create a copy of loop variables stepIdx := stepIdx - rawCmd := rawCmd + rawCmd := rawCmds[stepIdx] - log.Infof("Executing> %s", rawCmd) templateName := fmt.Sprintf("Runbook: %s Action: %s Step: %d", runbook.Name, runbookAction.Name, stepIdx) rawCmdLines, err := runbooks.RenderTemplates(templateName, rawCmd, runbookStringArguments, runbookAction.Variables) @@ -238,6 +259,24 @@ func initRunbookRunCommands() *cobra.Command { cancelFunc() return err } + + joinedString := strings.Join(rawCmdLines, "\n") + renderedCmd := []string{} + + err = yaml.Unmarshal([]byte(joinedString), &renderedCmd) + + if err == nil { + log.Tracef("Line %d is a Yaml array %s, inserting into stack", stepIdx, joinedString) + newCmds := make([]string, 0, len(rawCmds)+len(renderedCmd)-1) + newCmds = append(newCmds, rawCmds[0:stepIdx]...) + newCmds = append(newCmds, renderedCmd...) + newCmds = append(newCmds, rawCmds[stepIdx+1:]...) + rawCmds = newCmds + *origIndex-- + continue + } + + log.Infof("Executing> %s", rawCmd) resultChan := make(chan *commandResult, *maxConcurrency*2) funcs := make([]func(), 0, len(rawCmdLines)) diff --git a/docs/runbook-development.md b/docs/runbook-development.md index 5b53840..021e887 100644 --- a/docs/runbook-development.md +++ b/docs/runbook-development.md @@ -4,6 +4,8 @@ This document outlines the syntax and capabilities of runbooks. + + ## Prerequisites Users developing runbooks should have familiarity with: @@ -158,6 +160,72 @@ epcc create customer-address "Hello World" name "address_0" first_name "John" l epcc create customer-address "Hello World" name "address_1" first_name "John" last_name "Smith" line_1 "1234 Main Street" county "XX" "postcode" "H0H 0H0" country "DE" ``` +### Dynamic Steps + +Templates are only renderable for a particular step, but you can instead of generating a command, actually generate a Yaml array, that will be interpreted as steps. +This can give you more control over control flow, where-as otherwise you are stuck with all rendered commands being in the same sequence. + +```yaml +name: hello-world +description: + short: "A hello world runbook" +actions: + sequential-sleeps: + variables: + count: + type: INT + default: 2 + description: + short: "The number of sleeps" + commands: + - |2 + {{- range untilStep 0 .count 1}} + - sleep 1 + {{- end -}} + concurrent-sleeps: + variables: + count: + type: INT + default: 2 + description: + short: "The number of sleeps" + commands: + - | + {{- range untilStep 0 .count 1}} + sleep 1 + {{- end -}} +``` + +It's important for all commands to be indended the same amount, and start with a `-` this will cause the entire string to be valid as a Yaml array. + +You can see the results of executing these runbooks below, without the `-` the commands all execute concurrently. With the `-` the execute one after another. + +```bash +$time epcc runbooks run hello-world concurrent-sleeps --count 4 +INFO[0000] Executing> {{- range untilStep 0 .count 1}} +sleep 1 +{{- end -}} +INFO[0000] Sleeping for 1 seconds +INFO[0000] Sleeping for 1 seconds +INFO[0000] Sleeping for 1 seconds +INFO[0000] Sleeping for 1 seconds +real 0m1.302s +user 0m0.441s +sys 0m0.059s +$time epcc runbooks run hello-world sequential-sleeps --count 4 +INFO[0000] Executing> sleep 1 +INFO[0000] Sleeping for 1 seconds +INFO[0001] Executing> sleep 1 +INFO[0001] Sleeping for 1 seconds +INFO[0002] Executing> sleep 1 +INFO[0002] Sleeping for 1 seconds +INFO[0003] Executing> sleep 1 +INFO[0003] Sleeping for 1 seconds +real 0m4.289s +user 0m0.405s +sys 0m0.050s +``` + ### Error Handling By default, an error in a command will stop execution, when operating concurrently all commands will finish in that block, and then abort. In some cases, it may be the case that errors are unavoidable, in which case the **ignore_errors** block can be used. In the future more granular error handling could be implemented based on need. diff --git a/external/runbooks/account-management.epcc.yml b/external/runbooks/account-management.epcc.yml index 6de7aed..1c29ec2 100644 --- a/external/runbooks/account-management.epcc.yml +++ b/external/runbooks/account-management.epcc.yml @@ -10,6 +10,37 @@ actions: # Initialize alias for Authentication Realm - epcc get account-authentication-settings - epcc create password-profile related_authentication_realm_for_account_authentication_settings_last_read=entity name "Username and Password Authentication" + create-deep-hierarchy: + description: + short: "Create a hierarchy" + variables: + depth: + type: INT + default: 4 + description: + short: "Depth of the hierarchy" + width: + type: INT + default: 2 + description: + short: "Width of the hierarchy" + commands: + # language=YAML + - |2 + {{- range untilStep 0 $.depth 1 -}} + {{- $d := . -}} + {{- $total := 1 -}} + {{- range untilStep 0 $d 1 -}} + {{- $total = mul $total $.width | int }} + {{ end }} + - + {{ range untilStep 0 $total 1 -}} + {{- $x := . }} + epcc create -s account name "Account_{{ $d }}_{{ $x }}" legal_name "Account at {{ $d }} {{ $x }}" {{ if ne $d 0 }} parent_id name=Account_{{ sub $d 1 }}_{{ div $x $.width }} {{ end }} + {{ end }} + {{ end }} + + create-singleton-account-member: description: short: "Create an account member with an account" @@ -115,7 +146,7 @@ actions: epcc create pcm-catalog-release name=Account_1_Catalog --save-as-alias account-management-catalog-rule-account-1-catalog epcc create pcm-catalog-release name=Account_2_Catalog --save-as-alias account-management-catalog-rule-account-2-catalog # Wait for Catalogs to be Published - - | + - | epcc get pcm-catalog-release --retry-while-jq '.data.meta.release_status != "PUBLISHED"' name=Account_1_Catalog account-management-catalog-rule-account-1-catalog epcc get pcm-catalog-release --retry-while-jq '.data.meta.release_status != "PUBLISHED"' name=Account_2_Catalog account-management-catalog-rule-account-2-catalog - epcc get pcm-catalog-releases name=Account_1_Catalog @@ -127,7 +158,7 @@ actions: short: "Reset the store configuration" ignore_errors: true commands: - - | + - | epcc delete accounts name=Account_1 epcc delete accounts name=Account_2 epcc delete pcm-catalog-rule name=Account_1_Rule @@ -135,7 +166,7 @@ actions: - | epcc delete pcm-catalog-release name=Account_1_Catalog name=Account_1_Catalog epcc delete pcm-catalog-release name=Account_2_Catalog name=Account_2_Catalog - - | + - | epcc delete pcm-catalog name=Account_1_Catalog epcc delete pcm-catalog name=Account_2_Catalog - | @@ -148,7 +179,3 @@ actions: epcc delete pcm-pricebook name=Account_1_Pricebook epcc delete pcm-pricebook name=Account_2_Pricebook epcc delete pcm-pricebook name=Account_2_Pricebook - - - - diff --git a/external/runbooks/hello-world.epcc.yml b/external/runbooks/hello-world.epcc.yml index bdb0a24..7d86f7a 100644 --- a/external/runbooks/hello-world.epcc.yml +++ b/external/runbooks/hello-world.epcc.yml @@ -48,6 +48,30 @@ actions: {{- range untilStep 0 .number_of_addresses 1 }} epcc create customer-address "{{$.customer_id }}" name "address_{{.}}" first_name "John" last_name "Smith" line_1 "1234 Main Street" county "XX" "postcode" "H0H 0H0" country "{{$.country}}" {{- end -}} + sequential-sleeps: + variables: + count: + type: INT + default: 2 + description: + short: "The number of sleeps" + commands: + - |2 + {{- range untilStep 0 .count 1}} + - sleep 1 + {{- end -}} + concurrent-sleeps: + variables: + count: + type: INT + default: 2 + description: + short: "The number of sleeps" + commands: + - | + {{- range untilStep 0 .count 1}} + sleep 1 + {{- end -}} reset: ignore_errors: true commands: diff --git a/external/runbooks/run-all-runbooks.sh b/external/runbooks/run-all-runbooks.sh index 25cc8d1..88fec3f 100755 --- a/external/runbooks/run-all-runbooks.sh +++ b/external/runbooks/run-all-runbooks.sh @@ -38,6 +38,9 @@ epcc runbooks run hello-world create-10-customers epcc create customer --auto-fill epcc runbooks run hello-world create-some-customer-addresses --customer_id last_read=entity +epcc runbooks run hello-world concurrent-sleeps --count 2 +epcc runbooks run hello-world sequential-sleeps --count 2 + epcc runbooks run hello-world reset echo "Starting Extend Customer Resources" diff --git a/external/runbooks/runbook_validation.go b/external/runbooks/runbook_validation.go index 5506483..78bdfe2 100644 --- a/external/runbooks/runbook_validation.go +++ b/external/runbooks/runbook_validation.go @@ -3,6 +3,8 @@ package runbooks import ( "fmt" "github.com/buildkite/shellwords" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" "strconv" "strings" ) @@ -24,7 +26,9 @@ func ValidateRunbook(runbook *Runbook) error { argumentsWithDefaults := CreateMapForRunbookArgumentPointers(runbookAction) - for stepIdx, rawCmd := range runbookAction.RawCommands { + cmds := runbookAction.RawCommands + for stepIdx := 0; stepIdx < len(cmds); stepIdx++ { + rawCmd := cmds[stepIdx] templateName := fmt.Sprintf("Runbook: %s Action: %s Step: %d", runbook.Name, runbookAction.Name, stepIdx+1) rawCmdLines, err := RenderTemplates(templateName, rawCmd, argumentsWithDefaults, runbookAction.Variables) @@ -33,6 +37,21 @@ func ValidateRunbook(runbook *Runbook) error { return fmt.Errorf("error rendering template: %w", err) } + joinedString := strings.Join(rawCmdLines, "\n") + renderedCmd := []string{} + err = yaml.Unmarshal([]byte(joinedString), &renderedCmd) + + if err == nil { + log.Tracef("Line %d is a Yaml array %s, inserting into stack", stepIdx, joinedString) + newCmds := make([]string, 0, len(cmds)+len(renderedCmd)-1) + newCmds = append(newCmds, cmds[0:stepIdx]...) + newCmds = append(newCmds, renderedCmd...) + newCmds = append(newCmds, cmds[stepIdx+1:]...) + cmds = newCmds + stepIdx-- + continue + } + for commandIdx, rawCmdLine := range rawCmdLines { rawCmdLine := strings.Trim(rawCmdLine, " \n")