From 435569b182f687c7b40d3873b7d2ee828489bd34 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Thu, 10 Nov 2016 12:46:06 -0800 Subject: [PATCH] support for setup/teardown commands in cmd tests (#49) * add ability to substitute env vars into command output/error * add support for setup/teardown commands. remove flags field in favor of just providing command string. * fix file/directory existence logging * gofmt * change build script to take entire image path instead of just tag * add back flags field * remove env vars field. hardcode replacing PWD in output/error regexes * remove flags field. change command field to a list of strings, containing flags * unify running setup/teardown/normal commands. restructure methods * update documentation * update tag check test config * remove duplicate code. fix readme * remove envvar substitution function and move PWD hack to regex compiler * remove pwd hack --- check_if_image_tag_exists/structure_test.yaml | 6 +- structure_tests/README.md | 23 ++-- structure_tests/build.sh | 9 +- structure_tests/cloudbuild.yaml.in | 4 +- structure_tests/command_test_v1.go | 7 +- structure_tests/structure_test_v1.go | 117 ++++++++++++------ 6 files changed, 100 insertions(+), 66 deletions(-) diff --git a/check_if_image_tag_exists/structure_test.yaml b/check_if_image_tag_exists/structure_test.yaml index fcfa19c018..ba9113c8f4 100644 --- a/check_if_image_tag_exists/structure_test.yaml +++ b/check_if_image_tag_exists/structure_test.yaml @@ -1,13 +1,11 @@ commandTests: - name: 'uname' - command: 'uname' - flags: '-s' + command: ['uname', '-s'] expectedError: ['foo'] expectedOutput: ['Linux\n'] - name: 'broken uname' - command: 'uname' - flags: '-s -asdf' + command: ['uname', '-s', '-asdf'] expectedError: ['.*[invalid | unrecognized].*'] expectedOutput: [] diff --git a/structure_tests/README.md b/structure_tests/README.md index e3cdef6219..e503b42974 100644 --- a/structure_tests/README.md +++ b/structure_tests/README.md @@ -23,8 +23,9 @@ Command tests ensure that certain commands run properly on top of the shell of t #### Supported Fields: - Name (string, **required**): The name of the test -- Command (string, **required**): The command to run -- Flags (string[], *optional*): Optional list of flags to pass to the command +- Setup ([][]string, *optional*): A list of commands (each with optional flags) to run before the actual command under test. +- Teardown ([][]string, *optional*): A list of commands (each with optional flags) to run after the actual command under test. +- Command ([]string, **required**): The command to run, along with the flags to pass to it. - Expected Output (string[], *optional*): List of regexes that should match the stdout from running the command. - Excluded Output (string[], *optional*): List of regexes that should **not** match the stdout from running the command. - Expected Error (string[], *optional*): List of regexes that should match the stderr from running the command. @@ -34,17 +35,16 @@ Example: ```json "commandTests": [ { - "name": "apt-get", - "command": "apt-get", - "flags": ["help"], - "expectedOutput": [".*Usage.*"], - "excludedError": [".*FAIL.*"] - },{ "name": "apt-get upgrade", - "command": "apt-get", - "flags": ["-qqs", "upgrade"], + "command": ["apt-get", "-qqs", "upgrade"], "excludedOutput": [".*Inst.*Security.* | .*Security.*Inst.*"], "excludedError": [".*Inst.*Security.* | .*Security.*Inst.*"] + },{ + "name": "Custom Node Version", + "setup": [["install_node", "v5.9.0"]], + "teardown": [["install_node", "v6.9.1"]], + "command": ["node", "-v"], + "expectedOutput": ["v5.9.0\n"] } ] ``` @@ -52,8 +52,7 @@ Example: ```yaml commandTests: - name: 'apt-get' - command: 'apt-get' - flags: 'help' + command: ['apt-get', 'help'] expectedError: ['.*Usage.*'] excludedError: ['*FAIL.*'] ``` diff --git a/structure_tests/build.sh b/structure_tests/build.sh index 522ebb89ec..f14466385f 100755 --- a/structure_tests/build.sh +++ b/structure_tests/build.sh @@ -14,13 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +usage() { echo "Usage: ./build.sh [target_image_path]"; exit 1; } + set -e -export VERSION=$1 +export IMAGE=$1 -if [ -z "$1" ]; then - echo "Please provide valid version to tag image." - exit 1 +if [ -z "$IMAGE" ]; then + usage fi envsubst < cloudbuild.yaml.in > cloudbuild.yaml diff --git a/structure_tests/cloudbuild.yaml.in b/structure_tests/cloudbuild.yaml.in index 052e551736..d7d15b1014 100644 --- a/structure_tests/cloudbuild.yaml.in +++ b/structure_tests/cloudbuild.yaml.in @@ -3,6 +3,6 @@ steps: args: ['test', '-c', 'github.com/GoogleCloudPlatform/runtimes-common/structure_tests', '-o', '/workspace/structure_tests/structure_test'] env: ['PROJECT_ROOT=github.com/GoogleCloudPlatform/runtimes-common'] - name: gcr.io/cloud-builders/docker - args: ['build', '-t', 'gcr.io/gcp-runtimes/structure_test:${VERSION}', './structure_tests'] + args: ['build', '-t', '${IMAGE}', './structure_tests'] images: - - 'gcr.io/gcp-runtimes/structure_test:${VERSION}' + - '${IMAGE}' diff --git a/structure_tests/command_test_v1.go b/structure_tests/command_test_v1.go index 257b687c03..560fc5d622 100644 --- a/structure_tests/command_test_v1.go +++ b/structure_tests/command_test_v1.go @@ -20,8 +20,9 @@ import ( type CommandTestv1 struct { Name string - Command string - Flags []string + Setup [][]string + Teardown [][]string + Command []string ExpectedOutput []string ExcludedOutput []string ExpectedError []string @@ -32,7 +33,7 @@ func validateCommandTestV1(t *testing.T, tt CommandTestv1) { if tt.Name == "" { t.Fatalf("Please provide a valid name for every test!") } - if tt.Command == "" { + if tt.Command == nil || len(tt.Command) == 0 { t.Fatalf("Please provide a valid command to run for test %s", tt.Name) } t.Logf("COMMAND TEST: %s", tt.Name) diff --git a/structure_tests/structure_test_v1.go b/structure_tests/structure_test_v1.go index 8e38202b41..8c8b8757c6 100644 --- a/structure_tests/structure_test_v1.go +++ b/structure_tests/structure_test_v1.go @@ -37,48 +37,15 @@ func (st StructureTestv1) RunAll(t *testing.T) { func (st StructureTestv1) RunCommandTests(t *testing.T) { for _, tt := range st.CommandTests { validateCommandTestV1(t, tt) - var cmd *exec.Cmd - if tt.Flags != nil && len(tt.Flags) > 0 { - cmd = exec.Command(tt.Command, tt.Flags...) - } else { - cmd = exec.Command(tt.Command) - } - t.Logf("Executing: %s", cmd.Args) - var outbuf, errbuf bytes.Buffer - - cmd.Stdout = &outbuf - cmd.Stderr = &errbuf - - if err := cmd.Run(); err != nil { - // The test might be designed to run a command that exits with an error. - t.Logf("Error running command: %s. Continuing.", err) - } - - stdout := outbuf.String() - if stdout != "" { - t.Logf("stdout: %s", stdout) - } - stderr := errbuf.String() - if stderr != "" { - t.Logf("stderr: %s", stderr) + for _, setup := range tt.Setup { + ProcessCommand(t, setup, false) } - for _, errStr := range tt.ExpectedError { - errMsg := fmt.Sprintf("Expected string '%s' not found in error!", errStr) - compileAndRunRegex(errStr, stderr, t, errMsg, true) - } - for _, errStr := range tt.ExcludedError { - errMsg := fmt.Sprintf("Excluded string '%s' found in error!", errStr) - compileAndRunRegex(errStr, stderr, t, errMsg, false) - } + stdout, stderr := ProcessCommand(t, tt.Command, true) + CheckOutput(t, tt, stdout, stderr) - for _, outStr := range tt.ExpectedOutput { - errMsg := fmt.Sprintf("Expected string '%s' not found in output!", outStr) - compileAndRunRegex(outStr, stdout, t, errMsg, true) - } - for _, outStr := range tt.ExcludedOutput { - errMsg := fmt.Sprintf("Excluded string '%s' found in output!", outStr) - compileAndRunRegex(outStr, stdout, t, errMsg, false) + for _, teardown := range tt.Teardown { + ProcessCommand(t, teardown, false) } } } @@ -93,9 +60,17 @@ func (st StructureTestv1) RunFileExistenceTests(t *testing.T) { _, err = ioutil.ReadFile(tt.Path) } if tt.ShouldExist && err != nil { - t.Errorf("File %s should exist but does not!", tt.Path) + if tt.IsDirectory { + t.Errorf("Directory %s should exist but does not!", tt.Path) + } else { + t.Errorf("File %s should exist but does not!", tt.Path) + } } else if !tt.ShouldExist && err == nil { - t.Errorf("File %s should not exist but does!", tt.Path) + if tt.IsDirectory { + t.Errorf("Directory %s should not exist but does!", tt.Path) + } else { + t.Errorf("File %s should not exist but does!", tt.Path) + } } } } @@ -121,3 +96,63 @@ func (st StructureTestv1) RunFileContentTests(t *testing.T) { } } } + +func ProcessCommand(t *testing.T, fullCommand []string, checkOutput bool) (string, string) { + var cmd *exec.Cmd + command := fullCommand[0] + flags := fullCommand[1:] + if len(flags) > 0 { + cmd = exec.Command(command, flags...) + } else { + cmd = exec.Command(command) + } + + if checkOutput { + t.Logf("Executing: %s", cmd.Args) + } else { + t.Logf("Executing setup/teardown: %s", cmd.Args) + } + + var outbuf, errbuf bytes.Buffer + + cmd.Stdout = &outbuf + cmd.Stderr = &errbuf + + err := cmd.Run() + stdout := outbuf.String() + if stdout != "" { + t.Logf("stdout: %s", stdout) + } + stderr := errbuf.String() + if stderr != "" { + t.Logf("stderr: %s", stderr) + } + if err != nil { + if checkOutput { + // The test might be designed to run a command that exits with an error. + t.Logf("Error running command: %s. Continuing.", err) + } else { + t.Fatalf("Error running setup/teardown command: %s.", err) + } + } + return stdout, stderr +} + +func CheckOutput(t *testing.T, tt CommandTestv1, stdout string, stderr string) { + for _, errStr := range tt.ExpectedError { + errMsg := fmt.Sprintf("Expected string '%s' not found in error!", errStr) + compileAndRunRegex(errStr, stderr, t, errMsg, true) + } + for _, errStr := range tt.ExcludedError { + errMsg := fmt.Sprintf("Excluded string '%s' found in error!", errStr) + compileAndRunRegex(errStr, stderr, t, errMsg, false) + } + for _, outStr := range tt.ExpectedOutput { + errMsg := fmt.Sprintf("Expected string '%s' not found in output!", outStr) + compileAndRunRegex(outStr, stdout, t, errMsg, true) + } + for _, outStr := range tt.ExcludedOutput { + errMsg := fmt.Sprintf("Excluded string '%s' found in output!", outStr) + compileAndRunRegex(outStr, stdout, t, errMsg, false) + } +}