diff --git a/.github/workflows/acceptance-tests-pr.yml b/.github/workflows/acceptance-tests-pr.yml new file mode 100644 index 0000000..768276d --- /dev/null +++ b/.github/workflows/acceptance-tests-pr.yml @@ -0,0 +1,23 @@ +name: acceptance-tests-pr + +on: + pull_request: + branches: + - master + +jobs: + run: + runs-on: ubuntu-latest + steps: + - name: Checkout source code + uses: actions/checkout@v1 + + - name: Run acceptance tests + run: make github-actions-ci + + - uses: actions/upload-artifact@master + name: Upload test report + with: + name: helm-acceptance-testing-report-${{ github.sha }} + path: acceptance-testing-reports/${{ github.sha }}/ + if: always() diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml new file mode 100644 index 0000000..44e7530 --- /dev/null +++ b/.github/workflows/acceptance-tests.yml @@ -0,0 +1,22 @@ +name: acceptance-tests + +on: + push: + branches: + - master +jobs: + run: + runs-on: ubuntu-latest + steps: + - name: Checkout source code + uses: actions/checkout@v1 + + - name: Run acceptance tests + run: make github-actions-ci + + - uses: actions/upload-artifact@master + name: Upload test report + with: + name: helm-acceptance-testing-report-${{ github.sha }} + path: acceptance-testing-reports/${{ github.sha }}/ + if: always() diff --git a/.gitignore b/.gitignore index 56ae284..b9da8cd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ .acceptance/ .idea/ + +acceptance-testing-reports/ +bin/ diff --git a/Makefile b/Makefile index 37a72c5..e175d6b 100644 --- a/Makefile +++ b/Makefile @@ -3,3 +3,16 @@ SHELL = /bin/bash .PHONY: acceptance acceptance: @scripts/acceptance.sh + +.PHONY: github-actions-ci +github-actions-ci: + @scripts/github-actions-ci.sh + +.PHONY: github-actions-ci-local +github-actions-ci-local: + docker run -it --rm \ + -v $(shell pwd):/tmp/acceptance-testing \ + -w /tmp/acceptance-testing \ + --privileged -v /var/run/docker.sock:/var/run/docker.sock \ + --entrypoint=/bin/bash ubuntu:latest \ + -c 'set +e; scripts/github-actions-ci.sh; echo "Exited $?. (Ctrl+D to exit shell)"; bash' \ No newline at end of file diff --git a/README.md b/README.md index ea889d9..2848f19 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,51 @@ # Helm Acceptance Tests -*Note: these tests have only been run against Helm 3 ([dev-v3](https://github.com/helm/helm/tree/dev-v3))* +[![GitHub Actions status](https://github.com/helm/acceptance-testing/workflows/acceptance-tests/badge.svg)](https://github.com/helm/acceptance-testing/actions) This repo contains the source for Helm acceptance tests. - The tests are written using [Robot Framework](https://robotframework.org/). +*Note: these tests have only been run against Helm 3 ([dev-v3](https://github.com/helm/helm/tree/dev-v3))* + +## Test Summary + +### Kubernetes Versions + +Helm is tested to work against the following versions of Kubernetes: + + + +- [1.15.3](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.15.md) +- [1.14.6](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md) + +Test suite: [kubernetes_versions.robot](./testsuites/kubernetes_versions.robot) + + +### Shell Completion + +Helm's shell completion functionality is tested against the following shells: + +- Bash +- Zsh + +Test suite: [shells.robot](./testsuites/shells.robot) + +### Helm Repositories + +Basic functionality of the chart repository subsystem is tested. + +Test suite: [repos.robot](./testsuites/repos.robot) + ## System requirements The following tools/commands are expected to be present on the base system @@ -25,6 +65,13 @@ From the root of this repo, run the following: make acceptance ``` +Alternatively, if you have Docker installed, +the system requirements above are not needed, and you can run the following +command which will simulate CI: +``` +make github-actions-ci-local +``` + Note: by default, the tests will use helm as found on your PATH. To specify a different helm to test, set and export the `ROBOT_HELM_PATH` environment variable. For example, if you have helm v2 installed, but want @@ -107,7 +154,6 @@ contains a base class called `CommandRunner` that you will likely want to leverage when adding support for a new external tool. The test run is wrapped by [acceptance.sh](./scripts/acceptance.sh) - -in this file the environment is validated (i.e. check if required tools present). - -sinstalled (including Robot Framework itself). If any additional Python libraries -are required for a new library, it can be appended to `ROBOT_PY_REQUIRES`. +in this file the environment is validated (i.e. check if required tools present). +If any additional Python libraries are required for a new library, +it can be appended to `ROBOT_PY_REQUIRES`. diff --git a/lib/Kind.py b/lib/Kind.py index 8129f3b..90e0215 100644 --- a/lib/Kind.py +++ b/lib/Kind.py @@ -9,7 +9,7 @@ MAX_WAIT_KIND_NODE_SECONDS = 60 KIND_NODE_INTERVAL_SECONDS = 2 -MAX_WAIT_KIND_POD_SECONDS = 60 +MAX_WAIT_KIND_POD_SECONDS = 120 KIND_POD_INTERVAL_SECONDS = 2 KIND_POD_EXPECTED_NUMBER = 8 @@ -39,6 +39,12 @@ def create_test_cluster_with_kubernetes_version(self, kube_version): cmd += ' --image='+DOCKER_HUB_REPO+':v'+kube_version self.run_command(cmd) + # Fix for running kind in docker, switch the port+IP in the kubeconfig + if os.path.exists('/.dockerenv'): + print('Running in Docker, modifying IP in kubeconfig') + fixcmd = 'export KIND_IP=$(docker inspect '+LAST_CLUSTER_NAME+'-control-plane | grep \'IPAddress": "\' | head -1 | awk \'{print $2}\' | tr -d \\",) && sed -i "s/https:\/\/127\.0\.0\.1:.*/https:\/\/${KIND_IP}:6443/" $(kind get kubeconfig-path --name="'+LAST_CLUSTER_NAME+'")' + self.run_command(fixcmd) + def delete_test_cluster(self): if LAST_CLUSTER_EXISTING: print('Not deleting cluster (cluster existed prior to test run)') diff --git a/scripts/acceptance.sh b/scripts/acceptance.sh index 275f807..324e5f5 100755 --- a/scripts/acceptance.sh +++ b/scripts/acceptance.sh @@ -1,4 +1,18 @@ #!/bin/bash -e +# +# Copyright The Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # Turn on debug printouts if the user requested a debug level >= $1 set_shell_debug_level() @@ -64,7 +78,8 @@ set_shell_debug_level 2 # Only use the -d flag for mktemp as many other flags don't # work on every plateform -export TMP_DIR=$(mktemp -d ${ROBOT_OUTPUT_DIR}/helm-acceptance.XXXXXX) +mkdir -p ${ROBOT_OUTPUT_DIR} +export TMP_DIR="$(mktemp -d ${ROBOT_OUTPUT_DIR}/helm-acceptance.XXXXXX)" trap "rm -rf ${TMP_DIR}" EXIT SUITES_TO_RUN="" diff --git a/scripts/completion-tests/test-completion.sh b/scripts/completion-tests/test-completion.sh index 8b29c21..6a94c0f 100755 --- a/scripts/completion-tests/test-completion.sh +++ b/scripts/completion-tests/test-completion.sh @@ -20,6 +20,17 @@ set -e SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") +# TODO: this is redeclared, but shouldnt have to be? +# getting error "scripts/completion-tests/test-completion.sh: line 23: set_shell_debug_level: command not found" +set_shell_debug_level() +{ + set +x + if [ $ROBOT_DEBUG_LEVEL -ge $1 ]; then + set -x + fi +} +export -f set_shell_debug_level + set_shell_debug_level 2 BINARY_NAME=helm @@ -46,12 +57,18 @@ mkdir -p ${COMP_DIR}/bin cp ${SCRIPT_DIR}/${COMP_SCRIPT_NAME} ${COMP_DIR} cp ${SCRIPT_DIR}/lib/completionTests-base.sh ${COMP_DIR}/lib -if ! [ -f ${BINARY_PATH_DOCKER}/${BINARY_NAME} ]; then - echo "These tests require a helm binary located at ${BINARY_PATH_DOCKER}/${BINARY_NAME}" - echo "Hint: Run 'make build-cross' in a clone of helm repo" - exit 2 +if [[ "${GITHUB_SHA}" == "" ]]; then + CHECK_BINARY_PATH="$(cd ${BINARY_PATH_DOCKER} && pwd)/${BINARY_NAME}" + if [[ ! -f ${CHECK_BINARY_PATH} ]] && [[ -L ${CHECK_BINARY_PATH} ]]; then + echo "These tests require a helm binary located at ${CHECK_BINARY_PATH}" + echo "Hint: Run 'make build-cross' in a clone of helm repo" + exit 2 + fi + cp ${CHECK_BINARY_PATH} ${COMP_DIR}/bin +else + echo "Running on GitHub Actions CI - using system-wide Helm 3 binary." + cp $(which helm) ${COMP_DIR}/bin fi -cp ${BINARY_PATH_DOCKER}/${BINARY_NAME} ${COMP_DIR}/bin # kubectl stub cat > ${COMP_DIR}/bin/kubectl << EOF diff --git a/scripts/github-actions-ci.sh b/scripts/github-actions-ci.sh new file mode 100755 index 0000000..a84a214 --- /dev/null +++ b/scripts/github-actions-ci.sh @@ -0,0 +1,81 @@ +#!/bin/bash -ex +# +# Copyright The Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +export KUBECTL_VERSION="v1.16.1" +export KIND_VERSION="v0.5.1" + +rm -rf bin/ +mkdir -p bin/ +export PATH="${PWD}/bin:${HOME}/.local/bin:${PATH}" +export GITHUB_SHA="${GITHUB_SHA:-latest}" + +# Build helm from dev-v3 source +which helm || true +mkdir -p /tmp/gopath/src/helm.sh +pushd /tmp/gopath/src/helm.sh +git clone https://github.com/helm/helm.git -b dev-v3 +pushd helm/ +GOPATH=/tmp/gopath make build +popd +popd +mv /tmp/gopath/src/helm.sh/helm/bin/helm bin/helm +helm version +which helm + +# These tools appear to be in the GitHub "ubuntu-latest" environment, but not in +# the ubuntu:latest image from Docker Hub +if ! [[ -x "$(command -v curl)" || -x "$(command -v pip3)" || -x "$(command -v docker)" ]]; then + apt-get update + apt-get install -y apt-transport-https ca-certificates gnupg-agent software-properties-common curl python3-pip + + # Docker install + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + apt-get update + apt-get install -y docker-ce +fi +if ! [[ -x "$(command -v pip)" ]]; then + ln -sf $(which pip3) bin/pip +fi + +# Install kubectl +which kubectl || true +curl -LO https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl +chmod +x kubectl +mv kubectl bin/kubectl +kubectl version --client +which kubectl + +# Install kind +which helm || true +curl -LO https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64 +chmod +x kind-linux-amd64 +mv kind-linux-amd64 bin/kind +which kind + +# Install virtualenv +which virtualenv || true +pip3 install --user virtualenv +virtualenv --version +which virtualenv + +export ROBOT_OUTPUT_DIR="${PWD}/acceptance-testing-reports/${GITHUB_SHA}" +rm -rf ${ROBOT_OUTPUT_DIR} +mkdir -p ${ROBOT_OUTPUT_DIR} +trap "rm -rf ${ROBOT_OUTPUT_DIR}/.venv/" EXIT + +# Run +make acceptance diff --git a/testsuites/kubernetes_versions.robot b/testsuites/kubernetes_versions.robot index 74cf5b2..45efa1d 100644 --- a/testsuites/kubernetes_versions.robot +++ b/testsuites/kubernetes_versions.robot @@ -1,3 +1,18 @@ +# +# Copyright The Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + *** Settings *** Documentation Verify Helm functionality on multiple Kubernetes versions. ... @@ -6,8 +21,9 @@ Documentation Verify Helm functionality on multiple Kubernetes versions. ... kind cluster can be used by specifying it in an env var ... representing the version, for example: ... -... export KIND_CLUSTER_1_14_3="helm-ac-keepalive-1.14.3" -... export KIND_CLUSTER_1_15_0="helm-ac-keepalive-1.15.0" +... export KIND_CLUSTER_1_16_1="helm-ac-keepalive-1.16.1" +... export KIND_CLUSTER_1_15_4="helm-ac-keepalive-1.15.4" +... export KIND_CLUSTER_1_14_7="helm-ac-keepalive-1.14.7" ... Library String Library OperatingSystem @@ -19,11 +35,14 @@ Suite Setup Suite Setup Suite Teardown Suite Teardown *** Test Cases *** -Helm works with Kubernetes 1.14.3 - Test Helm on Kubernetes version 1.14.3 +#Helm works with Kubernetes 1.16.1 +# Test Helm on Kubernetes version 1.16.1 + +Helm works with Kubernetes 1.15.3 + Test Helm on Kubernetes version 1.15.3 -Helm works with Kubernetes 1.15.0 - Test Helm on Kubernetes version 1.15.0 +Helm works with Kubernetes 1.14.6 + Test Helm on Kubernetes version 1.14.6 *** Keyword *** Test Helm on Kubernetes version diff --git a/testsuites/repos.robot b/testsuites/repos.robot index cfe14e2..daab240 100644 --- a/testsuites/repos.robot +++ b/testsuites/repos.robot @@ -61,7 +61,7 @@ Make sure both repos get updated Output contains Successfully got an update from the "jfrog" chart repository Output contains Update Complete. ⎈ Happy Helming!⎈ -Try to remove inexistant repo +Try to remove nonexistent repo Check helm version Should fail helm repo remove badname Output contains Error: no repo named "badname" found