Add initial github actions (#53)

* Add initial github actions

Signed-off-by: Josh Dolitsky <[email protected]>

.github/workflows/acceptance-tests-pr.yml

@@ -0,0 +1,23 @@
+name: acceptance-tests-pr
+
+on:
+  pull_request:
+    branches:
+    - master
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout source code
+      uses: actions/checkout@v1
+
+    - name: Run acceptance tests
+      run: make github-actions-ci
+
+    - uses: actions/upload-artifact@master
+      name: Upload test report
+      with:
+        name: helm-acceptance-testing-report-${{ github.sha }}
+        path: acceptance-testing-reports/${{ github.sha }}/
+      if: always()

.github/workflows/acceptance-tests.yml

@@ -0,0 +1,22 @@
+name: acceptance-tests
+
+on:
+  push:
+    branches:
+      - master
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v1
+
+      - name: Run acceptance tests
+        run: make github-actions-ci
+
+      - uses: actions/upload-artifact@master
+        name: Upload test report
+        with:
+          name: helm-acceptance-testing-report-${{ github.sha }}
+          path: acceptance-testing-reports/${{ github.sha }}/
+        if: always()

.gitignore

@@ -1,2 +1,5 @@
 .acceptance/
 .idea/
+
+acceptance-testing-reports/
+bin/

Makefile

@@ -3,3 +3,16 @@ SHELL = /bin/bash
 .PHONY: acceptance
 acceptance:
 	@scripts/acceptance.sh
+
+.PHONY: github-actions-ci
+github-actions-ci:
+	@scripts/github-actions-ci.sh
+
+.PHONY: github-actions-ci-local
+github-actions-ci-local:
+	docker run -it --rm \
+	    -v $(shell pwd):/tmp/acceptance-testing \
+	    -w /tmp/acceptance-testing  \
+	    --privileged -v /var/run/docker.sock:/var/run/docker.sock \
+	    --entrypoint=/bin/bash ubuntu:latest \
+	    -c 'set +e; scripts/github-actions-ci.sh; echo "Exited $?. (Ctrl+D to exit shell)"; bash'

README.md

@@ -1,11 +1,51 @@
 # Helm Acceptance Tests
 
-*Note: these tests have only been run against Helm 3 ([dev-v3](https://github.com/helm/helm/tree/dev-v3))*
+[![GitHub Actions status](https://github.com/helm/acceptance-testing/workflows/acceptance-tests/badge.svg)](https://github.com/helm/acceptance-testing/actions)
 
 This repo contains the source for Helm acceptance tests.
-
 The tests are written using [Robot Framework](https://robotframework.org/).
 
+*Note: these tests have only been run against Helm 3 ([dev-v3](https://github.com/helm/helm/tree/dev-v3))*
+
+## Test Summary
+
+### Kubernetes Versions
+
+Helm is tested to work against the following versions of Kubernetes:
+
+<!-- 
+TODO
+
+Add support for 1.16+, getting the following error:
+Error: apiVersion "apps/v1beta1" in nginx/templates/deployment.yaml is not available
+[1.16.1](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.16.md)
+
+Also, upgrade to 1.15.4 and 1.14.7
+(see issue on kind: https://github.com/kubernetes-sigs/kind/issues/948)
+
+-->
+
+- [1.15.3](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.15.md)
+- [1.14.6](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md)
+
+Test suite: [kubernetes_versions.robot](./testsuites/kubernetes_versions.robot)
+
+
+### Shell Completion
+
+Helm's shell completion functionality is tested against the following shells:
+
+- Bash
+- Zsh
+
+Test suite: [shells.robot](./testsuites/shells.robot)
+
+### Helm Repositories
+
+Basic functionality of the chart repository subsystem is tested.
+
+Test suite: [repos.robot](./testsuites/repos.robot)
+
 ## System requirements
 
 The following tools/commands are expected to be present on the base system
@@ -25,6 +65,13 @@ From the root of this repo, run the following:
 make acceptance
 ```
 
+Alternatively, if you have Docker installed, 
+the system requirements above are not needed, and you can run the following
+command which will simulate CI:
+```
+make github-actions-ci-local
+```
+
 Note: by default, the tests will use helm as found on your PATH.
 To specify a different helm to test, set and export the `ROBOT_HELM_PATH`
 environment variable.  For example, if you have helm v2 installed, but want
@@ -107,7 +154,6 @@ contains a base class called `CommandRunner` that you will likely want to
 leverage when adding support for a new external tool.
 
 The test run is wrapped by [acceptance.sh](./scripts/acceptance.sh) -
-in this file the environment is validated (i.e. check if required tools present).
-
-sinstalled (including Robot Framework itself). If any additional Python libraries
-are required for a new library, it can be appended to `ROBOT_PY_REQUIRES`.
+in this file the environment is validated (i.e. check if required tools present). 
+If any additional Python libraries are required for a new library, 
+it can be appended to `ROBOT_PY_REQUIRES`.

lib/Kind.py

@@ -9,7 +9,7 @@
 MAX_WAIT_KIND_NODE_SECONDS = 60
 KIND_NODE_INTERVAL_SECONDS = 2
 
-MAX_WAIT_KIND_POD_SECONDS = 60
+MAX_WAIT_KIND_POD_SECONDS = 120
 KIND_POD_INTERVAL_SECONDS = 2
 
 KIND_POD_EXPECTED_NUMBER = 8
@@ -39,6 +39,12 @@ def create_test_cluster_with_kubernetes_version(self, kube_version):
             cmd += ' --image='+DOCKER_HUB_REPO+':v'+kube_version
             self.run_command(cmd)
 
+        # Fix for running kind in docker, switch the port+IP in the kubeconfig
+        if os.path.exists('/.dockerenv'):
+            print('Running in Docker, modifying IP in kubeconfig')
+            fixcmd = 'export KIND_IP=$(docker inspect '+LAST_CLUSTER_NAME+'-control-plane | grep \'IPAddress": "\' | head -1 | awk \'{print $2}\' | tr -d \\",) && sed -i "s/https:\/\/127\.0\.0\.1:.*/https:\/\/${KIND_IP}:6443/" $(kind get kubeconfig-path --name="'+LAST_CLUSTER_NAME+'")'
+            self.run_command(fixcmd)
+
     def delete_test_cluster(self):
         if LAST_CLUSTER_EXISTING:
             print('Not deleting cluster (cluster existed prior to test run)')

scripts/acceptance.sh

@@ -1,4 +1,18 @@
 #!/bin/bash -e
+#
+# Copyright The Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 # Turn on debug printouts if the user requested a debug level >= $1
 set_shell_debug_level()
@@ -64,7 +78,8 @@ set_shell_debug_level 2
 
 # Only use the -d flag for mktemp as many other flags don't
 # work on every plateform
-export TMP_DIR=$(mktemp -d ${ROBOT_OUTPUT_DIR}/helm-acceptance.XXXXXX)
+mkdir -p ${ROBOT_OUTPUT_DIR}
+export TMP_DIR="$(mktemp -d ${ROBOT_OUTPUT_DIR}/helm-acceptance.XXXXXX)"
 trap "rm -rf ${TMP_DIR}" EXIT
 
 SUITES_TO_RUN=""

scripts/completion-tests/test-completion.sh

@@ -20,6 +20,17 @@
 set -e
 SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
 
+# TODO: this is redeclared, but shouldnt have to be?
+# getting error "scripts/completion-tests/test-completion.sh: line 23: set_shell_debug_level: command not found"
+set_shell_debug_level()
+{
+    set +x
+    if [ $ROBOT_DEBUG_LEVEL -ge $1 ]; then
+       set -x
+    fi
+}
+export -f set_shell_debug_level
+
 set_shell_debug_level 2
 
 BINARY_NAME=helm
@@ -46,12 +57,18 @@ mkdir -p ${COMP_DIR}/bin
 cp ${SCRIPT_DIR}/${COMP_SCRIPT_NAME} ${COMP_DIR}
 cp ${SCRIPT_DIR}/lib/completionTests-base.sh ${COMP_DIR}/lib
 
-if ! [ -f ${BINARY_PATH_DOCKER}/${BINARY_NAME} ]; then
-    echo "These tests require a helm binary located at ${BINARY_PATH_DOCKER}/${BINARY_NAME}"
-    echo "Hint: Run 'make build-cross' in a clone of helm repo"
-    exit 2
+if [[ "${GITHUB_SHA}" == "" ]]; then
+  CHECK_BINARY_PATH="$(cd ${BINARY_PATH_DOCKER} && pwd)/${BINARY_NAME}"
+  if [[ ! -f ${CHECK_BINARY_PATH} ]] && [[ -L ${CHECK_BINARY_PATH} ]]; then
+      echo "These tests require a helm binary located at ${CHECK_BINARY_PATH}"
+      echo "Hint: Run 'make build-cross' in a clone of helm repo"
+      exit 2
+  fi
+  cp ${CHECK_BINARY_PATH} ${COMP_DIR}/bin
+else
+  echo "Running on GitHub Actions CI - using system-wide Helm 3 binary."
+  cp $(which helm) ${COMP_DIR}/bin
 fi
-cp ${BINARY_PATH_DOCKER}/${BINARY_NAME} ${COMP_DIR}/bin
 
 # kubectl stub
 cat > ${COMP_DIR}/bin/kubectl << EOF

scripts/github-actions-ci.sh

@@ -0,0 +1,81 @@
+#!/bin/bash -ex
+#
+# Copyright The Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export KUBECTL_VERSION="v1.16.1"
+export KIND_VERSION="v0.5.1"
+
+rm -rf bin/
+mkdir -p bin/
+export PATH="${PWD}/bin:${HOME}/.local/bin:${PATH}"
+export GITHUB_SHA="${GITHUB_SHA:-latest}"
+
+# Build helm from dev-v3 source
+which helm || true
+mkdir -p /tmp/gopath/src/helm.sh
+pushd /tmp/gopath/src/helm.sh
+git clone https://github.com/helm/helm.git -b dev-v3
+pushd helm/
+GOPATH=/tmp/gopath make build
+popd
+popd
+mv /tmp/gopath/src/helm.sh/helm/bin/helm bin/helm
+helm version
+which helm
+
+# These tools appear to be in the GitHub "ubuntu-latest" environment, but not in
+# the ubuntu:latest image from Docker Hub
+if ! [[ -x "$(command -v curl)" || -x "$(command -v pip3)" || -x "$(command -v docker)" ]]; then
+  apt-get update
+  apt-get install -y apt-transport-https ca-certificates gnupg-agent software-properties-common curl python3-pip
+
+  # Docker install
+  curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+  add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+  apt-get update
+  apt-get install -y docker-ce
+fi
+if ! [[ -x "$(command -v pip)" ]]; then
+  ln -sf $(which pip3) bin/pip
+fi
+
+# Install kubectl
+which kubectl || true
+curl -LO https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl
+chmod +x kubectl
+mv kubectl bin/kubectl
+kubectl version --client
+which kubectl
+
+# Install kind
+which helm || true
+curl -LO https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64
+chmod +x kind-linux-amd64
+mv kind-linux-amd64 bin/kind
+which kind
+
+# Install virtualenv
+which virtualenv || true
+pip3 install --user virtualenv
+virtualenv --version
+which virtualenv
+
+export ROBOT_OUTPUT_DIR="${PWD}/acceptance-testing-reports/${GITHUB_SHA}"
+rm -rf ${ROBOT_OUTPUT_DIR}
+mkdir -p ${ROBOT_OUTPUT_DIR}
+trap "rm -rf ${ROBOT_OUTPUT_DIR}/.venv/" EXIT
+
+# Run
+make acceptance

testsuites/kubernetes_versions.robot

@@ -1,3 +1,18 @@
+#
+# Copyright The Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 *** Settings ***
 Documentation     Verify Helm functionality on multiple Kubernetes versions.
 ...
@@ -6,8 +21,9 @@ Documentation     Verify Helm functionality on multiple Kubernetes versions.
 ...               kind cluster can be used by specifying it in an env var
 ...               representing the version, for example:
 ...
-...                  export KIND_CLUSTER_1_14_3="helm-ac-keepalive-1.14.3"
-...                  export KIND_CLUSTER_1_15_0="helm-ac-keepalive-1.15.0"
+...                  export KIND_CLUSTER_1_16_1="helm-ac-keepalive-1.16.1"
+...                  export KIND_CLUSTER_1_15_4="helm-ac-keepalive-1.15.4"
+...                  export KIND_CLUSTER_1_14_7="helm-ac-keepalive-1.14.7"
 ...
 Library           String
 Library           OperatingSystem
@@ -19,11 +35,14 @@ Suite Setup       Suite Setup
 Suite Teardown    Suite Teardown
 
 *** Test Cases ***
-Helm works with Kubernetes 1.14.3
-    Test Helm on Kubernetes version   1.14.3
+#Helm works with Kubernetes 1.16.1
+#    Test Helm on Kubernetes version   1.16.1
+
+Helm works with Kubernetes 1.15.3
+    Test Helm on Kubernetes version   1.15.3
 
-Helm works with Kubernetes 1.15.0
-    Test Helm on Kubernetes version   1.15.0
+Helm works with Kubernetes 1.14.6
+    Test Helm on Kubernetes version   1.14.6
 
 *** Keyword ***
 Test Helm on Kubernetes version

testsuites/repos.robot

@@ -61,7 +61,7 @@ Make sure both repos get updated
     Output contains  Successfully got an update from the "jfrog" chart repository
     Output contains  Update Complete. ⎈ Happy Helming!⎈
 
-Try to remove inexistant repo
+Try to remove nonexistent repo
     Check helm version
     Should fail  helm repo remove badname
     Output contains  Error: no repo named "badname" found