Merge pull request FairwindsOps#235 from reactiveops/rs/helm-updates

Helm Updates, Initial Work on Ephemeral Environments, Better Tests
wework · Feb 27, 2019 · 1ead3a8 · 1ead3a8
2 parents fe8b8e0 + 937e27e
commit 1ead3a8
Show file tree

Hide file tree

Showing 18 changed files with 376 additions and 196 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,65 +1,6 @@
 version: 2
 
 references:
-  install_test_dependencies: &install_test_dependencies
-    run:
-      name: Install dependencies
-      command: |
-        scversion="v0.5.0"
-        wget "https://storage.googleapis.com/shellcheck/shellcheck-$scversion.linux.x86_64.tar.xz"
-        tar --xz -xvf "shellcheck-$scversion.linux.x86_64.tar.xz"
-        mv shellcheck-$scversion/shellcheck /usr/bin/shellcheck
-        chmod +x /usr/bin/shellcheck
-        touch /tmp/libc.so.6
-        export LD_LIBRARY_PATH=/tmp # hack to work around a bug with a statically linked lib https://github.com/koalaman/shellcheck/issues/1053
-        shellcheck --version
-
-  npm_install: &npm_install
-    run:
-      name: Test npm install
-      command: npm install -g git://github.com/reactiveops/rok8s-scripts.git#${CIRCLE_SHA1}
-
-  pip_install: &pip_install
-    run:
-      name: Test pip install
-      command: pip install .
-
-  run_shellcheck: &run_shellcheck
-    run:
-      name: Run Shellcheck
-      command: |
-        cd bin
-        export LD_LIBRARY_PATH=/tmp # hack to work around a bug with a statically linked lib https://github.com/koalaman/shellcheck/issues/1053
-        shellcheck -x docker-* \
-          install-rok8s-requirements \
-          k8s-apply \
-          verify-deployment \
-          k8s-lint \
-          prepare-* \
-          helm-*
-
-  run_basic_linting: &run_basic_linting
-    run:
-      name: Run basic linting
-      command: |
-        cd examples/production-ready
-        k8s-lint -f staging.config
-        k8s-lint -f production.config
-
-  check_setup_py_version: &check_setup_py_version
-    run:
-      name: Check setup.py version number against git release tag
-      command: |
-        pip install semver
-        latest_release="$(git describe --abbrev=0 --tags)"
-        latest_release="${latest_release#*v}"
-        setup_py_version="$(python setup.py --version)"
-
-        if ! python -c "import semver, sys; sys.exit(0 if semver.match('${setup_py_version}', '>=${latest_release}') else 1)"; then
-          echo "Repo setup.py version is stale: $setup_py_version Latest Git tag release: $latest_release"
-          exit 1
-        fi
-
   docker_login: &docker_login
     run:
       name: Docker login
@@ -123,26 +64,13 @@ references:
         echo "export DOCKER_MINOR_TAG=$(echo ${DOCKER_BASE_TAG} | sed -r 's/^(v[0-9]+)(\.[0-9]+)(\.[0-9]+)$/\1\2/')" >> $BASH_ENV
 
 jobs:
-  test_npm:
-    docker:
-      - image: node:8.9.4
-    steps:
-      - checkout
-      - *install_test_dependencies
-      - *npm_install
-      - *run_shellcheck
-      - *run_basic_linting
-
-  test_pip:
+  test:
     docker:
-      - image: python:2.7
+      - image: circleci/python:2.7-stretch-node
     steps:
       - checkout
-      - *install_test_dependencies
-      - *pip_install
-      - *run_shellcheck
-      - *check_setup_py_version
-      - *run_basic_linting
+      - run: sudo test/setup.sh
+      - run: test/run.sh
 
   build:
     docker:
@@ -178,8 +106,7 @@ workflows:
   version: 2
   build_and_test:
     jobs:
-      - test_npm
-      - test_pip
+      - test
       - build
   release:
     jobs:

diff --git a/bin/helm-deploy b/bin/helm-deploy
@@ -11,33 +11,35 @@ CI_REF="${CI_TAG}"
 CI_REF="${CI_REF:-$CI_BRANCH}"
 CI_REF=$(echo "${CI_REF}" | tr / _)
 
-if [[ ! $(kubectl get namespace $NAMESPACE) ]]; then
-  echo "Creating ${NAMESPACE} namespace"
-  # shellcheck disable=SC2086
-  kubectl create namespace $NAMESPACE;
-fi
+if [ "${HELM_TEMPLATE_ONLY}" != "true" ]; then
+  if [[ ! $(kubectl get namespace "$NAMESPACE") ]]; then
+    rok8s_echo "Creating ${NAMESPACE} namespace"
+    # shellcheck disable=SC2086
+    kubectl create namespace $NAMESPACE;
+  fi
 
-# Set tiller_namespace if it isn't already set.
-# Allows us to use helm even if it's not installed
-# in kube-system.
-if [ -z "$TILLER_NAMESPACE" ]; then
-  TILLER_NAMESPACE="$(kubectl get deploy --all-namespaces -l app=helm,name=tiller | tail -n 1 | cut -d ' ' -f 1 | tr -d ' ')"
+  # Set tiller_namespace if it isn't already set.
+  # Allows us to use helm even if it's not installed
+  # in kube-system.
+  if [ -z "$TILLER_NAMESPACE" ]; then
+    TILLER_NAMESPACE="$(kubectl get deploy --all-namespaces -l app=helm,name=tiller | tail -n 1 | cut -d ' ' -f 1 | tr -d ' ')"
 
-  # just in case variable didn't get set, give it a default
-  TILLER_NAMESPACE=${TILLER_NAMESPACE:-kube-system}
-  echo "Set tiller namespace to $TILLER_NAMESPACE"
-fi
+    # just in case variable didn't get set, give it a default
+    TILLER_NAMESPACE=${TILLER_NAMESPACE:-kube-system}
+    rok8s_echo "Set tiller namespace to $TILLER_NAMESPACE"
+  fi
 
-# If using k8s-deploy in combination with helm-deploy
-#  while transitioning to helm, set this to avoid deploying
-#  secrets twice
-if [[ -z "${ROK8S_HELM_TRANSITIONING}" ]]; then
-  . k8s-deploy-secrets
-fi
+  # If using k8s-deploy in combination with helm-deploy
+  #  while transitioning to helm, set this to avoid deploying
+  #  secrets twice
+  if [[ -z "${ROK8S_HELM_TRANSITIONING}" ]]; then
+    . k8s-deploy-secrets
+  fi
 
-HELM_DEFAULT_TIMEOUT=300
-HELM_HOME=${HELM_HOME}
-helm init --client-only
+  HELM_DEFAULT_TIMEOUT=300
+  HELM_HOME=${HELM_HOME}
+  helm init --client-only
+fi
 
 format_multiple_values_files() {
   values_files="$1"
@@ -52,9 +54,10 @@ format_multiple_values_files() {
 helm_upgrade() {
   # shellcheck disable=SC2086
   helm upgrade --install "${CHART_RELEASE_NAME}" \
-    "./deploy/${CHART_PATH}" -f "${CHART_VALUES}" \
+    "${CHART_PATH}" -f "${CHART_VALUES}" \
     --set image.tag="${CI_SHA1}" \
     --set rok8sCIRef="${CI_REF}" \
+    --set sanitizedBranch="${SANITIZED_BRANCH}" \
     --namespace="${NAMESPACE}" \
     --tiller-namespace="${TILLER_NAMESPACE}" \
     --wait \
@@ -63,11 +66,36 @@ helm_upgrade() {
       2>&1 | tee "${ROK8S_TMP}/helm.out"
 }
 
-echo "Deploying Helm Charts"
+helm_template() {
+  helm template "${CHART_PATH}" \
+    -f "${CHART_VALUES}" \
+    --namespace="${NAMESPACE}" \
+    --set image.tag="${CI_SHA1}" \
+    --set rok8sCIRef="${CI_REF}" \
+    --set sanitizedBranch="${SANITIZED_BRANCH}"
+}
+
+
+for index in "${!HELM_REPO_NAMES[@]}"
+do
+  rok8s_echo "Adding Helm Repo: ${HELM_REPO_NAMES[index]}"
+  helm repo add "${HELM_REPO_NAMES[index]}" "${HELM_REPO_URLS[index]}"
+done
+
+rok8s_echo "Deploying Helm Charts"
 for index in "${!HELM_CHARTS[@]}"
 do
 
-  CHART_PATH=${HELM_CHARTS[$index]}
+  CHART_PATH_CONFIG=${HELM_CHARTS[$index]}
+  CHART_PATH_FIRST_SECTION=$(echo "${CHART_PATH_CONFIG}" | cut -d/ -f1)
+
+  if [[ " ${HELM_REPO_NAMES[*]} " == *" ${CHART_PATH_FIRST_SECTION} "* ]]; then
+    USING_REMOTE_CHART="true"
+    CHART_PATH="${CHART_PATH_CONFIG}"
+  else
+    CHART_PATH="./deploy/${CHART_PATH_CONFIG}"
+  fi
+
   CHART_RELEASE_NAME=${HELM_RELEASE_NAMES[$index]:-${HELM_CHARTS[$index]}}
   CHART_VALUES=${HELM_VALUES[$index]}
 
@@ -79,26 +107,37 @@ do
     CHART_VALUES="./deploy/${CHART_VALUES}.values.yml"
   fi
 
-  echo "Applying ${CHART_PATH} with ${CHART_VALUES}"
-  helm dep up "./deploy/${CHART_PATH}"
-  set +e
-  helm_upgrade
-  helm_ret=$?
-  set -e
-  if [ $helm_ret -gt 0 ] && grep -q "already exists" "${ROK8S_TMP}/helm.out"; then
-    echo "Detected that there are existing resources."
-    if [ -n "${ROK8S_HELM_ADOPT_EXISTING}" ]; then
-      echo "Attempting to force adoption of existing resources."
-      latest_revision=$(helm get "${CHART_RELEASE_NAME}" | grep REVISION | awk '{print $2}')
-      kubectl -n kube-system label cm "${CHART_RELEASE_NAME}.v${latest_revision}" STATUS=DEPLOYED --overwrite
-      helm_upgrade
-    else
-      echo "Not attempting adoption. Set ROK8ROK8S_HELM_ADOPT_EXISTING=1 to try adopting existing resources."
+  rok8s_echo "Applying ${CHART_PATH} with ${CHART_VALUES}"
+
+  if [ "${HELM_TEMPLATE_ONLY}" == "true" ]; then
+    helm_template
+
+  else
+    if [ "${USING_REMOTE_CHART}" != "true" ]; then
+      helm dep up "${CHART_PATH}"
+    fi
+
+    set +e
+    helm_upgrade
+    helm_ret=$?
+    set -e
+
+    if [ $helm_ret -gt 0 ] && grep -q "already exists" "${ROK8S_TMP}/helm.out"; then
+      rok8s_echo "Detected that there are existing resources."
+      if [ -n "${ROK8S_HELM_ADOPT_EXISTING}" ]; then
+        rok8s_echo "Attempting to force adoption of existing resources."
+        latest_revision=$(helm get "${CHART_RELEASE_NAME}" | grep REVISION | awk '{print $2}')
+        kubectl -n kube-system label cm "${CHART_RELEASE_NAME}.v${latest_revision}" STATUS=DEPLOYED --overwrite
+        helm_upgrade
+      else
+        rok8s_echo "Not attempting adoption. Set ROK8S_HELM_ADOPT_EXISTING=1 to try adopting existing resources."
+        exit $helm_ret
+      fi
+    elif [ $helm_ret -gt 0 ]; then
       exit $helm_ret
     fi
-  elif [ $helm_ret -gt 0 ]; then
-    exit $helm_ret
+
+    echo "Done deploying Helm Charts"
+    echo ""
   fi
 done
-echo "Done deploying Helm Charts"
-echo ""
diff --git a/bin/helm-template b/bin/helm-template
@@ -2,47 +2,4 @@
 
 set -eo pipefail
 
-. k8s-read-config
-
-# Set a CI_REF from branch or tag
-CI_REF="${CI_TAG}"
-CI_REF="${CI_REF:-$CI_BRANCH}"
-CI_REF=$(echo "${CI_REF}" | tr / _)
-
-format_multiple_values_files() {
-  values_files="$1"
-  formatted_files=""
-  echo "${values_files}" | sed -n 1'p' | tr ',' '\n' | ( while read -r file_name; do
-    file="./deploy/${file_name}.values.yml,"
-    formatted_files="${formatted_files}${file}"
-  done
-  echo "${formatted_files%?}" )
-}
-
-HELM_HOME=${HELM_HOME}
-helm init --client-only
-
-for index in "${!HELM_CHARTS[@]}"
-do
-
-  CHART_PATH=${HELM_CHARTS[$index]}
-  CHART_VALUES=${HELM_VALUES[$index]}
-
-  # if there is more than one values files listed
-  # then pass all values file to helm upgrade command
-  if [[ "${CHART_VALUES}" = *","* ]]; then
-    CHART_VALUES=$(format_multiple_values_files "${CHART_VALUES}")
-  else
-    CHART_VALUES="./deploy/${CHART_VALUES}.values.yml"
-  fi
-
-  helm dep up "./deploy/${CHART_PATH}"
-  helm template "./deploy/${CHART_PATH}" \
-      -f "${CHART_VALUES}" \
-      --namespace="${NAMESPACE}" \
-      --set image.tag="${CI_SHA1}" \
-      --set rok8sCIRef="${CI_REF}" > "${ROK8S_TMP}/helm.out"
-  cat "${ROK8S_TMP}/helm.out"
-  echo ""
-
-done
+LOG_LEVEL="${LOG_LEVEL:=none}" HELM_TEMPLATE_ONLY="true" helm-deploy "$@"