From 14cf4abc9318815807aacf9c89aee251e91b5734 Mon Sep 17 00:00:00 2001 From: Sebastian Alvis Date: Thu, 24 Sep 2020 10:26:21 -0700 Subject: [PATCH 01/24] Basic GitHub Action production hub parts are commented out for now --- .github/build-and-deploy.yaml | 40 +++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/build-and-deploy.yaml diff --git a/.github/build-and-deploy.yaml b/.github/build-and-deploy.yaml new file mode 100644 index 00000000..220ed839 --- /dev/null +++ b/.github/build-and-deploy.yaml @@ -0,0 +1,40 @@ +name: Build and Deploy +on: + push: + branches: + - staging + #- prod + +jobs: + build: + name: + # This job runs on Linux + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: docker://yuvipanda/hubploy:20200826083951674280 + name: Uunlock git-crypt Secrets + env: + GIT_CRYPT_KEY: ${{ secrets.GIT_CRYPT_KEY }} + with: + entrypoint: /bin/bash + args: -c "echo ${GIT_CRYPT_KEY} | base64 -d | git crypt unlock - && git crypt status" + - uses: docker://yuvipanda/hubploy:20200826083951674280 + name: Build & Push AWS image if needed + with: + args: -c "build icesat2 --check-registry --push" + - uses: docker://yuvipanda/hubploy:20200826083951674280 + name: Setup Helm + with: + entrypoint: /bin/bash + args: -c "helm init --client-only && helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/ && helm repo update" + - uses: docker://yuvipanda/hubploy:20200826083951674280 + name: Deploy AWS Staging Hub + if: github.ref == 'refs/heads/staging' + with: + args: deploy icesat2 pangeo-deploy staging --timeout 1200s --cleanup-on-fail + #- uses: docker://yuvipanda/hubploy:20200826083951674280 + # name: Deploy AWS Production Hub + # if: github.ref == 'refs/heads/prod' + # with: + # args: deploy icesat2 pangeo-deploy prod --timeout 1200s --cleanup-on-fail From 3ba39542377ad2e71e639bf25f100057a5ac1f08 Mon Sep 17 00:00:00 2001 From: Sebastian Alvis Date: Thu, 24 Sep 2020 10:30:37 -0700 Subject: [PATCH 02/24] move hubploy image to env var --- .github/build-and-deploy.yaml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/build-and-deploy.yaml b/.github/build-and-deploy.yaml index 220ed839..29451b94 100644 --- a/.github/build-and-deploy.yaml +++ b/.github/build-and-deploy.yaml @@ -5,6 +5,9 @@ on: - staging #- prod +env: + HUBPLOY_IMAGE: docker://yuvipanda/hubploy:20200826083951674280 + jobs: build: name: @@ -12,28 +15,28 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 - - uses: docker://yuvipanda/hubploy:20200826083951674280 + - uses: $HUBPLOY_IMAGE name: Uunlock git-crypt Secrets env: GIT_CRYPT_KEY: ${{ secrets.GIT_CRYPT_KEY }} with: entrypoint: /bin/bash args: -c "echo ${GIT_CRYPT_KEY} | base64 -d | git crypt unlock - && git crypt status" - - uses: docker://yuvipanda/hubploy:20200826083951674280 + - uses: $HUBPLOY_IMAGE name: Build & Push AWS image if needed with: args: -c "build icesat2 --check-registry --push" - - uses: docker://yuvipanda/hubploy:20200826083951674280 + - uses: $HUBPLOY_IMAGE name: Setup Helm with: entrypoint: /bin/bash args: -c "helm init --client-only && helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/ && helm repo update" - - uses: docker://yuvipanda/hubploy:20200826083951674280 + - uses: $HUBPLOY_IMAGE name: Deploy AWS Staging Hub if: github.ref == 'refs/heads/staging' with: args: deploy icesat2 pangeo-deploy staging --timeout 1200s --cleanup-on-fail - #- uses: docker://yuvipanda/hubploy:20200826083951674280 + #- uses: $HUBPLOY_IMAGE # name: Deploy AWS Production Hub # if: github.ref == 'refs/heads/prod' # with: From 7294c88395f0335adbd6f636844b631983969ea6 Mon Sep 17 00:00:00 2001 From: Sebastian Alvis Date: Thu, 24 Sep 2020 10:33:39 -0700 Subject: [PATCH 03/24] Typos / formatting of step names --- .github/build-and-deploy.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/build-and-deploy.yaml b/.github/build-and-deploy.yaml index 29451b94..a2860894 100644 --- a/.github/build-and-deploy.yaml +++ b/.github/build-and-deploy.yaml @@ -16,14 +16,14 @@ jobs: steps: - uses: actions/checkout@v1 - uses: $HUBPLOY_IMAGE - name: Uunlock git-crypt Secrets + name: Unlock git-crypt Secrets env: GIT_CRYPT_KEY: ${{ secrets.GIT_CRYPT_KEY }} with: entrypoint: /bin/bash args: -c "echo ${GIT_CRYPT_KEY} | base64 -d | git crypt unlock - && git crypt status" - uses: $HUBPLOY_IMAGE - name: Build & Push AWS image if needed + name: Build & Push AWS Image if Needed with: args: -c "build icesat2 --check-registry --push" - uses: $HUBPLOY_IMAGE From ccc5daf04f03dfe05a7a10d3887a49609c5cc68e Mon Sep 17 00:00:00 2001 From: Sebastian Alvis Date: Thu, 24 Sep 2020 10:43:13 -0700 Subject: [PATCH 04/24] Make helm install command many lines, add in other helm repos --- .github/build-and-deploy.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/build-and-deploy.yaml b/.github/build-and-deploy.yaml index a2860894..ecb496a5 100644 --- a/.github/build-and-deploy.yaml +++ b/.github/build-and-deploy.yaml @@ -30,7 +30,15 @@ jobs: name: Setup Helm with: entrypoint: /bin/bash - args: -c "helm init --client-only && helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/ && helm repo update" + args: > + -c "helm init --client-only && \ + helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/ &&\ + helm repo add dask https://helm.dask.org/ + helm repo add dask-gateway https://dask.org/dask-gateway-helm-repo/ &&\ + helm repo add stable https://kubernetes-charts.storage.googleapis.com &&\ + helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx &&\ + helm repo add prometheus-operator https://kubernetes-charts.storage.googleapis.com &&\ + helm repo update" - uses: $HUBPLOY_IMAGE name: Deploy AWS Staging Hub if: github.ref == 'refs/heads/staging' From c2110731e7b51251a53099d938fc868f5268938c Mon Sep 17 00:00:00 2001 From: Sebastian Alvis Date: Thu, 24 Sep 2020 10:47:01 -0700 Subject: [PATCH 05/24] Clean up multi-line yaml --- .github/build-and-deploy.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/build-and-deploy.yaml b/.github/build-and-deploy.yaml index ecb496a5..4b8e3431 100644 --- a/.github/build-and-deploy.yaml +++ b/.github/build-and-deploy.yaml @@ -31,13 +31,13 @@ jobs: with: entrypoint: /bin/bash args: > - -c "helm init --client-only && \ - helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/ &&\ - helm repo add dask https://helm.dask.org/ - helm repo add dask-gateway https://dask.org/dask-gateway-helm-repo/ &&\ - helm repo add stable https://kubernetes-charts.storage.googleapis.com &&\ - helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx &&\ - helm repo add prometheus-operator https://kubernetes-charts.storage.googleapis.com &&\ + -c "helm init --client-only && + helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/ && + helm repo add dask https://helm.dask.org/ && + helm repo add dask-gateway https://dask.org/dask-gateway-helm-repo/ && + helm repo add stable https://kubernetes-charts.storage.googleapis.com && + helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx && + helm repo add prometheus-operator https://kubernetes-charts.storage.googleapis.com && helm repo update" - uses: $HUBPLOY_IMAGE name: Deploy AWS Staging Hub From cede57e5215f4d29792efe53b66a865dd5b56ab4 Mon Sep 17 00:00:00 2001 From: Sebastian Alvis Date: Thu, 24 Sep 2020 10:55:12 -0700 Subject: [PATCH 06/24] Add EKS IP Whitelisting --- .github/build-and-deploy.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/build-and-deploy.yaml b/.github/build-and-deploy.yaml index 4b8e3431..2bfbff49 100644 --- a/.github/build-and-deploy.yaml +++ b/.github/build-and-deploy.yaml @@ -39,11 +39,30 @@ jobs: helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx && helm repo add prometheus-operator https://kubernetes-charts.storage.googleapis.com && helm repo update" + - uses: $HUBPLOY_IMAGE + name: Add Runner IP to EKS Kubernetes API Whitelist + with: + entrypoint: /bin/bash + args: > + -c "export AWS_SHARED_CREDENTIALS_FILE=./deployments/icesat2/secrets/aws-config.txt && + RUNNERIP=`curl --silent https://checkip.amazonaws.com` && + aws --version && + aws eks update-cluster-config --region us-west-2 --name pangeo --resources-vpc-config publicAccessCidrs=${RUNNERIP}/32 > /dev/null && + sleep 120" - uses: $HUBPLOY_IMAGE name: Deploy AWS Staging Hub if: github.ref == 'refs/heads/staging' with: args: deploy icesat2 pangeo-deploy staging --timeout 1200s --cleanup-on-fail + - uses: $HUBPLOY_IMAGE + name: Revert to Original EKS IP Whitelist + env: + AWS_IP_WHITELIST: ${{ secrets.AWS_IP_WHITELIST }} + with: + entrypoint: /bin/bash + args: > + -c "export AWS_SHARED_CREDENTIALS_FILE=./deployments/icesat2/secrets/aws-config.txt && + aws eks update-cluster-config --region us-west-2 --name pangeo --resources-vpc-config publicAccessCidrs=${AWS_IP_WHITELIST} > /dev/null" #- uses: $HUBPLOY_IMAGE # name: Deploy AWS Production Hub # if: github.ref == 'refs/heads/prod' From 57528a6aa9309db2955b0259b5f34b11cf2c0979 Mon Sep 17 00:00:00 2001 From: Sebastian Alvis Date: Thu, 24 Sep 2020 11:21:34 -0700 Subject: [PATCH 07/24] Remove AWS CI actions from CircleCI --- .circleci/config.yml | 70 ++++++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1f25111a..cbb98c33 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -53,13 +53,13 @@ jobs: command: | hubploy build gcp-uscentral1b --commit-range ${COMMIT_RANGE} - - run: - name: Test building aws-uswest2.pangeo.io image if needed - when: always - command: | - hubploy build icesat2 --commit-range ${COMMIT_RANGE} - environment: - PYTHONIOENCODING: utf-8 + #- run: + # name: Test building aws-uswest2.pangeo.io image if needed + # when: always + # command: | + # hubploy build icesat2 --commit-range ${COMMIT_RANGE} + # environment: + # PYTHONIOENCODING: utf-8 # This is currently disabled becauses it always runs and because the Azure # container registry is not open to annonymous read access. This can be changed @@ -142,13 +142,13 @@ jobs: command: | hubploy build gcp-uscentral1b --check-registry --push - - run: - name: Build aws-uswest2.pangeo.io image if needed - when: always - command: | - hubploy build icesat2 --check-registry --push - environment: - PYTHONIOENCODING: utf-8 + #- run: + # name: Build aws-uswest2.pangeo.io image if needed + # when: always + # command: | + # hubploy build icesat2 --check-registry --push + # environment: + # PYTHONIOENCODING: utf-8 - run: name: Build ooi.pangeo.io image if needed @@ -202,28 +202,28 @@ jobs: # NOTE: should move the dynamic IP into hubploy where credentials and awscli version already dealt with # sleep 2min for now, but better to poll for readiness https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html - - run: - name: Add Runner IP to EKS Kubernetes API Whitelist - when: always - command: | - export AWS_SHARED_CREDENTIALS_FILE=./deployments/icesat2/secrets/aws-config.txt - RUNNERIP=`curl --silent https://checkip.amazonaws.com` - aws --version - aws eks update-cluster-config --region us-west-2 --name pangeo --resources-vpc-config publicAccessCidrs=${RUNNERIP}/32 > /dev/null - sleep 120 - - run: - name: Deploy aws-uswest2.pangeo.io - when: always - no_output_timeout: 1200 - command: | - hubploy deploy icesat2 pangeo-deploy ${CIRCLE_BRANCH} --timeout 1200s --cleanup-on-fail + #- run: + # name: Add Runner IP to EKS Kubernetes API Whitelist + # when: always + # command: | + # export AWS_SHARED_CREDENTIALS_FILE=./deployments/icesat2/secrets/aws-config.txt + # RUNNERIP=`curl --silent https://checkip.amazonaws.com` + # aws --version + # aws eks update-cluster-config --region us-west-2 --name pangeo --resources-vpc-config publicAccessCidrs=${RUNNERIP}/32 > /dev/null + # sleep 120 + #- run: + # name: Deploy aws-uswest2.pangeo.io + # when: always + # no_output_timeout: 1200 + # command: | + # hubploy deploy icesat2 pangeo-deploy ${CIRCLE_BRANCH} --timeout 1200s --cleanup-on-fail - - run: - name: Revert to Original EKS IP Whitelist - when: always - command: | - export AWS_SHARED_CREDENTIALS_FILE=./deployments/icesat2/secrets/aws-config.txt - aws eks update-cluster-config --region us-west-2 --name pangeo --resources-vpc-config publicAccessCidrs=${AWS_IP_WHITELIST} > /dev/null + #- run: + # name: Revert to Original EKS IP Whitelist + # when: always + # command: | + # export AWS_SHARED_CREDENTIALS_FILE=./deployments/icesat2/secrets/aws-config.txt + # aws eks update-cluster-config --region us-west-2 --name pangeo --resources-vpc-config publicAccessCidrs=${AWS_IP_WHITELIST} > /dev/null - run: name: Deploy ooi.pangeo.io From d8dc6c2704ffc4d09c5563d329aea6682a931424 Mon Sep 17 00:00:00 2001 From: Sebastian Alvis Date: Wed, 30 Sep 2020 15:20:44 -0700 Subject: [PATCH 08/24] Move and rename aws-specific github action --- .../build-and-deploy-aws.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/{build-and-deploy.yaml => workflows/build-and-deploy-aws.yaml} (100%) diff --git a/.github/build-and-deploy.yaml b/.github/workflows/build-and-deploy-aws.yaml similarity index 100% rename from .github/build-and-deploy.yaml rename to .github/workflows/build-and-deploy-aws.yaml From 2bc87d2d5ba60db41e05ee947a37ad18503920c3 Mon Sep 17 00:00:00 2001 From: Sebastian Alvis Date: Wed, 30 Sep 2020 15:46:25 -0700 Subject: [PATCH 09/24] Move image building to its own action This allows us to let image building happen on PRs AND pushes, but keep building only on pushes --- .github/workflows/build-and-deploy-aws.yaml | 8 +++--- .github/workflows/build-aws-image.yaml | 31 +++++++++++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/build-aws-image.yaml diff --git a/.github/workflows/build-and-deploy-aws.yaml b/.github/workflows/build-and-deploy-aws.yaml index 2bfbff49..57e4f21a 100644 --- a/.github/workflows/build-and-deploy-aws.yaml +++ b/.github/workflows/build-and-deploy-aws.yaml @@ -22,10 +22,10 @@ jobs: with: entrypoint: /bin/bash args: -c "echo ${GIT_CRYPT_KEY} | base64 -d | git crypt unlock - && git crypt status" - - uses: $HUBPLOY_IMAGE - name: Build & Push AWS Image if Needed - with: - args: -c "build icesat2 --check-registry --push" + #- uses: $HUBPLOY_IMAGE + # name: Build & Push AWS Image if Needed + # with: + # args: -c "build icesat2 --check-registry --push" - uses: $HUBPLOY_IMAGE name: Setup Helm with: diff --git a/.github/workflows/build-aws-image.yaml b/.github/workflows/build-aws-image.yaml new file mode 100644 index 00000000..976af4aa --- /dev/null +++ b/.github/workflows/build-aws-image.yaml @@ -0,0 +1,31 @@ +name: Build AWS Image +on: + push: + branches: + - staging + #- prod + pull_request: + branches: + - staging + #- prod + +env: + HUBPLOY_IMAGE: docker://yuvipanda/hubploy:20200826083951674280 + +jobs: + build: + name: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: $HUBPLOY_IMAGE + name: Unlock git-crypt Secrets + env: + GIT_CRYPT_KEY: ${{ secrets.GIT_CRYPT_KEY }} + with: + entrypoint: /bin/bash + args: -c "echo ${GIT_CRYPT_KEY} | base64 -d | git crypt unlock - && git crypt status" + - uses: $HUBPLOY_IMAGE + name: Build & Push AWS Image if Needed + with: + args: -c "build icesat2 --check-registry --push" From 9c96d2e3e96611de7194e52cae185b30bc98f0e0 Mon Sep 17 00:00:00 2001 From: Sebastian Alvis Date: Wed, 30 Sep 2020 15:51:55 -0700 Subject: [PATCH 10/24] Change action name to Deploy AWS hub Also changed tabbing --- .github/workflows/build-and-deploy-aws.yaml | 118 ++++++++++---------- 1 file changed, 59 insertions(+), 59 deletions(-) diff --git a/.github/workflows/build-and-deploy-aws.yaml b/.github/workflows/build-and-deploy-aws.yaml index 57e4f21a..93bfbf82 100644 --- a/.github/workflows/build-and-deploy-aws.yaml +++ b/.github/workflows/build-and-deploy-aws.yaml @@ -1,4 +1,4 @@ -name: Build and Deploy +name: Deploy AWS Hub on: push: branches: @@ -9,62 +9,62 @@ env: HUBPLOY_IMAGE: docker://yuvipanda/hubploy:20200826083951674280 jobs: - build: - name: - # This job runs on Linux - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - uses: $HUBPLOY_IMAGE - name: Unlock git-crypt Secrets - env: - GIT_CRYPT_KEY: ${{ secrets.GIT_CRYPT_KEY }} - with: - entrypoint: /bin/bash - args: -c "echo ${GIT_CRYPT_KEY} | base64 -d | git crypt unlock - && git crypt status" - #- uses: $HUBPLOY_IMAGE - # name: Build & Push AWS Image if Needed - # with: - # args: -c "build icesat2 --check-registry --push" - - uses: $HUBPLOY_IMAGE - name: Setup Helm - with: - entrypoint: /bin/bash - args: > - -c "helm init --client-only && - helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/ && - helm repo add dask https://helm.dask.org/ && - helm repo add dask-gateway https://dask.org/dask-gateway-helm-repo/ && - helm repo add stable https://kubernetes-charts.storage.googleapis.com && - helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx && - helm repo add prometheus-operator https://kubernetes-charts.storage.googleapis.com && - helm repo update" - - uses: $HUBPLOY_IMAGE - name: Add Runner IP to EKS Kubernetes API Whitelist - with: - entrypoint: /bin/bash - args: > - -c "export AWS_SHARED_CREDENTIALS_FILE=./deployments/icesat2/secrets/aws-config.txt && - RUNNERIP=`curl --silent https://checkip.amazonaws.com` && - aws --version && - aws eks update-cluster-config --region us-west-2 --name pangeo --resources-vpc-config publicAccessCidrs=${RUNNERIP}/32 > /dev/null && - sleep 120" - - uses: $HUBPLOY_IMAGE - name: Deploy AWS Staging Hub - if: github.ref == 'refs/heads/staging' - with: - args: deploy icesat2 pangeo-deploy staging --timeout 1200s --cleanup-on-fail - - uses: $HUBPLOY_IMAGE - name: Revert to Original EKS IP Whitelist - env: - AWS_IP_WHITELIST: ${{ secrets.AWS_IP_WHITELIST }} - with: - entrypoint: /bin/bash - args: > - -c "export AWS_SHARED_CREDENTIALS_FILE=./deployments/icesat2/secrets/aws-config.txt && - aws eks update-cluster-config --region us-west-2 --name pangeo --resources-vpc-config publicAccessCidrs=${AWS_IP_WHITELIST} > /dev/null" - #- uses: $HUBPLOY_IMAGE - # name: Deploy AWS Production Hub - # if: github.ref == 'refs/heads/prod' - # with: + deploy_aws: + name: Deploy AWS Hub + # This job runs on Linux + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: $HUBPLOY_IMAGE + name: Unlock git-crypt Secrets + env: + GIT_CRYPT_KEY: ${{ secrets.GIT_CRYPT_KEY }} + with: + entrypoint: /bin/bash + args: -c "echo ${GIT_CRYPT_KEY} | base64 -d | git crypt unlock - && git crypt status" + #- uses: $HUBPLOY_IMAGE + # name: Build & Push AWS Image if Needed + # with: + # args: -c "build icesat2 --check-registry --push" + - uses: $HUBPLOY_IMAGE + name: Setup Helm + with: + entrypoint: /bin/bash + args: > + -c "helm init --client-only && + helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/ && + helm repo add dask https://helm.dask.org/ && + helm repo add dask-gateway https://dask.org/dask-gateway-helm-repo/ && + helm repo add stable https://kubernetes-charts.storage.googleapis.com && + helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx && + helm repo add prometheus-operator https://kubernetes-charts.storage.googleapis.com && + helm repo update" + - uses: $HUBPLOY_IMAGE + name: Add Runner IP to EKS Kubernetes API Whitelist + with: + entrypoint: /bin/bash + args: > + -c "export AWS_SHARED_CREDENTIALS_FILE=./deployments/icesat2/secrets/aws-config.txt && + RUNNERIP=`curl --silent https://checkip.amazonaws.com` && + aws --version && + aws eks update-cluster-config --region us-west-2 --name pangeo --resources-vpc-config publicAccessCidrs=${RUNNERIP}/32 > /dev/null && + sleep 120" + - uses: $HUBPLOY_IMAGE + name: Deploy AWS Staging Hub + if: github.ref == 'refs/heads/staging' + with: + args: deploy icesat2 pangeo-deploy staging --timeout 1200s --cleanup-on-fail + - uses: $HUBPLOY_IMAGE + name: Revert to Original EKS IP Whitelist + env: + AWS_IP_WHITELIST: ${{ secrets.AWS_IP_WHITELIST }} + with: + entrypoint: /bin/bash + args: > + -c "export AWS_SHARED_CREDENTIALS_FILE=./deployments/icesat2/secrets/aws-config.txt && + aws eks update-cluster-config --region us-west-2 --name pangeo --resources-vpc-config publicAccessCidrs=${AWS_IP_WHITELIST} > /dev/null" + #- uses: $HUBPLOY_IMAGE + # name: Deploy AWS Production Hub + # if: github.ref == 'refs/heads/prod' + # with: # args: deploy icesat2 pangeo-deploy prod --timeout 1200s --cleanup-on-fail From 71c4fedf3f1668e8727645795893f83c1421d70e Mon Sep 17 00:00:00 2001 From: Sebastian Alvis Date: Wed, 30 Sep 2020 15:53:00 -0700 Subject: [PATCH 11/24] Make more specific name to Build AWS Image Changed tabbing --- .github/workflows/build-aws-image.yaml | 32 +++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build-aws-image.yaml b/.github/workflows/build-aws-image.yaml index 976af4aa..51169849 100644 --- a/.github/workflows/build-aws-image.yaml +++ b/.github/workflows/build-aws-image.yaml @@ -13,19 +13,19 @@ env: HUBPLOY_IMAGE: docker://yuvipanda/hubploy:20200826083951674280 jobs: - build: - name: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - uses: $HUBPLOY_IMAGE - name: Unlock git-crypt Secrets - env: - GIT_CRYPT_KEY: ${{ secrets.GIT_CRYPT_KEY }} - with: - entrypoint: /bin/bash - args: -c "echo ${GIT_CRYPT_KEY} | base64 -d | git crypt unlock - && git crypt status" - - uses: $HUBPLOY_IMAGE - name: Build & Push AWS Image if Needed - with: - args: -c "build icesat2 --check-registry --push" + build_aws_image: + name: Build AWS Image + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: $HUBPLOY_IMAGE + name: Unlock git-crypt Secrets + env: + GIT_CRYPT_KEY: ${{ secrets.GIT_CRYPT_KEY }} + with: + entrypoint: /bin/bash + args: -c "echo ${GIT_CRYPT_KEY} | base64 -d | git crypt unlock - && git crypt status" + - uses: $HUBPLOY_IMAGE + name: Build & Push AWS Image if Needed + with: + args: -c "build icesat2 --check-registry --push" From 8a508b384d02e89fe61b25bd7cf19e769a3852f4 Mon Sep 17 00:00:00 2001 From: Sebastian Alvis Date: Wed, 30 Sep 2020 16:23:48 -0700 Subject: [PATCH 12/24] Add paths for filtering action triggers --- .github/workflows/build-and-deploy-aws.yaml | 5 +++++ .github/workflows/build-aws-image.yaml | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/build-and-deploy-aws.yaml b/.github/workflows/build-and-deploy-aws.yaml index 93bfbf82..9e898801 100644 --- a/.github/workflows/build-and-deploy-aws.yaml +++ b/.github/workflows/build-and-deploy-aws.yaml @@ -4,6 +4,11 @@ on: branches: - staging #- prod + paths: + - 'deployments/icesat2/**' + - 'pangeo-deploy/**' + paths_ignore: + - 'deployments/icesat2/image/binder/*' env: HUBPLOY_IMAGE: docker://yuvipanda/hubploy:20200826083951674280 diff --git a/.github/workflows/build-aws-image.yaml b/.github/workflows/build-aws-image.yaml index 51169849..614f3c3f 100644 --- a/.github/workflows/build-aws-image.yaml +++ b/.github/workflows/build-aws-image.yaml @@ -4,10 +4,14 @@ on: branches: - staging #- prod + paths: + - 'deployments/icesat2/image/binder/*' pull_request: branches: - staging #- prod + paths: + - 'deployments/icesat2/image/binder/*' env: HUBPLOY_IMAGE: docker://yuvipanda/hubploy:20200826083951674280 From 7ab78f7a6b210ecb99de69a070fbd5b04696373b Mon Sep 17 00:00:00 2001 From: Sebastian Alvis Date: Thu, 1 Oct 2020 13:20:04 -0700 Subject: [PATCH 13/24] Rename build action file for clarity --- .../workflows/{build-and-deploy-aws.yaml => build-aws-hub.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{build-and-deploy-aws.yaml => build-aws-hub.yaml} (100%) diff --git a/.github/workflows/build-and-deploy-aws.yaml b/.github/workflows/build-aws-hub.yaml similarity index 100% rename from .github/workflows/build-and-deploy-aws.yaml rename to .github/workflows/build-aws-hub.yaml From 509abacd8d3c64dc31e3b3d80a48e7fceff7b625 Mon Sep 17 00:00:00 2001 From: Sebastian Alvis Date: Thu, 1 Oct 2020 13:24:00 -0700 Subject: [PATCH 14/24] Remove check-registry flag --- .github/workflows/build-aws-image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-aws-image.yaml b/.github/workflows/build-aws-image.yaml index 614f3c3f..1f91e008 100644 --- a/.github/workflows/build-aws-image.yaml +++ b/.github/workflows/build-aws-image.yaml @@ -32,4 +32,4 @@ jobs: - uses: $HUBPLOY_IMAGE name: Build & Push AWS Image if Needed with: - args: -c "build icesat2 --check-registry --push" + args: -c "build icesat2 --push" From 927f18c723f2de99c284abbed7c23a9822605e0c Mon Sep 17 00:00:00 2001 From: Sebastian Alvis Date: Thu, 1 Oct 2020 13:53:37 -0700 Subject: [PATCH 15/24] Test changes to this PR branch on my fork --- .github/workflows/build-aws-hub.yaml | 1 + .github/workflows/build-aws-image.yaml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.github/workflows/build-aws-hub.yaml b/.github/workflows/build-aws-hub.yaml index 9e898801..8361710f 100644 --- a/.github/workflows/build-aws-hub.yaml +++ b/.github/workflows/build-aws-hub.yaml @@ -3,6 +3,7 @@ on: push: branches: - staging + - switch-to-github-actions #- prod paths: - 'deployments/icesat2/**' diff --git a/.github/workflows/build-aws-image.yaml b/.github/workflows/build-aws-image.yaml index 1f91e008..a1039eb0 100644 --- a/.github/workflows/build-aws-image.yaml +++ b/.github/workflows/build-aws-image.yaml @@ -3,12 +3,14 @@ on: push: branches: - staging + - switch-to-github-actions #- prod paths: - 'deployments/icesat2/image/binder/*' pull_request: branches: - staging + - switch-to-github-actions #- prod paths: - 'deployments/icesat2/image/binder/*' From e35fa828a5c87a4cd38f4c5a338a3af8798c7408 Mon Sep 17 00:00:00 2001 From: Sebastian Alvis Date: Thu, 1 Oct 2020 13:58:20 -0700 Subject: [PATCH 16/24] Allow the deployment of staging on this branch --- .github/workflows/build-aws-hub.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-aws-hub.yaml b/.github/workflows/build-aws-hub.yaml index 8361710f..92377ca8 100644 --- a/.github/workflows/build-aws-hub.yaml +++ b/.github/workflows/build-aws-hub.yaml @@ -57,7 +57,7 @@ jobs: sleep 120" - uses: $HUBPLOY_IMAGE name: Deploy AWS Staging Hub - if: github.ref == 'refs/heads/staging' + if: (github.ref == 'refs/heads/staging') || (github.ref == 'refs/heads/switch-to-github-actions') with: args: deploy icesat2 pangeo-deploy staging --timeout 1200s --cleanup-on-fail - uses: $HUBPLOY_IMAGE From dbab31b782f3f700a4603e6b679b8ed8ca6a0933 Mon Sep 17 00:00:00 2001 From: Sebastian Alvis Date: Thu, 1 Oct 2020 14:02:24 -0700 Subject: [PATCH 17/24] Correct workflow files so that Actions run correctly --- .github/workflows/build-aws-image.yaml | 10 +++--- ...build-aws-hub.yaml => deploy-aws-hub.yaml} | 35 ++++++++++--------- 2 files changed, 22 insertions(+), 23 deletions(-) rename .github/workflows/{build-aws-hub.yaml => deploy-aws-hub.yaml} (67%) diff --git a/.github/workflows/build-aws-image.yaml b/.github/workflows/build-aws-image.yaml index a1039eb0..689d05ee 100644 --- a/.github/workflows/build-aws-image.yaml +++ b/.github/workflows/build-aws-image.yaml @@ -7,6 +7,7 @@ on: #- prod paths: - 'deployments/icesat2/image/binder/*' + - '.github/workflows/build-aws-image.yaml' pull_request: branches: - staging @@ -15,23 +16,20 @@ on: paths: - 'deployments/icesat2/image/binder/*' -env: - HUBPLOY_IMAGE: docker://yuvipanda/hubploy:20200826083951674280 - jobs: build_aws_image: name: Build AWS Image runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 - - uses: $HUBPLOY_IMAGE + - uses: docker://yuvipanda/hubploy:20200826083951674280 name: Unlock git-crypt Secrets env: GIT_CRYPT_KEY: ${{ secrets.GIT_CRYPT_KEY }} with: entrypoint: /bin/bash args: -c "echo ${GIT_CRYPT_KEY} | base64 -d | git crypt unlock - && git crypt status" - - uses: $HUBPLOY_IMAGE + - uses: docker://yuvipanda/hubploy:20200826083951674280 name: Build & Push AWS Image if Needed with: - args: -c "build icesat2 --push" + args: build icesat2 --push diff --git a/.github/workflows/build-aws-hub.yaml b/.github/workflows/deploy-aws-hub.yaml similarity index 67% rename from .github/workflows/build-aws-hub.yaml rename to .github/workflows/deploy-aws-hub.yaml index 92377ca8..809e1100 100644 --- a/.github/workflows/build-aws-hub.yaml +++ b/.github/workflows/deploy-aws-hub.yaml @@ -8,11 +8,12 @@ on: paths: - 'deployments/icesat2/**' - 'pangeo-deploy/**' + - '.github/workflows/deploy-aws-hub.yaml' paths_ignore: - 'deployments/icesat2/image/binder/*' env: - HUBPLOY_IMAGE: docker://yuvipanda/hubploy:20200826083951674280 + HELM_EXECUTABLE: /usr/local/bin/helm3 jobs: deploy_aws: @@ -21,7 +22,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 - - uses: $HUBPLOY_IMAGE + - uses: docker://yuvipanda/hubploy:20200826083951674280 name: Unlock git-crypt Secrets env: GIT_CRYPT_KEY: ${{ secrets.GIT_CRYPT_KEY }} @@ -32,20 +33,20 @@ jobs: # name: Build & Push AWS Image if Needed # with: # args: -c "build icesat2 --check-registry --push" - - uses: $HUBPLOY_IMAGE + - #uses: docker://yuvipanda/hubploy:20200826083951674280 name: Setup Helm - with: - entrypoint: /bin/bash - args: > - -c "helm init --client-only && - helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/ && - helm repo add dask https://helm.dask.org/ && - helm repo add dask-gateway https://dask.org/dask-gateway-helm-repo/ && - helm repo add stable https://kubernetes-charts.storage.googleapis.com && - helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx && - helm repo add prometheus-operator https://kubernetes-charts.storage.googleapis.com && - helm repo update" - - uses: $HUBPLOY_IMAGE + run: | + curl https://get.helm.sh/helm-v3.1.2-linux-amd64.tar.gz | tar -xzf - + sudo mv linux-amd64/helm $HELM_EXECUTABLE + helm3 version + helm3 repo add jupyterhub https://jupyterhub.github.io/helm-chart/ + helm3 repo add dask https://helm.dask.org/ + helm3 repo add dask-gateway https://dask.org/dask-gateway-helm-repo/ + helm3 repo add stable https://kubernetes-charts.storage.googleapis.com + helm3 repo add ingress-nginx https://kubernetes.github.io/ingress-nginx + helm3 repo add prometheus-operator https://kubernetes-charts.storage.googleapis.com + helm3 repo update + - uses: docker://yuvipanda/hubploy:20200826083951674280 name: Add Runner IP to EKS Kubernetes API Whitelist with: entrypoint: /bin/bash @@ -55,12 +56,12 @@ jobs: aws --version && aws eks update-cluster-config --region us-west-2 --name pangeo --resources-vpc-config publicAccessCidrs=${RUNNERIP}/32 > /dev/null && sleep 120" - - uses: $HUBPLOY_IMAGE + - uses: docker://yuvipanda/hubploy:20200826083951674280 name: Deploy AWS Staging Hub if: (github.ref == 'refs/heads/staging') || (github.ref == 'refs/heads/switch-to-github-actions') with: args: deploy icesat2 pangeo-deploy staging --timeout 1200s --cleanup-on-fail - - uses: $HUBPLOY_IMAGE + - uses: docker://yuvipanda/hubploy:20200826083951674280 name: Revert to Original EKS IP Whitelist env: AWS_IP_WHITELIST: ${{ secrets.AWS_IP_WHITELIST }} From c51b32658cae1bf4537e07a7b7d6baefa9181b7e Mon Sep 17 00:00:00 2001 From: Sebastian Alvis Date: Fri, 2 Oct 2020 10:41:23 -0700 Subject: [PATCH 18/24] Switch to checkoutv2 --- .github/workflows/build-aws-image.yaml | 2 +- .github/workflows/deploy-aws-hub.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-aws-image.yaml b/.github/workflows/build-aws-image.yaml index 689d05ee..9ef9edf8 100644 --- a/.github/workflows/build-aws-image.yaml +++ b/.github/workflows/build-aws-image.yaml @@ -21,7 +21,7 @@ jobs: name: Build AWS Image runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - uses: docker://yuvipanda/hubploy:20200826083951674280 name: Unlock git-crypt Secrets env: diff --git a/.github/workflows/deploy-aws-hub.yaml b/.github/workflows/deploy-aws-hub.yaml index 809e1100..b48ed0a8 100644 --- a/.github/workflows/deploy-aws-hub.yaml +++ b/.github/workflows/deploy-aws-hub.yaml @@ -21,7 +21,7 @@ jobs: # This job runs on Linux runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - uses: docker://yuvipanda/hubploy:20200826083951674280 name: Unlock git-crypt Secrets env: From 4f003f7c7ccdc5afdd69fd5e3113ea08299d794e Mon Sep 17 00:00:00 2001 From: Sebastian Alvis Date: Fri, 2 Oct 2020 10:42:51 -0700 Subject: [PATCH 19/24] use if always to make sure we always revert the IP list --- .github/workflows/deploy-aws-hub.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy-aws-hub.yaml b/.github/workflows/deploy-aws-hub.yaml index b48ed0a8..1cec7f53 100644 --- a/.github/workflows/deploy-aws-hub.yaml +++ b/.github/workflows/deploy-aws-hub.yaml @@ -63,6 +63,7 @@ jobs: args: deploy icesat2 pangeo-deploy staging --timeout 1200s --cleanup-on-fail - uses: docker://yuvipanda/hubploy:20200826083951674280 name: Revert to Original EKS IP Whitelist + if: always() env: AWS_IP_WHITELIST: ${{ secrets.AWS_IP_WHITELIST }} with: From 02c15e692175131a58be7036b21e2a68b69de263 Mon Sep 17 00:00:00 2001 From: Sebastian Alvis Date: Fri, 2 Oct 2020 11:04:27 -0700 Subject: [PATCH 20/24] Remove hubploy build blocks that were commented out --- .github/workflows/deploy-aws-hub.yaml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/deploy-aws-hub.yaml b/.github/workflows/deploy-aws-hub.yaml index 1cec7f53..7b683a78 100644 --- a/.github/workflows/deploy-aws-hub.yaml +++ b/.github/workflows/deploy-aws-hub.yaml @@ -29,11 +29,6 @@ jobs: with: entrypoint: /bin/bash args: -c "echo ${GIT_CRYPT_KEY} | base64 -d | git crypt unlock - && git crypt status" - #- uses: $HUBPLOY_IMAGE - # name: Build & Push AWS Image if Needed - # with: - # args: -c "build icesat2 --check-registry --push" - - #uses: docker://yuvipanda/hubploy:20200826083951674280 name: Setup Helm run: | curl https://get.helm.sh/helm-v3.1.2-linux-amd64.tar.gz | tar -xzf - @@ -71,8 +66,3 @@ jobs: args: > -c "export AWS_SHARED_CREDENTIALS_FILE=./deployments/icesat2/secrets/aws-config.txt && aws eks update-cluster-config --region us-west-2 --name pangeo --resources-vpc-config publicAccessCidrs=${AWS_IP_WHITELIST} > /dev/null" - #- uses: $HUBPLOY_IMAGE - # name: Deploy AWS Production Hub - # if: github.ref == 'refs/heads/prod' - # with: - # args: deploy icesat2 pangeo-deploy prod --timeout 1200s --cleanup-on-fail From b04302a041894bd1773fa6bd7d42fb5b696e5cd4 Mon Sep 17 00:00:00 2001 From: Sebastian Alvis Date: Fri, 2 Oct 2020 11:06:13 -0700 Subject: [PATCH 21/24] Add workflow file as a path to trigger action on PRs --- .github/workflows/build-aws-image.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-aws-image.yaml b/.github/workflows/build-aws-image.yaml index 9ef9edf8..d8d1abec 100644 --- a/.github/workflows/build-aws-image.yaml +++ b/.github/workflows/build-aws-image.yaml @@ -15,6 +15,7 @@ on: #- prod paths: - 'deployments/icesat2/image/binder/*' + - '.github/workflows/build-aws-image.yaml' jobs: build_aws_image: From f5564554c83d45503a6c5831c36e1808fd3d095b Mon Sep 17 00:00:00 2001 From: Sebastian Alvis Date: Fri, 2 Oct 2020 11:07:19 -0700 Subject: [PATCH 22/24] Be more specific with acceptable paths on deploy command I was hoping that the paths_ignore of the image/ folder would override accepting anything in the icesat2/ folder, but that didn't work. Now I am more specific to exclude the image/ folder. --- .github/workflows/deploy-aws-hub.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy-aws-hub.yaml b/.github/workflows/deploy-aws-hub.yaml index 7b683a78..e7ffe8b3 100644 --- a/.github/workflows/deploy-aws-hub.yaml +++ b/.github/workflows/deploy-aws-hub.yaml @@ -6,11 +6,11 @@ on: - switch-to-github-actions #- prod paths: - - 'deployments/icesat2/**' + - 'deployments/icesat2/config/**' + - 'deployments/icesat2/secrets/**' + - 'deployments/icesat2/hubploy.yaml' - 'pangeo-deploy/**' - '.github/workflows/deploy-aws-hub.yaml' - paths_ignore: - - 'deployments/icesat2/image/binder/*' env: HELM_EXECUTABLE: /usr/local/bin/helm3 From a6b75e8c9efa108178f45f031d3ec5a8b377ad69 Mon Sep 17 00:00:00 2001 From: Sebastian Alvis Date: Fri, 2 Oct 2020 11:10:50 -0700 Subject: [PATCH 23/24] Add in the hyphen to make the Helm commands a step It got deleted in deleting the image build blocks --- .github/workflows/deploy-aws-hub.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-aws-hub.yaml b/.github/workflows/deploy-aws-hub.yaml index e7ffe8b3..97bdaa35 100644 --- a/.github/workflows/deploy-aws-hub.yaml +++ b/.github/workflows/deploy-aws-hub.yaml @@ -29,7 +29,7 @@ jobs: with: entrypoint: /bin/bash args: -c "echo ${GIT_CRYPT_KEY} | base64 -d | git crypt unlock - && git crypt status" - name: Setup Helm + - name: Setup Helm run: | curl https://get.helm.sh/helm-v3.1.2-linux-amd64.tar.gz | tar -xzf - sudo mv linux-amd64/helm $HELM_EXECUTABLE From 272c4ad3c5303c6cb2b3268eaa12ddbccd5c59b2 Mon Sep 17 00:00:00 2001 From: Sebastian Alvis Date: Mon, 5 Oct 2020 16:53:13 -0700 Subject: [PATCH 24/24] Remove workflow triggers on the switch-to-github-actions branch --- .github/workflows/build-aws-image.yaml | 2 -- .github/workflows/deploy-aws-hub.yaml | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/build-aws-image.yaml b/.github/workflows/build-aws-image.yaml index d8d1abec..3ad911e3 100644 --- a/.github/workflows/build-aws-image.yaml +++ b/.github/workflows/build-aws-image.yaml @@ -3,7 +3,6 @@ on: push: branches: - staging - - switch-to-github-actions #- prod paths: - 'deployments/icesat2/image/binder/*' @@ -11,7 +10,6 @@ on: pull_request: branches: - staging - - switch-to-github-actions #- prod paths: - 'deployments/icesat2/image/binder/*' diff --git a/.github/workflows/deploy-aws-hub.yaml b/.github/workflows/deploy-aws-hub.yaml index 97bdaa35..2ee880cd 100644 --- a/.github/workflows/deploy-aws-hub.yaml +++ b/.github/workflows/deploy-aws-hub.yaml @@ -3,7 +3,6 @@ on: push: branches: - staging - - switch-to-github-actions #- prod paths: - 'deployments/icesat2/config/**' @@ -53,7 +52,7 @@ jobs: sleep 120" - uses: docker://yuvipanda/hubploy:20200826083951674280 name: Deploy AWS Staging Hub - if: (github.ref == 'refs/heads/staging') || (github.ref == 'refs/heads/switch-to-github-actions') + if: github.ref == 'refs/heads/staging' with: args: deploy icesat2 pangeo-deploy staging --timeout 1200s --cleanup-on-fail - uses: docker://yuvipanda/hubploy:20200826083951674280