Merge pull request #107 from JarvusInnovations/develop

Release: gatekeeper v2.5.0
JarvusInnovations · Jan 12, 2021 · 41fe97f · 41fe97f
2 parents 55c351e + 5463b91
commit 41fe97f
Show file tree

Hide file tree

Showing 46 changed files with 1,043 additions and 182 deletions.
diff --git a/.github/workflows/k8s-deploy-pr.yml b/.github/workflows/k8s-deploy-pr.yml
@@ -11,16 +11,19 @@ env:
   KUBE_NAMESPACE: ${{ secrets.kube_namespace }}
   KUBE_CONFIG_DATA: ${{ secrets.kube_config }}
 
+  PACKAGE_ORIGIN: jarvus
   PACKAGE_NAME: gatekeeper-composite
   PACKAGE_REGISTRY: docker.pkg.github.com
 
   PR_NAME: pr-${{ github.event.number }}
   KUBE_HOSTNAME: ${{ secrets.kube_hostname }}
 
+  DATABASE_NAME: gatekeeper
+
   HAB_LICENSE: accept-no-persist
 
 jobs:
-  kubernetes-deploy:
+  k8s-deploy:
 
     runs-on: ubuntu-latest
     steps:
@@ -29,11 +32,17 @@ jobs:
       with:
         ref: ${{ github.head_ref }}
 
+    - name: 'Initialize Chef Habitat environment'
+      uses: JarvusInnovations/habitat-action@action/v1
+      with:
+        deps: |
+          jarvus/hologit
+
     - name: Create Github Deployment
       run: |
         set -e
         # Create deployment
-        hub api /repos/${{ github.repository }}/deployments -X POST --input <(cat <<EOF
+        hub api "/repos/${{ github.repository }}/deployments" -X POST --input <(cat <<EOF
         {
           "ref": "${{ github.head_ref }}",
           "auto_merge": false,
@@ -43,15 +52,15 @@ jobs:
         }
         EOF) > /tmp/deployment.json
 
-        DEPLOYMENT_ID=$(jq .id < /tmp/deployment.json)
-        echo ::set-env name=GH_DEPLOYMENT_ID::$(echo $DEPLOYMENT_ID)
+        DEPLOYMENT_ID="$(jq .id < /tmp/deployment.json)"
+        echo "GH_DEPLOYMENT_ID=${DEPLOYMENT_ID}" >> $GITHUB_ENV
 
     - name: Update GH Deployment Status
       run: |
         set -e
 
         # Set status to pending
-        hub api /repos/${{ github.repository }}/deployments/${{ env.GH_DEPLOYMENT_ID }}/statuses  \
+        hub api "/repos/${{ github.repository }}/deployments/${GH_DEPLOYMENT_ID}/statuses"  \
           -X POST                                                                                      \
           -H "Accept: application/json, application/vnd.github.flash-preview+json"                     \
           --input <(cat <<EOF
@@ -64,8 +73,19 @@ jobs:
 
     - name: Update Environment
       run: |
-        echo ::set-env name=COMMIT_MSG::$(git log --format=%B -n 1 ${{ github.event.after }})
-        echo ::set-env name=REPO_NAME::$(echo ${GITHUB_REPOSITORY,,})
+        echo "REPO_NAME=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV
+
+    - id: site-projection
+      name: 'Project holobranch: emergence-site'
+      uses: JarvusInnovations/hologit@actions/projector/v1
+      with:
+        holobranch: emergence-site
+
+    - id: fixtures-projection
+      name: 'Project holobranch: fixtures'
+      uses: JarvusInnovations/hologit@actions/projector/v1
+      with:
+        holobranch: fixtures
 
     - name: Build & Publish Docker image
       uses: whoan/docker-build-with-cache-action@v5
@@ -77,6 +97,8 @@ jobs:
         image_name: ${{ env.REPO_NAME }}/${{ env.PACKAGE_NAME }}
         image_tag: ${{ env.PR_NAME }}
         build_extra_args: |
+          --build-arg=SITE_TREE=${{ steps.site-projection.outputs.tree }}
+          --build-arg=SITE_VERSION=0.0.0-pr.${{ github.event.number }}
           --build-arg=SOURCE_COMMIT=${{ github.sha }}
           --build-arg=SOURCE_TAG=${{ env.PR_NAME }}
           --build-arg=HAB_LICENSE=${{ env.HAB_LICENSE }}
@@ -89,48 +111,101 @@ jobs:
         $(printf '%s' "$KUBE_CONFIG_DATA" | base64 -d)
         EOF
 
-    - uses: azure/setup-kubectl@v1
-
-    - name: Create K8S Deployment from Template
+    - name: Deploy instance via Helm template
       run: |
         set -e
 
-        image_id=$(echo ${{ env.REPO_NAME }}/${{ env.PACKAGE_NAME }}:${{ env.PR_NAME }})
-        image_url=$(echo docker.pkg.github.com/$image_id)
-        hostname=$(echo ${{ env.PR_NAME }}.${{ env.KUBE_HOSTNAME }})
+        image_id="${REPO_NAME}/${PACKAGE_NAME}:${PR_NAME}"
+        image_url="docker.pkg.github.com/${image_id}"
+        hostname="${PR_NAME}.${KUBE_HOSTNAME}"
 
-        kubectl config set-context --current --namespace=${KUBE_NAMESPACE}
+        kubectl config set-context --current --namespace="${KUBE_NAMESPACE}"
 
-        # delete any existing pod first to force image to re-pull without changing tag
-        # kubectl delete pod -l app.kubernetes.io/instance="${PR_NAME}"
-        # helm uninstall ${{ env.PR_NAME }} -n ${{ env.KUBE_NAMESPACE }}
+        echo "Listing pods existing before deploy"
+        kubectl get pods \
+          -l app.kubernetes.io/instance="${PR_NAME}" \
+          --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}' \
+          | sort \
+          | tee ./.pods-before
 
-        helm upgrade ${{ env.PR_NAME }} ./k8s/charts/deployment \
+        echo "Using helm upgrade to apply ./helm-chart to release ${PR_NAME}"
+        helm upgrade "${PR_NAME}" ./helm-chart \
           --install \
-          --set name=${{ env.PR_NAME }} \
-          --set namespace=${{ env.KUBE_NAMESPACE }} \
-          --set image=${image_url} \
-          --set hostname=${hostname}
-
+          --set name="${PR_NAME}" \
+          --set namespace="${KUBE_NAMESPACE}" \
+          --set image="${image_url}" \
+          --set hostname="${hostname}" \
+          --set hab.runtime.error.display=true
+
+        echo "Listing pods existing after deploy"
+        kubectl get pods \
+          -l app.kubernetes.io/instance="${PR_NAME}" \
+          --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}' \
+          | sort \
+          | tee ./.pods-after
+
+        echo "Deleting stale pods to force image refresh"
+        comm -12 ./.pods-before ./.pods-after \
+          | xargs --no-run-if-empty kubectl delete pod
 
     - name: Wait for Deployment to be Ready
+      timeout-minutes: 10
       run: |
         set -e
-        until kubectl rollout status deployment "${PR_NAME}" 2>/dev/null >/dev/null; do echo -n "."; sleep .5; done;
+        until kubectl rollout status deployment "${PR_NAME}" 2>/dev/null >/dev/null; do
+          echo -n "."
+          sleep .5
+        done
 
     - name: Retrieve/Store Pod Name
       run: |
-        echo ::set-env name=POD_NAME::$(kubectl get pod -l app.kubernetes.io/instance="${PR_NAME}" -o jsonpath='{.items[0].metadata.name}')
+        echo "POD_NAME=$(kubectl get pod -l app.kubernetes.io/instance="${PR_NAME}" -o jsonpath='{.items[0].metadata.name}')" >> $GITHUB_ENV
 
     - name: Wait For Pod to be Ready
       run: |
         set -e
         kubectl wait --for condition=ready "pod/${POD_NAME}" --timeout=30s
 
+    - name: Wait for MySQL to be Ready
+      timeout-minutes: 5
+      run: |
+        set -e
+        until kubectl exec "${POD_NAME}" -- hab pkg exec "${PACKAGE_ORIGIN}/${PACKAGE_NAME}" mysqladmin ping; do
+          sleep .5
+        done
+
+    - name: Load fixtures into database
+      run: |
+        echo "Dropping any existing database..."
+        kubectl exec "${POD_NAME}" -- \
+          hab pkg exec "${PACKAGE_ORIGIN}/${PACKAGE_NAME}" \
+          mysqladmin drop "${DATABASE_NAME}" --force \
+        || true
+
+        echo "Creating an empty database..."
+        kubectl exec "${POD_NAME}" -- \
+          hab pkg exec "${PACKAGE_ORIGIN}/${PACKAGE_NAME}" \
+          mysqladmin create "${DATABASE_NAME}"
+
+        echo "Loading fixtures..."
+        (
+          for fixture_file in $(git ls-tree -r --name-only ${{ steps.fixtures-projection.outputs.tree }}); do
+            git cat-file -p "${{ steps.fixtures-projection.outputs.tree }}:${fixture_file}"
+          done
+        ) | kubectl exec -i "${POD_NAME}" -- \
+            hab pkg exec "${PACKAGE_ORIGIN}/${PACKAGE_NAME}" \
+            mysql "${DATABASE_NAME}"
+
+        echo "Running migrations..."
+        kubectl exec "${POD_NAME}" -- \
+          hab pkg exec "${PACKAGE_ORIGIN}/${PACKAGE_NAME}" \
+           emergence-console-run migrations:execute --all
+
+
     - name: Mark deployment as failed
       if: failure()
       run: |
-        hub api /repos/${{ github.repository }}/deployments/${{ env.GH_DEPLOYMENT_ID }}/statuses  \
+        hub api "/repos/${{ github.repository }}/deployments/${GH_DEPLOYMENT_ID}/statuses"  \
           -X POST                                                                                      \
           -H "Accept: application/json, application/vnd.github.flash-preview+json"                     \
           --input <(cat <<EOF
@@ -143,7 +218,7 @@ jobs:
 
     - name: Mark deployment completed
       run: |
-        hub api /repos/${{ github.repository }}/deployments/${{ env.GH_DEPLOYMENT_ID }}/statuses  \
+        hub api "/repos/${{ github.repository }}/deployments/${GH_DEPLOYMENT_ID}/statuses"  \
           -X POST                                                                                      \
           -H "Accept: application/json, application/vnd.github.flash-preview+json"                     \
           --input <(cat <<EOF

diff --git a/.github/workflows/k8s-destroy-pr.yml b/.github/workflows/k8s-destroy-pr.yml
@@ -0,0 +1,37 @@
+name: K8s PR Sandbox Destroy
+
+on:
+  pull_request:
+    branches: [develop]
+    types: [closed]
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  KUBE_NAMESPACE: ${{ secrets.kube_namespace }}
+  KUBE_CONFIG_DATA: ${{ secrets.kube_config }}
+
+  PR_NAME: pr-${{ github.event.number }}
+
+jobs:
+
+  k8s-destroy:
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Add kubeconfig to environment
+      run: |
+        set -e
+        test -e ~/.kube || mkdir ~/.kube
+        cat <<EOF > ~/.kube/config
+        $(printf '%s' "$KUBE_CONFIG_DATA" | base64 -d)
+        EOF
+
+    - uses: azure/setup-kubectl@v1
+
+    - name: Delete PR Deployment
+      run: |
+        set -e
+        kubectl config set-context --current --namespace="${KUBE_NAMESPACE}"
+        kubectl delete all,ingress -l "app.kubernetes.io/instance=${PR_NAME}"
+        kubectl delete secret "${PR_NAME}-tls"
diff --git a/.holo/branches/emergence-site/_gatekeeper.toml b/.holo/branches/emergence-site/_gatekeeper.toml
@@ -1,7 +1,15 @@
 [holomapping]
 files = [
     "*/**",
+
+    # exclude CI and developer assets
+    "!.github/",
     "!.vscode/",
-    "!habitat/"
+    "!docs/",
+    "!fixtures/",
+    "!habitat/",
+    "!helm-chart/",
+    "!php-config/Git.config.d/",
+    "!script/",
 ]
 after = "*"
diff --git a/.holo/branches/fixtures/_gatekeeper.toml b/.holo/branches/fixtures/_gatekeeper.toml
@@ -0,0 +1,3 @@
+[holomapping]
+root = "fixtures"
+files = "*.sql"
diff --git a/.holo/sources/skeleton-v2.toml b/.holo/sources/skeleton-v2.toml
@@ -1,6 +1,6 @@
 [holosource]
 url = "https://github.com/JarvusInnovations/emergence-skeleton-v2"
-ref = "refs/tags/v2.4.1"
+ref = "refs/tags/v2.5.1"
 
 [holosource.project]
 holobranch = "emergence-skeleton"
diff --git a/Dockerfile b/Dockerfile
@@ -1,18 +1,21 @@
 # This Dockerfile is hyper-optimized to minimize layer changes
 
 FROM jarvus/habitat-compose:latest as habitat
+
 ARG HAB_LICENSE=no-accept
 ENV HAB_LICENSE=$HAB_LICENSE
 ENV STUDIO_TYPE=Dockerfile
 ENV HAB_ORIGIN=jarvus
 RUN hab origin key generate
+
 # pre-layer all external runtime plan deps
 COPY habitat/plan.sh /habitat/plan.sh
 RUN hab pkg install \
     core/bash \
     emergence/php-runtime \
     $({ cat '/habitat/plan.sh' && echo && echo 'echo "${pkg_deps[@]/$pkg_origin\/*/}"'; } | hab pkg exec core/bash bash) \
     && hab pkg exec core/coreutils rm -rf /hab/cache/{artifacts,src}/
+
 # pre-layer all external runtime composite deps
 COPY habitat/composite/plan.sh /habitat/composite/plan.sh
 RUN hab pkg install \
@@ -23,38 +26,49 @@ RUN hab pkg install \
 
 
 FROM habitat as projector
+
 # pre-layer all build-time plan deps
 RUN hab pkg install \
     core/hab-plan-build \
     jarvus/hologit \
     jarvus/toml-merge \
     $({ cat '/habitat/plan.sh' && echo && echo 'echo "${pkg_build_deps[@]/$pkg_origin\/*/}"'; } | hab pkg exec core/bash bash) \
     && hab pkg exec core/coreutils rm -rf /hab/cache/{artifacts,src}/
+
 # pre-layer all build-time composite deps
 RUN hab pkg install \
     jarvus/toml-merge \
     $({ cat '/habitat/composite/plan.sh' && echo && echo 'echo "${pkg_build_deps[@]/$pkg_origin\/*/}"'; } | hab pkg exec core/bash bash) \
     && hab pkg exec core/coreutils rm -rf /hab/cache/{artifacts,src}/
+
 # build application
 COPY . /src
+ARG SITE_TREE
+ENV SITE_TREE=$SITE_TREE
+ARG SITE_VERSION
+ENV SITE_VERSION=$SITE_VERSION
 RUN hab pkg exec core/hab-plan-build hab-plan-build /src
 RUN hab pkg exec core/hab-plan-build hab-plan-build /src/habitat/composite
 
 
 FROM habitat as runtime
-# install .hart artifact from builder stage
-COPY --from=projector /hab/cache/artifacts/$HAB_ORIGIN-* /hab/cache/artifacts/
-RUN hab pkg install /hab/cache/artifacts/$HAB_ORIGIN-* \
-    && hab pkg exec core/coreutils rm -rf /hab/cache/{artifacts,src}/
-
 
 # configure persistent volumes
 RUN hab pkg exec core/coreutils mkdir -p '/hab/svc/mysql/data' '/hab/svc/gatekeeper/data' '/hab/svc/nginx/files' \
     && hab pkg exec core/coreutils chown hab:hab -R '/hab/svc/mysql/data' '/hab/svc/gatekeeper/data' '/hab/svc/nginx/files'
 
-VOLUME ["/hab/svc/mysql/data", "/hab/svc/gatekeeper/data", "/hab/svc/nginx/files"]
-
-
 # configure entrypoint
+VOLUME ["/hab/svc/mysql/data", "/hab/svc/gatekeeper/data", "/hab/svc/nginx/files"]
 ENTRYPOINT ["hab", "sup", "run"]
 CMD ["jarvus/gatekeeper-composite"]
+
+# install .hart artifact from builder stage
+COPY --from=projector /hab/cache/artifacts/$HAB_ORIGIN-* /hab/cache/artifacts/
+RUN hab pkg install /hab/cache/artifacts/$HAB_ORIGIN-* \
+    && hab pkg exec core/coreutils rm -rf /hab/cache/{artifacts,src}/
+
+# add source metadata to environment
+ARG SOURCE_TAG
+ENV SOURCE_TAG=$SOURCE_TAG
+ARG SOURCE_COMMIT
+ENV SOURCE_COMMIT=$SOURCE_COMMIT
diff --git a/event-handlers/Gatekeeper/ApiRequestHandler/afterApiRequest/10_register-response.php b/event-handlers/Gatekeeper/ApiRequestHandler/afterApiRequest/10_register-response.php
@@ -7,7 +7,6 @@
 
 $Endpoint = $_EVENT['request']->getEndpoint();
 $userIdentifier = $_EVENT['request']->getUserIdentifier();
-$Key = $_EVENT['request']->getKey();
 
 
 // append metrics
@@ -21,10 +20,7 @@
 
 
 // drip bandwidth bucket
-if (
-    (!$Key || !$Key->RateLimitExempt) &&
-    ($Endpoint->GlobalBandwidthPeriod && $Endpoint->GlobalBandwidthCount)
-) {
+if ($Endpoint->GlobalBandwidthPeriod && $Endpoint->GlobalBandwidthCount) {
     HitBuckets::drip("endpoints/$Endpoint->ID/bandwidth", function() use ($Endpoint) {
         return [
             'seconds' => $Endpoint->GlobalBandwidthPeriod,

diff --git a/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/10_load-client-ip.php b/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/10_load-client-ip.php
@@ -0,0 +1,9 @@
+<?php
+
+namespace Gatekeeper;
+
+use Emergence\Site\Client;
+
+
+// populate request with client IP address
+$_EVENT['request']->setClientAddress(Client::getAddress());