Merge pull request #96 from JarvusInnovations/develop

Release: gatekeeper v2.4.0

.env.example

@@ -0,0 +1,11 @@
+# - Copy .env.example to .env to initialize a new instance
+# - DO NOT commit .env to source control
+# - See docs/deployment/docker-compose.md for configuration reference
+
+
+# Accept Chef Habitat's license by activating this .env
+HAB_LICENSE=accept-no-persist
+
+# MySQL database (local and remote)
+MYSQL_USERNAME=gatekeeper
+MYSQL_PASSWORD=8edvN7xotT49qLZReA2Cx1TFKpvvxK9

.github/workflows/k8s-deploy-pr.yml

@@ -0,0 +1,156 @@
+name: K8s PR Sandbox Build+Deploy
+
+on:
+  pull_request:
+    branches: [develop]
+    types: [opened, reopened, synchronize]
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  KUBE_NAMESPACE: ${{ secrets.kube_namespace }}
+  KUBE_CONFIG_DATA: ${{ secrets.kube_config }}
+
+  PACKAGE_NAME: gatekeeper-composite
+  PACKAGE_REGISTRY: docker.pkg.github.com
+
+  PR_NAME: pr-${{ github.event.number }}
+  KUBE_HOSTNAME: ${{ secrets.kube_hostname }}
+
+  HAB_LICENSE: accept-no-persist
+
+jobs:
+  kubernetes-deploy:
+
+    runs-on: ubuntu-latest
+    steps:
+
+    - uses: actions/checkout@v2
+      with:
+        ref: ${{ github.head_ref }}
+
+    - name: Create Github Deployment
+      run: |
+        set -e
+        # Create deployment
+        hub api /repos/${{ github.repository }}/deployments -X POST --input <(cat <<EOF
+        {
+          "ref": "${{ github.head_ref }}",
+          "auto_merge": false,
+          "required_contexts": [],
+          "environment": "${{ env.PR_NAME }}",
+          "transient_environment": true
+        }
+        EOF) > /tmp/deployment.json
+
+        DEPLOYMENT_ID=$(jq .id < /tmp/deployment.json)
+        echo ::set-env name=GH_DEPLOYMENT_ID::$(echo $DEPLOYMENT_ID)
+
+    - name: Update GH Deployment Status
+      run: |
+        set -e
+
+        # Set status to pending
+        hub api /repos/${{ github.repository }}/deployments/${{ env.GH_DEPLOYMENT_ID }}/statuses  \
+          -X POST                                                                                      \
+          -H "Accept: application/json, application/vnd.github.flash-preview+json"                     \
+          --input <(cat <<EOF
+        {
+          "state": "pending",
+          "description": "Building deployment artifacts",
+          "log_url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+        }
+        EOF)
+
+    - name: Update Environment
+      run: |
+        echo ::set-env name=COMMIT_MSG::$(git log --format=%B -n 1 ${{ github.event.after }})
+        echo ::set-env name=REPO_NAME::$(echo ${GITHUB_REPOSITORY,,})
+
+    - name: Build & Publish Docker image
+      uses: whoan/docker-build-with-cache-action@v5
+      with:
+        dockerfile: Dockerfile
+        username: ${{ github.actor }}
+        password: ${{ env.GITHUB_TOKEN }}
+        registry: ${{ env.PACKAGE_REGISTRY }}
+        image_name: ${{ env.REPO_NAME }}/${{ env.PACKAGE_NAME }}
+        image_tag: ${{ env.PR_NAME }}
+        build_extra_args: |
+          --build-arg=SOURCE_COMMIT=${{ github.sha }}
+          --build-arg=SOURCE_TAG=${{ env.PR_NAME }}
+          --build-arg=HAB_LICENSE=${{ env.HAB_LICENSE }}
+
+    - name: Add kubeconfig to environment
+      run: |
+        set -e
+        test -e ~/.kube || mkdir ~/.kube
+        cat <<EOF > ~/.kube/config
+        $(printf '%s' "$KUBE_CONFIG_DATA" | base64 -d)
+        EOF
+
+    - uses: azure/setup-kubectl@v1
+
+    - name: Create K8S Deployment from Template
+      run: |
+        set -e
+
+        image_id=$(echo ${{ env.REPO_NAME }}/${{ env.PACKAGE_NAME }}:${{ env.PR_NAME }})
+        image_url=$(echo docker.pkg.github.com/$image_id)
+        hostname=$(echo ${{ env.PR_NAME }}.${{ env.KUBE_HOSTNAME }})
+
+        kubectl config set-context --current --namespace=${KUBE_NAMESPACE}
+
+        # delete any existing pod first to force image to re-pull without changing tag
+        # kubectl delete pod -l app.kubernetes.io/instance="${PR_NAME}"
+        # helm uninstall ${{ env.PR_NAME }} -n ${{ env.KUBE_NAMESPACE }}
+
+        helm upgrade ${{ env.PR_NAME }} ./k8s/charts/deployment \
+          --install \
+          --set name=${{ env.PR_NAME }} \
+          --set namespace=${{ env.KUBE_NAMESPACE }} \
+          --set image=${image_url} \
+          --set hostname=${hostname}
+
+
+    - name: Wait for Deployment to be Ready
+      run: |
+        set -e
+        until kubectl rollout status deployment "${PR_NAME}" 2>/dev/null >/dev/null; do echo -n "."; sleep .5; done;
+
+    - name: Retrieve/Store Pod Name
+      run: |
+        echo ::set-env name=POD_NAME::$(kubectl get pod -l app.kubernetes.io/instance="${PR_NAME}" -o jsonpath='{.items[0].metadata.name}')
+
+    - name: Wait For Pod to be Ready
+      run: |
+        set -e
+        kubectl wait --for condition=ready "pod/${POD_NAME}" --timeout=30s
+
+    - name: Mark deployment as failed
+      if: failure()
+      run: |
+        hub api /repos/${{ github.repository }}/deployments/${{ env.GH_DEPLOYMENT_ID }}/statuses  \
+          -X POST                                                                                      \
+          -H "Accept: application/json, application/vnd.github.flash-preview+json"                     \
+          --input <(cat <<EOF
+        {
+          "state": "failure",
+          "description": "Error in job kubernetes-deploy",
+          "log_url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+        }
+        EOF)
+
+    - name: Mark deployment completed
+      run: |
+        hub api /repos/${{ github.repository }}/deployments/${{ env.GH_DEPLOYMENT_ID }}/statuses  \
+          -X POST                                                                                      \
+          -H "Accept: application/json, application/vnd.github.flash-preview+json"                     \
+          --input <(cat <<EOF
+        {
+          "state": "success",
+          "description": "${{ github.head_ref }} deployed to Kubernetes Sandbox at: https://${{env.PR_NAME}}.${{env.KUBE_HOSTNAME}}",
+          "log_url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}",
+          "environment_url": "https://${{ env.PR_NAME}}.${{ env.KUBE_HOSTNAME }}"
+        }
+        EOF)

.gitignore

@@ -5,3 +5,7 @@
 # exclude built files from source control
 /site-root/js/pages/
 /site-root/css/
+
+# ignore habitat build results
+results/
+.env

.holo/sources/skeleton-v2.toml

@@ -1,6 +1,6 @@
 [holosource]
 url = "https://github.com/JarvusInnovations/emergence-skeleton-v2"
-ref = "refs/tags/v2.4.0"
+ref = "refs/tags/v2.4.1"
 
 [holosource.project]
 holobranch = "emergence-skeleton"

Dockerfile

@@ -0,0 +1,60 @@
+# This Dockerfile is hyper-optimized to minimize layer changes
+
+FROM jarvus/habitat-compose:latest as habitat
+ARG HAB_LICENSE=no-accept
+ENV HAB_LICENSE=$HAB_LICENSE
+ENV STUDIO_TYPE=Dockerfile
+ENV HAB_ORIGIN=jarvus
+RUN hab origin key generate
+# pre-layer all external runtime plan deps
+COPY habitat/plan.sh /habitat/plan.sh
+RUN hab pkg install \
+    core/bash \
+    emergence/php-runtime \
+    $({ cat '/habitat/plan.sh' && echo && echo 'echo "${pkg_deps[@]/$pkg_origin\/*/}"'; } | hab pkg exec core/bash bash) \
+    && hab pkg exec core/coreutils rm -rf /hab/cache/{artifacts,src}/
+# pre-layer all external runtime composite deps
+COPY habitat/composite/plan.sh /habitat/composite/plan.sh
+RUN hab pkg install \
+    jarvus/habitat-compose \
+    emergence/nginx \
+    $({ cat '/habitat/composite/plan.sh' && echo && echo 'echo "${pkg_deps[@]/$pkg_origin\/*/} ${composite_mysql_pkg}'; } | hab pkg exec core/bash bash) \
+    && hab pkg exec core/coreutils rm -rf /hab/cache/{artifacts,src}/
+
+
+FROM habitat as projector
+# pre-layer all build-time plan deps
+RUN hab pkg install \
+    core/hab-plan-build \
+    jarvus/hologit \
+    jarvus/toml-merge \
+    $({ cat '/habitat/plan.sh' && echo && echo 'echo "${pkg_build_deps[@]/$pkg_origin\/*/}"'; } | hab pkg exec core/bash bash) \
+    && hab pkg exec core/coreutils rm -rf /hab/cache/{artifacts,src}/
+# pre-layer all build-time composite deps
+RUN hab pkg install \
+    jarvus/toml-merge \
+    $({ cat '/habitat/composite/plan.sh' && echo && echo 'echo "${pkg_build_deps[@]/$pkg_origin\/*/}"'; } | hab pkg exec core/bash bash) \
+    && hab pkg exec core/coreutils rm -rf /hab/cache/{artifacts,src}/
+# build application
+COPY . /src
+RUN hab pkg exec core/hab-plan-build hab-plan-build /src
+RUN hab pkg exec core/hab-plan-build hab-plan-build /src/habitat/composite
+
+
+FROM habitat as runtime
+# install .hart artifact from builder stage
+COPY --from=projector /hab/cache/artifacts/$HAB_ORIGIN-* /hab/cache/artifacts/
+RUN hab pkg install /hab/cache/artifacts/$HAB_ORIGIN-* \
+    && hab pkg exec core/coreutils rm -rf /hab/cache/{artifacts,src}/
+
+
+# configure persistent volumes
+RUN hab pkg exec core/coreutils mkdir -p '/hab/svc/mysql/data' '/hab/svc/gatekeeper/data' '/hab/svc/nginx/files' \
+    && hab pkg exec core/coreutils chown hab:hab -R '/hab/svc/mysql/data' '/hab/svc/gatekeeper/data' '/hab/svc/nginx/files'
+
+VOLUME ["/hab/svc/mysql/data", "/hab/svc/gatekeeper/data", "/hab/svc/nginx/files"]
+
+
+# configure entrypoint
+ENTRYPOINT ["hab", "sup", "run"]
+CMD ["jarvus/gatekeeper-composite"]

docker-compose.override.yml

@@ -0,0 +1,14 @@
+version: '3'
+services:
+  app:
+    environment:
+
+      # assign any username/password--both app and db will pick it up and initialize
+      HAB_MYSQL: |
+        # these can be made up:
+        app_username = '${MYSQL_USERNAME}'
+        app_password = '${MYSQL_PASSWORD}'
+        bind = '0.0.0.0'
+
+    ports:
+      - 127.0.0.1:3306:3306 # expose mysql on localhost

docker-compose.yml

@@ -0,0 +1,19 @@
+version: '3'
+services:
+  app:
+    # create via `hab pkg export docker --multi-layer ./gatekeeper-composite.hart`:
+    image: jarvus/gatekeeper-composite:latest
+    volumes:
+      # persist uploaded media:
+      - gatekeeper-app-data:/hab/svc/gatekeeper/data
+      # persist uploaded SSL certificates:
+      - ./nginx-files:/hab/svc/nginx/files
+    ports:
+      - 80:80
+    environment:
+      HAB_LICENSE: ${HAB_LICENSE}
+volumes:
+  gatekeeper-app-data:
+    driver: local
+  gatekeeper-nginx-files:
+    driver: local

docs/gatekeeper/deployments/pr-deployments.md

@@ -0,0 +1,29 @@
+# PR Deployments:
+
+## Github Secrets:
+
+- `kube_namespace` (required)
+Set this to the kubernetes namespace you wish for the app to be deployed to.
+
+- `kube_config` (required)
+Set this to the value **base64 encoded** kubeconfig file used to configure access to the cluster.
+
+- `kube_hostname` (required)
+Set this to the hostname of the kubernetes cluster.
+*Hint:* If the `kube_hostname` is set to example.com, the app will be deployed to `pr-1.example.com`
+
+## Kubernetes Secrets
+
+- `regcred` (required)
+Set this in the kubernetes cluster, within the `kube_namespace` to the login credentials for docker.
+
+
+## Helm Configuration
+
+#### Helm Template
+
+Located at `k8s/charts/deployment/`
+
+- `templates/` - Helm templates for PR Deployments
+- `values.yaml` - Default configurable values for PR deployments
+

event-handlers/Gatekeeper/ApiRequestHandler/afterApiRequest/10_register-response.php

@@ -7,6 +7,7 @@
 
 $Endpoint = $_EVENT['request']->getEndpoint();
 $userIdentifier = $_EVENT['request']->getUserIdentifier();
+$Key = $_EVENT['request']->getKey();
 
 
 // append metrics
@@ -20,11 +21,14 @@
 
 
 // drip bandwidth bucket
-if ($Endpoint->GlobalBandwidthPeriod && $Endpoint->GlobalBandwidthCount) {
+if (
+    (!$Key || !$Key->RateLimitExempt) &&
+    ($Endpoint->GlobalBandwidthPeriod && $Endpoint->GlobalBandwidthCount)
+) {
     HitBuckets::drip("endpoints/$Endpoint->ID/bandwidth", function() use ($Endpoint) {
         return [
             'seconds' => $Endpoint->GlobalBandwidthPeriod,
             'count' => $Endpoint->GlobalBandwidthCount
         ];
     }, $_EVENT['Transaction']->ResponseBytes);
-}
+}

event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/75_reject-endpoint-user-rate.php

@@ -5,10 +5,14 @@
 
 $Endpoint = $_EVENT['request']->getEndpoint();
 $userIdentifier = $_EVENT['request']->getUserIdentifier();
+$Key = $_EVENT['request']->getKey();
 
 
 // drip into endpoint+user bucket first so that abusive users can't pollute the global bucket
-if ($Endpoint->UserRatePeriod && $Endpoint->UserRateCount) {
+if (
+    (!$Key || !$Key->RateLimitExempt) &&
+    ($Endpoint->UserRatePeriod && $Endpoint->UserRateCount)
+) {
     $bucket = HitBuckets::drip("endpoints/$Endpoint->ID/$userIdentifier", function() use ($Endpoint) {
         return array('seconds' => $Endpoint->UserRatePeriod, 'count' => $Endpoint->UserRateCount);
     });

event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/95_reject-endpoint-rate.php

@@ -8,10 +8,14 @@
 
 
 $Endpoint = $_EVENT['request']->getEndpoint();
+$Key = $_EVENT['request']->getKey();
 
 
 // drip into endpoint requests bucket
-if ($Endpoint->GlobalRatePeriod && $Endpoint->GlobalRateCount) {
+if (
+    (!$Key || !$Key->RateLimitExempt) &&
+    ($Endpoint->GlobalRatePeriod && $Endpoint->GlobalRateCount)
+) {
     $flagKey = "alerts/endpoints/$Endpoint->ID/rate-flagged";
 
     $bucket = HitBuckets::drip("endpoints/$Endpoint->ID/requests", function() use ($Endpoint) {

habitat/composite/default.toml

@@ -0,0 +1,2 @@
+[services.app.config]
+    default_timezone = "America/New_York"

habitat/composite/plan.sh

@@ -0,0 +1,10 @@
+composite_app_pkg_name=gatekeeper
+pkg_name="${composite_app_pkg_name}-composite"
+pkg_origin=jarvus
+pkg_maintainer="Jarvus Innovations <[email protected]>"
+pkg_scaffolding=emergence/scaffolding-composite
+composite_mysql_pkg=core/mysql
+
+pkg_version() {
+  scaffolding_detect_pkg_version
+}

habitat/default.toml

@@ -0,0 +1,2 @@
+[sites.default]
+database = "gatekeeper"

habitat/plan.sh

@@ -0,0 +1,8 @@
+pkg_name=gatekeeper
+pkg_origin=jarvus
+pkg_maintainer="Jarvus Innovations <[email protected]>"
+pkg_scaffolding=emergence/scaffolding-site
+
+pkg_version() {
+  scaffolding_detect_pkg_version
+}