From 8f070faac72d8ccfeea621cfe1e38bf8c17cafdd Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Fri, 4 Dec 2020 04:29:12 +0000 Subject: [PATCH 01/40] refactor: move k8s/charts/deployment to helm-chart --- .github/workflows/k8s-deploy-pr.yml | 4 ++-- {k8s/charts/deployment => helm-chart}/Chart.yaml | 0 {k8s/charts/deployment => helm-chart}/templates/_helpers.tpl | 0 .../deployment => helm-chart}/templates/deployment.yaml | 0 {k8s/charts/deployment => helm-chart}/values.yaml | 0 5 files changed, 2 insertions(+), 2 deletions(-) rename {k8s/charts/deployment => helm-chart}/Chart.yaml (100%) rename {k8s/charts/deployment => helm-chart}/templates/_helpers.tpl (100%) rename {k8s/charts/deployment => helm-chart}/templates/deployment.yaml (100%) rename {k8s/charts/deployment => helm-chart}/values.yaml (100%) diff --git a/.github/workflows/k8s-deploy-pr.yml b/.github/workflows/k8s-deploy-pr.yml index c2107ee..904876c 100644 --- a/.github/workflows/k8s-deploy-pr.yml +++ b/.github/workflows/k8s-deploy-pr.yml @@ -20,7 +20,7 @@ env: HAB_LICENSE: accept-no-persist jobs: - kubernetes-deploy: + k8s-deploy: runs-on: ubuntu-latest steps: @@ -105,7 +105,7 @@ jobs: # kubectl delete pod -l app.kubernetes.io/instance="${PR_NAME}" # helm uninstall ${{ env.PR_NAME }} -n ${{ env.KUBE_NAMESPACE }} - helm upgrade ${{ env.PR_NAME }} ./k8s/charts/deployment \ + helm upgrade ${{ env.PR_NAME }} ./helm-chart \ --install \ --set name=${{ env.PR_NAME }} \ --set namespace=${{ env.KUBE_NAMESPACE }} \ diff --git a/k8s/charts/deployment/Chart.yaml b/helm-chart/Chart.yaml similarity index 100% rename from k8s/charts/deployment/Chart.yaml rename to helm-chart/Chart.yaml diff --git a/k8s/charts/deployment/templates/_helpers.tpl b/helm-chart/templates/_helpers.tpl similarity index 100% rename from k8s/charts/deployment/templates/_helpers.tpl rename to helm-chart/templates/_helpers.tpl diff --git a/k8s/charts/deployment/templates/deployment.yaml b/helm-chart/templates/deployment.yaml similarity index 100% rename from k8s/charts/deployment/templates/deployment.yaml rename to helm-chart/templates/deployment.yaml diff --git a/k8s/charts/deployment/values.yaml b/helm-chart/values.yaml similarity index 100% rename from k8s/charts/deployment/values.yaml rename to helm-chart/values.yaml From 749932a36611e51f0922cb64eeadcf3c6f41b623 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Fri, 4 Dec 2020 04:29:18 +0000 Subject: [PATCH 02/40] feat: add destroy workflow --- .github/workflows/k8s-destroy-pr.yml | 36 ++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/k8s-destroy-pr.yml diff --git a/.github/workflows/k8s-destroy-pr.yml b/.github/workflows/k8s-destroy-pr.yml new file mode 100644 index 0000000..fa72d04 --- /dev/null +++ b/.github/workflows/k8s-destroy-pr.yml @@ -0,0 +1,36 @@ +name: K8s PR Sandbox Destroy + +on: + pull_request: + branches: [develop] + types: [closed] + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + KUBE_NAMESPACE: ${{ secrets.kube_namespace }} + KUBE_CONFIG_DATA: ${{ secrets.kube_config }} + + PR_NAME: pr-${{ github.event.number }} + +jobs: + + k8s-destroy: + runs-on: ubuntu-latest + steps: + + - name: Add kubeconfig to environment + run: | + set -e + test -e ~/.kube || mkdir ~/.kube + cat < ~/.kube/config + $(printf '%s' "$KUBE_CONFIG_DATA" | base64 -d) + EOF + + - uses: azure/setup-kubectl@v1 + + - name: Delete PR Deployment + run: | + set -e + kubectl config set-context --current --namespace=$KUBE_NAMESPACE + kubectl delete all,ing -l pr=$PR_NAME \ No newline at end of file From 81451016d9621810655c25fde814cc2433db3bb2 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Fri, 4 Dec 2020 04:31:16 +0000 Subject: [PATCH 03/40] fix: upgrade set-env usage in github workflow --- .github/workflows/k8s-deploy-pr.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/k8s-deploy-pr.yml b/.github/workflows/k8s-deploy-pr.yml index 904876c..b5f47c3 100644 --- a/.github/workflows/k8s-deploy-pr.yml +++ b/.github/workflows/k8s-deploy-pr.yml @@ -44,7 +44,7 @@ jobs: EOF) > /tmp/deployment.json DEPLOYMENT_ID=$(jq .id < /tmp/deployment.json) - echo ::set-env name=GH_DEPLOYMENT_ID::$(echo $DEPLOYMENT_ID) + echo "GH_DEPLOYMENT_ID=$(echo $DEPLOYMENT_ID)" >> $GITHUB_ENV - name: Update GH Deployment Status run: | @@ -64,8 +64,8 @@ jobs: - name: Update Environment run: | - echo ::set-env name=COMMIT_MSG::$(git log --format=%B -n 1 ${{ github.event.after }}) - echo ::set-env name=REPO_NAME::$(echo ${GITHUB_REPOSITORY,,}) + echo "COMMIT_MSG=$(git log --format=%B -n 1 ${{ github.event.after }})" >> $GITHUB_ENV + echo "REPO_NAME=$(echo ${GITHUB_REPOSITORY,,})" >> $GITHUB_ENV - name: Build & Publish Docker image uses: whoan/docker-build-with-cache-action@v5 @@ -120,7 +120,7 @@ jobs: - name: Retrieve/Store Pod Name run: | - echo ::set-env name=POD_NAME::$(kubectl get pod -l app.kubernetes.io/instance="${PR_NAME}" -o jsonpath='{.items[0].metadata.name}') + echo "POD_NAME=$(kubectl get pod -l app.kubernetes.io/instance="${PR_NAME}" -o jsonpath='{.items[0].metadata.name}')" >> $GITHUB_ENV - name: Wait For Pod to be Ready run: | From d91b6c8e58b3778fd00e86432f3ac2d9fb8b3fb1 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Fri, 4 Dec 2020 04:46:48 +0000 Subject: [PATCH 04/40] fix: remove unused COMMIT_MSG env variable --- .github/workflows/k8s-deploy-pr.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/k8s-deploy-pr.yml b/.github/workflows/k8s-deploy-pr.yml index b5f47c3..a6930d6 100644 --- a/.github/workflows/k8s-deploy-pr.yml +++ b/.github/workflows/k8s-deploy-pr.yml @@ -64,7 +64,6 @@ jobs: - name: Update Environment run: | - echo "COMMIT_MSG=$(git log --format=%B -n 1 ${{ github.event.after }})" >> $GITHUB_ENV echo "REPO_NAME=$(echo ${GITHUB_REPOSITORY,,})" >> $GITHUB_ENV - name: Build & Publish Docker image From 78d2faeb38f858f14fb71c42d685b9b0469ea449 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Fri, 4 Dec 2020 04:49:49 +0000 Subject: [PATCH 05/40] feat: add timeout to wait for deployment step --- .github/workflows/k8s-deploy-pr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/k8s-deploy-pr.yml b/.github/workflows/k8s-deploy-pr.yml index a6930d6..4ff2776 100644 --- a/.github/workflows/k8s-deploy-pr.yml +++ b/.github/workflows/k8s-deploy-pr.yml @@ -113,6 +113,7 @@ jobs: - name: Wait for Deployment to be Ready + timeout-minutes: 10 run: | set -e until kubectl rollout status deployment "${PR_NAME}" 2>/dev/null >/dev/null; do echo -n "."; sleep .5; done; From dbaf2a8795722d42e47e0a2b3c4544821624d353 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Mon, 28 Sep 2020 16:14:42 -0400 Subject: [PATCH 06/40] fix: convert integer IPs to dotted-decimal during migration to IPPattern --- .../Gatekeeper/Ban/20200617000000_ippattern-column.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/php-migrations/Gatekeeper/Ban/20200617000000_ippattern-column.php b/php-migrations/Gatekeeper/Ban/20200617000000_ippattern-column.php index 0fca117..e95aff2 100644 --- a/php-migrations/Gatekeeper/Ban/20200617000000_ippattern-column.php +++ b/php-migrations/Gatekeeper/Ban/20200617000000_ippattern-column.php @@ -34,7 +34,7 @@ printf("Migrating values for deprecated column: `%s` -> `%s`", $deprecatedColumn, $columnName); DB::nonQuery( ' - UPDATE `%1$s` SET %2$s = %3$s + UPDATE `%1$s` SET %2$s = INET_NTOA(%3$s) WHERE %3$s IS NOT NULL ', [ @@ -53,4 +53,4 @@ ] ); -return static::STATUS_EXECUTED; \ No newline at end of file +return static::STATUS_EXECUTED; From f8e29b3231c5d337362387440d077e9d5c23e3d1 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Fri, 4 Dec 2020 05:11:54 +0000 Subject: [PATCH 07/40] fix: delete specific types to comply with limited access --- .github/workflows/k8s-destroy-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/k8s-destroy-pr.yml b/.github/workflows/k8s-destroy-pr.yml index fa72d04..9afcc97 100644 --- a/.github/workflows/k8s-destroy-pr.yml +++ b/.github/workflows/k8s-destroy-pr.yml @@ -33,4 +33,4 @@ jobs: run: | set -e kubectl config set-context --current --namespace=$KUBE_NAMESPACE - kubectl delete all,ing -l pr=$PR_NAME \ No newline at end of file + kubectl delete deployment,service,ingress -l pr=$PR_NAME From b841f5c56ed1ed49b1f844565f6c7408f8ce3688 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Tue, 8 Dec 2020 05:39:58 +0000 Subject: [PATCH 08/40] fix: delete closed PR environments --- .github/workflows/k8s-destroy-pr.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/k8s-destroy-pr.yml b/.github/workflows/k8s-destroy-pr.yml index 9afcc97..aa52abe 100644 --- a/.github/workflows/k8s-destroy-pr.yml +++ b/.github/workflows/k8s-destroy-pr.yml @@ -32,5 +32,6 @@ jobs: - name: Delete PR Deployment run: | set -e - kubectl config set-context --current --namespace=$KUBE_NAMESPACE - kubectl delete deployment,service,ingress -l pr=$PR_NAME + kubectl config set-context --current --namespace="${KUBE_NAMESPACE}" + kubectl delete all,ingress -l "app.kubernetes.io/instance=${PR_NAME}" + kubectl delete secret "${PR_NAME}-tls" From 0418b3bb02040a3e78d50c97fa72f80370d8918c Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Tue, 22 Dec 2020 14:15:45 +0000 Subject: [PATCH 09/40] chore(ci): upgrade k8s deploy flow --- .github/workflows/k8s-deploy-pr.yml | 117 ++++++++++++++++++++++------ 1 file changed, 93 insertions(+), 24 deletions(-) diff --git a/.github/workflows/k8s-deploy-pr.yml b/.github/workflows/k8s-deploy-pr.yml index 4ff2776..05b730b 100644 --- a/.github/workflows/k8s-deploy-pr.yml +++ b/.github/workflows/k8s-deploy-pr.yml @@ -11,12 +11,15 @@ env: KUBE_NAMESPACE: ${{ secrets.kube_namespace }} KUBE_CONFIG_DATA: ${{ secrets.kube_config }} + PACKAGE_ORIGIN: jarvus PACKAGE_NAME: gatekeeper-composite PACKAGE_REGISTRY: docker.pkg.github.com PR_NAME: pr-${{ github.event.number }} KUBE_HOSTNAME: ${{ secrets.kube_hostname }} + DATABASE_NAME: gatekeeper + HAB_LICENSE: accept-no-persist jobs: @@ -29,11 +32,17 @@ jobs: with: ref: ${{ github.head_ref }} + - name: 'Initialize Chef Habitat environment' + uses: JarvusInnovations/habitat-action@action/v1 + with: + deps: | + jarvus/hologit + - name: Create Github Deployment run: | set -e # Create deployment - hub api /repos/${{ github.repository }}/deployments -X POST --input <(cat < /tmp/deployment.json - DEPLOYMENT_ID=$(jq .id < /tmp/deployment.json) - echo "GH_DEPLOYMENT_ID=$(echo $DEPLOYMENT_ID)" >> $GITHUB_ENV + DEPLOYMENT_ID="$(jq .id < /tmp/deployment.json)" + echo "GH_DEPLOYMENT_ID=${DEPLOYMENT_ID}" >> $GITHUB_ENV - name: Update GH Deployment Status run: | set -e # Set status to pending - hub api /repos/${{ github.repository }}/deployments/${{ env.GH_DEPLOYMENT_ID }}/statuses \ + hub api "/repos/${{ github.repository }}/deployments/${GH_DEPLOYMENT_ID}/statuses" \ -X POST \ -H "Accept: application/json, application/vnd.github.flash-preview+json" \ --input <(cat <> $GITHUB_ENV + echo "REPO_NAME=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV + + - id: site-projection + name: 'Project holobranch: emergence-site' + uses: JarvusInnovations/hologit@actions/projector/v1 + with: + holobranch: emergence-site + + - id: fixtures-projection + name: 'Project holobranch: fixtures' + uses: JarvusInnovations/hologit@actions/projector/v1 + with: + holobranch: fixtures - name: Build & Publish Docker image uses: whoan/docker-build-with-cache-action@v5 @@ -76,6 +97,8 @@ jobs: image_name: ${{ env.REPO_NAME }}/${{ env.PACKAGE_NAME }} image_tag: ${{ env.PR_NAME }} build_extra_args: | + --build-arg=SITE_TREE=${{ steps.site-projection.outputs.tree }} + --build-arg=SITE_VERSION=0.0.0-pr.${{ github.event.number }} --build-arg=SOURCE_COMMIT=${{ github.sha }} --build-arg=SOURCE_TAG=${{ env.PR_NAME }} --build-arg=HAB_LICENSE=${{ env.HAB_LICENSE }} @@ -88,35 +111,50 @@ jobs: $(printf '%s' "$KUBE_CONFIG_DATA" | base64 -d) EOF - - uses: azure/setup-kubectl@v1 - - - name: Create K8S Deployment from Template + - name: Deploy instance via Helm template run: | set -e - image_id=$(echo ${{ env.REPO_NAME }}/${{ env.PACKAGE_NAME }}:${{ env.PR_NAME }}) - image_url=$(echo docker.pkg.github.com/$image_id) - hostname=$(echo ${{ env.PR_NAME }}.${{ env.KUBE_HOSTNAME }}) + image_id="${REPO_NAME}/${PACKAGE_NAME}:${PR_NAME}" + image_url="docker.pkg.github.com/${image_id}" + hostname="${PR_NAME}.${KUBE_HOSTNAME}" - kubectl config set-context --current --namespace=${KUBE_NAMESPACE} + kubectl config set-context --current --namespace="${KUBE_NAMESPACE}" - # delete any existing pod first to force image to re-pull without changing tag - # kubectl delete pod -l app.kubernetes.io/instance="${PR_NAME}" - # helm uninstall ${{ env.PR_NAME }} -n ${{ env.KUBE_NAMESPACE }} + echo "Listing pods existing before deploy" + kubectl get pods \ + -l app.kubernetes.io/instance="${PR_NAME}" \ + --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}' \ + | sort \ + | tee ./.pods-before - helm upgrade ${{ env.PR_NAME }} ./helm-chart \ + echo "Using helm upgrade to apply ./helm-chart to release ${PR_NAME}" + helm upgrade "${PR_NAME}" ./helm-chart \ --install \ - --set name=${{ env.PR_NAME }} \ - --set namespace=${{ env.KUBE_NAMESPACE }} \ - --set image=${image_url} \ - --set hostname=${hostname} - + --set name="${PR_NAME}" \ + --set namespace="${KUBE_NAMESPACE}" \ + --set image="${image_url}" \ + --set hostname="${hostname}" + + echo "Listing pods existing after deploy" + kubectl get pods \ + -l app.kubernetes.io/instance="${PR_NAME}" \ + --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}' \ + | sort \ + | tee ./.pods-after + + echo "Deleting stale pods to force image refresh" + comm -12 ./.pods-before ./.pods-after \ + | xargs --no-run-if-empty kubectl delete pod - name: Wait for Deployment to be Ready timeout-minutes: 10 run: | set -e - until kubectl rollout status deployment "${PR_NAME}" 2>/dev/null >/dev/null; do echo -n "."; sleep .5; done; + until kubectl rollout status deployment "${PR_NAME}" 2>/dev/null >/dev/null; do + echo -n "." + sleep .5 + done - name: Retrieve/Store Pod Name run: | @@ -127,10 +165,41 @@ jobs: set -e kubectl wait --for condition=ready "pod/${POD_NAME}" --timeout=30s + - name: Wait for MySQL to be Ready + timeout-minutes: 5 + run: | + set -e + until kubectl exec "${POD_NAME}" -- hab pkg exec "${PACKAGE_ORIGIN}/${PACKAGE_NAME}" mysqladmin ping; do + sleep .5 + done + + - name: Load fixtures into database + run: | + echo "Dropping any existing database..." + kubectl exec "${POD_NAME}" -- \ + hab pkg exec "${PACKAGE_ORIGIN}/${PACKAGE_NAME}" \ + mysqladmin drop "${DATABASE_NAME}" --force \ + || true + + echo "Creating an empty database..." + kubectl exec "${POD_NAME}" -- \ + hab pkg exec "${PACKAGE_ORIGIN}/${PACKAGE_NAME}" \ + mysqladmin create "${DATABASE_NAME}" + + echo "Loading fixtures..." + ( + for fixture_file in $(git ls-tree -r --name-only ${{ steps.fixtures-projection.outputs.tree }}); do + git cat-file -p "${{ steps.fixtures-projection.outputs.tree }}:${fixture_file}" + done + ) | kubectl exec -i "${POD_NAME}" -- \ + hab pkg exec "${PACKAGE_ORIGIN}/${PACKAGE_NAME}" \ + mysql "${DATABASE_NAME}" + + - name: Mark deployment as failed if: failure() run: | - hub api /repos/${{ github.repository }}/deployments/${{ env.GH_DEPLOYMENT_ID }}/statuses \ + hub api "/repos/${{ github.repository }}/deployments/${GH_DEPLOYMENT_ID}/statuses" \ -X POST \ -H "Accept: application/json, application/vnd.github.flash-preview+json" \ --input <(cat < Date: Tue, 22 Dec 2020 15:47:42 +0000 Subject: [PATCH 10/40] feat(ci): declare fixtures holobranch --- .holo/branches/fixtures/_gatekeeper.toml | 3 +++ fixtures/people.sql | 0 2 files changed, 3 insertions(+) create mode 100644 .holo/branches/fixtures/_gatekeeper.toml create mode 100644 fixtures/people.sql diff --git a/.holo/branches/fixtures/_gatekeeper.toml b/.holo/branches/fixtures/_gatekeeper.toml new file mode 100644 index 0000000..9cadfd8 --- /dev/null +++ b/.holo/branches/fixtures/_gatekeeper.toml @@ -0,0 +1,3 @@ +[holomapping] +root = "fixtures" +files = "*.sql" diff --git a/fixtures/people.sql b/fixtures/people.sql new file mode 100644 index 0000000..e69de29 From 23166e515dd37e97c21e7ee901fd42d59070d086 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Tue, 22 Dec 2020 15:47:59 +0000 Subject: [PATCH 11/40] feat(stra): add dump-table developer command --- script/dump-table | 56 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100755 script/dump-table diff --git a/script/dump-table b/script/dump-table new file mode 100755 index 0000000..a66020d --- /dev/null +++ b/script/dump-table @@ -0,0 +1,56 @@ +#!/bin/bash + + +# get input +SITE_NAMESPACE="${SITE_NAMESPACE:-gatekeeper}" +SITE_DATABASE="${SITE_DATABASE:-gatekeeper}" +SITE_COMPOSITE="${SITE_COMPOSITE:-jarvus/gatekeeper-composite}" + +instance="${1:?Usage: dump-table }" +table="${2:?Usage: dump-table
}" + + +# configuration +DUMP_OPTIONS=( + --force + --skip-opt + --skip-comments + --skip-dump-date + --tz-utc + --create-options + --order-by-primary + --single-transaction + --quick +) + + +# collect identifiers +>&2 echo +>&2 echo "==> dump-table: examining pod…" + +pod=$( + kubectl -n "${SITE_NAMESPACE}" get pod \ + -l app.kubernetes.io/instance="${instance}" \ + -o jsonpath='{.items[0].metadata.name}' +) + +if [ -z "${pod}" ]; then + >&2 echo " Failed to find pod for instance ${instance} in namespace ${SITE_NAMESPACE}" + >&2 echo + >&2 echo " Is the correct KUBECONFIG exported in the current shell?" + exit 1 +fi + +>&2 echo " pod=${SITE_NAMESPACE}/${pod}" + + +# dump table +>&2 echo +>&2 echo "==> dump-table: dumping table ${table}…" + +kubectl -n "${SITE_NAMESPACE}" exec -i "${pod}" -- \ + hab pkg exec "${SITE_COMPOSITE}" \ + mysqldump "${DUMP_OPTIONS[@]}" \ + "${SITE_DATABASE}" \ + "${table}" \ + "history_${table}" From 60235721d034105230a7eed77929ffad6498903c Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Tue, 22 Dec 2020 15:50:20 +0000 Subject: [PATCH 12/40] chore(ci): filter extra assets out of site projection --- .holo/branches/emergence-site/_gatekeeper.toml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.holo/branches/emergence-site/_gatekeeper.toml b/.holo/branches/emergence-site/_gatekeeper.toml index 5cbf66c..15d7d9b 100644 --- a/.holo/branches/emergence-site/_gatekeeper.toml +++ b/.holo/branches/emergence-site/_gatekeeper.toml @@ -1,7 +1,15 @@ [holomapping] files = [ "*/**", + + # exclude CI and developer assets + "!.github/", "!.vscode/", - "!habitat/" + "!docs/", + "!fixtures/", + "!habitat/", + "!helm-chart/", + "!php-config/Git.config.d/", + "!script/", ] after = "*" From 3930771c87793f987b51b68def6c1a7319dc3be4 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Wed, 23 Dec 2020 03:27:01 +0000 Subject: [PATCH 13/40] feat(fixtures): add people --- fixtures/people.sql | 59 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/fixtures/people.sql b/fixtures/people.sql index e69de29..349eea2 100644 --- a/fixtures/people.sql +++ b/fixtures/people.sql @@ -0,0 +1,59 @@ +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40101 SET character_set_client = utf8 */; + +CREATE TABLE `people` ( + `ID` int(10) unsigned NOT NULL AUTO_INCREMENT, + `Class` enum('Emergence\\People\\Person','Emergence\\People\\User') NOT NULL, + `Created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `CreatorID` int(11) DEFAULT NULL, + `Modified` timestamp NULL DEFAULT NULL, + `ModifierID` int(10) unsigned DEFAULT NULL, + `FirstName` varchar(255) NOT NULL, + `LastName` varchar(255) NOT NULL, + `MiddleName` varchar(255) DEFAULT NULL, + `PreferredName` varchar(255) DEFAULT NULL, + `Gender` enum('Male','Female') DEFAULT NULL, + `BirthDate` date DEFAULT NULL, + `Email` varchar(255) DEFAULT NULL, + `Phone` decimal(15,0) unsigned DEFAULT NULL, + `Location` varchar(255) DEFAULT NULL, + `About` text, + `PrimaryPhotoID` int(10) unsigned DEFAULT NULL, + `Username` varchar(255) DEFAULT NULL, + `Password` varchar(255) DEFAULT NULL, + `AccountLevel` enum('Disabled','Contact','User','Staff','Administrator','Developer') DEFAULT 'User', + PRIMARY KEY (`ID`), + UNIQUE KEY `Email` (`Email`), + UNIQUE KEY `Username` (`Username`) +) ENGINE=MyISAM AUTO_INCREMENT=2 DEFAULT CHARSET=utf8; + +INSERT INTO `people` VALUES (1,'Emergence\\People\\User','2019-01-02 03:04:05',1,NULL,NULL,'Admin','Person',NULL,NULL,NULL,NULL,'admin@example.com',NULL,NULL,NULL,NULL,'admin','$2y$10$rAOnrPHjxdyr40NnSphAaOqMptte76N2BwmFeMlwulpjQNKHKZ1uK','Developer'); + + +CREATE TABLE `history_people` ( + `RevisionID` int(10) unsigned NOT NULL AUTO_INCREMENT, + `ID` int(10) unsigned NOT NULL, + `Class` enum('Emergence\\People\\Person','Emergence\\People\\User') NOT NULL, + `Created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `CreatorID` int(11) DEFAULT NULL, + `Modified` timestamp NULL DEFAULT NULL, + `ModifierID` int(10) unsigned DEFAULT NULL, + `FirstName` varchar(255) NOT NULL, + `LastName` varchar(255) NOT NULL, + `MiddleName` varchar(255) DEFAULT NULL, + `PreferredName` varchar(255) DEFAULT NULL, + `Gender` enum('Male','Female') DEFAULT NULL, + `BirthDate` date DEFAULT NULL, + `Email` varchar(255) DEFAULT NULL, + `Phone` decimal(15,0) unsigned DEFAULT NULL, + `Location` varchar(255) DEFAULT NULL, + `About` text, + `PrimaryPhotoID` int(10) unsigned DEFAULT NULL, + `Username` varchar(255) DEFAULT NULL, + `Password` varchar(255) DEFAULT NULL, + `AccountLevel` enum('Disabled','Contact','User','Staff','Administrator','Developer') DEFAULT 'User', + PRIMARY KEY (`RevisionID`), + KEY `ID` (`ID`) +) ENGINE=MyISAM DEFAULT CHARSET=utf8; + +INSERT INTO `history_people` SELECT NULL AS RevisionID, `people`.* FROM `people`; From 9541b4f776e368070f43aba4e8ac6018bc484f9b Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Wed, 23 Dec 2020 03:47:35 +0000 Subject: [PATCH 14/40] feat(fixtures): add endpoints --- fixtures/endpoints.sql | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 fixtures/endpoints.sql diff --git a/fixtures/endpoints.sql b/fixtures/endpoints.sql new file mode 100644 index 0000000..bdbf150 --- /dev/null +++ b/fixtures/endpoints.sql @@ -0,0 +1,37 @@ +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40101 SET character_set_client = utf8 */; + +CREATE TABLE `endpoints` ( + `ID` int(10) unsigned NOT NULL AUTO_INCREMENT, + `Class` enum('Gatekeeper\\Endpoints\\Endpoint') NOT NULL, + `Created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `CreatorID` int(11) DEFAULT NULL, + `Title` varchar(255) NOT NULL, + `Handle` varchar(255) NOT NULL, + `Path` varchar(255) NOT NULL, + `InternalEndpoint` varchar(255) NOT NULL, + `AdminName` varchar(255) DEFAULT NULL, + `AdminEmail` varchar(255) DEFAULT NULL, + `Public` tinyint(1) NOT NULL DEFAULT '0', + `Description` text, + `DeprecationDate` timestamp NULL DEFAULT NULL, + `GlobalRateCount` int(10) unsigned DEFAULT NULL, + `GlobalRatePeriod` int(10) unsigned DEFAULT NULL, + `UserRateCount` int(10) unsigned DEFAULT NULL, + `UserRatePeriod` int(10) unsigned DEFAULT NULL, + `GlobalBandwidthCount` int(10) unsigned DEFAULT NULL, + `GlobalBandwidthPeriod` int(10) unsigned DEFAULT NULL, + `KeyRequired` tinyint(1) NOT NULL DEFAULT '0', + `KeySelfRegistration` tinyint(1) NOT NULL DEFAULT '0', + `CachingEnabled` tinyint(1) NOT NULL DEFAULT '1', + `AlertOnError` tinyint(1) NOT NULL DEFAULT '1', + `AlertNearMaxRequests` decimal(3,2) DEFAULT NULL, + `PingFrequency` int(10) unsigned DEFAULT NULL, + `PingURI` varchar(255) DEFAULT NULL, + `PingTestPattern` varchar(255) DEFAULT NULL, + PRIMARY KEY (`ID`), + UNIQUE KEY `Handle` (`Handle`), + UNIQUE KEY `Path` (`Path`) +) ENGINE=MyISAM AUTO_INCREMENT=2 DEFAULT CHARSET=utf8; + +INSERT INTO `endpoints` VALUES (1,'Gatekeeper\\Endpoints\\Endpoint','2019-01-02 03:04:05',1,'TODOs v1','todos-v1','todos/v1','https://jsonplaceholder.typicode.com/todos',NULL,NULL,1,NULL,NULL,10,60,1,1,NULL,NULL,0,0,1,0,NULL,NULL,NULL,NULL); From 9e49cfcbbd1d9f1ceedf19af514c3c22f0df606a Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 27 Dec 2020 03:12:57 +0000 Subject: [PATCH 15/40] feat(ci): run migrations after loading fixtures --- .github/workflows/k8s-deploy-pr.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/k8s-deploy-pr.yml b/.github/workflows/k8s-deploy-pr.yml index 05b730b..0f532f5 100644 --- a/.github/workflows/k8s-deploy-pr.yml +++ b/.github/workflows/k8s-deploy-pr.yml @@ -195,6 +195,11 @@ jobs: hab pkg exec "${PACKAGE_ORIGIN}/${PACKAGE_NAME}" \ mysql "${DATABASE_NAME}" + echo "Running migrations..." + kubectl exec "${POD_NAME}" -- \ + hab pkg exec "${PACKAGE_ORIGIN}/${PACKAGE_NAME}" \ + emergence-console-run migrations:execute --all + - name: Mark deployment as failed if: failure() From 1f6c449e7ad33a2b9e1e8bb4a4258b0b6de0c5bb Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 3 Jan 2021 19:08:36 +0000 Subject: [PATCH 16/40] refactor(helm): generalize composite service env name --- helm-chart/templates/deployment.yaml | 4 ++-- helm-chart/values.yaml | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/helm-chart/templates/deployment.yaml b/helm-chart/templates/deployment.yaml index a997a70..da4fcb3 100644 --- a/helm-chart/templates/deployment.yaml +++ b/helm-chart/templates/deployment.yaml @@ -27,10 +27,10 @@ spec: imagePullPolicy: Always env: - - name: HAB_GATEKEEPER_COMPOSITE + - name: {{ .Values.hab.composite.env_name }} value: | [services.mysql] - pkg_ident = {{ .Values.hab.gatekeeper_composite.mysql.pkg_ident | quote }} + pkg_ident = {{ .Values.hab.composite.mysql.pkg_ident | quote }} - name: HAB_MYSQL value: | diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index 3c84a51..f55b7f2 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -2,7 +2,8 @@ replicaCount: 1 hab: - gatekeeper_composite: + composite: + env_name: HAB_GATEKEEPER_COMPOSITE mysql: pkg_ident: 'core/mysql' From 1dd624841c419739a585a12fdfdf5a662305b220 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 3 Jan 2021 20:21:40 +0000 Subject: [PATCH 17/40] feat: enable tracking history for endpoints records --- php-classes/Gatekeeper/Endpoints/Endpoint.php | 3 +- .../Endpoint/20210103_endpoint-history.php | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 php-migrations/Gatekeeper/Endpoint/20210103_endpoint-history.php diff --git a/php-classes/Gatekeeper/Endpoints/Endpoint.php b/php-classes/Gatekeeper/Endpoints/Endpoint.php index d73db40..88cfaff 100644 --- a/php-classes/Gatekeeper/Endpoints/Endpoint.php +++ b/php-classes/Gatekeeper/Endpoints/Endpoint.php @@ -4,7 +4,6 @@ use Site; use Cache; -use ActiveRecord; use HandleBehavior; use RecordValidator; use TableNotFoundException; @@ -14,7 +13,7 @@ use Gatekeeper\Alerts\AbstractAlert; use Symfony\Component\Yaml\Yaml; -class Endpoint extends ActiveRecord +class Endpoint extends \VersionedRecord { public static $metricTTL = 60; public static $validPathRegex = '/^[a-zA-Z][a-zA-Z0-9_\\-\\.]*(\\/[a-zA-Z][a-zA-Z0-9_\\-\\.]*)*$/'; diff --git a/php-migrations/Gatekeeper/Endpoint/20210103_endpoint-history.php b/php-migrations/Gatekeeper/Endpoint/20210103_endpoint-history.php new file mode 100644 index 0000000..71a731d --- /dev/null +++ b/php-migrations/Gatekeeper/Endpoint/20210103_endpoint-history.php @@ -0,0 +1,45 @@ + Date: Sun, 3 Jan 2021 19:40:20 +0000 Subject: [PATCH 18/40] feat(helm-chart): add runtime.error.display option --- helm-chart/templates/deployment.yaml | 7 +++++++ helm-chart/values.yaml | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/helm-chart/templates/deployment.yaml b/helm-chart/templates/deployment.yaml index da4fcb3..02caf16 100644 --- a/helm-chart/templates/deployment.yaml +++ b/helm-chart/templates/deployment.yaml @@ -32,6 +32,13 @@ spec: [services.mysql] pkg_ident = {{ .Values.hab.composite.mysql.pkg_ident | quote }} + - name: {{ .Values.hab.runtime.env_name }} + value: | + [error] + {{- if .Values.hab.runtime.error.display }} + display = true + {{- end }} + - name: HAB_MYSQL value: | {{- if .Values.hab.mysql }} diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index f55b7f2..2660454 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -2,6 +2,10 @@ replicaCount: 1 hab: + runtime: + env_name: HAB_GATEKEEPER + error: + display: false composite: env_name: HAB_GATEKEEPER_COMPOSITE mysql: From e025fff04820aa68f6ab33218eaf3734497c72c4 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 3 Jan 2021 19:43:09 +0000 Subject: [PATCH 19/40] feat(ci): enable runtime error display in PR envs --- .github/workflows/k8s-deploy-pr.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/k8s-deploy-pr.yml b/.github/workflows/k8s-deploy-pr.yml index 0f532f5..f3ac83e 100644 --- a/.github/workflows/k8s-deploy-pr.yml +++ b/.github/workflows/k8s-deploy-pr.yml @@ -134,7 +134,8 @@ jobs: --set name="${PR_NAME}" \ --set namespace="${KUBE_NAMESPACE}" \ --set image="${image_url}" \ - --set hostname="${hostname}" + --set hostname="${hostname}" \ + --set hab.runtime.error.display=true echo "Listing pods existing after deploy" kubectl get pods \ From 94f403eb90d8cfaae50cf451b80647e441f3e9d6 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 3 Jan 2021 20:30:55 +0000 Subject: [PATCH 20/40] feat: backfill new history table with initial versions --- .../Endpoint/20210103_endpoint-history.php | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/php-migrations/Gatekeeper/Endpoint/20210103_endpoint-history.php b/php-migrations/Gatekeeper/Endpoint/20210103_endpoint-history.php index 71a731d..08bf6ba 100644 --- a/php-migrations/Gatekeeper/Endpoint/20210103_endpoint-history.php +++ b/php-migrations/Gatekeeper/Endpoint/20210103_endpoint-history.php @@ -19,14 +19,6 @@ } -// create history table if needed -if (!static::tableExists($historyTableName)) { - printf("Creating history table `%s`\n", $historyTableName); - DB::multiQuery(SQL::getCreateTable(Endpoint::class)); - $skipped = false; -} - - // add modified/modifier columns if (!static::columnExists($tableName, 'Modified')) { printf("Adding `Modified` column to `%s` table\n", $tableName); @@ -41,5 +33,22 @@ } +// create history table if needed +if (!static::tableExists($historyTableName)) { + printf("Creating history table `%s`\n", $historyTableName); + DB::multiQuery(SQL::getCreateTable(Endpoint::class)); + $skipped = false; + + printf("Backfilling history table `%s` from `%s`\n", $historyTableName, $tableName); + DB::nonQuery( + 'INSERT INTO `%2$s` SELECT NULL AS RevisionID, `%1$s`.* FROM `%1$s`', + [ + $tableName, + $historyTableName + ] + ); +} + + // finish return $skipped ? static::STATUS_SKIPPED : static::STATUS_EXECUTED; From be27b74342c0965f5d4e9468831baac39bd14571 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 3 Jan 2021 22:25:05 +0000 Subject: [PATCH 21/40] chore: upgrade Dockerfile pattern --- Dockerfile | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index bb42fa6..578e3ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,13 @@ # This Dockerfile is hyper-optimized to minimize layer changes FROM jarvus/habitat-compose:latest as habitat + ARG HAB_LICENSE=no-accept ENV HAB_LICENSE=$HAB_LICENSE ENV STUDIO_TYPE=Dockerfile ENV HAB_ORIGIN=jarvus RUN hab origin key generate + # pre-layer all external runtime plan deps COPY habitat/plan.sh /habitat/plan.sh RUN hab pkg install \ @@ -13,6 +15,7 @@ RUN hab pkg install \ emergence/php-runtime \ $({ cat '/habitat/plan.sh' && echo && echo 'echo "${pkg_deps[@]/$pkg_origin\/*/}"'; } | hab pkg exec core/bash bash) \ && hab pkg exec core/coreutils rm -rf /hab/cache/{artifacts,src}/ + # pre-layer all external runtime composite deps COPY habitat/composite/plan.sh /habitat/composite/plan.sh RUN hab pkg install \ @@ -23,6 +26,7 @@ RUN hab pkg install \ FROM habitat as projector + # pre-layer all build-time plan deps RUN hab pkg install \ core/hab-plan-build \ @@ -30,31 +34,41 @@ RUN hab pkg install \ jarvus/toml-merge \ $({ cat '/habitat/plan.sh' && echo && echo 'echo "${pkg_build_deps[@]/$pkg_origin\/*/}"'; } | hab pkg exec core/bash bash) \ && hab pkg exec core/coreutils rm -rf /hab/cache/{artifacts,src}/ + # pre-layer all build-time composite deps RUN hab pkg install \ jarvus/toml-merge \ $({ cat '/habitat/composite/plan.sh' && echo && echo 'echo "${pkg_build_deps[@]/$pkg_origin\/*/}"'; } | hab pkg exec core/bash bash) \ && hab pkg exec core/coreutils rm -rf /hab/cache/{artifacts,src}/ + # build application COPY . /src +ARG SITE_TREE +ENV SITE_TREE=$SITE_TREE +ARG SITE_VERSION +ENV SITE_VERSION=$SITE_VERSION RUN hab pkg exec core/hab-plan-build hab-plan-build /src RUN hab pkg exec core/hab-plan-build hab-plan-build /src/habitat/composite FROM habitat as runtime -# install .hart artifact from builder stage -COPY --from=projector /hab/cache/artifacts/$HAB_ORIGIN-* /hab/cache/artifacts/ -RUN hab pkg install /hab/cache/artifacts/$HAB_ORIGIN-* \ - && hab pkg exec core/coreutils rm -rf /hab/cache/{artifacts,src}/ - # configure persistent volumes RUN hab pkg exec core/coreutils mkdir -p '/hab/svc/mysql/data' '/hab/svc/gatekeeper/data' '/hab/svc/nginx/files' \ && hab pkg exec core/coreutils chown hab:hab -R '/hab/svc/mysql/data' '/hab/svc/gatekeeper/data' '/hab/svc/nginx/files' -VOLUME ["/hab/svc/mysql/data", "/hab/svc/gatekeeper/data", "/hab/svc/nginx/files"] - - # configure entrypoint +VOLUME ["/hab/svc/mysql/data", "/hab/svc/gatekeeper/data", "/hab/svc/nginx/files"] ENTRYPOINT ["hab", "sup", "run"] CMD ["jarvus/gatekeeper-composite"] + +# install .hart artifact from builder stage +COPY --from=projector /hab/cache/artifacts/$HAB_ORIGIN-* /hab/cache/artifacts/ +RUN hab pkg install /hab/cache/artifacts/$HAB_ORIGIN-* \ + && hab pkg exec core/coreutils rm -rf /hab/cache/{artifacts,src}/ + +# add source metadata to environment +ARG SOURCE_TAG +ENV SOURCE_TAG=$SOURCE_TAG +ARG SOURCE_COMMIT +ENV SOURCE_COMMIT=$SOURCE_COMMIT From d4af615377b72bc12ce49e74a7a09aa31acf1bf0 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 2 Jan 2021 04:34:07 +0000 Subject: [PATCH 22/40] chore: bump skeleton-v2 version to v2.5.0 --- .holo/sources/skeleton-v2.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.holo/sources/skeleton-v2.toml b/.holo/sources/skeleton-v2.toml index 68a5ede..0f193da 100644 --- a/.holo/sources/skeleton-v2.toml +++ b/.holo/sources/skeleton-v2.toml @@ -1,6 +1,6 @@ [holosource] url = "https://github.com/JarvusInnovations/emergence-skeleton-v2" -ref = "refs/tags/v2.4.1" +ref = "refs/tags/v2.5.0" [holosource.project] holobranch = "emergence-skeleton" From 624af54f1e53a5c2f7e18ad12c5641f902c5e29b Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 2 Jan 2021 04:47:30 +0000 Subject: [PATCH 23/40] feat: use Client::getAddress in place of REMOTE_ADDR --- .../beforeApiRequest/15_reject-banned-ip.php | 3 ++- php-classes/Gatekeeper/ApiRequest.php | 5 +++-- php-classes/Gatekeeper/ApiRequestHandler.php | 11 ++++++++--- php-classes/Gatekeeper/Gatekeeper.php | 7 +++++-- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/15_reject-banned-ip.php b/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/15_reject-banned-ip.php index 0d5fc7f..ff380ec 100644 --- a/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/15_reject-banned-ip.php +++ b/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/15_reject-banned-ip.php @@ -3,9 +3,10 @@ namespace Gatekeeper; use Gatekeeper\Bans\Ban; +use Emergence\Site\Client; -if (Ban::isIPAddressBanned($_SERVER['REMOTE_ADDR'])) { +if (Ban::isIPAddressBanned(Client::getAddress())) { header('HTTP/1.1 403 Forbidden'); \JSON::error('Your IP address is currently banned from using this service'); } \ No newline at end of file diff --git a/php-classes/Gatekeeper/ApiRequest.php b/php-classes/Gatekeeper/ApiRequest.php index 9d0d83c..ded2a57 100644 --- a/php-classes/Gatekeeper/ApiRequest.php +++ b/php-classes/Gatekeeper/ApiRequest.php @@ -5,6 +5,7 @@ use Gatekeeper\Endpoints\Endpoint; use Gatekeeper\Keys\Key; use Gatekeeper\Transactions\Transaction; +use Emergence\Site\Client; class ApiRequest { @@ -96,6 +97,6 @@ public function isReady() public function getUserIdentifier() { - return $this->key ? 'key:' . $this->key->ID : 'ip:' . $_SERVER['REMOTE_ADDR']; + return $this->key ? 'key:' . $this->key->ID : 'ip:' . Client::getAddress(); } -} \ No newline at end of file +} diff --git a/php-classes/Gatekeeper/ApiRequestHandler.php b/php-classes/Gatekeeper/ApiRequestHandler.php index d4d669d..0b534df 100644 --- a/php-classes/Gatekeeper/ApiRequestHandler.php +++ b/php-classes/Gatekeeper/ApiRequestHandler.php @@ -6,6 +6,7 @@ use Exception; use HttpProxy; use Emergence\EventBus; +use Emergence\Site\Client; use Gatekeeper\Transactions\Transaction; class ApiRequestHandler extends \RequestHandler @@ -46,6 +47,10 @@ public static function handleRequest() { } + // get client IP + $clientIp = Client::getAddress(); + + // execute request against internal API HttpProxy::relayRequest([ 'autoAppend' => false @@ -53,7 +58,7 @@ public static function handleRequest() { ,'url' => rtrim($request->getEndpoint()->InternalEndpoint, '/') . $request->getUrl() ,'interface' => static::$sourceInterface ,'headers' => [ - "X-Forwarded-For: {$_SERVER['REMOTE_ADDR']}" + "X-Forwarded-For: {$clientIp}" ] ,'passthruHeaders' => static::$passthruHeaders ,'forwardHeaders' => array_merge(HttpProxy::$defaultForwardHeaders, static::$forwardHeaders) @@ -61,7 +66,7 @@ public static function handleRequest() { ,'timeoutConnect' => static::$defaultTimeoutConnect // ,'debug' => true // uncomment to debug proxy process and see output following response // ,'afterResponseSync' => true // true to debug afterResponse code from browser - ,'afterResponse' => function ($responseBody, $responseHeaders, $options, $curlHandle) use ($request, &$metrics, &$beforeEvent) { + ,'afterResponse' => function ($responseBody, $responseHeaders, $options, $curlHandle) use ($request, $clientIp, &$metrics, &$beforeEvent) { $curlInfo = curl_getinfo($curlHandle); list($path, $query) = explode('?', $request->getUrl()); @@ -73,7 +78,7 @@ public static function handleRequest() { $Transaction = Transaction::create([ 'Endpoint' => $request->getEndpoint() ,'Key' => $request->getKey() - ,'ClientIP' => ip2long($_SERVER['REMOTE_ADDR']) + ,'ClientIP' => ip2long($clientIp) ,'Method' => $_SERVER['REQUEST_METHOD'] ,'Path' => $path ,'Query' => $query diff --git a/php-classes/Gatekeeper/Gatekeeper.php b/php-classes/Gatekeeper/Gatekeeper.php index 186dcd8..4fdc13f 100644 --- a/php-classes/Gatekeeper/Gatekeeper.php +++ b/php-classes/Gatekeeper/Gatekeeper.php @@ -4,6 +4,7 @@ use Site; use JSON; +use Emergence\Site\Client; class Gatekeeper { @@ -30,9 +31,11 @@ public static function getApiBaseUrl() public static function authorizeTestApiAccess() { + $clientIp = Client::getAddress(); + if ( - $_SERVER['REMOTE_ADDR'] != $_SERVER['SERVER_ADDR'] && - $_SERVER['REMOTE_ADDR'] != gethostbyname($_SERVER['HTTP_HOST']) + $clientIp != $_SERVER['SERVER_ADDR'] && + $clientIp != gethostbyname($_SERVER['HTTP_HOST']) ) { JSON::error('Access to test API denied', 403); exit(); From 4ddfc8e682ecd9a28fcf0b9112a384044d3b29f1 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 2 Jan 2021 19:44:36 +0000 Subject: [PATCH 24/40] chore: bump skeleton-v2 to 2.5.1 --- .holo/sources/skeleton-v2.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.holo/sources/skeleton-v2.toml b/.holo/sources/skeleton-v2.toml index 0f193da..93d0189 100644 --- a/.holo/sources/skeleton-v2.toml +++ b/.holo/sources/skeleton-v2.toml @@ -1,6 +1,6 @@ [holosource] url = "https://github.com/JarvusInnovations/emergence-skeleton-v2" -ref = "refs/tags/v2.5.0" +ref = "refs/tags/v2.5.1" [holosource.project] holobranch = "emergence-skeleton" From 9e3b098c1f1c5a7773dcc9b4b49d0037689f0d81 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 3 Jan 2021 01:31:38 +0000 Subject: [PATCH 25/40] fix: revert method of checking test API access The original method worked better --- php-classes/Gatekeeper/Gatekeeper.php | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/php-classes/Gatekeeper/Gatekeeper.php b/php-classes/Gatekeeper/Gatekeeper.php index 4fdc13f..186dcd8 100644 --- a/php-classes/Gatekeeper/Gatekeeper.php +++ b/php-classes/Gatekeeper/Gatekeeper.php @@ -4,7 +4,6 @@ use Site; use JSON; -use Emergence\Site\Client; class Gatekeeper { @@ -31,11 +30,9 @@ public static function getApiBaseUrl() public static function authorizeTestApiAccess() { - $clientIp = Client::getAddress(); - if ( - $clientIp != $_SERVER['SERVER_ADDR'] && - $clientIp != gethostbyname($_SERVER['HTTP_HOST']) + $_SERVER['REMOTE_ADDR'] != $_SERVER['SERVER_ADDR'] && + $_SERVER['REMOTE_ADDR'] != gethostbyname($_SERVER['HTTP_HOST']) ) { JSON::error('Access to test API denied', 403); exit(); From cb02b07f095fa61458802010583d740160af03ee Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 3 Jan 2021 02:52:28 +0000 Subject: [PATCH 26/40] feat: add client IP test API --- site-root/test-api/client.php | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 site-root/test-api/client.php diff --git a/site-root/test-api/client.php b/site-root/test-api/client.php new file mode 100644 index 0000000..eee8134 --- /dev/null +++ b/site-root/test-api/client.php @@ -0,0 +1,10 @@ + $clientIp != null, + 'client_ip' => $clientIp +]); From 57c7dfcec9b18acb08b5b251693e29258b1f018f Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 3 Jan 2021 02:52:44 +0000 Subject: [PATCH 27/40] feat(fixtures): add endpoints for all test API endpoints --- fixtures/endpoints.sql | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fixtures/endpoints.sql b/fixtures/endpoints.sql index bdbf150..52302fa 100644 --- a/fixtures/endpoints.sql +++ b/fixtures/endpoints.sql @@ -32,6 +32,11 @@ CREATE TABLE `endpoints` ( PRIMARY KEY (`ID`), UNIQUE KEY `Handle` (`Handle`), UNIQUE KEY `Path` (`Path`) -) ENGINE=MyISAM AUTO_INCREMENT=2 DEFAULT CHARSET=utf8; +) ENGINE=MyISAM DEFAULT CHARSET=utf8; INSERT INTO `endpoints` VALUES (1,'Gatekeeper\\Endpoints\\Endpoint','2019-01-02 03:04:05',1,'TODOs v1','todos-v1','todos/v1','https://jsonplaceholder.typicode.com/todos',NULL,NULL,1,NULL,NULL,10,60,1,1,NULL,NULL,0,0,1,0,NULL,NULL,NULL,NULL); +INSERT INTO `endpoints` VALUES (2,'Gatekeeper\\Endpoints\\Endpoint','2019-01-02 03:04:05',1,'Test: Cacheable','test-cacheable','test/cacheable','http://localhost/test-api/cachable',NULL,NULL,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,0,1,1,NULL,NULL,NULL,NULL); +INSERT INTO `endpoints` VALUES (3,'Gatekeeper\\Endpoints\\Endpoint','2019-01-02 03:04:05',1,'Test: Cookies','test-cookies','test/cookies','http://localhost/test-api/cookies',NULL,NULL,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,0,1,1,NULL,NULL,NULL,NULL); +INSERT INTO `endpoints` VALUES (4,'Gatekeeper\\Endpoints\\Endpoint','2019-01-02 03:04:05',1,'Test: Slow','test-slow','test/slow','http://localhost/test-api/slow',NULL,NULL,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,0,1,1,NULL,NULL,NULL,NULL); +INSERT INTO `endpoints` VALUES (5,'Gatekeeper\\Endpoints\\Endpoint','2019-01-02 03:04:05',1,'Test: Status','test-status','test/status','http://localhost/test-api/status',NULL,NULL,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,0,1,1,NULL,NULL,NULL,NULL); +INSERT INTO `endpoints` VALUES (6,'Gatekeeper\\Endpoints\\Endpoint','2019-01-02 03:04:05',1,'Test: Client IP','test-client','test/client','http://localhost/test-api/client',NULL,NULL,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,0,1,1,NULL,NULL,NULL,NULL); From f9f9d61f6dff30dde71bb35315055d2f8a1d6c10 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 3 Jan 2021 20:09:16 +0000 Subject: [PATCH 28/40] feat: add headers test endpoint --- fixtures/endpoints.sql | 1 + site-root/test-api/headers.php | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 site-root/test-api/headers.php diff --git a/fixtures/endpoints.sql b/fixtures/endpoints.sql index 52302fa..490aeb4 100644 --- a/fixtures/endpoints.sql +++ b/fixtures/endpoints.sql @@ -40,3 +40,4 @@ INSERT INTO `endpoints` VALUES (3,'Gatekeeper\\Endpoints\\Endpoint','2019-01-02 INSERT INTO `endpoints` VALUES (4,'Gatekeeper\\Endpoints\\Endpoint','2019-01-02 03:04:05',1,'Test: Slow','test-slow','test/slow','http://localhost/test-api/slow',NULL,NULL,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,0,1,1,NULL,NULL,NULL,NULL); INSERT INTO `endpoints` VALUES (5,'Gatekeeper\\Endpoints\\Endpoint','2019-01-02 03:04:05',1,'Test: Status','test-status','test/status','http://localhost/test-api/status',NULL,NULL,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,0,1,1,NULL,NULL,NULL,NULL); INSERT INTO `endpoints` VALUES (6,'Gatekeeper\\Endpoints\\Endpoint','2019-01-02 03:04:05',1,'Test: Client IP','test-client','test/client','http://localhost/test-api/client',NULL,NULL,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,0,1,1,NULL,NULL,NULL,NULL); +INSERT INTO `endpoints` VALUES (7,'Gatekeeper\\Endpoints\\Endpoint','2019-01-02 03:04:05',1,'Test: Headers','test-headers','test/headers','http://localhost/test-api/headers',NULL,NULL,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,0,1,1,NULL,NULL,NULL,NULL); diff --git a/site-root/test-api/headers.php b/site-root/test-api/headers.php new file mode 100644 index 0000000..32c0fcc --- /dev/null +++ b/site-root/test-api/headers.php @@ -0,0 +1,20 @@ + $value) { + if (substr($key, 0, 5) != 'HTTP_') { + continue; + } + + $key = substr($key, 5); + $key = str_replace('_', '-', $key); + $key = ucwords(strtolower($key), '-'); + + $headers[$key] = $value; +} + +JSON::respond($headers); From 699526755c2776629da41a48e5bf76fad2b51ae3 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Mon, 4 Jan 2021 15:41:44 +0000 Subject: [PATCH 29/40] refactor: track client address in ApiRequest state --- .../beforeApiRequest/10_load-client-ip.php | 9 +++++++++ .../beforeApiRequest/15_reject-banned-ip.php | 3 +-- php-classes/Gatekeeper/ApiRequest.php | 15 ++++++++++++++- php-classes/Gatekeeper/ApiRequestHandler.php | 10 +++------- 4 files changed, 27 insertions(+), 10 deletions(-) create mode 100644 event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/10_load-client-ip.php diff --git a/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/10_load-client-ip.php b/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/10_load-client-ip.php new file mode 100644 index 0000000..c17291f --- /dev/null +++ b/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/10_load-client-ip.php @@ -0,0 +1,9 @@ +setClientAddress(Client::getAddress()); diff --git a/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/15_reject-banned-ip.php b/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/15_reject-banned-ip.php index ff380ec..29916f5 100644 --- a/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/15_reject-banned-ip.php +++ b/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/15_reject-banned-ip.php @@ -3,10 +3,9 @@ namespace Gatekeeper; use Gatekeeper\Bans\Ban; -use Emergence\Site\Client; -if (Ban::isIPAddressBanned(Client::getAddress())) { +if (Ban::isIPAddressBanned($_EVENT['request']->getClientAddress())) { header('HTTP/1.1 403 Forbidden'); \JSON::error('Your IP address is currently banned from using this service'); } \ No newline at end of file diff --git a/php-classes/Gatekeeper/ApiRequest.php b/php-classes/Gatekeeper/ApiRequest.php index ded2a57..1beffec 100644 --- a/php-classes/Gatekeeper/ApiRequest.php +++ b/php-classes/Gatekeeper/ApiRequest.php @@ -12,6 +12,7 @@ class ApiRequest protected $startTime; protected $pathStack = []; protected $endpoint; + protected $clientAddress; protected $key; protected $url = ''; protected $transaction; @@ -60,6 +61,16 @@ public function setEndpoint(Endpoint $Endpoint) $this->endpoint = $Endpoint; } + public function getClientAddress() + { + return $this->clientAddress; + } + + public function setClientAddress($clientAddress) + { + $this->clientAddress = $clientAddress; + } + public function getKey() { return $this->key; @@ -97,6 +108,8 @@ public function isReady() public function getUserIdentifier() { - return $this->key ? 'key:' . $this->key->ID : 'ip:' . Client::getAddress(); + return $this->key + ? 'key:' . $this->key->ID + : 'ip:' . ($this->clientAddress ?: Client::getAddress()); } } diff --git a/php-classes/Gatekeeper/ApiRequestHandler.php b/php-classes/Gatekeeper/ApiRequestHandler.php index 0b534df..824875e 100644 --- a/php-classes/Gatekeeper/ApiRequestHandler.php +++ b/php-classes/Gatekeeper/ApiRequestHandler.php @@ -47,10 +47,6 @@ public static function handleRequest() { } - // get client IP - $clientIp = Client::getAddress(); - - // execute request against internal API HttpProxy::relayRequest([ 'autoAppend' => false @@ -58,7 +54,7 @@ public static function handleRequest() { ,'url' => rtrim($request->getEndpoint()->InternalEndpoint, '/') . $request->getUrl() ,'interface' => static::$sourceInterface ,'headers' => [ - "X-Forwarded-For: {$clientIp}" + "X-Forwarded-For: {$request->getClientAddress()}" ] ,'passthruHeaders' => static::$passthruHeaders ,'forwardHeaders' => array_merge(HttpProxy::$defaultForwardHeaders, static::$forwardHeaders) @@ -66,7 +62,7 @@ public static function handleRequest() { ,'timeoutConnect' => static::$defaultTimeoutConnect // ,'debug' => true // uncomment to debug proxy process and see output following response // ,'afterResponseSync' => true // true to debug afterResponse code from browser - ,'afterResponse' => function ($responseBody, $responseHeaders, $options, $curlHandle) use ($request, $clientIp, &$metrics, &$beforeEvent) { + ,'afterResponse' => function ($responseBody, $responseHeaders, $options, $curlHandle) use ($request, &$metrics, &$beforeEvent) { $curlInfo = curl_getinfo($curlHandle); list($path, $query) = explode('?', $request->getUrl()); @@ -78,7 +74,7 @@ public static function handleRequest() { $Transaction = Transaction::create([ 'Endpoint' => $request->getEndpoint() ,'Key' => $request->getKey() - ,'ClientIP' => ip2long($clientIp) + ,'ClientIP' => ip2long($request->getClientAddress()) ,'Method' => $_SERVER['REQUEST_METHOD'] ,'Path' => $path ,'Query' => $query From bb409280bcd6327a38248a6f55240667fcda8c16 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Mon, 4 Jan 2021 15:45:47 +0000 Subject: [PATCH 30/40] feat: add basic CRUD for exemptions --- html-templates/exemptions/exemption.tpl | 30 ++++++ .../exemptions/exemptionDeleted.tpl | 11 ++ html-templates/exemptions/exemptionEdit.tpl | 44 ++++++++ html-templates/exemptions/exemptionSaved.tpl | 17 +++ html-templates/exemptions/exemptions.tpl | 54 ++++++++++ html-templates/includes/site.nav.tpl | 1 + html-templates/subtemplates/contextLinks.tpl | 10 ++ .../Gatekeeper/Exemptions/Exemption.php | 102 ++++++++++++++++++ .../Exemptions/ExemptionsRequestHandler.php | 26 +++++ site-root/exemptions.php | 7 ++ site-root/sass/site/_modules.scss | 1 + site-root/sass/site/modules/_exemptions.scss | 16 +++ 12 files changed, 319 insertions(+) create mode 100644 html-templates/exemptions/exemption.tpl create mode 100644 html-templates/exemptions/exemptionDeleted.tpl create mode 100644 html-templates/exemptions/exemptionEdit.tpl create mode 100644 html-templates/exemptions/exemptionSaved.tpl create mode 100644 html-templates/exemptions/exemptions.tpl create mode 100644 php-classes/Gatekeeper/Exemptions/Exemption.php create mode 100644 php-classes/Gatekeeper/Exemptions/ExemptionsRequestHandler.php create mode 100644 site-root/exemptions.php create mode 100644 site-root/sass/site/modules/_exemptions.scss diff --git a/html-templates/exemptions/exemption.tpl b/html-templates/exemptions/exemption.tpl new file mode 100644 index 0000000..de9f98d --- /dev/null +++ b/html-templates/exemptions/exemption.tpl @@ -0,0 +1,30 @@ +{extends designs/site.tpl} + +{block title}Exemption #{$data->ID} — Exemptions — {$dwoo.parent}{/block} + +{block content} + {$Exemption = $data} + +
+
+
+

+ {if $Exemption->IPPattern} + IP Pattern: {$Exemption->IPPattern} + {else} + Key: {apiKey $Exemption->Key} + {/if} +

+ +
Exempted {if $Exemption->ExpirationDate}until {$Exemption->ExpirationDate|date_format}{else}indefinitely{/if}.
+
+ {if $Exemption->Notes} + 
{$Exemption->Notes|escape}
+ {/if} +
+ +
+{/block} \ No newline at end of file diff --git a/html-templates/exemptions/exemptionDeleted.tpl b/html-templates/exemptions/exemptionDeleted.tpl new file mode 100644 index 0000000..2e7fd74 --- /dev/null +++ b/html-templates/exemptions/exemptionDeleted.tpl @@ -0,0 +1,11 @@ +{extends "designs/site.tpl"} + +{block "title"}Exemption deleted — {$dwoo.parent}{/block} + +{block "content"} + {$Exemption = $data} + +

Exemption on {if $Exemption->IPPattern}IP Pattern: {$Exemption->IPPattern}{else}Key: {apiKey $Exemption->Key}{/if} deleted.

+ +

Browse all exemptions

+{/block} \ No newline at end of file diff --git a/html-templates/exemptions/exemptionEdit.tpl b/html-templates/exemptions/exemptionEdit.tpl new file mode 100644 index 0000000..a9175f5 --- /dev/null +++ b/html-templates/exemptions/exemptionEdit.tpl @@ -0,0 +1,44 @@ +{extends "designs/site.tpl"} + +{block "title"}{tif $data->isPhantom ? "Create Exemption" : escape("Edit Exemption")} — {$dwoo.parent}{/block} + +{block "content"} + {$Exemption = $data} + {$errors = $Exemption->validationErrors} + +
+

{if $Exemption->isPhantom}New Exemption{else}Edit Exemption{/if}

+
+ {if $Exemption->isPhantom} + Cancel + {else} + Remove Exemption + {/if} +
+
+ +
+ {if $errors} +
+ Please double-check the fields highlighted below. +
+ {/if} + +
+ +
+ {field inputName=IPPattern label='IP Pattern' error=$errors.IPPattern default=$Exemption->IPPattern hint="192.168.1.1,192.168.1.*,192.168.1.1/24"} +
—or—
+ {field inputName=KeyID label='API Key' error=$errors.KeyID default=$Exemption->Key->Key} +
+ + {field inputName=ExpirationDate label='Expiration Date' type=date default=tif($Exemption->ExpirationDate, date('Y-m-d', $Exemption->ExpirationDate)) hint="Leave blank for indefinate exemption"} + + {textarea inputName=Notes label='Notes' default=$Exemption->Notes} + +
+ +
+
+ +{/block} \ No newline at end of file diff --git a/html-templates/exemptions/exemptionSaved.tpl b/html-templates/exemptions/exemptionSaved.tpl new file mode 100644 index 0000000..d14d9af --- /dev/null +++ b/html-templates/exemptions/exemptionSaved.tpl @@ -0,0 +1,17 @@ +{extends "designs/site.tpl"} + +{block "title"}Exemption saved — {$dwoo.parent}{/block} + +{block "content"} + {$Exemption = $data} + +

+ Exemption on + {if $Exemption->IPPattern}IP Address Pattern: {$Exemption->IPPattern} + {else}Key: {apiKey $Exemption->Key} + {/if} + {tif $Exemption->isNew ? created : saved}. +

+ +

← Browse all exemptions

+{/block} \ No newline at end of file diff --git a/html-templates/exemptions/exemptions.tpl b/html-templates/exemptions/exemptions.tpl new file mode 100644 index 0000000..1a88961 --- /dev/null +++ b/html-templates/exemptions/exemptions.tpl @@ -0,0 +1,54 @@ +{extends designs/site.tpl} + +{block title}Exemptions — {$dwoo.parent}{/block} + +{block content} + +
+

Exemptions

+
+ Issue Exemption +
+
+ +
+ + + +
+ + {foreach item=Exemption from=$data} +
+
+
+

+ {if $Exemption->IPPattern} + IP Pattern: {$Exemption->IPPattern} + {else} + Key: {apiKey $Exemption->Key} + {/if} +

+ +
Exempted {if $Exemption->ExpirationDate}until {$Exemption->ExpirationDate|date_format}{else}indefinitely{/if}.
+
+ {if $Exemption->Notes} + 
{$Exemption->Notes|escape}
+ {/if} +
+ +
+ {/foreach} + +
+ +{/block} \ No newline at end of file diff --git a/html-templates/includes/site.nav.tpl b/html-templates/includes/site.nav.tpl index f4a40bc..adc422f 100644 --- a/html-templates/includes/site.nav.tpl +++ b/html-templates/includes/site.nav.tpl @@ -5,6 +5,7 @@
  • Alerts
  • Keys
  • Bans
    • +
  • Exemptions
  • Transactions Log
  • Top Users
    • diff --git a/html-templates/subtemplates/contextLinks.tpl b/html-templates/subtemplates/contextLinks.tpl index 9ac3069..5df1a87 100644 --- a/html-templates/subtemplates/contextLinks.tpl +++ b/html-templates/subtemplates/contextLinks.tpl @@ -24,6 +24,16 @@ Key: {$Context->Key->OwnerName|escape} {$Context->Key->Key} {/if}{$suffix} + {elseif is_a($Context, Gatekeeper\Exemptions\Exemption::class)} + + {$prefix}Exemption #{$Context->ID} + — + {if $Context->IPPattern} + IP Pattern: {$Context->IPPattern} + {else} + Key: {$Context->Key->OwnerName|escape} {$Context->Key->Key} + {/if}{$suffix} + {elseif is_a($Context, Gatekeeper\Keys\Key::class)} {$prefix}{apiKey $Context}{$suffix} {elseif is_a($Context, Gatekeeper\Endpoints\Endpoint::class)} diff --git a/php-classes/Gatekeeper/Exemptions/Exemption.php b/php-classes/Gatekeeper/Exemptions/Exemption.php new file mode 100644 index 0000000..c47d8bf --- /dev/null +++ b/php-classes/Gatekeeper/Exemptions/Exemption.php @@ -0,0 +1,102 @@ + [ + 'type' => 'uint', + 'default' => null + ], + 'IPPattern' => [ + 'default' => null + ], + 'ExpirationDate' => [ + 'type' => 'timestamp', + 'default' => null + ], + 'BypassGlobalLimit' => [ + 'type' => 'boolean', + 'default' => false + ], + 'Notes' => [ + 'type' => 'clob', + 'default' => null, + 'fulltext' => true + ] + ]; + + public static $relationships = [ + 'Key' => [ + 'type' => 'one-one', + 'class' => Key::class + ] + ]; + + public static $dynamicFields = [ + 'Key' + ]; + + public static $validators = [ + 'ExpirationDate' => [ + 'validator' => 'datetime', + 'required' => false + ] + ]; + + public static $sorters = [ + 'created' => [__CLASS__, 'sortCreated'], + 'expiration' => [__CLASS__, 'sortExpiration'] + ]; + + public function validate($deep = true) + { + parent::validate($deep); + + if (!$this->KeyID == !$this->IPPattern) { + $this->_validator->addError('Exemption', 'Exemption must specify either a API key or an IP pattern'); + } + + return $this->finishValidation(); + } + + public function save($deep = true) + { + parent::save($deep); + + if ($this->isUpdated || $this->isNew) { + Cache::delete('exemptions'); + } + } + + public function destroy() + { + $success = parent::destroy(); + Cache::delete('exemptions'); + return $success; + } + + public static function sortExpiration($dir, $name) + { + return "ExpirationDate $dir"; + } + + public static function sortCreated($dir, $name) + { + return "ID $dir"; + } +} diff --git a/php-classes/Gatekeeper/Exemptions/ExemptionsRequestHandler.php b/php-classes/Gatekeeper/Exemptions/ExemptionsRequestHandler.php new file mode 100644 index 0000000..0147a37 --- /dev/null +++ b/php-classes/Gatekeeper/Exemptions/ExemptionsRequestHandler.php @@ -0,0 +1,26 @@ +ID : null; + } + + return parent::applyRecordDelta($Record, $data); + } +} diff --git a/site-root/exemptions.php b/site-root/exemptions.php new file mode 100644 index 0000000..35d00f5 --- /dev/null +++ b/site-root/exemptions.php @@ -0,0 +1,7 @@ + .details { + padding: .5em; + } +} + +.exemption-notes { + margin: 1em 1em 0 0; +} + +textarea[name=Notes] { + font-family: $mono-font; +} \ No newline at end of file From 16f8b499b8ea90325ddce35d549716ad9854ddf2 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Mon, 4 Jan 2021 15:45:47 +0000 Subject: [PATCH 31/40] chore: remove unneeded import and unused use --- php-classes/Gatekeeper/Bans/Ban.php | 1 - .../Gatekeeper/Ban/20200617000000_ippattern-column.php | 1 - 2 files changed, 2 deletions(-) diff --git a/php-classes/Gatekeeper/Bans/Ban.php b/php-classes/Gatekeeper/Bans/Ban.php index 80c212c..81cbf13 100644 --- a/php-classes/Gatekeeper/Bans/Ban.php +++ b/php-classes/Gatekeeper/Bans/Ban.php @@ -3,7 +3,6 @@ namespace Gatekeeper\Bans; use Cache; -use Emergence\Site\Storage; use Gatekeeper\Keys\Key; use Gatekeeper\Utils\IPPattern; diff --git a/php-migrations/Gatekeeper/Ban/20200617000000_ippattern-column.php b/php-migrations/Gatekeeper/Ban/20200617000000_ippattern-column.php index e95aff2..0897494 100644 --- a/php-migrations/Gatekeeper/Ban/20200617000000_ippattern-column.php +++ b/php-migrations/Gatekeeper/Ban/20200617000000_ippattern-column.php @@ -1,6 +1,5 @@ Date: Mon, 4 Jan 2021 17:55:05 +0000 Subject: [PATCH 32/40] refactor: embed all caching within IPPattern util and simplify API --- php-classes/Gatekeeper/Bans/Ban.php | 28 ++--- php-classes/Gatekeeper/Utils/IPPattern.php | 113 +++++++++------------ 2 files changed, 56 insertions(+), 85 deletions(-) diff --git a/php-classes/Gatekeeper/Bans/Ban.php b/php-classes/Gatekeeper/Bans/Ban.php index 81cbf13..a1f252f 100644 --- a/php-classes/Gatekeeper/Bans/Ban.php +++ b/php-classes/Gatekeeper/Bans/Ban.php @@ -116,8 +116,11 @@ public static function getActiveBansTable() foreach (Ban::getAllByWhere('ExpirationDate IS NULL OR ExpirationDate > CURRENT_TIMESTAMP') AS $Ban) { if (!empty($Ban->IPPattern)) { - if (is_array(static::getIPPatternBanClosure($Ban->IPPattern))) { // ip pattern ONLY contains static IPs - static::$_activeBans['ips'] = array_merge(static::$_activeBans['ips'], static::getIPPatternBanClosure($Ban->IPPattern)); + $parsed = IPPattern::parse($Ban->IPPattern); + + if (is_array($parsed)) { + // ip pattern ONLY contains static IPs + static::$_activeBans['ips'] = array_merge(static::$_activeBans['ips'], $parsed); } else { static::$_activeBans['patterns'][] = $Ban->IPPattern; } @@ -131,25 +134,6 @@ public static function getActiveBansTable() return static::$_activeBans; } - public static function getIPPatternBanClosure($ipPattern) - { - static $ipPatternCaches = []; - - $ipPatternHash = sha1($ipPattern); - - if (!empty($ipPatternCaches[$ipPatternHash])) { - return $ipPatternCaches[$ipPatternHash]; - } - - try { - $closure = include(IPPattern::getFilenameFromHash($ipPatternHash)); - } catch (\Exception $e) { - $closure = IPPattern::parse($ipPattern, $ipPatternHash); - } - - return $ipPatternCaches[$ipPatternHash] = $closure; - } - public static function isIPAddressBanned($ip) { $activeBans = static::getActiveBansTable(); @@ -161,7 +145,7 @@ public static function isIPAddressBanned($ip) // check IP Patterns individually foreach ($activeBans['patterns'] as $ipPattern) { - $matcher = static::getIPPatternBanClosure($ipPattern); + $matcher = IPPattern::parse($ipPattern); if (call_user_func($matcher, $ip) === true) { return true; } diff --git a/php-classes/Gatekeeper/Utils/IPPattern.php b/php-classes/Gatekeeper/Utils/IPPattern.php index d0976c1..6671d51 100644 --- a/php-classes/Gatekeeper/Utils/IPPattern.php +++ b/php-classes/Gatekeeper/Utils/IPPattern.php @@ -16,15 +16,42 @@ class IPPattern { * Parse IP pattens by splitting them by spaces or commas and grouping them * in the available groups: ip, cidr, or wildcard. * - * @param string $ipPattern The IP Pattern string to parse (ex. 10.0.0.1/24,192.168.1.1*) - * @param string $returnType The type of IP patterns to return ONLY. (ex. ip) + * @param string $pattern The IP Pattern string to parse (ex. 10.0.0.1/24,192.168.1.1*) * * @return array|closure Returns an array if pattern contains ONLY static IPs, or returns a closure function * that can be used to compare if an IP matches the pattern. * */ - public static function parse($pattern, $uid) + public static function parse($pattern) { + // maintain results in a static cache + static $cache = []; + + // return from statci cache early if available + $patternHash = sha1($pattern); + + if (!empty($cache[$patternHash])) { + return $cache[$patternHash]; + } + + // try to load from filesystem cache + $cacheFilePath = join('/', [ + Storage::getLocalStorageRoot(), + static::$fsRootDir, + $patternHash . '.php' + ]); + + try { + $closure = include($cacheFilePath); + + if ($closure) { + return $cache[$patternHash] = $closure; + } + } catch (\Exception $e) { + // continue... + } + + // parse patterns and organize by type $subPatternsByType = [ 'ip' => [], 'cidr' => [], @@ -40,81 +67,42 @@ public static function parse($pattern, $uid) } } - if ($count === 0) { throw new InvalidArgumentException("Unable to parse IP pattern: $pattern"); } + // fast path for static IP lists if (count($subPatternsByType['cidr']) === 0 && count($subPatternsByType['wildcard']) === 0) { - return $subPatternsByType['ip']; + // TODO: cache to a file? + return $cache[$patternHash] = $subPatternsByType['ip']; } - return static::generateClosure($subPatternsByType, $uid); - } + // generate source code for closure + $closureFunctionString = "write( - $fileName, - $closureFunctionString - ); + // write to filesystem + file_put_contents($cacheFilePath, $closureFunctionString); - return include join('/', [Storage::getLocalStorageRoot(), static::$fsRootDir, $fileName]); + return $cache[$patternHash] = include($cacheFilePath); } /** @@ -133,7 +121,7 @@ protected static function generateClosureCondition($pattern, $patternType) $ipLong = ip2long($pattern); $conditionString = <<= $min && \$ipLong <= $max) { return true; } From d9494b3b7dca1670b97eda1bede52f73bea86cbf Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Mon, 4 Jan 2021 22:31:39 +0000 Subject: [PATCH 33/40] refactor: add match() method to IPPattern utility --- php-classes/Gatekeeper/Bans/Ban.php | 3 +-- php-classes/Gatekeeper/Utils/IPPattern.php | 7 +++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/php-classes/Gatekeeper/Bans/Ban.php b/php-classes/Gatekeeper/Bans/Ban.php index a1f252f..195146e 100644 --- a/php-classes/Gatekeeper/Bans/Ban.php +++ b/php-classes/Gatekeeper/Bans/Ban.php @@ -145,8 +145,7 @@ public static function isIPAddressBanned($ip) // check IP Patterns individually foreach ($activeBans['patterns'] as $ipPattern) { - $matcher = IPPattern::parse($ipPattern); - if (call_user_func($matcher, $ip) === true) { + if (IPPattern::match($ipPattern, $ip)) { return true; } } diff --git a/php-classes/Gatekeeper/Utils/IPPattern.php b/php-classes/Gatekeeper/Utils/IPPattern.php index 6671d51..bd75dc5 100644 --- a/php-classes/Gatekeeper/Utils/IPPattern.php +++ b/php-classes/Gatekeeper/Utils/IPPattern.php @@ -11,6 +11,13 @@ class IPPattern { public static $fsRootDir = 'ip-patterns/matchers'; + + public static function match($pattern, $ip) + { + $matcher = static::parse($pattern); + return call_user_func($matcher, $ip) === true; + } + /** * * Parse IP pattens by splitting them by spaces or commas and grouping them From bb8aa738753ca6cd630cbc8e3dd30fc9c3cb320b Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Mon, 4 Jan 2021 22:32:24 +0000 Subject: [PATCH 34/40] feat: load exemption record into ApiRequest early in pipeline --- .../beforeApiRequest/41_load-exemption.php | 12 +++ php-classes/Gatekeeper/ApiRequest.php | 12 +++ .../Gatekeeper/Exemptions/Exemption.php | 77 +++++++++++++++++++ 3 files changed, 101 insertions(+) create mode 100644 event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/41_load-exemption.php diff --git a/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/41_load-exemption.php b/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/41_load-exemption.php new file mode 100644 index 0000000..e8283d5 --- /dev/null +++ b/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/41_load-exemption.php @@ -0,0 +1,12 @@ +setExemption($Exemption); +} diff --git a/php-classes/Gatekeeper/ApiRequest.php b/php-classes/Gatekeeper/ApiRequest.php index 1beffec..4682c00 100644 --- a/php-classes/Gatekeeper/ApiRequest.php +++ b/php-classes/Gatekeeper/ApiRequest.php @@ -4,6 +4,7 @@ use Gatekeeper\Endpoints\Endpoint; use Gatekeeper\Keys\Key; +use Gatekeeper\Exemptions\Exemption; use Gatekeeper\Transactions\Transaction; use Emergence\Site\Client; @@ -14,6 +15,7 @@ class ApiRequest protected $endpoint; protected $clientAddress; protected $key; + protected $exemption; protected $url = ''; protected $transaction; @@ -81,6 +83,16 @@ public function setKey(Key $Key) $this->key = $Key; } + public function getExemption() + { + return $this->exemption; + } + + public function setExemption(Exemption $Exemption) + { + $this->exemption = $Exemption; + } + public function getUrl() { return $this->url; diff --git a/php-classes/Gatekeeper/Exemptions/Exemption.php b/php-classes/Gatekeeper/Exemptions/Exemption.php index c47d8bf..c5a4984 100644 --- a/php-classes/Gatekeeper/Exemptions/Exemption.php +++ b/php-classes/Gatekeeper/Exemptions/Exemption.php @@ -3,6 +3,7 @@ namespace Gatekeeper\Exemptions; use Cache; +use Gatekeeper\ApiRequest; use Gatekeeper\Keys\Key; use Gatekeeper\Utils\IPPattern; @@ -99,4 +100,80 @@ public static function sortCreated($dir, $name) { return "ID $dir"; } + + public static function getForApiRequest(ApiRequest $request) + { + $exemptions = static::getActiveExemptionsTable(); + + // look for IP match first + $clientAddress = $request->getClientAddress(); + if ($clientAddress && !empty($exemptions['ips'])) { + foreach ($exemptions['ips'] as $ip => $exemptionId) { + if ($ip == $clientAddress) { + return static::getById($exemptionId); + } + } + } + + // look for key match second + $Key = $request->getKey(); + if ($Key && !empty($exemptions['keys'])) { + foreach ($exemptions['keys'] as $keyId => $exemptionId) { + if ($keyId == $Key->ID) { + return static::getById($exemptionId); + } + } + } + + // finally, execute pattern matches + if (!empty($exemptions['patterns'])) { + foreach ($exemptions['patterns'] as $pattern => $exemptionId) { + if (IPPattern::match($pattern, $ip)) { + return static::getById($exemptionId); + } + } + } + + return null; + } + + public static function getActiveExemptionsTable() + { + static $cache = null; + + if ($cache) { + return $cache; + } + + if ($cache = Cache::fetch('exemptions')) { + return $cache; + } + + $cache = [ + 'patterns' => [], + 'ips' => [], + 'keys' => [] + ]; + + foreach (static::getAllByWhere('ExpirationDate IS NULL OR ExpirationDate > CURRENT_TIMESTAMP') AS $Exemption) { + if (!empty($Exemption->IPPattern)) { + $parsed = IPPattern::parse($Exemption->IPPattern); + + if (is_array($parsed)) { + // ip pattern ONLY contains static IPs + foreach ($parsed as $ip) { + $cache['ips'][$ip] = $Exemption->ID; + } + } else { + $cache['patterns'][$Exemption->IPPattern] = $Exemption->ID; + } + } elseif ($Exemption->KeyID) { + $cache['keys'][$Exemption->KeyID] = $Exemption->ID; + } + } + + Cache::store('exemptions', $cache, static::$tableCachePeriod); + + return $cache; + } } From 46804e909e1f296f0372d1b6d3ec239bab1d69dd Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Mon, 4 Jan 2021 22:43:04 +0000 Subject: [PATCH 35/40] feat: apply exemptions to user-level rate limits --- .../beforeApiRequest/75_reject-endpoint-user-rate.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/75_reject-endpoint-user-rate.php b/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/75_reject-endpoint-user-rate.php index 80e6d07..16f9d6a 100644 --- a/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/75_reject-endpoint-user-rate.php +++ b/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/75_reject-endpoint-user-rate.php @@ -6,12 +6,14 @@ $Endpoint = $_EVENT['request']->getEndpoint(); $userIdentifier = $_EVENT['request']->getUserIdentifier(); $Key = $_EVENT['request']->getKey(); +$Exemption = $_EVENT['request']->getExemption(); // drip into endpoint+user bucket first so that abusive users can't pollute the global bucket if ( - (!$Key || !$Key->RateLimitExempt) && - ($Endpoint->UserRatePeriod && $Endpoint->UserRateCount) + !$Exemption + && (!$Key || !$Key->RateLimitExempt) + && ($Endpoint->UserRatePeriod && $Endpoint->UserRateCount) ) { $bucket = HitBuckets::drip("endpoints/$Endpoint->ID/$userIdentifier", function() use ($Endpoint) { return array('seconds' => $Endpoint->UserRatePeriod, 'count' => $Endpoint->UserRateCount); From 772ae4083d4521626741f49dd5e843b7cb383abd Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Mon, 4 Jan 2021 23:06:33 +0000 Subject: [PATCH 36/40] refactor: rename BypassGlobalLimit to BypassEndpointLimits --- php-classes/Gatekeeper/Exemptions/Exemption.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/php-classes/Gatekeeper/Exemptions/Exemption.php b/php-classes/Gatekeeper/Exemptions/Exemption.php index c5a4984..10dcbae 100644 --- a/php-classes/Gatekeeper/Exemptions/Exemption.php +++ b/php-classes/Gatekeeper/Exemptions/Exemption.php @@ -30,7 +30,7 @@ class Exemption extends \ActiveRecord 'type' => 'timestamp', 'default' => null ], - 'BypassGlobalLimit' => [ + 'BypassEndpointLimits' => [ 'type' => 'boolean', 'default' => false ], From 6b55e95d637f6625877ed08c161450cf50245e38 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Mon, 4 Jan 2021 23:06:44 +0000 Subject: [PATCH 37/40] feat: make BypassEndpointLimits editable --- html-templates/exemptions/exemptionEdit.tpl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/html-templates/exemptions/exemptionEdit.tpl b/html-templates/exemptions/exemptionEdit.tpl index a9175f5..2470308 100644 --- a/html-templates/exemptions/exemptionEdit.tpl +++ b/html-templates/exemptions/exemptionEdit.tpl @@ -34,6 +34,8 @@ {field inputName=ExpirationDate label='Expiration Date' type=date default=tif($Exemption->ExpirationDate, date('Y-m-d', $Exemption->ExpirationDate)) hint="Leave blank for indefinate exemption"} + {checkbox inputName=BypassEndpointLimits value=1 unsetValue=0 label='Bypass endpoint rate limits?' default=$Exemption->BypassEndpointLimits hint="Check this option to always allow requests regardless of any endpoint-level aggregate rate limit, and to skip counting any hits against such rate limits."} + {textarea inputName=Notes label='Notes' default=$Exemption->Notes}
    From e4fa0207ae4d0d00e2cf25bd32a4ed2dee563f79 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Mon, 4 Jan 2021 23:07:05 +0000 Subject: [PATCH 38/40] feat: apply exemptions with BypassEndpointLimits to endpoint rate limits --- .../beforeApiRequest/95_reject-endpoint-rate.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/95_reject-endpoint-rate.php b/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/95_reject-endpoint-rate.php index c373229..e37d9e4 100644 --- a/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/95_reject-endpoint-rate.php +++ b/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/95_reject-endpoint-rate.php @@ -9,12 +9,14 @@ $Endpoint = $_EVENT['request']->getEndpoint(); $Key = $_EVENT['request']->getKey(); +$Exemption = $_EVENT['request']->getExemption(); // drip into endpoint requests bucket if ( - (!$Key || !$Key->RateLimitExempt) && - ($Endpoint->GlobalRatePeriod && $Endpoint->GlobalRateCount) + (!$Exemption || !$Exemption->BypassEndpointLimits) + && (!$Key || !$Key->RateLimitExempt) + && ($Endpoint->GlobalRatePeriod && $Endpoint->GlobalRateCount) ) { $flagKey = "alerts/endpoints/$Endpoint->ID/rate-flagged"; From 0bbdeb961da1f7fb104580b36d94d5d0516c4f9b Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Mon, 4 Jan 2021 23:09:08 +0000 Subject: [PATCH 39/40] feat(fixtures): add exemption and key fixtures --- fixtures/exemptions.sql | 21 +++++++++++++++++++++ fixtures/keys.sql | 21 +++++++++++++++++++++ fixtures/people.sql | 2 +- 3 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 fixtures/exemptions.sql create mode 100644 fixtures/keys.sql diff --git a/fixtures/exemptions.sql b/fixtures/exemptions.sql new file mode 100644 index 0000000..47da384 --- /dev/null +++ b/fixtures/exemptions.sql @@ -0,0 +1,21 @@ +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40101 SET character_set_client = utf8 */; + +CREATE TABLE `exemptions` ( + `ID` int(10) unsigned NOT NULL AUTO_INCREMENT, + `Class` enum('Gatekeeper\\Exemptions\\Exemption') NOT NULL, + `Created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `CreatorID` int(11) DEFAULT NULL, + `KeyID` int(10) unsigned DEFAULT NULL, + `IPPattern` varchar(255) DEFAULT NULL, + `ExpirationDate` timestamp NULL DEFAULT NULL, + `BypassEndpointLimits` tinyint(1) NOT NULL DEFAULT '0', + `Notes` text, + PRIMARY KEY (`ID`), + FULLTEXT KEY `FULLTEXT` (`Notes`) +) ENGINE=MyISAM DEFAULT CHARSET=utf8; + +INSERT INTO `exemptions` VALUES (1,'Gatekeeper\\Exemptions\\Exemption','2019-01-02 03:04:05',1,NULL,'192.168.1.1,192.168.1.*,192.168.1.1/24','2021-01-09 05:00:00',0,'future patterns'); +INSERT INTO `exemptions` VALUES (2,'Gatekeeper\\Exemptions\\Exemption','2019-01-02 03:04:05',1,NULL,'8.8.8.8,4.2.2.2','2021-01-03 05:00:00',0,'expired patterns'); +INSERT INTO `exemptions` VALUES (3,'Gatekeeper\\Exemptions\\Exemption','2019-01-02 03:04:05',1,NULL,'141.158.45.69',NULL,0,'perm static'); +INSERT INTO `exemptions` VALUES (4,'Gatekeeper\\Exemptions\\Exemption','2019-01-02 03:04:05',1,1,NULL,NULL,1,NULL); diff --git a/fixtures/keys.sql b/fixtures/keys.sql new file mode 100644 index 0000000..751d0e4 --- /dev/null +++ b/fixtures/keys.sql @@ -0,0 +1,21 @@ +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40101 SET character_set_client = utf8 */; + +CREATE TABLE `keys` ( + `ID` int(10) unsigned NOT NULL AUTO_INCREMENT, + `Class` enum('Gatekeeper\\Keys\\Key') NOT NULL, + `Created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `CreatorID` int(11) DEFAULT NULL, + `Key` varchar(255) NOT NULL, + `Status` enum('active','revoked') NOT NULL DEFAULT 'active', + `OwnerName` varchar(255) NOT NULL, + `ContactName` varchar(255) DEFAULT NULL, + `ContactEmail` varchar(255) DEFAULT NULL, + `ExpirationDate` timestamp NULL DEFAULT NULL, + `AllEndpoints` tinyint(1) NOT NULL DEFAULT '0', + `RateLimitExempt` tinyint(1) NOT NULL DEFAULT '0', + PRIMARY KEY (`ID`), + UNIQUE KEY `Key` (`Key`) +) ENGINE=MyISAM DEFAULT CHARSET=utf8; + +INSERT INTO `keys` VALUES (1,'Gatekeeper\\Keys\\Key','2019-01-02 03:04:05',1,'56e877567dc444c6a4e06e45f1560ee2','active','Keymaster','Key Master','keymaster@example.com',NULL,1,0); diff --git a/fixtures/people.sql b/fixtures/people.sql index 349eea2..eed4db3 100644 --- a/fixtures/people.sql +++ b/fixtures/people.sql @@ -25,7 +25,7 @@ CREATE TABLE `people` ( PRIMARY KEY (`ID`), UNIQUE KEY `Email` (`Email`), UNIQUE KEY `Username` (`Username`) -) ENGINE=MyISAM AUTO_INCREMENT=2 DEFAULT CHARSET=utf8; +) ENGINE=MyISAM DEFAULT CHARSET=utf8; INSERT INTO `people` VALUES (1,'Emergence\\People\\User','2019-01-02 03:04:05',1,NULL,NULL,'Admin','Person',NULL,NULL,NULL,NULL,'admin@example.com',NULL,NULL,NULL,NULL,'admin','$2y$10$rAOnrPHjxdyr40NnSphAaOqMptte76N2BwmFeMlwulpjQNKHKZ1uK','Developer'); From 2ea5fc6695dbdbd86d7412081305f1b3c76f313f Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Tue, 5 Jan 2021 00:04:36 +0000 Subject: [PATCH 40/40] feat: migrate Key->RateLimitExempt to Exemption records --- .../afterApiRequest/10_register-response.php | 6 +-- .../75_reject-endpoint-user-rate.php | 2 - .../95_reject-endpoint-rate.php | 2 - fixtures/keys.sql | 3 +- html-templates/keys/keyEdit.tpl | 1 - php-classes/Gatekeeper/Keys/Key.php | 4 -- .../20210104_migrate-key-exemptions.php | 46 +++++++++++++++++++ .../20200524000000_add-rate-limit-exempt.php | 30 +----------- 8 files changed, 49 insertions(+), 45 deletions(-) create mode 100644 php-migrations/Gatekeeper/Exemption/20210104_migrate-key-exemptions.php diff --git a/event-handlers/Gatekeeper/ApiRequestHandler/afterApiRequest/10_register-response.php b/event-handlers/Gatekeeper/ApiRequestHandler/afterApiRequest/10_register-response.php index 15d2ba4..2ef427b 100644 --- a/event-handlers/Gatekeeper/ApiRequestHandler/afterApiRequest/10_register-response.php +++ b/event-handlers/Gatekeeper/ApiRequestHandler/afterApiRequest/10_register-response.php @@ -7,7 +7,6 @@ $Endpoint = $_EVENT['request']->getEndpoint(); $userIdentifier = $_EVENT['request']->getUserIdentifier(); -$Key = $_EVENT['request']->getKey(); // append metrics @@ -21,10 +20,7 @@ // drip bandwidth bucket -if ( - (!$Key || !$Key->RateLimitExempt) && - ($Endpoint->GlobalBandwidthPeriod && $Endpoint->GlobalBandwidthCount) -) { +if ($Endpoint->GlobalBandwidthPeriod && $Endpoint->GlobalBandwidthCount) { HitBuckets::drip("endpoints/$Endpoint->ID/bandwidth", function() use ($Endpoint) { return [ 'seconds' => $Endpoint->GlobalBandwidthPeriod, diff --git a/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/75_reject-endpoint-user-rate.php b/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/75_reject-endpoint-user-rate.php index 16f9d6a..43cbee7 100644 --- a/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/75_reject-endpoint-user-rate.php +++ b/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/75_reject-endpoint-user-rate.php @@ -5,14 +5,12 @@ $Endpoint = $_EVENT['request']->getEndpoint(); $userIdentifier = $_EVENT['request']->getUserIdentifier(); -$Key = $_EVENT['request']->getKey(); $Exemption = $_EVENT['request']->getExemption(); // drip into endpoint+user bucket first so that abusive users can't pollute the global bucket if ( !$Exemption - && (!$Key || !$Key->RateLimitExempt) && ($Endpoint->UserRatePeriod && $Endpoint->UserRateCount) ) { $bucket = HitBuckets::drip("endpoints/$Endpoint->ID/$userIdentifier", function() use ($Endpoint) { diff --git a/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/95_reject-endpoint-rate.php b/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/95_reject-endpoint-rate.php index e37d9e4..b8fc0be 100644 --- a/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/95_reject-endpoint-rate.php +++ b/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/95_reject-endpoint-rate.php @@ -8,14 +8,12 @@ $Endpoint = $_EVENT['request']->getEndpoint(); -$Key = $_EVENT['request']->getKey(); $Exemption = $_EVENT['request']->getExemption(); // drip into endpoint requests bucket if ( (!$Exemption || !$Exemption->BypassEndpointLimits) - && (!$Key || !$Key->RateLimitExempt) && ($Endpoint->GlobalRatePeriod && $Endpoint->GlobalRateCount) ) { $flagKey = "alerts/endpoints/$Endpoint->ID/rate-flagged"; diff --git a/fixtures/keys.sql b/fixtures/keys.sql index 751d0e4..53d84bf 100644 --- a/fixtures/keys.sql +++ b/fixtures/keys.sql @@ -13,9 +13,8 @@ CREATE TABLE `keys` ( `ContactEmail` varchar(255) DEFAULT NULL, `ExpirationDate` timestamp NULL DEFAULT NULL, `AllEndpoints` tinyint(1) NOT NULL DEFAULT '0', - `RateLimitExempt` tinyint(1) NOT NULL DEFAULT '0', PRIMARY KEY (`ID`), UNIQUE KEY `Key` (`Key`) ) ENGINE=MyISAM DEFAULT CHARSET=utf8; -INSERT INTO `keys` VALUES (1,'Gatekeeper\\Keys\\Key','2019-01-02 03:04:05',1,'56e877567dc444c6a4e06e45f1560ee2','active','Keymaster','Key Master','keymaster@example.com',NULL,1,0); +INSERT INTO `keys` VALUES (1,'Gatekeeper\\Keys\\Key','2019-01-02 03:04:05',1,'56e877567dc444c6a4e06e45f1560ee2','active','Keymaster','Key Master','keymaster@example.com',NULL,1); diff --git a/html-templates/keys/keyEdit.tpl b/html-templates/keys/keyEdit.tpl index 5e871ee..8a8be59 100644 --- a/html-templates/keys/keyEdit.tpl +++ b/html-templates/keys/keyEdit.tpl @@ -35,7 +35,6 @@ {field inputName=ExpirationDate label='Expiration Date' type=date default=tif($Key->ExpirationDate, date('Y-m-d', $Key->ExpirationDate)) hint="Leave blank if none"} - {checkbox inputName=RateLimitExempt value=1 unsetValue=0 label='Exempt from Rate Limits?' default=$Key->RateLimitExempt hint="Check this option to exempt this key from rate limit thresholds and impacting other consumers of the API."} {checkbox inputName=AllEndpoints value=1 unsetValue=0 label='Allow all endpoints?' default=$Key->AllEndpoints hint="Uncheck this option to allow more fine-grained access control to endpoints on the key page."} {checkbox inputName=Status value=revoked unsetValue=active label='Revoked' default=$Key->Status} diff --git a/php-classes/Gatekeeper/Keys/Key.php b/php-classes/Gatekeeper/Keys/Key.php index a6eabb4..bd519e0 100644 --- a/php-classes/Gatekeeper/Keys/Key.php +++ b/php-classes/Gatekeeper/Keys/Key.php @@ -46,10 +46,6 @@ class Key extends \ActiveRecord 'AllEndpoints' => [ 'type' => 'boolean', 'default' => false - ], - 'RateLimitExempt' => [ - 'type' => 'boolean', - 'default' => false ] ]; diff --git a/php-migrations/Gatekeeper/Exemption/20210104_migrate-key-exemptions.php b/php-migrations/Gatekeeper/Exemption/20210104_migrate-key-exemptions.php new file mode 100644 index 0000000..8ca6d4c --- /dev/null +++ b/php-migrations/Gatekeeper/Exemption/20210104_migrate-key-exemptions.php @@ -0,0 +1,46 @@ +getTitle()); + Exemption::create([ + 'KeyID' => $Key->ID, + 'BypassEndpointLimits' => true, // old behavior was to always do this + 'Notes' => 'Generated automatically from legacy Key->RateLimitExempt value' + ], true); +} + +// drop exemption column +static::dropColumn('keys', 'RateLimitExempt'); + + +return static::STATUS_EXECUTED; diff --git a/php-migrations/Gatekeeper/Key/20200524000000_add-rate-limit-exempt.php b/php-migrations/Gatekeeper/Key/20200524000000_add-rate-limit-exempt.php index 1d37872..159fb9d 100644 --- a/php-migrations/Gatekeeper/Key/20200524000000_add-rate-limit-exempt.php +++ b/php-migrations/Gatekeeper/Key/20200524000000_add-rate-limit-exempt.php @@ -1,31 +1,3 @@ RateLimitExempt field deprecated in favor of $Exemption model