From 616bb8602741d9d23a381d370ec6007e610570c9 Mon Sep 17 00:00:00 2001 From: Nafis Bey Date: Sun, 24 May 2020 23:44:05 +0000 Subject: [PATCH 01/29] feat: add support for RateLimitExempt column --- html-templates/keys/keyEdit.tpl | 1 + php-classes/Gatekeeper/Keys/Key.php | 4 +++ .../20200524000000_add-rate-limit-exempt.php | 31 +++++++++++++++++++ 3 files changed, 36 insertions(+) create mode 100644 php-migrations/Gatekeeper/Key/20200524000000_add-rate-limit-exempt.php diff --git a/html-templates/keys/keyEdit.tpl b/html-templates/keys/keyEdit.tpl index 8a8be59..5e871ee 100644 --- a/html-templates/keys/keyEdit.tpl +++ b/html-templates/keys/keyEdit.tpl @@ -35,6 +35,7 @@ {field inputName=ExpirationDate label='Expiration Date' type=date default=tif($Key->ExpirationDate, date('Y-m-d', $Key->ExpirationDate)) hint="Leave blank if none"} + {checkbox inputName=RateLimitExempt value=1 unsetValue=0 label='Exempt from Rate Limits?' default=$Key->RateLimitExempt hint="Check this option to exempt this key from rate limit thresholds and impacting other consumers of the API."} {checkbox inputName=AllEndpoints value=1 unsetValue=0 label='Allow all endpoints?' default=$Key->AllEndpoints hint="Uncheck this option to allow more fine-grained access control to endpoints on the key page."} {checkbox inputName=Status value=revoked unsetValue=active label='Revoked' default=$Key->Status} diff --git a/php-classes/Gatekeeper/Keys/Key.php b/php-classes/Gatekeeper/Keys/Key.php index bd519e0..a6eabb4 100644 --- a/php-classes/Gatekeeper/Keys/Key.php +++ b/php-classes/Gatekeeper/Keys/Key.php @@ -46,6 +46,10 @@ class Key extends \ActiveRecord 'AllEndpoints' => [ 'type' => 'boolean', 'default' => false + ], + 'RateLimitExempt' => [ + 'type' => 'boolean', + 'default' => false ] ]; diff --git a/php-migrations/Gatekeeper/Key/20200524000000_add-rate-limit-exempt.php b/php-migrations/Gatekeeper/Key/20200524000000_add-rate-limit-exempt.php new file mode 100644 index 0000000..1d37872 --- /dev/null +++ b/php-migrations/Gatekeeper/Key/20200524000000_add-rate-limit-exempt.php @@ -0,0 +1,31 @@ + Date: Mon, 25 May 2020 03:26:39 +0000 Subject: [PATCH 02/29] feat: allow RateLimitExempt keys to bypass metrics --- .../afterApiRequest/10_register-response.php | 8 ++++++-- .../beforeApiRequest/75_reject-endpoint-user-rate.php | 6 +++++- .../beforeApiRequest/95_reject-endpoint-rate.php | 6 +++++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/event-handlers/Gatekeeper/ApiRequestHandler/afterApiRequest/10_register-response.php b/event-handlers/Gatekeeper/ApiRequestHandler/afterApiRequest/10_register-response.php index 7a24d5c..15d2ba4 100644 --- a/event-handlers/Gatekeeper/ApiRequestHandler/afterApiRequest/10_register-response.php +++ b/event-handlers/Gatekeeper/ApiRequestHandler/afterApiRequest/10_register-response.php @@ -7,6 +7,7 @@ $Endpoint = $_EVENT['request']->getEndpoint(); $userIdentifier = $_EVENT['request']->getUserIdentifier(); +$Key = $_EVENT['request']->getKey(); // append metrics @@ -20,11 +21,14 @@ // drip bandwidth bucket -if ($Endpoint->GlobalBandwidthPeriod && $Endpoint->GlobalBandwidthCount) { +if ( + (!$Key || !$Key->RateLimitExempt) && + ($Endpoint->GlobalBandwidthPeriod && $Endpoint->GlobalBandwidthCount) +) { HitBuckets::drip("endpoints/$Endpoint->ID/bandwidth", function() use ($Endpoint) { return [ 'seconds' => $Endpoint->GlobalBandwidthPeriod, 'count' => $Endpoint->GlobalBandwidthCount ]; }, $_EVENT['Transaction']->ResponseBytes); -} \ No newline at end of file +} diff --git a/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/75_reject-endpoint-user-rate.php b/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/75_reject-endpoint-user-rate.php index 442a7ab..80e6d07 100644 --- a/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/75_reject-endpoint-user-rate.php +++ b/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/75_reject-endpoint-user-rate.php @@ -5,10 +5,14 @@ $Endpoint = $_EVENT['request']->getEndpoint(); $userIdentifier = $_EVENT['request']->getUserIdentifier(); +$Key = $_EVENT['request']->getKey(); // drip into endpoint+user bucket first so that abusive users can't pollute the global bucket -if ($Endpoint->UserRatePeriod && $Endpoint->UserRateCount) { +if ( + (!$Key || !$Key->RateLimitExempt) && + ($Endpoint->UserRatePeriod && $Endpoint->UserRateCount) +) { $bucket = HitBuckets::drip("endpoints/$Endpoint->ID/$userIdentifier", function() use ($Endpoint) { return array('seconds' => $Endpoint->UserRatePeriod, 'count' => $Endpoint->UserRateCount); }); diff --git a/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/95_reject-endpoint-rate.php b/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/95_reject-endpoint-rate.php index 039d88b..c373229 100644 --- a/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/95_reject-endpoint-rate.php +++ b/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/95_reject-endpoint-rate.php @@ -8,10 +8,14 @@ $Endpoint = $_EVENT['request']->getEndpoint(); +$Key = $_EVENT['request']->getKey(); // drip into endpoint requests bucket -if ($Endpoint->GlobalRatePeriod && $Endpoint->GlobalRateCount) { +if ( + (!$Key || !$Key->RateLimitExempt) && + ($Endpoint->GlobalRatePeriod && $Endpoint->GlobalRateCount) +) { $flagKey = "alerts/endpoints/$Endpoint->ID/rate-flagged"; $bucket = HitBuckets::drip("endpoints/$Endpoint->ID/requests", function() use ($Endpoint) { From 9225c6f14154f069bec7166df2d1eccac81e321a Mon Sep 17 00:00:00 2001 From: Nafis Bey Date: Fri, 14 Aug 2020 03:33:43 +0000 Subject: [PATCH 03/29] feat: implement docker builds - create gatekeeper-composite habitat plan --- .env.example | 11 +++++++ .gitignore | 4 +++ .holo/sources/skeleton-v2.toml | 2 +- Dockerfile | 60 ++++++++++++++++++++++++++++++++++ docker-compose.override.yml | 14 ++++++++ docker-compose.yml | 19 +++++++++++ habitat/composite/default.toml | 2 ++ habitat/composite/plan.sh | 10 ++++++ habitat/default.toml | 2 ++ habitat/plan.sh | 8 +++++ 10 files changed, 131 insertions(+), 1 deletion(-) create mode 100644 .env.example create mode 100644 Dockerfile create mode 100644 docker-compose.override.yml create mode 100644 docker-compose.yml create mode 100644 habitat/composite/default.toml create mode 100644 habitat/composite/plan.sh create mode 100644 habitat/default.toml create mode 100644 habitat/plan.sh diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..af80492 --- /dev/null +++ b/.env.example @@ -0,0 +1,11 @@ +# - Copy .env.example to .env to initialize a new instance +# - DO NOT commit .env to source control +# - See docs/deployment/docker-compose.md for configuration reference + + +# Accept Chef Habitat's license by activating this .env +HAB_LICENSE=accept-no-persist + +# MySQL database (local and remote) +MYSQL_USERNAME=gatekeeper +MYSQL_PASSWORD=8edvN7xotT49qLZReA2Cx1TFKpvvxK9 diff --git a/.gitignore b/.gitignore index 23cc375..61da8c4 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,7 @@ # exclude built files from source control /site-root/js/pages/ /site-root/css/ + +# ignore habitat build results +results/ +.env diff --git a/.holo/sources/skeleton-v2.toml b/.holo/sources/skeleton-v2.toml index 0d566d9..68a5ede 100644 --- a/.holo/sources/skeleton-v2.toml +++ b/.holo/sources/skeleton-v2.toml @@ -1,6 +1,6 @@ [holosource] url = "https://github.com/JarvusInnovations/emergence-skeleton-v2" -ref = "refs/tags/v2.4.0" +ref = "refs/tags/v2.4.1" [holosource.project] holobranch = "emergence-skeleton" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..bb42fa6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,60 @@ +# This Dockerfile is hyper-optimized to minimize layer changes + +FROM jarvus/habitat-compose:latest as habitat +ARG HAB_LICENSE=no-accept +ENV HAB_LICENSE=$HAB_LICENSE +ENV STUDIO_TYPE=Dockerfile +ENV HAB_ORIGIN=jarvus +RUN hab origin key generate +# pre-layer all external runtime plan deps +COPY habitat/plan.sh /habitat/plan.sh +RUN hab pkg install \ + core/bash \ + emergence/php-runtime \ + $({ cat '/habitat/plan.sh' && echo && echo 'echo "${pkg_deps[@]/$pkg_origin\/*/}"'; } | hab pkg exec core/bash bash) \ + && hab pkg exec core/coreutils rm -rf /hab/cache/{artifacts,src}/ +# pre-layer all external runtime composite deps +COPY habitat/composite/plan.sh /habitat/composite/plan.sh +RUN hab pkg install \ + jarvus/habitat-compose \ + emergence/nginx \ + $({ cat '/habitat/composite/plan.sh' && echo && echo 'echo "${pkg_deps[@]/$pkg_origin\/*/} ${composite_mysql_pkg}'; } | hab pkg exec core/bash bash) \ + && hab pkg exec core/coreutils rm -rf /hab/cache/{artifacts,src}/ + + +FROM habitat as projector +# pre-layer all build-time plan deps +RUN hab pkg install \ + core/hab-plan-build \ + jarvus/hologit \ + jarvus/toml-merge \ + $({ cat '/habitat/plan.sh' && echo && echo 'echo "${pkg_build_deps[@]/$pkg_origin\/*/}"'; } | hab pkg exec core/bash bash) \ + && hab pkg exec core/coreutils rm -rf /hab/cache/{artifacts,src}/ +# pre-layer all build-time composite deps +RUN hab pkg install \ + jarvus/toml-merge \ + $({ cat '/habitat/composite/plan.sh' && echo && echo 'echo "${pkg_build_deps[@]/$pkg_origin\/*/}"'; } | hab pkg exec core/bash bash) \ + && hab pkg exec core/coreutils rm -rf /hab/cache/{artifacts,src}/ +# build application +COPY . /src +RUN hab pkg exec core/hab-plan-build hab-plan-build /src +RUN hab pkg exec core/hab-plan-build hab-plan-build /src/habitat/composite + + +FROM habitat as runtime +# install .hart artifact from builder stage +COPY --from=projector /hab/cache/artifacts/$HAB_ORIGIN-* /hab/cache/artifacts/ +RUN hab pkg install /hab/cache/artifacts/$HAB_ORIGIN-* \ + && hab pkg exec core/coreutils rm -rf /hab/cache/{artifacts,src}/ + + +# configure persistent volumes +RUN hab pkg exec core/coreutils mkdir -p '/hab/svc/mysql/data' '/hab/svc/gatekeeper/data' '/hab/svc/nginx/files' \ + && hab pkg exec core/coreutils chown hab:hab -R '/hab/svc/mysql/data' '/hab/svc/gatekeeper/data' '/hab/svc/nginx/files' + +VOLUME ["/hab/svc/mysql/data", "/hab/svc/gatekeeper/data", "/hab/svc/nginx/files"] + + +# configure entrypoint +ENTRYPOINT ["hab", "sup", "run"] +CMD ["jarvus/gatekeeper-composite"] diff --git a/docker-compose.override.yml b/docker-compose.override.yml new file mode 100644 index 0000000..173dfcd --- /dev/null +++ b/docker-compose.override.yml @@ -0,0 +1,14 @@ +version: '3' +services: + app: + environment: + + # assign any username/password--both app and db will pick it up and initialize + HAB_MYSQL: | + # these can be made up: + app_username = '${MYSQL_USERNAME}' + app_password = '${MYSQL_PASSWORD}' + bind = '0.0.0.0' + + ports: + - 127.0.0.1:3306:3306 # expose mysql on localhost diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..b63fe87 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,19 @@ +version: '3' +services: + app: + # create via `hab pkg export docker --multi-layer ./gatekeeper-composite.hart`: + image: jarvus/gatekeeper-composite:latest + volumes: + # persist uploaded media: + - gatekeeper-app-data:/hab/svc/gatekeeper/data + # persist uploaded SSL certificates: + - ./nginx-files:/hab/svc/nginx/files + ports: + - 80:80 + environment: + HAB_LICENSE: ${HAB_LICENSE} +volumes: + gatekeeper-app-data: + driver: local + gatekeeper-nginx-files: + driver: local diff --git a/habitat/composite/default.toml b/habitat/composite/default.toml new file mode 100644 index 0000000..0ac11b5 --- /dev/null +++ b/habitat/composite/default.toml @@ -0,0 +1,2 @@ +[services.app.config] + default_timezone = "America/New_York" diff --git a/habitat/composite/plan.sh b/habitat/composite/plan.sh new file mode 100644 index 0000000..3bee95d --- /dev/null +++ b/habitat/composite/plan.sh @@ -0,0 +1,10 @@ +composite_app_pkg_name=gatekeeper +pkg_name="${composite_app_pkg_name}-composite" +pkg_origin=jarvus +pkg_maintainer="Jarvus Innovations " +pkg_scaffolding=emergence/scaffolding-composite +composite_mysql_pkg=core/mysql + +pkg_version() { + scaffolding_detect_pkg_version +} diff --git a/habitat/default.toml b/habitat/default.toml new file mode 100644 index 0000000..2065aaa --- /dev/null +++ b/habitat/default.toml @@ -0,0 +1,2 @@ +[sites.default] +database = "gatekeeper" diff --git a/habitat/plan.sh b/habitat/plan.sh new file mode 100644 index 0000000..c01a1a6 --- /dev/null +++ b/habitat/plan.sh @@ -0,0 +1,8 @@ +pkg_name=gatekeeper +pkg_origin=jarvus +pkg_maintainer="Jarvus Innovations " +pkg_scaffolding=emergence/scaffolding-site + +pkg_version() { + scaffolding_detect_pkg_version +} From 6c9858ade786a507ba0d06438cc847142b890a23 Mon Sep 17 00:00:00 2001 From: Nafis Bey Date: Fri, 14 Aug 2020 17:50:50 +0000 Subject: [PATCH 04/29] feat: deploy PR builds - generate charts via helm --- .github/workflows/k8s-deploy-pr.yml | 150 ++++++++++++++++++ k8s/charts/deployment/Chart.yaml | 23 +++ k8s/charts/deployment/templates/_helpers.tpl | 34 ++++ .../deployment/templates/deployment.yaml | 88 ++++++++++ k8s/charts/deployment/values.yaml | 2 + 5 files changed, 297 insertions(+) create mode 100644 .github/workflows/k8s-deploy-pr.yml create mode 100644 k8s/charts/deployment/Chart.yaml create mode 100644 k8s/charts/deployment/templates/_helpers.tpl create mode 100644 k8s/charts/deployment/templates/deployment.yaml create mode 100644 k8s/charts/deployment/values.yaml diff --git a/.github/workflows/k8s-deploy-pr.yml b/.github/workflows/k8s-deploy-pr.yml new file mode 100644 index 0000000..b22ac19 --- /dev/null +++ b/.github/workflows/k8s-deploy-pr.yml @@ -0,0 +1,150 @@ +name: K8s PR Sandbox Build+Deploy + +on: + pull_request: + branches: [develop] + types: [opened, reopened, synchronize] + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + KUBE_NAMESPACE: ${{ secrets.kube_namespace }} + KUBE_CONFIG_DATA: ${{ secrets.kube_config }} + + PACKAGE_NAME: gatekeeper-composite + PACKAGE_REGISTRY: docker.pkg.github.com + + PR_NAME: pr-${{ github.event.number }} + KUBE_HOSTNAME: ${{ secrets.kube_hostname }} + +jobs: + kubernetes-deploy: + + runs-on: ubuntu-latest + steps: + + - uses: actions/checkout@v2 + with: + ref: ${{ github.head_ref }} + + - name: Create Github Deployment + run: | + set -e + # Create deployment + hub api /repos/${{ github.repository }}/deployments -X POST --input <(cat < /tmp/deployment.json + + DEPLOYMENT_ID=$(jq .id < /tmp/deployment.json) + echo ::set-env name=GH_DEPLOYMENT_ID::$(echo $DEPLOYMENT_ID) + + - name: Update GH Deployment Status + run: | + set -e + + # Set status to pending + hub api /repos/${{ github.repository }}/deployments/${{ env.GH_DEPLOYMENT_ID }}/statuses \ + -X POST \ + -H "Accept: application/json, application/vnd.github.flash-preview+json" \ + --input <(cat < ~/.kube/config + $(printf '%s' "$KUBE_CONFIG_DATA" | base64 -d) + EOF + + - uses: azure/setup-kubectl@v1 + + - name: Create K8S Deployment from Template + run: | + set -e + + image_id=$(echo ${{ env.REPO_NAME }}/${{ env.PACKAGE_NAME }}:${{ env.PR_NAME }}) + image_url=$(echo docker.pkg.github.com/$image_id) + hostname=$(echo ${{ env.PR_NAME }}.${{ env.KUBE_HOSTNAME }}) + + kubectl config set-context --current --namespace=${KUBE_NAMESPACE} + + # uninstall any existing deployment first to force image to re-pull without changing tag + helm uninstall ${PR_NAME} + + helm install ${PR_NAME} ./k8s/charts/deployment \ + --set name={$PR_NAME} \ + --set namespace=${KUBE_NAMESPACE } \ + --set image=${image_url} \ + --set host=${hostname} + + - name: Wait for Deployment to be Ready + run: | + set -e + until kubectl rollout status deployment "${PR_NAME}" 2>/dev/null >/dev/null; do echo -n "."; sleep .5; done; + + - name: Retrieve/Store Pod Name + run: | + echo ::set-env name=POD_NAME::$(kubectl get pod -l pr="${PR_NAME}" -o jsonpath='{.items[0].metadata.name}') + + - name: Wait For Pod to be Ready + run: | + set -e + kubectl wait --for condition=ready "pod/${POD_NAME}" --timeout=30s + + - name: Mark deployment as failed + if: failure() + run: | + hub api /repos/${{ github.repository }}/deployments/${{ env.GH_DEPLOYMENT_ID }}/statuses \ + -X POST \ + -H "Accept: application/json, application/vnd.github.flash-preview+json" \ + --input <(cat < Date: Fri, 14 Aug 2020 18:18:12 +0000 Subject: [PATCH 05/29] fix: Dockerfile location --- .github/workflows/k8s-deploy-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/k8s-deploy-pr.yml b/.github/workflows/k8s-deploy-pr.yml index b22ac19..92e0242 100644 --- a/.github/workflows/k8s-deploy-pr.yml +++ b/.github/workflows/k8s-deploy-pr.yml @@ -68,7 +68,7 @@ jobs: - name: Build & Publish Docker image uses: whoan/docker-build-with-cache-action@v5 with: - dockerfile: docker/php73/Dockerfile + dockerfile: Dockerfile username: ${{ github.actor }} password: ${{ env.GITHUB_TOKEN }} registry: ${{ env.PACKAGE_REGISTRY }} From 47bfcc6ec3f808b5990b78cc549d0be941af5b39 Mon Sep 17 00:00:00 2001 From: Nafis Bey Date: Mon, 17 Aug 2020 14:33:50 +0000 Subject: [PATCH 06/29] fix: set env variables --- .github/workflows/k8s-deploy-pr.yml | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/k8s-deploy-pr.yml b/.github/workflows/k8s-deploy-pr.yml index 92e0242..9847863 100644 --- a/.github/workflows/k8s-deploy-pr.yml +++ b/.github/workflows/k8s-deploy-pr.yml @@ -17,6 +17,11 @@ env: PR_NAME: pr-${{ github.event.number }} KUBE_HOSTNAME: ${{ secrets.kube_hostname }} + HAB_LICENSE: accept-no-persist + + MYSQL_USERNAME: gatekeeper # ${{ secrets.mysql_username }} + MYSQL_PASSWORD: zyxw1234! # ${{ secrets.mysql_password }} + jobs: kubernetes-deploy: @@ -77,6 +82,7 @@ jobs: build_extra_args: | --build-arg=SOURCE_COMMIT=${{ github.sha }} --build-arg=SOURCE_TAG=${{ env.PR_NAME }} + --build-arg=HAB_LICENSE=${{ env.HAB_LICENSE }} - name: Add kubeconfig to environment run: | @@ -98,15 +104,19 @@ jobs: kubectl config set-context --current --namespace=${KUBE_NAMESPACE} - # uninstall any existing deployment first to force image to re-pull without changing tag - helm uninstall ${PR_NAME} + # delete any existing pod first to force image to re-pull without changing tag + kubectl delete pod -l app.kubernetes.io/instance="${PR_NAME}" - helm install ${PR_NAME} ./k8s/charts/deployment \ - --set name={$PR_NAME} \ - --set namespace=${KUBE_NAMESPACE } \ + helm install ${{ env.PR_NAME }} ./k8s/charts/deployment \ + --set name=${{ env.PR_NAME }} \ + --set namespace=${{ env.KUBE_NAMESPACE }} \ --set image=${image_url} \ --set host=${hostname} + - name: Setup tmate session + if: always() + uses: mxschmitt/action-tmate@v2 + - name: Wait for Deployment to be Ready run: | set -e From a8b04889c64d3f0e73eca256834493f1e274535f Mon Sep 17 00:00:00 2001 From: Nafis Bey Date: Tue, 18 Aug 2020 01:18:05 +0000 Subject: [PATCH 07/29] fix: helm deployment template - remove namespace definition - loop through hab env values --- .github/workflows/k8s-deploy-pr.yml | 39 ++++++++------- .../deployment/templates/deployment.yaml | 47 ++++++++++++------- k8s/charts/deployment/values.yaml | 11 +++++ 3 files changed, 59 insertions(+), 38 deletions(-) diff --git a/.github/workflows/k8s-deploy-pr.yml b/.github/workflows/k8s-deploy-pr.yml index 9847863..56aff04 100644 --- a/.github/workflows/k8s-deploy-pr.yml +++ b/.github/workflows/k8s-deploy-pr.yml @@ -70,19 +70,19 @@ jobs: echo ::set-env name=COMMIT_MSG::$(git log --format=%B -n 1 ${{ github.event.after }}) echo ::set-env name=REPO_NAME::$(echo ${GITHUB_REPOSITORY,,}) - - name: Build & Publish Docker image - uses: whoan/docker-build-with-cache-action@v5 - with: - dockerfile: Dockerfile - username: ${{ github.actor }} - password: ${{ env.GITHUB_TOKEN }} - registry: ${{ env.PACKAGE_REGISTRY }} - image_name: ${{ env.REPO_NAME }}/${{ env.PACKAGE_NAME }} - image_tag: ${{ env.PR_NAME }} - build_extra_args: | - --build-arg=SOURCE_COMMIT=${{ github.sha }} - --build-arg=SOURCE_TAG=${{ env.PR_NAME }} - --build-arg=HAB_LICENSE=${{ env.HAB_LICENSE }} + # - name: Build & Publish Docker image + # uses: whoan/docker-build-with-cache-action@v5 + # with: + # dockerfile: Dockerfile + # username: ${{ github.actor }} + # password: ${{ env.GITHUB_TOKEN }} + # registry: ${{ env.PACKAGE_REGISTRY }} + # image_name: ${{ env.REPO_NAME }}/${{ env.PACKAGE_NAME }} + # image_tag: ${{ env.PR_NAME }} + # build_extra_args: | + # --build-arg=SOURCE_COMMIT=${{ github.sha }} + # --build-arg=SOURCE_TAG=${{ env.PR_NAME }} + # --build-arg=HAB_LICENSE=${{ env.HAB_LICENSE }} - name: Add kubeconfig to environment run: | @@ -105,17 +105,16 @@ jobs: kubectl config set-context --current --namespace=${KUBE_NAMESPACE} # delete any existing pod first to force image to re-pull without changing tag - kubectl delete pod -l app.kubernetes.io/instance="${PR_NAME}" + # kubectl delete pod -l app.kubernetes.io/instance="${PR_NAME}" + # helm uninstall ${{ env.PR_NAME }} -n ${{ env.KUBE_NAMESPACE }} - helm install ${{ env.PR_NAME }} ./k8s/charts/deployment \ + helm upgrade ${{ env.PR_NAME }} ./k8s/charts/deployment \ + --install \ --set name=${{ env.PR_NAME }} \ --set namespace=${{ env.KUBE_NAMESPACE }} \ --set image=${image_url} \ - --set host=${hostname} + --set hostname=${hostname} - - name: Setup tmate session - if: always() - uses: mxschmitt/action-tmate@v2 - name: Wait for Deployment to be Ready run: | @@ -124,7 +123,7 @@ jobs: - name: Retrieve/Store Pod Name run: | - echo ::set-env name=POD_NAME::$(kubectl get pod -l pr="${PR_NAME}" -o jsonpath='{.items[0].metadata.name}') + echo ::set-env name=POD_NAME::$(kubectl get pod -l app.kubernetes.io/instance="${PR_NAME}" -o jsonpath='{.items[0].metadata.name}') - name: Wait For Pod to be Ready run: | diff --git a/k8s/charts/deployment/templates/deployment.yaml b/k8s/charts/deployment/templates/deployment.yaml index da9df10..6d2c2af 100644 --- a/k8s/charts/deployment/templates/deployment.yaml +++ b/k8s/charts/deployment/templates/deployment.yaml @@ -1,14 +1,7 @@ -kind: Namespace -apiVersion: v1 -metadata: - name: {{ .Values.namespace }} - ---- - apiVersion: apps/v1 kind: Deployment metadata: - name: {{ .Values.name }} + name: {{ .Release.Name }} namespace: {{ .Values.namespace }} labels: {{- include "gatekeeper-deployment.labels" . | nindent 4 }} @@ -22,7 +15,7 @@ spec: template: metadata: labels: - {{- include "gatekeeper-deployment.labels" . | nindent 4 }} + {{- include "gatekeeper-deployment.labels" . | nindent 8 }} spec: imagePullSecrets: - name: regcred @@ -31,9 +24,27 @@ spec: containers: - image: {{ .Values.image }} - name: {{ .Values.name }}-app + name: {{ .Release.Name }}-app imagePullPolicy: Always + env: + - name: HAB_GATEKEEPER_COMPOSITE + value: | + {{- if .Values.hab.gatekeeper_composite.mysql }} + [services.mysql] + {{- range $key, $value := .Values.hab.gatekeeper_composite.mysql }} + {{ $key }} = {{ $value | quote }} + {{- end }} + {{- end }} + + - name: HAB_MYSQL + value: | + {{- if .Values.hab.mysql }} + {{- range $key, $value := .Values.hab.mysql }} + {{ $key }} = {{ $value | quote }} + {{- end }} + {{- end }} + ports: - containerPort: 80 name: http @@ -43,15 +54,15 @@ spec: apiVersion: v1 kind: Service metadata: - name: {{ .Values.name }} + name: {{ .Release.Name }} namespace: {{ .Values.namespace }} labels: {{- include "gatekeeper-deployment.labels" . | nindent 4 }} - app: {{ .Values.name }} - name: {{ .Values.name }}-app + app: {{ .Release.Name }} + name: {{ .Release.Name }}-app spec: selector: - {{- include "gatekeeper-deployment.selectorLabels" . | nindent 8 }} + {{- include "gatekeeper-deployment.selectorLabels" . | nindent 4 }} ports: - name: http port: 80 @@ -62,7 +73,7 @@ spec: apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: - name: {{ .Values.name }} + name: {{ .Release.Name }} namespace: {{ .Values.namespace }} labels: {{- include "gatekeeper-deployment.labels" . | nindent 4 }} @@ -74,9 +85,9 @@ spec: tls: - hosts: - {{ .Values.hostname }} - secretName: {{ .Values.name }}-tls + secretName: {{ .Release.Name }}-tls backend: - serviceName: {{ .Values.name }} + serviceName: {{ .Release.Name }} servicePort: 80 rules: - host: {{ .Values.hostname }} @@ -84,5 +95,5 @@ spec: paths: - path: / backend: - serviceName: {{ .Values.name }} + serviceName: {{ .Release.Name }} servicePort: 80 diff --git a/k8s/charts/deployment/values.yaml b/k8s/charts/deployment/values.yaml index 8cfbc94..64472db 100644 --- a/k8s/charts/deployment/values.yaml +++ b/k8s/charts/deployment/values.yaml @@ -1,2 +1,13 @@ # Default values for deployment replicaCount: 1 + +hab: + gatekeeper_composite: + mysql: + pkg_ident: 'core/mysql' + +# these can be made up + mysql: + app_username: admin + app_password: xTw9wYFe70 + bind: '0.0.0.0' From b85288814febfd2e3c199be821899a99aff2bedd Mon Sep 17 00:00:00 2001 From: Nafis Bey Date: Wed, 19 Aug 2020 15:31:20 +0000 Subject: [PATCH 08/29] docs: pr deployments --- docs/gatekeeper/deployments/pr-deployments.md | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 docs/gatekeeper/deployments/pr-deployments.md diff --git a/docs/gatekeeper/deployments/pr-deployments.md b/docs/gatekeeper/deployments/pr-deployments.md new file mode 100644 index 0000000..8a616dc --- /dev/null +++ b/docs/gatekeeper/deployments/pr-deployments.md @@ -0,0 +1,29 @@ +# PR Deployments: + +## Github Secrets: + +- `kube_namespace` (required) +Set this to the kubernetes namespace you wish for the app to be deployed to. + +- `kube_config` (required) +Set this to the value **base64 encoded** kubeconfig file used to configure access to the cluster. + +- `kube_hostname` (required) +Set this to the hostname of the kubernetes cluster. +*Hint:* If the `kube_hostname` is set to example.com, the app will be deployed to `pr-1.example.com` + +## Kubernetes Secrets + +- `regcred` (required) +Set this in the kubernetes cluster, within the `kube_namespace` to the login credentials for docker. + + +## Helm Configuration + +#### Helm Template + +Located at `k8s/charts/deployment/` + +- `templates/` - Helm templates for PR Deployments +- `values.yaml` - Default configurable values for PR deployments + From 388a3a417a43df72aecdcae72c0d234a68d64ecf Mon Sep 17 00:00:00 2001 From: Nafis Bey Date: Wed, 19 Aug 2020 15:40:20 +0000 Subject: [PATCH 09/29] chore: remove deprecated secrets --- .github/workflows/k8s-deploy-pr.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/k8s-deploy-pr.yml b/.github/workflows/k8s-deploy-pr.yml index 56aff04..6e0b53a 100644 --- a/.github/workflows/k8s-deploy-pr.yml +++ b/.github/workflows/k8s-deploy-pr.yml @@ -19,9 +19,6 @@ env: HAB_LICENSE: accept-no-persist - MYSQL_USERNAME: gatekeeper # ${{ secrets.mysql_username }} - MYSQL_PASSWORD: zyxw1234! # ${{ secrets.mysql_password }} - jobs: kubernetes-deploy: From a90592d69e186ce61aa040131a418863d5332c3d Mon Sep 17 00:00:00 2001 From: Nafis Bey Date: Wed, 19 Aug 2020 15:46:07 +0000 Subject: [PATCH 10/29] fix: ingress annotations for pr deployments --- k8s/charts/deployment/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/charts/deployment/templates/deployment.yaml b/k8s/charts/deployment/templates/deployment.yaml index 6d2c2af..d8899fd 100644 --- a/k8s/charts/deployment/templates/deployment.yaml +++ b/k8s/charts/deployment/templates/deployment.yaml @@ -79,7 +79,7 @@ metadata: {{- include "gatekeeper-deployment.labels" . | nindent 4 }} annotations: kubernetes.io/ingress.class: "nginx" - certmanager.k8s.io/cluster-issuer: letsencrypt-prod + cert-manager.io/cluster-issuer: letsencrypt-prod ingress.kubernetes.io/rewrite-target: / spec: tls: From 244feb380f1d167e46836eb8c4fe8d69be928af0 Mon Sep 17 00:00:00 2001 From: Nafis Bey Date: Wed, 19 Aug 2020 16:01:32 +0000 Subject: [PATCH 11/29] chore: re-enable docker build --- .github/workflows/k8s-deploy-pr.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/k8s-deploy-pr.yml b/.github/workflows/k8s-deploy-pr.yml index 6e0b53a..c2107ee 100644 --- a/.github/workflows/k8s-deploy-pr.yml +++ b/.github/workflows/k8s-deploy-pr.yml @@ -67,19 +67,19 @@ jobs: echo ::set-env name=COMMIT_MSG::$(git log --format=%B -n 1 ${{ github.event.after }}) echo ::set-env name=REPO_NAME::$(echo ${GITHUB_REPOSITORY,,}) - # - name: Build & Publish Docker image - # uses: whoan/docker-build-with-cache-action@v5 - # with: - # dockerfile: Dockerfile - # username: ${{ github.actor }} - # password: ${{ env.GITHUB_TOKEN }} - # registry: ${{ env.PACKAGE_REGISTRY }} - # image_name: ${{ env.REPO_NAME }}/${{ env.PACKAGE_NAME }} - # image_tag: ${{ env.PR_NAME }} - # build_extra_args: | - # --build-arg=SOURCE_COMMIT=${{ github.sha }} - # --build-arg=SOURCE_TAG=${{ env.PR_NAME }} - # --build-arg=HAB_LICENSE=${{ env.HAB_LICENSE }} + - name: Build & Publish Docker image + uses: whoan/docker-build-with-cache-action@v5 + with: + dockerfile: Dockerfile + username: ${{ github.actor }} + password: ${{ env.GITHUB_TOKEN }} + registry: ${{ env.PACKAGE_REGISTRY }} + image_name: ${{ env.REPO_NAME }}/${{ env.PACKAGE_NAME }} + image_tag: ${{ env.PR_NAME }} + build_extra_args: | + --build-arg=SOURCE_COMMIT=${{ github.sha }} + --build-arg=SOURCE_TAG=${{ env.PR_NAME }} + --build-arg=HAB_LICENSE=${{ env.HAB_LICENSE }} - name: Add kubeconfig to environment run: | From 9b4ade5a63074b2f64a810f5d9b75b7b6dc0c333 Mon Sep 17 00:00:00 2001 From: Nafis Bey Date: Thu, 20 Aug 2020 14:16:18 +0000 Subject: [PATCH 12/29] chore: minor tweaks - use helm template for cert manager annotations - update helm chart description - simplify `pkg_ident` helm config --- k8s/charts/deployment/Chart.yaml | 2 +- k8s/charts/deployment/templates/_helpers.tpl | 7 +++++++ k8s/charts/deployment/templates/deployment.yaml | 9 ++------- k8s/charts/deployment/values.yaml | 4 ++++ 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/k8s/charts/deployment/Chart.yaml b/k8s/charts/deployment/Chart.yaml index 940a805..97358e2 100644 --- a/k8s/charts/deployment/Chart.yaml +++ b/k8s/charts/deployment/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: gatekeeper-deployment -description: A Helm chart for Gatekeeper Kubernetes PR Deployments +description: A Helm chart for Gatekeeper Deployments # A chart can be either an 'application' or a 'library' chart. # diff --git a/k8s/charts/deployment/templates/_helpers.tpl b/k8s/charts/deployment/templates/_helpers.tpl index fb7bea9..00070af 100644 --- a/k8s/charts/deployment/templates/_helpers.tpl +++ b/k8s/charts/deployment/templates/_helpers.tpl @@ -32,3 +32,10 @@ Selector labels app.kubernetes.io/name: {{ include "gatekeeper-deployment.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} + +{{/* +Cert Manager Annotations +*/}} +{{- define "gatekeeper-deployment.cert-manager-annotations" -}} +cert-manager.io/cluster-issuer: {{ .Values.cert_manager.annotations.cluster_issuer }} +{{- end }} diff --git a/k8s/charts/deployment/templates/deployment.yaml b/k8s/charts/deployment/templates/deployment.yaml index d8899fd..a4a440d 100644 --- a/k8s/charts/deployment/templates/deployment.yaml +++ b/k8s/charts/deployment/templates/deployment.yaml @@ -21,7 +21,6 @@ spec: - name: regcred restartPolicy: Always - containers: - image: {{ .Values.image }} name: {{ .Release.Name }}-app @@ -30,12 +29,8 @@ spec: env: - name: HAB_GATEKEEPER_COMPOSITE value: | - {{- if .Values.hab.gatekeeper_composite.mysql }} [services.mysql] - {{- range $key, $value := .Values.hab.gatekeeper_composite.mysql }} - {{ $key }} = {{ $value | quote }} - {{- end }} - {{- end }} + pkg_ident: {{ .Values.hab.gatekeeper_composite.mysql.pkg_ident | quote }} - name: HAB_MYSQL value: | @@ -79,8 +74,8 @@ metadata: {{- include "gatekeeper-deployment.labels" . | nindent 4 }} annotations: kubernetes.io/ingress.class: "nginx" - cert-manager.io/cluster-issuer: letsencrypt-prod ingress.kubernetes.io/rewrite-target: / + {{- include "gatekeeper-deployment.cert-manager-annotations" . | nindent 4 }} spec: tls: - hosts: diff --git a/k8s/charts/deployment/values.yaml b/k8s/charts/deployment/values.yaml index 64472db..3c84a51 100644 --- a/k8s/charts/deployment/values.yaml +++ b/k8s/charts/deployment/values.yaml @@ -11,3 +11,7 @@ hab: app_username: admin app_password: xTw9wYFe70 bind: '0.0.0.0' + +cert_manager: + annotations: + cluster_issuer: letsencrypt-prod From 39790456d27a8a69db575d1ffb9370eeb9f79c22 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Tue, 21 Jul 2020 03:53:26 +0000 Subject: [PATCH 13/29] feat: add script/studio command --- script/-studio-bootstrap | 91 ++++++++++++++++++++++++++++++++++++++++ script/studio | 69 ++++++++++++++++++++++++++++++ 2 files changed, 160 insertions(+) create mode 100755 script/-studio-bootstrap create mode 100755 script/studio diff --git a/script/-studio-bootstrap b/script/-studio-bootstrap new file mode 100755 index 0000000..7f3c8ac --- /dev/null +++ b/script/-studio-bootstrap @@ -0,0 +1,91 @@ +#!/bin/bash + +# script/-studio-bootstrap: Check dependencies for Chef Habitat studio. + +set -e +cd "$(dirname "$0")/.." + + +echo +echo "==> studio-bootstrap: verifying Docker…" + +if ! [ -x "$(command -v docker)" ]; then + echo "Please install Docker Engine: https://docs.docker.com/engine/install/" + exit 1 +fi + +if ! docker info > /dev/null 2>&1; then + echo "Docker Engine is not running, or your user does not have access to connect." + echo "Try starting Docker Engine, and adding your user to the docker group: sudo gpasswd -a $USER docker" + exit 1 +fi + + +echo +echo "==> studio-bootstrap: verifying Chef Habitat…" + +if ! [ -x "$(command -v hab)" ]; then + echo "Please install Chef Habitat: https://www.habitat.sh/docs/install-habitat/" + exit 1 +fi + +set +e +hab_version="$(hab --version < /dev/null)" +if [ $? -ne 0 ]; then + echo + echo " Failed to read hab version, you may need to accept the Chef Habitat" + echo " license. Please run \`hab setup\` or configure HAB_LICENSE in the environment" + exit 1 +fi +set -e + +if ! [[ $hab_version =~ ^hab[[:space:]][0-9]+\.[0-9]+\.[0-9]+/[0-9]+$ ]]; then + echo + echo " Could not parse hab version: ${hab_version}" + echo " Please install hab 1.6+" + exit 1 +fi + +hab_version="$(echo "${hab_version}" | awk '{print $2}' | awk -F'/' '{print $1}')" +echo " Found hab version: ${hab_version}" + + +# check that node >= MAJOR.MINOR +hab_min_major="1" +hab_min_minor="6" + +IFS='.' read -ra hab_version_split <<< "${hab_version#v}" +if [ "${hab_version_split[0]}" -lt "${hab_min_major}" ] || [[ "${hab_version_split[0]}" -le "${hab_min_major}" && "${hab_version_split[1]}" -lt "${hab_min_minor}" ]]; then + echo + echo " Please install hab >= ${hab_min_major}.${hab_min_minor}.x" + exit 1 +fi + +if ! [ -f ~/.hab/etc/cli.toml ] || ! grep -q '^origin =' ~/.hab/etc/cli.toml; then + echo "Please re-run \`hab setup\` and choose to set a default origin, it can be anything" + exit 1 +fi + +_origin=$(awk -F'"' '/^origin = /{print $2}' ~/.hab/etc/cli.toml) + + +echo +echo "==> studio-bootstrap: verifying origin '${_origin}'…" + +_root_owned_key_count=$(ls -l ~/.hab/cache/keys | cut -f 3,4 -d " " | grep "root root" | wc -l) + +if [ "$_root_owned_key_count" -gt 0 ]; then + echo "Working around: https://github.com/habitat-sh/habitat/issues/7737" + echo "Found ${_root_owned_key_count} keys owned by root. Chowning them to $USER:$USER." + sudo chown $USER:$USER ~/.hab/cache/keys/* +fi + +if ! hab origin key export --type secret "${_origin}" > /dev/null; then + echo "No key has been generated for origin ${_origin}, run: hab origin key generate ${_origin}" + exit 1 +fi + + +echo +echo "==> studio-bootstrap: all set 👍" +echo diff --git a/script/studio b/script/studio new file mode 100755 index 0000000..a47f3f2 --- /dev/null +++ b/script/studio @@ -0,0 +1,69 @@ +#!/bin/sh + +# script/studio: Enter a Chef Habitat studio for the application. + +set -e +cd "$(dirname "$0")/.." + + +script/-studio-bootstrap + + +unset DEBUG + + +echo +echo "==> studio: configuring Chef Habitat studio Docker options…" +STUDIO_NAME="${STUDIO_NAME:-gatekeeper-studio}" +export HAB_DOCKER_OPTS=" + --name ${STUDIO_NAME} + -p 7080:80 + -p 7043:443 + -p 7036:3306 + -p 7099:19999 + -p 7800:8000 + -v ${STUDIO_NAME}-mysql-data:/hab/svc/mysql/data + -v $(cd ~/.ssh; pwd)/known_hosts:/root/.ssh/known_hosts:ro + --env STUDIO_DEVELOPER_UID=$(id -u) + --env STUDIO_DEVELOPER_GID=$(id -g) +" +echo "${HAB_DOCKER_OPTS}" + + +launch_studio=true +if [ "$(docker ps -aq -f name="${STUDIO_NAME}")" ]; then + echo + echo "==> studio: a ${STUDIO_NAME} container is already running…" + echo + while true; do + read -p "==> studio: would you like to (A)ttach to it, (s)top it, or do (n)othing? [A/s/n] " choice + case "${choice}" in + r|R|a|A|"") + echo + echo "==> studio: you can run studio-help at anytime to get a list of commands" + echo + docker attach "${STUDIO_NAME}" + launch_studio=false + break;; + s|S) + echo "==> studio: stopping existing container…" + docker stop "${STUDIO_NAME}" > /dev/null + break;; + n|N) + echo "==> studio: doing nothing with existing container… an error is likely to occur" + break ;; + *) + echo "==> studio: $choice is invalid";; + esac + done +fi + +if [ $launch_studio = true ]; then + echo + echo "==> studio: launching Docker-powered Chef Habitat studio…" + set +e + if ! hab studio enter -D; then + echo "===> studio: failed to launch studio… try executing the following and try again:" + echo "docker rm -f ${STUDIO_NAME}" + fi +fi From 42dd82ef3511fad238ad1dc03fc90cc7bd173c65 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Fri, 21 Aug 2020 03:39:04 +0000 Subject: [PATCH 14/29] fix(ci): use correct TOML syntax within Helm chart --- k8s/charts/deployment/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/charts/deployment/templates/deployment.yaml b/k8s/charts/deployment/templates/deployment.yaml index a4a440d..a997a70 100644 --- a/k8s/charts/deployment/templates/deployment.yaml +++ b/k8s/charts/deployment/templates/deployment.yaml @@ -30,7 +30,7 @@ spec: - name: HAB_GATEKEEPER_COMPOSITE value: | [services.mysql] - pkg_ident: {{ .Values.hab.gatekeeper_composite.mysql.pkg_ident | quote }} + pkg_ident = {{ .Values.hab.gatekeeper_composite.mysql.pkg_ident | quote }} - name: HAB_MYSQL value: | From 8e398b341b83aef9031f7ce74b61ee7af75d5686 Mon Sep 17 00:00:00 2001 From: Nafis Bey Date: Sat, 27 Jun 2020 23:50:01 +0000 Subject: [PATCH 15/29] fix: proxy header config - passthru ALL headers - forward `Authorization` header - send `X-Forwarded-For` header --- php-classes/Gatekeeper/ApiRequestHandler.php | 21 ++++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/php-classes/Gatekeeper/ApiRequestHandler.php b/php-classes/Gatekeeper/ApiRequestHandler.php index 8064297..6223562 100644 --- a/php-classes/Gatekeeper/ApiRequestHandler.php +++ b/php-classes/Gatekeeper/ApiRequestHandler.php @@ -15,16 +15,11 @@ class ApiRequestHandler extends \RequestHandler public static $defaultTimeout = 30; public static $defaultTimeoutConnect = 5; public static $passthruHeaders = [ - '/^HTTP\//' - ,'/^Content-Type:/i' - ,'/^Date:/i' - ,'/^Set-Cookie:/i' - ,'/^Location:/i' - ,'/^ETag:/i' - ,'/^Last-Modified:/i' - ,'/^Cache-Control:/i' - ,'/^Pragma:/i' - ,'/^Expires:/i' + '/.*/' + ]; + + public static $forwardHeaders = [ + 'Authorization' ]; public static $responseMode = 'json'; // override RequestHandler::$responseMode @@ -55,10 +50,14 @@ public static function handleRequest() { ,'autoQuery' => false ,'url' => rtrim($request->getEndpoint()->InternalEndpoint, '/') . $request->getUrl() ,'interface' => static::$sourceInterface + ,'headers' => [ + 'X-Forwarded-For' => $_SERVER['REMOTE_ADDR'] + ] ,'passthruHeaders' => static::$passthruHeaders + ,'forwardHeaders' => array_merge(HttpProxy::$defaultForwardHeaders, static::$forwardHeaders) ,'timeout' => static::$defaultTimeout ,'timeoutConnect' => static::$defaultTimeoutConnect -# ,'debug' => true // uncomment to debug proxy process and see output following response + // ,'debug' => true // uncomment to debug proxy process and see output following response // ,'afterResponseSync' => true // true to debug afterResponse code from browser ,'afterResponse' => function ($responseBody, $responseHeaders, $options, $curlHandle) use ($request, &$metrics, &$beforeEvent) { From ead202a8e1e50bdda478a29ac43d344d1ccbc330 Mon Sep 17 00:00:00 2001 From: Nafis Bey Date: Wed, 1 Jul 2020 14:07:45 +0000 Subject: [PATCH 16/29] fix: x-forwarded-for header --- php-classes/Gatekeeper/ApiRequestHandler.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/php-classes/Gatekeeper/ApiRequestHandler.php b/php-classes/Gatekeeper/ApiRequestHandler.php index 6223562..e3ea363 100644 --- a/php-classes/Gatekeeper/ApiRequestHandler.php +++ b/php-classes/Gatekeeper/ApiRequestHandler.php @@ -51,7 +51,7 @@ public static function handleRequest() { ,'url' => rtrim($request->getEndpoint()->InternalEndpoint, '/') . $request->getUrl() ,'interface' => static::$sourceInterface ,'headers' => [ - 'X-Forwarded-For' => $_SERVER['REMOTE_ADDR'] + "X-Forwarded-For: {$_SERVER['REMOTE_ADDR']}" ] ,'passthruHeaders' => static::$passthruHeaders ,'forwardHeaders' => array_merge(HttpProxy::$defaultForwardHeaders, static::$forwardHeaders) From d30f6c9b51078d208c0f78c93b494d80e948d045 Mon Sep 17 00:00:00 2001 From: Nafis Bey Date: Wed, 24 Jun 2020 00:50:09 +0000 Subject: [PATCH 17/29] feat: degradation mode - skip inserting transactions - add manual toggle in Site Admin tasks --- .../skip-insert-transactions.tpl | 26 ++++++++++++++ php-classes/Gatekeeper/ApiRequestHandler.php | 36 +++++++++++++------ .../configuration/skip-insert-transaction.php | 24 +++++++++++++ 3 files changed, 75 insertions(+), 11 deletions(-) create mode 100644 html-templates/site-admin/tasks/configuration/skip-insert-transactions.tpl create mode 100644 site-tasks/configuration/skip-insert-transaction.php diff --git a/html-templates/site-admin/tasks/configuration/skip-insert-transactions.tpl b/html-templates/site-admin/tasks/configuration/skip-insert-transactions.tpl new file mode 100644 index 0000000..1e8a001 --- /dev/null +++ b/html-templates/site-admin/tasks/configuration/skip-insert-transactions.tpl @@ -0,0 +1,26 @@ +{extends "task.tpl"} + +{block content} +

+ Skip Insert Transaction (Degradation Mode): {tif $enabled ? Active : Inactive} +

+ +
+
+ + +
+ +
+ +
+ + +
+{/block} \ No newline at end of file diff --git a/php-classes/Gatekeeper/ApiRequestHandler.php b/php-classes/Gatekeeper/ApiRequestHandler.php index 8064297..d16af31 100644 --- a/php-classes/Gatekeeper/ApiRequestHandler.php +++ b/php-classes/Gatekeeper/ApiRequestHandler.php @@ -29,6 +29,8 @@ class ApiRequestHandler extends \RequestHandler public static $responseMode = 'json'; // override RequestHandler::$responseMode + public static $degradationTimeout = 60; + public static function handleRequest() { // initialize request object @@ -68,17 +70,29 @@ public static function handleRequest() { // initialize log record if (!Cache::fetch('flags/gatekeeper/skip-insert-transaction')) { - $Transaction = Transaction::create([ - 'Endpoint' => $request->getEndpoint() - ,'Key' => $request->getKey() - ,'ClientIP' => ip2long($_SERVER['REMOTE_ADDR']) - ,'Method' => $_SERVER['REQUEST_METHOD'] - ,'Path' => $path - ,'Query' => $query - ,'ResponseTime' => $curlInfo['starttransfer_time'] * 1000 - ,'ResponseCode' => $curlInfo['http_code'] - ,'ResponseBytes' => $curlInfo['size_download'] - ]); + try { + $Transaction = Transaction::create([ + 'Endpoint' => $request->getEndpoint() + ,'Key' => $request->getKey() + ,'ClientIP' => ip2long($_SERVER['REMOTE_ADDR']) + ,'Method' => $_SERVER['REQUEST_METHOD'] + ,'Path' => $path + ,'Query' => $query + ,'ResponseTime' => $curlInfo['starttransfer_time'] * 1000 + ,'ResponseCode' => $curlInfo['http_code'] + ,'ResponseBytes' => $curlInfo['size_download'] + ]); + } catch (\Exception $e) { + Cache::store('flags/gatekeeper/skip-insert-transaction', true, static::$degradationTimeout); + \Emergence\Logger::general_warning( + 'Transaction Exception: {exceptionMessage}. Setting degredation flag for {seconds} seconds', + [ + 'exception' => $e, + 'exceptionMessage' => $e->getMessage(), + 'seconds' => static::$degradationTimeout + ] + ); + } } diff --git a/site-tasks/configuration/skip-insert-transaction.php b/site-tasks/configuration/skip-insert-transaction.php new file mode 100644 index 0000000..de6e64a --- /dev/null +++ b/site-tasks/configuration/skip-insert-transaction.php @@ -0,0 +1,24 @@ + 'Skip Transaction Inserts (Degradation Mode)', + 'description' => 'Enable or disable degredation mode which skips saving transactions to the DB. This functionality aids with the speed of requests with sites while under failure or high load', + 'icon' => 'power-off', + 'handler' => function () { + $flag = 'flags/gatekeeper/skip-insert-transaction'; + $ttl = $_REQUEST['ttl'] ?: Gatekeeper\ApiRequestHandler::$degradationTimeout; + + if ($_SERVER['REQUEST_METHOD'] == 'POST') { + if ($_REQUEST['status'] == 'enable') { + Cache::store($flag, true, $ttl); + } else { + Cache::delete($flag); + } + } + + return static::respond('skip-insert-transactions', [ + 'ttl' => $ttl, + 'enabled' => !!Cache::fetch($flag) + ]); + } +]; \ No newline at end of file From 16cb28f56ccf3e9201997d98483352c0a25d72a6 Mon Sep 17 00:00:00 2001 From: Nafis Bey Date: Thu, 18 Jun 2020 07:14:46 +0000 Subject: [PATCH 18/29] feat: advanced ip pattern syntax todo: - migrate column IP -> IPPattern - write tests? - optimize? --- dwoo-plugins/ip_cidr_range.php | 25 +++++ dwoo-plugins/ip_wildcard_regex.php | 17 ++++ html-templates/bans/banEdit.tpl | 2 +- html-templates/bans/banSaved.tpl | 1 + html-templates/bans/bans.tpl | 2 + html-templates/ip-patterns/ip-pattern.tpl | 37 +++++++ php-classes/Gatekeeper/Bans/Ban.php | 97 +++++++++++++++++-- .../Gatekeeper/Bans/BansRequestHandler.php | 2 +- php-classes/Gatekeeper/Utils/IP.php | 87 +++++++++++++++++ .../Ban/20200617000000_ippattern-column.php | 32 ++++++ 10 files changed, 293 insertions(+), 9 deletions(-) create mode 100644 dwoo-plugins/ip_cidr_range.php create mode 100644 dwoo-plugins/ip_wildcard_regex.php create mode 100644 html-templates/ip-patterns/ip-pattern.tpl create mode 100644 php-classes/Gatekeeper/Utils/IP.php create mode 100644 php-migrations/Gatekeeper/Ban/20200617000000_ippattern-column.php diff --git a/dwoo-plugins/ip_cidr_range.php b/dwoo-plugins/ip_cidr_range.php new file mode 100644 index 0000000..155cd60 --- /dev/null +++ b/dwoo-plugins/ip_cidr_range.php @@ -0,0 +1,25 @@ + cidr_range_min($subnet, $mask), + 'max' => cidr_range_max($subnet, $mask) + ]; +} + + diff --git a/dwoo-plugins/ip_wildcard_regex.php b/dwoo-plugins/ip_wildcard_regex.php new file mode 100644 index 0000000..f194ed5 --- /dev/null +++ b/dwoo-plugins/ip_wildcard_regex.php @@ -0,0 +1,17 @@ +
- {field inputName=IP label='IP Address' error=$errors.IP default=tif($Ban->IP, long2ip($Ban->IP))} + {field inputName=IPPattern label='IP Pattern' error=$errors.IPPattern default=$Ban->IPPattern hint="192.168.1.1,192.168.1.*,192.168.1.1/24"}
—or—
{field inputName=KeyID label='API Key' error=$errors.KeyID default=$Ban->Key->Key}
diff --git a/html-templates/bans/banSaved.tpl b/html-templates/bans/banSaved.tpl index 8c6f4a9..f4a22e1 100644 --- a/html-templates/bans/banSaved.tpl +++ b/html-templates/bans/banSaved.tpl @@ -8,6 +8,7 @@

Ban on {if $Ban->IP}IP Address: {$Ban->IP|long2ip} + {elseif $Ban->IPPattern}IP Address Pattern: {$Ban->IPPattern} {else}Key: {apiKey $Ban->Key} {/if} {tif $Ban->isNew ? created : saved}. diff --git a/html-templates/bans/bans.tpl b/html-templates/bans/bans.tpl index 387e7d2..f15a202 100644 --- a/html-templates/bans/bans.tpl +++ b/html-templates/bans/bans.tpl @@ -31,6 +31,8 @@

{if $Ban->IP} IP Address: {$Ban->IP|long2ip} + {elseif $Ban->IPPattern} + IP Pattern: {$Ban->IPPattern} {else} Key: {apiKey $Ban->Key} {/if} diff --git a/html-templates/ip-patterns/ip-pattern.tpl b/html-templates/ip-patterns/ip-pattern.tpl new file mode 100644 index 0000000..d9f740a --- /dev/null +++ b/html-templates/ip-patterns/ip-pattern.tpl @@ -0,0 +1,37 @@ +{template exactMatch ipPattern} + # IP Address = {$ipPattern} + if ($ipLong === {ip2long($ipPattern)}){literal} { + return true; + }{/literal} +{/template} +{template wildcardMatch ipPattern} + # Wildcard IP Address = {$ipPattern} + if (preg_match("{$ipPattern|ip_wildcard_regex}", $ipInput)){literal} { + return true; + }{/literal} +{/template} +{template cidrMatch ipPattern} + {$ranges = $ipPattern|ip_cidr_range} + # CIDR IP Range = {$ipPattern} + # Min: {$ranges.min|long2ip} Max: {$ranges.max|long2ip} + if ($ipLong >= {$ranges.min} && $ipLong <= {$ranges.max}){literal} { + return true; + }{/literal} +{/template} + +{""} \ No newline at end of file diff --git a/php-classes/Gatekeeper/Bans/Ban.php b/php-classes/Gatekeeper/Bans/Ban.php index 0bf1947..c6611d3 100644 --- a/php-classes/Gatekeeper/Bans/Ban.php +++ b/php-classes/Gatekeeper/Bans/Ban.php @@ -3,6 +3,8 @@ namespace Gatekeeper\Bans; use Cache; +use Emergence\Dwoo\Engine as DwooEngine; +use Emergence\Site\Storage; use Gatekeeper\Keys\Key; class Ban extends \ActiveRecord @@ -25,6 +27,9 @@ class Ban extends \ActiveRecord 'type' => 'uint', 'notnull' => false ], + 'IPPattern' => [ + 'notnull' => false + ], 'ExpirationDate' => [ 'type' => 'timestamp', 'notnull' => false @@ -63,7 +68,7 @@ public function validate($deep = true) { parent::validate($deep); - if (!$this->KeyID == !$this->IP) { + if (!$this->KeyID == !($this->IP || $this->IPPattern)) { // todo: replace when column is migrated $this->_validator->addError('Ban', 'Ban must specifiy either a API key or an IP address'); } @@ -96,7 +101,7 @@ public static function sortCreated($dir, $name) return "ID $dir"; } - protected static $_activeBans; + protected static $_activeBans; public static function getActiveBansTable() { if (isset(static::$_activeBans)) { @@ -108,13 +113,13 @@ public static function getActiveBansTable() } static::$_activeBans = [ - 'ips' => [] - ,'keys' => [] + 'patterns' => [], + 'keys' => [] ]; foreach (Ban::getAllByWhere('ExpirationDate IS NULL OR ExpirationDate > CURRENT_TIMESTAMP') AS $Ban) { - if ($Ban->IP) { - static::$_activeBans['ips'][] = long2ip($Ban->IP); + if ($Ban->IPPattern) { + static::$_activeBans['patterns'][] = $Ban->IPPattern; } elseif($Ban->KeyID) { static::$_activeBans['keys'][] = $Ban->KeyID; } @@ -125,9 +130,87 @@ public static function getActiveBansTable() return static::$_activeBans; } + public static function getIPPatternBanClosure($ipPattern) + { + static $ipPatternCaches = []; + + $ipPatternSafe = static::getIPPatternSafe($ipPattern); + + if (in_array($ipPatternSafe, $ipPatternCaches)) { + return $ipPatternCaches[$ipPatternSafe]; + } + + if ($ipPatternCache = Cache::fetch("ip-pattern:{$ipPatternSafe}")) { + return $ipPatternCache; + } + + // todo: use or remove? + //$localStorageRoot = Storage::getLocalStorageRoot(); + $fileName = "/patterns/matchers/{$ipPatternSafe}.php"; + + try { + return Storage::getFileSystem('site-root')->read(ltrim($fileName, '/')); + } catch (\Exception $e) { + $ipPatternSplit = []; + foreach (preg_split("/[\s,]+/", $ipPattern) as $pattern) { + $ipPatternSplit[$pattern] = static::getPatternType($pattern); + } + $order = ['ip', 'cidr', 'wildcard']; + uasort($ipPatternSplit, function($a, $b) use ($order) { + if ($a === $b) { + return 0; + } else { + return (array_search($a, $order) - array_search($b, $order)); + } + }); + + $phpScript = DwooEngine::getSource('ip-patterns/ip-pattern', [ + 'data' => $ipPatternSplit + ]); + + Storage::getFileSystem('site-root')->write(ltrim($fileName, '/'), $phpScript); + + return $phpScript; + } + + return null; + + } + + protected static function getIPPatternSafe($ipPattern) + { + return str_replace(['/', '*', ',', ' '], ['-', 'x', '_', '_'], $ipPattern); + } + + protected static function getPatternType($ipPattern) + { + if (preg_match('/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2}/', $ipPattern)) { + return 'cidr'; + } elseif (preg_match('/(\d{1,3})\.(\d{1,3})\.([0-9]{1,3}|\*)\.(\*)/', $ipPattern)) { + return 'wildcard'; + } elseif (preg_match('/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/', $ipPattern)) { + return 'ip'; + } + } + public static function isIPAddressBanned($ip) { - return in_array($ip, static::getActiveBansTable()['ips']); + $bannedPatterns = static::getActiveBansTable()['patterns']; + + if (in_array($ip, $bannedPatterns)) { + return true; + } + + $isBanned = false; + foreach($bannedPatterns as $ipPattern) { + $closure = eval( '?>' . static::getIPPatternBanClosure($ipPattern)); + if (call_user_func($closure, $ip)) { + $isBanned = true; + break; + } + } + + return $isBanned; } public static function isKeyBanned(Key $Key) diff --git a/php-classes/Gatekeeper/Bans/BansRequestHandler.php b/php-classes/Gatekeeper/Bans/BansRequestHandler.php index 6ae60b1..39aca33 100644 --- a/php-classes/Gatekeeper/Bans/BansRequestHandler.php +++ b/php-classes/Gatekeeper/Bans/BansRequestHandler.php @@ -17,7 +17,7 @@ class BansRequestHandler extends \RecordsRequestHandler protected static function applyRecordDelta(\ActiveRecord $Ban, $data) { if (isset($data['IP']) && !is_numeric($data['IP'])) { - $data['IP'] = ip2long($data['IP']); + $data['IPPattern'] = $data['IP']; } if (isset($data['KeyID']) && !is_numeric($data['KeyID'])) { diff --git a/php-classes/Gatekeeper/Utils/IP.php b/php-classes/Gatekeeper/Utils/IP.php new file mode 100644 index 0000000..185b2de --- /dev/null +++ b/php-classes/Gatekeeper/Utils/IP.php @@ -0,0 +1,87 @@ + $end) { + return false; + } + + if(is_null($start)) { + $start = 0; + } + + if(is_null($end)) { + $end = count($ranges) - 1; + } + + $ipLong = ip2long($ip); + $mid = (int)floor(($end + $start) / 2); + + switch(static::inRange($ipLong, $ranges[$mid])) { + case 0: + return $ranges[$mid][2]; + case -1: + return static::findRangeByIP($ip, $ranges, $start, $mid-1); + case 1: + return static::findRangeByIP($ip, $ranges, $mid+1, $end); + } + } + + + protected static function inRange($ipLong, $range) + { + list($start, $end) = $range; + + if ($ipLong < $start) { + return -1; + } elseif ($ipLong > $end) { + return 1; + } else { + return 0; + } + } + + protected static function prepareRange($range) + { + list ($subnet, $bits) = explode('/', $range); + $subnet = ip2long($subnet); + $mask = -1 << (32 - $bits); + $min = $subnet & $mask; + $max = $subnet | ~$mask; + + return [ + $min, + $max, + $range + ]; + } + + protected static function sortRanges(array &$ranges = []) + { + // sort by start, then by end. aka from narrowest overlapping range to widest + usort($ranges, function($a, $b) { + return $a[0] - $b[0] === 0 ? + $a[1] - $b[1] : + $a[0] - $b[0] ; + }); + } +} \ No newline at end of file diff --git a/php-migrations/Gatekeeper/Ban/20200617000000_ippattern-column.php b/php-migrations/Gatekeeper/Ban/20200617000000_ippattern-column.php new file mode 100644 index 0000000..9768dbe --- /dev/null +++ b/php-migrations/Gatekeeper/Ban/20200617000000_ippattern-column.php @@ -0,0 +1,32 @@ + Date: Tue, 23 Jun 2020 02:57:48 +0000 Subject: [PATCH 19/29] wip: cache closures --- html-templates/ip-patterns/ip-pattern.tpl | 23 +++--- php-classes/Gatekeeper/Bans/Ban.php | 89 +++++++++++++++-------- 2 files changed, 72 insertions(+), 40 deletions(-) diff --git a/html-templates/ip-patterns/ip-pattern.tpl b/html-templates/ip-patterns/ip-pattern.tpl index d9f740a..46d8b9c 100644 --- a/html-templates/ip-patterns/ip-pattern.tpl +++ b/html-templates/ip-patterns/ip-pattern.tpl @@ -19,19 +19,22 @@ }{/literal} {/template} -{""} \ No newline at end of file +{"?>"} +{/block} \ No newline at end of file diff --git a/php-classes/Gatekeeper/Bans/Ban.php b/php-classes/Gatekeeper/Bans/Ban.php index c6611d3..680ef4b 100644 --- a/php-classes/Gatekeeper/Bans/Ban.php +++ b/php-classes/Gatekeeper/Bans/Ban.php @@ -81,6 +81,7 @@ public function save($deep = true) if ($this->isUpdated || $this->isNew) { Cache::delete('bans'); + Cache::delete("ip-pattern:${static::getIPPatternSafe($this->IPPattern)}"); } } @@ -88,6 +89,7 @@ public function destroy() { $success = parent::destroy(); Cache::delete('bans'); + Cache::delete("ip-pattern:${static::getIPPatternSafe($this->IPPattern)}"); return $success; } @@ -101,6 +103,28 @@ public static function sortCreated($dir, $name) return "ID $dir"; } + + public static function parseIPPatterns($ipPattern, $returnType = null) + { + $bansByType = [ + 'ip' => [], + 'cidr' => [], + 'wildcard' => [] + ]; + + foreach (preg_split("/[\s,]+/", $ipPattern) as $pattern) { + if (!empty($pattern)) { + $bansByType[static::getPatternType($pattern)][] = $pattern; + } + } + + if (!empty($returnType)) { + return $bansByType[$returnType]; + } + + return $bansByType; + } + protected static $_activeBans; public static function getActiveBansTable() { @@ -114,12 +138,14 @@ public static function getActiveBansTable() static::$_activeBans = [ 'patterns' => [], + 'ips' => [], 'keys' => [] ]; foreach (Ban::getAllByWhere('ExpirationDate IS NULL OR ExpirationDate > CURRENT_TIMESTAMP') AS $Ban) { if ($Ban->IPPattern) { static::$_activeBans['patterns'][] = $Ban->IPPattern; + static::$_activeBans['ips'] = array_merge(static::$_activeBans['ips'], static::parseIPPatterns($Ban, 'ip')); } elseif($Ban->KeyID) { static::$_activeBans['keys'][] = $Ban->KeyID; } @@ -130,51 +156,49 @@ public static function getActiveBansTable() return static::$_activeBans; } - public static function getIPPatternBanClosure($ipPattern) + public static function getIPPatternBanClosure($ipPattern, $ignoreCache = false) { static $ipPatternCaches = []; + // todo: confirm if this is needed $ipPatternSafe = static::getIPPatternSafe($ipPattern); - if (in_array($ipPatternSafe, $ipPatternCaches)) { + if (!$ignoreCache && !empty($ipPatternCaches[$ipPatternSafe])) { return $ipPatternCaches[$ipPatternSafe]; } - if ($ipPatternCache = Cache::fetch("ip-pattern:{$ipPatternSafe}")) { + if (!$ignoreCache && $ipPatternCache = Cache::fetch("ip-pattern:{$ipPatternSafe}")) { return $ipPatternCache; } - // todo: use or remove? - //$localStorageRoot = Storage::getLocalStorageRoot(); - $fileName = "/patterns/matchers/{$ipPatternSafe}.php"; + $bucketId = 'ip-patterns'; + $filesystem = Storage::getFileSystem($bucketId); + $fileName = "matchers/{$ipPatternSafe}.php"; + $matcher = null; - try { - return Storage::getFileSystem('site-root')->read(ltrim($fileName, '/')); - } catch (\Exception $e) { - $ipPatternSplit = []; - foreach (preg_split("/[\s,]+/", $ipPattern) as $pattern) { - $ipPatternSplit[$pattern] = static::getPatternType($pattern); - } - $order = ['ip', 'cidr', 'wildcard']; - uasort($ipPatternSplit, function($a, $b) use ($order) { - if ($a === $b) { - return 0; - } else { - return (array_search($a, $order) - array_search($b, $order)); - } - }); - - $phpScript = DwooEngine::getSource('ip-patterns/ip-pattern', [ - 'data' => $ipPatternSplit - ]); + // try { + // $matcher = $filesystem->read($fileName); + + // } catch (\Exception $e) { + // } - Storage::getFileSystem('site-root')->write(ltrim($fileName, '/'), $phpScript); + if (!$filesystem->has($fileName)) { + $matcher = DwooEngine::getSource('ip-patterns/ip-pattern', [ + 'data' => static::parseIPPatterns($ipPattern) + ]); - return $phpScript; + $filesystem->write($fileName, $matcher); } - return null; + $closure = require join('/', [Storage::getLocalStorageRoot(), $bucketId, $fileName]); + + // cache matcher + $ipPatternCaches[$ipPatternSafe] = $closure;// $matcher; + // todo: confirm if we want to cache for a specific time period + Cache::store("ip-pattern:{$ipPatternSafe}", $closure); // $matcher + return $closure; + // return $matcher; } protected static function getIPPatternSafe($ipPattern) @@ -193,17 +217,22 @@ protected static function getPatternType($ipPattern) } } - public static function isIPAddressBanned($ip) + public static function isIPAddressBanned($ip, $ignoreCache = false) { $bannedPatterns = static::getActiveBansTable()['patterns']; + // check for explicit IP ban if (in_array($ip, $bannedPatterns)) { return true; } + // check IP Patterns individually $isBanned = false; foreach($bannedPatterns as $ipPattern) { - $closure = eval( '?>' . static::getIPPatternBanClosure($ipPattern)); + $matcher = static::getIPPatternBanClosure($ipPattern, $ignoreCache); + \MICS::dump(trim($matcher), 'matcher', !empty($_REQUEST['debug'])); + $closure = eval($matcher . " ?>"); + \MICS::dump($closure, 'matcher', isset($_REQUEST['debug'])); if (call_user_func($closure, $ip)) { $isBanned = true; break; From 226bcf00a54782a6cd8ea2fabf2541794e1b85c6 Mon Sep 17 00:00:00 2001 From: Nafis Bey Date: Wed, 24 Jun 2020 17:00:42 +0000 Subject: [PATCH 20/29] refactor: use include statement --- php-classes/Gatekeeper/Bans/Ban.php | 43 +++++++++-------------------- 1 file changed, 13 insertions(+), 30 deletions(-) diff --git a/php-classes/Gatekeeper/Bans/Ban.php b/php-classes/Gatekeeper/Bans/Ban.php index 680ef4b..afe9914 100644 --- a/php-classes/Gatekeeper/Bans/Ban.php +++ b/php-classes/Gatekeeper/Bans/Ban.php @@ -167,38 +167,24 @@ public static function getIPPatternBanClosure($ipPattern, $ignoreCache = false) return $ipPatternCaches[$ipPatternSafe]; } - if (!$ignoreCache && $ipPatternCache = Cache::fetch("ip-pattern:{$ipPatternSafe}")) { - return $ipPatternCache; - } - $bucketId = 'ip-patterns'; $filesystem = Storage::getFileSystem($bucketId); $fileName = "matchers/{$ipPatternSafe}.php"; - $matcher = null; - - // try { - // $matcher = $filesystem->read($fileName); - - // } catch (\Exception $e) { - // } - if (!$filesystem->has($fileName)) { - $matcher = DwooEngine::getSource('ip-patterns/ip-pattern', [ - 'data' => static::parseIPPatterns($ipPattern) - ]); - - $filesystem->write($fileName, $matcher); + try { + $closure = include join('/', [Storage::getLocalStorageRoot(), $bucketId, $fileName]); + } catch (\Exception $e) { + $filesystem->write( + $fileName, + DwooEngine::getSource('ip-patterns/ip-pattern', [ + 'data' => static::parseIPPatterns($ipPattern) + ]) + ); + + return static::getIPPatternBanClosure($ipPattern); } - $closure = require join('/', [Storage::getLocalStorageRoot(), $bucketId, $fileName]); - - // cache matcher - $ipPatternCaches[$ipPatternSafe] = $closure;// $matcher; - // todo: confirm if we want to cache for a specific time period - Cache::store("ip-pattern:{$ipPatternSafe}", $closure); // $matcher - - return $closure; - // return $matcher; + return $ipPatternCaches[$ipPatternSafe] = $closure; } protected static function getIPPatternSafe($ipPattern) @@ -230,10 +216,7 @@ public static function isIPAddressBanned($ip, $ignoreCache = false) $isBanned = false; foreach($bannedPatterns as $ipPattern) { $matcher = static::getIPPatternBanClosure($ipPattern, $ignoreCache); - \MICS::dump(trim($matcher), 'matcher', !empty($_REQUEST['debug'])); - $closure = eval($matcher . " ?>"); - \MICS::dump($closure, 'matcher', isset($_REQUEST['debug'])); - if (call_user_func($closure, $ip)) { + if (call_user_func($matcher, $ip)) { $isBanned = true; break; } From c0cc87c81366f0cf01445b6e2bf8c457a82c76f4 Mon Sep 17 00:00:00 2001 From: Nafis Bey Date: Fri, 26 Jun 2020 22:13:49 +0000 Subject: [PATCH 21/29] refactor: advanced ip syntax --- dwoo-plugins/ip_cidr_range.php | 25 --- dwoo-plugins/ip_wildcard_regex.php | 17 -- html-templates/ip-patterns/ip-pattern.tpl | 40 ---- php-classes/Gatekeeper/Bans/Ban.php | 102 +++------ php-classes/Gatekeeper/Utils/IP.php | 86 ++------ php-classes/Gatekeeper/Utils/IPPattern.php | 230 +++++++++++++++++++++ 6 files changed, 270 insertions(+), 230 deletions(-) delete mode 100644 dwoo-plugins/ip_cidr_range.php delete mode 100644 dwoo-plugins/ip_wildcard_regex.php delete mode 100644 html-templates/ip-patterns/ip-pattern.tpl create mode 100644 php-classes/Gatekeeper/Utils/IPPattern.php diff --git a/dwoo-plugins/ip_cidr_range.php b/dwoo-plugins/ip_cidr_range.php deleted file mode 100644 index 155cd60..0000000 --- a/dwoo-plugins/ip_cidr_range.php +++ /dev/null @@ -1,25 +0,0 @@ - cidr_range_min($subnet, $mask), - 'max' => cidr_range_max($subnet, $mask) - ]; -} - - diff --git a/dwoo-plugins/ip_wildcard_regex.php b/dwoo-plugins/ip_wildcard_regex.php deleted file mode 100644 index f194ed5..0000000 --- a/dwoo-plugins/ip_wildcard_regex.php +++ /dev/null @@ -1,17 +0,0 @@ -= {$ranges.min} && $ipLong <= {$ranges.max}){literal} { - return true; - }{/literal} -{/template} - -{block closure-method} -{""} -{/block} \ No newline at end of file diff --git a/php-classes/Gatekeeper/Bans/Ban.php b/php-classes/Gatekeeper/Bans/Ban.php index afe9914..36d6fbe 100644 --- a/php-classes/Gatekeeper/Bans/Ban.php +++ b/php-classes/Gatekeeper/Bans/Ban.php @@ -3,9 +3,9 @@ namespace Gatekeeper\Bans; use Cache; -use Emergence\Dwoo\Engine as DwooEngine; use Emergence\Site\Storage; use Gatekeeper\Keys\Key; +use Gatekeeper\Utils\IPPattern; class Ban extends \ActiveRecord { @@ -23,10 +23,6 @@ class Ban extends \ActiveRecord 'type' => 'uint', 'notnull' => false ], - 'IP' => [ - 'type' => 'uint', - 'notnull' => false - ], 'IPPattern' => [ 'notnull' => false ], @@ -68,8 +64,8 @@ public function validate($deep = true) { parent::validate($deep); - if (!$this->KeyID == !($this->IP || $this->IPPattern)) { // todo: replace when column is migrated - $this->_validator->addError('Ban', 'Ban must specifiy either a API key or an IP address'); + if (!$this->KeyID == !$this->IPPattern) { + $this->_validator->addError('Ban', 'Ban must specify either a API key or an IP address'); } return $this->finishValidation(); @@ -81,7 +77,6 @@ public function save($deep = true) if ($this->isUpdated || $this->isNew) { Cache::delete('bans'); - Cache::delete("ip-pattern:${static::getIPPatternSafe($this->IPPattern)}"); } } @@ -89,7 +84,6 @@ public function destroy() { $success = parent::destroy(); Cache::delete('bans'); - Cache::delete("ip-pattern:${static::getIPPatternSafe($this->IPPattern)}"); return $success; } @@ -104,27 +98,6 @@ public static function sortCreated($dir, $name) } - public static function parseIPPatterns($ipPattern, $returnType = null) - { - $bansByType = [ - 'ip' => [], - 'cidr' => [], - 'wildcard' => [] - ]; - - foreach (preg_split("/[\s,]+/", $ipPattern) as $pattern) { - if (!empty($pattern)) { - $bansByType[static::getPatternType($pattern)][] = $pattern; - } - } - - if (!empty($returnType)) { - return $bansByType[$returnType]; - } - - return $bansByType; - } - protected static $_activeBans; public static function getActiveBansTable() { @@ -143,9 +116,12 @@ public static function getActiveBansTable() ]; foreach (Ban::getAllByWhere('ExpirationDate IS NULL OR ExpirationDate > CURRENT_TIMESTAMP') AS $Ban) { - if ($Ban->IPPattern) { - static::$_activeBans['patterns'][] = $Ban->IPPattern; - static::$_activeBans['ips'] = array_merge(static::$_activeBans['ips'], static::parseIPPatterns($Ban, 'ip')); + if (!empty($Ban->IPPattern)) { + if (is_array(static::getIPPatternBanClosure($Ban->IPPattern))) { // ip pattern ONLY contains static IPs + static::$_activeBans['ips'] = array_merge(static::$_activeBans['ips'], static::getIPPatternBanClosure($Ban->IPPattern)); + } else { + static::$_activeBans['patterns'][] = $Ban->IPPattern; + } } elseif($Ban->KeyID) { static::$_activeBans['keys'][] = $Ban->KeyID; } @@ -160,69 +136,45 @@ public static function getIPPatternBanClosure($ipPattern, $ignoreCache = false) { static $ipPatternCaches = []; - // todo: confirm if this is needed - $ipPatternSafe = static::getIPPatternSafe($ipPattern); + $ipPatternHash = sha1($ipPattern); - if (!$ignoreCache && !empty($ipPatternCaches[$ipPatternSafe])) { - return $ipPatternCaches[$ipPatternSafe]; + if (!$ignoreCache && !empty($ipPatternCaches[$ipPatternHash])) { + return $ipPatternCaches[$ipPatternHash]; } - $bucketId = 'ip-patterns'; - $filesystem = Storage::getFileSystem($bucketId); - $fileName = "matchers/{$ipPatternSafe}.php"; - try { - $closure = include join('/', [Storage::getLocalStorageRoot(), $bucketId, $fileName]); - } catch (\Exception $e) { - $filesystem->write( - $fileName, - DwooEngine::getSource('ip-patterns/ip-pattern', [ - 'data' => static::parseIPPatterns($ipPattern) - ]) + $closure = include join('/', + [ + Storage::getLocalStorageRoot(), + IPPattern::$fsRootDir, + $ipPatternHash . '.php' + ] ); - - return static::getIPPatternBanClosure($ipPattern); + } catch (\Exception $e) { + $closure = IPPattern::parse($ipPattern, $ipPatternHash); } - return $ipPatternCaches[$ipPatternSafe] = $closure; - } - - protected static function getIPPatternSafe($ipPattern) - { - return str_replace(['/', '*', ',', ' '], ['-', 'x', '_', '_'], $ipPattern); - } - - protected static function getPatternType($ipPattern) - { - if (preg_match('/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2}/', $ipPattern)) { - return 'cidr'; - } elseif (preg_match('/(\d{1,3})\.(\d{1,3})\.([0-9]{1,3}|\*)\.(\*)/', $ipPattern)) { - return 'wildcard'; - } elseif (preg_match('/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/', $ipPattern)) { - return 'ip'; - } + return $ipPatternCaches[$ipPatternHash] = $closure; } public static function isIPAddressBanned($ip, $ignoreCache = false) { - $bannedPatterns = static::getActiveBansTable()['patterns']; + $activeBans = static::getActiveBansTable(); // check for explicit IP ban - if (in_array($ip, $bannedPatterns)) { + if (in_array($ip, $activeBans['ips'])) { return true; } // check IP Patterns individually - $isBanned = false; - foreach($bannedPatterns as $ipPattern) { + foreach($activeBans['patterns'] as $ipPattern) { $matcher = static::getIPPatternBanClosure($ipPattern, $ignoreCache); - if (call_user_func($matcher, $ip)) { - $isBanned = true; - break; + if (call_user_func($matcher, $ip) === true) { + return true; } } - return $isBanned; + return false; } public static function isKeyBanned(Key $Key) diff --git a/php-classes/Gatekeeper/Utils/IP.php b/php-classes/Gatekeeper/Utils/IP.php index 185b2de..107a786 100644 --- a/php-classes/Gatekeeper/Utils/IP.php +++ b/php-classes/Gatekeeper/Utils/IP.php @@ -4,84 +4,24 @@ class IP { - - public static function isInRange($ip, $ranges = []) - { - if (is_string($ranges)) { - $ranges = [$ranges]; - } - - $formattedRanges = array_map(['static', 'prepareRange'], $ranges); - - static::sortRanges($formattedRanges); - - $rangeByIP = static::findRangeByIP($ip, $formattedRanges); - - return $rangeByIP; - } - - protected static function findRangeByIP($ip, $ranges = [], $start = null, $end = null) { - if ($end < $start || $start > $end) { - return false; - } - - if(is_null($start)) { - $start = 0; - } - - if(is_null($end)) { - $end = count($ranges) - 1; - } - - $ipLong = ip2long($ip); - $mid = (int)floor(($end + $start) / 2); - - switch(static::inRange($ipLong, $ranges[$mid])) { - case 0: - return $ranges[$mid][2]; - case -1: - return static::findRangeByIP($ip, $ranges, $start, $mid-1); - case 1: - return static::findRangeByIP($ip, $ranges, $mid+1, $end); - } - } - - - protected static function inRange($ipLong, $range) - { - list($start, $end) = $range; - - if ($ipLong < $start) { - return -1; - } elseif ($ipLong > $end) { - return 1; - } else { - return 0; - } - } - - protected static function prepareRange($range) - { - list ($subnet, $bits) = explode('/', $range); - $subnet = ip2long($subnet); - $mask = -1 << (32 - $bits); - $min = $subnet & $mask; - $max = $subnet | ~$mask; - - return [ - $min, - $max, - $range - ]; - } - - protected static function sortRanges(array &$ranges = []) + // todo: remove if not needed + /** + * + * Sort IP ranges by start, then by end (from narrowest overlapping range to widest). + * + * @param array $ranges The array of IP ranges to sort + * @return string + * + */ + + public static function sortRanges(array &$ranges = []) { - // sort by start, then by end. aka from narrowest overlapping range to widest + // sort usort($ranges, function($a, $b) { return $a[0] - $b[0] === 0 ? $a[1] - $b[1] : $a[0] - $b[0] ; }); } + } \ No newline at end of file diff --git a/php-classes/Gatekeeper/Utils/IPPattern.php b/php-classes/Gatekeeper/Utils/IPPattern.php new file mode 100644 index 0000000..7d050bf --- /dev/null +++ b/php-classes/Gatekeeper/Utils/IPPattern.php @@ -0,0 +1,230 @@ + [], + 'cidr' => [], + 'wildcard' => [] + ]; + + $count = 0; + foreach (preg_split("/[\s,]+/", $pattern) as $subPattern) { + $subPatternType = static::getPatternType($subPattern); + if (!empty($subPattern) && array_key_exists($subPatternType, $subPatternsByType)) { + $subPatternsByType[$subPatternType][] = $subPattern; + $count++; + } + } + + + if ($count === 0) { + throw new InvalidArgumentException("Unable to parse IP pattern: $pattern"); + } + + if (count($subPatternsByType['cidr']) === 0 && count($subPatternsByType['wildcard']) === 0) { + return $subPatternsByType['ip']; + } + + return static::generateClosure($subPatternsByType, $uid); + } + + /** + * + * Generate Closure function and write to filesystem with the name of the pattern hashed + * + * @param array $patterns Multi-dimensional array of sub-patterns, keyed by pattern type. + * @param string $patternHash Original Pattern hashed. + * + * @return closure Closure function + */ + protected static function generateClosure(array $patterns, $patternHash) + { + + $closureFunctionString = <<write( + $fileName, + $closureFunctionString + ); + + return include join('/', [Storage::getLocalStorageRoot(), static::$fsRootDir, $fileName]); + } + + /** + * + * Generate Closure condition string for use in the generateClosure() function. + * + * @param string $pattern Sub-pattern to create condition for. + * @param string $patternType Type of ip pattern (ip, cidr, wilcdcard). + * + * @return string If Condition in string format + */ + protected static function generateClosureCondition($pattern, $patternType) + { + switch ($patternType) { + case 'ip': + $ipLong = ip2long($pattern); + $conditionString = <<= $min && \$ipLong <= $max) { + return true; + } + +DOC; + + break; + + case 'wildcard': + $patternRegex = static::getWildcardRegex($pattern); + return << Date: Fri, 26 Jun 2020 22:46:53 +0000 Subject: [PATCH 22/29] refactor: remove todos --- php-classes/Gatekeeper/Utils/IPPattern.php | 26 +++++++++++----------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/php-classes/Gatekeeper/Utils/IPPattern.php b/php-classes/Gatekeeper/Utils/IPPattern.php index 7d050bf..367fc70 100644 --- a/php-classes/Gatekeeper/Utils/IPPattern.php +++ b/php-classes/Gatekeeper/Utils/IPPattern.php @@ -11,8 +11,7 @@ class IPPattern { public static $fsRootDir = 'ip-patterns/matchers'; - // tODO: returns either a closure (containing pattern checks -- including static IPs) or array (of integers) - /** + /** * * Parse IP pattens by splitting them by spaces or commas and grouping them * in the available groups: ip, cidr, or wildcard. @@ -53,19 +52,20 @@ public static function parse($pattern, $uid) return static::generateClosure($subPatternsByType, $uid); } - /** - * - * Generate Closure function and write to filesystem with the name of the pattern hashed - * - * @param array $patterns Multi-dimensional array of sub-patterns, keyed by pattern type. - * @param string $patternHash Original Pattern hashed. - * - * @return closure Closure function - */ + /** + * + * Generate Closure function and write to filesystem with the name of the pattern hashed + * + * @param array $patterns Multi-dimensional array of sub-patterns, keyed by pattern type. + * @param string $patternHash Original Pattern hashed. + * + * @return closure Closure function + */ protected static function generateClosure(array $patterns, $patternHash) { - $closureFunctionString = <<write( $fileName, From 5ad163cbcecc8d28d0085a56c7193913bd3ee587 Mon Sep 17 00:00:00 2001 From: Nafis Bey Date: Fri, 26 Jun 2020 22:47:51 +0000 Subject: [PATCH 23/29] fix: migration script - migrate values from deprecated column --- .../Ban/20200617000000_ippattern-column.php | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/php-migrations/Gatekeeper/Ban/20200617000000_ippattern-column.php b/php-migrations/Gatekeeper/Ban/20200617000000_ippattern-column.php index 9768dbe..0fca117 100644 --- a/php-migrations/Gatekeeper/Ban/20200617000000_ippattern-column.php +++ b/php-migrations/Gatekeeper/Ban/20200617000000_ippattern-column.php @@ -29,4 +29,28 @@ ] ); +$deprecatedColumn = 'IP'; + +printf("Migrating values for deprecated column: `%s` -> `%s`", $deprecatedColumn, $columnName); +DB::nonQuery( + ' + UPDATE `%1$s` SET %2$s = %3$s + WHERE %3$s IS NOT NULL + ', + [ + Ban::$tableName, + $columnName, + $deprecatedColumn + ] +); + +printf("Removing deprecated column: IP"); +DB::nonQuery( + 'ALTER TABLE `%s` DROP COLUMN %s', + [ + Ban::$tableName, + $deprecatedColumn + ] +); + return static::STATUS_EXECUTED; \ No newline at end of file From 23e107ea2ecf6229941a778cae3f6fde6aa8ea04 Mon Sep 17 00:00:00 2001 From: Nafis Bey Date: Fri, 26 Jun 2020 23:36:43 +0000 Subject: [PATCH 24/29] feat: bulk bans --- html-templates/bans/bans.tpl | 1 + html-templates/bans/bulk/banCreate.tpl | 35 +++++++++++++++++++ html-templates/bans/bulk/bansSaved.tpl | 12 +++++++ php-classes/Gatekeeper/Bans/Ban.php | 2 +- .../Gatekeeper/Bans/BansRequestHandler.php | 35 +++++++++++++++++-- 5 files changed, 81 insertions(+), 4 deletions(-) create mode 100644 html-templates/bans/bulk/banCreate.tpl create mode 100644 html-templates/bans/bulk/bansSaved.tpl diff --git a/html-templates/bans/bans.tpl b/html-templates/bans/bans.tpl index f15a202..3f0faf8 100644 --- a/html-templates/bans/bans.tpl +++ b/html-templates/bans/bans.tpl @@ -8,6 +8,7 @@

Bans

Issue Ban + Bulk Bans
diff --git a/html-templates/bans/bulk/banCreate.tpl b/html-templates/bans/bulk/banCreate.tpl new file mode 100644 index 0000000..ef71154 --- /dev/null +++ b/html-templates/bans/bulk/banCreate.tpl @@ -0,0 +1,35 @@ +{extends "designs/site.tpl"} + +{block "title"}Create Bans in Bulk — {$dwoo.parent}{/block} + +{block "content"} + {$Ban = $data} + {$errors = $Ban->validationErrors} + +
+

New Bans

+
+ Cancel +
+
+ +
+ {if $errors} +
+ Please double-check the following lines highlighted below. + + {foreach from=$invalidPatterns item=invalidPattern} +

{$invalidPattern}

+ {/foreach} +
+ {/if} + +
+ {textarea inputName=IPPatterns label='IP Patterns' hint="Separate IP Patterns by line"} + +
+ +
+
+
+{/block} \ No newline at end of file diff --git a/html-templates/bans/bulk/bansSaved.tpl b/html-templates/bans/bulk/bansSaved.tpl new file mode 100644 index 0000000..ef5aa1c --- /dev/null +++ b/html-templates/bans/bulk/bansSaved.tpl @@ -0,0 +1,12 @@ +{extends "designs/site.tpl"} + +{block "title"}Ban saved — {$dwoo.parent}{/block} + +{block "content"} +

Bans created for the following patterns:

+ {foreach from=$data item=Ban} +

{$Ban->IPPattern}

+ {/foreach} + +

← Browse all bans

+{/block} \ No newline at end of file diff --git a/php-classes/Gatekeeper/Bans/Ban.php b/php-classes/Gatekeeper/Bans/Ban.php index 36d6fbe..1b542df 100644 --- a/php-classes/Gatekeeper/Bans/Ban.php +++ b/php-classes/Gatekeeper/Bans/Ban.php @@ -65,7 +65,7 @@ public function validate($deep = true) parent::validate($deep); if (!$this->KeyID == !$this->IPPattern) { - $this->_validator->addError('Ban', 'Ban must specify either a API key or an IP address'); + $this->_validator->addError('Ban', 'Ban must specify either a API key or an IP pattern'); } return $this->finishValidation(); diff --git a/php-classes/Gatekeeper/Bans/BansRequestHandler.php b/php-classes/Gatekeeper/Bans/BansRequestHandler.php index 39aca33..09fc92d 100644 --- a/php-classes/Gatekeeper/Bans/BansRequestHandler.php +++ b/php-classes/Gatekeeper/Bans/BansRequestHandler.php @@ -14,12 +14,41 @@ class BansRequestHandler extends \RecordsRequestHandler public static $accountLevelWrite = 'Staff'; public static $accountLevelAPI = 'Staff'; - protected static function applyRecordDelta(\ActiveRecord $Ban, $data) + public static function handleCreateRequest(\ActiveRecord $Record = null) + { + if (static::shiftPath() === 'bulk') { + return static::handleBulkCreationRequest(); + } + + return parent::handleCreateRequest($Record); + } + + protected static function handleBulkCreationRequest() { - if (isset($data['IP']) && !is_numeric($data['IP'])) { - $data['IPPattern'] = $data['IP']; + if ($_SERVER['REQUEST_METHOD'] === 'POST') { + // process request + $bans = []; + foreach (explode("\r\n", $_REQUEST['IPPatterns']) as $ipPattern) { + $trimmedPattern = trim($ipPattern); + if (empty($trimmedPattern)) { + continue; + } + + $bans[] = Ban::create([ + 'IPPattern' => $trimmedPattern + ], true); + } + + return static::respond('bulk/bansSaved', [ + 'data' => $bans + ]); } + return static::respond('bulk/banCreate'); + } + + protected static function applyRecordDelta(\ActiveRecord $Ban, $data) + { if (isset($data['KeyID']) && !is_numeric($data['KeyID'])) { $Key = Key::getByHandle($data['KeyID']); $data['KeyID'] = $Key ? $Key->ID : null; From ef7ac278c5555a94c0763dfd5d25ef79027a92b8 Mon Sep 17 00:00:00 2001 From: Nafis Bey Date: Sat, 27 Jun 2020 17:15:39 +0000 Subject: [PATCH 25/29] fix: references to deprecated 'IP' column --- html-templates/bans/ban.tpl | 4 ++-- html-templates/bans/banDeleted.tpl | 2 +- html-templates/bans/banSaved.tpl | 3 +-- html-templates/bans/bans.tpl | 4 +--- html-templates/subtemplates/contextLinks.tpl | 4 ++-- php-config/SearchRequestHandler.config.d/gatekeeper.php | 5 ++--- sencha-workspace/pages/src/widget/model/Ban.js | 2 +- 7 files changed, 10 insertions(+), 14 deletions(-) diff --git a/html-templates/bans/ban.tpl b/html-templates/bans/ban.tpl index 7c00354..0ff7a95 100644 --- a/html-templates/bans/ban.tpl +++ b/html-templates/bans/ban.tpl @@ -9,8 +9,8 @@

- {if $Ban->IP} - IP Address: {$Ban->IP|long2ip} + {if $Ban->IPPattern} + IP Pattern: {$Ban->IPPattern} {else} Key: {apiKey $Ban->Key} {/if} diff --git a/html-templates/bans/banDeleted.tpl b/html-templates/bans/banDeleted.tpl index f99c810..344bf04 100644 --- a/html-templates/bans/banDeleted.tpl +++ b/html-templates/bans/banDeleted.tpl @@ -5,7 +5,7 @@ {block "content"} {$Ban = $data} -

Ban on {if $Ban->IP}IP Address: {$Ban->IP|long2ip}{else}Key: {apiKey $Ban->Key}{/if} deleted.

+

Ban on {if $Ban->IPPattern}IP Pattern: {$Ban->IPPattern}{else}Key: {apiKey $Ban->Key}{/if} deleted.

Browse all bans

{/block} \ No newline at end of file diff --git a/html-templates/bans/banSaved.tpl b/html-templates/bans/banSaved.tpl index f4a22e1..26c7671 100644 --- a/html-templates/bans/banSaved.tpl +++ b/html-templates/bans/banSaved.tpl @@ -7,8 +7,7 @@

Ban on - {if $Ban->IP}IP Address: {$Ban->IP|long2ip} - {elseif $Ban->IPPattern}IP Address Pattern: {$Ban->IPPattern} + {if $Ban->IPPattern}IP Address Pattern: {$Ban->IPPattern} {else}Key: {apiKey $Ban->Key} {/if} {tif $Ban->isNew ? created : saved}. diff --git a/html-templates/bans/bans.tpl b/html-templates/bans/bans.tpl index 3f0faf8..b78548a 100644 --- a/html-templates/bans/bans.tpl +++ b/html-templates/bans/bans.tpl @@ -30,9 +30,7 @@

- {if $Ban->IP} - IP Address: {$Ban->IP|long2ip} - {elseif $Ban->IPPattern} + {if $Ban->IPPattern} IP Pattern: {$Ban->IPPattern} {else} Key: {apiKey $Ban->Key} diff --git a/html-templates/subtemplates/contextLinks.tpl b/html-templates/subtemplates/contextLinks.tpl index 749ffed..9ac3069 100644 --- a/html-templates/subtemplates/contextLinks.tpl +++ b/html-templates/subtemplates/contextLinks.tpl @@ -18,8 +18,8 @@ {$prefix}Ban #{$Context->ID} — - {if $Context->IP} - IP Address: {$Context->IP|long2ip} + {if $Context->IPPattern} + IP Pattern: {$Context->IPPattern} {else} Key: {$Context->Key->OwnerName|escape} {$Context->Key->Key} {/if}{$suffix} diff --git a/php-config/SearchRequestHandler.config.d/gatekeeper.php b/php-config/SearchRequestHandler.config.d/gatekeeper.php index 477e0b4..5300405 100644 --- a/php-config/SearchRequestHandler.config.d/gatekeeper.php +++ b/php-config/SearchRequestHandler.config.d/gatekeeper.php @@ -51,9 +51,8 @@ 'method' => 'like' ], [ - 'field' => 'IP', - 'method' => 'sql', - 'sql' => 'INET_NTOA(IP) LIKE "%%%s%%"' + 'field' => 'IPPattern', + 'method' => 'like' ] ] ]; diff --git a/sencha-workspace/pages/src/widget/model/Ban.js b/sencha-workspace/pages/src/widget/model/Ban.js index 7afb2d7..dde38ad 100644 --- a/sencha-workspace/pages/src/widget/model/Ban.js +++ b/sencha-workspace/pages/src/widget/model/Ban.js @@ -10,7 +10,7 @@ Ext.define('Site.widget.model.Ban', { tpl: [ '', - '#{ID} — IP: {[this.long2ip(values.IP)]}Key ', + '#{ID} — IP Pattern: {IPPattern}Key ', 'Expires {[this.formatTimestamp(values.ExpirationDate)]}', '', { From f4224ad3b1daef6282b584485ac16bd5f7c95a50 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sat, 12 Sep 2020 22:50:52 +0000 Subject: [PATCH 26/29] feat: add endpoint column to transactions table --- html-templates/transactions/transactions.tpl | 2 ++ site-root/sass/site/modules/_tables.scss | 10 +++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/html-templates/transactions/transactions.tpl b/html-templates/transactions/transactions.tpl index d567207..3428d62 100644 --- a/html-templates/transactions/transactions.tpl +++ b/html-templates/transactions/transactions.tpl @@ -75,6 +75,7 @@ + @@ -88,6 +89,7 @@ {foreach item=Request from=$data} + diff --git a/site-root/sass/site/modules/_tables.scss b/site-root/sass/site/modules/_tables.scss index 55610e8..522dfc8 100644 --- a/site-root/sass/site/modules/_tables.scss +++ b/site-root/sass/site/modules/_tables.scss @@ -16,7 +16,7 @@ vertical-align: -.2em; white-space: nowrap; width: 16px; - + // gimme crisp pixels image-rendering:optimizeSpeed; /* Legal fallback */ image-rendering:-moz-crisp-edges; /* Firefox */ @@ -24,12 +24,12 @@ image-rendering:-webkit-optimize-contrast; /* Chrome (and eventually Safari) */ image-rendering:crisp-edges; /* CSS3 Proposed */ -ms-interpolation-mode:bicubic; /* IE8+ */ - + &:hover, &:focus { border-color: rgba($text-color, .2); } - + &:active, &.selected { background-color: rgba($text-color, .1); @@ -58,6 +58,10 @@ text-align: right; } +.col-endpoint { + width: 200px +} + .col-key { width: 1%; } From 65ac9569a9af01f7a8c1eba82185a2a854e53127 Mon Sep 17 00:00:00 2001 From: Nafis Bey Date: Tue, 22 Sep 2020 10:00:29 -0400 Subject: [PATCH 27/29] chore: remove ignoreCache arguments --- php-classes/Gatekeeper/Bans/Ban.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/php-classes/Gatekeeper/Bans/Ban.php b/php-classes/Gatekeeper/Bans/Ban.php index 1b542df..f9b1cdf 100644 --- a/php-classes/Gatekeeper/Bans/Ban.php +++ b/php-classes/Gatekeeper/Bans/Ban.php @@ -132,13 +132,13 @@ public static function getActiveBansTable() return static::$_activeBans; } - public static function getIPPatternBanClosure($ipPattern, $ignoreCache = false) + public static function getIPPatternBanClosure($ipPattern) { static $ipPatternCaches = []; $ipPatternHash = sha1($ipPattern); - if (!$ignoreCache && !empty($ipPatternCaches[$ipPatternHash])) { + if (!empty($ipPatternCaches[$ipPatternHash])) { return $ipPatternCaches[$ipPatternHash]; } @@ -157,7 +157,7 @@ public static function getIPPatternBanClosure($ipPattern, $ignoreCache = false) return $ipPatternCaches[$ipPatternHash] = $closure; } - public static function isIPAddressBanned($ip, $ignoreCache = false) + public static function isIPAddressBanned($ip) { $activeBans = static::getActiveBansTable(); @@ -168,7 +168,7 @@ public static function isIPAddressBanned($ip, $ignoreCache = false) // check IP Patterns individually foreach($activeBans['patterns'] as $ipPattern) { - $matcher = static::getIPPatternBanClosure($ipPattern, $ignoreCache); + $matcher = static::getIPPatternBanClosure($ipPattern); if (call_user_func($matcher, $ip) === true) { return true; } From 2f756d7a2370dc394c0da36488eb5c8d69c1294b Mon Sep 17 00:00:00 2001 From: Nafis Bey Date: Tue, 22 Sep 2020 10:01:11 -0400 Subject: [PATCH 28/29] refactor: move logic for getting filename --- php-classes/Gatekeeper/Bans/Ban.php | 7 +------ php-classes/Gatekeeper/Utils/IPPattern.php | 20 +++++++++++++++++++- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/php-classes/Gatekeeper/Bans/Ban.php b/php-classes/Gatekeeper/Bans/Ban.php index f9b1cdf..d787637 100644 --- a/php-classes/Gatekeeper/Bans/Ban.php +++ b/php-classes/Gatekeeper/Bans/Ban.php @@ -143,12 +143,7 @@ public static function getIPPatternBanClosure($ipPattern) } try { - $closure = include join('/', - [ - Storage::getLocalStorageRoot(), - IPPattern::$fsRootDir, - $ipPatternHash . '.php' - ] + $closure = include IPPattern::getFilenameFromHash($ipPatternHash); ); } catch (\Exception $e) { $closure = IPPattern::parse($ipPattern, $ipPatternHash); diff --git a/php-classes/Gatekeeper/Utils/IPPattern.php b/php-classes/Gatekeeper/Utils/IPPattern.php index 367fc70..d0976c1 100644 --- a/php-classes/Gatekeeper/Utils/IPPattern.php +++ b/php-classes/Gatekeeper/Utils/IPPattern.php @@ -52,6 +52,24 @@ public static function parse($pattern, $uid) return static::generateClosure($subPatternsByType, $uid); } + /** + * + * Get IPPattern filename (containing closure method) from the hashed IP Pattern + * @param string $patternHash The hashed IP Pattern + * + * @return string Returns a string containing the full path of the ip pattern file + */ + public static function getFilenameFromHash($patternHash) + { + return join('/', + [ + Storage::getLocalStorageRoot(), + static::$fsRootDir, + $patternHash . '.php' + ] + ); + } + /** * * Generate Closure function and write to filesystem with the name of the pattern hashed @@ -227,4 +245,4 @@ protected static function getPatternType($pattern) throw new UnexpectedValueException("IP Pattern could not be validated: $pattern"); } -} \ No newline at end of file +} From 7a65a7592406ca222931f3bbf09785e6e573941b Mon Sep 17 00:00:00 2001 From: Nafis Bey Date: Tue, 22 Sep 2020 10:08:59 -0400 Subject: [PATCH 29/29] chore: remove deprecated IP util class --- php-classes/Gatekeeper/Utils/IP.php | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 php-classes/Gatekeeper/Utils/IP.php diff --git a/php-classes/Gatekeeper/Utils/IP.php b/php-classes/Gatekeeper/Utils/IP.php deleted file mode 100644 index 107a786..0000000 --- a/php-classes/Gatekeeper/Utils/IP.php +++ /dev/null @@ -1,27 +0,0 @@ -
Endpoint Request Timestamp Response Code
{contextLink $Request->Endpoint} {$Request->Method} {$Request->Path|default:/}{tif $Request->Query ? "?$Request->Query"|query_string} {$Request->Created|date_format:'%Y-%m-%d %H:%M:%S'} {$Request->ResponseCode}