diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..af80492 --- /dev/null +++ b/.env.example @@ -0,0 +1,11 @@ +# - Copy .env.example to .env to initialize a new instance +# - DO NOT commit .env to source control +# - See docs/deployment/docker-compose.md for configuration reference + + +# Accept Chef Habitat's license by activating this .env +HAB_LICENSE=accept-no-persist + +# MySQL database (local and remote) +MYSQL_USERNAME=gatekeeper +MYSQL_PASSWORD=8edvN7xotT49qLZReA2Cx1TFKpvvxK9 diff --git a/.github/workflows/k8s-deploy-pr.yml b/.github/workflows/k8s-deploy-pr.yml new file mode 100644 index 0000000..c2107ee --- /dev/null +++ b/.github/workflows/k8s-deploy-pr.yml @@ -0,0 +1,156 @@ +name: K8s PR Sandbox Build+Deploy + +on: + pull_request: + branches: [develop] + types: [opened, reopened, synchronize] + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + KUBE_NAMESPACE: ${{ secrets.kube_namespace }} + KUBE_CONFIG_DATA: ${{ secrets.kube_config }} + + PACKAGE_NAME: gatekeeper-composite + PACKAGE_REGISTRY: docker.pkg.github.com + + PR_NAME: pr-${{ github.event.number }} + KUBE_HOSTNAME: ${{ secrets.kube_hostname }} + + HAB_LICENSE: accept-no-persist + +jobs: + kubernetes-deploy: + + runs-on: ubuntu-latest + steps: + + - uses: actions/checkout@v2 + with: + ref: ${{ github.head_ref }} + + - name: Create Github Deployment + run: | + set -e + # Create deployment + hub api /repos/${{ github.repository }}/deployments -X POST --input <(cat < /tmp/deployment.json + + DEPLOYMENT_ID=$(jq .id < /tmp/deployment.json) + echo ::set-env name=GH_DEPLOYMENT_ID::$(echo $DEPLOYMENT_ID) + + - name: Update GH Deployment Status + run: | + set -e + + # Set status to pending + hub api /repos/${{ github.repository }}/deployments/${{ env.GH_DEPLOYMENT_ID }}/statuses \ + -X POST \ + -H "Accept: application/json, application/vnd.github.flash-preview+json" \ + --input <(cat < ~/.kube/config + $(printf '%s' "$KUBE_CONFIG_DATA" | base64 -d) + EOF + + - uses: azure/setup-kubectl@v1 + + - name: Create K8S Deployment from Template + run: | + set -e + + image_id=$(echo ${{ env.REPO_NAME }}/${{ env.PACKAGE_NAME }}:${{ env.PR_NAME }}) + image_url=$(echo docker.pkg.github.com/$image_id) + hostname=$(echo ${{ env.PR_NAME }}.${{ env.KUBE_HOSTNAME }}) + + kubectl config set-context --current --namespace=${KUBE_NAMESPACE} + + # delete any existing pod first to force image to re-pull without changing tag + # kubectl delete pod -l app.kubernetes.io/instance="${PR_NAME}" + # helm uninstall ${{ env.PR_NAME }} -n ${{ env.KUBE_NAMESPACE }} + + helm upgrade ${{ env.PR_NAME }} ./k8s/charts/deployment \ + --install \ + --set name=${{ env.PR_NAME }} \ + --set namespace=${{ env.KUBE_NAMESPACE }} \ + --set image=${image_url} \ + --set hostname=${hostname} + + + - name: Wait for Deployment to be Ready + run: | + set -e + until kubectl rollout status deployment "${PR_NAME}" 2>/dev/null >/dev/null; do echo -n "."; sleep .5; done; + + - name: Retrieve/Store Pod Name + run: | + echo ::set-env name=POD_NAME::$(kubectl get pod -l app.kubernetes.io/instance="${PR_NAME}" -o jsonpath='{.items[0].metadata.name}') + + - name: Wait For Pod to be Ready + run: | + set -e + kubectl wait --for condition=ready "pod/${POD_NAME}" --timeout=30s + + - name: Mark deployment as failed + if: failure() + run: | + hub api /repos/${{ github.repository }}/deployments/${{ env.GH_DEPLOYMENT_ID }}/statuses \ + -X POST \ + -H "Accept: application/json, application/vnd.github.flash-preview+json" \ + --input <(cat <getEndpoint(); $userIdentifier = $_EVENT['request']->getUserIdentifier(); +$Key = $_EVENT['request']->getKey(); // append metrics @@ -20,11 +21,14 @@ // drip bandwidth bucket -if ($Endpoint->GlobalBandwidthPeriod && $Endpoint->GlobalBandwidthCount) { +if ( + (!$Key || !$Key->RateLimitExempt) && + ($Endpoint->GlobalBandwidthPeriod && $Endpoint->GlobalBandwidthCount) +) { HitBuckets::drip("endpoints/$Endpoint->ID/bandwidth", function() use ($Endpoint) { return [ 'seconds' => $Endpoint->GlobalBandwidthPeriod, 'count' => $Endpoint->GlobalBandwidthCount ]; }, $_EVENT['Transaction']->ResponseBytes); -} \ No newline at end of file +} diff --git a/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/75_reject-endpoint-user-rate.php b/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/75_reject-endpoint-user-rate.php index 442a7ab..80e6d07 100644 --- a/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/75_reject-endpoint-user-rate.php +++ b/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/75_reject-endpoint-user-rate.php @@ -5,10 +5,14 @@ $Endpoint = $_EVENT['request']->getEndpoint(); $userIdentifier = $_EVENT['request']->getUserIdentifier(); +$Key = $_EVENT['request']->getKey(); // drip into endpoint+user bucket first so that abusive users can't pollute the global bucket -if ($Endpoint->UserRatePeriod && $Endpoint->UserRateCount) { +if ( + (!$Key || !$Key->RateLimitExempt) && + ($Endpoint->UserRatePeriod && $Endpoint->UserRateCount) +) { $bucket = HitBuckets::drip("endpoints/$Endpoint->ID/$userIdentifier", function() use ($Endpoint) { return array('seconds' => $Endpoint->UserRatePeriod, 'count' => $Endpoint->UserRateCount); }); diff --git a/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/95_reject-endpoint-rate.php b/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/95_reject-endpoint-rate.php index 039d88b..c373229 100644 --- a/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/95_reject-endpoint-rate.php +++ b/event-handlers/Gatekeeper/ApiRequestHandler/beforeApiRequest/95_reject-endpoint-rate.php @@ -8,10 +8,14 @@ $Endpoint = $_EVENT['request']->getEndpoint(); +$Key = $_EVENT['request']->getKey(); // drip into endpoint requests bucket -if ($Endpoint->GlobalRatePeriod && $Endpoint->GlobalRateCount) { +if ( + (!$Key || !$Key->RateLimitExempt) && + ($Endpoint->GlobalRatePeriod && $Endpoint->GlobalRateCount) +) { $flagKey = "alerts/endpoints/$Endpoint->ID/rate-flagged"; $bucket = HitBuckets::drip("endpoints/$Endpoint->ID/requests", function() use ($Endpoint) { diff --git a/habitat/composite/default.toml b/habitat/composite/default.toml new file mode 100644 index 0000000..0ac11b5 --- /dev/null +++ b/habitat/composite/default.toml @@ -0,0 +1,2 @@ +[services.app.config] + default_timezone = "America/New_York" diff --git a/habitat/composite/plan.sh b/habitat/composite/plan.sh new file mode 100644 index 0000000..3bee95d --- /dev/null +++ b/habitat/composite/plan.sh @@ -0,0 +1,10 @@ +composite_app_pkg_name=gatekeeper +pkg_name="${composite_app_pkg_name}-composite" +pkg_origin=jarvus +pkg_maintainer="Jarvus Innovations " +pkg_scaffolding=emergence/scaffolding-composite +composite_mysql_pkg=core/mysql + +pkg_version() { + scaffolding_detect_pkg_version +} diff --git a/habitat/default.toml b/habitat/default.toml new file mode 100644 index 0000000..2065aaa --- /dev/null +++ b/habitat/default.toml @@ -0,0 +1,2 @@ +[sites.default] +database = "gatekeeper" diff --git a/habitat/plan.sh b/habitat/plan.sh new file mode 100644 index 0000000..c01a1a6 --- /dev/null +++ b/habitat/plan.sh @@ -0,0 +1,8 @@ +pkg_name=gatekeeper +pkg_origin=jarvus +pkg_maintainer="Jarvus Innovations " +pkg_scaffolding=emergence/scaffolding-site + +pkg_version() { + scaffolding_detect_pkg_version +} diff --git a/html-templates/bans/ban.tpl b/html-templates/bans/ban.tpl index 7c00354..0ff7a95 100644 --- a/html-templates/bans/ban.tpl +++ b/html-templates/bans/ban.tpl @@ -9,8 +9,8 @@

- {if $Ban->IP} - IP Address: {$Ban->IP|long2ip} + {if $Ban->IPPattern} + IP Pattern: {$Ban->IPPattern} {else} Key: {apiKey $Ban->Key} {/if} diff --git a/html-templates/bans/banDeleted.tpl b/html-templates/bans/banDeleted.tpl index f99c810..344bf04 100644 --- a/html-templates/bans/banDeleted.tpl +++ b/html-templates/bans/banDeleted.tpl @@ -5,7 +5,7 @@ {block "content"} {$Ban = $data} -

Ban on {if $Ban->IP}IP Address: {$Ban->IP|long2ip}{else}Key: {apiKey $Ban->Key}{/if} deleted.

+

Ban on {if $Ban->IPPattern}IP Pattern: {$Ban->IPPattern}{else}Key: {apiKey $Ban->Key}{/if} deleted.

Browse all bans

{/block} \ No newline at end of file diff --git a/html-templates/bans/banEdit.tpl b/html-templates/bans/banEdit.tpl index 1a8307d..6ee4ba9 100644 --- a/html-templates/bans/banEdit.tpl +++ b/html-templates/bans/banEdit.tpl @@ -27,7 +27,7 @@
- {field inputName=IP label='IP Address' error=$errors.IP default=tif($Ban->IP, long2ip($Ban->IP))} + {field inputName=IPPattern label='IP Pattern' error=$errors.IPPattern default=$Ban->IPPattern hint="192.168.1.1,192.168.1.*,192.168.1.1/24"}
—or—
{field inputName=KeyID label='API Key' error=$errors.KeyID default=$Ban->Key->Key}
diff --git a/html-templates/bans/banSaved.tpl b/html-templates/bans/banSaved.tpl index 8c6f4a9..26c7671 100644 --- a/html-templates/bans/banSaved.tpl +++ b/html-templates/bans/banSaved.tpl @@ -7,7 +7,7 @@

Ban on - {if $Ban->IP}IP Address: {$Ban->IP|long2ip} + {if $Ban->IPPattern}IP Address Pattern: {$Ban->IPPattern} {else}Key: {apiKey $Ban->Key} {/if} {tif $Ban->isNew ? created : saved}. diff --git a/html-templates/bans/bans.tpl b/html-templates/bans/bans.tpl index 387e7d2..b78548a 100644 --- a/html-templates/bans/bans.tpl +++ b/html-templates/bans/bans.tpl @@ -8,6 +8,7 @@

Bans

Issue Ban + Bulk Bans

@@ -29,8 +30,8 @@

- {if $Ban->IP} - IP Address: {$Ban->IP|long2ip} + {if $Ban->IPPattern} + IP Pattern: {$Ban->IPPattern} {else} Key: {apiKey $Ban->Key} {/if} diff --git a/html-templates/bans/bulk/banCreate.tpl b/html-templates/bans/bulk/banCreate.tpl new file mode 100644 index 0000000..ef71154 --- /dev/null +++ b/html-templates/bans/bulk/banCreate.tpl @@ -0,0 +1,35 @@ +{extends "designs/site.tpl"} + +{block "title"}Create Bans in Bulk — {$dwoo.parent}{/block} + +{block "content"} + {$Ban = $data} + {$errors = $Ban->validationErrors} + +
+

New Bans

+
+ Cancel +
+
+ +
+ {if $errors} +
+ Please double-check the following lines highlighted below. + + {foreach from=$invalidPatterns item=invalidPattern} +

{$invalidPattern}

+ {/foreach} +
+ {/if} + +
+ {textarea inputName=IPPatterns label='IP Patterns' hint="Separate IP Patterns by line"} + +
+ +
+
+
+{/block} \ No newline at end of file diff --git a/html-templates/bans/bulk/bansSaved.tpl b/html-templates/bans/bulk/bansSaved.tpl new file mode 100644 index 0000000..ef5aa1c --- /dev/null +++ b/html-templates/bans/bulk/bansSaved.tpl @@ -0,0 +1,12 @@ +{extends "designs/site.tpl"} + +{block "title"}Ban saved — {$dwoo.parent}{/block} + +{block "content"} +

Bans created for the following patterns:

+ {foreach from=$data item=Ban} +

{$Ban->IPPattern}

+ {/foreach} + +

← Browse all bans

+{/block} \ No newline at end of file diff --git a/html-templates/keys/keyEdit.tpl b/html-templates/keys/keyEdit.tpl index 8a8be59..5e871ee 100644 --- a/html-templates/keys/keyEdit.tpl +++ b/html-templates/keys/keyEdit.tpl @@ -35,6 +35,7 @@ {field inputName=ExpirationDate label='Expiration Date' type=date default=tif($Key->ExpirationDate, date('Y-m-d', $Key->ExpirationDate)) hint="Leave blank if none"} + {checkbox inputName=RateLimitExempt value=1 unsetValue=0 label='Exempt from Rate Limits?' default=$Key->RateLimitExempt hint="Check this option to exempt this key from rate limit thresholds and impacting other consumers of the API."} {checkbox inputName=AllEndpoints value=1 unsetValue=0 label='Allow all endpoints?' default=$Key->AllEndpoints hint="Uncheck this option to allow more fine-grained access control to endpoints on the key page."} {checkbox inputName=Status value=revoked unsetValue=active label='Revoked' default=$Key->Status} diff --git a/html-templates/site-admin/tasks/configuration/skip-insert-transactions.tpl b/html-templates/site-admin/tasks/configuration/skip-insert-transactions.tpl new file mode 100644 index 0000000..1e8a001 --- /dev/null +++ b/html-templates/site-admin/tasks/configuration/skip-insert-transactions.tpl @@ -0,0 +1,26 @@ +{extends "task.tpl"} + +{block content} +

+ Skip Insert Transaction (Degradation Mode): {tif $enabled ? Active : Inactive} +

+ +
+
+ + +
+ +
+ +
+ + +
+{/block} \ No newline at end of file diff --git a/html-templates/subtemplates/contextLinks.tpl b/html-templates/subtemplates/contextLinks.tpl index 749ffed..9ac3069 100644 --- a/html-templates/subtemplates/contextLinks.tpl +++ b/html-templates/subtemplates/contextLinks.tpl @@ -18,8 +18,8 @@ {$prefix}Ban #{$Context->ID} — - {if $Context->IP} - IP Address: {$Context->IP|long2ip} + {if $Context->IPPattern} + IP Pattern: {$Context->IPPattern} {else} Key: {$Context->Key->OwnerName|escape} {$Context->Key->Key} {/if}{$suffix} diff --git a/html-templates/transactions/transactions.tpl b/html-templates/transactions/transactions.tpl index d567207..3428d62 100644 --- a/html-templates/transactions/transactions.tpl +++ b/html-templates/transactions/transactions.tpl @@ -75,6 +75,7 @@ + @@ -88,6 +89,7 @@ {foreach item=Request from=$data} + diff --git a/k8s/charts/deployment/Chart.yaml b/k8s/charts/deployment/Chart.yaml new file mode 100644 index 0000000..97358e2 --- /dev/null +++ b/k8s/charts/deployment/Chart.yaml @@ -0,0 +1,23 @@ +apiVersion: v2 +name: gatekeeper-deployment +description: A Helm chart for Gatekeeper Deployments + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.0.1 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +appVersion: 2.3.1 diff --git a/k8s/charts/deployment/templates/_helpers.tpl b/k8s/charts/deployment/templates/_helpers.tpl new file mode 100644 index 0000000..00070af --- /dev/null +++ b/k8s/charts/deployment/templates/_helpers.tpl @@ -0,0 +1,41 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "gatekeeper-deployment.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "gatekeeper-deployment.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "gatekeeper-deployment.labels" -}} +helm.sh/chart: {{ include "gatekeeper-deployment.chart" . }} +{{ include "gatekeeper-deployment.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "gatekeeper-deployment.selectorLabels" -}} +app.kubernetes.io/name: {{ include "gatekeeper-deployment.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Cert Manager Annotations +*/}} +{{- define "gatekeeper-deployment.cert-manager-annotations" -}} +cert-manager.io/cluster-issuer: {{ .Values.cert_manager.annotations.cluster_issuer }} +{{- end }} diff --git a/k8s/charts/deployment/templates/deployment.yaml b/k8s/charts/deployment/templates/deployment.yaml new file mode 100644 index 0000000..a997a70 --- /dev/null +++ b/k8s/charts/deployment/templates/deployment.yaml @@ -0,0 +1,94 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }} + namespace: {{ .Values.namespace }} + labels: + {{- include "gatekeeper-deployment.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + strategy: + type: RollingUpdate + selector: + matchLabels: + {{- include "gatekeeper-deployment.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "gatekeeper-deployment.labels" . | nindent 8 }} + spec: + imagePullSecrets: + - name: regcred + restartPolicy: Always + + containers: + - image: {{ .Values.image }} + name: {{ .Release.Name }}-app + imagePullPolicy: Always + + env: + - name: HAB_GATEKEEPER_COMPOSITE + value: | + [services.mysql] + pkg_ident = {{ .Values.hab.gatekeeper_composite.mysql.pkg_ident | quote }} + + - name: HAB_MYSQL + value: | + {{- if .Values.hab.mysql }} + {{- range $key, $value := .Values.hab.mysql }} + {{ $key }} = {{ $value | quote }} + {{- end }} + {{- end }} + + ports: + - containerPort: 80 + name: http + protocol: TCP +--- + +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }} + namespace: {{ .Values.namespace }} + labels: + {{- include "gatekeeper-deployment.labels" . | nindent 4 }} + app: {{ .Release.Name }} + name: {{ .Release.Name }}-app +spec: + selector: + {{- include "gatekeeper-deployment.selectorLabels" . | nindent 4 }} + ports: + - name: http + port: 80 + protocol: TCP +--- + + +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: {{ .Release.Name }} + namespace: {{ .Values.namespace }} + labels: + {{- include "gatekeeper-deployment.labels" . | nindent 4 }} + annotations: + kubernetes.io/ingress.class: "nginx" + ingress.kubernetes.io/rewrite-target: / + {{- include "gatekeeper-deployment.cert-manager-annotations" . | nindent 4 }} +spec: + tls: + - hosts: + - {{ .Values.hostname }} + secretName: {{ .Release.Name }}-tls + backend: + serviceName: {{ .Release.Name }} + servicePort: 80 + rules: + - host: {{ .Values.hostname }} + http: + paths: + - path: / + backend: + serviceName: {{ .Release.Name }} + servicePort: 80 diff --git a/k8s/charts/deployment/values.yaml b/k8s/charts/deployment/values.yaml new file mode 100644 index 0000000..3c84a51 --- /dev/null +++ b/k8s/charts/deployment/values.yaml @@ -0,0 +1,17 @@ +# Default values for deployment +replicaCount: 1 + +hab: + gatekeeper_composite: + mysql: + pkg_ident: 'core/mysql' + +# these can be made up + mysql: + app_username: admin + app_password: xTw9wYFe70 + bind: '0.0.0.0' + +cert_manager: + annotations: + cluster_issuer: letsencrypt-prod diff --git a/php-classes/Gatekeeper/ApiRequestHandler.php b/php-classes/Gatekeeper/ApiRequestHandler.php index 8064297..d4d669d 100644 --- a/php-classes/Gatekeeper/ApiRequestHandler.php +++ b/php-classes/Gatekeeper/ApiRequestHandler.php @@ -15,20 +15,17 @@ class ApiRequestHandler extends \RequestHandler public static $defaultTimeout = 30; public static $defaultTimeoutConnect = 5; public static $passthruHeaders = [ - '/^HTTP\//' - ,'/^Content-Type:/i' - ,'/^Date:/i' - ,'/^Set-Cookie:/i' - ,'/^Location:/i' - ,'/^ETag:/i' - ,'/^Last-Modified:/i' - ,'/^Cache-Control:/i' - ,'/^Pragma:/i' - ,'/^Expires:/i' + '/.*/' + ]; + + public static $forwardHeaders = [ + 'Authorization' ]; public static $responseMode = 'json'; // override RequestHandler::$responseMode + public static $degradationTimeout = 60; + public static function handleRequest() { // initialize request object @@ -55,10 +52,14 @@ public static function handleRequest() { ,'autoQuery' => false ,'url' => rtrim($request->getEndpoint()->InternalEndpoint, '/') . $request->getUrl() ,'interface' => static::$sourceInterface + ,'headers' => [ + "X-Forwarded-For: {$_SERVER['REMOTE_ADDR']}" + ] ,'passthruHeaders' => static::$passthruHeaders + ,'forwardHeaders' => array_merge(HttpProxy::$defaultForwardHeaders, static::$forwardHeaders) ,'timeout' => static::$defaultTimeout ,'timeoutConnect' => static::$defaultTimeoutConnect -# ,'debug' => true // uncomment to debug proxy process and see output following response + // ,'debug' => true // uncomment to debug proxy process and see output following response // ,'afterResponseSync' => true // true to debug afterResponse code from browser ,'afterResponse' => function ($responseBody, $responseHeaders, $options, $curlHandle) use ($request, &$metrics, &$beforeEvent) { @@ -68,17 +69,29 @@ public static function handleRequest() { // initialize log record if (!Cache::fetch('flags/gatekeeper/skip-insert-transaction')) { - $Transaction = Transaction::create([ - 'Endpoint' => $request->getEndpoint() - ,'Key' => $request->getKey() - ,'ClientIP' => ip2long($_SERVER['REMOTE_ADDR']) - ,'Method' => $_SERVER['REQUEST_METHOD'] - ,'Path' => $path - ,'Query' => $query - ,'ResponseTime' => $curlInfo['starttransfer_time'] * 1000 - ,'ResponseCode' => $curlInfo['http_code'] - ,'ResponseBytes' => $curlInfo['size_download'] - ]); + try { + $Transaction = Transaction::create([ + 'Endpoint' => $request->getEndpoint() + ,'Key' => $request->getKey() + ,'ClientIP' => ip2long($_SERVER['REMOTE_ADDR']) + ,'Method' => $_SERVER['REQUEST_METHOD'] + ,'Path' => $path + ,'Query' => $query + ,'ResponseTime' => $curlInfo['starttransfer_time'] * 1000 + ,'ResponseCode' => $curlInfo['http_code'] + ,'ResponseBytes' => $curlInfo['size_download'] + ]); + } catch (\Exception $e) { + Cache::store('flags/gatekeeper/skip-insert-transaction', true, static::$degradationTimeout); + \Emergence\Logger::general_warning( + 'Transaction Exception: {exceptionMessage}. Setting degredation flag for {seconds} seconds', + [ + 'exception' => $e, + 'exceptionMessage' => $e->getMessage(), + 'seconds' => static::$degradationTimeout + ] + ); + } } diff --git a/php-classes/Gatekeeper/Bans/Ban.php b/php-classes/Gatekeeper/Bans/Ban.php index 0bf1947..d787637 100644 --- a/php-classes/Gatekeeper/Bans/Ban.php +++ b/php-classes/Gatekeeper/Bans/Ban.php @@ -3,7 +3,9 @@ namespace Gatekeeper\Bans; use Cache; +use Emergence\Site\Storage; use Gatekeeper\Keys\Key; +use Gatekeeper\Utils\IPPattern; class Ban extends \ActiveRecord { @@ -21,8 +23,7 @@ class Ban extends \ActiveRecord 'type' => 'uint', 'notnull' => false ], - 'IP' => [ - 'type' => 'uint', + 'IPPattern' => [ 'notnull' => false ], 'ExpirationDate' => [ @@ -63,8 +64,8 @@ public function validate($deep = true) { parent::validate($deep); - if (!$this->KeyID == !$this->IP) { - $this->_validator->addError('Ban', 'Ban must specifiy either a API key or an IP address'); + if (!$this->KeyID == !$this->IPPattern) { + $this->_validator->addError('Ban', 'Ban must specify either a API key or an IP pattern'); } return $this->finishValidation(); @@ -96,7 +97,8 @@ public static function sortCreated($dir, $name) return "ID $dir"; } - protected static $_activeBans; + + protected static $_activeBans; public static function getActiveBansTable() { if (isset(static::$_activeBans)) { @@ -108,13 +110,18 @@ public static function getActiveBansTable() } static::$_activeBans = [ - 'ips' => [] - ,'keys' => [] + 'patterns' => [], + 'ips' => [], + 'keys' => [] ]; foreach (Ban::getAllByWhere('ExpirationDate IS NULL OR ExpirationDate > CURRENT_TIMESTAMP') AS $Ban) { - if ($Ban->IP) { - static::$_activeBans['ips'][] = long2ip($Ban->IP); + if (!empty($Ban->IPPattern)) { + if (is_array(static::getIPPatternBanClosure($Ban->IPPattern))) { // ip pattern ONLY contains static IPs + static::$_activeBans['ips'] = array_merge(static::$_activeBans['ips'], static::getIPPatternBanClosure($Ban->IPPattern)); + } else { + static::$_activeBans['patterns'][] = $Ban->IPPattern; + } } elseif($Ban->KeyID) { static::$_activeBans['keys'][] = $Ban->KeyID; } @@ -125,9 +132,44 @@ public static function getActiveBansTable() return static::$_activeBans; } + public static function getIPPatternBanClosure($ipPattern) + { + static $ipPatternCaches = []; + + $ipPatternHash = sha1($ipPattern); + + if (!empty($ipPatternCaches[$ipPatternHash])) { + return $ipPatternCaches[$ipPatternHash]; + } + + try { + $closure = include IPPattern::getFilenameFromHash($ipPatternHash); + ); + } catch (\Exception $e) { + $closure = IPPattern::parse($ipPattern, $ipPatternHash); + } + + return $ipPatternCaches[$ipPatternHash] = $closure; + } + public static function isIPAddressBanned($ip) { - return in_array($ip, static::getActiveBansTable()['ips']); + $activeBans = static::getActiveBansTable(); + + // check for explicit IP ban + if (in_array($ip, $activeBans['ips'])) { + return true; + } + + // check IP Patterns individually + foreach($activeBans['patterns'] as $ipPattern) { + $matcher = static::getIPPatternBanClosure($ipPattern); + if (call_user_func($matcher, $ip) === true) { + return true; + } + } + + return false; } public static function isKeyBanned(Key $Key) diff --git a/php-classes/Gatekeeper/Bans/BansRequestHandler.php b/php-classes/Gatekeeper/Bans/BansRequestHandler.php index 6ae60b1..09fc92d 100644 --- a/php-classes/Gatekeeper/Bans/BansRequestHandler.php +++ b/php-classes/Gatekeeper/Bans/BansRequestHandler.php @@ -14,12 +14,41 @@ class BansRequestHandler extends \RecordsRequestHandler public static $accountLevelWrite = 'Staff'; public static $accountLevelAPI = 'Staff'; - protected static function applyRecordDelta(\ActiveRecord $Ban, $data) + public static function handleCreateRequest(\ActiveRecord $Record = null) + { + if (static::shiftPath() === 'bulk') { + return static::handleBulkCreationRequest(); + } + + return parent::handleCreateRequest($Record); + } + + protected static function handleBulkCreationRequest() { - if (isset($data['IP']) && !is_numeric($data['IP'])) { - $data['IP'] = ip2long($data['IP']); + if ($_SERVER['REQUEST_METHOD'] === 'POST') { + // process request + $bans = []; + foreach (explode("\r\n", $_REQUEST['IPPatterns']) as $ipPattern) { + $trimmedPattern = trim($ipPattern); + if (empty($trimmedPattern)) { + continue; + } + + $bans[] = Ban::create([ + 'IPPattern' => $trimmedPattern + ], true); + } + + return static::respond('bulk/bansSaved', [ + 'data' => $bans + ]); } + return static::respond('bulk/banCreate'); + } + + protected static function applyRecordDelta(\ActiveRecord $Ban, $data) + { if (isset($data['KeyID']) && !is_numeric($data['KeyID'])) { $Key = Key::getByHandle($data['KeyID']); $data['KeyID'] = $Key ? $Key->ID : null; diff --git a/php-classes/Gatekeeper/Keys/Key.php b/php-classes/Gatekeeper/Keys/Key.php index bd519e0..a6eabb4 100644 --- a/php-classes/Gatekeeper/Keys/Key.php +++ b/php-classes/Gatekeeper/Keys/Key.php @@ -46,6 +46,10 @@ class Key extends \ActiveRecord 'AllEndpoints' => [ 'type' => 'boolean', 'default' => false + ], + 'RateLimitExempt' => [ + 'type' => 'boolean', + 'default' => false ] ]; diff --git a/php-classes/Gatekeeper/Utils/IPPattern.php b/php-classes/Gatekeeper/Utils/IPPattern.php new file mode 100644 index 0000000..d0976c1 --- /dev/null +++ b/php-classes/Gatekeeper/Utils/IPPattern.php @@ -0,0 +1,248 @@ + [], + 'cidr' => [], + 'wildcard' => [] + ]; + + $count = 0; + foreach (preg_split("/[\s,]+/", $pattern) as $subPattern) { + $subPatternType = static::getPatternType($subPattern); + if (!empty($subPattern) && array_key_exists($subPatternType, $subPatternsByType)) { + $subPatternsByType[$subPatternType][] = $subPattern; + $count++; + } + } + + + if ($count === 0) { + throw new InvalidArgumentException("Unable to parse IP pattern: $pattern"); + } + + if (count($subPatternsByType['cidr']) === 0 && count($subPatternsByType['wildcard']) === 0) { + return $subPatternsByType['ip']; + } + + return static::generateClosure($subPatternsByType, $uid); + } + + /** + * + * Get IPPattern filename (containing closure method) from the hashed IP Pattern + * @param string $patternHash The hashed IP Pattern + * + * @return string Returns a string containing the full path of the ip pattern file + */ + public static function getFilenameFromHash($patternHash) + { + return join('/', + [ + Storage::getLocalStorageRoot(), + static::$fsRootDir, + $patternHash . '.php' + ] + ); + } + + /** + * + * Generate Closure function and write to filesystem with the name of the pattern hashed + * + * @param array $patterns Multi-dimensional array of sub-patterns, keyed by pattern type. + * @param string $patternHash Original Pattern hashed. + * + * @return closure Closure function + */ + protected static function generateClosure(array $patterns, $patternHash) + { + + $closureFunctionString = +<<write( + $fileName, + $closureFunctionString + ); + + return include join('/', [Storage::getLocalStorageRoot(), static::$fsRootDir, $fileName]); + } + + /** + * + * Generate Closure condition string for use in the generateClosure() function. + * + * @param string $pattern Sub-pattern to create condition for. + * @param string $patternType Type of ip pattern (ip, cidr, wilcdcard). + * + * @return string If Condition in string format + */ + protected static function generateClosureCondition($pattern, $patternType) + { + switch ($patternType) { + case 'ip': + $ipLong = ip2long($pattern); + $conditionString = <<= $min && \$ipLong <= $max) { + return true; + } + +DOC; + + break; + + case 'wildcard': + $patternRegex = static::getWildcardRegex($pattern); + return << 'like' ], [ - 'field' => 'IP', - 'method' => 'sql', - 'sql' => 'INET_NTOA(IP) LIKE "%%%s%%"' + 'field' => 'IPPattern', + 'method' => 'like' ] ] ]; diff --git a/php-migrations/Gatekeeper/Ban/20200617000000_ippattern-column.php b/php-migrations/Gatekeeper/Ban/20200617000000_ippattern-column.php new file mode 100644 index 0000000..0fca117 --- /dev/null +++ b/php-migrations/Gatekeeper/Ban/20200617000000_ippattern-column.php @@ -0,0 +1,56 @@ + `%s`", $deprecatedColumn, $columnName); +DB::nonQuery( + ' + UPDATE `%1$s` SET %2$s = %3$s + WHERE %3$s IS NOT NULL + ', + [ + Ban::$tableName, + $columnName, + $deprecatedColumn + ] +); + +printf("Removing deprecated column: IP"); +DB::nonQuery( + 'ALTER TABLE `%s` DROP COLUMN %s', + [ + Ban::$tableName, + $deprecatedColumn + ] +); + +return static::STATUS_EXECUTED; \ No newline at end of file diff --git a/php-migrations/Gatekeeper/Key/20200524000000_add-rate-limit-exempt.php b/php-migrations/Gatekeeper/Key/20200524000000_add-rate-limit-exempt.php new file mode 100644 index 0000000..1d37872 --- /dev/null +++ b/php-migrations/Gatekeeper/Key/20200524000000_add-rate-limit-exempt.php @@ -0,0 +1,31 @@ + studio-bootstrap: verifying Docker…" + +if ! [ -x "$(command -v docker)" ]; then + echo "Please install Docker Engine: https://docs.docker.com/engine/install/" + exit 1 +fi + +if ! docker info > /dev/null 2>&1; then + echo "Docker Engine is not running, or your user does not have access to connect." + echo "Try starting Docker Engine, and adding your user to the docker group: sudo gpasswd -a $USER docker" + exit 1 +fi + + +echo +echo "==> studio-bootstrap: verifying Chef Habitat…" + +if ! [ -x "$(command -v hab)" ]; then + echo "Please install Chef Habitat: https://www.habitat.sh/docs/install-habitat/" + exit 1 +fi + +set +e +hab_version="$(hab --version < /dev/null)" +if [ $? -ne 0 ]; then + echo + echo " Failed to read hab version, you may need to accept the Chef Habitat" + echo " license. Please run \`hab setup\` or configure HAB_LICENSE in the environment" + exit 1 +fi +set -e + +if ! [[ $hab_version =~ ^hab[[:space:]][0-9]+\.[0-9]+\.[0-9]+/[0-9]+$ ]]; then + echo + echo " Could not parse hab version: ${hab_version}" + echo " Please install hab 1.6+" + exit 1 +fi + +hab_version="$(echo "${hab_version}" | awk '{print $2}' | awk -F'/' '{print $1}')" +echo " Found hab version: ${hab_version}" + + +# check that node >= MAJOR.MINOR +hab_min_major="1" +hab_min_minor="6" + +IFS='.' read -ra hab_version_split <<< "${hab_version#v}" +if [ "${hab_version_split[0]}" -lt "${hab_min_major}" ] || [[ "${hab_version_split[0]}" -le "${hab_min_major}" && "${hab_version_split[1]}" -lt "${hab_min_minor}" ]]; then + echo + echo " Please install hab >= ${hab_min_major}.${hab_min_minor}.x" + exit 1 +fi + +if ! [ -f ~/.hab/etc/cli.toml ] || ! grep -q '^origin =' ~/.hab/etc/cli.toml; then + echo "Please re-run \`hab setup\` and choose to set a default origin, it can be anything" + exit 1 +fi + +_origin=$(awk -F'"' '/^origin = /{print $2}' ~/.hab/etc/cli.toml) + + +echo +echo "==> studio-bootstrap: verifying origin '${_origin}'…" + +_root_owned_key_count=$(ls -l ~/.hab/cache/keys | cut -f 3,4 -d " " | grep "root root" | wc -l) + +if [ "$_root_owned_key_count" -gt 0 ]; then + echo "Working around: https://github.com/habitat-sh/habitat/issues/7737" + echo "Found ${_root_owned_key_count} keys owned by root. Chowning them to $USER:$USER." + sudo chown $USER:$USER ~/.hab/cache/keys/* +fi + +if ! hab origin key export --type secret "${_origin}" > /dev/null; then + echo "No key has been generated for origin ${_origin}, run: hab origin key generate ${_origin}" + exit 1 +fi + + +echo +echo "==> studio-bootstrap: all set 👍" +echo diff --git a/script/studio b/script/studio new file mode 100755 index 0000000..a47f3f2 --- /dev/null +++ b/script/studio @@ -0,0 +1,69 @@ +#!/bin/sh + +# script/studio: Enter a Chef Habitat studio for the application. + +set -e +cd "$(dirname "$0")/.." + + +script/-studio-bootstrap + + +unset DEBUG + + +echo +echo "==> studio: configuring Chef Habitat studio Docker options…" +STUDIO_NAME="${STUDIO_NAME:-gatekeeper-studio}" +export HAB_DOCKER_OPTS=" + --name ${STUDIO_NAME} + -p 7080:80 + -p 7043:443 + -p 7036:3306 + -p 7099:19999 + -p 7800:8000 + -v ${STUDIO_NAME}-mysql-data:/hab/svc/mysql/data + -v $(cd ~/.ssh; pwd)/known_hosts:/root/.ssh/known_hosts:ro + --env STUDIO_DEVELOPER_UID=$(id -u) + --env STUDIO_DEVELOPER_GID=$(id -g) +" +echo "${HAB_DOCKER_OPTS}" + + +launch_studio=true +if [ "$(docker ps -aq -f name="${STUDIO_NAME}")" ]; then + echo + echo "==> studio: a ${STUDIO_NAME} container is already running…" + echo + while true; do + read -p "==> studio: would you like to (A)ttach to it, (s)top it, or do (n)othing? [A/s/n] " choice + case "${choice}" in + r|R|a|A|"") + echo + echo "==> studio: you can run studio-help at anytime to get a list of commands" + echo + docker attach "${STUDIO_NAME}" + launch_studio=false + break;; + s|S) + echo "==> studio: stopping existing container…" + docker stop "${STUDIO_NAME}" > /dev/null + break;; + n|N) + echo "==> studio: doing nothing with existing container… an error is likely to occur" + break ;; + *) + echo "==> studio: $choice is invalid";; + esac + done +fi + +if [ $launch_studio = true ]; then + echo + echo "==> studio: launching Docker-powered Chef Habitat studio…" + set +e + if ! hab studio enter -D; then + echo "===> studio: failed to launch studio… try executing the following and try again:" + echo "docker rm -f ${STUDIO_NAME}" + fi +fi diff --git a/sencha-workspace/pages/src/widget/model/Ban.js b/sencha-workspace/pages/src/widget/model/Ban.js index 7afb2d7..dde38ad 100644 --- a/sencha-workspace/pages/src/widget/model/Ban.js +++ b/sencha-workspace/pages/src/widget/model/Ban.js @@ -10,7 +10,7 @@ Ext.define('Site.widget.model.Ban', { tpl: [ '', - '#{ID} — IP: {[this.long2ip(values.IP)]}Key ', + '#{ID} — IP Pattern: {IPPattern}Key ', 'Expires {[this.formatTimestamp(values.ExpirationDate)]}', '', { diff --git a/site-root/sass/site/modules/_tables.scss b/site-root/sass/site/modules/_tables.scss index 55610e8..522dfc8 100644 --- a/site-root/sass/site/modules/_tables.scss +++ b/site-root/sass/site/modules/_tables.scss @@ -16,7 +16,7 @@ vertical-align: -.2em; white-space: nowrap; width: 16px; - + // gimme crisp pixels image-rendering:optimizeSpeed; /* Legal fallback */ image-rendering:-moz-crisp-edges; /* Firefox */ @@ -24,12 +24,12 @@ image-rendering:-webkit-optimize-contrast; /* Chrome (and eventually Safari) */ image-rendering:crisp-edges; /* CSS3 Proposed */ -ms-interpolation-mode:bicubic; /* IE8+ */ - + &:hover, &:focus { border-color: rgba($text-color, .2); } - + &:active, &.selected { background-color: rgba($text-color, .1); @@ -58,6 +58,10 @@ text-align: right; } +.col-endpoint { + width: 200px +} + .col-key { width: 1%; } diff --git a/site-tasks/configuration/skip-insert-transaction.php b/site-tasks/configuration/skip-insert-transaction.php new file mode 100644 index 0000000..de6e64a --- /dev/null +++ b/site-tasks/configuration/skip-insert-transaction.php @@ -0,0 +1,24 @@ + 'Skip Transaction Inserts (Degradation Mode)', + 'description' => 'Enable or disable degredation mode which skips saving transactions to the DB. This functionality aids with the speed of requests with sites while under failure or high load', + 'icon' => 'power-off', + 'handler' => function () { + $flag = 'flags/gatekeeper/skip-insert-transaction'; + $ttl = $_REQUEST['ttl'] ?: Gatekeeper\ApiRequestHandler::$degradationTimeout; + + if ($_SERVER['REQUEST_METHOD'] == 'POST') { + if ($_REQUEST['status'] == 'enable') { + Cache::store($flag, true, $ttl); + } else { + Cache::delete($flag); + } + } + + return static::respond('skip-insert-transactions', [ + 'ttl' => $ttl, + 'enabled' => !!Cache::fetch($flag) + ]); + } +]; \ No newline at end of file
Endpoint Request Timestamp Response Code
{contextLink $Request->Endpoint} {$Request->Method} {$Request->Path|default:/}{tif $Request->Query ? "?$Request->Query"|query_string} {$Request->Created|date_format:'%Y-%m-%d %H:%M:%S'} {$Request->ResponseCode}