Merge branch 'main' into TT-9701-configure-mongodb-driver

components/tyk-bootstrap/values.yaml

@@ -82,7 +82,7 @@ bootstrap:
         # repository specifies image repository of pre-install job.
         repository: tykio/tyk-k8s-bootstrap-pre-install
         # tag specifies image tag of pre-install job.
-        tag: testing
+        tag: 1.5.0
 
     # postInstall configures post-install Helm hook
     postInstall:
@@ -91,7 +91,7 @@ bootstrap:
         # repository specifies image repository of post-install job.
         repository: tykio/tyk-k8s-bootstrap-post
         # tag specifies image tag of post-install job.
-        tag: testing
+        tag: 1.5.0
 
     # preDelete configures pre-delete Helm hook
     preDelete:
@@ -100,7 +100,7 @@ bootstrap:
         # repository specifies image repository of pre-delete job.
         repository: tykio/tyk-k8s-bootstrap-pre-delete
         # tag specifies image tag of pre-delete job.
-        tag: testing
+        tag: 1.5.0
 
   org:
     # The name for your organization inside Tyk

components/tyk-gateway/README.md

@@ -18,7 +18,7 @@ For typical usage, we recommend using following umbrella charts:
 ## Prerequisites
 * Kubernetes 1.19+
 * Helm 3+
-* [Redis](https://tyk.io/docs/planning-for-production/redis/) should already be installed or accessible by the gateway 
+* [Redis](https://tyk.io/docs/planning-for-production/redis/) should already be installed or accessible by the gateway
 
 ## Installing the Chart
 
@@ -27,7 +27,7 @@ To install the chart from the Helm repository in namespace `tyk` with the releas
     helm repo add tyk-helm https://helm.tyk.io/public/helm/charts/
     helm repo update
     helm show values tyk-helm/tyk-gateway > values.yaml --devel
-    
+
 Note: Set redis connection details first. See [Configuration](#configuration) below.
 
     helm install tyk-gateway tyk-helm/tyk-gateway -n tyk --create-namespace -f values.yaml --devel
@@ -49,7 +49,7 @@ Please see Migration notes in [tyk-oss](https://github.com/TykTechnologies/tyk-c
 See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To get all configurable options with detailed comments:
 
     helm show values tyk-helm/tyk-gateway > values.yaml --devel
-    
+
 You can update any value in your local values.yaml file and use `-f [filename]` flag to override default values during installation. Alternatively, you can use `--set` flag to set it in Tyk installation.
 
 ### Set Redis connection details (Required)
@@ -62,6 +62,33 @@ If you do not already have redis installed, you can use these charts provided by
 
 Follow the notes from the installation output to get connection details and password. The DNS name of your Redis as set by Bitnami is `tyk-redis-master.tyk.svc.cluster.local:6379` (Tyk needs the name including the port) You can update them in your local values.yaml file under `global.redis.addrs` and `global.redis.pass`. Alternatively, you can use `--set` flag to set it in Tyk installation. For example `--set global.redis.pass=$REDIS_PASSWORD`
 
+### Enable autoscaling
+
+This chart allows for easy configuration of autoscaling parameters. To simply enable autoscaling it's enough to add `--set gateway.autoscaling.enabled=true`. That will enable `Horizontal Pod Autoscaler` resource with default parameters (avg. CPU load at 60%, scaling between 1 and 3 instances). To customize those values you can add `--set gateway.autoscaling.averageCpuUtilization=75` or use `values.yaml` file:
+
+```yaml
+gateway:
+  autoscaling:
+    enabled: true
+    minReplicas: 3
+    maxReplicas: 30
+```
+
+Built-in rules include `gateway.autoscaling.averageCpuUtilization` for CPU utilization (set by default at 60%) and `gateway.autoscaling.averageMemoryUtilization` for memory (disabled by default). In addition to that you can define rules for custom metrics using `gateway.autoscaling.autoscalingTemplate` list:
+
+```yaml
+gateway:
+  autoscaling:
+    autoscalingTemplate:
+      - type: Pods
+        pods:
+          metric:
+            name: nginx_ingress_controller_nginx_process_requests_total
+          target:
+            type: AverageValue
+            averageValue: 10000m
+```
+
 ### Gateway Configurations
 
 #### Enabling TLS

components/tyk-gateway/templates/deployment-gw-repset.yaml

@@ -8,7 +8,7 @@ metadata:
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
 spec:
-{{- if eq .Values.gateway.kind "Deployment" }}
+{{- if and (eq .Values.gateway.kind "Deployment") (not .Values.gateway.autoscaling.enabled) }}
   replicas: {{ .Values.gateway.replicaCount }}
 {{- end }}
   minReadySeconds: 5

components/tyk-gateway/templates/hpa.yaml

@@ -0,0 +1,36 @@
+{{- if and (ne .Values.gateway.kind "DaemonSet") .Values.gateway.autoscaling.enabled }}
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: gateway-{{ include "tyk-gateway.fullname" . }}
+  labels:
+    app: gateway-{{ include "tyk-gateway.fullname" . }}
+    chart: {{ include "tyk-gateway.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: {{ .Values.gateway.kind }}
+    name: gateway-{{ include "tyk-gateway.fullname" . }}
+  minReplicas: {{ default 1 .Values.gateway.autoscaling.minReplicas }}
+  maxReplicas: {{ default 3 .Values.gateway.autoscaling.maxReplicas }}
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: {{ default 60 .Values.gateway.autoscaling.averageCpuUtilization }}
+{{- if .Values.gateway.autoscaling.averageMemoryUtilization }}
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.gateway.autoscaling.averageMemoryUtilization }}
+{{- end }}
+{{- with .Values.gateway.autoscaling.autoscalingTemplate }}
+{{ toYaml . | indent 4}}
+{{- end}}
+{{- end }}

components/tyk-gateway/values.yaml

@@ -149,6 +149,24 @@ gateway:
   # replicaCount specifies number of replicas to be created if kind is Deployment.
   replicaCount: 1
 
+  # autoscaling configuration if kind IS NOT DaemonSet
+  autoscaling: {}
+  #  enabled: true
+  #  minReplicas: 1
+  #  maxReplicas: 3
+  #  averageCpuUtilization: 60
+  #  averageMemoryUtilization: null
+  #  autoscalingTemplate:
+  #   Custom or additional autoscaling metrics
+  #   ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics
+  #    - type: Pods
+  #      pods:
+  #        metric:
+  #          name: nginx_ingress_controller_nginx_process_requests_total
+  #        target:
+  #          type: AverageValue
+  #          averageValue: 10000m
+
   image:
     # image repository for Tyk Gateway
     repository: docker.tyk.io/tyk-gateway/tyk-gateway

components/tyk-pump/README.md

@@ -61,14 +61,24 @@ You may set `global.redis.addr` and `global.redis.pass` with redis connection st
 
 ### Pump Configurations
 
-| Pump                      | Configuration                                                                                              |
-|---------------------------|------------------------------------------------------------------------------------------------------------| 
-| Prometheus Pump (Default) | Add `prometheus` to `pump.backend`, and add connection details for prometheus under `pump.prometheusPump`. |
-| Mongo Pump                | Add `mongo` to `pump.backend`, and add connection details for mongo under `.global.mongo`.                 |
-| SQL Pump                  | Add `postgres` to `.pump.backend`, and add connection details for postgres under `.global.postgres`.       |
-| Uptime Pump               | Set `pump.uptimePumpBackend` to `'mongo'` or `'postgres'` or `''`                                          |
-| Hybrid Pump               | Add `hybrid` to `.pump.backend`, and setup `.global.remoteControlPlane` section with the required adresses and tokens           |
-| Other Pumps               | Add the required environment variables in `pump.extraEnvs`                                                 |
+| Pump                      | Configuration                                                                                                         |
+|---------------------------|-----------------------------------------------------------------------------------------------------------------------| 
+| Prometheus Pump (Default) | Add the value `prometheus` to the `pump.backend` entry, and add connection details for Prometheus under `pump.prometheusPump`. |
+| Mongo Pump                | Add `mongo` to `pump.backend`, and add connection details for mongo under `global.mongo`.                             |
+| Mongo Selective Pump      | Add `mongo-selective` to `pump.backend`, and add connection details for mongo under `global.mongo`.                   |
+| Mongo Aggregate Pump      | Add `mongo-aggregate` to `pump.backend`, and add connection details for mongo under `global.mongo`.                   |
+| Postgres Pump             | Add `postgres` to `pump.backend`, and add connection details for postgres under `global.postgres`.                    |
+| Postgres Aggregate Pump   | Add `postgres-aggregate` to `pump.backend`, and add connection details for postgres under `global.postgres`.          |
+| Uptime Pump               | Set `pump.uptimePumpBackend` to `mongo` or `postgres` or `""`                                                         |
+| Hybrid Pump               | Add `hybrid` to `pump.backend`, and setup `global.remoteControlPlane` section with the required adresses and tokens   |
+| Other Pumps               | Add the required environment variables in `pump.extraEnvs`                                                            |
+
+
+> [!NOTE] 
+> For additional information on Tyk Pump configurations, 
+refer to the [Setup Dashboard Analytics](https://tyk.io/docs/tyk-pump/tyk-pump-configuration/tyk-pump-dashboard-config/) documentation. 
+
+> To explore the list of supported backends for Tyk Pump, please visit https://tyk.io/docs/tyk-stack/tyk-pump/other-data-stores/.
 
 #### Prometheus Pump
 Add `prometheus` to `pump.backend`, and add connection details for prometheus under `pump.prometheusPump`. 
@@ -77,7 +87,7 @@ We also support monitoring using Prometheus Operator. All you have to do is set
 This will create a PodMonitor resource for your Pump instance.
 
 #### Mongo Pump
-If you are using the MongoDB pumps in the tyk-oss installation you will require MongoDB installed for that as well.
+If you are using the MongoDB pumps in the `tyk-oss` installation you will require MongoDB installed for that as well.
 
 To install Mongo you can use these rather excellent charts provided by Bitnami:
 

components/tyk-pump/templates/deployment-pmp.yaml

@@ -94,7 +94,8 @@ spec:
             value: {{ toYaml .Values.pump.prometheusPump.customMetrics }}
             {{ end }}
           {{ end }}
-          {{ if has "postgres" .Values.pump.backend }}
+
+          {{ if or (has "postgres-pump" .Values.pump.backend) (has "postgres" .Values.pump.backend) }}
           # SQL Pump
           - name: TYK_PMP_PUMPS_SQL_TYPE
             value: "sql"
@@ -129,7 +130,8 @@ spec:
               secretKeyRef:
                 name: {{ include "tyk-pump.mongo_url_secret_name" . }}
                 key: {{ include "tyk-pump.mongo_url_secret_key" . }}
-
+          {{ end }}
+          {{ if or (has "mongo" .Values.pump.backend) (has "mongo-aggregate" .Values.pump.backend) }}
           # Mongo Aggregate Pump
           - name: TYK_PMP_PUMPS_MONGOAGGREGATE_TYPE
             value: "mongo-pump-aggregate"
@@ -145,6 +147,17 @@ spec:
           - name: TYK_PMP_PUMPS_MONGOAGGREGATE_META_USEMIXEDCOLLECTION
             value: "true"
           {{ end }}
+          {{ if or (has "mongo" .Values.pump.backend) (has "mongo-selective" .Values.pump.backend) }}
+          - name: TYK_PMP_PUMPS_MONGOSELECTIVE_TYPE
+            value: "mongo-pump-selective"
+          - name: TYK_PMP_PUMPS_MONGOSELECTIVE_META_MONGOURL
+            valueFrom:
+              secretKeyRef:
+                name: {{ include "tyk-pump.mongo_url_secret_name" . }}
+                key: {{ include "tyk-pump.mongo_url_secret_key" . }}
+          - name: TYK_PMP_PUMPS_MONGOSELECTIVE_META_MONGOUSESSL
+            value: "{{ default "false" .Values.global.mongo.useSSL }}"
+          {{ end }}
 
           # Uptime Pump configurations
           {{ if eq "disabled" (include "tyk-pump.uptimePump" .) }}
@@ -172,7 +185,7 @@ spec:
                 name: {{ include "tyk-pump.mongo_url_secret_name" . }}
                 key: {{ include "tyk-pump.mongo_url_secret_key" . }}
           - name: TYK_PMP_UPTIMEPUMPCONFIG_MONGOUSESSL
-            value: "{{ default "false" .Values.mongo.useSSL }}"
+            value: "{{ default "false" .Values.global.mongo.useSSL }}"
           {{ end }}
           {{ end }}
 

components/tyk-pump/values.yaml

@@ -9,7 +9,7 @@
 # Default value: tyk-pump.name
 nameOverride: ""
 
-# Overrides app name. IT is truncated to 63 characters.
+# Overrides app name. It is truncated to 63 characters.
 # Default value: tyk-pump.fullname
 fullnameOverride: ""
 
@@ -178,7 +178,15 @@ pump:
   containerPort: 9090
 
   # backend defines the pumps to be created by default, as an array of string.
-  # Supported backends are ["mongo", "postgres", "postgres-aggregate", "prometheus","hybrid"]
+  # Supported backends are:
+  #   - "mongo": Enables Mongo Aggregate and Mongo Selective Pump
+  #   - "mongo-aggregate": Enables ONLY Mongo Aggregate
+  #   - "mongo-selective": Enables ONLY Mongo Selective
+  #   - "postgres": Enables Postgres Aggregate and SQL Pump
+  #   - "postgres-aggregate": Enables ONLY SQL Aggregate
+  #   - "postgres-pump": Enables ONLY SQL Pump
+  #   - "prometheus": Enables Prometheus Pump. See pump.prometheusPump for Prometheus Pump configurations.
+  #   - "hybrid": Enables Hybrid Pump
   # If you would like to use other backends such as ElasticSearch, please
   # configure the backend via environment variables.
   backend:

tyk-mdcb-data-plane/README.md

@@ -1,6 +1,6 @@
 ## Tyk MDCB Data Plane
 
-> [!WARNING]  
+> [!WARNING]
 > To be renamed to tyk-data-plane
 
 `tyk-mdcb-data-plane` provides the default deployment of a Tyk data plane for Tyk Self Managed MDCB or Tyk Cloud users. It will deploy the data plane components that remotely connect to a MDCB control plane.
@@ -110,7 +110,7 @@ To get all configurable options with detailed comments:
 helm show values tyk-helm/tyk-mdcb-data-plane > values.yaml
 ```
 
-You can update any value in your local `values.yaml` file and use `-f [filename]` flag to override default values during installation. 
+You can update any value in your local `values.yaml` file and use `-f [filename]` flag to override default values during installation.
 Alternatively, you can use `--set` flag to set it in Tyk installation.
 
 ### Set Redis Connection Details (Required)
@@ -119,7 +119,7 @@ Tyk uses Redis for distributed rate-limiting and token storage. You may use the
 
 Set the following values after installing Redis:
 
-| Name | Description | 
+| Name | Description |
 |------|-------------|
 | `global.redis.addrs` | Redis addresses |
 | `global.redis.pass` | Redis password in plain text |
@@ -164,6 +164,35 @@ helm install redis tyk-helm/simple-redis -n tyk
 
 The Tyk Helm Chart can connect to `simple-redis` in the same namespace by default. You do not need to set Redis address and password in `values.yaml`.
 
+### Enable gateway autoscaling
+
+This chart allows for easy configuration of autoscaling parameters. To simply enable autoscaling it's enough to add `--set tyk-gateway.gateway.autoscaling.enabled=true`. That will enable `Horizontal Pod Autoscaler` resource with default parameters (avg. CPU load at 60%, scaling between 1 and 3 instances). To customize those values you can add `--set tyk-gateway.gateway.autoscaling.averageCpuUtilization=75` or use `values.yaml` file:
+
+```yaml
+tyk-gateway:
+  gateway:
+    autoscaling:
+      enabled: true
+      minReplicas: 3
+      maxReplicas: 30
+```
+
+Built-in rules include `tyk-gateway.gateway.autoscaling.averageCpuUtilization` for CPU utilization (set by default at 60%) and `tyk-gateway.gateway.autoscaling.averageMemoryUtilization` for memory (disabled by default). In addition to that you can define rules for custom metrics using `tyk-gateway.gateway.autoscaling.autoscalingTemplate` list:
+
+```yaml
+tyk-gateway:
+  gateway:
+    autoscaling:
+      autoscalingTemplate:
+        - type: Pods
+          pods:
+            metric:
+              name: nginx_ingress_controller_nginx_process_requests_total
+            target:
+              type: AverageValue
+              averageValue: 10000m
+```
+
 ### Gateway Configurations
 
 Configure below inside `tyk-gateway` section.
@@ -192,11 +221,11 @@ If you want to use your own key/cert pair, you must follow the following steps:
 To add your custom Certificate Authority(CA) to your containers, you can mount your CA certificate directly into /etc/ssl/certs folder.
 
 ```yaml
-   extraVolumes: 
+   extraVolumes:
      - name: self-signed-ca
        secret:
          secretName: self-signed-ca-secret
-   extraVolumeMounts: 
+   extraVolumeMounts:
      - name: self-signed-ca
        mountPath: "/etc/ssl/certs/myCA.pem"
        subPath: myCA.pem
@@ -270,14 +299,14 @@ Here is a reference of all [Tyk Gateway Configuration Options](https://tyk.io/do
 
 To enable Pump, set `global.components.pump` to true, and configure below inside `tyk-pump` section.
 
-| Pump                      | Configuration                                                                                              |
-|---------------------------|------------------------------------------------------------------------------------------------------------| 
-| Prometheus Pump (Default) | Add `prometheus` to `tyk-pump.pump.backend`, and add connection details for prometheus under `tyk-pump.pump.prometheusPump`. |
+| Pump                      | Configuration                                                                                                                                                                          |
+|---------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Prometheus Pump (Default) | Add `prometheus` to `tyk-pump.pump.backend`, and add connection details for prometheus under `tyk-pump.pump.prometheusPump`.                                                           |
 | Hybrid Pump (Default)     | Add `hybrid` to `tyk-pump.pump.backend`, and add remoteControlPlane details under `global.remoteControlPlane`. Change `tyk-gateway.gateway.analyticsConfigType` to `""` (empty string) |
-| Other Pumps               | Add the required environment variables in `tyk-pump.pump.extraEnvs`                                                |
+| Other Pumps               | Add the required environment variables in `tyk-pump.pump.extraEnvs`                                                                                                                    |
 
 #### Prometheus Pump
-Add `prometheus` to `tyk-pump.pump.backend`, and add connection details for prometheus under `tyk-pump.pump.prometheusPump`. 
+Add `prometheus` to `tyk-pump.pump.backend`, and add connection details for prometheus under `tyk-pump.pump.prometheusPump`.
 
 We also support monitoring using Prometheus Operator. All you have to do is set `tyk-pump.pump.prometheusPump.prometheusOperator.enabled` to true.
 This will create a *PodMonitor* resource for your Pump instance.
@@ -305,7 +334,7 @@ Add `hybrid` to `tyk-pump.pump.backend`, and add remoteControlPlane details unde
 ```yaml
   # hybridPump configures Tyk Pump to forward Tyk metrics to a Tyk Control Plane.
   # Please add "hybrid" to .Values.pump.backend in order to enable Hybrid Pump.
-  hybridPump: 
+  hybridPump:
     # Specify the frequency of the aggregation in minutes or simply turn it on by setting it to true
     enableAggregateAnalytics: true
     # Hybrid pump RPC calls timeout in seconds.