From 472f5f9207eacc43c0f00eee5cbe6135fb270f41 Mon Sep 17 00:00:00 2001 From: Burak Sekili Date: Tue, 24 Oct 2023 14:08:33 +0300 Subject: [PATCH] [TT-9703] - Add new options to enable specific pumps (#126) * Add new options to enable specific pumps Signed-off-by: Burak Sekili * Update comments Signed-off-by: Burak Sekili * add new options to readme and enable mongo-selective when selected Signed-off-by: Burak Sekili * Update pump-docs; rename sql to postgres Signed-off-by: Burak Sekili * Add a reference link to official Tyk docs about Tyk Pump configurations Signed-off-by: Burak Sekili * Update docs. Remove inconsistent naming reference. Signed-off-by: Burak Sekili * Add missing phrase in tyk-pump docs Signed-off-by: Burak Sekili * Update values.yaml * Fix mongo uptime Signed-off-by: Burak Sekili --------- Signed-off-by: Burak Sekili --- components/tyk-bootstrap/values.yaml | 6 ++-- components/tyk-pump/README.md | 28 +++++++++++++------ .../tyk-pump/templates/deployment-pmp.yaml | 19 +++++++++++-- components/tyk-pump/values.yaml | 12 ++++++-- tyk-mdcb-data-plane/README.md | 8 +++--- tyk-mdcb-data-plane/values.yaml | 10 ++++++- tyk-oss/README.md | 24 +++++++++++----- tyk-oss/values.yaml | 11 ++++++-- tyk-single-dc/README.md | 23 ++++++++++----- tyk-single-dc/values.yaml | 11 ++++++-- 10 files changed, 112 insertions(+), 40 deletions(-) diff --git a/components/tyk-bootstrap/values.yaml b/components/tyk-bootstrap/values.yaml index 9d460150..97f4e96c 100644 --- a/components/tyk-bootstrap/values.yaml +++ b/components/tyk-bootstrap/values.yaml @@ -82,7 +82,7 @@ bootstrap: # repository specifies image repository of pre-install job. repository: tykio/tyk-k8s-bootstrap-pre-install # tag specifies image tag of pre-install job. - tag: testing + tag: 1.5.0 # postInstall configures post-install Helm hook postInstall: @@ -91,7 +91,7 @@ bootstrap: # repository specifies image repository of post-install job. repository: tykio/tyk-k8s-bootstrap-post # tag specifies image tag of post-install job. - tag: testing + tag: 1.5.0 # preDelete configures pre-delete Helm hook preDelete: @@ -100,7 +100,7 @@ bootstrap: # repository specifies image repository of pre-delete job. repository: tykio/tyk-k8s-bootstrap-pre-delete # tag specifies image tag of pre-delete job. - tag: testing + tag: 1.5.0 org: # The name for your organization inside Tyk diff --git a/components/tyk-pump/README.md b/components/tyk-pump/README.md index 765185d6..0177afbf 100644 --- a/components/tyk-pump/README.md +++ b/components/tyk-pump/README.md @@ -61,14 +61,24 @@ You may set `global.redis.addr` and `global.redis.pass` with redis connection st ### Pump Configurations -| Pump | Configuration | -|---------------------------|------------------------------------------------------------------------------------------------------------| -| Prometheus Pump (Default) | Add `prometheus` to `pump.backend`, and add connection details for prometheus under `pump.prometheusPump`. | -| Mongo Pump | Add `mongo` to `pump.backend`, and add connection details for mongo under `.global.mongo`. | -| SQL Pump | Add `postgres` to `.pump.backend`, and add connection details for postgres under `.global.postgres`. | -| Uptime Pump | Set `pump.uptimePumpBackend` to `'mongo'` or `'postgres'` or `''` | -| Hybrid Pump | Add `hybrid` to `.pump.backend`, and setup `.global.remoteControlPlane` section with the required adresses and tokens | -| Other Pumps | Add the required environment variables in `pump.extraEnvs` | +| Pump | Configuration | +|---------------------------|-----------------------------------------------------------------------------------------------------------------------| +| Prometheus Pump (Default) | Add the value `prometheus` to the `pump.backend` entry, and add connection details for Prometheus under `pump.prometheusPump`. | +| Mongo Pump | Add `mongo` to `pump.backend`, and add connection details for mongo under `global.mongo`. | +| Mongo Selective Pump | Add `mongo-selective` to `pump.backend`, and add connection details for mongo under `global.mongo`. | +| Mongo Aggregate Pump | Add `mongo-aggregate` to `pump.backend`, and add connection details for mongo under `global.mongo`. | +| Postgres Pump | Add `postgres` to `pump.backend`, and add connection details for postgres under `global.postgres`. | +| Postgres Aggregate Pump | Add `postgres-aggregate` to `pump.backend`, and add connection details for postgres under `global.postgres`. | +| Uptime Pump | Set `pump.uptimePumpBackend` to `mongo` or `postgres` or `""` | +| Hybrid Pump | Add `hybrid` to `pump.backend`, and setup `global.remoteControlPlane` section with the required adresses and tokens | +| Other Pumps | Add the required environment variables in `pump.extraEnvs` | + + +> [!NOTE] +> For additional information on Tyk Pump configurations, +refer to the [Setup Dashboard Analytics](https://tyk.io/docs/tyk-pump/tyk-pump-configuration/tyk-pump-dashboard-config/) documentation. + +> To explore the list of supported backends for Tyk Pump, please visit https://tyk.io/docs/tyk-stack/tyk-pump/other-data-stores/. #### Prometheus Pump Add `prometheus` to `pump.backend`, and add connection details for prometheus under `pump.prometheusPump`. @@ -77,7 +87,7 @@ We also support monitoring using Prometheus Operator. All you have to do is set This will create a PodMonitor resource for your Pump instance. #### Mongo Pump -If you are using the MongoDB pumps in the tyk-oss installation you will require MongoDB installed for that as well. +If you are using the MongoDB pumps in the `tyk-oss` installation you will require MongoDB installed for that as well. To install Mongo you can use these rather excellent charts provided by Bitnami: diff --git a/components/tyk-pump/templates/deployment-pmp.yaml b/components/tyk-pump/templates/deployment-pmp.yaml index 8dc41244..a17d0e7f 100644 --- a/components/tyk-pump/templates/deployment-pmp.yaml +++ b/components/tyk-pump/templates/deployment-pmp.yaml @@ -94,7 +94,8 @@ spec: value: {{ toYaml .Values.pump.prometheusPump.customMetrics }} {{ end }} {{ end }} - {{ if has "postgres" .Values.pump.backend }} + + {{ if or (has "postgres-pump" .Values.pump.backend) (has "postgres" .Values.pump.backend) }} # SQL Pump - name: TYK_PMP_PUMPS_SQL_TYPE value: "sql" @@ -129,7 +130,8 @@ spec: secretKeyRef: name: {{ include "tyk-pump.mongo_url_secret_name" . }} key: {{ include "tyk-pump.mongo_url_secret_key" . }} - + {{ end }} + {{ if or (has "mongo" .Values.pump.backend) (has "mongo-aggregate" .Values.pump.backend) }} # Mongo Aggregate Pump - name: TYK_PMP_PUMPS_MONGOAGGREGATE_TYPE value: "mongo-pump-aggregate" @@ -145,6 +147,17 @@ spec: - name: TYK_PMP_PUMPS_MONGOAGGREGATE_META_USEMIXEDCOLLECTION value: "true" {{ end }} + {{ if or (has "mongo" .Values.pump.backend) (has "mongo-selective" .Values.pump.backend) }} + - name: TYK_PMP_PUMPS_MONGOSELECTIVE_TYPE + value: "mongo-pump-selective" + - name: TYK_PMP_PUMPS_MONGOSELECTIVE_META_MONGOURL + valueFrom: + secretKeyRef: + name: {{ include "tyk-pump.mongo_url_secret_name" . }} + key: {{ include "tyk-pump.mongo_url_secret_key" . }} + - name: TYK_PMP_PUMPS_MONGOSELECTIVE_META_MONGOUSESSL + value: "{{ default "false" .Values.global.mongo.useSSL }}" + {{ end }} # Uptime Pump configurations {{ if eq "disabled" (include "tyk-pump.uptimePump" .) }} @@ -172,7 +185,7 @@ spec: name: {{ include "tyk-pump.mongo_url_secret_name" . }} key: {{ include "tyk-pump.mongo_url_secret_key" . }} - name: TYK_PMP_UPTIMEPUMPCONFIG_MONGOUSESSL - value: "{{ default "false" .Values.mongo.useSSL }}" + value: "{{ default "false" .Values.global.mongo.useSSL }}" {{ end }} {{ end }} diff --git a/components/tyk-pump/values.yaml b/components/tyk-pump/values.yaml index 8d78b914..3c63cf9a 100644 --- a/components/tyk-pump/values.yaml +++ b/components/tyk-pump/values.yaml @@ -9,7 +9,7 @@ # Default value: tyk-pump.name nameOverride: "" -# Overrides app name. IT is truncated to 63 characters. +# Overrides app name. It is truncated to 63 characters. # Default value: tyk-pump.fullname fullnameOverride: "" @@ -171,7 +171,15 @@ pump: containerPort: 9090 # backend defines the pumps to be created by default, as an array of string. - # Supported backends are ["mongo", "postgres", "postgres-aggregate", "prometheus","hybrid"] + # Supported backends are: + # - "mongo": Enables Mongo Aggregate and Mongo Selective Pump + # - "mongo-aggregate": Enables ONLY Mongo Aggregate + # - "mongo-selective": Enables ONLY Mongo Selective + # - "postgres": Enables Postgres Aggregate and SQL Pump + # - "postgres-aggregate": Enables ONLY SQL Aggregate + # - "postgres-pump": Enables ONLY SQL Pump + # - "prometheus": Enables Prometheus Pump. See pump.prometheusPump for Prometheus Pump configurations. + # - "hybrid": Enables Hybrid Pump # If you would like to use other backends such as ElasticSearch, please # configure the backend via environment variables. backend: diff --git a/tyk-mdcb-data-plane/README.md b/tyk-mdcb-data-plane/README.md index 657b1516..a3611365 100644 --- a/tyk-mdcb-data-plane/README.md +++ b/tyk-mdcb-data-plane/README.md @@ -299,11 +299,11 @@ Here is a reference of all [Tyk Gateway Configuration Options](https://tyk.io/do To enable Pump, set `global.components.pump` to true, and configure below inside `tyk-pump` section. -| Pump | Configuration | -|---------------------------|------------------------------------------------------------------------------------------------------------| -| Prometheus Pump (Default) | Add `prometheus` to `tyk-pump.pump.backend`, and add connection details for prometheus under `tyk-pump.pump.prometheusPump`. | +| Pump | Configuration | +|---------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Prometheus Pump (Default) | Add `prometheus` to `tyk-pump.pump.backend`, and add connection details for prometheus under `tyk-pump.pump.prometheusPump`. | | Hybrid Pump (Default) | Add `hybrid` to `tyk-pump.pump.backend`, and add remoteControlPlane details under `global.remoteControlPlane`. Change `tyk-gateway.gateway.analyticsConfigType` to `""` (empty string) | -| Other Pumps | Add the required environment variables in `tyk-pump.pump.extraEnvs` | +| Other Pumps | Add the required environment variables in `tyk-pump.pump.extraEnvs` | #### Prometheus Pump Add `prometheus` to `tyk-pump.pump.backend`, and add connection details for prometheus under `tyk-pump.pump.prometheusPump`. diff --git a/tyk-mdcb-data-plane/values.yaml b/tyk-mdcb-data-plane/values.yaml index 4f174007..9374becd 100644 --- a/tyk-mdcb-data-plane/values.yaml +++ b/tyk-mdcb-data-plane/values.yaml @@ -387,7 +387,15 @@ tyk-pump: containerPort: 9090 # backend defines the pumps to be created by default, as an array of string. - # Supported backends are ["mongo", "postgres", "prometheus","hybrid"] + # Supported backends are: + # - "mongo": Enables Mongo Aggregate and Mongo Selective Pump + # - "mongo-aggregate": Enables ONLY Mongo Aggregate + # - "mongo-selective": Enables ONLY Mongo Selective + # - "postgres": Enables Postgres Aggregate and SQL Pump + # - "postgres-aggregate": Enables ONLY SQL Aggregate + # - "postgres-pump": Enables ONLY SQL Pump + # - "prometheus": Enables Prometheus Pump. See pump.prometheusPump for Prometheus Pump configurations. + # - "hybrid": Enables Hybrid Pump # If you would like to use other backends such as ElasticSearch, please # configure the backend via environment variables. backend: diff --git a/tyk-oss/README.md b/tyk-oss/README.md index d0b07454..098c5f67 100644 --- a/tyk-oss/README.md +++ b/tyk-oss/README.md @@ -298,13 +298,23 @@ Here is a reference of all [Tyk Gateway Configuration Options](https://tyk.io/do To enable Pump, set `global.components.pump` to true, and configure below inside `tyk-pump` section. -| Pump | Configuration | -|---------------------------|------------------------------------------------------------------------------------------------------------| -| Prometheus Pump (Default) | Add `prometheus` to `tyk-pump.pump.backend`, and add connection details for prometheus under `tyk-pump.pump.prometheusPump`. | -| Mongo Pump | Add `mongo` to `tyk-pump.pump.backend`, and add connection details for mongo under `global.mongo`. | -| SQL Pump | Add `postgres` to `tyk-pump.pump.backend`, and add connection details for postgres under `global.postgres`. | -| Uptime Pump | Set `tyk-pump.pump.uptimePumpBackend` to `'mongo'` or `'postgres'` | -| Other Pumps | Add the required environment variables in `tyk-pump.pump.extraEnvs` | + +| Pump | Configuration | +|---------------------------|-------------------------------------------------------------------------------------------------------------------------------| +| Prometheus Pump (Default) | Add the value `prometheus` to `tyk-pump.pump.backend`, and add connection details for prometheus under `tyk-pump.pump.prometheusPump`. | +| Mongo Pump | Add `mongo` to `tyk-pump.pump.backend`, and add connection details for mongo under `global.mongo`. | +| Mongo Selective Pump | Add `mongo-selective` to `pump.backend`, and add connection details for mongo under `.global.mongo`. | +| Mongo Aggregate Pump | Add `mongo-aggregate` to `pump.backend`, and add connection details for mongo under `.global.mongo`. | +| Postgres Pump | Add `postgres` to `tyk-pump.pump.backend`, and add connection details for postgres under `global.postgres`. | +| Postgres Aggregate Pump | Add `postgres-aggregate` to `.pump.backend`, and add connection details for postgres under `.global.postgres`. | +| Uptime Pump | Set `tyk-pump.pump.uptimePumpBackend` to `'mongo'` or `'postgres'` | +| Other Pumps | Add the required environment variables in `tyk-pump.pump.extraEnvs` | + +> [!NOTE] +> For additional information on Tyk Pump configurations, refer to the +[Setup Dashboard Analytics](https://tyk.io/docs/tyk-pump/tyk-pump-configuration/tyk-pump-dashboard-config/) documentation. + +> To explore the list of supported backends for Tyk Pump, please visit https://tyk.io/docs/tyk-stack/tyk-pump/other-data-stores/. #### Prometheus Pump Add `prometheus` to `tyk-pump.pump.backend`, and add connection details for prometheus under `tyk-pump.pump.prometheusPump`. diff --git a/tyk-oss/values.yaml b/tyk-oss/values.yaml index 57eeb2d7..ea875ad7 100644 --- a/tyk-oss/values.yaml +++ b/tyk-oss/values.yaml @@ -349,10 +349,17 @@ tyk-pump: containerPort: 9090 # backend defines the pumps to be created by default, as an array of string. - # Supported backends are ["mongo", "postgres", "prometheus"] + # Supported backends are: + # - "mongo": Enables Mongo Aggregate and Mongo Selective Pump + # - "mongo-aggregate": Enables ONLY Mongo Aggregate + # - "mongo-selective": Enables ONLY Mongo Selective + # - "postgres": Enables Postgres Aggregate and SQL Pump + # - "postgres-aggregate": Enables ONLY SQL Aggregate + # - "postgres-pump": Enables ONLY SQL Pump + # - "prometheus": Enables Prometheus Pump. See pump.prometheusPump for Prometheus Pump configurations. + # - "hybrid": Enables Hybrid Pump # If you would like to use other backends such as ElasticSearch, please # configure the backend via environment variables. - # By default, Tyk Pump uses Prometheus. backend: - "prometheus" diff --git a/tyk-single-dc/README.md b/tyk-single-dc/README.md index 08ba1514..fc120d7c 100644 --- a/tyk-single-dc/README.md +++ b/tyk-single-dc/README.md @@ -165,13 +165,22 @@ To enable Pump, set `global.components.pump` to true, and configure below inside -| Pump | Configuration | -|---------------------------|------------------------------------------------------------------------------------------------------------| -| Prometheus Pump (Default) | Add `prometheus` to `pump.backend`, and add connection details for prometheus under `pump.prometheusPump`. | -| Mongo Pump | Add `mongo` to `pump.backend`, and add connection details for mongo under `.mongo`. | -| SQL Pump | Add `postgres` to `pump.backend`, and add connection details for postgres under `.postgres`. | -| Uptime Pump | Set `pump.uptimePumpBackend` to `'mongo'` or `'postgres'` or `''` | -| Other Pumps | Add the required environment variables in `pump.extraEnvs` | +| Pump | Configuration | +|---------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------| +| Prometheus Pump (Default) | Add the value `prometheus` to the `tyk-pump.pump.backend` entry, and add connection details for Prometheus under `tyk-pump.pump.prometheusPump`. | +| Mongo Pump | Add `mongo` to `tyk-pump.pump.backend`, and add connection details for mongo under `global.mongo`. | +| Mongo Selective Pump | Add `mongo-selective` to `tyk-pump.pump.backend`, and add connection details for mongo under `global.mongo`. | +| Mongo Aggregate Pump | Add `mongo-aggregate` to `tyk-pump.pump.backend`, and add connection details for mongo under `global.mongo`. | +| Postgres Pump | Add `postgres` to `tyk-pump.pump.backend`, and add connection details for postgres under `global.postgres`. | +| Postgres Aggregate Pump | Add `postgres-aggregate` to `tyk-pump.pump.backend`, and add connection details for postgres under `global.postgres`. | +| Uptime Pump | Set `tyk-pump.pump.uptimePumpBackend` to `mongo` or `postgres` or `""` | +| Other Pumps | Add the required environment variables in `tyk-pump.pump.extraEnvs` | + +> [!NOTE] +> For additional information on Tyk Pump configurations, refer to the +[Setup Dashboard Analytics](https://tyk.io/docs/tyk-pump/tyk-pump-configuration/tyk-pump-dashboard-config/) documentation. + +> To explore the list of supported backends for Tyk Pump, please visit https://tyk.io/docs/tyk-stack/tyk-pump/other-data-stores/. #### Prometheus Pump Add `prometheus` to `pump.backend`, and add connection details for prometheus under `pump.prometheusPump`. diff --git a/tyk-single-dc/values.yaml b/tyk-single-dc/values.yaml index 179055cf..5a58d623 100644 --- a/tyk-single-dc/values.yaml +++ b/tyk-single-dc/values.yaml @@ -458,10 +458,17 @@ tyk-pump: containerPort: 9090 # backend defines the pumps to be created by default, as an array of string. - # Supported backends are ["mongo", "postgres", "prometheus"] + # Supported backends are: + # - "mongo": Enables Mongo Aggregate and Mongo Selective Pump + # - "mongo-aggregate": Enables ONLY Mongo Aggregate + # - "mongo-selective": Enables ONLY Mongo Selective + # - "postgres": Enables Postgres Aggregate and SQL Pump + # - "postgres-aggregate": Enables ONLY SQL Aggregate + # - "postgres-pump": Enables ONLY SQL Pump + # - "prometheus": Enables Prometheus Pump. See pump.prometheusPump for Prometheus Pump configurations. + # - "hybrid": Enables Hybrid Pump # If you would like to use other backends such as ElasticSearch, please # configure the backend via environment variables. - # By default, Tyk Pump uses Prometheus. backend: - "prometheus" - *globalStorageType