Role for auditing-timescaledb. (#310)

control-plane/roles/auditing-timescaledb/README.md

@@ -0,0 +1,9 @@
+# auditing-timescaledb
+
+This role provides a database for the metal-api that can be used for storing audit traces. The auditing feature has to be explicitly enabled in the metal-api in order to make use of this database.
+
+This role just wraps the [postgres-backup-restore](/control-plane/roles/postgres-backup-restore) role. Refer to this role for further documentation.
+
+## Variables
+
+The role should take the same variables as the wrapped role, but prefixed with `auditing_timescaledb_` instead of `postgres_`.

control-plane/roles/auditing-timescaledb/defaults/main/control-plane-defaults

@@ -0,0 +1 @@
+../../../../control-plane-defaults/

control-plane/roles/auditing-timescaledb/defaults/main/global-defaults

@@ -0,0 +1 @@
+../../../../../defaults

control-plane/roles/auditing-timescaledb/defaults/main/main.yaml

@@ -0,0 +1,44 @@
+---
+auditing_timescaledb_name: auditing-timescaledb
+auditing_timescaledb_namespace: "{{ metal_control_plane_namespace }}"
+
+auditing_timescaledb_image_pull_policy: "{{ metal_control_plane_image_pull_policy }}"
+
+auditing_timescaledb_storage_size: 10Gi
+auditing_timescaledb_storage_class:
+auditing_timescaledb_db: auditing
+auditing_timescaledb_user: postgres
+auditing_timescaledb_password: change-me
+auditing_timescaledb_max_connections: 100
+
+auditing_timescaledb_shared_libraries_preload:
+  - pg_stat_statements
+  - timescaledb
+
+auditing_timescaledb_backup_restore_sidecar_image_pull_policy: "{{ metal_control_plane_image_pull_policy }}"
+auditing_timescaledb_backup_restore_sidecar_provider: local
+auditing_timescaledb_backup_restore_sidecar_backup_cron_schedule: "0 * * * *"
+auditing_timescaledb_backup_restore_sidecar_log_level: debug
+auditing_timescaledb_backup_restore_sidecar_object_prefix: "{{ auditing_timescaledb_name }}-{{ metal_control_plane_stage_name }}"
+auditing_timescaledb_backup_restore_sidecar_object_max_keep:
+
+auditing_timescaledb_backup_restore_sidecar_gcp_bucket_name:
+auditing_timescaledb_backup_restore_sidecar_gcp_backup_location:
+auditing_timescaledb_backup_restore_sidecar_gcp_project_id:
+auditing_timescaledb_backup_restore_sidecar_gcp_serviceaccount_json:
+
+auditing_timescaledb_resources:
+  requests:
+    memory: "256Mi"
+    cpu: "500m"
+  limits:
+    memory: "1Gi"
+    cpu: "1"
+
+auditing_timescaledb_registry_auth_enabled: "{{ metal_registry_auth_enabled }}"
+auditing_timescaledb_registry_auth:
+  auths:
+    https://index.docker.io/v1/:
+      username: "{{ metal_registry_auth_user }}"
+      password: "{{ metal_registry_auth_password }}"
+      auth: "{{ (metal_registry_auth_user + ':' + metal_registry_auth_password) | b64encode }}"

control-plane/roles/auditing-timescaledb/tasks/main.yaml

@@ -0,0 +1,55 @@
+---
+- name: Gather release versions
+  setup_yaml:
+
+- name: Check mandatory variables for this role are set
+  assert:
+    fail_msg: "not all mandatory variables given, check role documentation"
+    quiet: yes
+    that:
+      - auditing_timescaledb_image_name is defined
+      - auditing_timescaledb_image_tag is defined
+      - auditing_timescaledb_backup_restore_sidecar_image_name is defined
+      - auditing_timescaledb_backup_restore_sidecar_image_tag is defined
+
+- name: Create namespace
+  k8s:
+    definition:
+      apiVersion: v1
+      kind: Namespace
+      metadata:
+        name: "{{ auditing_timescaledb_namespace }}"
+        labels:
+          name: "{{ auditing_timescaledb_namespace }}"
+
+- name: Deploy auditing timescale db
+  include_role:
+    name: metal-roles/control-plane/roles/postgres-backup-restore
+  vars:
+    postgres_name: "{{ auditing_timescaledb_name }}"
+    postgres_namespace: "{{ auditing_timescaledb_namespace }}"
+    postgres_image_pull_policy: "{{ auditing_timescaledb_image_pull_policy }}"
+    postgres_image_name: "{{ auditing_timescaledb_image_name }}"
+    postgres_image_tag: "{{ auditing_timescaledb_image_tag }}"
+    postgres_registry_auth_enabled: "{{ auditing_timescaledb_registry_auth_enabled }}"
+    postgres_registry_auth: "{{ auditing_timescaledb_registry_auth }}"
+    postgres_storage_size: "{{ auditing_timescaledb_storage_size }}"
+    postgres_storage_class: "{{ auditing_timescaledb_storage_class }}"
+    postgres_db: "{{ auditing_timescaledb_db }}"
+    postgres_user: "{{ auditing_timescaledb_user }}"
+    postgres_password: "{{ auditing_timescaledb_password }}"
+    postgres_max_connections: "{{ auditing_timescaledb_max_connections }}"
+    postgres_shared_libraries_preload: "{{ auditing_timescaledb_shared_libraries_preload }}"
+    postgres_backup_restore_sidecar_image_pull_policy: "{{ auditing_timescaledb_backup_restore_sidecar_image_pull_policy }}"
+    postgres_backup_restore_sidecar_image_name: "{{ auditing_timescaledb_backup_restore_sidecar_image_name }}"
+    postgres_backup_restore_sidecar_image_tag: "{{ auditing_timescaledb_backup_restore_sidecar_image_tag }}"
+    postgres_backup_restore_sidecar_provider: "{{ auditing_timescaledb_backup_restore_sidecar_provider }}"
+    postgres_backup_restore_sidecar_backup_cron_schedule: "{{ auditing_timescaledb_backup_restore_sidecar_backup_cron_schedule }}"
+    postgres_backup_restore_sidecar_log_level: "{{ auditing_timescaledb_backup_restore_sidecar_log_level }}"
+    postgres_backup_restore_sidecar_object_prefix: "{{ auditing_timescaledb_backup_restore_sidecar_object_prefix }}"
+    postgres_backup_restore_sidecar_gcp_bucket_name: "{{ auditing_timescaledb_backup_restore_sidecar_gcp_bucket_name }}"
+    postgres_backup_restore_sidecar_gcp_backup_location: "{{ auditing_timescaledb_backup_restore_sidecar_gcp_backup_location }}"
+    postgres_backup_restore_sidecar_gcp_project_id: "{{ auditing_timescaledb_backup_restore_sidecar_gcp_project_id }}"
+    postgres_backup_restore_sidecar_gcp_serviceaccount_json: "{{ auditing_timescaledb_backup_restore_sidecar_gcp_serviceaccount_json }}"
+    postgres_resources: "{{ auditing_timescaledb_resources }}"
+    postgres_backup_restore_sidecar_object_max_keep: "{{ auditing_timescaledb_backup_restore_sidecar_object_max_keep }}"

control-plane/roles/metal/README.md

@@ -154,11 +154,19 @@ You can look up all the default values of this role [here](defaults/main/main.ya
 
 ### Auditing
 
-| Name                             | Mandatory | Description                                                                  |
-|----------------------------------|-----------|------------------------------------------------------------------------------|
-| metal_auditing_enabled           |           | Whether to deploy or not to deploy the auditing. Default false.              |
-| metal_auditing_url               |           | The URL of the auditing server (required if enabled)                         |
-| metal_auditing_index_prefix      |           | auditing index prefix.                                                       |
-| metal_auditing_index_interval    |           | auditing index creation interval, can be one of @hourly / @daily / @monthly. |
-| metal_auditing_meili_secret_name |           | Secret name that holds the API key for meilisearch                           |
-| metal_auditing_meili_api_key     |           | API key for meilisearch                                                      |
+| Name                                 | Mandatory | Description                                                                  |
+| ------------------------------------ | --------- | ---------------------------------------------------------------------------- |
+| metal_auditing_meili_enabled         |           | Whether to deploy or not to configure meilisearch auditing. Default false.   |
+| metal_auditing_meili_url             |           | The URL of the auditing server (required if enabled)                         |
+| metal_auditing_meili_index_prefix    |           | auditing index prefix.                                                       |
+| metal_auditing_meili_index_interval  |           | auditing index creation interval, can be one of @hourly / @daily / @monthly. |
+| metal_auditing_meili_secret_name     |           | Secret name that holds the API key for meilisearch                           |
+| metal_auditing_meili_api_key         |           | API key for meilisearch                                                      |
+| metal_auditing_timescaledb_enabled   |           | Whether to deploy or not to configure timescaledb auditing. Default false.   |
+| metal_auditing_timescaledb_host      |           | The timescaledb host                                                         |
+| metal_auditing_timescaledb_port      |           | The timescaledb port                                                         |
+| metal_auditing_timescaledb_db        |           | The timescaledb database name                                                |
+| metal_auditing_timescaledb_user      |           | The timescaledb user                                                         |
+| metal_auditing_timescaledb_password  |           | The timescaledb password                                                     |
+| metal_auditing_timescaledb_retention |           | The timescaledb retention period, only configurable at first startup         |
+| metal_auditing_search_backend        |           | Explicitly sets a configured audit backend to be used for search             |

control-plane/roles/metal/defaults/main/main.yaml

@@ -113,8 +113,18 @@ metal_api_headscale_control_plane_address: "http{{ 's' if metal_api_headscale_tl
 metal_api_headscale_internal_api_address: "headscale:50443"
 
 # auditing
-metal_auditing_enabled: false
-metal_auditing_index_prefix: "auditing"
-metal_auditing_index_interval: "@daily"
-metal_auditing_url: "http://auditing-meili.{{ auditing_meili_namespace if auditing_meili_namespace is defined else metal_control_plane_namespace }}.svc.cluster.local:7700"
-metal_auditing_meili_api_key: "{{ lookup('k8s', api_version='v1', namespace=auditing_meili_namespace if auditing_meili_namespace is defined else metal_control_plane_namespace, kind='Secret', resource_name='auditing-meili').get('data', {}).get('MEILI_MASTER_KEY') | b64decode if metal_auditing_enabled else '' }}"
+metal_auditing_meili_enabled: false
+metal_auditing_meili_index_prefix: "auditing"
+metal_auditing_meili_index_interval: "@daily"
+metal_auditing_meili_url: "http://auditing-meili.{{ auditing_meili_namespace if auditing_meili_namespace is defined else metal_control_plane_namespace }}.svc.cluster.local:7700"
+metal_auditing_meili_api_key: "{{ lookup('k8s', api_version='v1', namespace=auditing_meili_namespace if auditing_meili_namespace is defined else metal_control_plane_namespace, kind='Secret', resource_name='auditing-meili').get('data', {}).get('MEILI_MASTER_KEY') | b64decode if metal_auditing_meili_enabled else '' }}"
+
+metal_auditing_timescaledb_enabled: false
+metal_auditing_timescaledb_host: "auditing-timescaledb"
+metal_auditing_timescaledb_port: "5432"
+metal_auditing_timescaledb_db: "auditing"
+metal_auditing_timescaledb_user: "postgres"
+metal_auditing_timescaledb_password: "change-me"
+metal_auditing_timescaledb_retention: "14 days"
+
+metal_auditing_search_backend:

control-plane/roles/metal/templates/metal-values.j2

@@ -216,8 +216,23 @@ deploy_ingress: {{ metal_deploy_ingress }}
 ingress: {{ metal_ingress | to_json }}
 
 auditing:
-  enabled: {{ metal_auditing_enabled }}
-  index_prefix: "{{ metal_auditing_index_prefix }}"
-  index_interval: "{{ metal_auditing_index_interval }}"
-  existingMasterKeySecret: "metal-auditing-meili-api-key"
-  url: "{{ metal_auditing_url }}"
+  enabled: {{ 'true' if metal_auditing_meili_enabled or metal_auditing_timescaledb_enabled else 'false' }}
+  search_backend: {{ metal_auditing_search_backend if metal_auditing_search_backend else '' }}
+{% if metal_auditing_meili_enabled %}
+  meilisearch:
+    enabled: true
+    index_prefix: "{{ metal_auditing_meili_index_prefix }}"
+    index_interval: "{{ metal_auditing_meili_index_interval }}"
+    existingMasterKeySecret: "metal-auditing-meili-api-key"
+    url: "{{ metal_auditing_meili_url }}"
+{% endif %}
+{% if metal_auditing_timescaledb_enabled %}
+  timescaledb:
+    enabled: true
+    host: "{{ metal_auditing_timescaledb_host }}"
+    port: "{{ metal_auditing_timescaledb_port }}"
+    db: "{{ metal_auditing_timescaledb_db }}"
+    user: "{{ metal_auditing_timescaledb_user }}"
+    password: "{{ metal_auditing_timescaledb_password }}"
+    retention: "{{ metal_auditing_timescaledb_retention }}"
+{% endif %}

defaults/main.yaml

@@ -46,6 +46,8 @@ metal_stack_release:
     headscale_db_backup_restore_sidecar_image_name: "docker-images.metal-stack.generic.backup-restore-sidecar.name"
     auditing_meili_backup_restore_sidecar_image_tag: "docker-images.metal-stack.generic.backup-restore-sidecar.tag"
     auditing_meili_backup_restore_sidecar_image_name: "docker-images.metal-stack.generic.backup-restore-sidecar.name"
+    auditing_timescaledb_backup_restore_sidecar_image_tag: "docker-images.metal-stack.generic.backup-restore-sidecar.tag"
+    auditing_timescaledb_backup_restore_sidecar_image_name: "docker-images.metal-stack.generic.backup-restore-sidecar.name"
     # gardener
     firewall_controller_manager_image_tag: "docker-images.metal-stack.gardener.firewall-controller-manager.tag"
     firewall_controller_manager_image_name: "docker-images.metal-stack.gardener.firewall-controller-manager.name"
@@ -100,6 +102,8 @@ metal_stack_release:
     headscale_db_image_name: "docker-images.third-party.control-plane.headscale-db.name"
     auditing_meili_image_name: "docker-images.third-party.control-plane.meilisearch.name"
     auditing_meili_image_tag: "docker-images.third-party.control-plane.meilisearch.tag"
+    auditing_timescaledb_image_name: "docker-images.third-party.control-plane.timescaledb.name"
+    auditing_timescaledb_image_tag: "docker-images.third-party.control-plane.timescaledb.tag"
     image_cache_coredns_image_tag: "docker-images.third-party.partition.image-cache-coredns.tag"
     image_cache_coredns_image_name: "docker-images.third-party.partition.image-cache-coredns.name"
     image_cache_haproxy_image_tag: "docker-images.third-party.partition.image-cache-haproxy.tag"