Feature/vault agent integration

.github/workflows/terraform_validate.yml

@@ -6,7 +6,7 @@ jobs:
   Deploy:
     name: Validate the Terraform files
     runs-on: ubuntu-22.04
-    container: hashicorp/terraform:1.2.9
+    container: hashicorp/terraform:1.3.10
     steps:
     - uses: actions/checkout@v2
     - name: Initialize the terraform modules

README.md

@@ -66,6 +66,7 @@ The module supports libvirt networks and macvtap (bridge mode).
     - **key**: Client key if certificate authentication is used.
     - **username**: Client username if certificate authentication is used.
     - **password**: Client password if certificate authentication is used.
+  - **vault_agent_secret_path**: Optional vault secret path for an optional vault agent to renew the etcd client credentials. The secret in vault is expected to have the **certificate** and **key** keys if certificate authentication is used or the **username** and **password** keys if password authentication is used.
 - **prometheus**: Parameters to customise the behavior of prometheus. It has the following keys:
   - **web**: Object containing the following keys:
     - **external_url**: Value for the **--web.external-url** prometheus command line parameter. Has to be defined.
@@ -107,6 +108,7 @@ The module supports libvirt networks and macvtap (bridge mode).
       - **key**: Client private tls key to authentify with. To be used for certificate authentication.
       - **username**: Client's username. To be used for username/password authentication.
       - **password**: Client's password. To be used for username/password authentication.
+    - **vault_agent_secret_path**: Optional vault secret path for an optional vault agent to renew the etcd client credentials. The secret in vault is expected to have the **certificate** and **key** keys if certificate authentication is used or the **username** and **password** keys if password authentication is used.
   - **git**: Parameters to fetch fluent-bit configurations dynamically from an git repo. It has the following keys:
     - **repo**: Url of the git repository. It should have the ssh format.
     - **ref**: Git reference (usually branch) to checkout in the repository
@@ -118,4 +120,12 @@ The module supports libvirt networks and macvtap (bridge mode).
 - **prometheus_secrets**: List of prometheus secrets (to access exporters, alertmanagers and other sattelite processes) to pass to the server's filesystem. The prometheus user that the prometheus process runs as will be made owner and given exclusive access to these files. Each element in the list takes the following keys:
   - **path**: Filesystem path where to store the secret on the server
   - **content**: Value of the secret
+- **vault_agent**: Parameters for the optional vault agent that will be used to manage the dynamic secrets in the vm.
+  - **enabled**: If set to true, a vault agent service will be setup and will run in the vm.
+  - **auth_method**: Auth method the vault agent will use to authenticate with vault. Currently, only approle is supported.
+    - **config**: Configuration parameters for the auth method.
+      - **role_id**: Id of the app role to us.
+      - **secret_id**: Authentication secret to use the app role.
+  - **vault_address**: Endpoint to use to talk to vault.
+  - **vault_ca_cert**: CA certificate to use to validate vault's certificate.
 - **install_dependencies**: Whether cloud-init should install external dependencies (should be set to false if you already provide an image with the external dependencies built-in).

main.tf

@@ -24,7 +24,7 @@ locals {
 }
 
 module "network_configs" {
-  source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//network?ref=v0.13.0"
+  source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//network?ref=v0.26.0"
   network_interfaces = concat(
     [for idx, libvirt_network in var.libvirt_networks: {
       ip = libvirt_network.ip
@@ -46,7 +46,7 @@ module "network_configs" {
 }
 
 module "prometheus_config_updater_configs" {
-  source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//configurations-auto-updater?ref=v0.13.0"
+  source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//configurations-auto-updater?ref=v0.26.0"
   install_dependencies = var.install_dependencies
   filesystem = {
     path = "/etc/prometheus/configs/"
@@ -77,21 +77,27 @@ module "prometheus_config_updater_configs" {
     service = "prometheus-config-updater"
   }
   user = "prometheus"
+  vault_agent = {
+    etcd_auth = {
+        enabled = var.etcd.vault_agent_secret_path != ""
+        secret_path = var.etcd.vault_agent_secret_path
+    }
+  }
 }
 
 module "prometheus_configs" {
-  source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//prometheus?ref=v0.13.0"
+  source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//prometheus?ref=v0.26.0"
   install_dependencies = var.install_dependencies
   prometheus = var.prometheus
 }
 
 module "prometheus_node_exporter_configs" {
-  source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//prometheus-node-exporter?ref=v0.13.0"
+  source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//prometheus-node-exporter?ref=v0.26.0"
   install_dependencies = var.install_dependencies
 }
 
 module "chrony_configs" {
-  source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//chrony?ref=v0.13.0"
+  source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//chrony?ref=v0.26.0"
   install_dependencies = var.install_dependencies
   chrony = {
     servers  = var.chrony.servers
@@ -101,7 +107,7 @@ module "chrony_configs" {
 }
 
 module "fluentbit_updater_etcd_configs" {
-  source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//configurations-auto-updater?ref=v0.13.0"
+  source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//configurations-auto-updater?ref=v0.26.0"
   install_dependencies = var.install_dependencies
   filesystem = {
     path = "/etc/fluent-bit-customization/dynamic-config"
@@ -132,10 +138,16 @@ module "fluentbit_updater_etcd_configs" {
     service = "fluent-bit-config-updater"
   }
   user = "fluentbit"
+  vault_agent = {
+    etcd_auth = {
+        enabled = var.fluentbit_dynamic_config.etcd.vault_agent_secret_path != ""
+        secret_path = var.fluentbit_dynamic_config.etcd.vault_agent_secret_path
+    }
+  }
 }
 
 module "fluentbit_updater_git_configs" {
-  source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//gitsync?ref=v0.13.0"
+  source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//gitsync?ref=v0.26.0"
   install_dependencies = var.install_dependencies
   filesystem = {
     path = "/etc/fluent-bit-customization/dynamic-config"
@@ -155,7 +167,7 @@ module "fluentbit_updater_git_configs" {
 }
 
 module "fluentbit_configs" {
-  source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//fluent-bit?ref=v0.13.0"
+  source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//fluent-bit?ref=v0.26.0"
   install_dependencies = var.install_dependencies
   fluentbit = {
     metrics = var.fluentbit.metrics
@@ -181,8 +193,19 @@ module "fluentbit_configs" {
   }
 }
 
+module "vault_agent_configs" {
+  source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//vault-agent?ref=v0.26.0"
+  install_dependencies = var.install_dependencies
+  vault_agent = {
+    auth_method = var.vault_agent.auth_method
+    vault_address = var.vault_agent.vault_address
+    vault_ca_cert = var.vault_agent.vault_ca_cert
+    extra_config = ""
+  }
+}
+
 module "data_volume_configs" {
-  source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//data-volumes?ref=v0.13.0"
+  source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//data-volumes?ref=v0.26.0"
   volumes = [{
     label         = "prometheus_data"
     device        = "vdb"
@@ -244,6 +267,11 @@ locals {
       content_type = "text/cloud-config"
       content      = module.fluentbit_configs.configuration
     }] : [],
+    var.vault_agent.enabled ? [{
+      filename     = "vault_agent.cfg"
+      content_type = "text/cloud-config"
+      content      = module.vault_agent_configs.configuration
+    }] : [],
     var.data_volume_id != "" ? [{
       filename     = "data_volume.cfg"
       content_type = "text/cloud-config"

variables.tf

@@ -30,13 +30,13 @@ variable "data_volume_id" {
 variable "libvirt_networks" {
   description = "Parameters of libvirt network connections if a libvirt networks are used."
   type = list(object({
-    network_name = string
-    network_id = string
+    network_name = optional(string, "")
+    network_id = optional(string, "")
     prefix_length = string
     ip = string
     mac = string
-    gateway = string
-    dns_servers = list(string)
+    gateway = optional(string, "")
+    dns_servers = optional(list(string), [])
   }))
   default = []
 }
@@ -48,8 +48,8 @@ variable "macvtap_interfaces" {
     prefix_length = string
     ip            = string
     mac           = string
-    gateway       = string
-    dns_servers   = list(string)
+    gateway       = optional(string, "")
+    dns_servers   = optional(list(string), [])
   }))
   default = []
 }
@@ -90,11 +90,12 @@ variable "etcd" {
     endpoints = list(string)
     ca_certificate = string
     client = object({
-      certificate = string
-      key = string
-      username = string
-      password = string
+      certificate = optional(string, "")
+      key = optional(string, "")
+      username = optional(string, "")
+      password = optional(string, "")
     })
+    vault_agent_secret_path = optional(string, "")
   })
 }
 
@@ -136,9 +137,12 @@ variable "fluentbit" {
     prometheus_tag = string
     prometheus_updater_tag = string
     node_exporter_tag = string
-    metrics = object({
+    metrics = optional(object({
       enabled = bool
       port    = number
+    }), {
+      enabled = false
+      port = 0
     })
     forward = object({
       domain = string
@@ -172,7 +176,7 @@ variable "fluentbit_dynamic_config" {
   type = object({
     enabled = bool
     source  = string
-    etcd    = object({
+    etcd    = optional(object({
       key_prefix     = string
       endpoints      = list(string)
       ca_certificate = string
@@ -182,8 +186,19 @@ variable "fluentbit_dynamic_config" {
         username    = string
         password    = string
       })
+      vault_agent_secret_path = optional(string, "")
+    }), {
+      key_prefix     = ""
+      endpoints      = []
+      ca_certificate = ""
+      client         = {
+        certificate = ""
+        key         = ""
+        username    = ""
+        password    = ""
+      }
     })
-    git     = object({
+    git     = optional(object({
       repo             = string
       ref              = string
       path             = string
@@ -192,6 +207,15 @@ variable "fluentbit_dynamic_config" {
         client_ssh_key         = string
         server_ssh_fingerprint = string
       })
+    }), {
+      repo             = ""
+      ref              = ""
+      path             = ""
+      trusted_gpg_keys = []
+      auth             = {
+        client_ssh_key         = ""
+        server_ssh_fingerprint = ""
+      }
     })
   })
   default = {
@@ -226,6 +250,31 @@ variable "fluentbit_dynamic_config" {
   }
 }
 
+variable "vault_agent" {
+  type = object({
+    enabled = bool
+    auth_method = object({
+      config = object({
+        role_id   = string
+        secret_id = string
+      })
+    })
+    vault_address   = string
+    vault_ca_cert   = string
+  })
+  default = {
+    enabled = false
+    auth_method = {
+      config = {
+        role_id   = ""
+        secret_id = ""
+      }
+    }
+    vault_address = ""
+    vault_ca_cert = ""
+  }
+}
+
 variable "prometheus" {
   description = "Prometheus configurations"
   type = object({

versions.tf

@@ -5,5 +5,5 @@ terraform {
       version = ">= 0.6.14, <= 0.7.1"
     }
   }
-  required_version = ">= 1.2.0"
+  required_version = ">= 1.3.0"
 }