From f57ecef75b46b72f83e719836743183a0d49c4c4 Mon Sep 17 00:00:00 2001 From: Eric Vallee Date: Thu, 5 Dec 2024 13:35:59 -0500 Subject: [PATCH 1/3] feat: INFRA-764 Added vault-agent integration for etcd client credentials renewal --- README.md | 2 ++ main.tf | 46 +++++++++++++++++++++++++------- variables.tf | 75 +++++++++++++++++++++++++++++++++++++++++++--------- versions.tf | 2 +- 4 files changed, 102 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 91e9295..e6b9b59 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,7 @@ The module supports libvirt networks and macvtap (bridge mode). - **key**: Client key if certificate authentication is used. - **username**: Client username if certificate authentication is used. - **password**: Client password if certificate authentication is used. + - **vault_agent_secret_path**: Optional vault secret path for an optional vault agent to renew the etcd client credentials. The secret in vault is expected to have the **certificate** and **key** keys if certificate authentication is used or the **username** and **password** keys if password authentication is used. - **prometheus**: Parameters to customise the behavior of prometheus. It has the following keys: - **web**: Object containing the following keys: - **external_url**: Value for the **--web.external-url** prometheus command line parameter. Has to be defined. @@ -107,6 +108,7 @@ The module supports libvirt networks and macvtap (bridge mode). - **key**: Client private tls key to authentify with. To be used for certificate authentication. - **username**: Client's username. To be used for username/password authentication. - **password**: Client's password. To be used for username/password authentication. + - **vault_agent_secret_path**: Optional vault secret path for an optional vault agent to renew the etcd client credentials. The secret in vault is expected to have the **certificate** and **key** keys if certificate authentication is used or the **username** and **password** keys if password authentication is used. - **git**: Parameters to fetch fluent-bit configurations dynamically from an git repo. It has the following keys: - **repo**: Url of the git repository. It should have the ssh format. - **ref**: Git reference (usually branch) to checkout in the repository diff --git a/main.tf b/main.tf index 8940eff..be0b4b8 100644 --- a/main.tf +++ b/main.tf @@ -24,7 +24,7 @@ locals { } module "network_configs" { - source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//network?ref=v0.13.0" + source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//network?ref=7c60f66d536fb2f8aa44987ba173a0d6cf414d12" network_interfaces = concat( [for idx, libvirt_network in var.libvirt_networks: { ip = libvirt_network.ip @@ -46,7 +46,7 @@ module "network_configs" { } module "prometheus_config_updater_configs" { - source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//configurations-auto-updater?ref=v0.13.0" + source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//configurations-auto-updater?ref=7c60f66d536fb2f8aa44987ba173a0d6cf414d12" install_dependencies = var.install_dependencies filesystem = { path = "/etc/prometheus/configs/" @@ -77,21 +77,27 @@ module "prometheus_config_updater_configs" { service = "prometheus-config-updater" } user = "prometheus" + vault_agent = { + etcd_auth = { + enabled = var.etcd.vault_agent_secret_path != "" + secret_path = var.etcd.vault_agent_secret_path + } + } } module "prometheus_configs" { - source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//prometheus?ref=v0.13.0" + source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//prometheus?ref=7c60f66d536fb2f8aa44987ba173a0d6cf414d12" install_dependencies = var.install_dependencies prometheus = var.prometheus } module "prometheus_node_exporter_configs" { - source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//prometheus-node-exporter?ref=v0.13.0" + source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//prometheus-node-exporter?ref=7c60f66d536fb2f8aa44987ba173a0d6cf414d12" install_dependencies = var.install_dependencies } module "chrony_configs" { - source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//chrony?ref=v0.13.0" + source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//chrony?ref=7c60f66d536fb2f8aa44987ba173a0d6cf414d12" install_dependencies = var.install_dependencies chrony = { servers = var.chrony.servers @@ -101,7 +107,7 @@ module "chrony_configs" { } module "fluentbit_updater_etcd_configs" { - source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//configurations-auto-updater?ref=v0.13.0" + source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//configurations-auto-updater?ref=7c60f66d536fb2f8aa44987ba173a0d6cf414d12" install_dependencies = var.install_dependencies filesystem = { path = "/etc/fluent-bit-customization/dynamic-config" @@ -132,10 +138,16 @@ module "fluentbit_updater_etcd_configs" { service = "fluent-bit-config-updater" } user = "fluentbit" + vault_agent = { + etcd_auth = { + enabled = var.fluentbit_dynamic_config.etcd.vault_agent_secret_path != "" + secret_path = var.fluentbit_dynamic_config.etcd.vault_agent_secret_path + } + } } module "fluentbit_updater_git_configs" { - source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//gitsync?ref=v0.13.0" + source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//gitsync?ref=7c60f66d536fb2f8aa44987ba173a0d6cf414d12" install_dependencies = var.install_dependencies filesystem = { path = "/etc/fluent-bit-customization/dynamic-config" @@ -155,7 +167,7 @@ module "fluentbit_updater_git_configs" { } module "fluentbit_configs" { - source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//fluent-bit?ref=v0.13.0" + source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//fluent-bit?ref=7c60f66d536fb2f8aa44987ba173a0d6cf414d12" install_dependencies = var.install_dependencies fluentbit = { metrics = var.fluentbit.metrics @@ -181,8 +193,19 @@ module "fluentbit_configs" { } } +module "vault_agent_configs" { + source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//vault-agent?ref=7c60f66d536fb2f8aa44987ba173a0d6cf414d12" + install_dependencies = var.install_dependencies + vault_agent = { + auth_method = var.vault_agent.auth_method + vault_address = var.vault_agent.vault_address + vault_ca_cert = var.vault_agent.vault_ca_cert + extra_config = "" + } +} + module "data_volume_configs" { - source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//data-volumes?ref=v0.13.0" + source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//data-volumes?ref=7c60f66d536fb2f8aa44987ba173a0d6cf414d12" volumes = [{ label = "prometheus_data" device = "vdb" @@ -244,6 +267,11 @@ locals { content_type = "text/cloud-config" content = module.fluentbit_configs.configuration }] : [], + var.vault_agent.enabled ? [{ + filename = "vault_agent.cfg" + content_type = "text/cloud-config" + content = module.vault_agent_configs.configuration + }] : [], var.data_volume_id != "" ? [{ filename = "data_volume.cfg" content_type = "text/cloud-config" diff --git a/variables.tf b/variables.tf index 298c00a..7bb75fa 100644 --- a/variables.tf +++ b/variables.tf @@ -30,13 +30,13 @@ variable "data_volume_id" { variable "libvirt_networks" { description = "Parameters of libvirt network connections if a libvirt networks are used." type = list(object({ - network_name = string - network_id = string + network_name = optional(string, "") + network_id = optional(string, "") prefix_length = string ip = string mac = string - gateway = string - dns_servers = list(string) + gateway = optional(string, "") + dns_servers = optional(list(string), []) })) default = [] } @@ -48,8 +48,8 @@ variable "macvtap_interfaces" { prefix_length = string ip = string mac = string - gateway = string - dns_servers = list(string) + gateway = optional(string, "") + dns_servers = optional(list(string), []) })) default = [] } @@ -90,11 +90,12 @@ variable "etcd" { endpoints = list(string) ca_certificate = string client = object({ - certificate = string - key = string - username = string - password = string + certificate = optional(string, "") + key = optional(string, "") + username = optional(string, "") + password = optional(string, "") }) + vault_agent_secret_path = optional(string, "") }) } @@ -136,9 +137,12 @@ variable "fluentbit" { prometheus_tag = string prometheus_updater_tag = string node_exporter_tag = string - metrics = object({ + metrics = optional(object({ enabled = bool port = number + }), { + enabled = false + port = 0 }) forward = object({ domain = string @@ -172,7 +176,7 @@ variable "fluentbit_dynamic_config" { type = object({ enabled = bool source = string - etcd = object({ + etcd = optional(object({ key_prefix = string endpoints = list(string) ca_certificate = string @@ -182,8 +186,19 @@ variable "fluentbit_dynamic_config" { username = string password = string }) + vault_agent_secret_path = optional(string, "") + }), { + key_prefix = "" + endpoints = [] + ca_certificate = "" + client = { + certificate = "" + key = "" + username = "" + password = "" + } }) - git = object({ + git = optional(object({ repo = string ref = string path = string @@ -192,6 +207,15 @@ variable "fluentbit_dynamic_config" { client_ssh_key = string server_ssh_fingerprint = string }) + }), { + repo = "" + ref = "" + path = "" + trusted_gpg_keys = [] + auth = { + client_ssh_key = "" + server_ssh_fingerprint = "" + } }) }) default = { @@ -226,6 +250,31 @@ variable "fluentbit_dynamic_config" { } } +variable "vault_agent" { + type = object({ + enabled = bool + auth_method = object({ + config = object({ + role_id = string + secret_id = string + }) + }) + vault_address = string + vault_ca_cert = string + }) + default = { + enabled = false + auth_method = { + config = { + role_id = "" + secret_id = "" + } + } + vault_address = "" + vault_ca_cert = "" + } +} + variable "prometheus" { description = "Prometheus configurations" type = object({ diff --git a/versions.tf b/versions.tf index 6f75596..8e39af0 100644 --- a/versions.tf +++ b/versions.tf @@ -5,5 +5,5 @@ terraform { version = ">= 0.6.14, <= 0.7.1" } } - required_version = ">= 1.2.0" + required_version = ">= 1.3.0" } \ No newline at end of file From cd18b04b8493f074a5b1cb3a6142df557142cc18 Mon Sep 17 00:00:00 2001 From: Eric Vallee Date: Thu, 5 Dec 2024 13:40:57 -0500 Subject: [PATCH 2/3] fix: INFRA-764 Incremented terraform version for validation pipeline --- .github/workflows/terraform_validate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/terraform_validate.yml b/.github/workflows/terraform_validate.yml index a0991b8..56d0241 100644 --- a/.github/workflows/terraform_validate.yml +++ b/.github/workflows/terraform_validate.yml @@ -6,7 +6,7 @@ jobs: Deploy: name: Validate the Terraform files runs-on: ubuntu-22.04 - container: hashicorp/terraform:1.2.9 + container: hashicorp/terraform:1.3.10 steps: - uses: actions/checkout@v2 - name: Initialize the terraform modules From 35aaaa6f87291d470da9364610d83456df900adc Mon Sep 17 00:00:00 2001 From: Eric Vallee Date: Thu, 12 Dec 2024 15:19:05 -0500 Subject: [PATCH 3/3] feat: INFRA-764 Added missing README arguments. Updated cloudinit references to last release tag --- README.md | 8 ++++++++ main.tf | 20 ++++++++++---------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index e6b9b59..b6662a9 100644 --- a/README.md +++ b/README.md @@ -120,4 +120,12 @@ The module supports libvirt networks and macvtap (bridge mode). - **prometheus_secrets**: List of prometheus secrets (to access exporters, alertmanagers and other sattelite processes) to pass to the server's filesystem. The prometheus user that the prometheus process runs as will be made owner and given exclusive access to these files. Each element in the list takes the following keys: - **path**: Filesystem path where to store the secret on the server - **content**: Value of the secret +- **vault_agent**: Parameters for the optional vault agent that will be used to manage the dynamic secrets in the vm. + - **enabled**: If set to true, a vault agent service will be setup and will run in the vm. + - **auth_method**: Auth method the vault agent will use to authenticate with vault. Currently, only approle is supported. + - **config**: Configuration parameters for the auth method. + - **role_id**: Id of the app role to us. + - **secret_id**: Authentication secret to use the app role. + - **vault_address**: Endpoint to use to talk to vault. + - **vault_ca_cert**: CA certificate to use to validate vault's certificate. - **install_dependencies**: Whether cloud-init should install external dependencies (should be set to false if you already provide an image with the external dependencies built-in). \ No newline at end of file diff --git a/main.tf b/main.tf index be0b4b8..4f32a82 100644 --- a/main.tf +++ b/main.tf @@ -24,7 +24,7 @@ locals { } module "network_configs" { - source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//network?ref=7c60f66d536fb2f8aa44987ba173a0d6cf414d12" + source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//network?ref=v0.26.0" network_interfaces = concat( [for idx, libvirt_network in var.libvirt_networks: { ip = libvirt_network.ip @@ -46,7 +46,7 @@ module "network_configs" { } module "prometheus_config_updater_configs" { - source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//configurations-auto-updater?ref=7c60f66d536fb2f8aa44987ba173a0d6cf414d12" + source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//configurations-auto-updater?ref=v0.26.0" install_dependencies = var.install_dependencies filesystem = { path = "/etc/prometheus/configs/" @@ -86,18 +86,18 @@ module "prometheus_config_updater_configs" { } module "prometheus_configs" { - source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//prometheus?ref=7c60f66d536fb2f8aa44987ba173a0d6cf414d12" + source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//prometheus?ref=v0.26.0" install_dependencies = var.install_dependencies prometheus = var.prometheus } module "prometheus_node_exporter_configs" { - source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//prometheus-node-exporter?ref=7c60f66d536fb2f8aa44987ba173a0d6cf414d12" + source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//prometheus-node-exporter?ref=v0.26.0" install_dependencies = var.install_dependencies } module "chrony_configs" { - source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//chrony?ref=7c60f66d536fb2f8aa44987ba173a0d6cf414d12" + source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//chrony?ref=v0.26.0" install_dependencies = var.install_dependencies chrony = { servers = var.chrony.servers @@ -107,7 +107,7 @@ module "chrony_configs" { } module "fluentbit_updater_etcd_configs" { - source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//configurations-auto-updater?ref=7c60f66d536fb2f8aa44987ba173a0d6cf414d12" + source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//configurations-auto-updater?ref=v0.26.0" install_dependencies = var.install_dependencies filesystem = { path = "/etc/fluent-bit-customization/dynamic-config" @@ -147,7 +147,7 @@ module "fluentbit_updater_etcd_configs" { } module "fluentbit_updater_git_configs" { - source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//gitsync?ref=7c60f66d536fb2f8aa44987ba173a0d6cf414d12" + source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//gitsync?ref=v0.26.0" install_dependencies = var.install_dependencies filesystem = { path = "/etc/fluent-bit-customization/dynamic-config" @@ -167,7 +167,7 @@ module "fluentbit_updater_git_configs" { } module "fluentbit_configs" { - source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//fluent-bit?ref=7c60f66d536fb2f8aa44987ba173a0d6cf414d12" + source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//fluent-bit?ref=v0.26.0" install_dependencies = var.install_dependencies fluentbit = { metrics = var.fluentbit.metrics @@ -194,7 +194,7 @@ module "fluentbit_configs" { } module "vault_agent_configs" { - source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//vault-agent?ref=7c60f66d536fb2f8aa44987ba173a0d6cf414d12" + source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//vault-agent?ref=v0.26.0" install_dependencies = var.install_dependencies vault_agent = { auth_method = var.vault_agent.auth_method @@ -205,7 +205,7 @@ module "vault_agent_configs" { } module "data_volume_configs" { - source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//data-volumes?ref=7c60f66d536fb2f8aa44987ba173a0d6cf414d12" + source = "git::https://github.com/Ferlab-Ste-Justine/terraform-cloudinit-templates.git//data-volumes?ref=v0.26.0" volumes = [{ label = "prometheus_data" device = "vdb"