From 764a34b5386b6c55f1f6c2a8935945d63a31d44b Mon Sep 17 00:00:00 2001 From: janli Date: Mon, 25 Nov 2024 13:13:46 -0800 Subject: [PATCH 1/3] feat: add support to use external db host --- CHANGELOG.md | 4 ++++ config-templates.tf | 24 ++++++++++++------------ rds.tf | 9 ++++++--- variables.tf | 16 ++++++++++------ 4 files changed, 32 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9656f2d..9331e5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [5.2.0] - 2024-11-25 +### Added +- Added new variable `beekeeper_db_external_hostname` to support external DB host. + ## [5.1.1] - 2024-11-07 ### Added - Kubernetes variables to adjust number replicas on deployments. diff --git a/config-templates.tf b/config-templates.tf index 8deb123..502dbd4 100644 --- a/config-templates.tf +++ b/config-templates.tf @@ -19,9 +19,9 @@ data "template_file" "beekeeper_scheduler_apiary_config" { template = file("${path.module}/files/beekeeper-scheduler-apiary-config.json") vars = { - db_endpoint = aws_db_instance.beekeeper.endpoint - db_name = aws_db_instance.beekeeper.name - db_username = aws_db_instance.beekeeper.username + db_endpoint = var.beekeeper_db_external_hostname == "" ? aws_db_instance.beekeeper[0].endpoint : var.beekeeper_db_external_hostname + db_name = var.beekeeper_db_name + db_username = var.db_username queue = aws_sqs_queue.beekeeper.id graphite_config = var.graphite_enabled == "false" ? "" : data.template_file.beekeeper_graphite_config.rendered } @@ -31,9 +31,9 @@ data "template_file" "beekeeper_path_cleanup_config" { template = file("${path.module}/files/beekeeper-path-cleanup-config.json") vars = { - db_endpoint = aws_db_instance.beekeeper.endpoint - db_name = aws_db_instance.beekeeper.name - db_username = aws_db_instance.beekeeper.username + db_endpoint = var.beekeeper_db_external_hostname == "" ? aws_db_instance.beekeeper[0].endpoint : var.beekeeper_db_external_hostname + db_name = var.beekeeper_db_name + db_username = var.db_username scheduler_delay_ms = var.scheduler_delay_ms dry_run_enabled = var.path_cleanup_dry_run_enabled graphite_config = var.graphite_enabled == "false" ? "" : data.template_file.beekeeper_graphite_config.rendered @@ -44,9 +44,9 @@ data "template_file" "beekeeper_metadata_cleanup_config" { template = file("${path.module}/files/beekeeper-metadata-cleanup-config.json") vars = { - db_endpoint = aws_db_instance.beekeeper.endpoint - db_name = aws_db_instance.beekeeper.name - db_username = aws_db_instance.beekeeper.username + db_endpoint = var.beekeeper_db_external_hostname == "" ? aws_db_instance.beekeeper[0].endpoint : var.beekeeper_db_external_hostname + db_name = var.beekeeper_db_name + db_username = var.db_username scheduler_delay_ms = var.scheduler_delay_ms dry_run_enabled = var.metadata_cleanup_dry_run_enabled metastore_uri = var.metastore_uri @@ -58,9 +58,9 @@ data "template_file" "beekeeper_api_config" { template = file("${path.module}/files/beekeeper-api-config.json") vars = { - db_endpoint = aws_db_instance.beekeeper.endpoint - db_name = aws_db_instance.beekeeper.name - db_username = aws_db_instance.beekeeper.username + db_endpoint = var.beekeeper_db_external_hostname == "" ? aws_db_instance.beekeeper[0].endpoint : var.beekeeper_db_external_hostname + db_name = var.beekeeper_db_name + db_username = var.db_username scheduler_delay_ms = var.scheduler_delay_ms metastore_uri = var.metastore_uri graphite_config = var.graphite_enabled == "false" ? "" : data.template_file.beekeeper_graphite_config.rendered diff --git a/rds.tf b/rds.tf index b4dab19..00c08a8 100644 --- a/rds.tf +++ b/rds.tf @@ -5,6 +5,7 @@ */ resource "aws_db_subnet_group" "beekeeper_db_subnet_group" { + count = var.beekeeper_db_external_hostname == "" ? 1 : 0 name = "${local.instance_alias}-db-subnet-group" subnet_ids = var.rds_subnets description = "Beekeeper DB Subnet Group for ${local.instance_alias}" @@ -13,6 +14,7 @@ resource "aws_db_subnet_group" "beekeeper_db_subnet_group" { } resource "aws_security_group" "beekeeper_db_sg" { + count = var.beekeeper_db_external_hostname == "" ? 1 : 0 name = "${local.instance_alias}-db" vpc_id = var.vpc_id tags = var.beekeeper_tags @@ -46,9 +48,10 @@ resource "random_id" "snapshot_id" { } resource "aws_db_instance" "beekeeper" { + count = var.beekeeper_db_external_hostname == "" ? 1 : 0 identifier = local.instance_alias - db_subnet_group_name = aws_db_subnet_group.beekeeper_db_subnet_group.name - vpc_security_group_ids = [aws_security_group.beekeeper_db_sg.id] + db_subnet_group_name = aws_db_subnet_group.beekeeper_db_subnet_group[0].name + vpc_security_group_ids = [aws_security_group.beekeeper_db_sg[0].id] allocated_storage = var.rds_allocated_storage max_allocated_storage = var.rds_max_allocated_storage storage_type = var.rds_storage_type @@ -57,7 +60,7 @@ resource "aws_db_instance" "beekeeper" { instance_class = var.rds_instance_class // Don't use instance alias as part of default DB name since all the Flyway scripts expect "beekeeper" as the db name. // No reason to parameterize the db name anyway, since we have a different RDS instance per Beekeeper instance. - db_name = "beekeeper" + db_name = var.beekeeper_db_name username = var.db_username password = chomp(data.aws_secretsmanager_secret_version.beekeeper_db.secret_string) parameter_group_name = var.rds_parameter_group_name diff --git a/variables.tf b/variables.tf index 0b4939c..aba4c0b 100644 --- a/variables.tf +++ b/variables.tf @@ -622,10 +622,14 @@ variable "beekeeper_path_cleanup_metrics" { ] } -variable "beekeeper_perf_scheduler_apiary_metrics" { - description = "Beekeeper metrics to be sent to Datadog." - type = list(string) - default = [ - "metadata_scheduled_seconds_count*" - ] +variable "beekeeper_db_external_hostname" { + description = "Hostname of the background Database. When not null, RDS DB wont be created." + type = string + default = "" +} + +variable "beekeeper_db_name" { + description = "Name of the beekeeper background Database" + type = string + default = "beekeeper" } From c00d6e33df873be3590142d20e84e2049e61ce5d Mon Sep 17 00:00:00 2001 From: janli Date: Mon, 25 Nov 2024 14:53:57 -0800 Subject: [PATCH 2/3] fix: set default value for rds_subnets --- variables.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/variables.tf b/variables.tf index aba4c0b..a68211f 100644 --- a/variables.tf +++ b/variables.tf @@ -43,6 +43,7 @@ variable "beekeeper_tags" { variable "rds_subnets" { description = "Subnets in which to provision Beekeeper RDS DB." type = list(string) + default = [] } variable "rds_allocated_storage" { From 221d55aea24579de675e95e2cd4582495cc3307b Mon Sep 17 00:00:00 2001 From: janli Date: Mon, 25 Nov 2024 15:06:30 -0800 Subject: [PATCH 3/3] feat: add db port --- config-templates.tf | 8 ++++---- variables.tf | 6 ++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/config-templates.tf b/config-templates.tf index 502dbd4..3c093ea 100644 --- a/config-templates.tf +++ b/config-templates.tf @@ -19,7 +19,7 @@ data "template_file" "beekeeper_scheduler_apiary_config" { template = file("${path.module}/files/beekeeper-scheduler-apiary-config.json") vars = { - db_endpoint = var.beekeeper_db_external_hostname == "" ? aws_db_instance.beekeeper[0].endpoint : var.beekeeper_db_external_hostname + db_endpoint = var.beekeeper_db_external_hostname == "" ? aws_db_instance.beekeeper[0].endpoint : "${var.beekeeper_db_external_hostname}:${var.beekeeper_db_port}" db_name = var.beekeeper_db_name db_username = var.db_username queue = aws_sqs_queue.beekeeper.id @@ -31,7 +31,7 @@ data "template_file" "beekeeper_path_cleanup_config" { template = file("${path.module}/files/beekeeper-path-cleanup-config.json") vars = { - db_endpoint = var.beekeeper_db_external_hostname == "" ? aws_db_instance.beekeeper[0].endpoint : var.beekeeper_db_external_hostname + db_endpoint = var.beekeeper_db_external_hostname == "" ? aws_db_instance.beekeeper[0].endpoint : "${var.beekeeper_db_external_hostname}:${var.beekeeper_db_port}" db_name = var.beekeeper_db_name db_username = var.db_username scheduler_delay_ms = var.scheduler_delay_ms @@ -44,7 +44,7 @@ data "template_file" "beekeeper_metadata_cleanup_config" { template = file("${path.module}/files/beekeeper-metadata-cleanup-config.json") vars = { - db_endpoint = var.beekeeper_db_external_hostname == "" ? aws_db_instance.beekeeper[0].endpoint : var.beekeeper_db_external_hostname + db_endpoint = var.beekeeper_db_external_hostname == "" ? aws_db_instance.beekeeper[0].endpoint : "${var.beekeeper_db_external_hostname}:${var.beekeeper_db_port}" db_name = var.beekeeper_db_name db_username = var.db_username scheduler_delay_ms = var.scheduler_delay_ms @@ -58,7 +58,7 @@ data "template_file" "beekeeper_api_config" { template = file("${path.module}/files/beekeeper-api-config.json") vars = { - db_endpoint = var.beekeeper_db_external_hostname == "" ? aws_db_instance.beekeeper[0].endpoint : var.beekeeper_db_external_hostname + db_endpoint = var.beekeeper_db_external_hostname == "" ? aws_db_instance.beekeeper[0].endpoint : "${var.beekeeper_db_external_hostname}:${var.beekeeper_db_port}" db_name = var.beekeeper_db_name db_username = var.db_username scheduler_delay_ms = var.scheduler_delay_ms diff --git a/variables.tf b/variables.tf index a68211f..fbba54a 100644 --- a/variables.tf +++ b/variables.tf @@ -634,3 +634,9 @@ variable "beekeeper_db_name" { type = string default = "beekeeper" } + +variable "beekeeper_db_port" { + description = "Port number of the beekeeper background Database" + type = number + default = 3306 +}