From 219abcc3e39af9e5e8292cd6b24d3521a04a2e3f Mon Sep 17 00:00:00 2001 From: Agniva De Sarker Date: Mon, 13 Jan 2025 19:48:08 +0530 Subject: [PATCH] MM-62533: Improvements for ExternalDBSettings - We bump the postgres version to 14.9 as that is the version after which "write forwarding" is available on a Global Aurora Cluster. - We also add the ClusterIdentifier for ExternalDBSettings because we need to attach the DB security group to the DB cluster like we do for a restored DB backup. https://mattermost.atlassian.net/browse/MM-62533 --- config/deployer.sample.json | 3 ++- config/deployer.sample.toml | 1 + deployment/config.go | 2 ++ deployment/terraform/assets/bindata.go | 4 ++-- deployment/terraform/assets/variables.tf | 2 +- deployment/terraform/create.go | 12 +++++++++--- docs/config/deployer.md | 6 ++++++ 7 files changed, 23 insertions(+), 7 deletions(-) diff --git a/config/deployer.sample.json b/config/deployer.sample.json index 6bdb391f8..064acd9ac 100644 --- a/config/deployer.sample.json +++ b/config/deployer.sample.json @@ -70,7 +70,8 @@ "DriverName": "cockroach", "DataSource": "", "DataSourceReplicas": [], - "DataSourceSearchReplicas": [] + "DataSourceSearchReplicas": [], + "ClusterIdentifier": "", }, "ExternalBucketSettings": { "AmazonS3AccessKeyId": "", diff --git a/config/deployer.sample.toml b/config/deployer.sample.toml index 3f36c3234..6d1417320 100644 --- a/config/deployer.sample.toml +++ b/config/deployer.sample.toml @@ -82,6 +82,7 @@ DataSource = '' DataSourceReplicas = [] DataSourceSearchReplicas = [] DriverName = 'cockroach' +ClusterIdentifier = '' [JobServerSettings] InstanceCount = 0 diff --git a/deployment/config.go b/deployment/config.go index a35286dc0..e6173ae75 100644 --- a/deployment/config.go +++ b/deployment/config.go @@ -257,6 +257,8 @@ type ExternalDBSettings struct { DataSourceReplicas []string `default:""` // DSN to connect to the database search replicas DataSourceSearchReplicas []string `default:""` + // ClusterIdentifier of the existing DB cluster. + ClusterIdentifier string `default:""` } // ExternalBucketSettings contains the necessary data diff --git a/deployment/terraform/assets/bindata.go b/deployment/terraform/assets/bindata.go index fec4ee56d..00375cdf9 100644 --- a/deployment/terraform/assets/bindata.go +++ b/deployment/terraform/assets/bindata.go @@ -468,7 +468,7 @@ func samlIdpCrt() (*asset, error) { return a, nil } -var _variablesTf = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x56\xbd\x72\xf3\x36\x10\xec\xfd\x14\x1c\x7e\x8d\x5d\x44\x23\xff\x4d\xd2\xa8\x48\x91\x26\x69\x32\x79\x01\xcc\x01\x38\x51\x90\x40\x00\xb9\x03\x25\xcb\x19\xbf\x7b\x86\xa0\x28\x93\x22\x28\x79\xbe\xb1\x4a\xee\x62\x71\xd8\x3b\x2c\xb4\x07\x32\x20\x2d\x16\xa5\xb2\x0d\x47\x24\xe1\xa0\xc6\xb2\xf8\xef\xee\xe3\xee\x6e\x0a\xee\x83\x12\x46\xcf\xc2\xdc\x48\x87\x51\x18\xcd\x2d\xa5\x28\xe2\x31\x60\xb1\x2a\xbc\xdc\xa2\x8a\xf7\xed\x97\xa2\x80\x10\x8a\x55\x61\x0d\xc7\x7b\x8e\x64\x5c\xf5\x90\x3e\x6f\xbd\xcc\x7d\x0e\xe4\xdf\x8e\x39\x00\x2a\x74\x31\x07\xa0\x05\x8e\x46\x31\x02\xa9\x4d\x8e\x50\x63\x24\xa3\x38\x07\xed\xf0\xa8\xac\x87\x5d\x0e\xd3\x10\x41\x02\x63\x0e\x23\xd4\x26\x23\xf8\xf1\x30\xb6\x09\x42\x10\x10\x23\xa8\x8d\x30\x50\x8b\x40\x7e\x6d\xec\xd4\xec\x96\x66\x1c\x47\x70\x0a\x85\xf2\x8d\x8b\xd7\x29\xad\xcb\x27\xc6\x8f\xe2\xf7\xe4\x4b\xcf\xe4\xd1\xa2\x16\xba\xa9\x3c\x26\x0d\xb4\x2f\x39\x60\xad\x57\x10\x51\x84\x46\x5a\xa3\x84\x09\x02\xb4\x26\x64\x3e\x17\xf3\xc7\xa8\x17\xd9\xa2\x90\x6f\x55\x34\x64\x64\xcb\x41\x16\x7b\x24\x36\xde\xe5\x20\x45\xd8\x16\x49\x3e\xe3\x34\xb2\x60\x07\x81\x37\x3e\x0a\xc2\xe0\xd9\x44\x4f\xc7\x1c\xed\xdd\x3b\x14\x70\x00\x42\x87\xcc\x02\x5d\x8b\x4c\x2f\xc2\x90\x99\x88\xb0\x07\x63\x41\x1a\x6b\xe2\xb1\x83\x66\x0f\xd9\x69\x0a\x65\x7d\xa3\x0f\x10\xd5\x46\x58\x5f\x7d\x7a\xf9\x77\xba\x09\x8c\xb4\x47\x1a\xae\x4c\x37\x24\xe3\xe1\xf9\xf6\xb9\xa6\x96\x48\x9d\xc8\x3f\x69\x50\xb3\x8d\x48\x33\x3c\x7b\xb0\x0e\x75\x5e\xcf\xf4\xa0\xc3\x03\x10\xd4\xa2\x22\xdf\x84\x7c\x8e\xf4\x9b\x54\xc6\xe1\x6c\xd3\x2e\x4e\x94\xdd\xaf\xe3\xdc\x1c\xc1\xcf\x05\x5a\xde\x9a\xb3\x21\xa3\xab\xf0\xba\x88\x85\xfc\x36\x7d\x18\x1a\x8d\x2e\x9a\xb5\x41\xca\xb1\xa6\x1e\x9c\x1b\x56\x43\xb8\x07\x77\x6c\x13\x44\xe3\x1a\x1a\xdb\xe6\x5c\x17\x9e\x25\x34\xe4\x09\x7e\xa9\x8f\xfc\xaf\x2d\x8b\xf4\x5b\x15\xe5\x6f\x8b\xe5\x22\x7d\x12\x1d\xbe\x78\x5e\x2c\x5f\x17\x4f\xe5\x68\x4d\xf0\x1c\x2b\xc2\xb4\x70\x55\x94\x8f\x2f\x8b\x5f\x5b\xc2\xc7\xa4\xb4\x86\x91\xb2\xdd\xd3\x52\x04\x60\x3e\x78\x9a\x0e\x48\x3a\x52\x1a\xe0\x80\xb4\xf6\x54\x27\x93\x8c\x63\x53\x6d\x62\xd6\xa7\x34\x2c\x18\x91\xc6\x8f\x45\xca\xd0\xd6\x82\x53\x8e\x5e\x64\x28\xf3\xa6\xef\xf6\x0e\xa7\x57\xb5\x86\x18\x91\x6a\xcf\x51\x58\xa3\xd0\x31\x8a\x6c\xc6\x4a\xeb\xd5\x4e\x68\xdc\x9b\xc1\x84\x9d\x6a\x48\x9e\x76\xbb\x8f\x5a\x50\x56\xe1\xb9\xbc\x22\xc3\xe6\x1d\x59\xa4\x70\xbc\x54\x3b\xdd\xc1\xa1\xda\xe3\xf2\xa6\x54\x1a\xf2\xef\x91\x82\x10\xbe\x47\xe8\xf4\x7a\x7e\x41\xec\xf5\xb6\xd8\xd6\xcb\xef\x11\x1a\x3d\xfa\x5f\x90\x7c\x5a\x76\x79\xf8\x57\xff\xe0\x7f\x46\xe2\xd5\x7d\xfa\x3f\x08\x3f\xe1\x65\xbf\x74\x18\xb1\xe7\xa1\x97\xde\xdb\x19\xfa\xf5\x1c\x3c\xd3\xa6\x41\x32\x3b\xc6\x4f\x2f\x8b\x65\x1b\x0e\x79\x21\x8d\x7b\xb4\x3e\xd4\xed\x13\x5f\x7b\x8d\xd9\x3a\x7f\x14\x7f\x7a\x79\x7a\x8c\xb2\xde\x6d\xbd\x14\x1d\x7c\x2b\x76\x73\xcc\xec\x51\xf9\x59\xc8\x46\xed\x30\x0a\xdd\xd4\x41\x34\x64\x72\x14\x7c\x8b\x6d\x7a\xd9\x9e\x9b\x0d\x32\x38\xf0\xfc\xbf\xaf\x03\x0b\xc2\x2a\xf7\x30\xb5\x10\xd4\xd3\x6d\x55\xc3\xd1\xd7\x22\x42\xc5\x93\x20\xef\xff\x0d\x4e\x85\xde\xa7\xe1\xd5\xdd\xad\x9c\x0f\xff\x07\x00\x00\xff\xff\x7e\x5e\x1b\x20\xa8\x0b\x00\x00") +var _variablesTf = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x56\xcd\x6e\xf3\x36\x10\xbc\xe7\x29\x04\x7d\x97\xe4\x50\xc3\xf9\x03\xda\x83\x0f\x3d\xf4\xd2\x5e\x8a\xbe\x00\xb1\x24\xd7\x32\x6d\x8a\x64\x77\x29\x3b\x4e\x91\x77\x2f\x44\x59\x8e\x64\x51\x76\xf0\x21\x3e\x6a\x86\xc3\xe5\xec\x72\xe8\x3d\x90\x01\x69\xb1\x28\x95\x6d\x38\x22\x09\x07\x35\x96\xc5\x7f\x77\x1f\x77\x77\x53\x70\x1f\x94\x30\x7a\x16\xe6\x46\x3a\x8c\xc2\x68\x6e\x29\x45\x11\x8f\x01\x8b\x55\xe1\xe5\x16\x55\xbc\x6f\xbf\x14\x05\x84\x50\xac\x0a\x6b\x38\xde\x73\x24\xe3\xaa\x87\xf4\x79\xeb\x65\xee\x73\x20\xff\x76\xcc\x01\x50\xa1\x8b\x39\x00\x2d\x70\x34\x8a\x11\x48\x6d\x72\x84\x1a\x23\x19\xc5\x39\x68\x87\x47\x65\x3d\xec\x72\x98\x86\x08\x12\x18\x73\x18\xa1\x36\x19\xc1\x8f\x87\xb1\x4d\x10\x82\x80\x18\x41\x6d\x84\x81\x5a\x04\xf2\x6b\x63\xa7\x66\xb7\x34\xe3\x38\x82\x53\x28\x94\x6f\x5c\xbc\x4e\x69\x5d\x3e\x31\x7e\x14\xbf\x27\x5f\x7a\x26\x8f\x16\xb5\xd0\x4d\xe5\x31\x69\xa0\x7d\xc9\x01\x6b\xbd\x82\x88\x22\x34\xd2\x1a\x25\x4c\x10\xa0\x35\x21\xf3\xb9\x98\x3f\x46\xbd\xc8\x16\x85\x7c\xab\xa2\x21\x23\x5b\x0e\xb2\xd8\x23\xb1\xf1\x2e\x07\x29\xc2\xb6\x48\xf2\x19\xa7\x91\x05\x3b\x08\xbc\xf1\x51\x10\x06\xcf\x26\x7a\x3a\xe6\x68\xef\xde\xa1\x80\x03\x10\x3a\x64\x16\xe8\x5a\x64\x7a\x11\x86\xcc\x44\x84\x3d\x18\x0b\xd2\x58\x13\x8f\x1d\x34\x7b\xc8\x4e\x53\x28\xeb\x1b\x7d\x80\xa8\x36\xc2\xfa\xea\xd3\xcb\xbf\xd3\x4d\x60\xa4\x3d\xd2\x70\x65\xba\x21\x19\x0f\xcf\xb7\xcf\x35\xb5\x44\xea\x44\xfe\x49\x83\x9a\x6d\x44\x9a\xe1\xd9\x83\x75\xa8\xf3\x7a\xa6\x07\x1d\x1e\x80\xa0\x16\x15\xf9\x26\xe4\x73\xa4\xdf\xa4\x32\x0e\x67\x9b\x76\x71\xa2\xec\x7e\x1d\xe7\xe6\x08\x7e\x2e\xd0\xf2\xd6\x9c\x0d\x19\x5d\x85\xd7\x45\x2c\xe4\xb7\xe9\xc3\xd0\x68\x74\xd1\xac\x0d\x52\x8e\x35\xf5\xe0\xdc\xb0\x1a\xc2\x3d\xb8\x63\x9b\x20\x1a\xd7\xd0\xd8\x36\xe7\xba\xf0\x2c\xa1\x21\x4f\xf0\x4b\x7d\xe4\x7f\x6d\x59\xa4\xdf\xaa\x28\x7f\x5d\x2c\x17\xe9\x93\xe8\xf0\xc5\xf3\x62\xf9\xba\x78\x2a\x47\x6b\x82\xe7\x58\x11\xa6\x85\xab\xa2\x7c\x7c\x59\xfc\xd6\x12\x3e\x26\xa5\x35\x8c\x94\xed\x9e\x96\x22\x00\xf3\xc1\xd3\x74\x40\xd2\x91\xd2\x00\x07\xa4\xb5\xa7\x3a\x99\x64\x1c\x9b\x6a\x13\xb3\x3e\xa5\x61\xc1\x88\x34\x7e\x2c\x52\x86\xb6\x16\x9c\x72\xf4\x22\x43\x99\x37\x7d\xb7\x77\x38\xbd\xaa\x35\xc4\x88\x54\x7b\x8e\xc2\x1a\x85\x8e\x51\x64\x33\x56\x5a\xaf\x76\x42\xe3\xde\x0c\x26\xec\x54\x43\xf2\xb4\xdb\x7d\xd4\x82\xb2\x0a\xcf\xe5\x15\x19\x36\xef\xc8\x22\x85\xe3\xa5\xda\xe9\x0e\x0e\xd5\x1e\x97\x37\xa5\xd2\x90\x7f\x8f\x14\x84\xf0\x3d\x42\xa7\xd7\xf3\x0b\x62\xaf\xb7\xc5\xb6\x5e\x7e\x8f\xd0\xe8\xd1\xff\x82\xe4\xd3\xb2\xcb\xc3\xbf\xfa\x07\xff\x33\x12\xaf\xee\xd3\xff\x41\xf8\x09\x2f\xfb\xa5\xc3\x88\x3d\x0f\xbd\xf4\xde\xce\xd0\xaf\xe7\xe0\x99\x36\x0d\x92\xd9\x31\x7e\x7a\x59\x2c\xdb\x70\xc8\x0b\x69\xdc\xa3\xf5\xa1\x6e\x9f\xf8\xda\x6b\xcc\xd6\xf9\xa3\xf8\xd3\xcb\xd3\x63\x94\xf5\x6e\xeb\xa5\xe8\xe0\x5b\xb1\x9b\x63\x66\x8f\xca\xcf\x42\x36\x6a\x87\x51\xe8\xa6\x0e\xa2\x21\x93\xa3\xe0\x5b\x6c\xd3\xcb\xf6\xdc\x6c\x90\xc1\x81\xe7\xff\x7d\x1d\x58\x10\x56\xb9\x87\xa9\x85\xa0\x9e\x6e\xab\x1a\x8e\xbe\x16\x11\x2a\x9e\x04\x79\xff\x6f\x70\x2a\xf4\x3e\x0d\xaf\xee\x6e\xe5\x7c\xf8\x3f\x00\x00\xff\xff\xb6\x2f\x7b\x73\xa8\x0b\x00\x00") func variablesTfBytes() ([]byte, error) { return bindataRead( @@ -484,7 +484,7 @@ func variablesTf() (*asset, error) { } info := bindataFileInfo{name: "variables.tf", size: 0, mode: os.FileMode(0644), modTime: time.Unix(0, 0)} - a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x2, 0x9f, 0x16, 0x4c, 0x82, 0x28, 0xdd, 0x3d, 0xfd, 0x8b, 0xf0, 0xba, 0x82, 0x55, 0x7b, 0x6f, 0xa8, 0x6f, 0xe5, 0xf0, 0x68, 0x21, 0xd3, 0x17, 0x4b, 0x78, 0x24, 0x32, 0x88, 0x88, 0xb2, 0x70}} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xb, 0x43, 0x40, 0xfd, 0xa1, 0xa0, 0xc, 0x26, 0x1b, 0x67, 0x4e, 0xa7, 0x10, 0x61, 0xa5, 0x9, 0x66, 0x8, 0x88, 0x1e, 0xba, 0xda, 0x17, 0xe8, 0xb5, 0x3c, 0xc5, 0x61, 0x99, 0xd7, 0x7, 0x44}} return a, nil } diff --git a/deployment/terraform/assets/variables.tf b/deployment/terraform/assets/variables.tf index c92cca640..c00165664 100644 --- a/deployment/terraform/assets/variables.tf +++ b/deployment/terraform/assets/variables.tf @@ -106,7 +106,7 @@ variable "db_engine_version" { type = map(any) default = { "aurora-mysql" = "8.0.mysql_aurora.3.05.2" - "aurora-postgresql" = "14.7" + "aurora-postgresql" = "14.9" } } diff --git a/deployment/terraform/create.go b/deployment/terraform/create.go index f3807b7fe..bf5b35b06 100644 --- a/deployment/terraform/create.go +++ b/deployment/terraform/create.go @@ -153,18 +153,24 @@ func (t *Terraform) Create(extAgent *ssh.ExtAgent, initData bool) error { return err } - // If we are restoring from a DB backup, then we need to hook up + // If we are restoring from a DB backup, or using an external database, then we need to hook up // the security group to it. - if t.config.TerraformDBSettings.ClusterIdentifier != "" { + if t.config.TerraformDBSettings.ClusterIdentifier != "" || t.config.ExternalDBSettings.ClusterIdentifier != "" { if len(t.output.DBSecurityGroup) == 0 { return errors.New("No DB security group created") } + var identifier string + if t.config.TerraformDBSettings.ClusterIdentifier != "" { + identifier = t.config.TerraformDBSettings.ClusterIdentifier + } else { + identifier = t.config.ExternalDBSettings.ClusterIdentifier + } sgID := t.output.DBSecurityGroup[0].Id args := []string{ "rds", "modify-db-cluster", - "--db-cluster-identifier=" + t.config.TerraformDBSettings.ClusterIdentifier, + "--db-cluster-identifier=" + identifier, "--vpc-security-group-ids=" + sgID, "--region=" + t.config.AWSRegion, } diff --git a/docs/config/deployer.md b/docs/config/deployer.md index 516a75182..315e05279 100644 --- a/docs/config/deployer.md +++ b/docs/config/deployer.md @@ -366,6 +366,12 @@ The list of dsn for external database read replicas The list of dsn for external database search replicas +### ClusterIdentifier + +*string* + +ClusterIdentifier of the existing DB cluster. + ## ExternalBucketSettings ### AmazonS3AccessKeyId