Postgresql flexible server support (#14)

* Added terraform HCL integration tests and integrated test execution into CI pipeline.

* Added end to end test framework and integrated with CI pipeline.

* Updated README with e2e test details.

* Updated full deployment test to query and assert on each resource.

* Removed parellism from e2e tests to avoid collisions of tfstate and resources.

* Completed full deployment end to end test.

* Johnc/end to end tests parallel (#10)

* Added tfstate and backend to end to end tests.

* Added TF state storage container variable to tests and workflow.

* Added copy of terraform to temp folder to avoid clashes during parellel execution.

* Updated README.

* Added environment variable null checks.

* Updated README.

* Refactored backup policies and instances into modules and added variables that can be used to configure the backups.

* Update docs.

* Fix basic deployment end to end test.

* Updated docs with usage info.

* Made vault name variable mandatory.

* Made test teardown deferred so it runs in all cases.

* Added test helper for end to end environment config.

* Updated info on end to end test debugging.

* Added end to end test for blob storage backups.

* Added managed disk end to end test.

* wip

* Fix issue with managed disk role assignment at resource group level.

* Fix bug in e2e managed disk backup test.

* Updated test logging to use the go log package.

* Added postgresql flexible server support.

* Removed duplication from README, introduced in error.

* Removed more duplicated content added in erroneous merge.

* Removed files left in error.

docs/developer-guide.md

@@ -12,9 +12,11 @@ The following are pre-requisites to working with the solution:
 * An Azure identity which has been assigned the following roles at the subscription level:
     * Contributor (required to create resources)
     * Role Based Access Control Administrator (to assign roles to the backup vault managed identity) **with a condition that limits the roles which can be assigned to:**
-        * Storage Account Backup Contributor
-        * Disk Snapshot Contributor
         * Disk Backup Reader
+        * Disk Snapshot Contributor
+        * PostgreSQL Flexible Server Long Term Retention Backup Role
+        * Storage Account Backup Contributor
+        * Reader
 * [Azure CLI installed](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli-windows?tabs=azure-cli)
 * [Terraform installed](https://developer.hashicorp.com/terraform/install)
 * [Go installed (to run the end-to-end tests)](https://go.dev/dl/)

docs/usage.md

@@ -54,6 +54,22 @@ module "my_backup" {
       }
     }
   }
+  postgresql_flexible_server_backups = {
+    backup1 = {
+      backup_name      = "server1"
+      retention_period = "P7D"
+      backup_intervals = ["R/2024-01-01T00:00:00+00:00/P1D"]
+      server_id  = azurerm_postgresql_flexible_server.my_server_1.id
+      server_resource_group_id = azurerm_resource_group.my_resource_group.id
+    }
+    backup2 = {
+      backup_name      = "server2"
+      retention_period = "P30D"
+      backup_intervals = ["R/2024-01-01T00:00:00+00:00/P2D"]
+      server_id  = azurerm_postgresql_flexible_server.my_server_2.id
+      server_resource_group_id = azurerm_resource_group.my_resource_group.id
+    }
+  }
 }
 ```
 
@@ -63,9 +79,11 @@ To deploy the module an Azure identity (typically an app registration with clien
 
 * Contributor (required to create resources)
 * Role Based Access Control Administrator (to assign roles to the backup vault managed identity) **with a condition that limits the roles which can be assigned to:**
-    * Storage Account Backup Contributor
-    * Disk Snapshot Contributor
     * Disk Backup Reader
+    * Disk Snapshot Contributor
+    * PostgreSQL Flexible Server Long Term Retention Backup Role
+    * Storage Account Backup Contributor
+    * Reader
 
 ## Module Variables
 
@@ -83,3 +101,9 @@ To deploy the module an Azure identity (typically an app registration with clien
 | `managed_disk_backups.backup_name` | The name of the backup, which must be unique across managed disk backups. | Yes | n/a |
 | `managed_disk_backups.retention_period` | How long the backed up data will be retained for, which should be in `ISO 8601` duration format. [See the following link for the possible values](https://en.wikipedia.org/wiki/ISO_8601#Durations). | Yes | n/a |
 | `managed_disk_backups.backup_intervals` | A list of intervals at which backups should be taken, which should be in `ISO 8601` duration format. [See the following link for the possible values](https://en.wikipedia.org/wiki/ISO_8601#Time_intervals). | Yes | n/a |
+| `postgresql_flexible_server_backups` | A map of postgresql flexible server backups that should be created. For each backup the following values should be provided: `backup_name`, `server_id`, `server_resource_group_id`, `retention_period` and `backup_intervals`. When no value is provided then no backups are created. | No | n/a |
+| `postgresql_flexible_server_backups.backup_name` | The name of the backup, which must be unique across postgresql flexible server backups. | Yes | n/a |
+| `postgresql_flexible_server_backups.server_id` | The id of the postgresql flexible server that should be backed up. | Yes | n/a |
+| `postgresql_flexible_server_backups.server_resource_group_id` | The id of the resource group which the postgresql flexible server resides in. | Yes | n/a |
+| `postgresql_flexible_server_backups.retention_period` | How long the backed up data will be retained for, which should be in `ISO 8601` duration format. [See the following link for the possible values](https://en.wikipedia.org/wiki/ISO_8601#Durations). | Yes | n/a |
+| `postgresql_flexible_server_backups.backup_intervals` | A list of intervals at which backups should be taken, which should be in `ISO 8601` duration format. [See the following link for the possible values](https://en.wikipedia.org/wiki/ISO_8601#Time_intervals). | Yes | n/a |

infrastructure/backup_modules.tf

@@ -24,3 +24,18 @@ module "managed_disk_backup" {
   vault_principal_id                = azurerm_data_protection_backup_vault.backup_vault.identity[0].principal_id
   assign_resource_group_level_roles = each.key == keys(var.managed_disk_backups)[0] ? true : false
 }
+
+module "postgresql_flexible_server_backup" {
+  for_each                          = var.postgresql_flexible_server_backups
+  source                            = "./modules/backup/postgresql_flexible_server"
+  vault_id                          = azurerm_data_protection_backup_vault.backup_vault.id
+  vault_name                        = var.vault_name
+  vault_location                    = var.vault_location
+  backup_name                       = each.value.backup_name
+  retention_period                  = each.value.retention_period
+  backup_intervals                  = each.value.backup_intervals
+  server_id                         = each.value.server_id
+  server_resource_group_id          = each.value.server_resource_group_id
+  vault_principal_id                = azurerm_data_protection_backup_vault.backup_vault.identity[0].principal_id
+  assign_resource_group_level_roles = each.key == keys(var.postgresql_flexible_server_backups)[0] ? true : false
+}

infrastructure/modules/backup/postgresql_flexible_server/backup_instance.tf

@@ -0,0 +1,25 @@
+resource "azurerm_role_assignment" "role_assignment_reader" {
+  count                = var.assign_resource_group_level_roles == true ? 1 : 0
+  scope                = var.server_resource_group_id
+  role_definition_name = "Reader"
+  principal_id         = var.vault_principal_id
+}
+
+resource "azurerm_role_assignment" "role_assignment_long_term_retention_backup_role" {
+  scope                = var.server_id
+  role_definition_name = "PostgreSQL Flexible Server Long Term Retention Backup Role"
+  principal_id         = var.vault_principal_id
+}
+
+resource "azurerm_data_protection_backup_instance_postgresql_flexible_server" "backup_instance" {
+  name             = "bkinst-${var.vault_name}-pgflexserver-${var.backup_name}"
+  vault_id         = var.vault_id
+  location         = var.vault_location
+  server_id        = var.server_id
+  backup_policy_id = azurerm_data_protection_backup_policy_postgresql_flexible_server.backup_policy.id
+
+  depends_on = [
+    azurerm_role_assignment.role_assignment_reader,
+    azurerm_role_assignment.role_assignment_long_term_retention_backup_role
+  ]
+}

infrastructure/modules/backup/postgresql_flexible_server/backup_policy.tf

@@ -0,0 +1,12 @@
+resource "azurerm_data_protection_backup_policy_postgresql_flexible_server" "backup_policy" {
+  name                            = "bkpol-${var.vault_name}-pgflexserver-${var.backup_name}"
+  vault_id                        = var.vault_id
+  backup_repeating_time_intervals = var.backup_intervals
+
+  default_retention_rule {
+    life_cycle {
+      duration        = var.retention_period
+      data_store_type = "VaultStore"
+    }
+  }
+}

infrastructure/modules/backup/postgresql_flexible_server/output.tf

@@ -0,0 +1,7 @@
+output "backup_policy" {
+  value = azurerm_data_protection_backup_policy_postgresql_flexible_server.backup_policy
+}
+
+output "backup_instance" {
+  value = azurerm_data_protection_backup_instance_postgresql_flexible_server.backup_instance
+}

infrastructure/modules/backup/postgresql_flexible_server/variables.tf

@@ -0,0 +1,39 @@
+variable "vault_id" {
+  type = string
+}
+
+variable "vault_name" {
+  type = string
+}
+
+variable "vault_location" {
+  type = string
+}
+
+variable "vault_principal_id" {
+  type = string
+}
+
+variable "backup_name" {
+  type = string
+}
+
+variable "retention_period" {
+  type = string
+}
+
+variable "backup_intervals" {
+  type = list(string)
+}
+
+variable "server_id" {
+  type = string
+}
+
+variable "server_resource_group_id" {
+  type = string
+}
+
+variable "assign_resource_group_level_roles" {
+  type = bool
+}

infrastructure/variables.tf

@@ -33,4 +33,15 @@ variable "managed_disk_backups" {
     })
   }))
   default = {}
-}
+}
+
+variable "postgresql_flexible_server_backups" {
+  type = map(object({
+    backup_name              = string
+    retention_period         = string
+    backup_intervals         = list(string)
+    server_id                = string
+    server_resource_group_id = string
+  }))
+  default = {}
+}

tests/end-to-end-tests/go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization v1.0.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute v1.0.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dataprotection/armdataprotection v1.0.0
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/postgresql/armpostgresqlflexibleservers v1.1.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.0.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.0.0
 	github.com/gruntwork-io/terratest v0.47.1

tests/end-to-end-tests/go.sum

@@ -198,16 +198,24 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthoriza
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization v1.0.0/go.mod h1:lPneRe3TwsoDRKY4O6YDLXHhEWrD+TIRa8XrV/3/fqw=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute v1.0.0 h1:/Di3vB4sNeQ+7A8efjUVENvyB945Wruvstucqp7ZArg=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute v1.0.0/go.mod h1:gM3K25LQlsET3QR+4V74zxCsFAy0r6xMNN9n80SZn+4=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization v1.0.0 h1:qtRcg5Y7jNJ4jEzPq4GpWLfTspHdNe2ZK6LjwGcjgmU=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization v1.0.0/go.mod h1:lPneRe3TwsoDRKY4O6YDLXHhEWrD+TIRa8XrV/3/fqw=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute v1.0.0 h1:/Di3vB4sNeQ+7A8efjUVENvyB945Wruvstucqp7ZArg=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute v1.0.0/go.mod h1:gM3K25LQlsET3QR+4V74zxCsFAy0r6xMNN9n80SZn+4=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dataprotection/armdataprotection v1.0.0 h1:VFqjVi532z3gdltbAkYrPl9Ez0czn3ZPM+bjmvLq6fk=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dataprotection/armdataprotection v1.0.0/go.mod h1:CmZQSRwBPP7KNjDA+PHaoR2m8wgOsbTd9ncqZgSzgHA=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.0.0 h1:lMW1lD/17LUA5z1XTURo7LcVG2ICBPlyMHjIUrcFZNQ=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.0.0/go.mod h1:ceIuwmxDWptoW3eCqSXlnPsZFKh4X+R38dWPv7GS9Vs=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.0.0 h1:nBy98uKOIfun5z6wx6jwWLrULcM0+cjBalBFZlEZ7CA=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.0.0/go.mod h1:243D9iHbcQXoFUtgHJwL7gl2zx1aDuDMjvBZVGr2uW0=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/postgresql/armpostgresqlflexibleservers v1.1.0 h1:HzqcSJWx32XQdr8KtxAu/SZJj0PqDo9tKf2YGPdynV0=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/postgresql/armpostgresqlflexibleservers v1.1.0/go.mod h1:nKcJObAisSPDrO9lMuuCBoYY7Ki7ADt8p6XmBhpKNTk=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.0.0 h1:ECsQtyERDVz3NP3kvDOTLvbQhqWp/x9EsGKtb4ogUr8=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.0.0/go.mod h1:s1tW/At+xHqjNFvWU4G0c0Qv33KOhvbGNj0RCTQDV8s=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.0.0 h1:TMEyRFKh1zaSPmoQh3kxK+xRAYVq8guCI/7SMO0F3KY=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.0.0/go.mod h1:c+Lifp3EDEamAkPVzMooRNOK6CZjNSdEnf1A7jsI9u4=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.0.0 h1:TMEyRFKh1zaSPmoQh3kxK+xRAYVq8guCI/7SMO0F3KY=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.0.0/go.mod h1:c+Lifp3EDEamAkPVzMooRNOK6CZjNSdEnf1A7jsI9u4=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=

tests/end-to-end-tests/helpers.go

@@ -13,6 +13,7 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dataprotection/armdataprotection"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/postgresql/armpostgresqlflexibleservers"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage"
 	test_structure "github.com/gruntwork-io/terratest/modules/test-structure"
@@ -357,6 +358,48 @@ func CreateManagedDisk(t *testing.T, credential *azidentity.ClientSecretCredenti
 	return resp.Disk
 }
 
+/*
+ * Creates a postgresql flexible server that can be used for testing purposes.
+ */
+func CreatePostgresqlFlexibleServer(t *testing.T, credential *azidentity.ClientSecretCredential, subscriptionID string,
+	resourceGroupName string, serverName string, serverLocation string, storageSizeGB int32) armpostgresqlflexibleservers.Server {
+	client, err := armpostgresqlflexibleservers.NewServersClient(subscriptionID, credential, nil)
+	assert.NoError(t, err, "Failed to create servers client: %v", err)
+
+	log.Printf("Creating postgresql flexible server %s in location %s", serverName, serverLocation)
+
+	pollerResp, err := client.BeginCreate(
+		context.Background(),
+		resourceGroupName,
+		serverName,
+		armpostgresqlflexibleservers.Server{
+			Location: &serverLocation,
+			SKU: &armpostgresqlflexibleservers.SKU{
+				Name: to.Ptr("Standard_B1ms"),
+				Tier: to.Ptr(armpostgresqlflexibleservers.SKUTierBurstable),
+			},
+			Properties: &armpostgresqlflexibleservers.ServerProperties{
+				AdministratorLogin:         to.Ptr("supersecurelogin"),
+				AdministratorLoginPassword: to.Ptr("supersecurepassword"),
+				Version:                    to.Ptr(armpostgresqlflexibleservers.ServerVersionFourteen),
+				Storage: &armpostgresqlflexibleservers.Storage{
+					StorageSizeGB: &storageSizeGB,
+				},
+			},
+		},
+		nil,
+	)
+	assert.NoError(t, err, "Failed to begin creating postgresql flexible server: %v", err)
+
+	// Wait for the creation to complete
+	resp, err := pollerResp.PollUntilDone(context.Background(), nil)
+	assert.NoError(t, err, "Failed to create postgresql flexible server: %v", err)
+
+	log.Printf("Postgresql flexible server %s created successfully", serverName)
+
+	return resp.Server
+}
+
 /*
  * Deletes a resource group.
  */

tests/end-to-end-tests/managed_disk_backup_test.go

@@ -45,7 +45,7 @@ func setupExternalResourcesForManagedDiskBackupTest(t *testing.T, credential *az
 }
 
 /*
- * TestManagedDiskBackup tests the deployment of a backup vault and backup policies for blob storage accounts.
+ * TestManagedDiskBackup tests the deployment of a backup vault and backup policies for managed disks.
  */
 func TestManagedDiskBackup(t *testing.T) {
 	t.Parallel()