feat: [PAGOPA-2527] Refactoring FdR-Fase3 collections (#2737)

* [PAGOPA-2627] feat: adding fdr_flow and fdr_payment collections

* [PAGOPA-2527] feat: including EnableUniqueCompoundNestedDocs capability

* [PAGOPA-2527] fix: refactored indexes

* [PAGOPA-2527] feat: removing unused collections

* [PAGOPA-2527] feat: add new index on fdr_flow container

* [PAGOPA-2527] feat: add container TTLs as explicit variable

---------

Co-authored-by: Angelo Caporaso <[email protected]>
Co-authored-by: Francesco Cesareo <[email protected]>

src/domains/fdr-common/03_cosmos_mongodb_fdr.tf

@@ -44,124 +44,50 @@ resource "azurerm_cosmosdb_mongo_database" "fdr" {
 
 }
 
-
-# fdr_history
-# fdr_insert
-# fdr_publish
-
-# fdr_payment_history
-# fdr_payment_insert
-# fdr_payment_publish
-
 # Collections
-
-# https://learn.microsoft.com/en-us/azure/cosmos-db/mongodb/indexing#compound-indexes-mongodb-server-version-36
-# Compound indexes are required if your query needs the ability to sort on multiple fields at once.
-# For queries with multiple filters that don't need to sort, create multiple single field indexes instead
-# of a compound index to save on indexing costs.
-
 locals {
   collections = [
     {
-      name = "fdr_history"
-      indexes = [{
-        keys   = ["_id"] # reporting_flow_name + revision
-        unique = true
-        }
-      ]
-      shard_key = null
-    },
-    {
-      name = "fdr_insert"
+      name = "fdr_flow"
       indexes = [
         {
-          keys   = ["_id"] # reporting_flow_name
+          keys   = ["_id"] # document UID
           unique = true
         },
         {
-          keys   = ["fdr"]
-          unique = false
-        },
-        {
-          keys   = ["sender.psp_id"]
-          unique = false
-        },
-        {
-          keys   = ["fdr", "revision"]
+          keys   = ["sender.psp_id", "name", "revision"] # flow_revision_idx
           unique = true
-        }
-      ]
-      shard_key = null
-    },
-    {
-      name = "fdr_publish"
-      indexes = [{
-        keys   = ["_id"] # reporting_flow_name
-        unique = true
         },
         {
-          keys   = ["fdr"]
+          keys   = ["sender.psp_id", "receiver.organization_id", "published"] # published_flow_by_psp_idx
           unique = false
         },
         {
-          keys   = ["sender.psp_id"]
+          keys   = ["receiver.organization_id", "sender.psp_id", "published"] # published_flow_by_organization_idx
           unique = false
-        },
-        {
-          keys   = ["receiver.organization_id"]
-          unique = false
-        },
-        {
-          keys   = ["fdr", "revision"]
-          unique = true
-        }
-      ]
-      shard_key = null
-    },
-    {
-      name = "fdr_payment_history"
-      indexes = [{
-        keys   = ["_id"] # index + ref_fdr_reporting_flow_name + ref_fdr_id
-        unique = true
         }
       ]
-      shard_key = null
+      shard_key   = null,
+      ttl_seconds = var.cosmos_mongo_db_fdr_params.fdr_flow_container_ttl
     },
     {
-      name = "fdr_payment_insert"
+      name = "fdr_payment"
       indexes = [
         {
-          keys   = ["_id"] # index + ref_fdr_reporting_flow_name + ref_fdr_id
+          keys   = ["_id"] # document UID
           unique = true
         },
         {
-          keys   = ["ref_fdr"]
-          unique = false
-        },
-        {
-          keys   = ["ref_fdr_sender_psp_id"]
-          unique = false
-        },
-      ]
-      shard_key = null
-    },
-    {
-      name = "fdr_payment_publish"
-      indexes = [
-        {
-          keys   = ["_id"] # index + ref_fdr_reporting_flow_name + ref_fdr_id
+          keys   = ["ref_fdr.id", "index"] # payment_by_fdr_idx
           unique = true
         },
-        {
-          keys   = ["ref_fdr"]
-          unique = false
-        },
-        {
-          keys   = ["ref_fdr_sender_psp_id"]
-          unique = false
-        }
+        #{
+        #  keys   = ["ref_fdr.sender_psp_id", "iuv", "created"] # payment_by_iuv_idx
+        #  unique = false
+        #}
       ]
-      shard_key = null
+      shard_key   = null,
+      ttl_seconds = var.cosmos_mongo_db_fdr_params.fdr_payment_container_ttl
     },
   ]
 }

src/domains/fdr-common/99_variables.tf

@@ -289,6 +289,9 @@ variable "cosmos_mongo_db_fdr_params" {
     throughput                        = number
     max_throughput                    = number
     container_default_ttl             = number
+    # single container variables
+    fdr_flow_container_ttl    = string
+    fdr_payment_container_ttl = string
   })
 }
 

src/domains/fdr-common/README.md

@@ -143,7 +143,7 @@
 | <a name="input_cidr_subnet_cosmosdb_fdr"></a> [cidr\_subnet\_cosmosdb\_fdr](#input\_cidr\_subnet\_cosmosdb\_fdr) | Cosmos DB address space for fdr. | `list(string)` | n/a | yes |
 | <a name="input_cidr_subnet_flex_dbms"></a> [cidr\_subnet\_flex\_dbms](#input\_cidr\_subnet\_flex\_dbms) | Postgresql network address space. | `list(string)` | n/a | yes |
 | <a name="input_cidr_subnet_storage_account"></a> [cidr\_subnet\_storage\_account](#input\_cidr\_subnet\_storage\_account) | Storage account network address space. | `list(string)` | n/a | yes |
-| <a name="input_cosmos_mongo_db_fdr_params"></a> [cosmos\_mongo\_db\_fdr\_params](#input\_cosmos\_mongo\_db\_fdr\_params) | n/a | <pre>object({<br/>    enabled        = bool<br/>    capabilities   = list(string)<br/>    offer_type     = string<br/>    server_version = string<br/>    kind           = string<br/>    consistency_policy = object({<br/>      consistency_level       = string<br/>      max_interval_in_seconds = number<br/>      max_staleness_prefix    = number<br/>    })<br/>    main_geo_location_zone_redundant = bool<br/>    enable_free_tier                 = bool<br/>    additional_geo_locations = list(object({<br/>      location          = string<br/>      failover_priority = number<br/>      zone_redundant    = bool<br/>    }))<br/>    private_endpoint_enabled          = bool<br/>    public_network_access_enabled     = bool<br/>    is_virtual_network_filter_enabled = bool<br/>    backup_continuous_enabled         = bool<br/>    enable_serverless                 = bool<br/>    enable_autoscaling                = bool<br/>    throughput                        = number<br/>    max_throughput                    = number<br/>    container_default_ttl             = number<br/>  })</pre> | n/a | yes |
+| <a name="input_cosmos_mongo_db_fdr_params"></a> [cosmos\_mongo\_db\_fdr\_params](#input\_cosmos\_mongo\_db\_fdr\_params) | n/a | <pre>object({<br/>    enabled        = bool<br/>    capabilities   = list(string)<br/>    offer_type     = string<br/>    server_version = string<br/>    kind           = string<br/>    consistency_policy = object({<br/>      consistency_level       = string<br/>      max_interval_in_seconds = number<br/>      max_staleness_prefix    = number<br/>    })<br/>    main_geo_location_zone_redundant = bool<br/>    enable_free_tier                 = bool<br/>    additional_geo_locations = list(object({<br/>      location          = string<br/>      failover_priority = number<br/>      zone_redundant    = bool<br/>    }))<br/>    private_endpoint_enabled          = bool<br/>    public_network_access_enabled     = bool<br/>    is_virtual_network_filter_enabled = bool<br/>    backup_continuous_enabled         = bool<br/>    enable_serverless                 = bool<br/>    enable_autoscaling                = bool<br/>    throughput                        = number<br/>    max_throughput                    = number<br/>    container_default_ttl             = number<br/>    # single container variables<br/>    fdr_flow_container_ttl    = string<br/>    fdr_payment_container_ttl = string<br/>  })</pre> | n/a | yes |
 | <a name="input_cosmos_mongo_db_fdr_re_params"></a> [cosmos\_mongo\_db\_fdr\_re\_params](#input\_cosmos\_mongo\_db\_fdr\_re\_params) | n/a | <pre>object({<br/>    capabilities   = list(string)<br/>    offer_type     = string<br/>    server_version = string<br/>    kind           = string<br/>    consistency_policy = object({<br/>      consistency_level       = string<br/>      max_interval_in_seconds = number<br/>      max_staleness_prefix    = number<br/>    })<br/>    main_geo_location_zone_redundant = bool<br/>    enable_free_tier                 = bool<br/>    additional_geo_locations = list(object({<br/>      location          = string<br/>      failover_priority = number<br/>      zone_redundant    = bool<br/>    }))<br/>    private_endpoint_enabled          = bool<br/>    public_network_access_enabled     = bool<br/>    is_virtual_network_filter_enabled = bool<br/>    backup_continuous_enabled         = bool<br/>    enable_serverless                 = bool<br/>    enable_autoscaling                = bool<br/>    throughput                        = number<br/>    max_throughput                    = number<br/>    container_default_ttl             = number<br/>  })</pre> | n/a | yes |
 | <a name="input_custom_metric_alerts"></a> [custom\_metric\_alerts](#input\_custom\_metric\_alerts) | Map of name = criteria objects | <pre>map(object({<br/>    # criteria.*.aggregation to be one of [Average Count Minimum Maximum Total]<br/>    aggregation = string<br/>    metric_name = string<br/>    # "Insights.Container/pods" "Insights.Container/nodes"<br/>    metric_namespace = string<br/>    # criteria.0.operator to be one of [Equals NotEquals GreaterThan GreaterThanOrEqual LessThan LessThanOrEqual]<br/>    operator  = string<br/>    threshold = number<br/>    # Possible values are PT1M, PT5M, PT15M, PT30M and PT1H<br/>    frequency = string<br/>    # Possible values are PT1M, PT5M, PT15M, PT30M, PT1H, PT6H, PT12H and P1D.<br/>    window_size = string<br/>    # severity: The severity of this Metric Alert. Possible values are 0, 1, 2, 3 and 4. Defaults to 3.<br/>    severity = number<br/>  }))</pre> | <pre>{<br/>  "active_connections": {<br/>    "aggregation": "Average",<br/>    "frequency": "PT5M",<br/>    "metric_name": "active_connections",<br/>    "metric_namespace": "Microsoft.DBforPostgreSQL/flexibleServers",<br/>    "operator": "GreaterThan",<br/>    "severity": 2,<br/>    "threshold": 80,<br/>    "window_size": "PT30M"<br/>  },<br/>  "connections_failed": {<br/>    "aggregation": "Total",<br/>    "frequency": "PT5M",<br/>    "metric_name": "connections_failed",<br/>    "metric_namespace": "Microsoft.DBforPostgreSQL/flexibleServers",<br/>    "operator": "GreaterThan",<br/>    "severity": 2,<br/>    "threshold": 80,<br/>    "window_size": "PT30M"<br/>  },<br/>  "cpu_percent": {<br/>    "aggregation": "Average",<br/>    "frequency": "PT5M",<br/>    "metric_name": "cpu_percent",<br/>    "metric_namespace": "Microsoft.DBforPostgreSQL/flexibleServers",<br/>    "operator": "GreaterThan",<br/>    "severity": 2,<br/>    "threshold": 4500,<br/>    "window_size": "PT30M"<br/>  },<br/>  "memory_percent": {<br/>    "aggregation": "Average",<br/>    "frequency": "PT5M",<br/>    "metric_name": "memory_percent",<br/>    "metric_namespace": "Microsoft.DBforPostgreSQL/flexibleServers",<br/>    "operator": "GreaterThan",<br/>    "severity": 2,<br/>    "threshold": 80,<br/>    "window_size": "PT30M"<br/>  },<br/>  "storage_percent": {<br/>    "aggregation": "Average",<br/>    "frequency": "PT5M",<br/>    "metric_name": "storage_percent",<br/>    "metric_namespace": "Microsoft.DBforPostgreSQL/flexibleServers",<br/>    "operator": "GreaterThan",<br/>    "severity": 2,<br/>    "threshold": 80,<br/>    "window_size": "PT30M"<br/>  }<br/>}</pre> | no |
 | <a name="input_dns_zone_internal_prefix"></a> [dns\_zone\_internal\_prefix](#input\_dns\_zone\_internal\_prefix) | The dns subdomain. | `string` | `null` | no |

src/domains/fdr-common/env/weu-dev/terraform.tfvars

@@ -116,7 +116,7 @@ cidr_subnet_cosmosdb_fdr = ["10.1.136.0/24"]
 cosmos_mongo_db_fdr_params = {
   enabled      = true
   kind         = "MongoDB"
-  capabilities = ["EnableMongo"]
+  capabilities = ["EnableMongo", "EnableUniqueCompoundNestedDocs"]
   offer_type   = "Standard"
   consistency_policy = {
     consistency_level       = "BoundedStaleness"
@@ -140,6 +140,9 @@ cosmos_mongo_db_fdr_params = {
   throughput         = 1000
 
   container_default_ttl = 604800 # 7 days
+
+  fdr_flow_container_ttl    = "604800" # 7 days
+  fdr_payment_container_ttl = "604800" # 7 days
 }
 
 cosmos_mongo_db_fdr_re_params = {

src/domains/fdr-common/env/weu-prod/terraform.tfvars

@@ -118,7 +118,7 @@ cidr_subnet_cosmosdb_fdr = ["10.1.136.0/24"]
 cosmos_mongo_db_fdr_params = {
   enabled      = true
   kind         = "MongoDB"
-  capabilities = ["EnableMongo"] # Serverless accounts do not support multiple regions
+  capabilities = ["EnableMongo", "EnableUniqueCompoundNestedDocs"] # Serverless accounts do not support multiple regions
   offer_type   = "Standard"
   consistency_policy = {
     consistency_level       = "BoundedStaleness"
@@ -143,6 +143,9 @@ cosmos_mongo_db_fdr_params = {
 
   container_default_ttl = 315576000 # 10 year in second
 
+  fdr_flow_container_ttl    = "3024000" # 30 days + 5 days (deltaTime)
+  fdr_payment_container_ttl = "3024000" # 30 days + 5 days (deltaTime)
+
   enable_serverless  = false
   enable_autoscaling = true
   max_throughput     = 2000

src/domains/fdr-common/env/weu-uat/terraform.tfvars

@@ -116,7 +116,7 @@ cidr_subnet_cosmosdb_fdr = ["10.1.136.0/24"]
 cosmos_mongo_db_fdr_params = {
   enabled      = true
   kind         = "MongoDB"
-  capabilities = ["EnableMongo"]
+  capabilities = ["EnableMongo", "EnableUniqueCompoundNestedDocs"]
   offer_type   = "Standard"
   consistency_policy = {
     consistency_level       = "BoundedStaleness"
@@ -136,6 +136,9 @@ cosmos_mongo_db_fdr_params = {
 
   container_default_ttl = 2629800 # 1 month in second
 
+  fdr_flow_container_ttl    = "3024000" # 30 days + 5 days (deltaTime)
+  fdr_payment_container_ttl = "3024000" # 30 days + 5 days (deltaTime)
+
   enable_serverless  = false
   enable_autoscaling = true
   max_throughput     = 15000