feat: Enabled cdc gpd ingestion PROD (#2680)

* enabled cdd gpd ingestion PROD * fix * add secrets * disable cdp app gps domain
pagopa · Dec 18, 2024 · a5fd034 · a5fd034
1 parent 2189bb6
commit a5fd034
Show file tree

Hide file tree

Showing 5 changed files with 81 additions and 70 deletions.
diff --git a/src/domains/gps-common/env/weu-prod/terraform.tfvars b/src/domains/gps-common/env/weu-prod/terraform.tfvars
@@ -166,4 +166,4 @@ gpd_sftp_sa_delete                                             = 60
 # GPD Archive account
 gpd_archive_replication_type = "GZRS"
 gpd_sftp_ip_rules            = ["37.179.98.148"]
-gpd_cdc_enabled              = false
+gpd_cdc_enabled              = true
diff --git a/src/domains/gps-secret/secret/weu-prod/noedit_secret_enc.json b/src/domains/gps-secret/secret/weu-prod/noedit_secret_enc.json
@@ -2,7 +2,7 @@
 	"config-cache-subscription-key": "ENC[AES256_GCM,data:Boy/LRbM7n6jeI3f8JIj77dSop9bmSYcClF5yOPnHP8=,iv:mYwuAQdw90B2tsXMiGLNrbDVu2RtXFwDU6Umc78DzKI=,tag:B1fh1tExQVpAd4ZpU0khWA==,type:str]",
 	"cdc-logical-replication-apd-user": "ENC[AES256_GCM,data:2bYTx4PH,iv:NMEyW6KBgwEMmWdbBIkWcI+fn5fSJ3tmIr6BXtCQOCo=,tag:e9QnridAM2ih7Kw1x3e0Zg==,type:str]",
 	"cdc-logical-replication-apd-pwd": "ENC[AES256_GCM,data:qocdlc1Z5Z4/F5F0fjGxJ0j4,iv:wEl471Vupsq00KkAeZ4iHddeXTiDCk/Noe4hKFW9wUw=,tag:S9yYq2mkq3bIBPTYsbTE5Q==,type:str]",
-	"tokenizer-api-key": "ENC[AES256_GCM,data:AVr5/g==,iv:DjYiP+y4W9WGiyBY5aNUiIRvK/4YQcFcoz5Bo7H7zwE=,tag:ffgLum7cTTXjrG4gR7zobg==,type:str]",
+	"tokenizer-api-key": "ENC[AES256_GCM,data:XzvJZX/FHIO6uPHZUjyrmV6iLQpeb/IylmXruODIGlXTJBrlAT/LZg==,iv:NxOqm6+RZaLXh5c9NsxC3tUBCW1lX/pt4mu9ELn6dWE=,tag:7zv6LL+UjEkZgH4g6fKOyQ==,type:str]",
 	"pagopa-platform-domain-github-bot-cd-pat": "ENC[AES256_GCM,data:/lVJ1Ew8C+yu/tTNweZR6Pv6lelGfbMnOsaIKgDIF1DlbnFd/8QAjw==,iv:kOHYJCUuYm90oNpJfXYbrVC1UNgETwJvbGmkuItC5Aw=,tag:vaFMYUX9yws7IMZ/H+kJcg==,type:str]",
 	"pagopa-platform-domain-github-bot-pwd": "ENC[AES256_GCM,data:dotkpOEa1sMxH4TRy4WD,iv:+SKoRE7cmESieOyRauuPVkkwgAPGBd6RdfatamyX8co=,tag:d9ZNPbEYQITGKS/MNW2qFA==,type:str]",
 	"sops": {
@@ -19,8 +19,8 @@
 		],
 		"hc_vault": null,
 		"age": null,
-		"lastmodified": "2024-12-18T09:58:19Z",
-		"mac": "ENC[AES256_GCM,data:zqr4vXqugtVgun/jvQ6GQCHZstmVKHTQe+OYrY8/4o6xKM73hpeXegDy0VfjH4ie7cdRWjtTTbYkpHBbFJbkFixdXerSFytb2SoufUgvSGMO5j9OHwmyzLSCPnc6GScvwPn6oqt3gSNNmz1h/BShn8B6l3DTQgYtqiD5bSaJqcM=,iv:OzczCexUqzao9FpJRlbdW5P/M9bgSl3pBB8ARsqm7No=,tag:DS0nrOPgeL4pCduSfpfLRg==,type:str]",
+		"lastmodified": "2024-12-18T11:48:38Z",
+		"mac": "ENC[AES256_GCM,data:HeCJwXo2u9fSNerHd0RBOaqOoaNwIWlC2q7uUT4YFX8SzGnSov06vZp/XDxQ/ZkLMm3wLJd0ZtDQsXKAXb4n9XBjtYUF3ikeacApvSzsiVDKgAkAqZmTzTn9DOl9p3qUpBhwEdny2b1i6umtrQtcoEI2N4pHjsiCmF/w+H/Xlno=,iv:HqrsXeXETHL8a7fxOzjhCoE+LPzW6JbcHINBuOKl7Lw=,tag:W31+sWzgoIUvai9pMRewhw==,type:str]",
 		"pgp": null,
 		"unencrypted_suffix": "_unencrypted",
 		"version": "3.9.1"

diff --git a/src/domains/observability/03_eventhub_msg_gdp.tf b/src/domains/observability/03_eventhub_msg_gdp.tf
@@ -37,7 +37,7 @@ module "eventhub_namespace_observability_gpd" {
   ]
 
   metric_alerts_create = var.ehns_alerts_enabled
-  # metric_alerts        = var.ehns_metric_alerts_gpd
+  metric_alerts        = var.ehns_metric_alerts_gpd
 
   tags = var.tags
 }

diff --git a/src/domains/observability/env/uat/terraform.tfvars b/src/domains/observability/env/uat/terraform.tfvars
@@ -286,7 +286,7 @@ eventhubs_gpd = [
     name              = "gpd-ingestion.apd.payment_option"
     partitions        = 1
     message_retention = 1
-    consumers         = ["gpd-ingestion.apd.payment_option-rx-dl", ]
+    consumers         = ["gpd-ingestion.apd.payment_option-rx-dl"]
     keys = [
       {
         name   = "gpd-ingestion.apd.payment_option-rx-dl"
@@ -306,7 +306,7 @@ eventhubs_gpd = [
     name              = "gpd-ingestion.apd.payment_position"
     partitions        = 1
     message_retention = 1
-    consumers         = ["gpd-ingestion.apd.payment_position-rx-dl", ]
+    consumers         = ["gpd-ingestion.apd.payment_position-rx-dl"]
     keys = [
       {
         name   = "gpd-ingestion.apd.payment_position-rx-dl"
@@ -326,7 +326,7 @@ eventhubs_gpd = [
     name              = "gpd-ingestion.apd.transfer"
     partitions        = 1
     message_retention = 1
-    consumers         = ["gpd-ingestion.apd.transfer-rx-dl", ]
+    consumers         = ["gpd-ingestion.apd.transfer-rx-dl"]
     keys = [
       {
         name   = "gpd-ingestion.apd.transfer-rx-dl"
@@ -345,53 +345,52 @@ eventhubs_gpd = [
 ]
 
 
-# alert evh
-# ehns_metric_alerts_gpd = {
-#   no_trx = {
-#     aggregation = "Total"
-#     metric_name = "IncomingMessages"
-#     description = "No transactions received from acquirer in the last 24h"
-#     operator    = "LessThanOrEqual"
-#     threshold   = 1000
-#     frequency   = "PT1H"
-#     window_size = "P1D"
-#     dimension = [
-#       {
-#         name     = "EntityName"
-#         operator = "Include"
-#         values   = ["gec-ingestion-bundles-evt-tx", "gec-ingestion-cibundles-evt-tx", "gec-ingestion-paymenttypes-evt-tx", "gec-ingestion-touchpoints-evt-tx"]
-#       }
-#     ],
-#   },
-#   active_connections = {
-#     aggregation = "Average"
-#     metric_name = "ActiveConnections"
-#     description = null
-#     operator    = "LessThanOrEqual"
-#     threshold   = 0
-#     frequency   = "PT5M"
-#     window_size = "PT15M"
-#     dimension   = [],
-#   },
-#   error_trx = {
-#     aggregation = "Total"
-#     metric_name = "IncomingMessages"
-#     description = "Transactions rejected from one acquirer file received. trx write on eventhub. check immediately"
-#     operator    = "GreaterThan"
-#     threshold   = 0
-#     frequency   = "PT5M"
-#     window_size = "PT30M"
-#     dimension = [
-#       {
-#         name     = "EntityName"
-#         operator = "Include"
-#         values = [
-#           "gec-ingestion-bundles-evt-rx-pdnd",
-#           "gec-ingestion-cibundles-evt-rx-pdnd",
-#           "gec-ingestion-paymenttypes-evt-rx-pdnd",
-#           "gec-ingestion-touchpoints-evt-rx-pdnd"
-#         ]
-#       }
-#     ],
-#   },
-# }
+#  alert evh
+ehns_metric_alerts_gpd = {
+  no_trx = {
+    aggregation = "Total"
+    metric_name = "IncomingMessages"
+    description = "No transactions received from acquirer in the last 24h"
+    operator    = "LessThanOrEqual"
+    threshold   = 1000
+    frequency   = "PT1H"
+    window_size = "P1D"
+    dimension = [
+      {
+        name     = "EntityName"
+        operator = "Include"
+        values   = ["gpd-ingestion.apd.payment_option-tx", "gpd-ingestion.apd.payment_position-tx","gpd-ingestion.apd.transfer-tx"]
+      }
+    ],
+  },
+  active_connections = {
+    aggregation = "Average"
+    metric_name = "ActiveConnections"
+    description = null
+    operator    = "LessThanOrEqual"
+    threshold   = 0
+    frequency   = "PT5M"
+    window_size = "PT15M"
+    dimension   = [],
+  },
+  error_trx = {
+    aggregation = "Total"
+    metric_name = "IncomingMessages"
+    description = "Transactions rejected from one acquirer file received. trx write on eventhub. check immediately"
+    operator    = "GreaterThan"
+    threshold   = 0
+    frequency   = "PT5M"
+    window_size = "PT30M"
+    dimension = [
+      {
+        name     = "EntityName"
+        operator = "Include"
+        values = [
+          "gpd-ingestion.apd.payment_option-rx-dl",
+          "gpd-ingestion.apd.payment_position-rx-dl",
+          "gpd-ingestion.apd.transfer-rx-dl"
+        ]
+      }
+    ],
+  },
+}
diff --git a/src/domains/observability/gpd_evh_create__az.sh b/src/domains/observability/gpd_evh_create__az.sh
@@ -8,6 +8,17 @@ set -e
 
 env=$1
 
+if [ "$env" != "p" ]; then
+    partitioncount=1
+    retentiontime=24 #hh = 1day
+else
+    partitioncount=32
+    retentiontime=168 #hh = 1week
+fi
+
+echo "partitioncount >> ${partitioncount}"
+echo "retentiontime  >> ${retentiontime}"
+
 
 echo ">>>>>> 1"
 
@@ -17,8 +28,8 @@ az eventhubs eventhub create \
 --namespace-name pagopa-$env-itn-observ-gpd-evh \
 --cleanup-policy "Compact" \
 --status "Active" \
---partition-count 1 \
---retention-time 24
+--partition-count ${partitioncount} \
+--retention-time ${retentiontime}
 
 echo ">>>>>> 2"
 
@@ -28,8 +39,8 @@ az eventhubs eventhub create \
 --namespace-name pagopa-$env-itn-observ-gpd-evh \
 --cleanup-policy "Compact" \
 --status "Active" \
---partition-count 1 \
---retention-time 24
+--partition-count ${partitioncount} \
+--retention-time ${retentiontime}
 
 echo ">>>>>> 3"
 
@@ -39,8 +50,8 @@ az eventhubs eventhub create \
 --namespace-name pagopa-$env-itn-observ-gpd-evh \
 --cleanup-policy "Compact" \
 --status "Active" \
---partition-count 1 \
---retention-time 24
+--partition-count ${partitioncount} \
+--retention-time ${retentiontime}
 
 # ==============================================================
 # logical topics
@@ -54,9 +65,10 @@ az eventhubs eventhub create \
 --namespace-name pagopa-$env-itn-observ-gpd-evh \
 --cleanup-policy "Compact" \
 --status "Active" \
---partition-count 1 \
---retention-time 24
+--partition-count ${partitioncount} \
+--retention-time ${retentiontime}
 
+# created by @aferracci tnx2 MS https://github.com/Azure/azure-cli/issues/30419
 az eventhubs eventhub authorization-rule create \
 --resource-group pagopa-$env-itn-observ-evh-rg \
 --namespace-name pagopa-$env-itn-observ-gpd-evh \
@@ -72,8 +84,8 @@ az eventhubs eventhub create \
 --namespace-name pagopa-$env-itn-observ-gpd-evh \
 --cleanup-policy "Compact" \
 --status "Active" \
---partition-count 1 \
---retention-time 24
+--partition-count ${partitioncount} \
+--retention-time ${retentiontime}
 
 az eventhubs eventhub authorization-rule create \
 --resource-group pagopa-$env-itn-observ-evh-rg \
@@ -90,8 +102,8 @@ az eventhubs eventhub create \
 --namespace-name pagopa-$env-itn-observ-gpd-evh \
 --cleanup-policy "Compact" \
 --status "Active" \
---partition-count 1 \
---retention-time 24
+--partition-count ${partitioncount} \
+--retention-time ${retentiontime}
 
 
 az eventhubs eventhub authorization-rule create \