Workflow Service Validation (#197)

Azure · Sep 12, 2024 · e2f2c01 · e2f2c01
1 parent 225907f
commit e2f2c01
Show file tree

Hide file tree

Showing 9 changed files with 330 additions and 50 deletions.
diff --git a/bicep/modules/blade_partition.bicep b/bicep/modules/blade_partition.bicep
@@ -207,7 +207,7 @@ var partitionLayerConfig = {
         ]
       }
       {
-        name: 'ReplayIndexStatus'
+        name: 'ReplayStatus'
         kind: 'Hash'
         paths: [
           '/id'
@@ -269,20 +269,48 @@ var partitionLayerConfig = {
           '/operatorId'
         ]
       }
+      {
+        name: 'WorkflowCustomOperatorV2'
+        kind: 'Hash'
+        paths: [
+          '/partitionKey'
+        ]
+      }
       {
         name: 'WorkflowRun'
         kind: 'Hash'
         paths: [
           '/partitionKey'
         ]
       }
+      {
+        name: 'WorkflowRunV2'
+        kind: 'Hash'
+        paths: [
+          '/partitionKey'
+        ]
+      }
       {
         name: 'WorkflowRunStatus'
         kind: 'Hash'
         paths: [
           '/partitionKey'
         ]
       }
+      // {
+      //   name: 'WorkflowTasksSharingInfoV2'
+      //   kind: 'Hash'
+      //   paths: [
+      //     '/partitionKey'
+      //   ]
+      // }
+      {
+        name: 'WorkflowV2'
+        kind: 'Hash'
+        paths: [
+          '/partitionKey'
+        ]
+      }
     ]
   }
   servicebus: {
@@ -294,6 +322,8 @@ var partitionLayerConfig = {
         subscriptions: [
           {
             name: 'indexing-progresssubscription'
+            maxDeliveryCount: 5
+            lockDuration: 'PT5M'
           }
         ]
       }
@@ -303,6 +333,8 @@ var partitionLayerConfig = {
         subscriptions: [
           {
             name: 'legaltagssubscription'
+            maxDeliveryCount: 5
+            lockDuration: 'PT5M'
           }
         ]
       }
@@ -312,6 +344,13 @@ var partitionLayerConfig = {
         subscriptions: [
           {
             name: 'recordstopicsubscription'
+            maxDeliveryCount: 5
+            lockDuration: 'PT5M'
+          }
+          {
+            name: 'wkssubscription'
+            maxDeliveryCount: 5
+            lockDuration: 'PT5M'
           }
         ]
       }
@@ -321,6 +360,8 @@ var partitionLayerConfig = {
         subscriptions: [
           {
             name: 'downstreamsub'
+            maxDeliveryCount: 5
+            lockDuration: 'PT5M'
           }
         ]
       }
@@ -330,6 +371,8 @@ var partitionLayerConfig = {
         subscriptions: [
           {
             name: 'eg_sb_wkssubscription'
+            maxDeliveryCount: 5
+            lockDuration: 'PT5M'
           }
         ]
       }
@@ -339,6 +382,8 @@ var partitionLayerConfig = {
         subscriptions: [
           {
             name: 'schemachangedtopicsubscription'
+            maxDeliveryCount: 5
+            lockDuration: 'PT5M'
           }
         ]
       }
@@ -348,9 +393,8 @@ var partitionLayerConfig = {
         subscriptions: [
           {
             name: 'eg_sb_schemasubscription'
-          }
-          {
-            name: 'schemachangedtopicsubscription'
+            maxDeliveryCount: 5
+            lockDuration: 'PT5M'
           }
         ]
       }
@@ -360,6 +404,8 @@ var partitionLayerConfig = {
         subscriptions: [
           {
             name: 'eg_sb_legaltagssubscription'
+            maxDeliveryCount: 5
+            lockDuration: 'PT5M'
           }
         ]
       }
@@ -368,29 +414,63 @@ var partitionLayerConfig = {
         maxSizeInMegabytes: 5120
         subscriptions: [
           {
-            name: 'eg_sb_statussubscription'
+            name: 'statuschangedtopicsubscription'
+            maxDeliveryCount: 5
+            lockDuration: 'PT5M'
           }
         ]
       }
       {
         name: 'statuschangedtopiceg'
         maxSizeInMegabytes: 1024
-        subscriptions: []
+        subscriptions: [
+          {
+            name: 'eg_sb_statussubscription'
+            maxDeliveryCount: 5
+            lockDuration: 'PT5M'
+          }
+        ]
       }
       {
-        name: 'replayrecordtopic'
+        name: 'recordstopic-v2'
         maxSizeInMegabytes: 1024
-        subscriptions: []
+        subscriptions: [
+          {
+            name: 'recordstopic-v2-subscription'
+            maxDeliveryCount: 5
+            lockDuration: 'PT5M'
+          }
+        ]
       }
       {
         name: 'reindextopic'
         maxSizeInMegabytes: 1024
         subscriptions: [
           {
             name: 'reindextopicsubscription'
+            maxDeliveryCount: 5
+            lockDuration: 'PT5M'
+            enableDeadLetteringOnMessageExpiration: false
           }
         ]
       }
+      {
+        name: 'entitlements-changed'
+        maxSizeInMegabytes: 1024
+        subscriptions: []
+      }
+      {
+        name: 'replaytopic'
+        maxSizeInMegabytes: 1024
+        subscriptions: [
+          {
+            name: 'replaytopicsubscription'
+            maxDeliveryCount: 5
+            lockDuration: 'PT5M'
+          }
+        ]
+      }
+
     ]
   }
 }
@@ -532,7 +612,7 @@ module partitionDbEndpoint './private-endpoint/main.bicep' = [for (partition, in
 }]
 
 
-module partitonNamespace 'br/public:avm/res/service-bus/namespace:0.4.2' = [for (partition, index) in partitions:  {
+module partitonNamespace 'br/public:avm/res/service-bus/namespace:0.9.0' = [for (partition, index) in partitions:  {
   name: '${bladeConfig.sectionName}-service-bus-${index}'
   params: {
     name: '${replace('data${index}${substring(uniqueString(partition.name), 0, 6)}', '-', '')}${uniqueString(resourceGroup().id, 'data${index}${substring(uniqueString(partition.name), 0, 6)}')}'
@@ -557,8 +637,11 @@ module partitonNamespace 'br/public:avm/res/service-bus/namespace:0.4.2' = [for
 
     skuObject: {
       name: partitionLayerConfig.servicebus.sku
+      capacity: partitionLayerConfig.servicebus.sku == 'Premium' ? 2 : null
     }
 
+    zoneRedundant: partitionLayerConfig.servicebus.sku == 'Premium' ? true : false
+
     disableLocalAuth: false
 
     authorizationRules: [

diff --git a/docs/src/osdu_core.md b/docs/src/osdu_core.md
@@ -288,13 +288,24 @@ Class: org.opengroup.osdu.search.provider.azure.SearchApplication
 | `AZURE_HOST`                         | `<your_host_ip>`                               | Azure host IP                              |
 | `PARTITION_SERVICE_ENDPOINT`         | `http://${AZURE_HOST}/api/partition/v1`        | Partition service endpoint                 |
 | `ENTITLEMENTS_SERVICE_ENDPOINT`      | `http://${AZURE_HOST}/api/entitlements/v2`     | Entitlements service endpoint              |
+| `POLICY_SERVICE_ENDPOINT`            | `http://${AZURE_HOST}/policy/api/policy/v1`    | Policy service endpoint                    |
 | `AAD_CLIENT_ID`                      | `<your_aad_client_id>`                         | Active Directory client ID                 |
 | `SPRING_APPLICATION_NAME`            | `search`                                       | Spring application name                    |
 | `AZURE_PAAS_PODIDENTITY`             | `false`                                        | Azure PaaS pod identity                    |
 | `AZURE_ISTIOAUTH_ENABLED`            | `true`                                         | Turn Istio auth on                         |
 | `AZURE_PAAS_PODIDENTITY_ISENABLED`   | `false`                                        | Azure PaaS pod identity enabled            |
 | `LOG_PREFIX`                         | `search`                                       | Log prefix                                 |
-
+| `SEARCH_SERVICE_SPRING_LOGGING_LEVEL`| `DEBUG`                                        | Logging level for the Search service       |
+| `ENTITLEMENTS_SERVICE_API_KEY`       | `OBSOLETE`                                     | API Key for Entitlements                   |
+| `POLICY_SERVICE_ENABLED`             | `false`                                        | Enable Policy Service                      |
+| `COSMOSDB_DATABASE`                  | `osdu-db`                                      | Cosmos DB database name                    |
+| `REDIS_DATABASE`                     | `5`                                            | Redis database number                      |
+| `ENVIRONMENT`                        | `evt`                                          | Environment                                |
+| `ELASTIC_CACHE_EXPIRATION`           | `1`                                            | Elastic cache expiration                   |
+| `MAX_CACHE_VALUE_SIZE`               | `60`                                           | Maximum cache value size                   |
+| `PARTITION_SERVICE_ENDPOINT`         | `http://partition/api/partition/v1`            | Partition service endpoint                 |
+| `ENTITLEMENTS_SERVICE_ENDPOINT`      | `http://entitlements/api/entitlements/v2`      | Entitlements service endpoint              |
+| `POLICY_SERVICE_ENDPOINT`            | `http://policy/api/policy/v1`                  | Policy service endpoint                    |
 
 ## File Service
 

diff --git a/software/applications/osdu-core/search.yaml b/software/applications/osdu-core/search.yaml
@@ -125,13 +125,13 @@ spec:
         value: 1
       - name: MAX_CACHE_VALUE_SIZE
         value: 60
-      - name: SERVICE_POLICY_ENABLED
-        value: "false"
-      - name: SERVICE_POLICY_ENABLED
+      - name: POLICY_SERVICE_ENABLED
         value: "false"
+      - name: ENTITLEMENTS_SERVICE_API_KEY
+        value: "OBSOLETE"
       - name: PARTITION_SERVICE_ENDPOINT
         value: "http://partition/api/partition/v1"
       - name: ENTITLEMENTS_SERVICE_ENDPOINT
         value: "http://entitlements/api/entitlements/v2"
-      - name: ENTITLEMENTS_SERVICE_API_KEY
-        value: "OBSOLETE"
+      - name: POLICY_SERVICE_ENDPOINT
+        value: http://policy/api/policy/v1
diff --git a/software/applications/osdu-core/workflow.yaml b/software/applications/osdu-core/workflow.yaml
@@ -96,14 +96,6 @@ spec:
         secret:
           name: azure-resources
           key: insights-connection
-      - name: OSDU_AIRFLOW_USERNAME
-        secret:
-          name: azure-resources
-          key: airflow-username
-      - name: OSDU_AIRFLOW_PASSWORD
-        secret:
-          name: azure-resources
-          key: airflow-password
       - name: AZURE_ISTIOAUTH_ENABLED
         value: "true"
       - name: AZURE_PAAS_PODIDENTITY_ISENABLED
@@ -122,23 +114,42 @@ spec:
         value: "workflow"
       - name: AZURE_STORAGE_ENABLE_HTTPS
         value: "true"
+
+
       - name: COSMOSDB_DATABASE
         value: "osdu-db"
+      - name: COSMOSDB_SYSTEM_DATABASE
+        value: osdu-system-db
+
+      - name: OSDU_AIRFLOW_USERNAME
+        secret:
+          name: azure-resources
+          key: airflow-username
+      - name: OSDU_AIRFLOW_PASSWORD
+        secret:
+          name: azure-resources
+          key: airflow-password
+
+      - name: AIRFLOW_STORAGE_ACCOUNT_NAME
+        secret:
+          name: azure-resources
+          key: azurestorageaccountname
+      - name: AIRFLOW_STORAGE_ACCOUNT_KEY
+        secret:
+          name: azure-resources
+          key: azurestorageaccountkey
+
+
+
+      - name: AUTHORIZEAPI
+        value: http://entitlements/api/entitlements/v2
+      - name: AUTHORIZEAPIKEY
+        value: "OBSOLETE"
       - name: OSDU_AIRFLOW_URL
         value: "http://airflow-webserver.airflow.svc.cluster.local/airflow"
       - name: DP_AIRFLOW_FOR_SYSTEM_DAG
         value: "false"
 
-
-      # - name: AIRFLOW_STORAGE_ACCOUNT_NAME
-      #   secret:
-      #     name: azure-resources
-      #     key: azurestorageaccountname
-      # - name: AIRFLOW_STORAGE_ACCOUNT_KEY
-      #   secret:
-      #     name: azure-resources
-      #     key: azurestorageaccountkey
-
       - name: OSDU_AIRFLOW_VERSION2_ENABLED
         value: true
       - name: PARTITION_SERVICE_ENDPOINT