From e2f2c01d778e37d4b27d1bdd69214bf4b2e61e4e Mon Sep 17 00:00:00 2001 From: danielscholl Date: Thu, 12 Sep 2024 18:58:07 -0500 Subject: [PATCH] Workflow Service Validation (#197) --- bicep/modules/blade_partition.bicep | 101 ++++++++++++++++-- docs/src/osdu_core.md | 13 ++- software/applications/osdu-core/search.yaml | 10 +- software/applications/osdu-core/workflow.yaml | 47 ++++---- software/components/airflow/release.yaml | 101 +++++++++++++++--- .../components/airflow/vault-secrets.yaml | 4 +- src/core/repos | 3 +- tools/rest-scripts/check-file.http | 2 +- tools/rest-scripts/workflow.http | 99 +++++++++++++++++ 9 files changed, 330 insertions(+), 50 deletions(-) create mode 100644 tools/rest-scripts/workflow.http diff --git a/bicep/modules/blade_partition.bicep b/bicep/modules/blade_partition.bicep index 67d30db5..21cdcf4c 100644 --- a/bicep/modules/blade_partition.bicep +++ b/bicep/modules/blade_partition.bicep @@ -207,7 +207,7 @@ var partitionLayerConfig = { ] } { - name: 'ReplayIndexStatus' + name: 'ReplayStatus' kind: 'Hash' paths: [ '/id' @@ -269,6 +269,13 @@ var partitionLayerConfig = { '/operatorId' ] } + { + name: 'WorkflowCustomOperatorV2' + kind: 'Hash' + paths: [ + '/partitionKey' + ] + } { name: 'WorkflowRun' kind: 'Hash' @@ -276,6 +283,13 @@ var partitionLayerConfig = { '/partitionKey' ] } + { + name: 'WorkflowRunV2' + kind: 'Hash' + paths: [ + '/partitionKey' + ] + } { name: 'WorkflowRunStatus' kind: 'Hash' @@ -283,6 +297,20 @@ var partitionLayerConfig = { '/partitionKey' ] } + // { + // name: 'WorkflowTasksSharingInfoV2' + // kind: 'Hash' + // paths: [ + // '/partitionKey' + // ] + // } + { + name: 'WorkflowV2' + kind: 'Hash' + paths: [ + '/partitionKey' + ] + } ] } servicebus: { @@ -294,6 +322,8 @@ var partitionLayerConfig = { subscriptions: [ { name: 'indexing-progresssubscription' + maxDeliveryCount: 5 + lockDuration: 'PT5M' } ] } @@ -303,6 +333,8 @@ var partitionLayerConfig = { subscriptions: [ { name: 'legaltagssubscription' + maxDeliveryCount: 5 + lockDuration: 'PT5M' } ] } @@ -312,6 +344,13 @@ var partitionLayerConfig = { subscriptions: [ { name: 'recordstopicsubscription' + maxDeliveryCount: 5 + lockDuration: 'PT5M' + } + { + name: 'wkssubscription' + maxDeliveryCount: 5 + lockDuration: 'PT5M' } ] } @@ -321,6 +360,8 @@ var partitionLayerConfig = { subscriptions: [ { name: 'downstreamsub' + maxDeliveryCount: 5 + lockDuration: 'PT5M' } ] } @@ -330,6 +371,8 @@ var partitionLayerConfig = { subscriptions: [ { name: 'eg_sb_wkssubscription' + maxDeliveryCount: 5 + lockDuration: 'PT5M' } ] } @@ -339,6 +382,8 @@ var partitionLayerConfig = { subscriptions: [ { name: 'schemachangedtopicsubscription' + maxDeliveryCount: 5 + lockDuration: 'PT5M' } ] } @@ -348,9 +393,8 @@ var partitionLayerConfig = { subscriptions: [ { name: 'eg_sb_schemasubscription' - } - { - name: 'schemachangedtopicsubscription' + maxDeliveryCount: 5 + lockDuration: 'PT5M' } ] } @@ -360,6 +404,8 @@ var partitionLayerConfig = { subscriptions: [ { name: 'eg_sb_legaltagssubscription' + maxDeliveryCount: 5 + lockDuration: 'PT5M' } ] } @@ -368,19 +414,33 @@ var partitionLayerConfig = { maxSizeInMegabytes: 5120 subscriptions: [ { - name: 'eg_sb_statussubscription' + name: 'statuschangedtopicsubscription' + maxDeliveryCount: 5 + lockDuration: 'PT5M' } ] } { name: 'statuschangedtopiceg' maxSizeInMegabytes: 1024 - subscriptions: [] + subscriptions: [ + { + name: 'eg_sb_statussubscription' + maxDeliveryCount: 5 + lockDuration: 'PT5M' + } + ] } { - name: 'replayrecordtopic' + name: 'recordstopic-v2' maxSizeInMegabytes: 1024 - subscriptions: [] + subscriptions: [ + { + name: 'recordstopic-v2-subscription' + maxDeliveryCount: 5 + lockDuration: 'PT5M' + } + ] } { name: 'reindextopic' @@ -388,9 +448,29 @@ var partitionLayerConfig = { subscriptions: [ { name: 'reindextopicsubscription' + maxDeliveryCount: 5 + lockDuration: 'PT5M' + enableDeadLetteringOnMessageExpiration: false } ] } + { + name: 'entitlements-changed' + maxSizeInMegabytes: 1024 + subscriptions: [] + } + { + name: 'replaytopic' + maxSizeInMegabytes: 1024 + subscriptions: [ + { + name: 'replaytopicsubscription' + maxDeliveryCount: 5 + lockDuration: 'PT5M' + } + ] + } + ] } } @@ -532,7 +612,7 @@ module partitionDbEndpoint './private-endpoint/main.bicep' = [for (partition, in }] -module partitonNamespace 'br/public:avm/res/service-bus/namespace:0.4.2' = [for (partition, index) in partitions: { +module partitonNamespace 'br/public:avm/res/service-bus/namespace:0.9.0' = [for (partition, index) in partitions: { name: '${bladeConfig.sectionName}-service-bus-${index}' params: { name: '${replace('data${index}${substring(uniqueString(partition.name), 0, 6)}', '-', '')}${uniqueString(resourceGroup().id, 'data${index}${substring(uniqueString(partition.name), 0, 6)}')}' @@ -557,8 +637,11 @@ module partitonNamespace 'br/public:avm/res/service-bus/namespace:0.4.2' = [for skuObject: { name: partitionLayerConfig.servicebus.sku + capacity: partitionLayerConfig.servicebus.sku == 'Premium' ? 2 : null } + zoneRedundant: partitionLayerConfig.servicebus.sku == 'Premium' ? true : false + disableLocalAuth: false authorizationRules: [ diff --git a/docs/src/osdu_core.md b/docs/src/osdu_core.md index 059e012f..573a2317 100644 --- a/docs/src/osdu_core.md +++ b/docs/src/osdu_core.md @@ -288,13 +288,24 @@ Class: org.opengroup.osdu.search.provider.azure.SearchApplication | `AZURE_HOST` | `` | Azure host IP | | `PARTITION_SERVICE_ENDPOINT` | `http://${AZURE_HOST}/api/partition/v1` | Partition service endpoint | | `ENTITLEMENTS_SERVICE_ENDPOINT` | `http://${AZURE_HOST}/api/entitlements/v2` | Entitlements service endpoint | +| `POLICY_SERVICE_ENDPOINT` | `http://${AZURE_HOST}/policy/api/policy/v1` | Policy service endpoint | | `AAD_CLIENT_ID` | `` | Active Directory client ID | | `SPRING_APPLICATION_NAME` | `search` | Spring application name | | `AZURE_PAAS_PODIDENTITY` | `false` | Azure PaaS pod identity | | `AZURE_ISTIOAUTH_ENABLED` | `true` | Turn Istio auth on | | `AZURE_PAAS_PODIDENTITY_ISENABLED` | `false` | Azure PaaS pod identity enabled | | `LOG_PREFIX` | `search` | Log prefix | - +| `SEARCH_SERVICE_SPRING_LOGGING_LEVEL`| `DEBUG` | Logging level for the Search service | +| `ENTITLEMENTS_SERVICE_API_KEY` | `OBSOLETE` | API Key for Entitlements | +| `POLICY_SERVICE_ENABLED` | `false` | Enable Policy Service | +| `COSMOSDB_DATABASE` | `osdu-db` | Cosmos DB database name | +| `REDIS_DATABASE` | `5` | Redis database number | +| `ENVIRONMENT` | `evt` | Environment | +| `ELASTIC_CACHE_EXPIRATION` | `1` | Elastic cache expiration | +| `MAX_CACHE_VALUE_SIZE` | `60` | Maximum cache value size | +| `PARTITION_SERVICE_ENDPOINT` | `http://partition/api/partition/v1` | Partition service endpoint | +| `ENTITLEMENTS_SERVICE_ENDPOINT` | `http://entitlements/api/entitlements/v2` | Entitlements service endpoint | +| `POLICY_SERVICE_ENDPOINT` | `http://policy/api/policy/v1` | Policy service endpoint | ## File Service diff --git a/software/applications/osdu-core/search.yaml b/software/applications/osdu-core/search.yaml index be1fa8af..ffb69963 100644 --- a/software/applications/osdu-core/search.yaml +++ b/software/applications/osdu-core/search.yaml @@ -125,13 +125,13 @@ spec: value: 1 - name: MAX_CACHE_VALUE_SIZE value: 60 - - name: SERVICE_POLICY_ENABLED - value: "false" - - name: SERVICE_POLICY_ENABLED + - name: POLICY_SERVICE_ENABLED value: "false" + - name: ENTITLEMENTS_SERVICE_API_KEY + value: "OBSOLETE" - name: PARTITION_SERVICE_ENDPOINT value: "http://partition/api/partition/v1" - name: ENTITLEMENTS_SERVICE_ENDPOINT value: "http://entitlements/api/entitlements/v2" - - name: ENTITLEMENTS_SERVICE_API_KEY - value: "OBSOLETE" \ No newline at end of file + - name: POLICY_SERVICE_ENDPOINT + value: http://policy/api/policy/v1 \ No newline at end of file diff --git a/software/applications/osdu-core/workflow.yaml b/software/applications/osdu-core/workflow.yaml index 00185503..cf8cf321 100644 --- a/software/applications/osdu-core/workflow.yaml +++ b/software/applications/osdu-core/workflow.yaml @@ -96,14 +96,6 @@ spec: secret: name: azure-resources key: insights-connection - - name: OSDU_AIRFLOW_USERNAME - secret: - name: azure-resources - key: airflow-username - - name: OSDU_AIRFLOW_PASSWORD - secret: - name: azure-resources - key: airflow-password - name: AZURE_ISTIOAUTH_ENABLED value: "true" - name: AZURE_PAAS_PODIDENTITY_ISENABLED @@ -122,23 +114,42 @@ spec: value: "workflow" - name: AZURE_STORAGE_ENABLE_HTTPS value: "true" + + - name: COSMOSDB_DATABASE value: "osdu-db" + - name: COSMOSDB_SYSTEM_DATABASE + value: osdu-system-db + + - name: OSDU_AIRFLOW_USERNAME + secret: + name: azure-resources + key: airflow-username + - name: OSDU_AIRFLOW_PASSWORD + secret: + name: azure-resources + key: airflow-password + + - name: AIRFLOW_STORAGE_ACCOUNT_NAME + secret: + name: azure-resources + key: azurestorageaccountname + - name: AIRFLOW_STORAGE_ACCOUNT_KEY + secret: + name: azure-resources + key: azurestorageaccountkey + + + + - name: AUTHORIZEAPI + value: http://entitlements/api/entitlements/v2 + - name: AUTHORIZEAPIKEY + value: "OBSOLETE" - name: OSDU_AIRFLOW_URL value: "http://airflow-webserver.airflow.svc.cluster.local/airflow" - name: DP_AIRFLOW_FOR_SYSTEM_DAG value: "false" - - # - name: AIRFLOW_STORAGE_ACCOUNT_NAME - # secret: - # name: azure-resources - # key: azurestorageaccountname - # - name: AIRFLOW_STORAGE_ACCOUNT_KEY - # secret: - # name: azure-resources - # key: azurestorageaccountkey - - name: OSDU_AIRFLOW_VERSION2_ENABLED value: true - name: PARTITION_SERVICE_ENDPOINT diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 78d1720f..15da14a2 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -39,7 +39,7 @@ spec: # Airflow executor executor: KubernetesExecutor - # Environment variables for all airflow containers + # Generic Environment Variables env: - name: ENVIRONMENT value: "dev" @@ -49,23 +49,45 @@ spec: value: "v0.12.0" - name: BUILD_TAG value: "v0.12.0" + - name: PYTHONPATH + value: "/opt/celery" + + # Extra Environment Variables Specific to Azure OSDU + extraEnv: | + - name: AIRFLOW_VAR_AZURE_ENABLE_MSI + value: "false" - name: AIRFLOW_VAR_ENTITLEMENTS_MODULE_NAME value: "entitlements_client" - extraEnv: | - - name: AIRFLOW_VAR_ANOTHER_KEY - value: 'value_1' - + # Environment Variables from ConfigMaps + # Airflow Settings are in airflow-variables + # Service Endpoints are in airflow-service-endpoints extraConfigMaps: 'airflow-variables': data: | - AIRFLOW_VAR_ENV_VARS_ENABLED: "True" - AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False" - AIRFLOW__LOGGING__LOGGING_LEVEL: "INFO" - AIRFLOW__LOGGING__ENABLE_TASK_CONTEXT_LOGGER: "False" AIRFLOW_VAR_NAMESPACE: "airflow" - AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.basic_auth" AIRFLOW__METRICS__USE_PATTERN_MATCH: "True" + AIRFLOW__LOGGING__LOGGING_LEVEL: "DEBUG" + AIRFLOW__LOGGING__ENABLE_TASK_CONTEXT_LOGGER: "False" + AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False" + AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "False" + AIRFLOW__WEBSERVER__AUTHENTICATE: "True" + AIRFLOW__WEBSERVER__AUTH_BACKEND: "airflow.contrib.auth.backends.password_auth" + AIRFLOW__WEBSERVER__RBAC: "True" + AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.default" + AIRFLOW__CELERY__SSL_ACTIVE: "True" + AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: "True" + AIRFLOW__CORE__PLUGINS_FOLDER: "/opt/airflow/plugins" + AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: "60" + AIRFLOW_VAR_CORE__CONFIG__DATALOAD_CONFIG_PATH: "/opt/airflow/dags/configs/dataload.ini" + AIRFLOW__WEBSERVER__WORKERS: "8" + AIRFLOW__WEBSERVER__WORKER_REFRESH_BATCH_SIZE: "0" + AIRFLOW__CORE__STORE_SERIALIZED_DAGS: "True" + AIRFLOW__CORE__STORE_DAG_CODE: "True" + AIRFLOW__WEBSERVER__WORKER_CLASS: "gevent" + AIRFLOW__CELERY__WORKER_CONCURRENCY: "16" + AIRFLOW_VAR_CORE__CONFIG__SHOW_SKIPPED_IDS: "True" + AIRFLOW_VAR_ENV_VARS_ENABLED: "True" 'airflow-service-endpoints': data: | @@ -80,6 +102,13 @@ spec: AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST: "http://workflow.osdu-core.svc.cluster.local/api/workflow/v1" AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST: "http://dataset.osdu-core.svc.cluster.local/api/dataset/v1" + ## Variables coming from Azure Key Vault + # AIRFLOW_VAR_AAD_CLIENT_ID + # AIRFLOW_VAR_APPINSIGHTS_KEY + # AIRFLOW_VAR_AZURE_CLIENT_ID + # AIRFLOW_VAR_AZURE_CLIENT_SECRET + # AIRFLOW_VAR_AZURE_TENANT_ID + # AIRFLOW_VAR_KEYVAULT_URI extraEnvFrom: | - configMapRef: name: "airflow-variables" @@ -88,6 +117,47 @@ spec: - secretRef: name: 'airflow-variables' + # Modify Create User Job command to use secrets + # Airflow create user job settings + createUserJob: + useHelmHooks: false + # Limit the lifetime of the job object after it finished execution. + ttlSecondsAfterFinished: 300 + # Command to use when running the create user job (templated). + command: ~ + # Args to use when running the create user job (templated). + args: + - "bash" + - "-c" + # The format below is necessary to get `helm lint` happy + - |- + exec \ + airflow {{ semverCompare ">=2.0.0" .Values.airflowVersion | ternary "users create" "create_user" }} "$@" + - -- + - "-r" + - "{{ .Values.webserver.defaultUser.role }}" + - "-u" + - "$(ADMIN_USERNAME)" + - "-e" + - "{{ .Values.webserver.defaultUser.email }}" + - "-f" + - "{{ .Values.webserver.defaultUser.firstName }}" + - "-l" + - "{{ .Values.webserver.defaultUser.lastName }}" + - "-p" + - "$(ADMIN_PASSWORD)" + env: + - name: ADMIN_USERNAME + valueFrom: + secretKeyRef: + name: airflow-secrets + key: username + - name: ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: airflow-secrets + key: password + # Admin user configuration admin: existingSecret: airflow-secrets @@ -143,6 +213,13 @@ spec: workers: logGroomerSidecar: enabled: false + webserver: + defaultUser: + enabled: true + role: Admin + email: admin@example.com + firstName: admin + lastName: user # Ensure we are using the secrets from Azure Key Vault enableBuiltInSecretEnvVars: @@ -151,10 +228,6 @@ spec: fernetKeySecretName: keyvault-secrets webserverSecretKeySecretName: keyvault-secrets - # Airflow create user job settings - createUserJob: - useHelmHooks: false - # Airflow database migration job settings migrateDatabaseJob: useHelmHooks: false diff --git a/software/components/airflow/vault-secrets.yaml b/software/components/airflow/vault-secrets.yaml index 30620d39..4d25e2cb 100644 --- a/software/components/airflow/vault-secrets.yaml +++ b/software/components/airflow/vault-secrets.yaml @@ -90,4 +90,6 @@ spec: - key: AIRFLOW_VAR_AAD_CLIENT_ID vaultSecret: app-dev-sp-id - key: AIRFLOW_VAR_APPINSIGHTS_KEY - vaultSecret: insights-key \ No newline at end of file + vaultSecret: insights-key + - key: AIRFLOW_VAR_KEYVAULT_URI + vaultSecret: keyvault-uri \ No newline at end of file diff --git a/src/core/repos b/src/core/repos index 32595cc5..38fb73be 100644 --- a/src/core/repos +++ b/src/core/repos @@ -6,4 +6,5 @@ https://community.opengroup.org/osdu/platform/system/indexer-service.git,indexer https://community.opengroup.org/osdu/platform/system/indexer-queue.git,indexer-queue,. https://community.opengroup.org/osdu/platform/system/storage.git,storage,. https://community.opengroup.org/osdu/platform/system/search-service.git,search,. -https://community.opengroup.org/osdu/platform/system/file.git,file,. \ No newline at end of file +https://community.opengroup.org/osdu/platform/system/file.git,file,. +https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-workflow.git,ingestion-workflow,. \ No newline at end of file diff --git a/tools/rest-scripts/check-file.http b/tools/rest-scripts/check-file.http index 75387c0b..3a5c4634 100644 --- a/tools/rest-scripts/check-file.http +++ b/tools/rest-scripts/check-file.http @@ -210,7 +210,7 @@ GET {{FILE_HOST}}/files/{{ID}}/downloadURL Authorization: Bearer {{access_token}} Accept: application/json Content-Type: application/json -data-partition-id: {{data_partition_id}} +data-partition-id: {{DATA_PARTITION}} @URL = {{getDownloadUrl.response.body.SignedUrl}} diff --git a/tools/rest-scripts/workflow.http b/tools/rest-scripts/workflow.http new file mode 100644 index 00000000..a564ab75 --- /dev/null +++ b/tools/rest-scripts/workflow.http @@ -0,0 +1,99 @@ +# -------HTTP REST CLIENT ------- +# https://marketplace.visualstudio.com/items?itemName=humao.rest-client + +## This script provides a few samples for calling unit. + + + +# ----------------------- +# OAUTH (Variables) +# ----------------------- +### +@login_base = login.microsoftonline.com/{{TENANT_ID}} +@oauth_token_host = {{login_base}}/oauth2/v2.0/token +@scopes = {{CLIENT_ID}}/.default openid profile offline_access + + + +# ----------------------- +# OAUTH refresh_token +# ----------------------- +### +# @name refresh +POST https://{{oauth_token_host}} HTTP/1.1 +Content-Type: application/x-www-form-urlencoded + +grant_type=refresh_token +&client_id={{CLIENT_ID}} +&client_secret={{CLIENT_SECRET}} +&refresh_token={{REFRESH_TOKEN}} +&scope={{scopes}} + + +# ----------------------- +# API (Variables) +# ----------------------- +### +@access_token = {{refresh.response.body.access_token}} +@SERVICE_HOST = {{HOST}}/api/workflow/v1 +@data_partition_id = opendes +@workflow_name = test_fetch_remote + + +# ----------------------- +# API: Version +# ----------------------- + +### +# @name info +GET {{SERVICE_HOST}}/info +Authorization: Bearer {{access_token}} +Accept: application/json + + + +# ----------------------- +# API: Ingestion Workflow +# ----------------------- + +### +# @name getAllWorkflow +GET {{SERVICE_HOST}}/workflow +Authorization: Bearer {{access_token}} +Content-Type: application/json +data-partition-id: {{data_partition_id}} + + +### +# @name createWorkflow +POST {{SERVICE_HOST}}/workflow +Authorization: Bearer {{access_token}} +Content-Type: application/json +data-partition-id: {{data_partition_id}} + +{ + "workflowName": "{{workflow_name}}", + "description": "Test Register a Workflow", + "registrationInstructions": { + "active": true, + "concurrentWorkflowRun": 5, + "concurrentTaskRun": 5, + "workflowDetailContent": "" + } +} + + +### +# @name getWorkflow +GET {{SERVICE_HOST}}/workflow/{{workflow_name}} +Authorization: Bearer {{access_token}} +Content-Type: application/json +data-partition-id: {{data_partition_id}} + + +### +# @name deleteWorkflow +DELETE {{SERVICE_HOST}}/workflow/{{workflow_name}} +Authorization: Bearer {{access_token}} +Content-Type: application/json +data-partition-id: {{data_partition_id}}