From ce6ba8afb40393513850e4fb9a65eb31ffc1ec9c Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 9 Sep 2024 10:33:46 -0500 Subject: [PATCH] Enable Workflow Service (#191) --- README.md | 6 +- charts/osdu-developer-base/Chart.yaml | 2 +- .../templates/kv-secrets.yaml | 10 ++ docs/src/software.md | 9 ++ software/applications/osdu-core/workflow.yaml | 149 ++++++++++++++++++ software/components/airflow/pvc.yaml | 8 +- software/components/airflow/release.yaml | 26 ++- src/core/ingestion-workflow | 1 + src/core/repos | 1 + 9 files changed, 191 insertions(+), 21 deletions(-) create mode 100644 software/applications/osdu-core/workflow.yaml create mode 160000 src/core/ingestion-workflow diff --git a/README.md b/README.md index 8389f770..a81a773e 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ For detailed instructions, view our online [Documentation](https://azure.github. Supported services of OSDU are based on the release branch of OSDU as specified in the [OSDU Milestones](https://community.opengroup.org/osdu/platform/-/milestones). (ie: release/0.25 release/0.26, release/0.27, master etc.) -| **Service** | **Description** | +| **Core Services** | **Description** | |-------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------| | [Partition Service](https://community.opengroup.org/osdu/platform/system/partition) | Manages data partitions to ensure efficient data management and scalability. | | [Entitlement Service](https://community.opengroup.org/osdu/platform/security-and-compliance/entitlements) | Provides access control and permissions management for data within the OSDU platform. | @@ -27,6 +27,10 @@ Supported services of OSDU are based on the release branch of OSDU as specified | [Storage Service](https://community.opengroup.org/osdu/platform/system/storage) | Provides scalable storage solutions for managing and retrieving large volumes of data. | | [Search Service](https://community.opengroup.org/osdu/platform/system/search-service) | Facilitates searching and querying across data stored within the OSDU platform. | | [File Service](https://community.opengroup.org/osdu/platform/system/file) | Handles file operations such as storage, retrieval, and management of data files. | +| [Workflow Service](https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-workflow/) | Initiates business processes within the system. During the prototype phase, it facilitates CRUD operations on workflow metadata and triggers workflows in Apache Airflow. Additionally, the service manages process startup records, acting as a wrapper around Airflow functions.. | + +| **Reference Helper Services** | **Description** | +|-------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------| | [Unit Service](https://community.opengroup.org/osdu/platform/system/reference/unit-service) | Provides dimension/measurement and unit definitions. | | [CRS Catalog Service](https://community.opengroup.org/osdu/platform/system/reference/crs-catalog-service) | Provides API endpoints to work with geodetic reference data, allowing developers to retrieve CRS definitions, select appropriate CRSs for data ingestion, and search for CRSs based on various constraints. | | [CRS Conversion Service](https://community.opengroup.org/osdu/platform/system/reference/crs-conversion-service) | Enables the conversion of coordinates from one coordinate reference system (CRS) to another. | diff --git a/charts/osdu-developer-base/Chart.yaml b/charts/osdu-developer-base/Chart.yaml index 00b27091..9150f73b 100644 --- a/charts/osdu-developer-base/Chart.yaml +++ b/charts/osdu-developer-base/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: osdu-developer-base type: application description: Installs the OSDU developer Base Components -version: 0.0.2 +version: 0.0.3 appVersion: 0.0.1 maintainers: - name: danielscholl diff --git a/charts/osdu-developer-base/templates/kv-secrets.yaml b/charts/osdu-developer-base/templates/kv-secrets.yaml index 3c8fb728..379c986c 100644 --- a/charts/osdu-developer-base/templates/kv-secrets.yaml +++ b/charts/osdu-developer-base/templates/kv-secrets.yaml @@ -32,6 +32,10 @@ spec: key: azurestorageaccountname - objectName: "system-storage-key" key: azurestorageaccountkey + - objectName: "airflow-admin-username" + key: airflow-username + - objectName: "airflow-admin-password" + key: airflow-password parameters: clientID: {{ .Values.azure.clientId }} # client id of the Azure AD app/identity to use for workload identity keyvaultName: {{ .Values.azure.keyvaultName }} # the name of the KeyVault @@ -67,5 +71,11 @@ spec: - | objectName: system-storage-key objectType: secret + - | + objectName: airflow-admin-username + objectType: secret + - | + objectName: airflow-admin-password + objectType: secret tenantId: {{ .Values.azure.tenantId }} # the tenant ID of the KeyVault {{- end }} diff --git a/docs/src/software.md b/docs/src/software.md index 2bf625f3..410e6b5b 100644 --- a/docs/src/software.md +++ b/docs/src/software.md @@ -66,6 +66,12 @@ flowchart TD ```bash ── components + ├── airflow + │   ├── namespace.yaml + │   ├── pvc.yaml + │   ├── release.yaml + │   ├── source.yaml + │   └── vault-secrets.yaml ├── certs │   ├── namespace.yaml │   ├── release.yaml @@ -175,6 +181,7 @@ flowchart TD storage("storage") file("file") search("search") + workflow("workflow") base-->partition partition-->entitlements @@ -185,6 +192,7 @@ flowchart TD legal-->indexer legal-->indexer_queue legal-->schema + legal-->workflow schema-->schema_init indexer_queue-->storage indexer_queue-->file @@ -204,6 +212,7 @@ flowchart TD    ├── search.yaml    ├── storage.yaml    └── user-init.yaml + └── workflow.yaml ``` __OSDU Reference Structure__ diff --git a/software/applications/osdu-core/workflow.yaml b/software/applications/osdu-core/workflow.yaml new file mode 100644 index 00000000..00185503 --- /dev/null +++ b/software/applications/osdu-core/workflow.yaml @@ -0,0 +1,149 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-workflow + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-partition + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-service + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + timeout: 6m + install: + remediation: + retries: 3 + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + values: + nameOverride: workflow + installationType: osduCore + subset: m24 + replicaCount: 1 + service: + type: ClusterIP + port: 80 + configuration: + - service: workflow + path: /api/workflow/ + hosts: + - "*" + gateways: + - istio-system/internal-gateway + - istio-system/external-gateway + repository: community.opengroup.org:5555/osdu/platform/data-flow/ingestion/ingestion-workflow/ingestion-workflow- + tag: latest + probe: + path: /actuator/health + port: 8081 + liveness: + delay: 250 + seconds: 10 + keyvault: true + auth: + disable: + - "*/actuator/health" + - "*/health" + - "*/_ah/**" + - "*/configuration/ui" + - "*/configuration/security" + - "/api/workflow/v3/info" + - "/api/workflow/v3/swagger*" + - "/api/workflow/v3/api-docs*" + - "/api/workflow/v3/webjars/*" + env: + - name: AZURE_TENANT_ID + secret: + name: active-directory + key: tenant-id + - name: AZURE_SUBSCRIPTION_ID + secret: + name: active-directory + key: subscription-id + - name: AZURE_CLIENT_ID + secret: + name: active-directory + key: principal-clientid + - name: AZURE_CLIENT_SECRET + secret: + name: active-directory + key: principal-clientpassword + - name: KEYVAULT_URI + secret: + name: azure-resources + key: keyvault-uri + - name: AAD_CLIENT_ID + secret: + name: active-directory + key: principal-clientid + - name: APPINSIGHTS_KEY + secret: + name: azure-resources + key: insights-key + - name: APPLICATIONINSIGHTS_CONNECTION_STRING + secret: + name: azure-resources + key: insights-connection + - name: OSDU_AIRFLOW_USERNAME + secret: + name: azure-resources + key: airflow-username + - name: OSDU_AIRFLOW_PASSWORD + secret: + name: azure-resources + key: airflow-password + - name: AZURE_ISTIOAUTH_ENABLED + value: "true" + - name: AZURE_PAAS_PODIDENTITY_ISENABLED + value: "false" + - name: SERVER_SERVLET_CONTEXTPATH + value: "/api/workflow/" + - name: SERVER_PORT + value: "80" + - name: ACCEPT_HTTP + value: "true" + - name: SPRING_APPLICATION_NAME + value: "workflow" + - name: SPRING_CONFIG_NAME + value: "common,application" + - name: LOG_PREFIX + value: "workflow" + - name: AZURE_STORAGE_ENABLE_HTTPS + value: "true" + - name: COSMOSDB_DATABASE + value: "osdu-db" + - name: OSDU_AIRFLOW_URL + value: "http://airflow-webserver.airflow.svc.cluster.local/airflow" + - name: DP_AIRFLOW_FOR_SYSTEM_DAG + value: "false" + + + # - name: AIRFLOW_STORAGE_ACCOUNT_NAME + # secret: + # name: azure-resources + # key: azurestorageaccountname + # - name: AIRFLOW_STORAGE_ACCOUNT_KEY + # secret: + # name: azure-resources + # key: azurestorageaccountkey + + - name: OSDU_AIRFLOW_VERSION2_ENABLED + value: true + - name: PARTITION_SERVICE_ENDPOINT + value: "http://partition/api/partition/v1" + - name: OSDU_ENTITLEMENTS_URL + value: "http://entitlements/api/entitlements/v2" + - name: OSDU_ENTITLEMENTS_APPKEY + value: "OBSOLETE" \ No newline at end of file diff --git a/software/components/airflow/pvc.yaml b/software/components/airflow/pvc.yaml index 2b81e07d..c3d9daea 100644 --- a/software/components/airflow/pvc.yaml +++ b/software/components/airflow/pvc.yaml @@ -10,7 +10,7 @@ spec: capacity: storage: 5Gi accessModes: - - ReadOnlyMany + - ReadWriteMany azureFile: secretName: keyvault-secrets shareName: airflow-logs @@ -30,7 +30,7 @@ metadata: namespace: airflow spec: accessModes: - - ReadOnlyMany + - ReadWriteMany resources: requests: storage: 5Gi @@ -47,7 +47,7 @@ spec: capacity: storage: 5Gi accessModes: - - ReadOnlyMany + - ReadWriteMany azureFile: secretName: keyvault-secrets shareName: airflow-dags @@ -67,7 +67,7 @@ metadata: namespace: airflow spec: accessModes: - - ReadOnlyMany + - ReadWriteMany resources: requests: storage: 5Gi diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 499d8537..524fd252 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -25,34 +25,30 @@ spec: chart: spec: chart: airflow - # version: 1.11.0 sourceRef: kind: HelmRepository name: airflow namespace: flux-system values: + # Isolate version + airflowVersion: 2.10.1 + # Airflow executor executor: KubernetesExecutor + # External PostgreSQL configuration + data: + metadataSecretName: airflow-secrets + # Environment variables for all airflow containers env: - name: ENVIRONMENT value: dev - # Admin user configuration - admin: - existingSecret: airflow-secrets - usernameKey: username - passwordKey: password - # Disable the internal PostgreSQL chart postgresql: enabled: false - # External PostgreSQL configuration - data: - metadataSecretName: airflow-secrets - # Disable pgbouncer. # CloudNativePG provides native support for connection pooling with PgBouncer pgbouncer: @@ -66,10 +62,10 @@ spec: branch: main wait: 40 subPath: dags - # persistence: - # enabled: false - # existingClaim: airflow-dags-pvc - # subPath: dags + persistence: + enabled: false + existingClaim: airflow-dags-pvc + subPath: dags logs: persistence: diff --git a/src/core/ingestion-workflow b/src/core/ingestion-workflow new file mode 160000 index 00000000..f61888de --- /dev/null +++ b/src/core/ingestion-workflow @@ -0,0 +1 @@ +Subproject commit f61888dec384f4d4d87208be819a00f654f33ee3 diff --git a/src/core/repos b/src/core/repos index 9cf8ff64..5032183b 100644 --- a/src/core/repos +++ b/src/core/repos @@ -7,3 +7,4 @@ https://community.opengroup.org/osdu/platform/system/indexer-queue.git,indexer-q https://community.opengroup.org/osdu/platform/system/storage.git,storage,. https://community.opengroup.org/osdu/platform/system/search-service.git,search,. https://community.opengroup.org/osdu/platform/system/file.git,file,. +https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-workflow.git,. \ No newline at end of file