Enable Workflow Service (#191)

Azure · Sep 9, 2024 · ce6ba8a · ce6ba8a
1 parent 23909c0
commit ce6ba8a
Show file tree

Hide file tree

Showing 9 changed files with 191 additions and 21 deletions.
diff --git a/README.md b/README.md
@@ -16,7 +16,7 @@ For detailed instructions, view our online [Documentation](https://azure.github.
 Supported services of OSDU are based on the release branch of OSDU as specified in the [OSDU Milestones](https://community.opengroup.org/osdu/platform/-/milestones). (ie: release/0.25 release/0.26, release/0.27, master etc.)
 
 
-| **Service**                                                                                     | **Description**                                                                                 |
+| **Core Services**                                                                               | **Description**                                                                                 |
 |-------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------|
 | [Partition Service](https://community.opengroup.org/osdu/platform/system/partition)             | Manages data partitions to ensure efficient data management and scalability.                    |
 | [Entitlement Service](https://community.opengroup.org/osdu/platform/security-and-compliance/entitlements) | Provides access control and permissions management for data within the OSDU platform. |
@@ -27,6 +27,10 @@ Supported services of OSDU are based on the release branch of OSDU as specified
 | [Storage Service](https://community.opengroup.org/osdu/platform/system/storage)                | Provides scalable storage solutions for managing and retrieving large volumes of data.           |
 | [Search Service](https://community.opengroup.org/osdu/platform/system/search-service)          | Facilitates searching and querying across data stored within the OSDU platform.                  |
 | [File Service](https://community.opengroup.org/osdu/platform/system/file)                      | Handles file operations such as storage, retrieval, and management of data files.                |
+| [Workflow Service](https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-workflow/)  | Initiates business processes within the system. During the prototype phase, it facilitates CRUD operations on workflow metadata and triggers workflows in Apache Airflow. Additionally, the service manages process startup records, acting as a wrapper around Airflow functions.. |
+
+| **Reference Helper Services**                                                                   | **Description**                                                                                 |
+|-------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------|
 | [Unit Service](https://community.opengroup.org/osdu/platform/system/reference/unit-service)    | Provides dimension/measurement and unit definitions.                                             |
 | [CRS Catalog Service](https://community.opengroup.org/osdu/platform/system/reference/crs-catalog-service) | Provides API endpoints to work with geodetic reference data, allowing developers to retrieve CRS definitions, select appropriate CRSs for data ingestion, and search for CRSs based on various constraints. |
 | [CRS Conversion Service](https://community.opengroup.org/osdu/platform/system/reference/crs-conversion-service)  | Enables the conversion of coordinates from one coordinate reference system (CRS) to another. |

diff --git a/charts/osdu-developer-base/Chart.yaml b/charts/osdu-developer-base/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: osdu-developer-base
 type: application
 description: Installs the OSDU developer Base Components
-version: 0.0.2
+version: 0.0.3
 appVersion: 0.0.1
 maintainers:
   - name: danielscholl

diff --git a/charts/osdu-developer-base/templates/kv-secrets.yaml b/charts/osdu-developer-base/templates/kv-secrets.yaml
@@ -32,6 +32,10 @@ spec:
       key: azurestorageaccountname
     - objectName: "system-storage-key"
       key: azurestorageaccountkey
+    - objectName: "airflow-admin-username"
+      key: airflow-username
+    - objectName: "airflow-admin-password"
+      key: airflow-password
   parameters:
     clientID: {{ .Values.azure.clientId }} # client id of the Azure AD app/identity to use for workload identity
     keyvaultName: {{ .Values.azure.keyvaultName }} # the name of the KeyVault
@@ -67,5 +71,11 @@ spec:
         - |
           objectName: system-storage-key
           objectType: secret
+        - |
+          objectName: airflow-admin-username
+          objectType: secret
+        - |
+          objectName: airflow-admin-password
+          objectType: secret
     tenantId: {{ .Values.azure.tenantId }} # the tenant ID of the KeyVault
 {{- end }}
diff --git a/docs/src/software.md b/docs/src/software.md
@@ -66,6 +66,12 @@ flowchart TD
 
 ```bash
 ── components
+    ├── airflow
+    │   ├── namespace.yaml
+    │   ├── pvc.yaml
+    │   ├── release.yaml
+    │   ├── source.yaml
+    │   └── vault-secrets.yaml
     ├── certs
     │   ├── namespace.yaml
     │   ├── release.yaml
@@ -175,6 +181,7 @@ flowchart TD
   storage("storage")
   file("file")
   search("search")
+  workflow("workflow")
 
   base-->partition
   partition-->entitlements
@@ -185,6 +192,7 @@ flowchart TD
   legal-->indexer
   legal-->indexer_queue
   legal-->schema
+  legal-->workflow
   schema-->schema_init
   indexer_queue-->storage
   indexer_queue-->file
@@ -204,6 +212,7 @@ flowchart TD
    ├── search.yaml
    ├── storage.yaml
    └── user-init.yaml
+   └── workflow.yaml
 ```
 
 __OSDU Reference Structure__

diff --git a/software/applications/osdu-core/workflow.yaml b/software/applications/osdu-core/workflow.yaml
@@ -0,0 +1,149 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: osdu-workflow
+  namespace: osdu-core
+  annotations:
+    clusterconfig.azure.com/use-managed-source: "true"
+spec:
+  dependsOn:
+    - name: osdu-partition
+      namespace: osdu-core
+  targetNamespace: osdu-core
+  chart:
+    spec:
+      chart: ./charts/osdu-developer-service
+      sourceRef:
+        kind: GitRepository
+        name: flux-system
+        namespace: flux-system
+  interval: 5m0s
+  timeout: 6m
+  install:
+    remediation:
+      retries: 3
+  valuesFrom:
+    - kind: ConfigMap
+      name: configmap-software
+      valuesKey: value.yaml
+  values:
+    nameOverride: workflow
+    installationType: osduCore
+    subset: m24
+    replicaCount: 1
+    service:
+      type: ClusterIP
+      port: 80
+    configuration:
+    - service: workflow
+      path: /api/workflow/
+      hosts:
+        - "*"
+      gateways:
+        - istio-system/internal-gateway
+        - istio-system/external-gateway
+      repository: community.opengroup.org:5555/osdu/platform/data-flow/ingestion/ingestion-workflow/ingestion-workflow-
+      tag: latest 
+      probe:
+        path: /actuator/health
+        port: 8081
+        liveness:
+          delay: 250
+          seconds: 10
+      keyvault: true
+      auth:
+        disable:
+          - "*/actuator/health"
+          - "*/health"
+          - "*/_ah/**"
+          - "*/configuration/ui"
+          - "*/configuration/security"
+          - "/api/workflow/v3/info"
+          - "/api/workflow/v3/swagger*"
+          - "/api/workflow/v3/api-docs*"
+          - "/api/workflow/v3/webjars/*"          
+      env:
+      - name: AZURE_TENANT_ID
+        secret:
+          name: active-directory
+          key: tenant-id
+      - name: AZURE_SUBSCRIPTION_ID
+        secret:
+          name: active-directory
+          key: subscription-id
+      - name: AZURE_CLIENT_ID
+        secret:
+          name: active-directory
+          key: principal-clientid
+      - name: AZURE_CLIENT_SECRET
+        secret:
+          name: active-directory
+          key: principal-clientpassword
+      - name: KEYVAULT_URI
+        secret:
+          name: azure-resources
+          key: keyvault-uri
+      - name: AAD_CLIENT_ID
+        secret:
+          name: active-directory
+          key: principal-clientid
+      - name: APPINSIGHTS_KEY
+        secret:
+          name: azure-resources
+          key: insights-key
+      - name: APPLICATIONINSIGHTS_CONNECTION_STRING
+        secret:
+          name: azure-resources
+          key: insights-connection
+      - name: OSDU_AIRFLOW_USERNAME
+        secret:
+          name: azure-resources
+          key: airflow-username
+      - name: OSDU_AIRFLOW_PASSWORD
+        secret:
+          name: azure-resources
+          key: airflow-password
+      - name: AZURE_ISTIOAUTH_ENABLED
+        value: "true"
+      - name: AZURE_PAAS_PODIDENTITY_ISENABLED
+        value: "false"
+      - name: SERVER_SERVLET_CONTEXTPATH
+        value: "/api/workflow/"
+      - name: SERVER_PORT
+        value: "80"
+      - name: ACCEPT_HTTP
+        value: "true"
+      - name: SPRING_APPLICATION_NAME
+        value: "workflow"
+      - name: SPRING_CONFIG_NAME
+        value: "common,application"
+      - name: LOG_PREFIX
+        value: "workflow"
+      - name: AZURE_STORAGE_ENABLE_HTTPS
+        value: "true"
+      - name: COSMOSDB_DATABASE
+        value: "osdu-db"
+      - name: OSDU_AIRFLOW_URL
+        value: "http://airflow-webserver.airflow.svc.cluster.local/airflow"
+      - name: DP_AIRFLOW_FOR_SYSTEM_DAG
+        value: "false"
+
+
+      # - name: AIRFLOW_STORAGE_ACCOUNT_NAME
+      #   secret:
+      #     name: azure-resources
+      #     key: azurestorageaccountname
+      # - name: AIRFLOW_STORAGE_ACCOUNT_KEY
+      #   secret:
+      #     name: azure-resources
+      #     key: azurestorageaccountkey
+
+      - name: OSDU_AIRFLOW_VERSION2_ENABLED
+        value: true
+      - name: PARTITION_SERVICE_ENDPOINT
+        value: "http://partition/api/partition/v1"
+      - name: OSDU_ENTITLEMENTS_URL
+        value: "http://entitlements/api/entitlements/v2"
+      - name: OSDU_ENTITLEMENTS_APPKEY
+        value: "OBSOLETE"
diff --git a/software/components/airflow/pvc.yaml b/software/components/airflow/pvc.yaml
@@ -10,7 +10,7 @@ spec:
   capacity:
     storage: 5Gi
   accessModes:
-    - ReadOnlyMany
+    - ReadWriteMany
   azureFile:
     secretName: keyvault-secrets
     shareName: airflow-logs
@@ -30,7 +30,7 @@ metadata:
   namespace: airflow
 spec:
   accessModes:
-    - ReadOnlyMany
+    - ReadWriteMany
   resources:
     requests:
       storage: 5Gi
@@ -47,7 +47,7 @@ spec:
   capacity:
     storage: 5Gi
   accessModes:
-    - ReadOnlyMany
+    - ReadWriteMany
   azureFile:
     secretName: keyvault-secrets
     shareName: airflow-dags
@@ -67,7 +67,7 @@ metadata:
   namespace: airflow
 spec:
   accessModes:
-    - ReadOnlyMany
+    - ReadWriteMany
   resources:
     requests:
       storage: 5Gi

diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml
@@ -25,34 +25,30 @@ spec:
   chart:
     spec:
       chart: airflow
-      # version: 1.11.0
       sourceRef:
         kind: HelmRepository
         name: airflow
         namespace: flux-system
   values:
+    # Isolate version
+    airflowVersion: 2.10.1
+
     # Airflow executor
     executor: KubernetesExecutor
 
+    # External PostgreSQL configuration
+    data:
+      metadataSecretName: airflow-secrets
+
     # Environment variables for all airflow containers
     env:
       - name: ENVIRONMENT
         value: dev
 
-    # Admin user configuration
-    admin:
-      existingSecret: airflow-secrets
-      usernameKey: username
-      passwordKey: password
-
     # Disable the internal PostgreSQL chart
     postgresql:
       enabled: false
 
-    # External PostgreSQL configuration
-    data:
-      metadataSecretName: airflow-secrets
-
     # Disable pgbouncer.
     # CloudNativePG provides native support for connection pooling with PgBouncer
     pgbouncer:
@@ -66,10 +62,10 @@ spec:
         branch: main
         wait: 40
         subPath: dags
-      # persistence:
-      #   enabled: false
-      #   existingClaim: airflow-dags-pvc
-      #   subPath: dags
+      persistence:
+        enabled: false
+        existingClaim: airflow-dags-pvc
+        subPath: dags
 
     logs:
       persistence:

diff --git a/src/core/ingestion-workflow b/src/core/ingestion-workflow
diff --git a/src/core/repos b/src/core/repos
@@ -7,3 +7,4 @@ https://community.opengroup.org/osdu/platform/system/indexer-queue.git,indexer-q
 https://community.opengroup.org/osdu/platform/system/storage.git,storage,.
 https://community.opengroup.org/osdu/platform/system/search-service.git,search,.
 https://community.opengroup.org/osdu/platform/system/file.git,file,.
+https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-workflow.git,.