Data Hub: Web API: Updated OpenAlex web config to use updated_date filter instead

deployments/data-hub/external-secrets/openalex-api.yaml

@@ -0,0 +1,16 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: openalex-api
+  namespace: data-hub
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: secret-store
+    kind: ClusterSecretStore
+  target:
+    name: openalex-api
+    creationPolicy: Owner
+  dataFrom:
+  - extract:
+      key: data-team/data-hub/openalex-api

kustomizations/apps/data-hub-configs/kubernetes-pipeline--stg.config.yaml

@@ -1501,6 +1501,8 @@ kubernetesPipelines:
         value: /dag_secret_files/gcloud/credentials.json
       - name: AWS_CONFIG_FILE
         value: /dag_secret_files/aws/credentials
+      - name: OPENALEX_API_KEY_FILE_PATH
+        value: /dag_secret_files/openalex-api/openalex_api_key
       - name: WEB_API_CONFIG_FILE_PATH
         value: /dag_config_files/web-api-data-pipeline.config.yaml
       - name: TMPDIR
@@ -1514,6 +1516,9 @@ kubernetesPipelines:
       - name: aws-secret-volume
         mountPath: /dag_secret_files/aws
         readOnly: true
+      - name: openalex-api-secret-volume
+        mountPath: /dag_secret_files/openalex-api
+        readOnly: true
       - name: data-hub-config-volume
         mountPath: /dag_config_files/
         readOnly: true
@@ -1527,6 +1532,9 @@ kubernetesPipelines:
       - name: gcloud-secret-volume
         secret:
           secretName: gcloud
+      - name: openalex-api-secret-volume
+        secret:
+          secretName: openalex-api
       - name: data-hub-config-volume
         configMap:
           name: data-hub-configs

kustomizations/apps/data-hub-configs/web-api-data-pipeline.config.yaml

@@ -686,16 +686,14 @@ webApi:
   - dataPipelineId: openalex_works_metadata_for_preprints
     description:
       Retrieve OpenAlex metadata for all of the preprints.
-      Currently this will use the publication date to chunk data.
-      It is recommended to use the `updated_date` which requires premium access.
     airflow:
       dagParameters:
         schedule: null
     dataset: '{ENV}'
     table: openalex_works_api_response
     stateFile:
       bucketName: '{ENV}-elife-data-pipeline'
-      objectName: 'airflow-config/generic-web-api/{ENV}-openalex_works_metadata_for_preprints-state.json'
+      objectName: 'airflow-config/generic-web-api/{ENV}-openalex_works_metadata_for_preprints-state-by-updated-date.json'
     retry:
       maxRetryCount: 10
       retryBackoffFactor: 1
@@ -704,6 +702,10 @@ webApi:
         - 500
         - 502
         - 504
+    headers:
+      parametersFromFile:
+        - parameterName: api_key
+          filePathEnvName: OPENALEX_API_KEY_FILE_PATH
     urlSourceType:
       # Note: We currently need to use `crossref_metadata_api` for the cursor parameter to work
       # It is also currently required for the `filter` parameter support
@@ -712,11 +714,12 @@ webApi:
       urlExcludingConfigurableParameters: 'https://api.openalex.org/works?filter=type:preprint'
       configurableParameters:
         nextPageCursorParameterName: 'cursor'
-        fromDateParameterName: 'from_publication_date'
-        toDateParameterName: 'to_publication_date'
+        fromDateParameterName: 'from_updated_date'
+        toDateParameterName: 'to_updated_date'
         daysDiffFromStartTillEnd: 30
-        defaultStartDate: '1744-01-01+00:00'
-        dateFormat: '%Y-%m-%d'
+        defaultStartDate: '2024-04-23+00:00'
+        # Note: OpenAlex do not seem to use timezones
+        dateFormat: '%Y-%m-%dT%H:%M:%S.%f'
         pageSizeParameterName: 'per-page'
         defaultPageSize: 200
     response:
@@ -730,7 +733,7 @@ webApi:
         - count
       recordTimestamp:
         itemTimestampKeyFromItemRoot:
-          - 'publication_date'
+          - 'updated_date'
       onSameNextCursor: Error
       provenanceEnabled: True
       fieldsToReturn: