Skip to content

Commit

Permalink
Fix Delta Lake and Hadoop AWS version incompatibility
Browse files Browse the repository at this point in the history
  • Loading branch information
Minutis committed Jun 13, 2024
1 parent d8f2e72 commit b94fd63
Showing 1 changed file with 24 additions and 0 deletions.
24 changes: 24 additions & 0 deletions src/main/resources/application.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,3 +41,27 @@ micronaut:
enabled: true
step: PT1M
descriptions: true

app:
resources:
- path: /internal/workspaces/{ws}/identities/{identifier}/{+user_id}
delta-path: s3a://stagecat-catfarm/internal/workspaces/{ws}/identity/{identifier}/
schema-path: /internal/workspaces/{ws}/schemas/identities/{identifier}
filter-variables:
- path-column: identifier
path-variable: user_id
- path: /internal/workspaces/{ws}/nbo/{app_id}/{identifier}/{+user_id}/recommendations
response-type: LIST
max-results: 50
delta-path: s3a://stagecat-catfarm/internal/workspaces/{ws}/nbo/APP_{app_id}/latest_recommendations/
schema-path: /api/workspaces/{ws}/nbo/{app_id}/recommendations/schema
filter-variables:
- path-column: identifier
path-variable: user_id
hadoop-props:
"fs.s3a.connection.maximum": 100
"fs.s3a.threads.keepalivetime": 60
"fs.s3a.aws.credentials.provider": "com.amazonaws.auth.EnvironmentVariableCredentialsProvider"
"fs.s3a.connection.establish.timeout": 50000
"fs.s3a.connection.timeout": 200000
"fs.s3a.multipart.purge.age": 86400

0 comments on commit b94fd63

Please sign in to comment.