From 11d4634f2093781db3e40beebfa2e91d9c539aa7 Mon Sep 17 00:00:00 2001 From: Maxim Lapan Date: Mon, 16 Oct 2023 12:57:37 +0200 Subject: [PATCH] Rely on default credentials providers if not explicitly set --- .../spark/s3/AbstractImportExportQueryGenerator.java | 1 + .../src/main/java/com/exasol/spark/s3/ExasolS3Table.java | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/exasol-s3/src/main/java/com/exasol/spark/s3/AbstractImportExportQueryGenerator.java b/exasol-s3/src/main/java/com/exasol/spark/s3/AbstractImportExportQueryGenerator.java index ab72ae59..c95ed699 100644 --- a/exasol-s3/src/main/java/com/exasol/spark/s3/AbstractImportExportQueryGenerator.java +++ b/exasol-s3/src/main/java/com/exasol/spark/s3/AbstractImportExportQueryGenerator.java @@ -33,6 +33,7 @@ public AbstractImportExportQueryGenerator(final ExasolOptions options) { * @return identifiedBy part of a query */ public String getIdentifier() { + // TODO: get credentials from ENV variable or not pass USER and IDENTIFIED BY at all final String awsAccessKeyId = this.options.get(Option.AWS_ACCESS_KEY_ID.key()); final String awsSecretAccessKey = this.options.get(Option.AWS_SECRET_ACCESS_KEY.key()); return "AT '" + escapeStringLiteral(getBucketURL()) + "'\nUSER '" + escapeStringLiteral(awsAccessKeyId) diff --git a/exasol-s3/src/main/java/com/exasol/spark/s3/ExasolS3Table.java b/exasol-s3/src/main/java/com/exasol/spark/s3/ExasolS3Table.java index f64c6b6c..bf5d203e 100644 --- a/exasol-s3/src/main/java/com/exasol/spark/s3/ExasolS3Table.java +++ b/exasol-s3/src/main/java/com/exasol/spark/s3/ExasolS3Table.java @@ -109,12 +109,12 @@ private void updateSparkConfigurationForS3(final ExasolOptions options) { final SparkSession sparkSession = SparkSession.active(); synchronized (sparkSession.sparkContext().hadoopConfiguration()) { final Configuration conf = sparkSession.sparkContext().hadoopConfiguration(); - conf.set("fs.s3a.access.key", options.get(Option.AWS_ACCESS_KEY_ID.key())); - conf.set("fs.s3a.secret.key", options.get(Option.AWS_SECRET_ACCESS_KEY.key())); + if (options.containsKey(Option.AWS_ACCESS_KEY_ID.key())) { + conf.set("fs.s3a.access.key", options.get(Option.AWS_ACCESS_KEY_ID.key())); + conf.set("fs.s3a.secret.key", options.get(Option.AWS_SECRET_ACCESS_KEY.key())); + } if (options.containsKey(Option.AWS_CREDENTIALS_PROVIDER.key())) { conf.set("fs.s3a.aws.credentials.provider", options.get(Option.AWS_CREDENTIALS_PROVIDER.key())); - } else { - conf.set("fs.s3a.aws.credentials.provider", "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider"); } if (options.containsKey(Option.S3_ENDPOINT_OVERRIDE.key())) { conf.set("fs.s3a.endpoint", "http://" + options.get(Option.S3_ENDPOINT_OVERRIDE.key()));