diff --git a/src/main/java/io/kestra/plugin/aws/AbstractConnection.java b/src/main/java/io/kestra/plugin/aws/AbstractConnection.java index e5adf960..20740201 100644 --- a/src/main/java/io/kestra/plugin/aws/AbstractConnection.java +++ b/src/main/java/io/kestra/plugin/aws/AbstractConnection.java @@ -26,6 +26,8 @@ public abstract class AbstractConnection extends Task implements AbstractConnect protected String endpointOverride; + private Boolean compatibilityMode; + protected AwsCredentialsProvider credentials(RunContext runContext) throws IllegalVariableEvaluationException { String accessKeyId = runContext.render(this.accessKeyId); String secretKeyId = runContext.render(this.secretKeyId); diff --git a/src/main/java/io/kestra/plugin/aws/s3/AbstractS3.java b/src/main/java/io/kestra/plugin/aws/s3/AbstractS3.java index 96c1f098..c7027b77 100644 --- a/src/main/java/io/kestra/plugin/aws/s3/AbstractS3.java +++ b/src/main/java/io/kestra/plugin/aws/s3/AbstractS3.java @@ -19,7 +19,7 @@ @EqualsAndHashCode @Getter @NoArgsConstructor -public abstract class AbstractS3 extends AbstractConnection { +public abstract class AbstractS3 extends AbstractConnection { protected S3Client client(RunContext runContext) throws IllegalVariableEvaluationException { S3ClientBuilder s3ClientBuilder = S3Client.builder() @@ -36,18 +36,36 @@ protected S3Client client(RunContext runContext) throws IllegalVariableEvaluatio return s3ClientBuilder.build(); } + protected S3AsyncClient asyncClient(RunContext runContext) throws IllegalVariableEvaluationException { - S3CrtAsyncClientBuilder s3ClientBuilder = S3AsyncClient.crtBuilder() - .credentialsProvider(this.credentials(runContext)); - if (this.region != null) { - s3ClientBuilder.region(Region.of(runContext.render(this.region))); - } + if (this.getCompatibilityMode()) { + S3AsyncClientBuilder s3ClientBuilder = S3AsyncClient.builder() + .credentialsProvider(this.credentials(runContext)); - if (this.endpointOverride != null) { - s3ClientBuilder.endpointOverride(URI.create(runContext.render(this.endpointOverride))); + if (this.region != null) { + s3ClientBuilder.region(Region.of(runContext.render(this.region))); + } + + if (this.endpointOverride != null) { + s3ClientBuilder.endpointOverride(URI.create(runContext.render(this.endpointOverride))); + } + return s3ClientBuilder.build(); + + } else { + S3CrtAsyncClientBuilder s3ClientBuilder = S3AsyncClient.crtBuilder() + .credentialsProvider(this.credentials(runContext)); + + if (this.region != null) { + s3ClientBuilder.region(Region.of(runContext.render(this.region))); + } + + if (this.endpointOverride != null) { + s3ClientBuilder.endpointOverride(URI.create(runContext.render(this.endpointOverride))); + } + + return s3ClientBuilder.build(); } - return s3ClientBuilder.build(); } } diff --git a/src/main/java/io/kestra/plugin/aws/s3/Download.java b/src/main/java/io/kestra/plugin/aws/s3/Download.java index 5e7cf66d..b8122df3 100644 --- a/src/main/java/io/kestra/plugin/aws/s3/Download.java +++ b/src/main/java/io/kestra/plugin/aws/s3/Download.java @@ -6,10 +6,7 @@ import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.swagger.v3.oas.annotations.media.Schema; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.ToString; +import lombok.*; import lombok.experimental.SuperBuilder; import org.apache.commons.lang3.tuple.Pair; import software.amazon.awssdk.services.s3.S3AsyncClient; @@ -54,6 +51,13 @@ public class Download extends AbstractS3Object implements RunnableTask metadata; + @Schema( + title = "this property will use the AsynS3Client instead of the S3CrtAsynClient which maximize compatibility with S3-compatible services but restrict uploads and downloads to 2GB" + ) + @PluginProperty + @Builder.Default + private Boolean compatibilityMode = false; + @Schema( title = "If you don't specify, S3 Standard is the default storage class. Amazon S3 supports other storage classes." ) @@ -79,6 +82,7 @@ public Output run(RunContext runContext) throws Exception { String key = runContext.render(this.key); try (S3AsyncClient client = this.asyncClient(runContext)) { + File tempFile = runContext.tempFile().toFile(); URI from = new URI(runContext.render(this.from)); Files.copy(runContext.uriToInputStream(from), tempFile.toPath(), StandardCopyOption.REPLACE_EXISTING);