diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml new file mode 100644 index 0000000..8a35cd5 --- /dev/null +++ b/.github/release-drafter.yml @@ -0,0 +1,30 @@ +name-template: 'v$RESOLVED_VERSION' +tag-template: 'v$RESOLVED_VERSION' +categories: + - title: '🚀 Features' + labels: + - 'feature' + - 'enhancement' + - title: '🐛 Bug Fixes' + labels: + - 'fix' + - 'bugfix' + - 'bug' + - title: '🧰 Maintenance' + label: 'chore' +change-template: '- $TITLE @$AUTHOR (#$NUMBER)' +version-resolver: + major: + labels: + - 'major' + minor: + labels: + - 'minor' + patch: + labels: + - 'patch' + default: patch +template: | + ## Changes + + $CHANGES \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..9441832 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,16 @@ +name: Build + +on: + pull_request: + paths-ignore: + - '*.md' + push: + +jobs: + check: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v2 + - name: check + run: ./gradlew check diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml new file mode 100644 index 0000000..3fb8ef3 --- /dev/null +++ b/.github/workflows/release-drafter.yml @@ -0,0 +1,14 @@ +name: Release Drafter + +on: + push: + branches: + - master + +jobs: + update_release_draft: + runs-on: ubuntu-latest + steps: + - uses: release-drafter/release-drafter@v5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..01c1ac9 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,25 @@ +name: Release + +on: + workflow_dispatch: + inputs: + version: + description: "Release Version" + required: true + +jobs: + deploy: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Publish to Maven Central + run: ./gradlew clean build bintrayUpload -PbintrayUser=${BINTRAY_USER} -PbintrayKey=${BINTRAY_KEY} -PdryRun=false + env: + RELEASE_VERSION: ${{ github.event.inputs.version }} + BINTRAY_KEY: ${{ secrets.BINTRAY_KEY }} + BINTRAY_USER: ${{ secrets.BINTRAY_USER }} \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 686b5b3..0000000 --- a/.travis.yml +++ /dev/null @@ -1,15 +0,0 @@ -language: java - -sudo: false - -before_cache: - - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock - - rm -fr $HOME/.gradle/caches/*/plugin-resolution/ - -cache: - directories: - - "$HOME/.gradle/caches/" - - "$HOME/.gradle/wrapper/" - - "$HOME/.m2/repository/" - -script: "./gradlew build --stacktrace" \ No newline at end of file diff --git a/README.md b/README.md index 1d7bf02..e86ea66 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,12 @@ connector.transfer(sftpFilePath = "sftp/file/path", s3File = "foo/MyFile.txt") You can optionally pass a KMS Key ID to request a server-side encryption with it ```kotlin -connector.transfer("sft/file/path", "foo/MyFile.txt", kmsKeyId = "aws:kms:mykeyid") +connector.transfer("sftp/file/path", "foo/MyFile.txt", kmsKeyId = "aws:kms:mykeyid") +``` + +You can optionally pass a Stream Transformer to process the file while it's being streamed. This can be useful for filters or event notifications of some sort. +```kotlin +connector.transfer("sftp/file/path", "foo/MyFile.txt", transformer = { inputStream, outputStream -> inputStream.copyTo(outputStream) }) ``` ## Features @@ -58,6 +63,7 @@ connector.transfer("sft/file/path", "foo/MyFile.txt", kmsKeyId = "aws:kms:mykeyi - The file will be streamed from one point to the other, therefore there won't be any problems regarding file size in-memory. Although unmeasured, the memory footprint of this library should be small - As per S3 specification, if there is any errors (such as an interrupted connection) during the transfer, no file chunks will be persisted - The files can be encrypted if provided with a KMS key ID +- It's possible to process the file (as an InputStream) while it's being streamed to S3 ## Limitations - Currently this library doesn't have a way to select which files you want transferred other than by specific path and name. diff --git a/build.gradle.kts b/build.gradle.kts index b1920a1..a690008 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,5 +1,4 @@ import com.novoda.gradle.release.PublishExtension -import io.gitlab.arturbosch.detekt.detekt import org.jetbrains.dokka.gradle.DokkaTask import org.jetbrains.kotlin.gradle.tasks.KotlinCompile @@ -16,16 +15,15 @@ buildscript { plugins { - kotlin("jvm") version "1.3.50" + kotlin("jvm") version "1.4.10" `maven-publish` id("org.jetbrains.dokka") version "0.9.17" - id("io.gitlab.arturbosch.detekt").version("1.1.1") + id("io.gitlab.arturbosch.detekt").version("1.14.2") } apply(plugin = "com.novoda.bintray-release") group = "br.com.guiabolso" -version = "0.1.2" repositories { mavenCentral() @@ -33,27 +31,29 @@ repositories { } dependencies { - // Kotlin - implementation(kotlin("stdlib-jdk8")) - // SFTP implementation("com.jcraft:jsch:0.1.55") implementation("org.apache.commons:commons-vfs2:2.4.1") testImplementation("com.github.stefanbirkner:fake-sftp-server-lambda:1.0.0") + testImplementation("org.apache.sshd:sshd-sftp:2.4.0") // S3 api("com.amazonaws:aws-java-sdk-s3:1.11.488") testImplementation("com.adobe.testing:s3mock-junit5:2.1.16") - // KotlinTest - testImplementation("io.kotlintest:kotlintest-runner-junit5:3.4.2") + // Kotest + testImplementation("io.kotest:kotest-runner-junit5:4.3.0") } tasks.withType { useJUnitPlatform() } +kotlin { + explicitApi() +} + tasks.withType { kotlinOptions.jvmTarget = "1.8" } @@ -72,11 +72,6 @@ val javadocJar by tasks.registering(Jar::class) { from(javadoc.outputDirectory) } -detekt { - toolVersion = "1.1.1" - input = files("src/main/kotlin", "src/test/kotlin") -} - publishing { publications { @@ -115,7 +110,7 @@ configure { groupId = "br.com.guiabolso" userOrg = "gb-opensource" setLicences("APACHE-2.0") - publishVersion = version.toString() + publishVersion = System.getenv("RELEASE_VERSION") ?: "local" uploadName = "SFTP-to-S3-Connector" website = "https://github.com/GuiaBolso/sftp-to-s3-connector" setPublications("maven") diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 87b738c..e708b1c 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 44e7c4d..be52383 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-5.2.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-6.7-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index af6708f..4f906e0 100755 --- a/gradlew +++ b/gradlew @@ -1,5 +1,21 @@ #!/usr/bin/env sh +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + ############################################################################## ## ## Gradle start up script for UN*X @@ -28,7 +44,7 @@ APP_NAME="Gradle" APP_BASE_NAME=`basename "$0"` # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m"' +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD="maximum" @@ -66,6 +82,7 @@ esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + # Determine the Java command to use to start the JVM. if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then @@ -109,10 +126,11 @@ if $darwin; then GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" fi -# For Cygwin, switch paths to Windows format before running java -if $cygwin ; then +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then APP_HOME=`cygpath --path --mixed "$APP_HOME"` CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` # We build the pattern for arguments to be converted via cygpath @@ -138,19 +156,19 @@ if $cygwin ; then else eval `echo args$i`="\"$arg\"" fi - i=$((i+1)) + i=`expr $i + 1` done case $i in - (0) set -- ;; - (1) set -- "$args0" ;; - (2) set -- "$args0" "$args1" ;; - (3) set -- "$args0" "$args1" "$args2" ;; - (4) set -- "$args0" "$args1" "$args2" "$args3" ;; - (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; esac fi @@ -159,14 +177,9 @@ save () { for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done echo " " } -APP_ARGS=$(save "$@") +APP_ARGS=`save "$@"` # Collect all arguments for the java command, following the shell quoting and substitution rules eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" -# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong -if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then - cd "$(dirname "$0")" -fi - exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat index 0f8d593..107acd3 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -1,84 +1,89 @@ -@if "%DEBUG%" == "" @echo off -@rem ########################################################################## -@rem -@rem Gradle startup script for Windows -@rem -@rem ########################################################################## - -@rem Set local scope for the variables with windows NT shell -if "%OS%"=="Windows_NT" setlocal - -set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. -set APP_BASE_NAME=%~n0 -set APP_HOME=%DIRNAME% - -@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS="-Xmx64m" - -@rem Find java.exe -if defined JAVA_HOME goto findJavaFromJavaHome - -set JAVA_EXE=java.exe -%JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto init - -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:findJavaFromJavaHome -set JAVA_HOME=%JAVA_HOME:"=% -set JAVA_EXE=%JAVA_HOME%/bin/java.exe - -if exist "%JAVA_EXE%" goto init - -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:init -@rem Get command-line arguments, handling Windows variants - -if not "%OS%" == "Windows_NT" goto win9xME_args - -:win9xME_args -@rem Slurp the command line arguments. -set CMD_LINE_ARGS= -set _SKIP=2 - -:win9xME_args_slurp -if "x%~1" == "x" goto execute - -set CMD_LINE_ARGS=%* - -:execute -@rem Setup the command line - -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar - -@rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% - -:end -@rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd - -:fail -rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of -rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 - -:mainEnd -if "%OS%"=="Windows_NT" endlocal - -:omega +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/src/main/kotlin/br/com/guiabolso/sftptos3connector/SftpToS3Connector.kt b/src/main/kotlin/br/com/guiabolso/sftptos3connector/SftpToS3Connector.kt index 27cd0f6..7329f95 100644 --- a/src/main/kotlin/br/com/guiabolso/sftptos3connector/SftpToS3Connector.kt +++ b/src/main/kotlin/br/com/guiabolso/sftptos3connector/SftpToS3Connector.kt @@ -20,6 +20,8 @@ import br.com.guiabolso.sftptos3connector.config.S3Config import br.com.guiabolso.sftptos3connector.config.SftpConfig import br.com.guiabolso.sftptos3connector.internal.s3.S3FileSender import br.com.guiabolso.sftptos3connector.internal.sftp.SftpFileStreamer +import java.io.InputStream +import java.io.OutputStream /** * Simple connector from a SFTP server to a S3 bucket @@ -51,10 +53,15 @@ public class SftpToS3Connector( * As per S3 specification, if there's a failure during the file transfer, no file chunks are stored at S3, so if * no exception is thrown during this execution, it's assumed that the file was transferred correctly. */ - public fun transfer(sftpFilePath: String, s3File: String, kmsKeyId: String? = null): Unit { + public fun transfer( + sftpFilePath: String, + s3File: String, + kmsKeyId: String? = null, + transformer: (InputStream, OutputStream) -> Unit = SimpleCopyTransformer + ): Unit { val sftpFile = sftpFileStreamer.getSftpFile(sftpFilePath) - s3FileSender.send(s3Config.bucket, sftpFile, s3File, kmsKeyId) + s3FileSender.send(s3Config.bucket, sftpFile, s3File, transformer, kmsKeyId) } - - } + +private val SimpleCopyTransformer: (InputStream, OutputStream) -> Unit = { i, o -> i.copyTo(o) } diff --git a/src/main/kotlin/br/com/guiabolso/sftptos3connector/internal/s3/S3FileSender.kt b/src/main/kotlin/br/com/guiabolso/sftptos3connector/internal/s3/S3FileSender.kt index e0d2aaa..92feb1c 100644 --- a/src/main/kotlin/br/com/guiabolso/sftptos3connector/internal/s3/S3FileSender.kt +++ b/src/main/kotlin/br/com/guiabolso/sftptos3connector/internal/s3/S3FileSender.kt @@ -18,9 +18,8 @@ package br.com.guiabolso.sftptos3connector.internal.s3 import br.com.guiabolso.sftptos3connector.internal.sftp.SftpFile import com.amazonaws.services.s3.AmazonS3 -import com.amazonaws.services.s3.model.ObjectMetadata -import com.amazonaws.services.s3.model.PutObjectRequest -import com.amazonaws.services.s3.model.SSEAwsKeyManagementParams +import java.io.InputStream +import java.io.OutputStream internal class S3FileSender( private val s3Client: AmazonS3 @@ -30,31 +29,41 @@ internal class S3FileSender( bucket: String, sftpFile: SftpFile, filename: String, + transformer: (InputStream, OutputStream) -> Unit, kmsKeyId: String? = null ) { if(kmsKeyId == null) { - putObjectUnencrypted(bucket, sftpFile, filename) + streamObjectUnencrypted(bucket, sftpFile, filename, transformer) } else { - putObjectEncrypted(bucket, sftpFile, filename, kmsKeyId) + streamObjectEncrypted(bucket, sftpFile, filename, transformer, kmsKeyId) } } - private fun putObjectUnencrypted(bucket: String, sftpFile: SftpFile, filename: String) { - s3Client.putObject(bucket, filename, sftpFile.stream, objectMetadata(sftpFile.contentLength)) + private fun streamObjectUnencrypted( + bucket: String, + sftpFile: SftpFile, + filename: String, + transformer: (InputStream, OutputStream) -> Unit + ) { + sftpFile.stream.use { sftpStream -> + S3OutputStream(s3Client, bucket, filename).use { s3Stream -> + transformer(sftpStream, s3Stream) + } + } } - private fun putObjectEncrypted(bucket: String, sftpFile: SftpFile, filename: String, kmsKey: String) { - s3Client.putObject( - PutObjectRequest(bucket, - filename, - sftpFile.stream, - objectMetadata(sftpFile.contentLength).withKmsEncryption() - ).withSSEAwsKeyManagementParams(SSEAwsKeyManagementParams(kmsKey)) - ) + private fun streamObjectEncrypted( + bucket: String, + sftpFile: SftpFile, + filename: String, + transformer: (InputStream, OutputStream) -> Unit, + kmsKey: String + ) { + sftpFile.stream.use { sftpStream -> + S3OutputStream(s3Client, bucket, filename, kmsKey).use { s3Stream -> + transformer(sftpStream, s3Stream) + } + } } - - private fun objectMetadata(contentLength: Long) = ObjectMetadata().also { it.contentLength = contentLength } - - private fun ObjectMetadata.withKmsEncryption() = apply { sseAlgorithm = "aws:kms" } } diff --git a/src/main/kotlin/br/com/guiabolso/sftptos3connector/internal/s3/S3OutputStream.kt b/src/main/kotlin/br/com/guiabolso/sftptos3connector/internal/s3/S3OutputStream.kt new file mode 100644 index 0000000..a00e279 --- /dev/null +++ b/src/main/kotlin/br/com/guiabolso/sftptos3connector/internal/s3/S3OutputStream.kt @@ -0,0 +1,117 @@ +package br.com.guiabolso.sftptos3connector.internal.s3 + +import com.amazonaws.services.s3.AmazonS3 +import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest +import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest +import com.amazonaws.services.s3.model.ObjectMetadata +import com.amazonaws.services.s3.model.PartETag +import com.amazonaws.services.s3.model.PutObjectRequest +import com.amazonaws.services.s3.model.SSEAwsKeyManagementParams +import com.amazonaws.services.s3.model.UploadPartRequest +import java.io.ByteArrayInputStream +import java.io.OutputStream + +@Suppress("MagicNumber") +/** + * Inspired on https://gist.github.com/blagerweij/ad1dbb7ee2fff8bcffd372815ad310eb + * by blagerweij + */ +internal class S3OutputStream( + private val s3Client: AmazonS3, + private val bucket: String, + private val path: String, + private val kmsKeyId: String? = null +) : OutputStream() { + + private var open = true + private val buffer = ByteArray(10 * 1024 * 1024) + private var position = 0 + private var uploadId: String? = null + private val etags = mutableListOf() + + override fun write(b: ByteArray) = write(b, 0, b.size) + + override fun write(b: ByteArray, off: Int, len: Int) { + require(open) + + var offset = off + var length = len + var size = buffer.size - position + + while(length > size) { + System.arraycopy(b, offset, buffer, position, size) + position += size + flushBufferAndRewind() + offset += size + length -= size + size = buffer.size - position + } + System.arraycopy(b, offset, buffer, position, length) + position += length + } + + private fun flushBufferAndRewind() { + if(uploadId == null) { + val request = InitiateMultipartUploadRequest(bucket, path) + val response = s3Client.initiateMultipartUpload(request) + uploadId = response.uploadId + } + uploadPart() + position = 0 + } + + private fun uploadPart() { + val result = s3Client.uploadPart(UploadPartRequest() + .withBucketName(bucket) + .withKey(path) + .withUploadId(uploadId) + .withInputStream(ByteArrayInputStream(buffer, 0, position)) + .withPartNumber(etags.size + 1) + .withPartSize(position.toLong()) + ) + etags += result.partETag + } + + override fun close() { + if(!open) return + open = false + + if(uploadId == null) { + val request = createPutObjectRequest() + s3Client.putObject(request) + } else { + if(position > 0 ) uploadPart() + s3Client.completeMultipartUpload(CompleteMultipartUploadRequest(bucket, path, uploadId, etags)) + } + + if(uploadId != null) { + if(position > 0) { + uploadPart() + } + s3Client.completeMultipartUpload(CompleteMultipartUploadRequest(bucket, path, uploadId, etags)) + } + } + + override fun flush() { require(open) } + + override fun write(b: Int) { + require(open) + if(position >= buffer.size) flushBufferAndRewind() + buffer[position++] = b.toByte() + } + + private fun createPutObjectRequest(): PutObjectRequest { + val metadata = objectMetadata(position.toLong()).withKmsEncryption() + val req = PutObjectRequest(bucket, path, ByteArrayInputStream(buffer, 0, position), metadata) + if(kmsKeyId != null) { + req.withSSEAwsKeyManagementParams(SSEAwsKeyManagementParams(kmsKeyId)) + } + return req + } + + private fun objectMetadata(contentLength: Long) = ObjectMetadata().also { it.contentLength = contentLength } + + private fun ObjectMetadata.withKmsEncryption() = apply { + if(kmsKeyId != null) sseAlgorithm = "aws:kms" + } +} diff --git a/src/test/kotlin/io/kotlintest/provided/ProjectConfig.kt b/src/test/kotlin/br/com/guiabolso/sftptos3connector/ProjectConfig.kt similarity index 72% rename from src/test/kotlin/io/kotlintest/provided/ProjectConfig.kt rename to src/test/kotlin/br/com/guiabolso/sftptos3connector/ProjectConfig.kt index 7ec63fe..09d1676 100644 --- a/src/test/kotlin/io/kotlintest/provided/ProjectConfig.kt +++ b/src/test/kotlin/br/com/guiabolso/sftptos3connector/ProjectConfig.kt @@ -14,13 +14,14 @@ * limitations under the License. */ -package io.kotlintest.provided +package br.com.guiabolso.sftptos3connector -import io.kotlintest.AbstractProjectConfig -import io.kotlintest.IsolationMode +import io.kotest.core.config.AbstractProjectConfig +import io.kotest.core.spec.IsolationMode -@Suppress("unused") // KotlinTest uses this object via reflection + +@Suppress("unused") object ProjectConfig : AbstractProjectConfig() { - - override fun isolationMode() = IsolationMode.InstancePerTest + + override val isolationMode = IsolationMode.InstancePerTest } diff --git a/src/test/kotlin/br/com/guiabolso/sftptos3connector/SftpToS3ConnectorTest.kt b/src/test/kotlin/br/com/guiabolso/sftptos3connector/SftpToS3ConnectorTest.kt index 0c79757..03d4322 100644 --- a/src/test/kotlin/br/com/guiabolso/sftptos3connector/SftpToS3ConnectorTest.kt +++ b/src/test/kotlin/br/com/guiabolso/sftptos3connector/SftpToS3ConnectorTest.kt @@ -24,41 +24,56 @@ import br.com.guiabolso.sftptos3connector.internal.sftp.sftpPassword import br.com.guiabolso.sftptos3connector.internal.sftp.sftpUsername import br.com.guiabolso.sftptos3connector.internal.sftp.withConfiguredSftpServer import com.adobe.testing.s3mock.junit5.S3MockExtension -import io.kotlintest.Spec -import io.kotlintest.shouldBe -import io.kotlintest.specs.FunSpec +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import java.io.InputStream +import java.io.OutputStream + +class SftpToS3ConnectorTest : ShouldSpec({ + val s3Mock = S3MockExtension.builder().silent().withSecureConnection(true).build() -class SftpToS3ConnectorTest : FunSpec() { - - private val s3MockExtension = S3MockExtension.builder().silent().withSecureConnection(true).build() - - init { - test("Should transfer a file from SFTP to S3") { - withConfiguredSftpServer { server -> - val s3Client = s3MockExtension.createS3Client() - s3Client.createBucket("bucket") - - val connector = SftpToS3Connector( - SftpConfig("localhost", server.port, sftpUsername, sftpPassword), - S3Config("bucket", s3Client) - ) - - connector.transfer(sftpFilePath = sftpFilePath, s3File = "folder/filename.txt") - - val s3Object = s3Client.getObject("bucket", "folder/filename.txt") - s3Object.objectContent.bufferedReader().readText() shouldBe sftpFileContent - } + should("transfer a file from SFTP to S3") { + withConfiguredSftpServer { server -> + val s3Client = s3Mock.createS3Client() + s3Client.createBucket("bucket") + + val connector = SftpToS3Connector( + SftpConfig("localhost", server.port, sftpUsername, sftpPassword), + S3Config("bucket", s3Client) + ) + + connector.transfer(sftpFilePath = sftpFilePath, s3File = "folder/filename.txt") + + val s3Object = s3Client.getObject("bucket", "folder/filename.txt") + s3Object.objectContent.bufferedReader().readText() shouldBe sftpFileContent } - } - override fun beforeSpec(spec: Spec) { - s3MockExtension.beforeAll(null) + should("process the file through input stream -> output stream pipeline") { + withConfiguredSftpServer { server -> + val s3Client = s3Mock.createS3Client() + s3Client.createBucket("bucket") + + val connector = SftpToS3Connector( + SftpConfig("localhost", server.port, sftpUsername, sftpPassword), + S3Config("bucket", s3Client) + ) + + connector.transfer(sftpFilePath = sftpFilePath, s3File = "folder/filename.txt", transformer = transformer) + + val s3Object = s3Client.getObject("bucket", "folder/filename.txt") + s3Object.objectContent.bufferedReader().readText() shouldBe "FileContent" + } } - - override fun afterSpec(spec: Spec) { - s3MockExtension.afterAll(null) + + beforeSpec { s3Mock.beforeAll(null) } + afterSpec { s3Mock.afterAll(null) } + +}) + +private val transformer: (InputStream, OutputStream) -> Unit = { inputStream, outputStream -> + inputStream.bufferedReader().useLines { + it.filter { it != "MoreContent" }.forEach { outputStream.write(it.encodeToByteArray()) } } - } diff --git a/src/test/kotlin/br/com/guiabolso/sftptos3connector/internal/s3/S3FileSenderTest.kt b/src/test/kotlin/br/com/guiabolso/sftptos3connector/internal/s3/S3FileSenderTest.kt index 61457b5..0138ab1 100644 --- a/src/test/kotlin/br/com/guiabolso/sftptos3connector/internal/s3/S3FileSenderTest.kt +++ b/src/test/kotlin/br/com/guiabolso/sftptos3connector/internal/s3/S3FileSenderTest.kt @@ -19,61 +19,52 @@ package br.com.guiabolso.sftptos3connector.internal.s3 import br.com.guiabolso.sftptos3connector.internal.sftp.SftpFile import com.adobe.testing.s3mock.junit5.S3MockExtension import com.amazonaws.services.s3.AmazonS3 -import io.kotlintest.Spec -import io.kotlintest.shouldBe -import io.kotlintest.specs.BehaviorSpec +import io.kotest.core.spec.style.BehaviorSpec +import io.kotest.matchers.shouldBe import java.io.ByteArrayInputStream import java.io.InputStream +import java.io.OutputStream -class S3FileSenderTest : BehaviorSpec() { - - private val kmsKey = "AWS::KMS::Key" - - private val s3MockExtension = S3MockExtension.builder().silent().withSecureConnection(true).build() - - init { - Given("A file stream with content") { - val content = "abc".toByteArray(Charsets.UTF_8) - val stream: InputStream = ByteArrayInputStream(content) - val sftpFile = SftpFile(stream, content.size.toLong()) - val filename = "abc.file" - - When("The S3FileSender is called to send it without a KMS key") { - val client = s3MockExtension.createS3Client().withBucket("bucket") - val target = S3FileSender(client) - target.send("bucket", sftpFile, filename) +class S3FileSenderTest : BehaviorSpec({ - Then("The file should be sent") { - val sentFile = client.getObject("bucket", filename) - sentFile.objectContent.readBytes() shouldBe content - } - } - - When("The S3FileSender is called to send it with a KMS key") { - s3MockExtension.registerKMSKeyRef(kmsKey) - val client = s3MockExtension.createS3Client().withBucket("bucket") - val target = S3FileSender(client) - target.send("bucket", sftpFile, filename, kmsKey) - - Then("The file should be sent with the kms key") { - val sentFileMetadata = client.getObjectMetadata("bucket", filename) - sentFileMetadata.sseAwsKmsKeyId shouldBe kmsKey - } + val kmsKey = "AWS::KMS::Key" + val s3Mock = S3MockExtension.builder().silent().withSecureConnection(true).build() + + Given("A file stream with content") { + val content = "abc".toByteArray(Charsets.UTF_8) + val stream: InputStream = ByteArrayInputStream(content) + val sftpFile = SftpFile(stream, content.size.toLong()) + val filename = "abc.file" + + When("The S3FileSender is called to send it without a KMS key") { + val client = s3Mock.createS3Client().withBucket("bucket") + val target = S3FileSender(client) + target.send("bucket", sftpFile, filename, simpleCopyTransformer) + + Then("The file should be sent") { + val sentFile = client.getObject("bucket", filename) + sentFile.objectContent.readBytes() shouldBe content } } - } - - private fun AmazonS3.withBucket(bucket: String) = apply { - createBucket(bucket) + + When("The S3FileSender is called to send it with a KMS key") { + s3Mock.registerKMSKeyRef(kmsKey) + val client = s3Mock.createS3Client().withBucket("bucket") + val target = S3FileSender(client) + target.send("bucket", sftpFile, filename, simpleCopyTransformer, kmsKey) + + Then("The file should be sent with the kms key") { + val sentFileMetadata = client.getObjectMetadata("bucket", filename) + sentFileMetadata.sseAwsKmsKeyId shouldBe kmsKey + } } - - - override fun beforeSpec(spec: Spec) { - s3MockExtension.beforeAll(null) - } - - override fun afterSpec(spec: Spec) { - s3MockExtension.afterAll(null) } -} + beforeSpec { s3Mock.beforeAll(null) } + afterSpec { s3Mock.afterAll(null) } + +}) + +private fun AmazonS3.withBucket(bucket: String) = apply { createBucket(bucket) } + +private val simpleCopyTransformer: (InputStream, OutputStream) -> Unit = { i, o -> i.copyTo(o) } diff --git a/src/test/kotlin/br/com/guiabolso/sftptos3connector/internal/sftp/SftpFileStreamerTest.kt b/src/test/kotlin/br/com/guiabolso/sftptos3connector/internal/sftp/SftpFileStreamerTest.kt index 2df40b4..885e6b2 100644 --- a/src/test/kotlin/br/com/guiabolso/sftptos3connector/internal/sftp/SftpFileStreamerTest.kt +++ b/src/test/kotlin/br/com/guiabolso/sftptos3connector/internal/sftp/SftpFileStreamerTest.kt @@ -17,33 +17,28 @@ package br.com.guiabolso.sftptos3connector.internal.sftp import br.com.guiabolso.sftptos3connector.config.SftpConfig -import io.kotlintest.shouldBe -import io.kotlintest.specs.FunSpec +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.shouldBe import java.io.InputStream -@ExperimentalStdlibApi -class SftpFileStreamerTest : FunSpec() { - - init { - test("Should stream the content from a SFTP server") { - withConfiguredSftpServer { server -> - val target = SftpFileStreamer( SftpConfig("localhost", server.port, sftpUsername, sftpPassword)) - - val fileStream: InputStream = target.getSftpFile(sftpFilePath).stream - - fileStream.reader().readText() shouldBe sftpFileContent - } - } - - test("Should return the file and it's content length") { - withConfiguredSftpServer { server -> - val target = SftpFileStreamer( SftpConfig("localhost", server.port, sftpUsername, sftpPassword)) - - val fileInfo = target.getSftpFile(sftpFilePath) - - fileInfo.contentLength shouldBe sftpFileContent.encodeToByteArray().size.toLong() - } +class SftpFileStreamerTest : FunSpec({ + test("Should stream the content from a SFTP server") { + withConfiguredSftpServer { server -> + val target = SftpFileStreamer(SftpConfig("localhost", server.port, sftpUsername, sftpPassword)) + + val fileStream: InputStream = target.getSftpFile(sftpFilePath).stream + + fileStream.reader().readText() shouldBe sftpFileContent } } -} + test("Should return the file and it's content length") { + withConfiguredSftpServer { server -> + val target = SftpFileStreamer(SftpConfig("localhost", server.port, sftpUsername, sftpPassword)) + + val fileInfo = target.getSftpFile(sftpFilePath) + + fileInfo.contentLength shouldBe sftpFileContent.encodeToByteArray().size.toLong() + } + } +}) diff --git a/src/test/kotlin/br/com/guiabolso/sftptos3connector/internal/sftp/SftpServer.kt b/src/test/kotlin/br/com/guiabolso/sftptos3connector/internal/sftp/SftpServer.kt index 40a8438..1f231af 100644 --- a/src/test/kotlin/br/com/guiabolso/sftptos3connector/internal/sftp/SftpServer.kt +++ b/src/test/kotlin/br/com/guiabolso/sftptos3connector/internal/sftp/SftpServer.kt @@ -22,7 +22,7 @@ import java.net.ServerSocket val sftpUsername = "username" val sftpPassword = "password" val sftpFilePath = "path/to/file" -val sftpFileContent = "FileContent" +val sftpFileContent = "FileContent\nMoreContent" fun withConfiguredSftpServer(block: (FakeSftpServer) -> Unit) = FakeSftpServer.withSftpServer { server -> diff --git a/upload-bintray.sh b/upload-bintray.sh deleted file mode 100755 index be5355a..0000000 --- a/upload-bintray.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -./gradlew clean build bintrayUpload -PbintrayUser=${BINTRAY_USER} -PbintrayKey=${BINTRAY_KEY} -PdryRun=false