From 80c0f0bc8025e64c96d6f1b7013a71c5f9d13cb7 Mon Sep 17 00:00:00 2001 From: Benedikt Schwab Date: Tue, 30 Aug 2022 11:08:14 +0200 Subject: [PATCH] added zstd support --- buildSrc/src/main/kotlin/Dependencies.kt | 10 +++++--- buildSrc/src/main/kotlin/Plugins.kt | 2 +- rtron-io/build.gradle.kts | 2 ++ .../rtron/io/files/CompressedFileExtension.kt | 1 + .../io/rtron/io/files/PathExtensions.kt | 23 ++++++++++++------- .../main/processor/AdditionalParameters.kt | 4 +++- .../readerwriter/opendrive/OpendriveReader.kt | 3 ++- 7 files changed, 31 insertions(+), 14 deletions(-) diff --git a/buildSrc/src/main/kotlin/Dependencies.kt b/buildSrc/src/main/kotlin/Dependencies.kt index 77524e98..cb5c20a9 100644 --- a/buildSrc/src/main/kotlin/Dependencies.kt +++ b/buildSrc/src/main/kotlin/Dependencies.kt @@ -24,7 +24,7 @@ object DependencyVersions { // testing libraries const val junit = "5.9.0" const val assertj = "3.23.1" - const val mockk = "1.12.5" + const val mockk = "1.12.7" // logging libraries const val kotlinLogging = "2.1.23" @@ -34,8 +34,8 @@ object DependencyVersions { const val orchid = "0.21.1" // object creation libraries - const val kotlinxSerializationJson = "1.3.3" - const val kaml = "0.46.0" + const val kotlinxSerializationJson = "1.4.0" + const val kaml = "0.47.0" const val jakartaActivationApi = "2.1.0" const val jakartaXmlBindApi = "4.0.0" const val jaxb = "4.0.0" @@ -49,6 +49,8 @@ object DependencyVersions { const val commonsIO = "2.11.0" const val commonsCSV = "1.9.0" const val commonsLang = "3.12.0" + const val commonsCompress = "1.21" + const val zstdJni = "1.5.2-3" const val emojiJava = "5.1.1" // math libraries @@ -98,6 +100,8 @@ object Dependencies { const val commonsIO = "commons-io:commons-io:${DependencyVersions.commonsIO}" const val commonsCSV = "org.apache.commons:commons-csv:${DependencyVersions.commonsCSV}" const val commonsLang = "org.apache.commons:commons-lang3:${DependencyVersions.commonsLang}" + const val commonsCompress = "org.apache.commons:commons-compress:${DependencyVersions.commonsCompress}" + const val zstdJni = "com.github.luben:zstd-jni:${DependencyVersions.zstdJni}" const val emojiJava = "com.vdurmont:emoji-java:${DependencyVersions.emojiJava}" // math libraries diff --git a/buildSrc/src/main/kotlin/Plugins.kt b/buildSrc/src/main/kotlin/Plugins.kt index bf3b156b..9925494c 100644 --- a/buildSrc/src/main/kotlin/Plugins.kt +++ b/buildSrc/src/main/kotlin/Plugins.kt @@ -17,7 +17,7 @@ object PluginVersions { const val shadowjar = "7.1.2" - const val ktlint = "10.3.0" + const val ktlint = "11.0.0" const val xjc = "1.6" const val versionChecker = "0.42.0" const val orchid = "0.21.1" diff --git a/rtron-io/build.gradle.kts b/rtron-io/build.gradle.kts index 49eaeb10..25971b06 100644 --- a/rtron-io/build.gradle.kts +++ b/rtron-io/build.gradle.kts @@ -20,6 +20,8 @@ dependencies { implementation(Dependencies.commonsIO) implementation(Dependencies.commonsCSV) implementation(Dependencies.commonsLang) + implementation(Dependencies.commonsCompress) + implementation(Dependencies.zstdJni) implementation(Dependencies.emojiJava) // math libraries diff --git a/rtron-io/src/main/kotlin/io/rtron/io/files/CompressedFileExtension.kt b/rtron-io/src/main/kotlin/io/rtron/io/files/CompressedFileExtension.kt index 18237736..0c20d2dc 100644 --- a/rtron-io/src/main/kotlin/io/rtron/io/files/CompressedFileExtension.kt +++ b/rtron-io/src/main/kotlin/io/rtron/io/files/CompressedFileExtension.kt @@ -22,4 +22,5 @@ package io.rtron.io.files enum class CompressedFileExtension(val extension: String, val extensionWithDot: String) { ZIP("zip", ".zip"), GZ("gz", ".gz"), + ZST("zst", ".zst"), } diff --git a/rtron-io/src/main/kotlin/io/rtron/io/files/PathExtensions.kt b/rtron-io/src/main/kotlin/io/rtron/io/files/PathExtensions.kt index a19dc3c6..69b6a8ef 100644 --- a/rtron-io/src/main/kotlin/io/rtron/io/files/PathExtensions.kt +++ b/rtron-io/src/main/kotlin/io/rtron/io/files/PathExtensions.kt @@ -16,14 +16,16 @@ package io.rtron.io.files +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream +import org.apache.commons.compress.compressors.zstandard.ZstdCompressorInputStream +import org.apache.commons.compress.compressors.zstandard.ZstdCompressorOutputStream import org.apache.commons.io.FileUtils import java.io.BufferedOutputStream import java.io.InputStream import java.io.OutputStream import java.nio.file.Files import java.nio.file.Path -import java.util.zip.GZIPInputStream -import java.util.zip.GZIPOutputStream import java.util.zip.ZipEntry import java.util.zip.ZipFile import java.util.zip.ZipOutputStream @@ -61,24 +63,29 @@ fun Path.inputStreamFromDirectOrCompressedFile(): InputStream = val zipEntry = zipFile.getEntry(this.nameWithoutExtension) zipFile.getInputStream(zipEntry) } - CompressedFileExtension.GZ.extension -> GZIPInputStream(this.inputStream()) + CompressedFileExtension.GZ.extension -> GzipCompressorInputStream(this.inputStream()) + CompressedFileExtension.ZST.extension -> ZstdCompressorInputStream(this.inputStream()) else -> this.inputStream() } /** * Constructs a new OutputStream of this file either directly or compressed according to the path's extension. */ -fun Path.outputStreamDirectOrCompressed(): OutputStream = - when (this.extension) { +fun Path.outputStreamDirectOrCompressed(): OutputStream { + val bufferedOutputStream = BufferedOutputStream(this.outputStream()) + + return when (this.extension) { CompressedFileExtension.ZIP.extension -> { - val bufferedOutputStream = BufferedOutputStream(this.outputStream()) val zippedOutputStream = ZipOutputStream(bufferedOutputStream) zippedOutputStream.putNextEntry(ZipEntry(this.nameWithoutExtension)) zippedOutputStream } CompressedFileExtension.GZ.extension -> { - val bufferedOutputStream = BufferedOutputStream(this.outputStream()) - GZIPOutputStream(bufferedOutputStream) + GzipCompressorOutputStream(bufferedOutputStream) + } + CompressedFileExtension.ZST.extension -> { + ZstdCompressorOutputStream(bufferedOutputStream) } else -> this.outputStream() } +} diff --git a/rtron-main/src/main/kotlin/io/rtron/main/processor/AdditionalParameters.kt b/rtron-main/src/main/kotlin/io/rtron/main/processor/AdditionalParameters.kt index 93d3b76c..68516991 100644 --- a/rtron-main/src/main/kotlin/io/rtron/main/processor/AdditionalParameters.kt +++ b/rtron-main/src/main/kotlin/io/rtron/main/processor/AdditionalParameters.kt @@ -24,11 +24,13 @@ import io.rtron.io.files.CompressedFileExtension enum class CompressionFormat { NONE, GZ, - ZIP + ZIP, + ZST } fun CompressionFormat.toOptionalCompressedFileExtension(): Option = when (this) { CompressionFormat.NONE -> None CompressionFormat.GZ -> CompressedFileExtension.GZ.some() CompressionFormat.ZIP -> CompressedFileExtension.ZIP.some() + CompressionFormat.ZST -> CompressedFileExtension.ZST.some() } diff --git a/rtron-readerwriter/src/main/kotlin/io/rtron/readerwriter/opendrive/OpendriveReader.kt b/rtron-readerwriter/src/main/kotlin/io/rtron/readerwriter/opendrive/OpendriveReader.kt index e3d421e0..609ea799 100644 --- a/rtron-readerwriter/src/main/kotlin/io/rtron/readerwriter/opendrive/OpendriveReader.kt +++ b/rtron-readerwriter/src/main/kotlin/io/rtron/readerwriter/opendrive/OpendriveReader.kt @@ -82,7 +82,8 @@ class OpendriveReader private constructor( val supportedFilenameEndings: Set = setOf( ".xodr", ".xodr.${CompressedFileExtension.ZIP.extension}", - ".xodr.${CompressedFileExtension.GZ.extension}" + ".xodr.${CompressedFileExtension.GZ.extension}", + ".xodr.${CompressedFileExtension.ZST.extension}", ) fun of(filePath: Path): Either = either.eager {