From 4d830a9d2d46dc07494f02cf50c797f88b3824f8 Mon Sep 17 00:00:00 2001 From: sullis Date: Mon, 1 Apr 2024 09:26:32 -0700 Subject: [PATCH 1/8] support zstd compression --- .../netty/request/NettyRequestFactory.java | 10 ++++++++++ .../java/org/asynchttpclient/util/HttpUtils.java | 9 +++++++++ .../java/org/asynchttpclient/netty/NettyTest.java | 13 +++++++++++++ pom.xml | 8 ++++++++ 4 files changed, 40 insertions(+) diff --git a/client/src/main/java/org/asynchttpclient/netty/request/NettyRequestFactory.java b/client/src/main/java/org/asynchttpclient/netty/request/NettyRequestFactory.java index db07a6323..48662636b 100755 --- a/client/src/main/java/org/asynchttpclient/netty/request/NettyRequestFactory.java +++ b/client/src/main/java/org/asynchttpclient/netty/request/NettyRequestFactory.java @@ -18,6 +18,7 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.handler.codec.compression.Brotli; +import io.netty.handler.codec.compression.Zstd; import io.netty.handler.codec.http.DefaultFullHttpRequest; import io.netty.handler.codec.http.DefaultHttpRequest; import io.netty.handler.codec.http.HttpHeaderValues; @@ -67,6 +68,7 @@ import static org.asynchttpclient.util.HttpUtils.ACCEPT_ALL_HEADER_VALUE; import static org.asynchttpclient.util.HttpUtils.GZIP_DEFLATE; import static org.asynchttpclient.util.HttpUtils.filterOutBrotliFromAcceptEncoding; +import static org.asynchttpclient.util.HttpUtils.filterOutZstdFromAcceptEncoding; import static org.asynchttpclient.util.HttpUtils.hostHeader; import static org.asynchttpclient.util.HttpUtils.originHeader; import static org.asynchttpclient.util.HttpUtils.urlEncodeFormParams; @@ -182,6 +184,11 @@ public NettyRequest newNettyRequest(Request request, boolean performConnectReque // For manual decompression by user, any encoding may suite, so leave untouched headers.set(ACCEPT_ENCODING, filterOutBrotliFromAcceptEncoding(userDefinedAcceptEncoding)); } + if (!Zstd.isAvailable()) { + // zstd is not available. + // For manual decompression by user, any encoding may suit, so leave untouched + headers.set(ACCEPT_ENCODING, filterOutZstdFromAcceptEncoding(userDefinedAcceptEncoding)); + } } } else if (config.isCompressionEnforced()) { // Add Accept Encoding header if compression is enforced @@ -189,6 +196,9 @@ public NettyRequest newNettyRequest(Request request, boolean performConnectReque if (Brotli.isAvailable()) { headers.add(ACCEPT_ENCODING, HttpHeaderValues.BR); } + if (Zstd.isAvailable()) { + headers.add(ACCEPT_ENCODING, HttpHeaderValues.ZSTD); + } } } diff --git a/client/src/main/java/org/asynchttpclient/util/HttpUtils.java b/client/src/main/java/org/asynchttpclient/util/HttpUtils.java index f62e2f235..3cca41e61 100644 --- a/client/src/main/java/org/asynchttpclient/util/HttpUtils.java +++ b/client/src/main/java/org/asynchttpclient/util/HttpUtils.java @@ -40,6 +40,7 @@ public final class HttpUtils { private static final String CONTENT_TYPE_CHARSET_ATTRIBUTE = "charset="; private static final String CONTENT_TYPE_BOUNDARY_ATTRIBUTE = "boundary="; private static final String BROTLY_ACCEPT_ENCODING_SUFFIX = ", br"; + private static final String ZSTD_ACCEPT_ENCODING_SUFFIX = ", zstd"; private HttpUtils() { // Prevent outside initialization @@ -173,4 +174,12 @@ public static CharSequence filterOutBrotliFromAcceptEncoding(String acceptEncodi } return acceptEncoding; } + + public static CharSequence filterOutZstdFromAcceptEncoding(String acceptEncoding) { + // we don't support zstd ATM + if (acceptEncoding.endsWith(ZSTD_ACCEPT_ENCODING_SUFFIX)) { + return acceptEncoding.subSequence(0, acceptEncoding.length() - ZSTD_ACCEPT_ENCODING_SUFFIX.length()); + } + return acceptEncoding; + } } diff --git a/client/src/test/java/org/asynchttpclient/netty/NettyTest.java b/client/src/test/java/org/asynchttpclient/netty/NettyTest.java index 9a0293be3..7878766f1 100644 --- a/client/src/test/java/org/asynchttpclient/netty/NettyTest.java +++ b/client/src/test/java/org/asynchttpclient/netty/NettyTest.java @@ -3,6 +3,7 @@ import io.netty.channel.epoll.Epoll; import io.netty.channel.kqueue.KQueue; import io.netty.handler.codec.compression.Brotli; +import io.netty.handler.codec.compression.Zstd; import io.netty.incubator.channel.uring.IOUring; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledOnOs; @@ -40,4 +41,16 @@ public void brotliIsAvailableOnLinux() { public void brotliIsAvailableOnMac() { assertTrue(Brotli.isAvailable()); } + + @Test + @EnabledOnOs(value = OS.LINUX) + public void zstdIsAvailableOnLinux() { + assertTrue(Zstd.isAvailable()); + } + + @Test + @EnabledOnOs(value = OS.MAC) + public void zstdIsAvailableOnMac() { + assertTrue(Zstd.isAvailable()); + } } diff --git a/pom.xml b/pom.xml index 42fcee0f7..7bf8b6df5 100644 --- a/pom.xml +++ b/pom.xml @@ -61,6 +61,7 @@ 4.1.111.Final 0.0.25.Final 1.16.0 + 1.5.6-1 2.0.13 2.0.1 1.4.11 @@ -224,6 +225,13 @@ true + + com.github.luben + zstd-jni + ${zstd-jni.version} + true + + com.aayushatharva.brotli4j brotli4j From 65f11e0bf662f4693caf9e696927a9d532c13c6c Mon Sep 17 00:00:00 2001 From: sullis Date: Tue, 2 Apr 2024 04:53:12 -0700 Subject: [PATCH 2/8] add license for zstd-jni --- LICENSES/LICENSE.zstd-jni.txt | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 LICENSES/LICENSE.zstd-jni.txt diff --git a/LICENSES/LICENSE.zstd-jni.txt b/LICENSES/LICENSE.zstd-jni.txt new file mode 100644 index 000000000..66abb8ae7 --- /dev/null +++ b/LICENSES/LICENSE.zstd-jni.txt @@ -0,0 +1,26 @@ +Zstd-jni: JNI bindings to Zstd Library + +Copyright (c) 2015-present, Luben Karavelov/ All rights reserved. + +BSD License + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. From 411d412cd6d619ef4e9d752a7318b1c95b252602 Mon Sep 17 00:00:00 2001 From: sullis Date: Thu, 4 Apr 2024 20:01:31 -0700 Subject: [PATCH 3/8] zstd-jni 1.5.6-2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7bf8b6df5..c7caef88e 100644 --- a/pom.xml +++ b/pom.xml @@ -61,8 +61,8 @@ 4.1.111.Final 0.0.25.Final 1.16.0 - 1.5.6-1 2.0.13 + 1.5.6-2 2.0.1 1.4.11 24.0.1 From c6e1e317cc18255369521c029e3d999b7ff62e83 Mon Sep 17 00:00:00 2001 From: sullis Date: Thu, 30 May 2024 14:34:56 -0700 Subject: [PATCH 4/8] add AutomaticDecompressionTest.java --- .../AutomaticDecompressionTest.java | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 client/src/test/java/org/asynchttpclient/AutomaticDecompressionTest.java diff --git a/client/src/test/java/org/asynchttpclient/AutomaticDecompressionTest.java b/client/src/test/java/org/asynchttpclient/AutomaticDecompressionTest.java new file mode 100644 index 000000000..eaffc5dcb --- /dev/null +++ b/client/src/test/java/org/asynchttpclient/AutomaticDecompressionTest.java @@ -0,0 +1,135 @@ +package org.asynchttpclient; + +import com.aayushatharva.brotli4j.encoder.BrotliOutputStream; +import com.aayushatharva.brotli4j.encoder.Encoder; +import com.sun.net.httpserver.Headers; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.zip.GZIPOutputStream; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import com.github.luben.zstd.Zstd; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class AutomaticDecompressionTest { + private static final String UNCOMPRESSED_PAYLOAD = "a".repeat(500); + + private static HttpServer HTTP_SERVER; + + private static AsyncHttpClient createClient() { + AsyncHttpClientConfig config = new DefaultAsyncHttpClientConfig.Builder() + .setEnableAutomaticDecompression(true) + .build(); + return new DefaultAsyncHttpClient(config); + } + + @BeforeAll + static void setupServer() throws Exception { + HTTP_SERVER = HttpServer.create(new InetSocketAddress(0), 0); + + HTTP_SERVER.createContext("/br").setHandler(new HttpHandler() { + @Override + public void handle(HttpExchange exchange) + throws IOException { + exchange.getResponseHeaders().set("Content-Encoding", "br"); + exchange.sendResponseHeaders(200, 0); + OutputStream out = exchange.getResponseBody(); + Encoder.Parameters params = new Encoder.Parameters(); + BrotliOutputStream brotliOutputStream = new BrotliOutputStream(out, params); + brotliOutputStream.write(UNCOMPRESSED_PAYLOAD.getBytes(StandardCharsets.UTF_8)); + brotliOutputStream.flush(); + brotliOutputStream.close(); + } + }); + + HTTP_SERVER.createContext("/zstd").setHandler(new HttpHandler() { + @Override + public void handle(HttpExchange exchange) + throws IOException { + exchange.getResponseHeaders().set("Content-Encoding", "zstd"); + byte[] compressedData = new byte[UNCOMPRESSED_PAYLOAD.length()]; + long n = Zstd.compress(compressedData, UNCOMPRESSED_PAYLOAD.getBytes(StandardCharsets.UTF_8), 2, true); + exchange.sendResponseHeaders(200, n); + OutputStream out = exchange.getResponseBody(); + out.write(compressedData, 0, (int) n); + out.flush(); + out.close(); + exchange.close(); + } + }); + + HTTP_SERVER.createContext("/gzip").setHandler(new HttpHandler() { + @Override + public void handle(HttpExchange exchange) + throws IOException { + exchange.getResponseHeaders().set("Content-Encoding", "gzip"); + exchange.sendResponseHeaders(200, 0); + OutputStream out = exchange.getResponseBody(); + try { + GZIPOutputStream gzip = new GZIPOutputStream(out); + gzip.write(UNCOMPRESSED_PAYLOAD.getBytes(StandardCharsets.UTF_8)); + gzip.flush(); + gzip.close(); + } catch (Exception exception) { + exception.printStackTrace(); + } + } + }); + + HTTP_SERVER.start(); + } + + @AfterAll + static void stopServer() { + if (HTTP_SERVER != null) { + HTTP_SERVER.stop(0); + } + } + + @Test + void zstd() throws Throwable { + io.netty.handler.codec.compression.Zstd.ensureAvailability(); + try (AsyncHttpClient client = createClient()) { + Request request = new RequestBuilder("GET") + .setUrl("http://localhost:" + HTTP_SERVER.getAddress().getPort() + "/zstd") + .build(); + Response response = client.executeRequest(request).get(); + assertEquals(200, response.getStatusCode()); + assertEquals(UNCOMPRESSED_PAYLOAD, response.getResponseBody()); + } + } + + @Test + void brotli() throws Throwable { + io.netty.handler.codec.compression.Brotli.ensureAvailability(); + try (AsyncHttpClient client = createClient()) { + Request request = new RequestBuilder("GET") + .setUrl("http://localhost:" + HTTP_SERVER.getAddress().getPort() + "/br") + .build(); + Response response = client.executeRequest(request).get(); + assertEquals(200, response.getStatusCode()); + assertEquals(UNCOMPRESSED_PAYLOAD, response.getResponseBody()); + } + } + + @Test + void gzip() throws Throwable { + try (AsyncHttpClient client = createClient()) { + Request request = new RequestBuilder("GET") + .setUrl("http://localhost:" + HTTP_SERVER.getAddress().getPort() + "/gzip") + .build(); + Response response = client.executeRequest(request).get(); + assertEquals(200, response.getStatusCode()); + assertEquals(UNCOMPRESSED_PAYLOAD, response.getResponseBody()); + } + } + + +} From 54efd6079294e4fa9c37a4c234a45d4061c852c8 Mon Sep 17 00:00:00 2001 From: sullis Date: Thu, 30 May 2024 14:38:12 -0700 Subject: [PATCH 5/8] cleanup --- .../asynchttpclient/AutomaticDecompressionTest.java | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/client/src/test/java/org/asynchttpclient/AutomaticDecompressionTest.java b/client/src/test/java/org/asynchttpclient/AutomaticDecompressionTest.java index eaffc5dcb..f9e96791d 100644 --- a/client/src/test/java/org/asynchttpclient/AutomaticDecompressionTest.java +++ b/client/src/test/java/org/asynchttpclient/AutomaticDecompressionTest.java @@ -61,7 +61,6 @@ public void handle(HttpExchange exchange) out.write(compressedData, 0, (int) n); out.flush(); out.close(); - exchange.close(); } }); @@ -72,14 +71,10 @@ public void handle(HttpExchange exchange) exchange.getResponseHeaders().set("Content-Encoding", "gzip"); exchange.sendResponseHeaders(200, 0); OutputStream out = exchange.getResponseBody(); - try { - GZIPOutputStream gzip = new GZIPOutputStream(out); - gzip.write(UNCOMPRESSED_PAYLOAD.getBytes(StandardCharsets.UTF_8)); - gzip.flush(); - gzip.close(); - } catch (Exception exception) { - exception.printStackTrace(); - } + GZIPOutputStream gzip = new GZIPOutputStream(out); + gzip.write(UNCOMPRESSED_PAYLOAD.getBytes(StandardCharsets.UTF_8)); + gzip.flush(); + gzip.close(); } }); From 3ef5b5a96c8a51e3f3ac9c288294fcde49fd99af Mon Sep 17 00:00:00 2001 From: sullis Date: Thu, 30 May 2024 14:45:45 -0700 Subject: [PATCH 6/8] zstd-jni 1.5.6-3 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c7caef88e..1a6de507d 100644 --- a/pom.xml +++ b/pom.xml @@ -62,7 +62,7 @@ 0.0.25.Final 1.16.0 2.0.13 - 1.5.6-2 + 1.5.6-3 2.0.1 1.4.11 24.0.1 From a77383558db95c1360d0193011eafba36f54cf3a Mon Sep 17 00:00:00 2001 From: sullis Date: Thu, 30 May 2024 15:04:40 -0700 Subject: [PATCH 7/8] validate Accept-Encoding header --- .../AutomaticDecompressionTest.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/client/src/test/java/org/asynchttpclient/AutomaticDecompressionTest.java b/client/src/test/java/org/asynchttpclient/AutomaticDecompressionTest.java index f9e96791d..45ffaafcc 100644 --- a/client/src/test/java/org/asynchttpclient/AutomaticDecompressionTest.java +++ b/client/src/test/java/org/asynchttpclient/AutomaticDecompressionTest.java @@ -10,6 +10,9 @@ import java.io.OutputStream; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; import java.util.zip.GZIPOutputStream; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -26,6 +29,7 @@ public class AutomaticDecompressionTest { private static AsyncHttpClient createClient() { AsyncHttpClientConfig config = new DefaultAsyncHttpClientConfig.Builder() .setEnableAutomaticDecompression(true) + .setCompressionEnforced(true) .build(); return new DefaultAsyncHttpClient(config); } @@ -38,6 +42,7 @@ static void setupServer() throws Exception { @Override public void handle(HttpExchange exchange) throws IOException { + validateAcceptEncodingHeader(exchange); exchange.getResponseHeaders().set("Content-Encoding", "br"); exchange.sendResponseHeaders(200, 0); OutputStream out = exchange.getResponseBody(); @@ -53,6 +58,7 @@ public void handle(HttpExchange exchange) @Override public void handle(HttpExchange exchange) throws IOException { + validateAcceptEncodingHeader(exchange); exchange.getResponseHeaders().set("Content-Encoding", "zstd"); byte[] compressedData = new byte[UNCOMPRESSED_PAYLOAD.length()]; long n = Zstd.compress(compressedData, UNCOMPRESSED_PAYLOAD.getBytes(StandardCharsets.UTF_8), 2, true); @@ -68,6 +74,7 @@ public void handle(HttpExchange exchange) @Override public void handle(HttpExchange exchange) throws IOException { + validateAcceptEncodingHeader(exchange); exchange.getResponseHeaders().set("Content-Encoding", "gzip"); exchange.sendResponseHeaders(200, 0); OutputStream out = exchange.getResponseBody(); @@ -81,6 +88,15 @@ public void handle(HttpExchange exchange) HTTP_SERVER.start(); } + private static void validateAcceptEncodingHeader(HttpExchange exchange) { + Headers requestHeaders = exchange.getRequestHeaders(); + List acceptEncodingList = requestHeaders.get("Accept-Encoding") + .stream() + .flatMap(x -> Arrays.asList(x.split(",")).stream()) + .collect(Collectors.toList()); + assertEquals(List.of("gzip", "deflate", "br", "zstd"), acceptEncodingList); + } + @AfterAll static void stopServer() { if (HTTP_SERVER != null) { From 19d929f32de4068cfbe90d1306bc4d168d62dca2 Mon Sep 17 00:00:00 2001 From: sullis Date: Fri, 31 May 2024 14:52:23 -0700 Subject: [PATCH 8/8] add license header --- .../AutomaticDecompressionTest.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/client/src/test/java/org/asynchttpclient/AutomaticDecompressionTest.java b/client/src/test/java/org/asynchttpclient/AutomaticDecompressionTest.java index 45ffaafcc..238bb7206 100644 --- a/client/src/test/java/org/asynchttpclient/AutomaticDecompressionTest.java +++ b/client/src/test/java/org/asynchttpclient/AutomaticDecompressionTest.java @@ -1,3 +1,18 @@ +/* + * Copyright (c) 2015-2024 AsyncHttpClient Project. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.asynchttpclient; import com.aayushatharva.brotli4j.encoder.BrotliOutputStream;