From b41039a2b97fd4d2cf717c03e0fd848ca657f4e0 Mon Sep 17 00:00:00 2001 From: Shivam Malhotra Date: Tue, 5 Nov 2024 20:50:20 +0530 Subject: [PATCH] test: Increased timeout for S3 unit tests (#6335) Should help fix some of the sporadic TimeoutExceptions that we get innightly jobs with the S3 unit tests. --- .../iceberg/util/S3WarehouseSqliteCatalogBase.java | 5 ++++- .../deephaven/extensions/s3/testlib/S3Helper.java | 14 +++++++++++--- .../s3/testlib/S3SeekableChannelTestSetup.java | 10 +++++++--- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/S3WarehouseSqliteCatalogBase.java b/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/S3WarehouseSqliteCatalogBase.java index 2c8d34e76fb..acd529037f5 100644 --- a/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/S3WarehouseSqliteCatalogBase.java +++ b/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/S3WarehouseSqliteCatalogBase.java @@ -18,6 +18,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import static io.deephaven.extensions.s3.testlib.S3Helper.TIMEOUT_SECONDS; + public abstract class S3WarehouseSqliteCatalogBase extends SqliteCatalogBase { public abstract S3Instructions s3Instructions(); @@ -31,7 +33,8 @@ protected IcebergCatalogAdapter catalogAdapter(TestInfo testInfo, Path rootDir, final String catalogName = methodName + "-catalog"; final String bucket = methodName.toLowerCase(Locale.US) + "-bucket"; try (final S3AsyncClient client = s3AsyncClient()) { - client.createBucket(CreateBucketRequest.builder().bucket(bucket).build()).get(5, TimeUnit.SECONDS); + client.createBucket(CreateBucketRequest.builder().bucket(bucket).build()) + .get(TIMEOUT_SECONDS, TimeUnit.SECONDS); } properties.put(CatalogProperties.WAREHOUSE_LOCATION, "s3://" + bucket + "/warehouse"); properties.put(CatalogProperties.FILE_IO_IMPL, S3FileIO.class.getName()); diff --git a/extensions/s3/src/test/java/io/deephaven/extensions/s3/testlib/S3Helper.java b/extensions/s3/src/test/java/io/deephaven/extensions/s3/testlib/S3Helper.java index 933c63bd288..2e03388bba5 100644 --- a/extensions/s3/src/test/java/io/deephaven/extensions/s3/testlib/S3Helper.java +++ b/extensions/s3/src/test/java/io/deephaven/extensions/s3/testlib/S3Helper.java @@ -26,6 +26,12 @@ import java.util.stream.Collectors; public final class S3Helper { + + /** + * Timeout in seconds for S3 operations for testing. + */ + public static final int TIMEOUT_SECONDS = 30; + public static void uploadDirectory( S3AsyncClient s3AsyncClient, Path dir, @@ -59,7 +65,8 @@ private static void uploadDirectory( public static void deleteAllKeys(S3AsyncClient s3AsyncClient, String bucket) throws ExecutionException, InterruptedException, TimeoutException { ListObjectsV2Response response = s3AsyncClient - .listObjectsV2(ListObjectsV2Request.builder().bucket(bucket).build()).get(5, TimeUnit.SECONDS); + .listObjectsV2(ListObjectsV2Request.builder().bucket(bucket).build()) + .get(TIMEOUT_SECONDS, TimeUnit.SECONDS); final List> futures = new ArrayList<>(); while (true) { final List deletes = response.contents() @@ -80,12 +87,13 @@ public static void deleteAllKeys(S3AsyncClient s3AsyncClient, String bucket) } response = s3AsyncClient.listObjectsV2( ListObjectsV2Request.builder().bucket(bucket).continuationToken(nextContinuationToken).build()) - .get(5, TimeUnit.SECONDS); + .get(TIMEOUT_SECONDS, TimeUnit.SECONDS); } if (futures.isEmpty()) { return; } - CompletableFuture.allOf(futures.stream().toArray(CompletableFuture[]::new)).get(5, TimeUnit.SECONDS); + CompletableFuture.allOf(futures.stream().toArray(CompletableFuture[]::new)) + .get(TIMEOUT_SECONDS, TimeUnit.SECONDS); } private static ObjectIdentifier objectId(String o) { diff --git a/extensions/s3/src/test/java/io/deephaven/extensions/s3/testlib/S3SeekableChannelTestSetup.java b/extensions/s3/src/test/java/io/deephaven/extensions/s3/testlib/S3SeekableChannelTestSetup.java index 2cb04c83364..6cc9fe05fa8 100644 --- a/extensions/s3/src/test/java/io/deephaven/extensions/s3/testlib/S3SeekableChannelTestSetup.java +++ b/extensions/s3/src/test/java/io/deephaven/extensions/s3/testlib/S3SeekableChannelTestSetup.java @@ -25,6 +25,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import static io.deephaven.extensions.s3.testlib.S3Helper.TIMEOUT_SECONDS; + public abstract class S3SeekableChannelTestSetup { protected ExecutorService executor; @@ -39,19 +41,21 @@ protected final void doSetUp() throws ExecutionException, InterruptedException, executor = Executors.newCachedThreadPool(); bucket = UUID.randomUUID().toString(); asyncClient = s3AsyncClient(); - asyncClient.createBucket(CreateBucketRequest.builder().bucket(bucket).build()).get(5, TimeUnit.SECONDS); + asyncClient.createBucket(CreateBucketRequest.builder().bucket(bucket).build()) + .get(TIMEOUT_SECONDS, TimeUnit.SECONDS); } protected final void doTearDown() throws ExecutionException, InterruptedException, TimeoutException { S3Helper.deleteAllKeys(asyncClient, bucket); - asyncClient.deleteBucket(DeleteBucketRequest.builder().bucket(bucket).build()).get(5, TimeUnit.SECONDS); + asyncClient.deleteBucket(DeleteBucketRequest.builder().bucket(bucket).build()) + .get(TIMEOUT_SECONDS, TimeUnit.SECONDS); asyncClient.close(); executor.shutdownNow(); } protected void uploadDirectory(final Path directory, final String prefix) throws ExecutionException, InterruptedException, TimeoutException { - S3Helper.uploadDirectory(asyncClient, directory, bucket, prefix, Duration.ofSeconds(5)); + S3Helper.uploadDirectory(asyncClient, directory, bucket, prefix, Duration.ofSeconds(TIMEOUT_SECONDS)); } protected final URI uri(String key) {