From 4bc43fc23c33a23af0b5840671adf53b697eb2dc Mon Sep 17 00:00:00 2001 From: Maxim Grischenko Date: Mon, 16 Sep 2024 16:57:21 +0200 Subject: [PATCH 1/5] #221 fix StaticBucketRouter removes bucketName in the middle of path --- .../business/impl/e2e/MultiDFSFunctionalityIT.java | 4 +++- .../adorsys/datasafe/cli/config/DatasafeFactory.java | 2 ++ .../business/s3/MultiDfsWithCredentialsExampleIT.java | 2 ++ ...rationsWithDefaultDatasafeOnVersionedStorageIT.java | 1 + .../datasafe/rest/impl/config/DatasafeConfig.java | 4 +++- .../simple/adapter/impl/SimpleDatasafeServiceImpl.java | 1 + .../datasafe/storage/impl/s3/S3StorageService.java | 7 ++++--- .../datasafe/storage/impl/s3/StaticBucketRouter.java | 6 ++++-- .../storage/impl/s3/S3SystemStorageServiceIT.java | 1 + .../datasafe/teststorage/WithStorageProvider.java | 10 +++++----- 10 files changed, 26 insertions(+), 12 deletions(-) diff --git a/datasafe-business/src/test/java/de/adorsys/datasafe/business/impl/e2e/MultiDFSFunctionalityIT.java b/datasafe-business/src/test/java/de/adorsys/datasafe/business/impl/e2e/MultiDFSFunctionalityIT.java index c58607ac9..5d547c676 100644 --- a/datasafe-business/src/test/java/de/adorsys/datasafe/business/impl/e2e/MultiDFSFunctionalityIT.java +++ b/datasafe-business/src/test/java/de/adorsys/datasafe/business/impl/e2e/MultiDFSFunctionalityIT.java @@ -137,7 +137,8 @@ void initDatasafe() { accessKey(CREDENTIALS), secretKey(CREDENTIALS) ), - CREDENTIALS, + REGION, + CREDENTIALS, EXECUTOR ); @@ -158,6 +159,7 @@ void initDatasafe() { acc.getAccessKey(), acc.getSecretKey() ), + acc.getRegion(), acc.getBucketName(), EXECUTOR ) diff --git a/datasafe-cli/src/main/java/de/adorsys/datasafe/cli/config/DatasafeFactory.java b/datasafe-cli/src/main/java/de/adorsys/datasafe/cli/config/DatasafeFactory.java index c59e94ac9..33c2c905e 100644 --- a/datasafe-cli/src/main/java/de/adorsys/datasafe/cli/config/DatasafeFactory.java +++ b/datasafe-cli/src/main/java/de/adorsys/datasafe/cli/config/DatasafeFactory.java @@ -86,6 +86,7 @@ private static StorageService amazonS3() { acc.getAccessKey(), acc.getSecretKey() ), + acc.getRegion(), // Bucket name is encoded in first path segment acc.getBucketName(), ExecutorServiceUtil.submitterExecutesOnStarvationExecutingService() @@ -134,6 +135,7 @@ private static S3StorageService getStorageService(String accessKey, String secre return new S3StorageService( amazons3, + region, bucket, ExecutorServiceUtil .submitterExecutesOnStarvationExecutingService( diff --git a/datasafe-examples/datasafe-examples-multidfs/src/test/java/de/adorsys/datasafe/examples/business/s3/MultiDfsWithCredentialsExampleIT.java b/datasafe-examples/datasafe-examples-multidfs/src/test/java/de/adorsys/datasafe/examples/business/s3/MultiDfsWithCredentialsExampleIT.java index 29b7e5f36..36d5ba611 100644 --- a/datasafe-examples/datasafe-examples-multidfs/src/test/java/de/adorsys/datasafe/examples/business/s3/MultiDfsWithCredentialsExampleIT.java +++ b/datasafe-examples/datasafe-examples-multidfs/src/test/java/de/adorsys/datasafe/examples/business/s3/MultiDfsWithCredentialsExampleIT.java @@ -108,6 +108,7 @@ void testMultiUserStorageUserSetup() { StorageService directoryStorage = new S3StorageService( directoryClient, DIRECTORY_BUCKET.getBucketName(), + REGION, EXECUTOR ); @@ -137,6 +138,7 @@ void testMultiUserStorageUserSetup() { ), // Bucket name is encoded in first path segment acc.getBucketName(), + acc.getRegion(), EXECUTOR ) ) diff --git a/datasafe-examples/datasafe-examples-versioned-s3/src/test/java/de/adorsys/datasafe/examples/business/s3/BaseUserOperationsWithDefaultDatasafeOnVersionedStorageIT.java b/datasafe-examples/datasafe-examples-versioned-s3/src/test/java/de/adorsys/datasafe/examples/business/s3/BaseUserOperationsWithDefaultDatasafeOnVersionedStorageIT.java index c8429a2cb..030d76dc3 100644 --- a/datasafe-examples/datasafe-examples-versioned-s3/src/test/java/de/adorsys/datasafe/examples/business/s3/BaseUserOperationsWithDefaultDatasafeOnVersionedStorageIT.java +++ b/datasafe-examples/datasafe-examples-versioned-s3/src/test/java/de/adorsys/datasafe/examples/business/s3/BaseUserOperationsWithDefaultDatasafeOnVersionedStorageIT.java @@ -121,6 +121,7 @@ void init() { .config(new DefaultDFSConfig(cephMappedUrl, "secret"::toCharArray)) .storage(new S3StorageService( cephS3, + "", VERSIONED_BUCKET_NAME, ExecutorServiceUtil.submitterExecutesOnStarvationExecutingService())) .build(); diff --git a/datasafe-rest-impl/src/main/java/de/adorsys/datasafe/rest/impl/config/DatasafeConfig.java b/datasafe-rest-impl/src/main/java/de/adorsys/datasafe/rest/impl/config/DatasafeConfig.java index 71c31b9fe..62c029573 100644 --- a/datasafe-rest-impl/src/main/java/de/adorsys/datasafe/rest/impl/config/DatasafeConfig.java +++ b/datasafe-rest-impl/src/main/java/de/adorsys/datasafe/rest/impl/config/DatasafeConfig.java @@ -132,6 +132,7 @@ StorageService clientCredentials(AmazonS3 s3, S3Factory factory, DatasafePropert ExecutorService executorService = ExecutorServiceUtil.submitterExecutesOnStarvationExecutingService(); S3StorageService basicStorage = new S3StorageService( s3, + properties.getAmazonRegion(), properties.getBucketName(), executorService ); @@ -185,6 +186,7 @@ StorageService singleStorageServiceFilesystem(DatasafeProperties properties) { StorageService singleStorageServiceS3(AmazonS3 s3, DatasafeProperties properties) { return new S3StorageService( s3, + properties.getAmazonRegion(), properties.getBucketName(), ExecutorServiceUtil.submitterExecutesOnStarvationExecutingService() ); @@ -203,7 +205,7 @@ StorageService multiStorageService(DatasafeProperties properties) { ) ); - S3StorageService s3StorageService = new S3StorageService(s3(properties), properties.getBucketName(), + S3StorageService s3StorageService = new S3StorageService(s3(properties), properties.getAmazonRegion(), properties.getBucketName(), ExecutorServiceUtil.submitterExecutesOnStarvationExecutingService() ); diff --git a/datasafe-simple-adapter/datasafe-simple-adapter-impl/src/main/java/de/adorsys/datasafe/simple/adapter/impl/SimpleDatasafeServiceImpl.java b/datasafe-simple-adapter/datasafe-simple-adapter-impl/src/main/java/de/adorsys/datasafe/simple/adapter/impl/SimpleDatasafeServiceImpl.java index 02f9cb024..ae72cfd76 100644 --- a/datasafe-simple-adapter/datasafe-simple-adapter-impl/src/main/java/de/adorsys/datasafe/simple/adapter/impl/SimpleDatasafeServiceImpl.java +++ b/datasafe-simple-adapter/datasafe-simple-adapter-impl/src/main/java/de/adorsys/datasafe/simple/adapter/impl/SimpleDatasafeServiceImpl.java @@ -299,6 +299,7 @@ private static SystemRootAndStorageService useAmazonS3(AmazonS3DFSCredentials df } StorageService storageService = new S3StorageService( amazons3, + amazonS3DFSCredentials.getRegion(), amazonS3DFSCredentials.getContainer(), ExecutorServiceUtil .submitterExecutesOnStarvationExecutingService( diff --git a/datasafe-storage/datasafe-storage-impl-s3/src/main/java/de/adorsys/datasafe/storage/impl/s3/S3StorageService.java b/datasafe-storage/datasafe-storage-impl-s3/src/main/java/de/adorsys/datasafe/storage/impl/s3/S3StorageService.java index 59d204569..439e207c9 100644 --- a/datasafe-storage/datasafe-storage-impl-s3/src/main/java/de/adorsys/datasafe/storage/impl/s3/S3StorageService.java +++ b/datasafe-storage/datasafe-storage-impl-s3/src/main/java/de/adorsys/datasafe/storage/impl/s3/S3StorageService.java @@ -45,12 +45,13 @@ public class S3StorageService implements StorageService { /** * @param s3 Connection to S3 - * @param bucketName Bucket to use + * @param region Region to use + * @param bucket Bucket to use * @param executorService Multipart sending threadpool (file chunks are sent in parallel) */ - public S3StorageService(AmazonS3 s3, String bucketName, ExecutorService executorService) { + public S3StorageService(AmazonS3 s3, String region, String bucket, ExecutorService executorService) { this.s3 = s3; - this.router = new StaticBucketRouter(bucketName); + this.router = new StaticBucketRouter(region, bucket); this.executorService = executorService; } diff --git a/datasafe-storage/datasafe-storage-impl-s3/src/main/java/de/adorsys/datasafe/storage/impl/s3/StaticBucketRouter.java b/datasafe-storage/datasafe-storage-impl-s3/src/main/java/de/adorsys/datasafe/storage/impl/s3/StaticBucketRouter.java index 2c43345b5..444ba1279 100644 --- a/datasafe-storage/datasafe-storage-impl-s3/src/main/java/de/adorsys/datasafe/storage/impl/s3/StaticBucketRouter.java +++ b/datasafe-storage/datasafe-storage-impl-s3/src/main/java/de/adorsys/datasafe/storage/impl/s3/StaticBucketRouter.java @@ -8,6 +8,7 @@ @RequiredArgsConstructor public class StaticBucketRouter implements BucketRouter { + private final String region; private final String bucketName; @Override @@ -20,10 +21,11 @@ public String resourceKey(AbsoluteLocation resource) { UnaryOperator trimStartingSlash = str -> str.replaceFirst("^/", ""); String resourcePath = trimStartingSlash.apply(resource.location().getRawPath()); - if (bucketName == null || "".equals(bucketName) || !resourcePath.contains(bucketName)) { + String bucketNameWithRegion = region + "/" + bucketName; + if (bucketName == null || "".equals(bucketName) || !resourcePath.startsWith(bucketNameWithRegion)) { return resourcePath; } - return trimStartingSlash.apply(resourcePath.substring(resourcePath.indexOf(bucketName) + bucketName.length())); + return trimStartingSlash.apply(resourcePath.substring(resourcePath.indexOf(bucketNameWithRegion) + bucketNameWithRegion.length())); } } diff --git a/datasafe-storage/datasafe-storage-impl-s3/src/test/java/de/adorsys/datasafe/storage/impl/s3/S3SystemStorageServiceIT.java b/datasafe-storage/datasafe-storage-impl-s3/src/test/java/de/adorsys/datasafe/storage/impl/s3/S3SystemStorageServiceIT.java index ee66097af..535c517af 100644 --- a/datasafe-storage/datasafe-storage-impl-s3/src/test/java/de/adorsys/datasafe/storage/impl/s3/S3SystemStorageServiceIT.java +++ b/datasafe-storage/datasafe-storage-impl-s3/src/test/java/de/adorsys/datasafe/storage/impl/s3/S3SystemStorageServiceIT.java @@ -82,6 +82,7 @@ static void beforeAll() { void init() { this.storageService = new S3StorageService( s3, + "eu-central-1", bucketName, ExecutorServiceUtil.submitterExecutesOnStarvationExecutingService() ); diff --git a/datasafe-test-storages/src/test/java/de/adorsys/datasafe/teststorage/WithStorageProvider.java b/datasafe-test-storages/src/test/java/de/adorsys/datasafe/teststorage/WithStorageProvider.java index ba03f52c4..29f6e0b61 100644 --- a/datasafe-test-storages/src/test/java/de/adorsys/datasafe/teststorage/WithStorageProvider.java +++ b/datasafe-test-storages/src/test/java/de/adorsys/datasafe/teststorage/WithStorageProvider.java @@ -235,7 +235,7 @@ protected static StorageDescriptor minio() { StorageDescriptorName.MINIO, () -> { minioStorage.get(); - return new S3StorageService(minio, primaryBucket, EXECUTOR_SERVICE); + return new S3StorageService(minio, minioRegion, primaryBucket, EXECUTOR_SERVICE); }, new Uri("s3://" + primaryBucket + "/" + bucketPath + "/"), minioAccessKeyID, @@ -254,7 +254,7 @@ protected static StorageDescriptor cephVersioned() { StorageDescriptorName.CEPH, () -> { cephStorage.get(); - return new S3StorageService(ceph, primaryBucket, EXECUTOR_SERVICE); + return new S3StorageService(ceph, cephRegion, primaryBucket, EXECUTOR_SERVICE); }, new Uri("s3://" + primaryBucket + "/" + bucketPath + "/"), cephAccessKeyID, @@ -275,10 +275,10 @@ private static boolean skipCeph() { protected static Function storageServiceByBucket() { if (null == amazonS3) { - return bucketName -> new S3StorageService(minio, bucketName, EXECUTOR_SERVICE); + return bucketName -> new S3StorageService(minio, amazonRegion, bucketName, EXECUTOR_SERVICE); } - return bucketName -> new S3StorageService(amazonS3, bucketName, EXECUTOR_SERVICE); + return bucketName -> new S3StorageService(amazonS3, amazonRegion, bucketName, EXECUTOR_SERVICE); } protected static StorageDescriptor s3() { @@ -290,7 +290,7 @@ protected static StorageDescriptor s3() { StorageDescriptorName.AMAZON, () -> { amazonStorage.get(); - return new S3StorageService(amazonS3, primaryBucket, EXECUTOR_SERVICE); + return new S3StorageService(amazonS3, amazonRegion, primaryBucket, EXECUTOR_SERVICE); }, new Uri("s3://" + primaryBucket + "/" + bucketPath + "/"), amazonAccessKeyID, From a6df36cbc32badf4c6a40a35318dbe73b7d95259 Mon Sep 17 00:00:00 2001 From: Maxim Grischenko Date: Tue, 17 Sep 2024 09:49:12 +0200 Subject: [PATCH 2/5] #221 fix test --- .../business/s3/MultiDfsWithCredentialsExampleIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datasafe-examples/datasafe-examples-multidfs/src/test/java/de/adorsys/datasafe/examples/business/s3/MultiDfsWithCredentialsExampleIT.java b/datasafe-examples/datasafe-examples-multidfs/src/test/java/de/adorsys/datasafe/examples/business/s3/MultiDfsWithCredentialsExampleIT.java index 36d5ba611..69e14b63b 100644 --- a/datasafe-examples/datasafe-examples-multidfs/src/test/java/de/adorsys/datasafe/examples/business/s3/MultiDfsWithCredentialsExampleIT.java +++ b/datasafe-examples/datasafe-examples-multidfs/src/test/java/de/adorsys/datasafe/examples/business/s3/MultiDfsWithCredentialsExampleIT.java @@ -107,8 +107,8 @@ void testMultiUserStorageUserSetup() { // static client that will be used to access `directory` bucket: StorageService directoryStorage = new S3StorageService( directoryClient, - DIRECTORY_BUCKET.getBucketName(), REGION, + DIRECTORY_BUCKET.getBucketName(), EXECUTOR ); @@ -136,9 +136,9 @@ void testMultiUserStorageUserSetup() { acc.getAccessKey(), acc.getSecretKey() ), + acc.getRegion(), // Bucket name is encoded in first path segment acc.getBucketName(), - acc.getRegion(), EXECUTOR ) ) From 9d3b489f6d5271f672209dc49ec23e9f364ee894 Mon Sep 17 00:00:00 2001 From: Maxim Grischenko Date: Tue, 24 Sep 2024 11:43:15 +0200 Subject: [PATCH 3/5] #221 fix test --- .../de/adorsys/datasafe/simple/adapter/spring/InjectionIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasafe-simple-adapter/datasafe-simple-adapter-spring/src/test/java/de/adorsys/datasafe/simple/adapter/spring/InjectionIT.java b/datasafe-simple-adapter/datasafe-simple-adapter-spring/src/test/java/de/adorsys/datasafe/simple/adapter/spring/InjectionIT.java index 76b2c0505..be553a90f 100644 --- a/datasafe-simple-adapter/datasafe-simple-adapter-spring/src/test/java/de/adorsys/datasafe/simple/adapter/spring/InjectionIT.java +++ b/datasafe-simple-adapter/datasafe-simple-adapter-spring/src/test/java/de/adorsys/datasafe/simple/adapter/spring/InjectionIT.java @@ -103,7 +103,7 @@ private AbsoluteLocation getPrivateResourceAbsoluteLocation(DFS } if (dfsCredentials instanceof AmazonS3DFSCredentials) { AmazonS3DFSCredentials a = (AmazonS3DFSCredentials) dfsCredentials; - return new AbsoluteLocation<>(BasePrivateResource.forPrivate(new URI(a.getUrl() + "/" + a.getRootBucket()))); + return new AbsoluteLocation<>(BasePrivateResource.forPrivate(new URI(a.getUrl() + "/" + a.getRegion() + "/" + a.getRootBucket()))); } throw new TestException("NYI"); } From d7a11786d54f7331a240cbed1418d3578e145cc3 Mon Sep 17 00:00:00 2001 From: Maxim Grischenko Date: Tue, 24 Sep 2024 14:55:05 +0200 Subject: [PATCH 4/5] #221 rewriting static bucket router + test --- .../storage/impl/s3/StaticBucketRouter.java | 15 ++++--- .../impl/s3/StaticBucketRouterTest.java | 45 +++++++++++++++++++ 2 files changed, 55 insertions(+), 5 deletions(-) create mode 100644 datasafe-storage/datasafe-storage-impl-s3/src/test/java/de/adorsys/datasafe/storage/impl/s3/StaticBucketRouterTest.java diff --git a/datasafe-storage/datasafe-storage-impl-s3/src/main/java/de/adorsys/datasafe/storage/impl/s3/StaticBucketRouter.java b/datasafe-storage/datasafe-storage-impl-s3/src/main/java/de/adorsys/datasafe/storage/impl/s3/StaticBucketRouter.java index 444ba1279..fa04a8e4c 100644 --- a/datasafe-storage/datasafe-storage-impl-s3/src/main/java/de/adorsys/datasafe/storage/impl/s3/StaticBucketRouter.java +++ b/datasafe-storage/datasafe-storage-impl-s3/src/main/java/de/adorsys/datasafe/storage/impl/s3/StaticBucketRouter.java @@ -21,11 +21,16 @@ public String resourceKey(AbsoluteLocation resource) { UnaryOperator trimStartingSlash = str -> str.replaceFirst("^/", ""); String resourcePath = trimStartingSlash.apply(resource.location().getRawPath()); - String bucketNameWithRegion = region + "/" + bucketName; - if (bucketName == null || "".equals(bucketName) || !resourcePath.startsWith(bucketNameWithRegion)) { - return resourcePath; - } - return trimStartingSlash.apply(resourcePath.substring(resourcePath.indexOf(bucketNameWithRegion) + bucketNameWithRegion.length())); + if (bucketName != null && !bucketName.isEmpty()) { + if (resourcePath.startsWith(bucketName)) { + return trimStartingSlash.apply(resourcePath.substring(resourcePath.indexOf(bucketName) + bucketName.length())); + } + String bucketNameWithRegion = region + "/" + bucketName; + if (resourcePath.startsWith(bucketNameWithRegion)) { + return trimStartingSlash.apply(resourcePath.substring(resourcePath.indexOf(bucketNameWithRegion) + bucketNameWithRegion.length())); + } + } + return resourcePath; } } diff --git a/datasafe-storage/datasafe-storage-impl-s3/src/test/java/de/adorsys/datasafe/storage/impl/s3/StaticBucketRouterTest.java b/datasafe-storage/datasafe-storage-impl-s3/src/test/java/de/adorsys/datasafe/storage/impl/s3/StaticBucketRouterTest.java new file mode 100644 index 000000000..16c71900a --- /dev/null +++ b/datasafe-storage/datasafe-storage-impl-s3/src/test/java/de/adorsys/datasafe/storage/impl/s3/StaticBucketRouterTest.java @@ -0,0 +1,45 @@ +package de.adorsys.datasafe.storage.impl.s3; + +import de.adorsys.datasafe.types.api.resource.AbsoluteLocation; +import de.adorsys.datasafe.types.api.resource.BasePrivateResource; +import de.adorsys.datasafe.types.api.resource.Uri; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +@Slf4j +public class StaticBucketRouterTest { + + private StaticBucketRouter router; + + @BeforeEach + void setup() { + String region = "region"; + String bucketName = "bucket"; + router = new StaticBucketRouter(region, bucketName); + } + + @Test + void resourceKeyTest() { + var root = new AbsoluteLocation<>(BasePrivateResource.forPrivate(new Uri("http://s3-us-west-2.amazonaws.com/bucket/users/myuserid/private/files/bucket/users/otheruser/private/files/somefile.aes"))); + log.info(String.valueOf(root)); + String resourcePath = router.resourceKey(root); + assertThat(resourcePath).hasToString("users/myuserid/private/files/bucket/users/otheruser/private/files/somefile.aes"); + } + + @Test + void noBucketInPath() { + var root = new AbsoluteLocation<>(BasePrivateResource.forPrivate(new Uri("http://bucket.s3-us-west-2.amazonaws.com/users/myuserid/private/files/bucket/users/otheruser/private/files/somefile.aes"))); + String resourcePath = router.resourceKey(root); + assertThat(resourcePath).hasToString("users/myuserid/private/files/bucket/users/otheruser/private/files/somefile.aes"); + } + + @Test + void regionAndBucketInPath() { + var root = new AbsoluteLocation<>(BasePrivateResource.forPrivate(new Uri("s3://host/region/bucket/users/myuserid/private/files/bucket/users/otheruser/private/files/somefile.aes"))); + String resourcePath = router.resourceKey(root); + assertThat(resourcePath).hasToString("users/myuserid/private/files/bucket/users/otheruser/private/files/somefile.aes"); + } +} From a6ca0a6ead29fa4b215f10ea672a7e78ad01824c Mon Sep 17 00:00:00 2001 From: AssahBismarkabah Date: Wed, 2 Oct 2024 10:58:01 +0100 Subject: [PATCH 5/5] fix checkstyle failures --- .../impl/e2e/MultiDFSFunctionalityIT.java | 62 +++--- .../s3/MultiDfsWithCredentialsExampleIT.java | 43 ++-- .../rest/impl/config/DatasafeConfig.java | 18 +- .../impl/SimpleDatasafeServiceImpl.java | 18 +- .../teststorage/WithStorageProvider.java | 188 +++++++++--------- 5 files changed, 170 insertions(+), 159 deletions(-) diff --git a/datasafe-business/src/test/java/de/adorsys/datasafe/business/impl/e2e/MultiDFSFunctionalityIT.java b/datasafe-business/src/test/java/de/adorsys/datasafe/business/impl/e2e/MultiDFSFunctionalityIT.java index 5b24490f9..556f54867 100644 --- a/datasafe-business/src/test/java/de/adorsys/datasafe/business/impl/e2e/MultiDFSFunctionalityIT.java +++ b/datasafe-business/src/test/java/de/adorsys/datasafe/business/impl/e2e/MultiDFSFunctionalityIT.java @@ -36,6 +36,7 @@ import de.adorsys.datasafe.types.api.types.ReadStorePassword; import de.adorsys.datasafe.types.api.utils.ExecutorServiceUtil; import de.adorsys.datasafe.types.api.utils.ReadKeyPasswordTestFactory; + import java.io.InputStream; import java.io.OutputStream; import java.security.UnrecoverableKeyException; @@ -47,6 +48,7 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; + import lombok.SneakyThrows; import lombok.experimental.Delegate; import lombok.extern.slf4j.Slf4j; @@ -58,9 +60,11 @@ import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; + import static de.adorsys.datasafe.types.api.shared.DockerUtil.getDockerUri; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; + /** * This test distributes users' storage access keystore, document encryption keystore, * users' private files into buckets that reside on different buckets. Bootstrap knows only how to @@ -128,41 +132,41 @@ static void stopAll() { @BeforeEach void initDatasafe() { StorageService directoryStorage = new S3StorageService( - S3ClientFactory.getClient( - endpointsByHostNoBucket.get(CREDENTIALS), - REGION, - accessKey(CREDENTIALS), - secretKey(CREDENTIALS) - ), + S3ClientFactory.getClient( + endpointsByHostNoBucket.get(CREDENTIALS), + REGION, + accessKey(CREDENTIALS), + secretKey(CREDENTIALS) + ), REGION, CREDENTIALS, - EXECUTOR + EXECUTOR ); OverridesRegistry registry = new BaseOverridesRegistry(); this.datasafeServices = DaggerDefaultDatasafeServices.builder() - .config(new DefaultDFSConfig(endpointsByHost.get(CREDENTIALS), new ReadStorePassword("PAZZWORT"))) - .overridesRegistry(registry) - .storage(new RegexDelegatingStorage( - ImmutableMap.builder() - .put(Pattern.compile(endpointsByHost.get(CREDENTIALS) + ".+"), directoryStorage) - .put( - Pattern.compile(LOCALHOST + ".+"), - new UriBasedAuthStorageService( - acc -> new S3StorageService( - S3ClientFactory.getClient( - acc.getEndpoint(), - acc.getRegion(), - acc.getAccessKey(), - acc.getSecretKey() - ), - acc.getRegion(), - acc.getBucketName(), - EXECUTOR - ) - ) - ).build()) - ).build(); + .config(new DefaultDFSConfig(endpointsByHost.get(CREDENTIALS), new ReadStorePassword("PAZZWORT"))) + .overridesRegistry(registry) + .storage(new RegexDelegatingStorage( + ImmutableMap.builder() + .put(Pattern.compile(endpointsByHost.get(CREDENTIALS) + ".+"), directoryStorage) + .put( + Pattern.compile(LOCALHOST + ".+"), + new UriBasedAuthStorageService( + acc -> new S3StorageService( + S3ClientFactory.getClient( + acc.getEndpoint(), + acc.getRegion(), + acc.getAccessKey(), + acc.getSecretKey() + ), + acc.getRegion(), + acc.getBucketName(), + EXECUTOR + ) + ) + ).build()) + ).build(); BucketAccessServiceImplRuntimeDelegatable.overrideWith( registry, args -> new WithCredentialProvider(args.getStorageKeyStoreOperations()) diff --git a/datasafe-examples/datasafe-examples-multidfs/src/test/java/de/adorsys/datasafe/examples/business/s3/MultiDfsWithCredentialsExampleIT.java b/datasafe-examples/datasafe-examples-multidfs/src/test/java/de/adorsys/datasafe/examples/business/s3/MultiDfsWithCredentialsExampleIT.java index 360603e0d..a17aff860 100644 --- a/datasafe-examples/datasafe-examples-multidfs/src/test/java/de/adorsys/datasafe/examples/business/s3/MultiDfsWithCredentialsExampleIT.java +++ b/datasafe-examples/datasafe-examples-multidfs/src/test/java/de/adorsys/datasafe/examples/business/s3/MultiDfsWithCredentialsExampleIT.java @@ -26,6 +26,7 @@ import de.adorsys.datasafe.types.api.resource.StorageIdentifier; import de.adorsys.datasafe.types.api.shared.AwsClientRetry; import de.adorsys.datasafe.types.api.utils.ExecutorServiceUtil; + import java.io.OutputStream; import java.net.URI; import java.nio.charset.StandardCharsets; @@ -35,6 +36,7 @@ import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.regex.Pattern; + import lombok.SneakyThrows; import lombok.experimental.Delegate; import lombok.extern.slf4j.Slf4j; @@ -44,6 +46,7 @@ import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; + import static de.adorsys.datasafe.examples.business.s3.MinioContainerId.DIRECTORY_BUCKET; import static de.adorsys.datasafe.examples.business.s3.MinioContainerId.FILES_BUCKET_ONE; import static de.adorsys.datasafe.examples.business.s3.MinioContainerId.FILES_BUCKET_TWO; @@ -121,26 +124,26 @@ void testMultiUserStorageUserSetup() { .storage( new RegexDelegatingStorage( ImmutableMap.builder() - // bind URI that contains `directoryBucket` to directoryStorage - .put(Pattern.compile(directoryBucketS3Uri + ".+"), directoryStorage) - .put( - Pattern.compile(getDockerUri("http://127.0.0.1") + ".+"), - // Dynamically creates S3 client with bucket name equal to host value - new UriBasedAuthStorageService( - acc -> new S3StorageService( - S3ClientFactory.getClient( - acc.getEndpoint(), - acc.getRegion(), - acc.getAccessKey(), - acc.getSecretKey() - ), - acc.getRegion(), - // Bucket name is encoded in first path segment - acc.getBucketName(), - EXECUTOR - ) - ) - ).build() + // bind URI that contains `directoryBucket` to directoryStorage + .put(Pattern.compile(directoryBucketS3Uri + ".+"), directoryStorage) + .put( + Pattern.compile(getDockerUri("http://127.0.0.1") + ".+"), + // Dynamically creates S3 client with bucket name equal to host value + new UriBasedAuthStorageService( + acc -> new S3StorageService( + S3ClientFactory.getClient( + acc.getEndpoint(), + acc.getRegion(), + acc.getAccessKey(), + acc.getSecretKey() + ), + acc.getRegion(), + // Bucket name is encoded in first path segment + acc.getBucketName(), + EXECUTOR + ) + ) + ).build() ) ) .overridesRegistry(registry) diff --git a/datasafe-rest-impl/src/main/java/de/adorsys/datasafe/rest/impl/config/DatasafeConfig.java b/datasafe-rest-impl/src/main/java/de/adorsys/datasafe/rest/impl/config/DatasafeConfig.java index 0a2e1cded..ae4dfb826 100644 --- a/datasafe-rest-impl/src/main/java/de/adorsys/datasafe/rest/impl/config/DatasafeConfig.java +++ b/datasafe-rest-impl/src/main/java/de/adorsys/datasafe/rest/impl/config/DatasafeConfig.java @@ -34,12 +34,14 @@ import de.adorsys.datasafe.types.api.context.overrides.OverridesRegistry; import de.adorsys.datasafe.types.api.types.ReadStorePassword; import de.adorsys.datasafe.types.api.utils.ExecutorServiceUtil; + import java.net.URI; import java.nio.file.Paths; import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.regex.Pattern; + import lombok.experimental.Delegate; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; @@ -130,10 +132,10 @@ VersionedDatasafeServices versionedDatasafeServices(StorageService storageServic StorageService clientCredentials(AmazonS3 s3, S3Factory factory, DatasafeProperties properties) { ExecutorService executorService = ExecutorServiceUtil.submitterExecutesOnStarvationExecutingService(); S3StorageService basicStorage = new S3StorageService( - s3, - properties.getAmazonRegion(), - properties.getBucketName(), - executorService + s3, + properties.getAmazonRegion(), + properties.getBucketName(), + executorService ); return new RegexDelegatingStorage( @@ -184,10 +186,10 @@ StorageService singleStorageServiceFilesystem(DatasafeProperties properties) { @ConditionalOnProperty(name = DATASAFE_S3_STORAGE, havingValue = "true") StorageService singleStorageServiceS3(AmazonS3 s3, DatasafeProperties properties) { return new S3StorageService( - s3, - properties.getAmazonRegion(), - properties.getBucketName(), - ExecutorServiceUtil.submitterExecutesOnStarvationExecutingService() + s3, + properties.getAmazonRegion(), + properties.getBucketName(), + ExecutorServiceUtil.submitterExecutesOnStarvationExecutingService() ); } diff --git a/datasafe-simple-adapter/datasafe-simple-adapter-impl/src/main/java/de/adorsys/datasafe/simple/adapter/impl/SimpleDatasafeServiceImpl.java b/datasafe-simple-adapter/datasafe-simple-adapter-impl/src/main/java/de/adorsys/datasafe/simple/adapter/impl/SimpleDatasafeServiceImpl.java index ba01874f6..17b5cc608 100644 --- a/datasafe-simple-adapter/datasafe-simple-adapter-impl/src/main/java/de/adorsys/datasafe/simple/adapter/impl/SimpleDatasafeServiceImpl.java +++ b/datasafe-simple-adapter/datasafe-simple-adapter-impl/src/main/java/de/adorsys/datasafe/simple/adapter/impl/SimpleDatasafeServiceImpl.java @@ -45,12 +45,14 @@ import de.adorsys.datasafe.types.api.types.ReadKeyPassword; import de.adorsys.datasafe.types.api.types.ReadStorePassword; import de.adorsys.datasafe.types.api.utils.ExecutorServiceUtil; + import java.io.InputStream; import java.io.OutputStream; import java.net.URI; import java.nio.file.FileSystems; import java.util.List; import java.util.stream.Collectors; + import lombok.AllArgsConstructor; import lombok.Getter; import lombok.SneakyThrows; @@ -297,14 +299,14 @@ private static SystemRootAndStorageService useAmazonS3(AmazonS3DFSCredentials df amazons3.createBucket(amazonS3DFSCredentials.getContainer()); } StorageService storageService = new S3StorageService( - amazons3, - amazonS3DFSCredentials.getRegion(), - amazonS3DFSCredentials.getContainer(), - ExecutorServiceUtil - .submitterExecutesOnStarvationExecutingService( - amazonS3DFSCredentials.getThreadPoolSize(), - amazonS3DFSCredentials.getQueueSize() - ) + amazons3, + amazonS3DFSCredentials.getRegion(), + amazonS3DFSCredentials.getContainer(), + ExecutorServiceUtil + .submitterExecutesOnStarvationExecutingService( + amazonS3DFSCredentials.getThreadPoolSize(), + amazonS3DFSCredentials.getQueueSize() + ) ); URI systemRoot = URI.create(S3_PREFIX + amazonS3DFSCredentials.getRootBucket()); log.info("build DFS to S3 with root " + amazonS3DFSCredentials.getRootBucket() + " and url " + amazonS3DFSCredentials.getUrl()); diff --git a/datasafe-test-storages/src/test/java/de/adorsys/datasafe/teststorage/WithStorageProvider.java b/datasafe-test-storages/src/test/java/de/adorsys/datasafe/teststorage/WithStorageProvider.java index 422dbe330..503632a12 100644 --- a/datasafe-test-storages/src/test/java/de/adorsys/datasafe/teststorage/WithStorageProvider.java +++ b/datasafe-test-storages/src/test/java/de/adorsys/datasafe/teststorage/WithStorageProvider.java @@ -165,83 +165,83 @@ static void shutdown() { @ValueSource protected static Stream allLocalDefaultStorages() { return Stream.of( - fs(), - minio() - /* No CEPH here because it is quite slow*/ + fs(), + minio() + /* No CEPH here because it is quite slow*/ ).filter(Objects::nonNull); } @ValueSource protected static Stream allLocalStorages() { return Stream.of( - fs(), - minio(), - cephVersioned() + fs(), + minio(), + cephVersioned() ).filter(Objects::nonNull); } @ValueSource protected static Stream allDefaultStorages() { return Stream.of( - fs(), - minio(), - s3() + fs(), + minio(), + s3() ).filter(Objects::nonNull); } @ValueSource protected static Stream allStorages() { return Stream.of( - fs(), - minio(), - cephVersioned(), - s3() + fs(), + minio(), + cephVersioned(), + s3() ).filter(Objects::nonNull); } @ValueSource protected static Stream fsOnly() { return Stream.of( - fs() + fs() ).filter(Objects::nonNull); } @ValueSource protected static Stream s3Only() { return Stream.of( - s3() + s3() ).filter(Objects::nonNull); } @ValueSource protected static Stream minioOnly() { return Stream.of( - minio() + minio() ).filter(Objects::nonNull); } protected static StorageDescriptor fs() { return new StorageDescriptor( - StorageDescriptorName.FILESYSTEM, + StorageDescriptorName.FILESYSTEM, () -> new FileSystemStorageService(new Uri(tempDir.toUri())), new Uri(tempDir.toUri()), - null, null, null, - tempDir.toString() + null, null, null, + tempDir.toString() ); } protected static StorageDescriptor minio() { return new StorageDescriptor( - StorageDescriptorName.MINIO, - () -> { - minioStorage.get(); - return new S3StorageService(minio, minioRegion, primaryBucket, EXECUTOR_SERVICE); - }, - new Uri("s3://" + primaryBucket + "/" + bucketPath + "/"), - minioAccessKeyID, - minioSecretAccessKey, - minioRegion, - primaryBucket + "/" + bucketPath + StorageDescriptorName.MINIO, + () -> { + minioStorage.get(); + return new S3StorageService(minio, minioRegion, primaryBucket, EXECUTOR_SERVICE); + }, + new Uri("s3://" + primaryBucket + "/" + bucketPath + "/"), + minioAccessKeyID, + minioSecretAccessKey, + minioRegion, + primaryBucket + "/" + bucketPath ); } @@ -251,16 +251,16 @@ protected static StorageDescriptor cephVersioned() { } return new StorageDescriptor( - StorageDescriptorName.CEPH, - () -> { - cephStorage.get(); - return new S3StorageService(ceph, cephRegion, primaryBucket, EXECUTOR_SERVICE); - }, - new Uri("s3://" + primaryBucket + "/" + bucketPath + "/"), - cephAccessKeyID, - cephSecretAccessKey, - cephRegion, - primaryBucket + "/" + bucketPath + StorageDescriptorName.CEPH, + () -> { + cephStorage.get(); + return new S3StorageService(ceph, cephRegion, primaryBucket, EXECUTOR_SERVICE); + }, + new Uri("s3://" + primaryBucket + "/" + bucketPath + "/"), + cephAccessKeyID, + cephSecretAccessKey, + cephRegion, + primaryBucket + "/" + bucketPath ); } @@ -287,16 +287,16 @@ protected static StorageDescriptor s3() { } return new StorageDescriptor( - StorageDescriptorName.AMAZON, - () -> { - amazonStorage.get(); - return new S3StorageService(amazonS3, amazonRegion, primaryBucket, EXECUTOR_SERVICE); - }, - new Uri("s3://" + primaryBucket + "/" + bucketPath + "/"), - amazonAccessKeyID, - amazonSecretAccessKey, - amazonRegion, - primaryBucket + "/" + bucketPath + StorageDescriptorName.AMAZON, + () -> { + amazonStorage.get(); + return new S3StorageService(amazonS3, amazonRegion, primaryBucket, EXECUTOR_SERVICE); + }, + new Uri("s3://" + primaryBucket + "/" + bucketPath + "/"), + amazonAccessKeyID, + amazonSecretAccessKey, + amazonRegion, + primaryBucket + "/" + bucketPath ); } @@ -325,7 +325,7 @@ private static void initS3() { AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder.standard() .withCredentials(new AWSStaticCredentialsProvider( new BasicAWSCredentials(amazonAccessKeyID, amazonSecretAccessKey)) - ); + ); if (buckets.size() > 1) { log.info("Using {} buckets:{}", buckets.size(), buckets); @@ -337,10 +337,10 @@ private static void initS3() { final boolean isRealAmazon = amazonUrl.endsWith(amazonDomain); amazonS3ClientBuilder = amazonS3ClientBuilder - .withClientConfiguration(new ClientConfiguration().withProtocol(Protocol.HTTP)) - .withEndpointConfiguration( - new AwsClientBuilder.EndpointConfiguration(amazonUrl, amazonRegion) - ); + .withClientConfiguration(new ClientConfiguration().withProtocol(Protocol.HTTP)) + .withEndpointConfiguration( + new AwsClientBuilder.EndpointConfiguration(amazonUrl, amazonRegion) + ); if (isRealAmazon) { amazonMappedUrl = amazonProtocol + primaryBucket + "." + amazonDomain; } else { @@ -356,27 +356,27 @@ private static void initS3() { private static void startMinio() { log.info("Starting MINIO"); minioContainer = new GenericContainer("minio/minio") - .withExposedPorts(9000) - .withEnv("MINIO_ACCESS_KEY", minioAccessKeyID) - .withEnv("MINIO_SECRET_KEY", minioSecretAccessKey) - .withCommand("server /data") - .waitingFor(Wait.defaultWaitStrategy()); + .withExposedPorts(9000) + .withEnv("MINIO_ACCESS_KEY", minioAccessKeyID) + .withEnv("MINIO_SECRET_KEY", minioSecretAccessKey) + .withCommand("server /data") + .waitingFor(Wait.defaultWaitStrategy()); minioContainer.start(); Integer mappedPort = minioContainer.getMappedPort(9000); minioMappedUrl = minioUrl + ":" + mappedPort; log.info("Minio mapped URL:" + minioMappedUrl); minio = AmazonS3ClientBuilder.standard() - .withEndpointConfiguration( - new AwsClientBuilder.EndpointConfiguration(minioMappedUrl, minioRegion) - ) - .withCredentials( - new AWSStaticCredentialsProvider( - new BasicAWSCredentials(minioAccessKeyID, minioSecretAccessKey) + .withEndpointConfiguration( + new AwsClientBuilder.EndpointConfiguration(minioMappedUrl, minioRegion) ) - ) - .enablePathStyleAccess() - .build(); + .withCredentials( + new AWSStaticCredentialsProvider( + new BasicAWSCredentials(minioAccessKeyID, minioSecretAccessKey) + ) + ) + .enablePathStyleAccess() + .build(); buckets.forEach(minio::createBucket); @@ -385,45 +385,45 @@ private static void startMinio() { private static void startCeph() { log.info("Starting CEPH"); cephContainer = new GenericContainer("ceph/daemon") - .withExposedPorts(8000) - .withEnv("RGW_FRONTEND_PORT", "8000") - .withEnv("SREE_PORT", "5000") - .withEnv("DEBUG", "verbose") - .withEnv("CEPH_DEMO_UID", "nano") - .withEnv("MON_IP", "127.0.0.1") - .withEnv("CEPH_PUBLIC_NETWORK", "0.0.0.0/0") - .withEnv("CEPH_DAEMON", "demo") - .withEnv("DEMO_DAEMONS", "mon,mgr,osd,rgw") - .withEnv("CEPH_DEMO_ACCESS_KEY", cephAccessKeyID) - .withEnv("CEPH_DEMO_SECRET_KEY", cephSecretAccessKey) - .withCommand("mkdir -p /etc/ceph && mkdir -p /var/lib/ceph && /entrypoint.sh") - .waitingFor(Wait.defaultWaitStrategy().withStartupTimeout(Duration.ofSeconds(180))); + .withExposedPorts(8000) + .withEnv("RGW_FRONTEND_PORT", "8000") + .withEnv("SREE_PORT", "5000") + .withEnv("DEBUG", "verbose") + .withEnv("CEPH_DEMO_UID", "nano") + .withEnv("MON_IP", "127.0.0.1") + .withEnv("CEPH_PUBLIC_NETWORK", "0.0.0.0/0") + .withEnv("CEPH_DAEMON", "demo") + .withEnv("DEMO_DAEMONS", "mon,mgr,osd,rgw") + .withEnv("CEPH_DEMO_ACCESS_KEY", cephAccessKeyID) + .withEnv("CEPH_DEMO_SECRET_KEY", cephSecretAccessKey) + .withCommand("mkdir -p /etc/ceph && mkdir -p /var/lib/ceph && /entrypoint.sh") + .waitingFor(Wait.defaultWaitStrategy().withStartupTimeout(Duration.ofSeconds(180))); cephContainer.start(); Integer mappedPort = cephContainer.getMappedPort(8000); cephMappedUrl = cephUrl + ":" + mappedPort; log.info("Ceph mapped URL:" + cephMappedUrl); ceph = AmazonS3ClientBuilder.standard() - .withEndpointConfiguration( - new AwsClientBuilder.EndpointConfiguration(cephMappedUrl, cephRegion) - ) - .withCredentials( - new AWSStaticCredentialsProvider( - new BasicAWSCredentials(cephAccessKeyID, cephSecretAccessKey) + .withEndpointConfiguration( + new AwsClientBuilder.EndpointConfiguration(cephMappedUrl, cephRegion) + ) + .withCredentials( + new AWSStaticCredentialsProvider( + new BasicAWSCredentials(cephAccessKeyID, cephSecretAccessKey) + ) ) - ) - .enablePathStyleAccess() - .build(); + .enablePathStyleAccess() + .build(); ceph.createBucket(buckets.get(0)); // curiously enough CEPH docs are incorrect, looks like they do support version id: // https://github.com/ceph/ceph/blame/bc065cae7857c352ca36d5f06cdb5107cf72ed41/src/rgw/rgw_rest_s3.cc // so for versioned local tests we can use CEPH ceph.setBucketVersioningConfiguration( - new SetBucketVersioningConfigurationRequest( - primaryBucket, - new BucketVersioningConfiguration(BucketVersioningConfiguration.ENABLED) - ) + new SetBucketVersioningConfigurationRequest( + primaryBucket, + new BucketVersioningConfiguration(BucketVersioningConfiguration.ENABLED) + ) ); }