From 2df0dba03bae7f0aee59f1b549be7b93fb3d8b6f Mon Sep 17 00:00:00 2001 From: PeeAyBee Date: Mon, 27 Nov 2023 16:17:38 +0300 Subject: [PATCH 1/3] EPMRPP-87433 || Add missed filename for attachment record mapper (#956) * EPMRPP-80519 || Add methods for removing buckets and files * EPMRPP-80519 || Remove attachments by project id * EPMRPP-86363 exclude system attributes in launches table widget (#932) * EPMRPP-86363 exclude system attributes in launches table widget * EPMRPP-86363 fixed check-style comments * EPMRPP-87433 || Add missed filename to the attachment record mapper --------- Co-authored-by: Ivan_Kustau Co-authored-by: Ivan Kustau <86599591+IvanKustau@users.noreply.github.com> Co-authored-by: Siarhei Hrabko <45555481+grabsefx@users.noreply.github.com> Co-authored-by: Pavel Bortnik --- .../reportportal/dao/util/RecordMappers.java | 85 ++++++++++--------- 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/src/main/java/com/epam/ta/reportportal/dao/util/RecordMappers.java b/src/main/java/com/epam/ta/reportportal/dao/util/RecordMappers.java index 67106e201..265eba6c9 100644 --- a/src/main/java/com/epam/ta/reportportal/dao/util/RecordMappers.java +++ b/src/main/java/com/epam/ta/reportportal/dao/util/RecordMappers.java @@ -129,7 +129,7 @@ */ public class RecordMappers { - private static final ObjectMapper objectMapper; + private static final ObjectMapper objectMapper; static { objectMapper = new ObjectMapper(); @@ -220,6 +220,7 @@ public class RecordMappers { attachment.setProjectId(r.get(ATTACHMENT.PROJECT_ID)); attachment.setLaunchId(r.get(ATTACHMENT.LAUNCH_ID)); attachment.setItemId(r.get(ATTACHMENT.ITEM_ID)); + attachment.setFileName(r.get(ATTACHMENT.FILE_NAME)); return attachment; }).orElse(null); @@ -323,8 +324,8 @@ public class RecordMappers { ); /** - * Maps record into {@link PatternTemplate} object (only {@link PatternTemplate#id} and - * {@link PatternTemplate#name} fields) + * Maps record into {@link PatternTemplate} object (only {@link PatternTemplate#id} and {@link + * PatternTemplate#name} fields) */ public static final Function> PATTERN_TEMPLATE_NAME_RECORD_MAPPER = r -> ofNullable( r.get( @@ -409,45 +410,45 @@ public class RecordMappers { return new ReportPortalUser.ProjectDetails(projectId, projectName, projectRole); }; - public static final RecordMapper ACTIVITY_MAPPER = r -> { - Activity activity = new Activity(); - activity.setId(r.get(ACTIVITY.ID)); - activity.setCreatedAt(r.get(ACTIVITY.CREATED_AT, LocalDateTime.class)); - activity.setAction(EventAction.valueOf(r.get(ACTIVITY.ACTION))); - activity.setEventName(r.get(ACTIVITY.EVENT_NAME)); - activity.setPriority(EventPriority.valueOf(r.get(ACTIVITY.PRIORITY))); - activity.setObjectId(r.get(ACTIVITY.OBJECT_ID)); - activity.setObjectName(r.get(ACTIVITY.OBJECT_NAME)); - activity.setObjectType(EventObject.valueOf(r.get(ACTIVITY.OBJECT_TYPE))); - activity.setProjectId(r.get(ACTIVITY.PROJECT_ID)); - activity.setProjectName(r.get(PROJECT.NAME)); - String detailsJson = r.get(ACTIVITY.DETAILS, String.class); - ofNullable(detailsJson).ifPresent(s -> { - try { - ActivityDetails details = objectMapper.readValue(s, ActivityDetails.class); - activity.setDetails(details); - } catch (IOException e) { - throw new ReportPortalException(ErrorType.UNCLASSIFIED_REPORT_PORTAL_ERROR); - } - }); - activity.setSubjectId(r.get(ACTIVITY.SUBJECT_ID)); - activity.setSubjectName(ofNullable(r.get(USERS.LOGIN)).orElse(r.get(ACTIVITY.SUBJECT_NAME))); - activity.setSubjectType(EventSubject.valueOf(r.get(ACTIVITY.SUBJECT_TYPE))); - return activity; - }; - - public static final RecordMapper OWNED_ENTITY_RECORD_MAPPER = r -> r.into( - OwnedEntity.class); - - private static final BiConsumer WIDGET_USER_FILTER_MAPPER = (widget, res) -> ofNullable( - res.get(FILTER.ID)).ifPresent( - id -> { - Set filters = ofNullable(widget.getFilters()).orElseGet(Sets::newLinkedHashSet); - UserFilter filter = new UserFilter(); - filter.setId(id); - filters.add(filter); - widget.setFilters(filters); - }); + public static final RecordMapper ACTIVITY_MAPPER = r -> { + Activity activity = new Activity(); + activity.setId(r.get(ACTIVITY.ID)); + activity.setCreatedAt(r.get(ACTIVITY.CREATED_AT, LocalDateTime.class)); + activity.setAction(EventAction.valueOf(r.get(ACTIVITY.ACTION))); + activity.setEventName(r.get(ACTIVITY.EVENT_NAME)); + activity.setPriority(EventPriority.valueOf(r.get(ACTIVITY.PRIORITY))); + activity.setObjectId(r.get(ACTIVITY.OBJECT_ID)); + activity.setObjectName(r.get(ACTIVITY.OBJECT_NAME)); + activity.setObjectType(EventObject.valueOf(r.get(ACTIVITY.OBJECT_TYPE))); + activity.setProjectId(r.get(ACTIVITY.PROJECT_ID)); + activity.setProjectName(r.get(PROJECT.NAME)); + String detailsJson = r.get(ACTIVITY.DETAILS, String.class); + ofNullable(detailsJson).ifPresent(s -> { + try { + ActivityDetails details = objectMapper.readValue(s, ActivityDetails.class); + activity.setDetails(details); + } catch (IOException e) { + throw new ReportPortalException(ErrorType.UNCLASSIFIED_REPORT_PORTAL_ERROR); + } + }); + activity.setSubjectId(r.get(ACTIVITY.SUBJECT_ID)); + activity.setSubjectName(ofNullable(r.get(USERS.LOGIN)).orElse(r.get(ACTIVITY.SUBJECT_NAME))); + activity.setSubjectType(EventSubject.valueOf(r.get(ACTIVITY.SUBJECT_TYPE))); + return activity; + }; + + public static final RecordMapper OWNED_ENTITY_RECORD_MAPPER = r -> r.into( + OwnedEntity.class); + + private static final BiConsumer WIDGET_USER_FILTER_MAPPER = (widget, res) -> ofNullable( + res.get(FILTER.ID)).ifPresent( + id -> { + Set filters = ofNullable(widget.getFilters()).orElseGet(Sets::newLinkedHashSet); + UserFilter filter = new UserFilter(); + filter.setId(id); + filters.add(filter); + widget.setFilters(filters); + }); private static final BiConsumer WIDGET_OPTIONS_MAPPER = (widget, res) -> { ofNullable(res.get(WIDGET.WIDGET_OPTIONS, String.class)).ifPresent(wo -> { From 126ee2ed792e3d7a484b053be2a88bf9185cf505 Mon Sep 17 00:00:00 2001 From: PeeAyBee Date: Mon, 27 Nov 2023 17:59:28 +0300 Subject: [PATCH 2/3] EPMRPP-87433 || Add missed attachment file name fetch (#957) --- .../ta/reportportal/commons/querygen/FilterTarget.java | 3 ++- .../reportportal/dao/AttachmentRepositoryCustomImpl.java | 9 ++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/epam/ta/reportportal/commons/querygen/FilterTarget.java b/src/main/java/com/epam/ta/reportportal/commons/querygen/FilterTarget.java index d9f820c62..935f1f703 100644 --- a/src/main/java/com/epam/ta/reportportal/commons/querygen/FilterTarget.java +++ b/src/main/java/com/epam/ta/reportportal/commons/querygen/FilterTarget.java @@ -1080,7 +1080,8 @@ protected Collection selectFields() { ATTACHMENT.FILE_SIZE, ATTACHMENT.PROJECT_ID, ATTACHMENT.LAUNCH_ID, - ATTACHMENT.ITEM_ID + ATTACHMENT.ITEM_ID, + ATTACHMENT.FILE_NAME ); } diff --git a/src/main/java/com/epam/ta/reportportal/dao/AttachmentRepositoryCustomImpl.java b/src/main/java/com/epam/ta/reportportal/dao/AttachmentRepositoryCustomImpl.java index 2223e5309..8db42fa94 100644 --- a/src/main/java/com/epam/ta/reportportal/dao/AttachmentRepositoryCustomImpl.java +++ b/src/main/java/com/epam/ta/reportportal/dao/AttachmentRepositoryCustomImpl.java @@ -150,7 +150,8 @@ public List findByItemIdsAndLogTimeBefore(Collection itemIds, ATTACHMENT.FILE_SIZE, ATTACHMENT.ITEM_ID, ATTACHMENT.LAUNCH_ID, - ATTACHMENT.PROJECT_ID + ATTACHMENT.PROJECT_ID, + ATTACHMENT.FILE_NAME ) .from(ATTACHMENT) .join(LOG) @@ -173,7 +174,8 @@ public List findByLaunchIdsAndLogTimeBefore(Collection launchI ATTACHMENT.FILE_SIZE, ATTACHMENT.ITEM_ID, ATTACHMENT.LAUNCH_ID, - ATTACHMENT.PROJECT_ID + ATTACHMENT.PROJECT_ID, + ATTACHMENT.FILE_NAME ) .from(ATTACHMENT) .join(LOG) @@ -194,7 +196,8 @@ public List findByProjectIdsAndLogTimeBefore(Long projectId, LocalDa ATTACHMENT.FILE_SIZE, ATTACHMENT.ITEM_ID, ATTACHMENT.LAUNCH_ID, - ATTACHMENT.PROJECT_ID + ATTACHMENT.PROJECT_ID, + ATTACHMENT.FILE_NAME ) .from(ATTACHMENT) .join(LOG) From 516ace351e02a0bbf5c192c106a97144552e1d6d Mon Sep 17 00:00:00 2001 From: Ivan Kustau <86599591+IvanKustau@users.noreply.github.com> Date: Thu, 14 Dec 2023 15:15:49 +0300 Subject: [PATCH 3/3] EPMRPP-88755 || Add prefix and postfix for filesystem (#963) * Add ability to use prefix and postfix in filesystem * Remove redundant code * Remove prefix and postfix for plugins bucket * Fix tests * Fix CommonDataStoreServiceTest * Test commit * Test commit * Change logback-test.xml * Change asserts * Make tests OS independent * Fix failed tests --- .../config/DataStoreConfiguration.java | 8 +++- .../filesystem/LocalDataStore.java | 41 ++++++++++++------- .../distributed/s3/S3DataStore.java | 2 +- .../impl/AttachmentDataStoreServiceTest.java | 33 ++++++++++----- .../impl/CommonDataStoreServiceTest.java | 38 ++++++++++++----- .../binary/impl/UserDataStoreServiceTest.java | 30 ++++++++++---- .../filesystem/LocalDataStoreTest.java | 29 +++++++++---- .../distributed/s3/S3DataStoreTest.java | 32 +++++++++------ src/test/resources/logback-test.xml | 5 +-- .../resources/test-application.properties | 3 ++ 10 files changed, 150 insertions(+), 71 deletions(-) diff --git a/src/main/java/com/epam/ta/reportportal/config/DataStoreConfiguration.java b/src/main/java/com/epam/ta/reportportal/config/DataStoreConfiguration.java index fe0ad3293..e4b2d1203 100644 --- a/src/main/java/com/epam/ta/reportportal/config/DataStoreConfiguration.java +++ b/src/main/java/com/epam/ta/reportportal/config/DataStoreConfiguration.java @@ -155,8 +155,12 @@ public BlobStore filesystemBlobStore( @Bean @ConditionalOnProperty(name = "datastore.type", havingValue = "filesystem") public DataStore localDataStore(@Autowired BlobStore blobStore, - FeatureFlagHandler featureFlagHandler) { - return new LocalDataStore(blobStore, featureFlagHandler); + FeatureFlagHandler featureFlagHandler, + @Value("${datastore.bucketPrefix}") String bucketPrefix, + @Value("${datastore.bucketPostfix}") String bucketPostfix, + @Value("${datastore.defaultBucketName}") String defaultBucketName) { + return new LocalDataStore( + blobStore, featureFlagHandler, bucketPrefix, bucketPostfix, defaultBucketName); } /** diff --git a/src/main/java/com/epam/ta/reportportal/filesystem/LocalDataStore.java b/src/main/java/com/epam/ta/reportportal/filesystem/LocalDataStore.java index 0a3617022..5b2ea84a0 100644 --- a/src/main/java/com/epam/ta/reportportal/filesystem/LocalDataStore.java +++ b/src/main/java/com/epam/ta/reportportal/filesystem/LocalDataStore.java @@ -26,6 +26,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; +import java.util.Objects; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.domain.Blob; import org.slf4j.Logger; @@ -41,13 +42,19 @@ public class LocalDataStore implements DataStore { private final FeatureFlagHandler featureFlagHandler; - private static final String SINGLE_BUCKET_NAME = "store"; + private final String bucketPrefix; - private static final String PLUGINS = "plugins"; + private final String bucketPostfix; - public LocalDataStore(BlobStore blobStore, FeatureFlagHandler featureFlagHandler) { + private final String defaultBucketName; + + public LocalDataStore(BlobStore blobStore, FeatureFlagHandler featureFlagHandler, + String bucketPrefix, String bucketPostfix, String defaultBucketName) { this.blobStore = blobStore; this.featureFlagHandler = featureFlagHandler; + this.bucketPrefix = bucketPrefix; + this.bucketPostfix = Objects.requireNonNullElse(bucketPostfix, ""); + this.defaultBucketName = defaultBucketName; } @Override @@ -117,9 +124,9 @@ public void delete(String filePath) { @Override public void deleteAll(List filePaths, String bucketName) { if (!featureFlagHandler.isEnabled(FeatureFlag.SINGLE_BUCKET)) { - blobStore.removeBlobs(bucketName, filePaths); + blobStore.removeBlobs(bucketPrefix + bucketName + bucketPostfix, filePaths); } else { - blobStore.removeBlobs(SINGLE_BUCKET_NAME, filePaths); + blobStore.removeBlobs(bucketName, filePaths); } } @@ -129,19 +136,23 @@ public void deleteContainer(String bucketName) { } private StoredFile getStoredFile(String filePath) { - Path targetPath = Paths.get(filePath); if (featureFlagHandler.isEnabled(FeatureFlag.SINGLE_BUCKET)) { - return new StoredFile(SINGLE_BUCKET_NAME, filePath); + return new StoredFile(defaultBucketName, filePath); + } + Path targetPath = Paths.get(filePath); + int nameCount = targetPath.getNameCount(); + String bucketName; + if (nameCount > 1) { + bucketName = bucketPrefix + retrievePath(targetPath, 0, 1) + bucketPostfix; + return new StoredFile(bucketName, retrievePath(targetPath, 1, nameCount)); } else { - int nameCount = targetPath.getNameCount(); - if (nameCount > 1) { - String bucketName = targetPath.getName(0).toString(); - String newFilePath = targetPath.subpath(1, nameCount).toString(); - return new StoredFile(bucketName, newFilePath); - } else { - return new StoredFile(PLUGINS, filePath); - } + bucketName = defaultBucketName; + return new StoredFile(bucketName, retrievePath(targetPath, 0, 1)); } } + + private String retrievePath(Path path, int beginIndex, int endIndex) { + return String.valueOf(path.subpath(beginIndex, endIndex)); + } } diff --git a/src/main/java/com/epam/ta/reportportal/filesystem/distributed/s3/S3DataStore.java b/src/main/java/com/epam/ta/reportportal/filesystem/distributed/s3/S3DataStore.java index 17ab82a47..159197246 100644 --- a/src/main/java/com/epam/ta/reportportal/filesystem/distributed/s3/S3DataStore.java +++ b/src/main/java/com/epam/ta/reportportal/filesystem/distributed/s3/S3DataStore.java @@ -175,7 +175,7 @@ private StoredFile getStoredFile(String filePath) { bucketName = bucketPrefix + retrievePath(targetPath, 0, 1) + bucketPostfix; return new StoredFile(bucketName, retrievePath(targetPath, 1, nameCount)); } else { - bucketName = bucketPrefix + defaultBucketName + bucketPostfix; + bucketName = defaultBucketName; return new StoredFile(bucketName, retrievePath(targetPath, 0, 1)); } } diff --git a/src/test/java/com/epam/ta/reportportal/binary/impl/AttachmentDataStoreServiceTest.java b/src/test/java/com/epam/ta/reportportal/binary/impl/AttachmentDataStoreServiceTest.java index ae1ef69b6..52f46af89 100644 --- a/src/test/java/com/epam/ta/reportportal/binary/impl/AttachmentDataStoreServiceTest.java +++ b/src/test/java/com/epam/ta/reportportal/binary/impl/AttachmentDataStoreServiceTest.java @@ -45,6 +45,12 @@ class AttachmentDataStoreServiceTest extends BaseTest { @Value("${datastore.path:/data/store}") private String storageRootPath; + @Value("${datastore.bucketPrefix:prj-}") + private String bucketPrefix; + + @Value("${datastore.bucketPostfix:}") + private String bucketPostfix; + private static final String BUCKET_NAME = "bucket"; private static Random random = new Random(); @@ -53,6 +59,8 @@ class AttachmentDataStoreServiceTest extends BaseTest { void saveLoadAndDeleteTest() throws IOException { InputStream inputStream = new ClassPathResource("meh.jpg").getInputStream(); + String bucketPath = bucketPrefix + BUCKET_NAME + bucketPostfix; + String fileId = attachmentDataStoreService.save(BUCKET_NAME + "/" + random.nextLong() + "meh.jpg", inputStream @@ -62,8 +70,9 @@ void saveLoadAndDeleteTest() throws IOException { assertTrue(loadedData.isPresent()); try (InputStream ignored = loadedData.get()) { - assertTrue(Files.exists( - Paths.get(storageRootPath, attachmentDataStoreService.dataEncoder.decode(fileId)))); + String decodedPath = attachmentDataStoreService.dataEncoder.decode(fileId); + decodedPath = decodedPath.replace(BUCKET_NAME, bucketPath); + assertTrue(Files.exists(Paths.get(storageRootPath, decodedPath))); } attachmentDataStoreService.delete(fileId); @@ -71,23 +80,26 @@ void saveLoadAndDeleteTest() throws IOException { ReportPortalException exception = assertThrows(ReportPortalException.class, () -> attachmentDataStoreService.load(fileId)); assertEquals("Unable to load binary data by id 'Unable to find file'", exception.getMessage()); - assertFalse(Files.exists( - Paths.get(storageRootPath, attachmentDataStoreService.dataEncoder.decode(fileId)))); + String decodedPath = attachmentDataStoreService.dataEncoder.decode(fileId); + decodedPath = decodedPath.replace(BUCKET_NAME, bucketPath); + assertFalse(Files.exists(Paths.get(storageRootPath, decodedPath))); } @Test void saveLoadAndDeleteThumbnailTest() throws IOException { try (InputStream inputStream = new ClassPathResource("meh.jpg").getInputStream()) { + String bucketPath = bucketPrefix + BUCKET_NAME + bucketPostfix; + String thumbnailId = attachmentDataStoreService.saveThumbnail( BUCKET_NAME + "/" + random.nextLong() + "thumbnail.jpg", inputStream); Optional loadedData = attachmentDataStoreService.load(thumbnailId); assertTrue(loadedData.isPresent()); - try (InputStream is = loadedData.get()) { - assertTrue(Files.exists(Paths.get(storageRootPath, - attachmentDataStoreService.dataEncoder.decode(thumbnailId) - ))); + try (InputStream ignored = loadedData.get()) { + String decodedPath = attachmentDataStoreService.dataEncoder.decode(thumbnailId); + decodedPath = decodedPath.replace(BUCKET_NAME, bucketPath); + assertTrue(Files.exists(Paths.get(storageRootPath, decodedPath))); } attachmentDataStoreService.delete(thumbnailId); @@ -97,8 +109,9 @@ void saveLoadAndDeleteThumbnailTest() throws IOException { ); assertEquals( "Unable to load binary data by id 'Unable to find file'", exception.getMessage()); - assertFalse(Files.exists( - Paths.get(storageRootPath, attachmentDataStoreService.dataEncoder.decode(thumbnailId)))); + String decodedPath = attachmentDataStoreService.dataEncoder.decode(thumbnailId); + decodedPath = decodedPath.replace(BUCKET_NAME, bucketPath); + assertFalse(Files.exists(Paths.get(storageRootPath, decodedPath))); } } } \ No newline at end of file diff --git a/src/test/java/com/epam/ta/reportportal/binary/impl/CommonDataStoreServiceTest.java b/src/test/java/com/epam/ta/reportportal/binary/impl/CommonDataStoreServiceTest.java index 050db9a70..a6cafb7db 100644 --- a/src/test/java/com/epam/ta/reportportal/binary/impl/CommonDataStoreServiceTest.java +++ b/src/test/java/com/epam/ta/reportportal/binary/impl/CommonDataStoreServiceTest.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.Optional; import java.util.Random; @@ -35,6 +36,8 @@ import org.apache.commons.fileupload.disk.DiskFileItem; import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; @@ -56,28 +59,43 @@ class CommonDataStoreServiceTest extends BaseTest { @Value("${datastore.path:/data/store}") private String storageRootPath; + @Value("${datastore.bucketPrefix:prj-}") + private String bucketPrefix; + + @Value("${datastore.bucketPostfix:}") + private String bucketPostfix; private static final String BUCKET_NAME = "bucket"; + private String getModifiedPath(String originalPath) { + String bucketPath = bucketPrefix + BUCKET_NAME + bucketPostfix; + return originalPath.replace(BUCKET_NAME, bucketPath); + } + @Test void saveTest() throws IOException { CommonsMultipartFile multipartFile = getMultipartFile("meh.jpg"); - String fileId = dataStoreService.save(BUCKET_NAME + "/" + multipartFile.getOriginalFilename(), - multipartFile.getInputStream() - ); + String fileId = + dataStoreService.save(BUCKET_NAME + File.separator + multipartFile.getOriginalFilename(), + multipartFile.getInputStream() + ); assertNotNull(fileId); - assertTrue(Files.exists(Paths.get(storageRootPath, dataEncoder.decode(fileId)))); + String decodedPath = getModifiedPath(dataEncoder.decode(fileId)); + Path filePath = Paths.get(storageRootPath, decodedPath); + assertTrue(filePath.toFile().exists(), "File " + filePath + " does not exist"); dataStoreService.delete(fileId); } @Test void saveThumbnailTest() throws IOException { CommonsMultipartFile multipartFile = getMultipartFile("meh.jpg"); - String fileId = - dataStoreService.saveThumbnail(BUCKET_NAME + "/" + multipartFile.getOriginalFilename(), - multipartFile.getInputStream() - ); + String fileId = dataStoreService.saveThumbnail( + BUCKET_NAME + File.separator + multipartFile.getOriginalFilename(), + multipartFile.getInputStream() + ); assertNotNull(fileId); - assertTrue(Files.exists(Paths.get(storageRootPath, dataEncoder.decode(fileId)))); + String decodedPath = getModifiedPath(dataEncoder.decode(fileId)); + Path filePath = Paths.get(storageRootPath, decodedPath); + assertTrue(filePath.toFile().exists(), "File " + filePath + " does not exist"); dataStoreService.delete(fileId); } @@ -106,7 +124,7 @@ void saveAndDeleteTest() throws IOException { dataStoreService.delete(fileId); - assertFalse(Files.exists(Paths.get(dataEncoder.decode(fileId)))); + assertFalse(Files.exists(Paths.get(dataEncoder.decode(getModifiedPath(fileId))))); } public static CommonsMultipartFile getMultipartFile(String path) throws IOException { diff --git a/src/test/java/com/epam/ta/reportportal/binary/impl/UserDataStoreServiceTest.java b/src/test/java/com/epam/ta/reportportal/binary/impl/UserDataStoreServiceTest.java index fd634c346..141678b78 100644 --- a/src/test/java/com/epam/ta/reportportal/binary/impl/UserDataStoreServiceTest.java +++ b/src/test/java/com/epam/ta/reportportal/binary/impl/UserDataStoreServiceTest.java @@ -45,6 +45,12 @@ class UserDataStoreServiceTest extends BaseTest { @Value("${datastore.path:/data/store}") private String storageRootPath; + @Value("${datastore.bucketPrefix:prj-}") + private String bucketPrefix; + + @Value("${datastore.bucketPostfix:}") + private String bucketPostfix; + private static final String BUCKET_NAME = "bucket"; private static Random random = new Random(); @@ -53,6 +59,8 @@ class UserDataStoreServiceTest extends BaseTest { void saveLoadAndDeleteTest() throws IOException { InputStream inputStream = new ClassPathResource("meh.jpg").getInputStream(); + String bucketPath = bucketPrefix + BUCKET_NAME + bucketPostfix; + String fileId = userDataStoreService.save(BUCKET_NAME + "/" + random.nextLong() + "meh.jpg", inputStream); @@ -60,8 +68,9 @@ void saveLoadAndDeleteTest() throws IOException { assertTrue(loadedData.isPresent()); try (InputStream ignored = loadedData.get()) { - assertTrue(Files.exists( - Paths.get(storageRootPath, userDataStoreService.dataEncoder.decode(fileId)))); + String decodedPath = userDataStoreService.dataEncoder.decode(fileId); + decodedPath = decodedPath.replace(BUCKET_NAME, bucketPath); + assertTrue(Files.exists(Paths.get(storageRootPath, decodedPath))); } userDataStoreService.delete(fileId); @@ -69,14 +78,17 @@ void saveLoadAndDeleteTest() throws IOException { ReportPortalException exception = assertThrows(ReportPortalException.class, () -> userDataStoreService.load(fileId)); assertEquals("Unable to load binary data by id 'Unable to find file'", exception.getMessage()); - assertFalse( - Files.exists(Paths.get(storageRootPath, userDataStoreService.dataEncoder.decode(fileId)))); + String decodedPath = userDataStoreService.dataEncoder.decode(fileId); + decodedPath = decodedPath.replace(BUCKET_NAME, bucketPath); + assertFalse(Files.exists(Paths.get(storageRootPath, decodedPath))); } @Test void saveLoadAndDeleteThumbnailTest() throws IOException { InputStream inputStream = new ClassPathResource("meh.jpg").getInputStream(); + String bucketPath = bucketPrefix + BUCKET_NAME + bucketPostfix; + String thumbnailId = userDataStoreService.saveThumbnail(BUCKET_NAME + "/" + random.nextLong() + "thmbnail.jpg", inputStream @@ -86,8 +98,9 @@ void saveLoadAndDeleteThumbnailTest() throws IOException { assertTrue(loadedData.isPresent()); try (InputStream ignored = loadedData.get()) { - assertTrue(Files.exists( - Paths.get(storageRootPath, userDataStoreService.dataEncoder.decode(thumbnailId)))); + String decodedPath = userDataStoreService.dataEncoder.decode(thumbnailId); + decodedPath = decodedPath.replace(BUCKET_NAME, bucketPath); + assertTrue(Files.exists(Paths.get(storageRootPath, decodedPath))); } userDataStoreService.delete(thumbnailId); @@ -95,7 +108,8 @@ void saveLoadAndDeleteThumbnailTest() throws IOException { ReportPortalException exception = assertThrows(ReportPortalException.class, () -> userDataStoreService.load(thumbnailId)); assertEquals("Unable to load binary data by id 'Unable to find file'", exception.getMessage()); - assertFalse(Files.exists( - Paths.get(storageRootPath, userDataStoreService.dataEncoder.decode(thumbnailId)))); + String decodedPath = userDataStoreService.dataEncoder.decode(thumbnailId); + decodedPath = decodedPath.replace(BUCKET_NAME, bucketPath); + assertFalse(Files.exists(Paths.get(storageRootPath, decodedPath))); } } \ No newline at end of file diff --git a/src/test/java/com/epam/ta/reportportal/filesystem/LocalDataStoreTest.java b/src/test/java/com/epam/ta/reportportal/filesystem/LocalDataStoreTest.java index b38ea634a..d11771157 100644 --- a/src/test/java/com/epam/ta/reportportal/filesystem/LocalDataStoreTest.java +++ b/src/test/java/com/epam/ta/reportportal/filesystem/LocalDataStoreTest.java @@ -45,12 +45,16 @@ class LocalDataStoreTest { private static final int ZERO = 0; - private static final String SINGLE_BUCKET_NAME = "store"; - private static final String FILE_PATH = "someFile.txt"; private static final String MULTI_BUCKET_NAME = "multiBucket"; + private static final String BUCKET_PREFIX = "prj-"; + + private static final String BUCKET_POSTFIX = "-tiest"; + + private static final String DEFAULT_BUCKET_NAME = "rp-bucket"; + private static final String MULTI_FILE_PATH = MULTI_BUCKET_NAME + "/" + FILE_PATH; @BeforeEach @@ -60,7 +64,10 @@ void setUp() { featureFlagHandler = Mockito.mock(FeatureFlagHandler.class); - localDataStore = new LocalDataStore(blobStore, featureFlagHandler); + localDataStore = + new LocalDataStore(blobStore, featureFlagHandler, BUCKET_PREFIX, BUCKET_POSTFIX, + DEFAULT_BUCKET_NAME + ); } @Test @@ -82,7 +89,7 @@ void whenSave_andSingleBucketIsEnabled_thenSaveToSingleBucket() throws Exception localDataStore.save(FILE_PATH, inputStream); - verify(blobStore, times(1)).putBlob(SINGLE_BUCKET_NAME, blobMock); + verify(blobStore, times(1)).putBlob(DEFAULT_BUCKET_NAME, blobMock); } @Test @@ -95,7 +102,7 @@ void whenLoad_andSingleBucketIsEnabled_thenReturnFromSingleBucket() throws Excep when(mockBlob.getPayload()).thenReturn(mockPayload); when(featureFlagHandler.isEnabled(FeatureFlag.SINGLE_BUCKET)).thenReturn(true); - when(blobStore.getBlob(SINGLE_BUCKET_NAME, FILE_PATH)).thenReturn(mockBlob); + when(blobStore.getBlob(DEFAULT_BUCKET_NAME, FILE_PATH)).thenReturn(mockBlob); InputStream loaded = localDataStore.load(FILE_PATH); Assertions.assertEquals(inputStream, loaded); @@ -108,7 +115,7 @@ void whenDelete_andSingleBucketIsEnabled_thenDeleteFromSingleBucket() throws Exc localDataStore.delete(FILE_PATH); - verify(blobStore, times(1)).removeBlob(SINGLE_BUCKET_NAME, FILE_PATH); + verify(blobStore, times(1)).removeBlob(DEFAULT_BUCKET_NAME, FILE_PATH); } @Test @@ -130,7 +137,8 @@ void whenSave_andSingleBucketIsDisabled_andBucketInName_thenSaveToThisBucket() t localDataStore.save(MULTI_FILE_PATH, inputStream); - verify(blobStore, times(1)).putBlob(MULTI_BUCKET_NAME, blobMock); + verify(blobStore, times(1)).putBlob( + BUCKET_PREFIX + MULTI_BUCKET_NAME + BUCKET_POSTFIX, blobMock); } @Test @@ -144,7 +152,9 @@ void whenLoad_andSingleBucketIsDisabled_andBucketInName_thenReturnFromThisBucket when(mockBlob.getPayload()).thenReturn(mockPayload); when(featureFlagHandler.isEnabled(FeatureFlag.SINGLE_BUCKET)).thenReturn(false); - when(blobStore.getBlob(MULTI_BUCKET_NAME, FILE_PATH)).thenReturn(mockBlob); + when(blobStore.getBlob(BUCKET_PREFIX + MULTI_BUCKET_NAME + BUCKET_POSTFIX, + FILE_PATH + )).thenReturn(mockBlob); InputStream loaded = localDataStore.load(MULTI_FILE_PATH); Assertions.assertEquals(inputStream, loaded); @@ -158,6 +168,7 @@ void whenDelete_andSingleBucketIsDisabled_andBucketInName_thenReturnFromThisBuck localDataStore.delete(MULTI_FILE_PATH); - verify(blobStore, times(1)).removeBlob(MULTI_BUCKET_NAME, FILE_PATH); + verify(blobStore, times(1)).removeBlob( + BUCKET_PREFIX + MULTI_BUCKET_NAME + BUCKET_POSTFIX, FILE_PATH); } } diff --git a/src/test/java/com/epam/ta/reportportal/filesystem/distributed/s3/S3DataStoreTest.java b/src/test/java/com/epam/ta/reportportal/filesystem/distributed/s3/S3DataStoreTest.java index 23f4cdc3d..e58278f19 100644 --- a/src/test/java/com/epam/ta/reportportal/filesystem/distributed/s3/S3DataStoreTest.java +++ b/src/test/java/com/epam/ta/reportportal/filesystem/distributed/s3/S3DataStoreTest.java @@ -37,7 +37,7 @@ */ class S3DataStoreTest { - private static final String FILE_PATH = "someFile"; + private static final String FILE_NAME = "someFile"; private static final String BUCKET_PREFIX = "prj-"; private static final String BUCKET_POSTFIX = "-postfix"; private static final String DEFAULT_BUCKET_NAME = "rp-bucket"; @@ -51,7 +51,8 @@ class S3DataStoreTest { private final S3DataStore s3DataStore = new S3DataStore(blobStore, BUCKET_PREFIX, BUCKET_POSTFIX, DEFAULT_BUCKET_NAME, REGION, - featureFlagHandler); + featureFlagHandler + ); @Test void save() throws Exception { @@ -61,21 +62,23 @@ void save() throws Exception { mock(BlobBuilder.PayloadBlobBuilder.class); Blob blobMock = mock(Blob.class); + String filePath = DEFAULT_BUCKET_NAME + "/" + FILE_NAME; + when(inputStream.available()).thenReturn(ZERO); - when(payloadBlobBuilderMock.contentDisposition(FILE_PATH)).thenReturn(payloadBlobBuilderMock); + when(payloadBlobBuilderMock.contentDisposition(FILE_NAME)).thenReturn(payloadBlobBuilderMock); when(payloadBlobBuilderMock.contentLength(ZERO)).thenReturn(payloadBlobBuilderMock); when(payloadBlobBuilderMock.build()).thenReturn(blobMock); when(blobBuilderMock.payload(inputStream)).thenReturn(payloadBlobBuilderMock); when(blobStore.containerExists(any(String.class))).thenReturn(true); - when(blobStore.blobBuilder(FILE_PATH)).thenReturn(blobBuilderMock); + when(blobStore.blobBuilder(FILE_NAME)).thenReturn(blobBuilderMock); when(featureFlagHandler.isEnabled(FeatureFlag.SINGLE_BUCKET)).thenReturn(false); - s3DataStore.save(FILE_PATH, inputStream); + s3DataStore.save(filePath, inputStream); - verify(blobStore, times(1)) - .putBlob(BUCKET_PREFIX + DEFAULT_BUCKET_NAME + BUCKET_POSTFIX, blobMock); + verify(blobStore, times(1)).putBlob( + BUCKET_PREFIX + DEFAULT_BUCKET_NAME + BUCKET_POSTFIX, blobMock); } @Test @@ -84,12 +87,15 @@ void load() throws Exception { Blob mockBlob = mock(Blob.class); Payload mockPayload = mock(Payload.class); + String filePath = DEFAULT_BUCKET_NAME + "/" + FILE_NAME; + when(mockPayload.openStream()).thenReturn(inputStream); when(mockBlob.getPayload()).thenReturn(mockPayload); when(blobStore.getBlob(BUCKET_PREFIX + DEFAULT_BUCKET_NAME + BUCKET_POSTFIX, - FILE_PATH)).thenReturn(mockBlob); - InputStream loaded = s3DataStore.load(FILE_PATH); + FILE_NAME + )).thenReturn(mockBlob); + InputStream loaded = s3DataStore.load(filePath); Assertions.assertEquals(inputStream, loaded); } @@ -97,9 +103,11 @@ void load() throws Exception { @Test void delete() throws Exception { - s3DataStore.delete(FILE_PATH); + String filePath = DEFAULT_BUCKET_NAME + "/" + FILE_NAME; + + s3DataStore.delete(filePath); - verify(blobStore, times(1)) - .removeBlob(BUCKET_PREFIX + DEFAULT_BUCKET_NAME + BUCKET_POSTFIX, FILE_PATH); + verify(blobStore, times(1)).removeBlob( + BUCKET_PREFIX + DEFAULT_BUCKET_NAME + BUCKET_POSTFIX, FILE_NAME); } } \ No newline at end of file diff --git a/src/test/resources/logback-test.xml b/src/test/resources/logback-test.xml index fa7189aa9..c4d877240 100644 --- a/src/test/resources/logback-test.xml +++ b/src/test/resources/logback-test.xml @@ -8,10 +8,7 @@ - - - - + \ No newline at end of file diff --git a/src/test/resources/test-application.properties b/src/test/resources/test-application.properties index 730f4eb11..74875f2bc 100644 --- a/src/test/resources/test-application.properties +++ b/src/test/resources/test-application.properties @@ -20,6 +20,9 @@ rp.binarystore.path=${java.io.tmpdir}/reportportal/datastore rp.feature.flags= datastore.path=${rp.binarystore.path:/data/storage} +datastore.bucketPrefix=prj- +datastore.defaultBucketName=rp-bucket +datastore.bucketPostfix= datastore.seaweed.master.host=${rp.binarystore.master.host:localhost} datastore.seaweed.master.port=${rp.binarystore.master.port:9333} datastore.s3.endpoint=${rp.binarystore.s3.endpoint:https://play.min.io}