From 08e2ee4226ab2489442954094a7ac4bbcb633afb Mon Sep 17 00:00:00 2001 From: Ivan_Kustau Date: Mon, 11 Sep 2023 18:16:58 +0300 Subject: [PATCH 1/9] EPMRPP-80519 || Add methods for removing buckets and files --- .../dao/AttachmentRepository.java | 24 ++-- .../ta/reportportal/filesystem/DataStore.java | 11 +- .../filesystem/LocalDataStore.java | 105 +++++++++++------- .../distributed/s3/S3DataStore.java | 19 ++++ 4 files changed, 103 insertions(+), 56 deletions(-) diff --git a/src/main/java/com/epam/ta/reportportal/dao/AttachmentRepository.java b/src/main/java/com/epam/ta/reportportal/dao/AttachmentRepository.java index fd210eb16..614d97a22 100644 --- a/src/main/java/com/epam/ta/reportportal/dao/AttachmentRepository.java +++ b/src/main/java/com/epam/ta/reportportal/dao/AttachmentRepository.java @@ -17,25 +17,27 @@ package com.epam.ta.reportportal.dao; import com.epam.ta.reportportal.entity.attachment.Attachment; -import org.springframework.data.jpa.repository.Modifying; -import org.springframework.data.jpa.repository.Query; -import org.springframework.data.repository.query.Param; - import java.util.Collection; import java.util.List; import java.util.Optional; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; /** * @author Ivan Budayeu */ -public interface AttachmentRepository extends ReportPortalRepository, AttachmentRepositoryCustom { +public interface AttachmentRepository + extends ReportPortalRepository, AttachmentRepositoryCustom { + + Optional findByFileId(String fileId); - Optional findByFileId(String fileId); + List findAllByLaunchIdIn(Collection launchIds); - List findAllByLaunchIdIn(Collection launchIds); + void deleteAllByProjectId(Long projectId); - @Modifying - @Query(value = "UPDATE attachment SET launch_id = :newLaunchId WHERE project_id = :projectId AND launch_id = :currentLaunchId", nativeQuery = true) - void updateLaunchIdByProjectIdAndLaunchId(@Param("projectId") Long projectId, @Param("currentLaunchId") Long currentLaunchId, - @Param("newLaunchId") Long newLaunchId); + @Modifying + @Query(value = "UPDATE attachment SET launch_id = :newLaunchId WHERE project_id = :projectId AND launch_id = :currentLaunchId", nativeQuery = true) + void updateLaunchIdByProjectIdAndLaunchId(@Param("projectId") Long projectId, + @Param("currentLaunchId") Long currentLaunchId, @Param("newLaunchId") Long newLaunchId); } diff --git a/src/main/java/com/epam/ta/reportportal/filesystem/DataStore.java b/src/main/java/com/epam/ta/reportportal/filesystem/DataStore.java index 58203495a..3d4b88ee0 100644 --- a/src/main/java/com/epam/ta/reportportal/filesystem/DataStore.java +++ b/src/main/java/com/epam/ta/reportportal/filesystem/DataStore.java @@ -17,15 +17,20 @@ package com.epam.ta.reportportal.filesystem; import java.io.InputStream; +import java.util.List; /** * @author Dzianis_Shybeka */ public interface DataStore { - String save(String fileName, InputStream inputStream); + String save(String fileName, InputStream inputStream); - InputStream load(String filePath); + InputStream load(String filePath); - void delete(String filePath); + void delete(String filePath); + + void deleteAll(List filePaths, String bucketName); + + void deleteContainer(String bucketName); } diff --git a/src/main/java/com/epam/ta/reportportal/filesystem/LocalDataStore.java b/src/main/java/com/epam/ta/reportportal/filesystem/LocalDataStore.java index ad74302c8..1276907e4 100644 --- a/src/main/java/com/epam/ta/reportportal/filesystem/LocalDataStore.java +++ b/src/main/java/com/epam/ta/reportportal/filesystem/LocalDataStore.java @@ -18,80 +18,101 @@ import com.epam.ta.reportportal.exception.ReportPortalException; import com.epam.ta.reportportal.ws.model.ErrorType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; +import java.util.List; import java.util.Objects; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * @author Dzianis_Shybeka */ public class LocalDataStore implements DataStore { - private static final Logger logger = LoggerFactory.getLogger(LocalDataStore.class); + private static final Logger logger = LoggerFactory.getLogger(LocalDataStore.class); + + private final String storageRootPath; + + public LocalDataStore(String storageRootPath) { + this.storageRootPath = storageRootPath; + } + + @Override + public String save(String filePath, InputStream inputStream) { + + try { + + Path targetPath = Paths.get(storageRootPath, filePath); + Path targetDirectory = targetPath.getParent(); + + if (Objects.nonNull(targetDirectory) && !Files.isDirectory(targetDirectory)) { + Files.createDirectories(targetDirectory); + } - private final String storageRootPath; + logger.debug("Saving to: {} ", targetPath.toAbsolutePath()); - public LocalDataStore(String storageRootPath) { - this.storageRootPath = storageRootPath; - } + Files.copy(inputStream, targetPath, StandardCopyOption.REPLACE_EXISTING); - @Override - public String save(String filePath, InputStream inputStream) { + return filePath; + } catch (IOException e) { - try { + logger.error("Unable to save log file '{}'", filePath, e); - Path targetPath = Paths.get(storageRootPath, filePath); - Path targetDirectory = targetPath.getParent(); + throw new ReportPortalException(ErrorType.INCORRECT_REQUEST, "Unable to save log file"); + } + } - if (Objects.nonNull(targetDirectory) && !Files.isDirectory(targetDirectory)) { - Files.createDirectories(targetDirectory); - } + @Override + public InputStream load(String filePath) { - logger.debug("Saving to: {} ", targetPath.toAbsolutePath()); + try { - Files.copy(inputStream, targetPath, StandardCopyOption.REPLACE_EXISTING); + return Files.newInputStream(Paths.get(storageRootPath, filePath)); + } catch (IOException e) { - return filePath; - } catch (IOException e) { + logger.error("Unable to find file '{}'", filePath, e); - logger.error("Unable to save log file '{}'", filePath, e); + throw new ReportPortalException(ErrorType.UNABLE_TO_LOAD_BINARY_DATA, "Unable to find file"); + } + } - throw new ReportPortalException(ErrorType.INCORRECT_REQUEST, "Unable to save log file"); - } - } + @Override + public void delete(String filePath) { - @Override - public InputStream load(String filePath) { + try { - try { + Files.deleteIfExists(Paths.get(storageRootPath, filePath)); + } catch (IOException e) { - return Files.newInputStream(Paths.get(storageRootPath, filePath)); - } catch (IOException e) { + logger.error("Unable to delete file '{}'", filePath, e); - logger.error("Unable to find file '{}'", filePath, e); + throw new ReportPortalException(ErrorType.INCORRECT_REQUEST, "Unable to delete file"); + } + } - throw new ReportPortalException(ErrorType.UNABLE_TO_LOAD_BINARY_DATA, "Unable to find file"); - } - } + @Override + public void deleteAll(List filePaths, String bucketName) { - @Override - public void delete(String filePath) { + for (String filePath : filePaths) { + delete(filePath); + } + } - try { + @Override + public void deleteContainer(String bucketName) { + try { - Files.deleteIfExists(Paths.get(storageRootPath, filePath)); - } catch (IOException e) { + Files.deleteIfExists(Paths.get(bucketName)); + } catch (IOException e) { - logger.error("Unable to delete file '{}'", filePath, e); + logger.error("Unable to delete bucket '{}'", bucketName, e); - throw new ReportPortalException(ErrorType.INCORRECT_REQUEST, "Unable to delete file"); - } - } + throw new ReportPortalException(ErrorType.INCORRECT_REQUEST, "Unable to delete file"); + } + } } diff --git a/src/main/java/com/epam/ta/reportportal/filesystem/distributed/s3/S3DataStore.java b/src/main/java/com/epam/ta/reportportal/filesystem/distributed/s3/S3DataStore.java index e9f7e2e49..1ac0d896f 100644 --- a/src/main/java/com/epam/ta/reportportal/filesystem/distributed/s3/S3DataStore.java +++ b/src/main/java/com/epam/ta/reportportal/filesystem/distributed/s3/S3DataStore.java @@ -25,6 +25,7 @@ import java.io.InputStream; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.List; import java.util.Objects; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -126,6 +127,24 @@ public void delete(String filePath) { } } + @Override + public void deleteAll(List filePaths, String bucketName) { + if (!featureFlagHandler.isEnabled(FeatureFlag.SINGLE_BUCKET)) { + blobStore.removeBlobs(bucketPrefix + bucketName + bucketPostfix, filePaths); + } else { + blobStore.removeBlobs(bucketName, filePaths); + } + } + + @Override + public void deleteContainer(String bucketName) { + if (!featureFlagHandler.isEnabled(FeatureFlag.SINGLE_BUCKET)) { + blobStore.deleteContainer(bucketPrefix + bucketName + bucketPostfix); + } else { + blobStore.deleteContainer(bucketName); + } + } + private S3File getS3File(String filePath) { if (featureFlagHandler.isEnabled(FeatureFlag.SINGLE_BUCKET)) { return new S3File(defaultBucketName, filePath); From a65d0b96addc6abf1cf14e69ea2951f64f17397d Mon Sep 17 00:00:00 2001 From: Ivan_Kustau Date: Tue, 12 Sep 2023 18:10:16 +0300 Subject: [PATCH 2/9] EPMRPP-80519 || Remove attachments by project id --- .../binary/AttachmentBinaryDataService.java | 16 ++++---- .../reportportal/binary/DataStoreService.java | 13 +++++-- .../impl/AttachmentBinaryDataServiceImpl.java | 38 ++++++++++++++----- .../binary/impl/CommonDataStoreService.java | 13 +++++++ .../dao/AttachmentRepository.java | 2 + .../distributed/s3/S3DataStore.java | 2 +- 6 files changed, 62 insertions(+), 22 deletions(-) diff --git a/src/main/java/com/epam/ta/reportportal/binary/AttachmentBinaryDataService.java b/src/main/java/com/epam/ta/reportportal/binary/AttachmentBinaryDataService.java index cec525d02..b4721620c 100644 --- a/src/main/java/com/epam/ta/reportportal/binary/AttachmentBinaryDataService.java +++ b/src/main/java/com/epam/ta/reportportal/binary/AttachmentBinaryDataService.java @@ -20,22 +20,24 @@ import com.epam.ta.reportportal.commons.ReportPortalUser; import com.epam.ta.reportportal.entity.attachment.AttachmentMetaInfo; import com.epam.ta.reportportal.entity.attachment.BinaryData; -import org.springframework.web.multipart.MultipartFile; - import java.util.Optional; +import org.springframework.web.multipart.MultipartFile; /** * @author Ihar Kahadouski */ public interface AttachmentBinaryDataService { - Optional saveAttachment(AttachmentMetaInfo attachmentMetaInfo, MultipartFile file); + Optional saveAttachment(AttachmentMetaInfo attachmentMetaInfo, + MultipartFile file); + + void saveFileAndAttachToLog(MultipartFile file, AttachmentMetaInfo attachmentMetaInfo); - void saveFileAndAttachToLog(MultipartFile file, AttachmentMetaInfo attachmentMetaInfo); + void attachToLog(BinaryDataMetaInfo binaryDataMetaInfo, AttachmentMetaInfo attachmentMetaInfo); - void attachToLog(BinaryDataMetaInfo binaryDataMetaInfo, AttachmentMetaInfo attachmentMetaInfo); + BinaryData load(Long fileId, ReportPortalUser.ProjectDetails projectDetails); - BinaryData load(Long fileId, ReportPortalUser.ProjectDetails projectDetails); + void delete(String fileId); - void delete(String fileId); + void deleteAllByProjectId(Long projectId); } diff --git a/src/main/java/com/epam/ta/reportportal/binary/DataStoreService.java b/src/main/java/com/epam/ta/reportportal/binary/DataStoreService.java index c98a2029f..0f80ba167 100644 --- a/src/main/java/com/epam/ta/reportportal/binary/DataStoreService.java +++ b/src/main/java/com/epam/ta/reportportal/binary/DataStoreService.java @@ -17,6 +17,7 @@ package com.epam.ta.reportportal.binary; import java.io.InputStream; +import java.util.List; import java.util.Optional; /** @@ -24,11 +25,15 @@ */ public interface DataStoreService { - String save(String fileName, InputStream data); + String save(String fileName, InputStream data); - String saveThumbnail(String fileName, InputStream data); + String saveThumbnail(String fileName, InputStream data); - void delete(String fileId); + void delete(String fileId); - Optional load(String fileId); + void deleteAll(List fileIds, String bucketName); + + void deleteContainer(String containerName); + + Optional load(String fileId); } diff --git a/src/main/java/com/epam/ta/reportportal/binary/impl/AttachmentBinaryDataServiceImpl.java b/src/main/java/com/epam/ta/reportportal/binary/impl/AttachmentBinaryDataServiceImpl.java index 2b8ffbfd4..b49b1a407 100644 --- a/src/main/java/com/epam/ta/reportportal/binary/impl/AttachmentBinaryDataServiceImpl.java +++ b/src/main/java/com/epam/ta/reportportal/binary/impl/AttachmentBinaryDataServiceImpl.java @@ -16,6 +16,12 @@ package com.epam.ta.reportportal.binary.impl; +import static com.epam.ta.reportportal.binary.impl.DataStoreUtils.PROJECT_PATH; +import static com.epam.ta.reportportal.binary.impl.DataStoreUtils.isContentTypePresent; +import static com.epam.ta.reportportal.binary.impl.DataStoreUtils.resolveExtension; +import static com.epam.ta.reportportal.commons.validation.BusinessRule.expect; +import static com.epam.ta.reportportal.commons.validation.Suppliers.formattedSupplier; + import com.epam.reportportal.commons.ContentTypeResolver; import com.epam.ta.reportportal.binary.AttachmentBinaryDataService; import com.epam.ta.reportportal.binary.CreateLogAttachmentService; @@ -31,6 +37,14 @@ import com.epam.ta.reportportal.filesystem.FilePathGenerator; import com.epam.ta.reportportal.util.FeatureFlagHandler; import com.epam.ta.reportportal.ws.model.ErrorType; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Paths; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,15 +54,6 @@ import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.commons.CommonsMultipartFile; -import java.io.*; -import java.nio.file.Paths; -import java.util.Optional; -import java.util.function.Predicate; - -import static com.epam.ta.reportportal.binary.impl.DataStoreUtils.*; -import static com.epam.ta.reportportal.commons.validation.BusinessRule.expect; -import static com.epam.ta.reportportal.commons.validation.Suppliers.formattedSupplier; - /** * @author Ihar Kahadouski */ @@ -179,7 +184,8 @@ public BinaryData load(Long fileId, ReportPortalUser.ProjectDetails projectDetai ErrorType.ACCESS_DENIED, formattedSupplier("You are not assigned to project '{}'", projectDetails.getProjectName()) ); - return new BinaryData(attachment.getFileName(), attachment.getContentType(), (long) data.available(), data); + return new BinaryData( + attachment.getFileName(), attachment.getContentType(), (long) data.available(), data); } catch (IOException e) { LOGGER.error("Unable to load binary data", e); throw new ReportPortalException( @@ -195,6 +201,18 @@ public void delete(String fileId) { } } + @Override + public void deleteAllByProjectId(Long projectId) { + if (featureFlagHandler.isEnabled(FeatureFlag.SINGLE_BUCKET)) { + dataStoreService.deleteAll( + attachmentRepository.findAllByProjectId(projectId).stream().map(Attachment::getFileId) + .collect(Collectors.toList()), projectId.toString()); + } else { + dataStoreService.deleteContainer(projectId.toString()); + } + attachmentRepository.deleteAllByProjectId(projectId); + } + private String resolveContentType(String contentType, ByteArrayOutputStream outputStream) throws IOException { if (isContentTypePresent(contentType)) { diff --git a/src/main/java/com/epam/ta/reportportal/binary/impl/CommonDataStoreService.java b/src/main/java/com/epam/ta/reportportal/binary/impl/CommonDataStoreService.java index 03c5ef21e..8bb4058ef 100644 --- a/src/main/java/com/epam/ta/reportportal/binary/impl/CommonDataStoreService.java +++ b/src/main/java/com/epam/ta/reportportal/binary/impl/CommonDataStoreService.java @@ -22,7 +22,9 @@ import com.epam.ta.reportportal.filesystem.DataEncoder; import com.epam.ta.reportportal.filesystem.DataStore; import java.io.InputStream; +import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; /** * @author Ihar Kahadouski @@ -51,6 +53,17 @@ public void delete(String fileId) { dataStore.delete(dataEncoder.decode(fileId)); } + @Override + public void deleteAll(List fileIds, String bucketName) { + dataStore.deleteAll( + fileIds.stream().map(dataEncoder::decode).collect(Collectors.toList()), bucketName); + } + + @Override + public void deleteContainer(String containerName) { + dataStore.deleteContainer(containerName); + } + @Override public Optional load(String fileId) { return ofNullable(dataStore.load(dataEncoder.decode(fileId))); diff --git a/src/main/java/com/epam/ta/reportportal/dao/AttachmentRepository.java b/src/main/java/com/epam/ta/reportportal/dao/AttachmentRepository.java index 614d97a22..b203c98c5 100644 --- a/src/main/java/com/epam/ta/reportportal/dao/AttachmentRepository.java +++ b/src/main/java/com/epam/ta/reportportal/dao/AttachmentRepository.java @@ -32,6 +32,8 @@ public interface AttachmentRepository Optional findByFileId(String fileId); + List findAllByProjectId(Long projectId); + List findAllByLaunchIdIn(Collection launchIds); void deleteAllByProjectId(Long projectId); diff --git a/src/main/java/com/epam/ta/reportportal/filesystem/distributed/s3/S3DataStore.java b/src/main/java/com/epam/ta/reportportal/filesystem/distributed/s3/S3DataStore.java index 1ac0d896f..abcad405b 100644 --- a/src/main/java/com/epam/ta/reportportal/filesystem/distributed/s3/S3DataStore.java +++ b/src/main/java/com/epam/ta/reportportal/filesystem/distributed/s3/S3DataStore.java @@ -132,7 +132,7 @@ public void deleteAll(List filePaths, String bucketName) { if (!featureFlagHandler.isEnabled(FeatureFlag.SINGLE_BUCKET)) { blobStore.removeBlobs(bucketPrefix + bucketName + bucketPostfix, filePaths); } else { - blobStore.removeBlobs(bucketName, filePaths); + blobStore.removeBlobs(defaultBucketName, filePaths); } } From 8e4b91cc929905442e27eec74bddadb70a3e58de Mon Sep 17 00:00:00 2001 From: Siarhei Hrabko <45555481+grabsefx@users.noreply.github.com> Date: Fri, 22 Sep 2023 13:15:14 +0300 Subject: [PATCH 3/9] EPMRPP-86363 exclude system attributes in launches table widget (#932) * EPMRPP-86363 exclude system attributes in launches table widget * EPMRPP-86363 fixed check-style comments --- .../dao/WidgetContentRepositoryImpl.java | 3 ++- .../dao/WidgetContentRepositoryTest.java | 25 +++++++++++++++++++ .../V001005__widget_content_init.sql | 4 +-- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/epam/ta/reportportal/dao/WidgetContentRepositoryImpl.java b/src/main/java/com/epam/ta/reportportal/dao/WidgetContentRepositoryImpl.java index 7290d5e73..3481ddabf 100644 --- a/src/main/java/com/epam/ta/reportportal/dao/WidgetContentRepositoryImpl.java +++ b/src/main/java/com/epam/ta/reportportal/dao/WidgetContentRepositoryImpl.java @@ -1227,7 +1227,8 @@ private SelectSeekStepN buildLaunchesTableQuery(Collection contentFields = buildLaunchesTableContentFields(); + + List launchStatisticsContents = widgetContentRepository.launchesTableStatistics( + filter, + contentFields, + sort, + 3 + ); + assertNotNull(launchStatisticsContents); + assertEquals(3, launchStatisticsContents.size()); + + launchStatisticsContents.forEach(content -> { + assertTrue(CollectionUtils.isNotEmpty(content.getAttributes())); + boolean isSystemAttributePresent = content.getAttributes() + .stream() + .anyMatch(attribute -> attribute.getValue().equals("true_system_attr")); + assertFalse(isSystemAttributePresent); + }); + } + @Test void activityStatistics() { Filter filter = buildDefaultActivityFilter(1L); diff --git a/src/test/resources/db/migration/V001005__widget_content_init.sql b/src/test/resources/db/migration/V001005__widget_content_init.sql index d840d13ba..e36ee9da1 100644 --- a/src/test/resources/db/migration/V001005__widget_content_init.sql +++ b/src/test/resources/db/migration/V001005__widget_content_init.sql @@ -1,8 +1,7 @@ create or replace function widget_content_init() RETURNS VOID as $$ -DECLARE - launch1 BIGINT; + DECLARE launch1 BIGINT; DECLARE launch2 BIGINT; DECLARE launch3 BIGINT; DECLARE launch4 BIGINT; @@ -98,6 +97,7 @@ BEGIN INSERT INTO item_attribute ("key", "value", item_id, launch_id, system) VALUES ('build', '1.3.2', null, launch2, false); INSERT INTO item_attribute ("key", "value", item_id, launch_id, system) VALUES ('build', '1.9.1', null, launch3, false); INSERT INTO item_attribute ("key", "value", item_id, launch_id, system) VALUES ('build', '3', null, launch4, false); + INSERT INTO item_attribute ("key", "value", item_id, launch_id, system) VALUES ('build', 'true_system_attr', null, launch1, true); INSERT INTO public.ticket (id, ticket_id, submitter, submit_date, bts_url, bts_project, url) From ad6bf2307afbb53a15540cbd25ca15fe9454c211 Mon Sep 17 00:00:00 2001 From: Pavel Bortnik Date: Thu, 12 Oct 2023 23:47:56 +0300 Subject: [PATCH 4/9] EPMRPP-80574 || Add possibility to provide encryption password by env --- .../config/EncryptConfiguration.java | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/epam/ta/reportportal/config/EncryptConfiguration.java b/src/main/java/com/epam/ta/reportportal/config/EncryptConfiguration.java index 36ff5207c..3fd1234ca 100644 --- a/src/main/java/com/epam/ta/reportportal/config/EncryptConfiguration.java +++ b/src/main/java/com/epam/ta/reportportal/config/EncryptConfiguration.java @@ -30,9 +30,11 @@ import java.nio.file.Paths; import java.security.SecureRandom; import java.util.Base64; +import java.util.Optional; import org.apache.commons.io.IOUtils; import org.jasypt.encryption.pbe.StandardPBEStringEncryptor; import org.jasypt.util.text.BasicTextEncryptor; +import org.jooq.tools.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.InitializingBean; @@ -51,6 +53,9 @@ public class EncryptConfiguration implements InitializingBean { private static final Logger LOGGER = LoggerFactory.getLogger(EncryptConfiguration.class); + @Value("${rp.encryptor.password}") + private String password; + @Value("${rp.integration.salt.path:keystore}") private String integrationSaltPath; @@ -58,6 +63,7 @@ public class EncryptConfiguration implements InitializingBean { private String integrationSaltFile; private String secretFilePath; + private final DataStore dataStore; private final FeatureFlagHandler featureFlagHandler; @@ -76,7 +82,7 @@ public EncryptConfiguration(DataStore dataStore, FeatureFlagHandler featureFlagH @Bean(name = "basicEncryptor") public BasicTextEncryptor getBasicEncrypt() throws IOException { BasicTextEncryptor basic = new BasicTextEncryptor(); - basic.setPassword(IOUtils.toString(dataStore.load(secretFilePath), StandardCharsets.UTF_8)); + basic.setPassword(getPassword()); return basic; } @@ -88,22 +94,29 @@ public BasicTextEncryptor getBasicEncrypt() throws IOException { @Bean(name = "strongEncryptor") public StandardPBEStringEncryptor getStrongEncryptor() throws IOException { StandardPBEStringEncryptor strong = new StandardPBEStringEncryptor(); - strong.setPassword(IOUtils.toString(dataStore.load(secretFilePath), StandardCharsets.UTF_8)); + strong.setPassword(getPassword()); strong.setAlgorithm("PBEWithMD5AndTripleDES"); return strong; } @Override - public void afterPropertiesSet() throws Exception { + public void afterPropertiesSet() { if (featureFlagHandler.isEnabled(FeatureFlag.SINGLE_BUCKET)) { secretFilePath = Paths.get(INTEGRATION_SECRETS_PATH, integrationSaltFile).toString(); } else { secretFilePath = integrationSaltPath + File.separator + integrationSaltFile; } - loadOrGenerateIntegrationSalt(dataStore); + if (password == null) { + loadOrGenerateEncryptorPassword(); + } + } + + private String getPassword() throws IOException { + return Optional.ofNullable(password) + .orElse(IOUtils.toString(dataStore.load(secretFilePath), StandardCharsets.UTF_8)); } - private void loadOrGenerateIntegrationSalt(DataStore dataStore) { + private void loadOrGenerateEncryptorPassword() { try { dataStore.load(secretFilePath); } catch (ReportPortalException ex) { From eb340d412b29dced9682e6b942601f1a1eec81d8 Mon Sep 17 00:00:00 2001 From: Pavel Bortnik Date: Thu, 12 Oct 2023 23:53:37 +0300 Subject: [PATCH 5/9] EPMRPP-80574 || Add file existence check to the DataStore interface --- .../epam/ta/reportportal/config/EncryptConfiguration.java | 4 +--- .../java/com/epam/ta/reportportal/filesystem/DataStore.java | 2 ++ .../com/epam/ta/reportportal/filesystem/LocalDataStore.java | 5 +++++ .../reportportal/filesystem/distributed/s3/S3DataStore.java | 6 ++++++ 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/epam/ta/reportportal/config/EncryptConfiguration.java b/src/main/java/com/epam/ta/reportportal/config/EncryptConfiguration.java index 3fd1234ca..fdfe72aef 100644 --- a/src/main/java/com/epam/ta/reportportal/config/EncryptConfiguration.java +++ b/src/main/java/com/epam/ta/reportportal/config/EncryptConfiguration.java @@ -117,9 +117,7 @@ private String getPassword() throws IOException { } private void loadOrGenerateEncryptorPassword() { - try { - dataStore.load(secretFilePath); - } catch (ReportPortalException ex) { + if (!dataStore.exists(secretFilePath)) { byte[] bytes = new byte[20]; new SecureRandom().nextBytes(bytes); try (InputStream secret = new ByteArrayInputStream( diff --git a/src/main/java/com/epam/ta/reportportal/filesystem/DataStore.java b/src/main/java/com/epam/ta/reportportal/filesystem/DataStore.java index 3d4b88ee0..2dbb43ef4 100644 --- a/src/main/java/com/epam/ta/reportportal/filesystem/DataStore.java +++ b/src/main/java/com/epam/ta/reportportal/filesystem/DataStore.java @@ -28,6 +28,8 @@ public interface DataStore { InputStream load(String filePath); + boolean exists(String filePath); + void delete(String filePath); void deleteAll(List filePaths, String bucketName); diff --git a/src/main/java/com/epam/ta/reportportal/filesystem/LocalDataStore.java b/src/main/java/com/epam/ta/reportportal/filesystem/LocalDataStore.java index 88e2b70a0..34bb83299 100644 --- a/src/main/java/com/epam/ta/reportportal/filesystem/LocalDataStore.java +++ b/src/main/java/com/epam/ta/reportportal/filesystem/LocalDataStore.java @@ -81,6 +81,11 @@ public InputStream load(String filePath) { } } + @Override + public boolean exists(String filePath) { + return Files.exists(Paths.get(storageRootPath, filePath)); + } + @Override public void delete(String filePath) { diff --git a/src/main/java/com/epam/ta/reportportal/filesystem/distributed/s3/S3DataStore.java b/src/main/java/com/epam/ta/reportportal/filesystem/distributed/s3/S3DataStore.java index 1ac0d896f..e00bf6e17 100644 --- a/src/main/java/com/epam/ta/reportportal/filesystem/distributed/s3/S3DataStore.java +++ b/src/main/java/com/epam/ta/reportportal/filesystem/distributed/s3/S3DataStore.java @@ -116,6 +116,12 @@ public InputStream load(String filePath) { } } + @Override + public boolean exists(String filePath) { + S3File s3File = getS3File(filePath); + return blobStore.blobExists(s3File.getBucket(), s3File.getFilePath()); + } + @Override public void delete(String filePath) { S3File s3File = getS3File(filePath); From 147b54c306bd11ea412ddb6d3e1b001f50d9d5ed Mon Sep 17 00:00:00 2001 From: Pavel Bortnik Date: Thu, 12 Oct 2023 23:58:46 +0300 Subject: [PATCH 6/9] EPMRPP-80574 || Add default password value --- .../com/epam/ta/reportportal/config/EncryptConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/epam/ta/reportportal/config/EncryptConfiguration.java b/src/main/java/com/epam/ta/reportportal/config/EncryptConfiguration.java index fdfe72aef..810536681 100644 --- a/src/main/java/com/epam/ta/reportportal/config/EncryptConfiguration.java +++ b/src/main/java/com/epam/ta/reportportal/config/EncryptConfiguration.java @@ -53,7 +53,7 @@ public class EncryptConfiguration implements InitializingBean { private static final Logger LOGGER = LoggerFactory.getLogger(EncryptConfiguration.class); - @Value("${rp.encryptor.password}") + @Value("${rp.encryptor.password:}") private String password; @Value("${rp.integration.salt.path:keystore}") From f27faa0f48aedd45eca114db73175fdb36eea925 Mon Sep 17 00:00:00 2001 From: Pavel Bortnik Date: Fri, 13 Oct 2023 00:44:44 +0300 Subject: [PATCH 7/9] EPMRPP-80574 || Add default null password value --- .../config/EncryptConfiguration.java | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/epam/ta/reportportal/config/EncryptConfiguration.java b/src/main/java/com/epam/ta/reportportal/config/EncryptConfiguration.java index 810536681..b2ccb134c 100644 --- a/src/main/java/com/epam/ta/reportportal/config/EncryptConfiguration.java +++ b/src/main/java/com/epam/ta/reportportal/config/EncryptConfiguration.java @@ -22,6 +22,7 @@ import com.epam.ta.reportportal.exception.ReportPortalException; import com.epam.ta.reportportal.filesystem.DataStore; import com.epam.ta.reportportal.util.FeatureFlagHandler; +import com.epam.ta.reportportal.ws.model.ErrorType; import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; @@ -53,7 +54,7 @@ public class EncryptConfiguration implements InitializingBean { private static final Logger LOGGER = LoggerFactory.getLogger(EncryptConfiguration.class); - @Value("${rp.encryptor.password:}") + @Value("${rp.encryptor.password:#{null}}") private String password; @Value("${rp.integration.salt.path:keystore}") @@ -80,7 +81,7 @@ public EncryptConfiguration(DataStore dataStore, FeatureFlagHandler featureFlagH * @return {@link BasicTextEncryptor} instance */ @Bean(name = "basicEncryptor") - public BasicTextEncryptor getBasicEncrypt() throws IOException { + public BasicTextEncryptor getBasicEncrypt() { BasicTextEncryptor basic = new BasicTextEncryptor(); basic.setPassword(getPassword()); return basic; @@ -92,7 +93,7 @@ public BasicTextEncryptor getBasicEncrypt() throws IOException { * @return {@link StandardPBEStringEncryptor} instance */ @Bean(name = "strongEncryptor") - public StandardPBEStringEncryptor getStrongEncryptor() throws IOException { + public StandardPBEStringEncryptor getStrongEncryptor() { StandardPBEStringEncryptor strong = new StandardPBEStringEncryptor(); strong.setPassword(getPassword()); strong.setAlgorithm("PBEWithMD5AndTripleDES"); @@ -111,9 +112,16 @@ public void afterPropertiesSet() { } } - private String getPassword() throws IOException { - return Optional.ofNullable(password) - .orElse(IOUtils.toString(dataStore.load(secretFilePath), StandardCharsets.UTF_8)); + private String getPassword() { + return Optional.ofNullable(password).orElseGet(this::loadFormDataStore); + } + + private String loadFormDataStore() { + try { + return IOUtils.toString(dataStore.load(secretFilePath), StandardCharsets.UTF_8); + } catch (IOException e) { + throw new ReportPortalException(ErrorType.UNABLE_TO_LOAD_BINARY_DATA, e.getMessage()); + } } private void loadOrGenerateEncryptorPassword() { From 46a3ae348519c7b207878e78772b9805c9652442 Mon Sep 17 00:00:00 2001 From: Pavel_Bortnik Date: Sat, 14 Oct 2023 21:29:09 +0300 Subject: [PATCH 8/9] EPMRPP-80574 || Update exception handling in s3 data store --- .../config/EncryptConfiguration.java | 10 +++++----- .../filesystem/distributed/s3/S3DataStore.java | 18 +++++++++--------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/epam/ta/reportportal/config/EncryptConfiguration.java b/src/main/java/com/epam/ta/reportportal/config/EncryptConfiguration.java index b2ccb134c..5b1c6f4e0 100644 --- a/src/main/java/com/epam/ta/reportportal/config/EncryptConfiguration.java +++ b/src/main/java/com/epam/ta/reportportal/config/EncryptConfiguration.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Paths; import java.security.SecureRandom; import java.util.Base64; @@ -35,7 +36,6 @@ import org.apache.commons.io.IOUtils; import org.jasypt.encryption.pbe.StandardPBEStringEncryptor; import org.jasypt.util.text.BasicTextEncryptor; -import org.jooq.tools.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.InitializingBean; @@ -58,10 +58,10 @@ public class EncryptConfiguration implements InitializingBean { private String password; @Value("${rp.integration.salt.path:keystore}") - private String integrationSaltPath; + private String passwordFilePath; @Value("${rp.integration.salt.file:secret-integration-salt}") - private String integrationSaltFile; + private String passwordFile; private String secretFilePath; @@ -103,9 +103,9 @@ public StandardPBEStringEncryptor getStrongEncryptor() { @Override public void afterPropertiesSet() { if (featureFlagHandler.isEnabled(FeatureFlag.SINGLE_BUCKET)) { - secretFilePath = Paths.get(INTEGRATION_SECRETS_PATH, integrationSaltFile).toString(); + secretFilePath = Paths.get(INTEGRATION_SECRETS_PATH, passwordFile).toString(); } else { - secretFilePath = integrationSaltPath + File.separator + integrationSaltFile; + secretFilePath = passwordFilePath + File.separator + passwordFile; } if (password == null) { loadOrGenerateEncryptorPassword(); diff --git a/src/main/java/com/epam/ta/reportportal/filesystem/distributed/s3/S3DataStore.java b/src/main/java/com/epam/ta/reportportal/filesystem/distributed/s3/S3DataStore.java index e00bf6e17..313e1102b 100644 --- a/src/main/java/com/epam/ta/reportportal/filesystem/distributed/s3/S3DataStore.java +++ b/src/main/java/com/epam/ta/reportportal/filesystem/distributed/s3/S3DataStore.java @@ -27,6 +27,7 @@ import java.nio.file.Paths; import java.util.List; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import org.jclouds.blobstore.BlobStore; @@ -103,17 +104,16 @@ public String save(String filePath, InputStream inputStream) { @Override public InputStream load(String filePath) { S3File s3File = getS3File(filePath); - try { - Blob fileBlob = blobStore.getBlob(s3File.getBucket(), s3File.getFilePath()); - if (fileBlob != null) { - return fileBlob.getPayload().openStream(); - } else { - throw new Exception(); + Blob fileBlob = blobStore.getBlob(s3File.getBucket(), s3File.getFilePath()); + if (fileBlob != null) { + try (InputStream inputStream = fileBlob.getPayload().openStream()) { + return inputStream; + } catch (IOException e) { + throw new ReportPortalException(ErrorType.UNABLE_TO_LOAD_BINARY_DATA, e.getMessage()); } - } catch (Exception e) { - LOGGER.error("Unable to find file '{}'", filePath, e); - throw new ReportPortalException(ErrorType.UNABLE_TO_LOAD_BINARY_DATA, "Unable to find file"); } + LOGGER.error("Unable to find file '{}'", filePath); + throw new ReportPortalException(ErrorType.UNABLE_TO_LOAD_BINARY_DATA, "Unable to find file"); } @Override From 44a2bc4d0d3878c5706946bd3faa9238a4aeb0be Mon Sep 17 00:00:00 2001 From: Pavel_Bortnik Date: Mon, 16 Oct 2023 13:59:48 +0300 Subject: [PATCH 9/9] EPMRPP-80574 || Fix spelling mistake --- .../epam/ta/reportportal/config/EncryptConfiguration.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/epam/ta/reportportal/config/EncryptConfiguration.java b/src/main/java/com/epam/ta/reportportal/config/EncryptConfiguration.java index 5b1c6f4e0..b865b6add 100644 --- a/src/main/java/com/epam/ta/reportportal/config/EncryptConfiguration.java +++ b/src/main/java/com/epam/ta/reportportal/config/EncryptConfiguration.java @@ -28,7 +28,6 @@ import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.nio.file.Paths; import java.security.SecureRandom; import java.util.Base64; @@ -113,10 +112,10 @@ public void afterPropertiesSet() { } private String getPassword() { - return Optional.ofNullable(password).orElseGet(this::loadFormDataStore); + return Optional.ofNullable(password).orElseGet(this::loadFromDataStore); } - private String loadFormDataStore() { + private String loadFromDataStore() { try { return IOUtils.toString(dataStore.load(secretFilePath), StandardCharsets.UTF_8); } catch (IOException e) {