Skip to content

Commit

Permalink
EPMRPP-88755 || Add prefix and postfix for filesystem (#963)
Browse files Browse the repository at this point in the history 
* Add ability to use prefix and postfix in filesystem

* Remove redundant code

* Remove prefix and postfix for plugins bucket

* Fix tests

* Fix CommonDataStoreServiceTest

* Test commit

* Test commit

* Change logback-test.xml

* Change asserts

* Make tests OS independent

* Fix failed tests
  • Loading branch information
@IvanKustau
IvanKustau authored Dec 14, 2023
1 parent 126ee2e commit 516ace3
Show file tree
Hide file tree
Showing 10 changed files with 150 additions and 71 deletions.
8 changes: 6 additions & 2 deletions src/main/java/com/epam/ta/reportportal/config/DataStoreConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,8 +155,12 @@ public BlobStore filesystemBlobStore(
@Bean
@ConditionalOnProperty(name = "datastore.type", havingValue = "filesystem")
public DataStore localDataStore(@Autowired BlobStore blobStore,
FeatureFlagHandler featureFlagHandler) {
return new LocalDataStore(blobStore, featureFlagHandler);
FeatureFlagHandler featureFlagHandler,
@Value("${datastore.bucketPrefix}") String bucketPrefix,
@Value("${datastore.bucketPostfix}") String bucketPostfix,
@Value("${datastore.defaultBucketName}") String defaultBucketName) {
return new LocalDataStore(
blobStore, featureFlagHandler, bucketPrefix, bucketPostfix, defaultBucketName);
}

/**
Expand Down
41 changes: 26 additions & 15 deletions src/main/java/com/epam/ta/reportportal/filesystem/LocalDataStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.Objects;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.domain.Blob;
import org.slf4j.Logger;
Expand All @@ -41,13 +42,19 @@ public class LocalDataStore implements DataStore {

private final FeatureFlagHandler featureFlagHandler;

private static final String SINGLE_BUCKET_NAME = "store";
private final String bucketPrefix;

private static final String PLUGINS = "plugins";
private final String bucketPostfix;

public LocalDataStore(BlobStore blobStore, FeatureFlagHandler featureFlagHandler) {
private final String defaultBucketName;

public LocalDataStore(BlobStore blobStore, FeatureFlagHandler featureFlagHandler,
String bucketPrefix, String bucketPostfix, String defaultBucketName) {
this.blobStore = blobStore;
this.featureFlagHandler = featureFlagHandler;
this.bucketPrefix = bucketPrefix;
this.bucketPostfix = Objects.requireNonNullElse(bucketPostfix, "");
this.defaultBucketName = defaultBucketName;
}

@Override
Expand Down Expand Up @@ -117,9 +124,9 @@ public void delete(String filePath) {
@Override
public void deleteAll(List<String> filePaths, String bucketName) {
if (!featureFlagHandler.isEnabled(FeatureFlag.SINGLE_BUCKET)) {
blobStore.removeBlobs(bucketName, filePaths);
blobStore.removeBlobs(bucketPrefix + bucketName + bucketPostfix, filePaths);
} else {
blobStore.removeBlobs(SINGLE_BUCKET_NAME, filePaths);
blobStore.removeBlobs(bucketName, filePaths);
}
}

Expand All @@ -129,19 +136,23 @@ public void deleteContainer(String bucketName) {
}

private StoredFile getStoredFile(String filePath) {
Path targetPath = Paths.get(filePath);
if (featureFlagHandler.isEnabled(FeatureFlag.SINGLE_BUCKET)) {
return new StoredFile(SINGLE_BUCKET_NAME, filePath);
return new StoredFile(defaultBucketName, filePath);
}
Path targetPath = Paths.get(filePath);
int nameCount = targetPath.getNameCount();
String bucketName;
if (nameCount > 1) {
bucketName = bucketPrefix + retrievePath(targetPath, 0, 1) + bucketPostfix;
return new StoredFile(bucketName, retrievePath(targetPath, 1, nameCount));
} else {
int nameCount = targetPath.getNameCount();
if (nameCount > 1) {
String bucketName = targetPath.getName(0).toString();
String newFilePath = targetPath.subpath(1, nameCount).toString();
return new StoredFile(bucketName, newFilePath);
} else {
return new StoredFile(PLUGINS, filePath);
}
bucketName = defaultBucketName;
return new StoredFile(bucketName, retrievePath(targetPath, 0, 1));
}
}

private String retrievePath(Path path, int beginIndex, int endIndex) {
return String.valueOf(path.subpath(beginIndex, endIndex));
}
}

2 changes: 1 addition & 1 deletion src/main/java/com/epam/ta/reportportal/filesystem/distributed/s3/S3DataStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ private StoredFile getStoredFile(String filePath) {
bucketName = bucketPrefix + retrievePath(targetPath, 0, 1) + bucketPostfix;
return new StoredFile(bucketName, retrievePath(targetPath, 1, nameCount));
} else {
bucketName = bucketPrefix + defaultBucketName + bucketPostfix;
bucketName = defaultBucketName;
return new StoredFile(bucketName, retrievePath(targetPath, 0, 1));
}
}
Expand Down
33 changes: 23 additions & 10 deletions src/test/java/com/epam/ta/reportportal/binary/impl/AttachmentDataStoreServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,12 @@ class AttachmentDataStoreServiceTest extends BaseTest {
@Value("${datastore.path:/data/store}")
private String storageRootPath;

@Value("${datastore.bucketPrefix:prj-}")
private String bucketPrefix;

@Value("${datastore.bucketPostfix:}")
private String bucketPostfix;

private static final String BUCKET_NAME = "bucket";

private static Random random = new Random();
Expand All @@ -53,6 +59,8 @@ class AttachmentDataStoreServiceTest extends BaseTest {
void saveLoadAndDeleteTest() throws IOException {
InputStream inputStream = new ClassPathResource("meh.jpg").getInputStream();

String bucketPath = bucketPrefix + BUCKET_NAME + bucketPostfix;

String fileId =
attachmentDataStoreService.save(BUCKET_NAME + "/" + random.nextLong() + "meh.jpg",
inputStream
Expand All @@ -62,32 +70,36 @@ void saveLoadAndDeleteTest() throws IOException {

assertTrue(loadedData.isPresent());
try (InputStream ignored = loadedData.get()) {
assertTrue(Files.exists(
Paths.get(storageRootPath, attachmentDataStoreService.dataEncoder.decode(fileId))));
String decodedPath = attachmentDataStoreService.dataEncoder.decode(fileId);
decodedPath = decodedPath.replace(BUCKET_NAME, bucketPath);
assertTrue(Files.exists(Paths.get(storageRootPath, decodedPath)));
}

attachmentDataStoreService.delete(fileId);

ReportPortalException exception =
assertThrows(ReportPortalException.class, () -> attachmentDataStoreService.load(fileId));
assertEquals("Unable to load binary data by id 'Unable to find file'", exception.getMessage());
assertFalse(Files.exists(
Paths.get(storageRootPath, attachmentDataStoreService.dataEncoder.decode(fileId))));
String decodedPath = attachmentDataStoreService.dataEncoder.decode(fileId);
decodedPath = decodedPath.replace(BUCKET_NAME, bucketPath);
assertFalse(Files.exists(Paths.get(storageRootPath, decodedPath)));
}

@Test
void saveLoadAndDeleteThumbnailTest() throws IOException {
try (InputStream inputStream = new ClassPathResource("meh.jpg").getInputStream()) {
String bucketPath = bucketPrefix + BUCKET_NAME + bucketPostfix;

String thumbnailId = attachmentDataStoreService.saveThumbnail(
BUCKET_NAME + "/" + random.nextLong() + "thumbnail.jpg", inputStream);

Optional<InputStream> loadedData = attachmentDataStoreService.load(thumbnailId);

assertTrue(loadedData.isPresent());
try (InputStream is = loadedData.get()) {
assertTrue(Files.exists(Paths.get(storageRootPath,
attachmentDataStoreService.dataEncoder.decode(thumbnailId)
)));
try (InputStream ignored = loadedData.get()) {
String decodedPath = attachmentDataStoreService.dataEncoder.decode(thumbnailId);
decodedPath = decodedPath.replace(BUCKET_NAME, bucketPath);
assertTrue(Files.exists(Paths.get(storageRootPath, decodedPath)));
}

attachmentDataStoreService.delete(thumbnailId);
Expand All @@ -97,8 +109,9 @@ void saveLoadAndDeleteThumbnailTest() throws IOException {
);
assertEquals(
"Unable to load binary data by id 'Unable to find file'", exception.getMessage());
assertFalse(Files.exists(
Paths.get(storageRootPath, attachmentDataStoreService.dataEncoder.decode(thumbnailId))));
String decodedPath = attachmentDataStoreService.dataEncoder.decode(thumbnailId);
decodedPath = decodedPath.replace(BUCKET_NAME, bucketPath);
assertFalse(Files.exists(Paths.get(storageRootPath, decodedPath)));
}
}
}
38 changes: 28 additions & 10 deletions src/test/java/com/epam/ta/reportportal/binary/impl/CommonDataStoreServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,16 @@
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Optional;
import java.util.Random;
import org.apache.commons.fileupload.FileItem;
import org.apache.commons.fileupload.disk.DiskFileItem;
import org.apache.commons.io.IOUtils;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
Expand All @@ -56,28 +59,43 @@ class CommonDataStoreServiceTest extends BaseTest {
@Value("${datastore.path:/data/store}")
private String storageRootPath;

@Value("${datastore.bucketPrefix:prj-}")
private String bucketPrefix;

@Value("${datastore.bucketPostfix:}")
private String bucketPostfix;
private static final String BUCKET_NAME = "bucket";

private String getModifiedPath(String originalPath) {
String bucketPath = bucketPrefix + BUCKET_NAME + bucketPostfix;
return originalPath.replace(BUCKET_NAME, bucketPath);
}

@Test
void saveTest() throws IOException {
CommonsMultipartFile multipartFile = getMultipartFile("meh.jpg");
String fileId = dataStoreService.save(BUCKET_NAME + "/" + multipartFile.getOriginalFilename(),
multipartFile.getInputStream()
);
String fileId =
dataStoreService.save(BUCKET_NAME + File.separator + multipartFile.getOriginalFilename(),
multipartFile.getInputStream()
);
assertNotNull(fileId);
assertTrue(Files.exists(Paths.get(storageRootPath, dataEncoder.decode(fileId))));
String decodedPath = getModifiedPath(dataEncoder.decode(fileId));
Path filePath = Paths.get(storageRootPath, decodedPath);
assertTrue(filePath.toFile().exists(), "File " + filePath + " does not exist");
dataStoreService.delete(fileId);
}

@Test
void saveThumbnailTest() throws IOException {
CommonsMultipartFile multipartFile = getMultipartFile("meh.jpg");
String fileId =
dataStoreService.saveThumbnail(BUCKET_NAME + "/" + multipartFile.getOriginalFilename(),
multipartFile.getInputStream()
);
String fileId = dataStoreService.saveThumbnail(
BUCKET_NAME + File.separator + multipartFile.getOriginalFilename(),
multipartFile.getInputStream()
);
assertNotNull(fileId);
assertTrue(Files.exists(Paths.get(storageRootPath, dataEncoder.decode(fileId))));
String decodedPath = getModifiedPath(dataEncoder.decode(fileId));
Path filePath = Paths.get(storageRootPath, decodedPath);
assertTrue(filePath.toFile().exists(), "File " + filePath + " does not exist");
dataStoreService.delete(fileId);
}

Expand Down Expand Up @@ -106,7 +124,7 @@ void saveAndDeleteTest() throws IOException {

dataStoreService.delete(fileId);

assertFalse(Files.exists(Paths.get(dataEncoder.decode(fileId))));
assertFalse(Files.exists(Paths.get(dataEncoder.decode(getModifiedPath(fileId)))));
}

public static CommonsMultipartFile getMultipartFile(String path) throws IOException {
Expand Down
30 changes: 22 additions & 8 deletions src/test/java/com/epam/ta/reportportal/binary/impl/UserDataStoreServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,12 @@ class UserDataStoreServiceTest extends BaseTest {
@Value("${datastore.path:/data/store}")
private String storageRootPath;

@Value("${datastore.bucketPrefix:prj-}")
private String bucketPrefix;

@Value("${datastore.bucketPostfix:}")
private String bucketPostfix;

private static final String BUCKET_NAME = "bucket";

private static Random random = new Random();
Expand All @@ -53,30 +59,36 @@ class UserDataStoreServiceTest extends BaseTest {
void saveLoadAndDeleteTest() throws IOException {
InputStream inputStream = new ClassPathResource("meh.jpg").getInputStream();

String bucketPath = bucketPrefix + BUCKET_NAME + bucketPostfix;

String fileId =
userDataStoreService.save(BUCKET_NAME + "/" + random.nextLong() + "meh.jpg", inputStream);

Optional<InputStream> loadedData = userDataStoreService.load(fileId);

assertTrue(loadedData.isPresent());
try (InputStream ignored = loadedData.get()) {
assertTrue(Files.exists(
Paths.get(storageRootPath, userDataStoreService.dataEncoder.decode(fileId))));
String decodedPath = userDataStoreService.dataEncoder.decode(fileId);
decodedPath = decodedPath.replace(BUCKET_NAME, bucketPath);
assertTrue(Files.exists(Paths.get(storageRootPath, decodedPath)));
}

userDataStoreService.delete(fileId);

ReportPortalException exception =
assertThrows(ReportPortalException.class, () -> userDataStoreService.load(fileId));
assertEquals("Unable to load binary data by id 'Unable to find file'", exception.getMessage());
assertFalse(
Files.exists(Paths.get(storageRootPath, userDataStoreService.dataEncoder.decode(fileId))));
String decodedPath = userDataStoreService.dataEncoder.decode(fileId);
decodedPath = decodedPath.replace(BUCKET_NAME, bucketPath);
assertFalse(Files.exists(Paths.get(storageRootPath, decodedPath)));
}

@Test
void saveLoadAndDeleteThumbnailTest() throws IOException {
InputStream inputStream = new ClassPathResource("meh.jpg").getInputStream();

String bucketPath = bucketPrefix + BUCKET_NAME + bucketPostfix;

String thumbnailId =
userDataStoreService.saveThumbnail(BUCKET_NAME + "/" + random.nextLong() + "thmbnail.jpg",
inputStream
Expand All @@ -86,16 +98,18 @@ void saveLoadAndDeleteThumbnailTest() throws IOException {

assertTrue(loadedData.isPresent());
try (InputStream ignored = loadedData.get()) {
assertTrue(Files.exists(
Paths.get(storageRootPath, userDataStoreService.dataEncoder.decode(thumbnailId))));
String decodedPath = userDataStoreService.dataEncoder.decode(thumbnailId);
decodedPath = decodedPath.replace(BUCKET_NAME, bucketPath);
assertTrue(Files.exists(Paths.get(storageRootPath, decodedPath)));
}

userDataStoreService.delete(thumbnailId);

ReportPortalException exception =
assertThrows(ReportPortalException.class, () -> userDataStoreService.load(thumbnailId));
assertEquals("Unable to load binary data by id 'Unable to find file'", exception.getMessage());
assertFalse(Files.exists(
Paths.get(storageRootPath, userDataStoreService.dataEncoder.decode(thumbnailId))));
String decodedPath = userDataStoreService.dataEncoder.decode(thumbnailId);
decodedPath = decodedPath.replace(BUCKET_NAME, bucketPath);
assertFalse(Files.exists(Paths.get(storageRootPath, decodedPath)));
}
}
Loading

0 comments on commit 516ace3

Please sign in to comment.