Skip to content

Commit

Permalink
Merge branch 'develop' of https://github.com/reportportal/commons-dao …
Browse files Browse the repository at this point in the history 
…into develop
  • Loading branch information
Pavel Bortnik committed Dec 15, 2023
2 parents a29704f + 516ace3 commit 1da74f0
Show file tree
Hide file tree
Showing 13 changed files with 201 additions and 117 deletions.
3 changes: 2 additions & 1 deletion src/main/java/com/epam/ta/reportportal/commons/querygen/FilterTarget.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1080,7 +1080,8 @@ protected Collection<? extends SelectField> selectFields() {
ATTACHMENT.FILE_SIZE,
ATTACHMENT.PROJECT_ID,
ATTACHMENT.LAUNCH_ID,
ATTACHMENT.ITEM_ID
ATTACHMENT.ITEM_ID,
ATTACHMENT.FILE_NAME
);
}

Expand Down
8 changes: 6 additions & 2 deletions src/main/java/com/epam/ta/reportportal/config/DataStoreConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,8 +155,12 @@ public BlobStore filesystemBlobStore(
@Bean
@ConditionalOnProperty(name = "datastore.type", havingValue = "filesystem")
public DataStore localDataStore(@Autowired BlobStore blobStore,
FeatureFlagHandler featureFlagHandler) {
return new LocalDataStore(blobStore, featureFlagHandler);
FeatureFlagHandler featureFlagHandler,
@Value("${datastore.bucketPrefix}") String bucketPrefix,
@Value("${datastore.bucketPostfix}") String bucketPostfix,
@Value("${datastore.defaultBucketName}") String defaultBucketName) {
return new LocalDataStore(
blobStore, featureFlagHandler, bucketPrefix, bucketPostfix, defaultBucketName);
}

/**
Expand Down
9 changes: 6 additions & 3 deletions src/main/java/com/epam/ta/reportportal/dao/AttachmentRepositoryCustomImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,8 @@ public List<Attachment> findByItemIdsAndLogTimeBefore(Collection<Long> itemIds,
ATTACHMENT.FILE_SIZE,
ATTACHMENT.ITEM_ID,
ATTACHMENT.LAUNCH_ID,
ATTACHMENT.PROJECT_ID
ATTACHMENT.PROJECT_ID,
ATTACHMENT.FILE_NAME
)
.from(ATTACHMENT)
.join(LOG)
Expand All @@ -173,7 +174,8 @@ public List<Attachment> findByLaunchIdsAndLogTimeBefore(Collection<Long> launchI
ATTACHMENT.FILE_SIZE,
ATTACHMENT.ITEM_ID,
ATTACHMENT.LAUNCH_ID,
ATTACHMENT.PROJECT_ID
ATTACHMENT.PROJECT_ID,
ATTACHMENT.FILE_NAME
)
.from(ATTACHMENT)
.join(LOG)
Expand All @@ -194,7 +196,8 @@ public List<Attachment> findByProjectIdsAndLogTimeBefore(Long projectId, LocalDa
ATTACHMENT.FILE_SIZE,
ATTACHMENT.ITEM_ID,
ATTACHMENT.LAUNCH_ID,
ATTACHMENT.PROJECT_ID
ATTACHMENT.PROJECT_ID,
ATTACHMENT.FILE_NAME
)
.from(ATTACHMENT)
.join(LOG)
Expand Down
85 changes: 43 additions & 42 deletions src/main/java/com/epam/ta/reportportal/dao/util/RecordMappers.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@
*/
public class RecordMappers {

private static final ObjectMapper objectMapper;
private static final ObjectMapper objectMapper;

static {
objectMapper = new ObjectMapper();
Expand Down Expand Up @@ -220,6 +220,7 @@ public class RecordMappers {
attachment.setProjectId(r.get(ATTACHMENT.PROJECT_ID));
attachment.setLaunchId(r.get(ATTACHMENT.LAUNCH_ID));
attachment.setItemId(r.get(ATTACHMENT.ITEM_ID));
attachment.setFileName(r.get(ATTACHMENT.FILE_NAME));

return attachment;
}).orElse(null);
Expand Down Expand Up @@ -323,8 +324,8 @@ public class RecordMappers {
);

/**
* Maps record into {@link PatternTemplate} object (only {@link PatternTemplate#id} and
* {@link PatternTemplate#name} fields)
* Maps record into {@link PatternTemplate} object (only {@link PatternTemplate#id} and {@link
* PatternTemplate#name} fields)
*/
public static final Function<? super Record, Optional<PatternTemplate>> PATTERN_TEMPLATE_NAME_RECORD_MAPPER = r -> ofNullable(
r.get(
Expand Down Expand Up @@ -409,45 +410,45 @@ public class RecordMappers {
return new ReportPortalUser.ProjectDetails(projectId, projectName, projectRole);
};

public static final RecordMapper<? super Record, Activity> ACTIVITY_MAPPER = r -> {
Activity activity = new Activity();
activity.setId(r.get(ACTIVITY.ID));
activity.setCreatedAt(r.get(ACTIVITY.CREATED_AT, LocalDateTime.class));
activity.setAction(EventAction.valueOf(r.get(ACTIVITY.ACTION)));
activity.setEventName(r.get(ACTIVITY.EVENT_NAME));
activity.setPriority(EventPriority.valueOf(r.get(ACTIVITY.PRIORITY)));
activity.setObjectId(r.get(ACTIVITY.OBJECT_ID));
activity.setObjectName(r.get(ACTIVITY.OBJECT_NAME));
activity.setObjectType(EventObject.valueOf(r.get(ACTIVITY.OBJECT_TYPE)));
activity.setProjectId(r.get(ACTIVITY.PROJECT_ID));
activity.setProjectName(r.get(PROJECT.NAME));
String detailsJson = r.get(ACTIVITY.DETAILS, String.class);
ofNullable(detailsJson).ifPresent(s -> {
try {
ActivityDetails details = objectMapper.readValue(s, ActivityDetails.class);
activity.setDetails(details);
} catch (IOException e) {
throw new ReportPortalException(ErrorType.UNCLASSIFIED_REPORT_PORTAL_ERROR);
}
});
activity.setSubjectId(r.get(ACTIVITY.SUBJECT_ID));
activity.setSubjectName(ofNullable(r.get(USERS.LOGIN)).orElse(r.get(ACTIVITY.SUBJECT_NAME)));
activity.setSubjectType(EventSubject.valueOf(r.get(ACTIVITY.SUBJECT_TYPE)));
return activity;
};

public static final RecordMapper<? super Record, OwnedEntity> OWNED_ENTITY_RECORD_MAPPER = r -> r.into(
OwnedEntity.class);

private static final BiConsumer<Widget, ? super Record> WIDGET_USER_FILTER_MAPPER = (widget, res) -> ofNullable(
res.get(FILTER.ID)).ifPresent(
id -> {
Set<UserFilter> filters = ofNullable(widget.getFilters()).orElseGet(Sets::newLinkedHashSet);
UserFilter filter = new UserFilter();
filter.setId(id);
filters.add(filter);
widget.setFilters(filters);
});
public static final RecordMapper<? super Record, Activity> ACTIVITY_MAPPER = r -> {
Activity activity = new Activity();
activity.setId(r.get(ACTIVITY.ID));
activity.setCreatedAt(r.get(ACTIVITY.CREATED_AT, LocalDateTime.class));
activity.setAction(EventAction.valueOf(r.get(ACTIVITY.ACTION)));
activity.setEventName(r.get(ACTIVITY.EVENT_NAME));
activity.setPriority(EventPriority.valueOf(r.get(ACTIVITY.PRIORITY)));
activity.setObjectId(r.get(ACTIVITY.OBJECT_ID));
activity.setObjectName(r.get(ACTIVITY.OBJECT_NAME));
activity.setObjectType(EventObject.valueOf(r.get(ACTIVITY.OBJECT_TYPE)));
activity.setProjectId(r.get(ACTIVITY.PROJECT_ID));
activity.setProjectName(r.get(PROJECT.NAME));
String detailsJson = r.get(ACTIVITY.DETAILS, String.class);
ofNullable(detailsJson).ifPresent(s -> {
try {
ActivityDetails details = objectMapper.readValue(s, ActivityDetails.class);
activity.setDetails(details);
} catch (IOException e) {
throw new ReportPortalException(ErrorType.UNCLASSIFIED_REPORT_PORTAL_ERROR);
}
});
activity.setSubjectId(r.get(ACTIVITY.SUBJECT_ID));
activity.setSubjectName(ofNullable(r.get(USERS.LOGIN)).orElse(r.get(ACTIVITY.SUBJECT_NAME)));
activity.setSubjectType(EventSubject.valueOf(r.get(ACTIVITY.SUBJECT_TYPE)));
return activity;
};

public static final RecordMapper<? super Record, OwnedEntity> OWNED_ENTITY_RECORD_MAPPER = r -> r.into(
OwnedEntity.class);

private static final BiConsumer<Widget, ? super Record> WIDGET_USER_FILTER_MAPPER = (widget, res) -> ofNullable(
res.get(FILTER.ID)).ifPresent(
id -> {
Set<UserFilter> filters = ofNullable(widget.getFilters()).orElseGet(Sets::newLinkedHashSet);
UserFilter filter = new UserFilter();
filter.setId(id);
filters.add(filter);
widget.setFilters(filters);
});

private static final BiConsumer<Widget, ? super Record> WIDGET_OPTIONS_MAPPER = (widget, res) -> {
ofNullable(res.get(WIDGET.WIDGET_OPTIONS, String.class)).ifPresent(wo -> {
Expand Down
41 changes: 26 additions & 15 deletions src/main/java/com/epam/ta/reportportal/filesystem/LocalDataStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.Objects;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.domain.Blob;
import org.slf4j.Logger;
Expand All @@ -41,13 +42,19 @@ public class LocalDataStore implements DataStore {

private final FeatureFlagHandler featureFlagHandler;

private static final String SINGLE_BUCKET_NAME = "store";
private final String bucketPrefix;

private static final String PLUGINS = "plugins";
private final String bucketPostfix;

public LocalDataStore(BlobStore blobStore, FeatureFlagHandler featureFlagHandler) {
private final String defaultBucketName;

public LocalDataStore(BlobStore blobStore, FeatureFlagHandler featureFlagHandler,
String bucketPrefix, String bucketPostfix, String defaultBucketName) {
this.blobStore = blobStore;
this.featureFlagHandler = featureFlagHandler;
this.bucketPrefix = bucketPrefix;
this.bucketPostfix = Objects.requireNonNullElse(bucketPostfix, "");
this.defaultBucketName = defaultBucketName;
}

@Override
Expand Down Expand Up @@ -117,9 +124,9 @@ public void delete(String filePath) {
@Override
public void deleteAll(List<String> filePaths, String bucketName) {
if (!featureFlagHandler.isEnabled(FeatureFlag.SINGLE_BUCKET)) {
blobStore.removeBlobs(bucketName, filePaths);
blobStore.removeBlobs(bucketPrefix + bucketName + bucketPostfix, filePaths);
} else {
blobStore.removeBlobs(SINGLE_BUCKET_NAME, filePaths);
blobStore.removeBlobs(bucketName, filePaths);
}
}

Expand All @@ -129,19 +136,23 @@ public void deleteContainer(String bucketName) {
}

private StoredFile getStoredFile(String filePath) {
Path targetPath = Paths.get(filePath);
if (featureFlagHandler.isEnabled(FeatureFlag.SINGLE_BUCKET)) {
return new StoredFile(SINGLE_BUCKET_NAME, filePath);
return new StoredFile(defaultBucketName, filePath);
}
Path targetPath = Paths.get(filePath);
int nameCount = targetPath.getNameCount();
String bucketName;
if (nameCount > 1) {
bucketName = bucketPrefix + retrievePath(targetPath, 0, 1) + bucketPostfix;
return new StoredFile(bucketName, retrievePath(targetPath, 1, nameCount));
} else {
int nameCount = targetPath.getNameCount();
if (nameCount > 1) {
String bucketName = targetPath.getName(0).toString();
String newFilePath = targetPath.subpath(1, nameCount).toString();
return new StoredFile(bucketName, newFilePath);
} else {
return new StoredFile(PLUGINS, filePath);
}
bucketName = defaultBucketName;
return new StoredFile(bucketName, retrievePath(targetPath, 0, 1));
}
}

private String retrievePath(Path path, int beginIndex, int endIndex) {
return String.valueOf(path.subpath(beginIndex, endIndex));
}
}

2 changes: 1 addition & 1 deletion src/main/java/com/epam/ta/reportportal/filesystem/distributed/s3/S3DataStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ private StoredFile getStoredFile(String filePath) {
bucketName = bucketPrefix + retrievePath(targetPath, 0, 1) + bucketPostfix;
return new StoredFile(bucketName, retrievePath(targetPath, 1, nameCount));
} else {
bucketName = bucketPrefix + defaultBucketName + bucketPostfix;
bucketName = defaultBucketName;
return new StoredFile(bucketName, retrievePath(targetPath, 0, 1));
}
}
Expand Down
33 changes: 23 additions & 10 deletions src/test/java/com/epam/ta/reportportal/binary/impl/AttachmentDataStoreServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,12 @@ class AttachmentDataStoreServiceTest extends BaseTest {
@Value("${datastore.path:/data/store}")
private String storageRootPath;

@Value("${datastore.bucketPrefix:prj-}")
private String bucketPrefix;

@Value("${datastore.bucketPostfix:}")
private String bucketPostfix;

private static final String BUCKET_NAME = "bucket";

private static Random random = new Random();
Expand All @@ -53,6 +59,8 @@ class AttachmentDataStoreServiceTest extends BaseTest {
void saveLoadAndDeleteTest() throws IOException {
InputStream inputStream = new ClassPathResource("meh.jpg").getInputStream();

String bucketPath = bucketPrefix + BUCKET_NAME + bucketPostfix;

String fileId =
attachmentDataStoreService.save(BUCKET_NAME + "/" + random.nextLong() + "meh.jpg",
inputStream
Expand All @@ -62,32 +70,36 @@ void saveLoadAndDeleteTest() throws IOException {

assertTrue(loadedData.isPresent());
try (InputStream ignored = loadedData.get()) {
assertTrue(Files.exists(
Paths.get(storageRootPath, attachmentDataStoreService.dataEncoder.decode(fileId))));
String decodedPath = attachmentDataStoreService.dataEncoder.decode(fileId);
decodedPath = decodedPath.replace(BUCKET_NAME, bucketPath);
assertTrue(Files.exists(Paths.get(storageRootPath, decodedPath)));
}

attachmentDataStoreService.delete(fileId);

ReportPortalException exception =
assertThrows(ReportPortalException.class, () -> attachmentDataStoreService.load(fileId));
assertEquals("Unable to load binary data by id 'Unable to find file'", exception.getMessage());
assertFalse(Files.exists(
Paths.get(storageRootPath, attachmentDataStoreService.dataEncoder.decode(fileId))));
String decodedPath = attachmentDataStoreService.dataEncoder.decode(fileId);
decodedPath = decodedPath.replace(BUCKET_NAME, bucketPath);
assertFalse(Files.exists(Paths.get(storageRootPath, decodedPath)));
}

@Test
void saveLoadAndDeleteThumbnailTest() throws IOException {
try (InputStream inputStream = new ClassPathResource("meh.jpg").getInputStream()) {
String bucketPath = bucketPrefix + BUCKET_NAME + bucketPostfix;

String thumbnailId = attachmentDataStoreService.saveThumbnail(
BUCKET_NAME + "/" + random.nextLong() + "thumbnail.jpg", inputStream);

Optional<InputStream> loadedData = attachmentDataStoreService.load(thumbnailId);

assertTrue(loadedData.isPresent());
try (InputStream is = loadedData.get()) {
assertTrue(Files.exists(Paths.get(storageRootPath,
attachmentDataStoreService.dataEncoder.decode(thumbnailId)
)));
try (InputStream ignored = loadedData.get()) {
String decodedPath = attachmentDataStoreService.dataEncoder.decode(thumbnailId);
decodedPath = decodedPath.replace(BUCKET_NAME, bucketPath);
assertTrue(Files.exists(Paths.get(storageRootPath, decodedPath)));
}

attachmentDataStoreService.delete(thumbnailId);
Expand All @@ -97,8 +109,9 @@ void saveLoadAndDeleteThumbnailTest() throws IOException {
);
assertEquals(
"Unable to load binary data by id 'Unable to find file'", exception.getMessage());
assertFalse(Files.exists(
Paths.get(storageRootPath, attachmentDataStoreService.dataEncoder.decode(thumbnailId))));
String decodedPath = attachmentDataStoreService.dataEncoder.decode(thumbnailId);
decodedPath = decodedPath.replace(BUCKET_NAME, bucketPath);
assertFalse(Files.exists(Paths.get(storageRootPath, decodedPath)));
}
}
}
Loading

0 comments on commit 1da74f0

Please sign in to comment.