Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DNM | EUSM Release Branch #103

Draft
wants to merge 40 commits into
base: gateway-enhancements
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
947ef56
Fix REL sync strategy Caching bug 🐛
ndegwamartin Oct 24, 2024
392bf3f
Refactor to support returning all paginated results
ndegwamartin Oct 25, 2024
578283f
Refactor REL upstream fetch to single request
ndegwamartin Oct 25, 2024
f37e396
Fix no _syncLocation specified exception
ndegwamartin Oct 28, 2024
5ed1a56
Refactor chunked REL requests data fetch
ndegwamartin Oct 28, 2024
967ebdd
Update REL requests data fetch batch size to 100
ndegwamartin Oct 28, 2024
533332c
add more tests
lincmba Nov 7, 2024
6af4db1
Test generateSyncStrategyIdsCacheKey
lincmba Nov 7, 2024
b30815b
Refactor getLocationHierarchyLocations to reuse Util helper
ndegwamartin Oct 29, 2024
ad53bfb
Refactor REL data fetch - Optimize
ndegwamartin Oct 29, 2024
a0f7762
Implement get Summary Mode for REL strategy
ndegwamartin Nov 7, 2024
c345b6b
Refactor/Implement REL data sync + pagination summary count
ndegwamartin Nov 14, 2024
bd3ee74
Revert chunk size for optimal REL search
ndegwamartin Nov 15, 2024
8c978c8
Release version 2.2.1 - 0 for QA
ndegwamartin Nov 15, 2024
28243a9
Fix build 💚
ndegwamartin Nov 15, 2024
28b5bc1
Write unit tests ✅
ndegwamartin Nov 15, 2024
8300fed
Update release version to v2.2.2 🔖
ndegwamartin Nov 15, 2024
cfce80e
Add Group resource to Sync Filter Ignored Queries
ndegwamartin Nov 15, 2024
aa3bc78
Release version 2.2.2 - 1 for Eusm QA
ndegwamartin Nov 15, 2024
7766d7b
Handle null in stream processing
lincmba Nov 19, 2024
d760bfa
Bump up version
lincmba Nov 19, 2024
47e25f9
Fix REL sync strategy Caching bug 🐛
ndegwamartin Oct 24, 2024
a2995cc
Refactor to support returning all paginated results
ndegwamartin Oct 25, 2024
14e4c86
Refactor REL upstream fetch to single request
ndegwamartin Oct 25, 2024
5150406
Fix no _syncLocation specified exception
ndegwamartin Oct 28, 2024
e3a37d5
Refactor chunked REL requests data fetch
ndegwamartin Oct 28, 2024
fcc35b3
Update REL requests data fetch batch size to 100
ndegwamartin Oct 28, 2024
47e48c9
add more tests
lincmba Nov 7, 2024
55676c5
Test generateSyncStrategyIdsCacheKey
lincmba Nov 7, 2024
9c0f288
Refactor getLocationHierarchyLocations to reuse Util helper
ndegwamartin Oct 29, 2024
e9cead3
Refactor REL data fetch - Optimize
ndegwamartin Oct 29, 2024
5593645
Implement get Summary Mode for REL strategy
ndegwamartin Nov 7, 2024
cfdffd4
Refactor/Implement REL data sync + pagination summary count
ndegwamartin Nov 14, 2024
1e9655a
Revert chunk size for optimal REL search
ndegwamartin Nov 15, 2024
0d19e62
Release version 2.2.1 - 0 for QA
ndegwamartin Nov 15, 2024
93857b0
Fix build 💚
ndegwamartin Nov 15, 2024
d6623c1
Write unit tests ✅
ndegwamartin Nov 15, 2024
ff13ce2
Update release version to v2.2.2 🔖
ndegwamartin Nov 15, 2024
2fb7958
Revert REL sync pagination to 100 record chunks
ndegwamartin Nov 20, 2024
95ba6ae
Merge branch 'gateway-enhancements' into 2.2.2.1-eusm-release
ndegwamartin Nov 20, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions exec/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<parent>
<groupId>org.smartregister</groupId>
<artifactId>opensrp-gateway-plugin</artifactId>
<version>2.2.1</version>
<version>2.2.2.3</version>
</parent>

<artifactId>exec</artifactId>
Expand Down Expand Up @@ -70,7 +70,7 @@
<dependency>
<groupId>org.smartregister</groupId>
<artifactId>plugins</artifactId>
<version>2.2.1</version>
<version>2.2.2.3</version>
</dependency>

<dependency>
Expand Down
2 changes: 1 addition & 1 deletion plugins/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<parent>
<groupId>org.smartregister</groupId>
<artifactId>opensrp-gateway-plugin</artifactId>
<version>2.2.1</version>
<version>2.2.2.3</version>
</parent>

<artifactId>plugins</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import java.util.concurrent.TimeUnit;

import org.apache.commons.lang3.StringUtils;
import org.hl7.fhir.r4.model.Bundle;
import org.hl7.fhir.r4.model.DomainResource;
import org.hl7.fhir.r4.model.Location;

Expand All @@ -23,6 +24,8 @@ public enum CacheHelper {

Cache<String, List<String>> listStringCache;

Cache<String, List<Bundle.BundleEntryComponent>> resourceListCache;

CacheHelper() {
cache =
Caffeine.newBuilder()
Expand All @@ -49,6 +52,11 @@ public enum CacheHelper {
.expireAfterWrite(getCacheExpiryDurationInSeconds(), TimeUnit.SECONDS)
.maximumSize(DEFAULT_CACHE_SIZE)
.build();
resourceListCache =
Caffeine.newBuilder()
.expireAfterWrite(getCacheExpiryDurationInSeconds(), TimeUnit.SECONDS)
.maximumSize(DEFAULT_CACHE_SIZE)
.build();
}

private int getCacheExpiryDurationInSeconds() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ public class Constants {
public static final int PAGINATION_DEFAULT_PAGE_SIZE = 20;
public static final int PAGINATION_DEFAULT_PAGE_NUMBER = 1;
public static final String SYNC_LOCATIONS_SEARCH_PARAM = "_syncLocations";
public static final String SYNC_LOCATIONS_HASH_PARAM = "_sLHash";
public static final String SYNC_LOCATIONS_PAGE_PARAM = "_sLPage";
public static final String RELATED_ENTITY_TAG_URL_ENV = "RELATED_ENTITY_TAG_URL";
public static final String DEFAULT_RELATED_ENTITY_TAG_URL =
"https://smartregister.org/related-entity-location-tag-id";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@ public List<LocationHierarchy> getLocationHierarchies(
Boolean filterInventory,
String lastUpdated) {

locationIds = locationIds != null ? locationIds : Collections.emptyList();

return locationIds.parallelStream()
.map(
locationId ->
Expand Down Expand Up @@ -114,6 +116,8 @@ public LocationHierarchy getLocationHierarchyCore(
locationHierarchy.setId(LOCATION_RESOURCE + locationId);

locationHierarchy.setLocationHierarchyTree(locationHierarchyTree);

logger.info("Finished building Location Hierarchy of Location Id : {}", locationId);
} else {
logger.error("LocationHierarchy with identifier: {} not found", locationId);
locationHierarchy.setId(LOCATION_RESOURCE_NOT_FOUND);
Expand Down Expand Up @@ -179,6 +183,9 @@ public List<Location> getDescendants(
allLocations.add(parentLocation);
}
if (childLocationBundle != null) {

Utils.fetchAllBundlePagesAndInject(r4FHIRClient, childLocationBundle);

childLocationBundle.getEntry().parallelStream()
.forEach(
childLocation -> {
Expand All @@ -191,24 +198,6 @@ public List<Location> getDescendants(
null,
adminLevels));
});

while (childLocationBundle.getLink(Bundle.LINK_NEXT) != null) {
childLocationBundle =
getFhirClientForR4().loadPage().next(childLocationBundle).execute();

childLocationBundle.getEntry().parallelStream()
.forEach(
childLocation -> {
Location childLocationEntity =
(Location) childLocation.getResource();
allLocations.add(childLocationEntity);
allLocations.addAll(
getDescendants(
childLocationEntity.getIdElement().getIdPart(),
null,
adminLevels));
});
}
}

return allLocations;
Expand Down Expand Up @@ -298,9 +287,11 @@ public Bundle handleNonIdentifierRequest(
filterInventory,
lastUpdated);
List<Resource> resourceList =
locationHierarchies.stream()
.map(locationHierarchy -> (Resource) locationHierarchy)
.collect(Collectors.toList());
locationHierarchies != null
? locationHierarchies.stream()
.map(locationHierarchy -> (Resource) locationHierarchy)
.collect(Collectors.toList())
: Collections.emptyList();
return Utils.createBundle(resourceList);
} else {
List<String> locationIds =
Expand All @@ -314,9 +305,11 @@ public Bundle handleNonIdentifierRequest(
filterInventory,
lastUpdated);
List<Resource> resourceList =
locationHierarchies.stream()
.map(locationHierarchy -> (Resource) locationHierarchy)
.collect(Collectors.toList());
locationHierarchies != null
? locationHierarchies.stream()
.map(locationHierarchy -> (Resource) locationHierarchy)
.collect(Collectors.toList())
: Collections.emptyList();
return Utils.createBundle(resourceList);
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package org.smartregister.fhir.gateway.plugins;

import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
Expand All @@ -11,7 +12,6 @@
import javax.inject.Named;

import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.hl7.fhir.r4.model.Binary;
import org.hl7.fhir.r4.model.Bundle;
import org.hl7.fhir.r4.model.CareTeam;
Expand Down Expand Up @@ -103,20 +103,22 @@ public AccessDecision checkAccess(RequestDetailsReader requestDetails) {
private void initSyncAccessDecision(RequestDetailsReader requestDetailsReader) {
Map<String, List<String>> syncStrategyIds;

Composition composition = fetchComposition();
String syncStrategy = readSyncStrategyFromComposition(composition);

if (CacheHelper.INSTANCE.skipCache()) {
syncStrategyIds =
getSyncStrategyIds(
jwt.getSubject(), applicationId, fhirContext, requestDetailsReader);
getSyncStrategyIds(jwt.getSubject(), syncStrategy, requestDetailsReader);
} else {
syncStrategyIds =
CacheHelper.INSTANCE.cache.get(
jwt.getSubject(),
userId ->
generateSyncStrategyIdsCacheKey(
jwt.getSubject(),
syncStrategy,
requestDetailsReader.getParameters()),
key ->
getSyncStrategyIds(
userId,
applicationId,
fhirContext,
requestDetailsReader));
jwt.getSubject(), syncStrategy, requestDetailsReader));
}

this.syncAccessDecision =
Expand All @@ -129,6 +131,38 @@ private void initSyncAccessDecision(RequestDetailsReader requestDetailsReader) {
userRoles);
}

@VisibleForTesting
protected static String generateSyncStrategyIdsCacheKey(
String userId, String syncStrategy, Map<String, String[]> parameters) {

String key = null;
switch (syncStrategy) {
case Constants.SyncStrategy.RELATED_ENTITY_LOCATION:
try {

String[] syncLocations =
parameters.getOrDefault(
Constants.SYNC_LOCATIONS_SEARCH_PARAM, new String[] {});

if (syncLocations.length == 0) {
key = userId;
} else {
key = Utils.generateHash(Utils.getSortedInput(syncLocations[0], ","));
}

} catch (NoSuchAlgorithmException exception) {
logger.error(exception.getMessage());
}

break;

default:
key = userId;
}

return key;
}

private boolean checkUserHasRole(String resourceName, String requestType) {
return StringUtils.isNotBlank(resourceName)
&& (checkIfRoleExists(getAdminRoleName(resourceName), this.userRoles)
Expand Down Expand Up @@ -216,8 +250,7 @@ private Composition readCompositionResource(String applicationId, FhirContext fh
return compositionEntry != null ? (Composition) compositionEntry.getResource() : null;
}

Pair<Composition, PractitionerDetails> fetchCompositionAndPractitionerDetails(
String subject, String applicationId, FhirContext fhirContext) {
PractitionerDetails fetchPractitionerDetails(String subject) {
fhirContext.registerCustomType(PractitionerDetails.class);

IGenericClient client = Utils.createFhirClientForR4(fhirContext);
Expand All @@ -227,53 +260,43 @@ Pair<Composition, PractitionerDetails> fetchCompositionAndPractitionerDetails(
PractitionerDetails practitionerDetails =
practitionerDetailsEndpointHelper.getPractitionerDetailsByKeycloakId(subject);

Composition composition = readCompositionResource(applicationId, fhirContext);

if (composition == null)
throw new IllegalStateException(
"No Composition resource found for application id '" + applicationId + "'");

if (practitionerDetails == null)
throw new IllegalStateException(
"No PractitionerDetail resource found for user with id '" + subject + "'");

return Pair.of(composition, practitionerDetails);
return practitionerDetails;
}

Pair<String, PractitionerDetails> fetchSyncStrategyDetails(
String subject, String applicationId, FhirContext fhirContext) {
private Composition fetchComposition() {
Composition composition = readCompositionResource(applicationId, fhirContext);
if (composition == null)
throw new IllegalStateException(
"No Composition resource found for application id '" + applicationId + "'");

Pair<Composition, PractitionerDetails> compositionPractitionerDetailsPair =
fetchCompositionAndPractitionerDetails(subject, applicationId, fhirContext);
Composition composition = compositionPractitionerDetailsPair.getLeft();
PractitionerDetails practitionerDetails = compositionPractitionerDetailsPair.getRight();
return composition;
}

private String readSyncStrategyFromComposition(Composition composition) {
String binaryResourceReference = Utils.getBinaryResourceReference(composition);
Binary binary =
Utils.readApplicationConfigBinaryResource(binaryResourceReference, fhirContext);

return Pair.of(Utils.findSyncStrategy(binary), practitionerDetails);
return Utils.findSyncStrategy(binary);
}

private Map<String, List<String>> getSyncStrategyIds(
String subjectId,
String applicationId,
FhirContext fhirContext,
RequestDetailsReader requestDetailsReader) {
Pair<String, PractitionerDetails> syncStrategyDetails =
fetchSyncStrategyDetails(subjectId, applicationId, fhirContext);
String subjectId, String syncStrategy, RequestDetailsReader requestDetailsReader) {

String syncStrategy = syncStrategyDetails.getLeft();
PractitionerDetails practitionerDetails = syncStrategyDetails.getRight();
PractitionerDetails practitionerDetails = fetchPractitionerDetails(subjectId);

return collateSyncStrategyIds(syncStrategy, practitionerDetails, requestDetailsReader);
}

private List<String> getLocationUuids(String[] syncLocations) {
List<String> locationUuids = new ArrayList<>();
String syncLocationParam;
for (int i = 0; i < syncLocations.length; i++) {
syncLocationParam = syncLocations[i];

for (String syncLocation : syncLocations) {
syncLocationParam = syncLocation;
if (!syncLocationParam.isEmpty())
locationUuids.addAll(
Set.of(syncLocationParam.split(Constants.PARAM_VALUES_SEPARATOR)));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,8 @@ protected Bundle getAttributedPractitionerDetailsByPractitioner(Practitioner pra

@Nonnull
public static Set<String> getAttributedLocations(List<LocationHierarchy> locationHierarchies) {
locationHierarchies =
locationHierarchies != null ? locationHierarchies : Collections.emptyList();
List<ParentChildrenMap> parentChildrenList =
locationHierarchies.stream()
.flatMap(
Expand Down
Loading