diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationsController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationsController.java index cbb476910..27ac4e34c 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationsController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationsController.java @@ -28,13 +28,17 @@ import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.modules.phenotype.ObservationsApi; +import org.brapi.v2.model.BrAPIIndexPagination; +import org.brapi.v2.model.BrAPIMetadata; +import org.brapi.v2.model.BrAPIStatus; import org.brapi.v2.model.BrAPIWSMIMEDataTypes; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.pheno.BrAPIObservation; -import org.brapi.v2.model.pheno.response.BrAPIObservationTableResponse; +import org.brapi.v2.model.pheno.response.*; import org.breedinginsight.api.auth.ProgramSecured; import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; import org.breedinginsight.brapi.v1.controller.BrapiVersion; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; import org.breedinginsight.brapi.v2.model.request.query.ObservationQuery; import org.breedinginsight.daos.ProgramDAO; @@ -47,6 +51,8 @@ import javax.annotation.Nullable; import javax.inject.Inject; import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; @Slf4j @Controller("/${micronaut.bi.api.version}/programs/{programId}" + BrapiVersion.BRAPI_V2) @@ -57,13 +63,20 @@ public class BrAPIObservationsController { private final ProgramDAO programDAO; private final BrAPIStudyDAO brAPIStudyDAO; private final BrAPIEndpointProvider brAPIEndpointProvider; + private final BrAPIObservationDAO observationDAO; @Inject - public BrAPIObservationsController(ProgramService programService, ProgramDAO programDAO, ProgramDAO programDAO1, BrAPIStudyDAO brAPIStudyDAO, BrAPIEndpointProvider brAPIEndpointProvider) { + public BrAPIObservationsController(ProgramService programService, + ProgramDAO programDAO, + ProgramDAO programDAO1, + BrAPIStudyDAO brAPIStudyDAO, + BrAPIEndpointProvider brAPIEndpointProvider, + BrAPIObservationDAO brAPIObservationDAO) { this.programService = programService; this.programDAO = programDAO1; this.brAPIStudyDAO = brAPIStudyDAO; this.brAPIEndpointProvider = brAPIEndpointProvider; + this.observationDAO = brAPIObservationDAO; } @Get("/observations") @@ -93,8 +106,99 @@ public HttpResponse observationsGet(@PathVariable("programId") UUID programId, @Nullable @QueryValue("externalReferenceSource") String externalReferenceSource, @Nullable @QueryValue("page") Integer page, @Nullable @QueryValue("pageSize") Integer pageSize) { - //TODO - return HttpResponse.notFound(); + log.debug("observationsGet: fetching observations by filters"); + Optional program = programService.getById(programId); + if(program.isEmpty()) { + log.warn("Program id: " + programId + " not found"); + return HttpResponse.notFound(); + } + + try { + + // TODO: BI-2506 - implement support for all query parameters. + List unsupportedParams = Stream.of( + observationDbId, + observationUnitDbId, + observationVariableDbId, + locationDbId, + seasonDbId, + observationTimeStampRangeStart, + observationTimeStampRangeEnd, + observationUnitLevelName, + observationUnitLevelOrder, + observationUnitLevelCode, + observationUnitLevelRelationshipName, + observationUnitLevelRelationshipOrder, + observationUnitLevelRelationshipCode, + observationUnitLevelRelationshipDbId, + commonCropName, + programDbId, + trialDbId, + germplasmDbId, + externalReferenceID, + externalReferenceId, + externalReferenceSource + ).filter(Objects::nonNull).collect(Collectors.toList()); + + if (!unsupportedParams.isEmpty()) { + return HttpResponse.status(HttpStatus.NOT_IMPLEMENTED).body( + new BrAPIObservationListResponse().metadata(new BrAPIMetadata().status(List.of(new BrAPIStatus().messageType(BrAPIStatus.MessageTypeEnum.ERROR) + .message("Unsupported query parameter. Only studyDbId, page, and pageSize are supported.")))) + ); + } + + // Get a filtered list of observations. + List observations = observationDAO.getObservationsByFilters(program.get(), studyDbId); + + // If page is not provided, set it to the default value 0. + if (page == null) page = 0; + // If pageSize is not provided, set it to the default value 1000. + if (pageSize == null) pageSize = 1000; + + // Total number of records in the unpaged super set. + int totalCount = observations.size(); + // The least of pageSize and totalCount, unless pageSize is zero, in which case use totalCount. + int requestedPageSize = pageSize > 0 ? Math.min(pageSize, totalCount) : totalCount; + // Integer division and round up. + int totalPages = totalCount / requestedPageSize + ((totalCount % requestedPageSize == 0) ? 0 : 1); + log.info("(Pagination) totalCount: " + totalCount + " page (0-indexed): " + page + " requestedPageSize: " + requestedPageSize + " totalPages: " + totalPages); + + // Determine validity of pagination query parameters. + boolean pageSizeValid = pageSize > 0; + boolean pageValid = page >= 0 && page < totalPages; + + // Only paginate if valid pagination values were sent. + if (pageSizeValid && pageValid) { + int start = page * requestedPageSize; + // Account for last page, which may have fewer than requestedPageSize items, or exactly requestedPageSize items. + int end = (page == (totalPages - 1) && totalCount % requestedPageSize != 0) ? (start + (totalCount % requestedPageSize)) : Math.min(((page + 1) * requestedPageSize), totalCount); + log.info("(Pagination) start " + start + " end " + end); + // Sort observations so that paging is consistent and coherent. + observations.sort(Comparator.comparing(BrAPIObservation::getObservationDbId)); + // Paginate response. + observations = observations.subList(start, end); + } else { + String errorMessage = "Invalid query parameters: page, pageSize"; + return HttpResponse.badRequest(new BrAPIObservationListResponse().metadata(new BrAPIMetadata().status(List.of(new BrAPIStatus().messageType(BrAPIStatus.MessageTypeEnum.ERROR) + .message(errorMessage))))); + } + + return HttpResponse.ok(new BrAPIObservationListResponse().metadata(new BrAPIMetadata().pagination(new BrAPIIndexPagination() + .currentPage(page) + .totalPages(totalPages) + .pageSize(observations.size()) + .totalCount(totalCount))) + .result(new BrAPIObservationListResponseResult().data(observations))); + + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + return HttpResponse.serverError(new BrAPIObservationListResponse().metadata(new BrAPIMetadata().status(List.of(new BrAPIStatus().messageType(BrAPIStatus.MessageTypeEnum.ERROR) + .message("Error fetching observations"))))); + } catch (Exception e) { + log.error("Error fetching Observations", e); + return HttpResponse.serverError(new BrAPIObservationListResponse().metadata(new BrAPIMetadata().status(List.of(new BrAPIStatus().messageType(BrAPIStatus.MessageTypeEnum.ERROR) + .message("Error fetching observations"))))); + } } @Get("/observations/{observationDbId}") diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java index fdfbd18a1..d0217748a 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java @@ -38,7 +38,10 @@ import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.daos.cache.ProgramCacheProvider; import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import org.breedinginsight.services.TraitService; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; +import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.BrAPIDAOUtil; import org.breedinginsight.utilities.Utilities; import org.jetbrains.annotations.NotNull; @@ -62,6 +65,7 @@ public class BrAPIObservationDAO extends BrAPICachedDAO { private final BrAPIEndpointProvider brAPIEndpointProvider; private final String referenceSource; private boolean runScheduledTasks; + private final TraitService traitService; @Inject public BrAPIObservationDAO(ProgramDAO programDAO, @@ -71,7 +75,7 @@ public BrAPIObservationDAO(ProgramDAO programDAO, BrAPIEndpointProvider brAPIEndpointProvider, @Property(name = "brapi.server.reference-source") String referenceSource, @Property(name = "micronaut.bi.api.run-scheduled-tasks") boolean runScheduledTasks, - ProgramCacheProvider programCacheProvider) { + ProgramCacheProvider programCacheProvider, TraitService traitService) { this.programDAO = programDAO; this.importDAO = importDAO; this.observationUnitDAO = observationUnitDAO; @@ -79,6 +83,7 @@ public BrAPIObservationDAO(ProgramDAO programDAO, this.brAPIEndpointProvider = brAPIEndpointProvider; this.referenceSource = referenceSource; this.runScheduledTasks = runScheduledTasks; + this.traitService = traitService; this.programCache = programCacheProvider.getProgramCache(this::fetchProgramObservations, BrAPIObservation.class); } @@ -242,6 +247,42 @@ public List getObservationsByObservationUnitsAndStudies(Collec .collect(Collectors.toList()); } + // TODO: implement other filters in BI-2506. + public List getObservationsByFilters(Program program, String studyDbId) throws ApiException, DoesNotExistException { + + String studySource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.STUDIES); + String observationUnitSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS); + String observationSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATIONS); + + // Get all observations for the program. + Collection observations = getProgramObservations(program.getId()).values(); + // Build a hashmap of traits for fast lookup. The key is ObservationVariableDbId, the value is the Trait Id. + HashMap traitIdsByObservationVariableDbId = traitService.getIdsByObservationVariableDbIds(program.getId(), observations.stream().map(BrAPIObservation::getObservationVariableDbId).collect(Collectors.toList())); + + // Lookup studyDbId. + return observations.stream() + .filter(o -> { + // Short circuit if filter is null. + if (studyDbId == null) return true; + Optional xref = Utilities.getExternalReference(o.getExternalReferences(), studySource); + return xref.filter(brAPIExternalReference -> studyDbId.equals(brAPIExternalReference.getReferenceId())).isPresent(); + }) + .peek(o -> { + // Translate ObservationVariableDbId. + o.setObservationVariableDbId(traitIdsByObservationVariableDbId.get(o.getObservationVariableDbId())); + // Translate ObservationUnitDbId. + o.setObservationUnitDbId(Utilities.getExternalReference(o.getExternalReferences(), observationUnitSource) + .orElseThrow(() -> new RuntimeException("observationUnit xref not found on observation")).getReferenceId()); + // Translate ObservationId. + o.setObservationDbId(Utilities.getExternalReference(o.getExternalReferences(), observationSource) + .orElseThrow(() -> new RuntimeException("observation xref not found on observation")).getReferenceId()); + // Translate StudyDbId. + o.setStudyDbId(Utilities.getExternalReference(o.getExternalReferences(), studySource) + .orElseThrow(() -> new RuntimeException("study xref not found on observation")).getReferenceId()); + // TODO: consider translating germplasmDbId in BI-2506. + }).collect(Collectors.toList()); + } + @NotNull private ApiResponse, Optional>> searchObservationsSearchResultsDbIdGet(UUID programId, String searchResultsDbId, Integer page, Integer pageSize) throws ApiException { ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationsApi.class); diff --git a/src/main/java/org/breedinginsight/services/TraitService.java b/src/main/java/org/breedinginsight/services/TraitService.java index 8394acd28..cdedac3ba 100644 --- a/src/main/java/org/breedinginsight/services/TraitService.java +++ b/src/main/java/org/breedinginsight/services/TraitService.java @@ -491,4 +491,29 @@ public List getByName(UUID programId, List names) throws DoesNotE return traitDAO.getTraitsByTraitName(programId, names.stream().map(name -> Trait.builder().observationVariableName(name).build()).collect(Collectors.toList())); } + + public Trait getByObservationVariableDbId(UUID programId, String observationVariableDbId) throws DoesNotExistException { + if (!programService.exists(programId)) { + throw new DoesNotExistException("Program does not exist"); + } + + return traitDAO.getTraitsFullByProgramId(programId).stream() + .filter(t -> t.getObservationVariableDbId().equals(observationVariableDbId)) + .findFirst().orElseThrow(() -> new DoesNotExistException("Trait not found for observationVariableDbId: " + observationVariableDbId)); + + } + + public HashMap getIdsByObservationVariableDbIds(UUID programId, List observationVariableDbIds) throws DoesNotExistException { + if (!programService.exists(programId)) { + throw new DoesNotExistException("Program does not exist"); + } + return traitDAO.getTraitsFullByProgramId(programId).stream() + .filter(t -> observationVariableDbIds.contains(t.getObservationVariableDbId())) + .collect(Collectors.toMap( + Trait::getObservationVariableDbId, + (t) -> t.getId().toString(), + (existing, replacement) -> existing, + HashMap::new + )); + } } diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java index b9ba07d43..27cf6defd 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java @@ -63,6 +63,7 @@ import java.util.*; import static io.micronaut.http.HttpRequest.*; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertEquals; @MicronautTest @@ -71,8 +72,10 @@ public class BrAPIObservationsControllerIntegrationTest extends BrAPITest { private Program program; - private String experimentId; + private String experimentId; private List envIds = new ArrayList<>(); + // Use hardcoded values for more deterministic test runs and easier assertions. + private List values = List.of(0.125F, 12.415F); private final List> rows = new ArrayList<>(); private final List columns = ExperimentFileColumns.getOrderedColumns(); private List traits; @@ -170,15 +173,12 @@ void setup() throws Exception { Map row2 = makeExpImportRow("Env2"); // Add test observation data - for (Trait trait : traits) { - Random random = new Random(); - + for (int i = 0; i < traits.size(); i++) { // TODO: test for sending obs data as double. // A float is returned from the backend instead of double. there is a separate card to fix this. - // Double val1 = Math.random(); - - Float val1 = random.nextFloat(); - row1.put(trait.getObservationVariableName(), val1); + int valueIndex = i % values.size(); // Prevent overflow. + row1.put(traits.get(i).getObservationVariableName(), values.get(valueIndex)); + row2.put(traits.get(i).getObservationVariableName(), values.get(valueIndex)); } rows.add(row1); @@ -273,7 +273,7 @@ public void testGetObsTableOK() { } @Test - @Disabled("Disabled until fetching of observations is implemented") + @Disabled("Disabled until fetching of observations is implemented in BI-2506.") public void testGetObsListByExpId() { Flowable> call = client.exchange( GET(String.format("/programs/%s/brapi/v2/observations?trialDbId=%s", program.getId(), experimentId)) @@ -286,7 +286,7 @@ public void testGetObsListByExpId() { } @Test - @Disabled("Disabled until fetching of observations is implemented") + @Disabled("Disabled until fetching of observations is implemented in BI-2506.") public void testGetOUById() { Flowable> call = client.exchange( GET(String.format("/programs/%s/brapi/v2/observations?trialDbId=%s", program.getId(), experimentId)) @@ -311,6 +311,99 @@ public void testGetOUById() { assertEquals(HttpStatus.OK, ouResponse.getStatus()); } + @Test + public void testGetObsByStudyDbId() { + // Make a GET request to the /observations endpoint with the studyDbId query parameter. + Flowable> call = client.exchange( + GET(String.format("/programs/%s/brapi/v2/observations?studyDbId=%s", program.getId(), envIds.get(0))) + .bearerAuth("test-registered-user"), + String.class + ); + + // Check for 200 OK response. + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + + // Check that two observations were returned. + JsonObject responseObj = gson.fromJson(response.body(), JsonObject.class); + JsonArray observations = responseObj.getAsJsonObject("result").getAsJsonArray("data"); + assertEquals(2, observations.size()); + + // Check the observation values, keep in mind the order of results is not guaranteed. + Float value1 = observations.get(0).getAsJsonObject().get("value").getAsFloat(); + Float value2 = observations.get(1).getAsJsonObject().get("value").getAsFloat(); + assertTrue(values.contains(value1), "Observation with value " + value1 + " not found."); + assertTrue(values.contains(value2), "Observation with value " + value2 + " not found."); + } + + @Test + public void testGetObsPagination() { + + // Check no pagination. + checkPagination(null, null, HttpStatus.OK, 4, 4, 1); + // Check page and pageSize defaults. + checkPagination(null, 2, HttpStatus.OK, 2, 4, 2); + checkPagination(0, null, HttpStatus.OK, 4, 4, 1); + // Check valid pagination, including last page edge case. + checkPagination(0, 1, HttpStatus.OK, 1, 4, 4); + checkPagination(1, 2, HttpStatus.OK, 2, 4, 2); + checkPagination(0, 3, HttpStatus.OK, 3, 4, 2); + checkPagination(1, 3, HttpStatus.OK, 1, 4, 2); + checkPagination(0, 100, HttpStatus.OK, 4, 4, 1); + // Check invalid pagination. + checkPagination(2, 2, HttpStatus.BAD_REQUEST, null, null, null); + checkPagination(0, 0, HttpStatus.BAD_REQUEST, null, null, null); + checkPagination(1, 0, HttpStatus.BAD_REQUEST, null, null, null); + checkPagination(10, 100, HttpStatus.BAD_REQUEST, null, null, null); + + } + + private void checkPagination(Integer page, Integer pageSize, HttpStatus expectedStatus, Integer expectedSize, Integer expectedTotalCount, Integer expectedTotalPages) { + // Build request URL. + String requestURL = String.format("/programs/%s/brapi/v2/observations", program.getId()); + if (page != null) { + requestURL = requestURL + "?page=" + page; + } + if (pageSize != null && page == null) { + requestURL = requestURL + "?pageSize=" + pageSize; + } else if (pageSize != null) { + requestURL = requestURL + "&pageSize=" + pageSize; + } + + // Make a GET request to the /observations endpoint with the supplied pagination parameters. + Flowable> call = client.exchange( + GET(requestURL).bearerAuth("test-registered-user"), + String.class + ); + + // Check for expected response. + try { + HttpResponse response = call.blockingFirst(); + assertEquals(expectedStatus, response.getStatus()); + + // If call.blockingFirst() doesn't throw, expect a 200 OK. + assertEquals(HttpStatus.OK, response.getStatus()); + + // Parse and check body and metadata. + JsonObject responseObj = gson.fromJson(response.body(), JsonObject.class); + // Get metadata. + JsonObject pagination = responseObj.getAsJsonObject("metadata").getAsJsonObject("pagination"); + assertEquals(expectedSize, pagination.get("pageSize").getAsInt()); + int expectedPage = page == null ? 0 : page; + assertEquals(expectedPage, pagination.get("currentPage").getAsInt()); + assertEquals(expectedTotalPages, pagination.get("totalPages").getAsInt()); + assertEquals(expectedTotalCount, pagination.get("totalCount").getAsInt()); + // Get observations. + JsonArray observations = responseObj.getAsJsonObject("result").getAsJsonArray("data"); + assertEquals(expectedSize, observations.size()); + + } catch (HttpClientResponseException e) { + // call.blockingFirst() will throw in the case of a non-200 code. + assertEquals(expectedStatus, e.getStatus()); + } + + } + private File writeDataToFile(List> data, List traits) throws IOException { File file = File.createTempFile("test", ".csv");