feat: Filter events by existing data element [DHIS2-15954]

dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/event/DefaultEventService.java

@@ -128,7 +128,12 @@ private FileResource getFileResourceMetadata(UID eventUid, UID dataElementUid)
           "this must be a bug in how the EventOperationParams are built");
     }
     if (events.getItems().isEmpty()) {
-      throw new NotFoundException(Event.class, eventUid);
+      throw new NotFoundException(
+          "Event "
+              + eventUid.getValue()
+              + " with data element "
+              + dataElementUid.getValue()
+              + " could not be found.");
     }
     Event event = events.getItems().get(0);
 

dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/event/EventQueryParams.java

@@ -414,7 +414,6 @@ public EventQueryParams orderBy(String field, SortDirection direction) {
   /** Order by the given data element {@code de} in given sort {@code direction}. */
   public EventQueryParams orderBy(DataElement de, SortDirection direction) {
     this.order.add(new Order(de, direction));
-    this.dataElements.putIfAbsent(de, new ArrayList<>());
     return this;
   }
 

dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/event/JdbcEventStore.java

@@ -893,16 +893,8 @@ private String getEventSelectQuery(
             .append(COLUMN_EVENT_ATTRIBUTE_OPTION_COMBO_NAME)
             .append(", coc_agg.attributevalues as ")
             .append(COLUMN_EVENT_ATTRIBUTE_OPTION_COMBO_ATTRIBUTE_VALUES)
-            .append(", coc_agg.co_values AS co_values, coc_agg.co_count AS option_size, ");
-
-    for (Order order : params.getOrder()) {
-      if (order.getField() instanceof TrackedEntityAttribute tea)
-        selectBuilder
-            .append(quote(tea.getUid()))
-            .append(".value AS ")
-            .append(tea.getUid())
-            .append("_value, ");
-    }
+            .append(", coc_agg.co_values AS co_values, coc_agg.co_count AS option_size, ")
+            .append(addOrderFieldsToSelectClause(params.getOrder()));
 
     return selectBuilder
         .append(
@@ -928,6 +920,32 @@ private String getEventSelectQuery(
         .toString();
   }
 
+  private String addOrderFieldsToSelectClause(List<Order> orders) {
+    StringBuilder selectBuilder = new StringBuilder();
+
+    for (Order order : orders) {
+      if (order.getField() instanceof TrackedEntityAttribute tea) {
+        selectBuilder
+            .append(quote(tea.getUid()))
+            .append(".value AS ")
+            .append(tea.getUid())
+            .append("_value, ");
+      } else if (order.getField() instanceof DataElement de) {
+        final String dataValueValueSql = "ev.eventdatavalues #>> '{" + de.getUid() + ", value}'";
+        selectBuilder
+            .append(
+                de.getValueType().isNumeric()
+                    ? castToNumber(dataValueValueSql)
+                    : lower(dataValueValueSql))
+            .append(" as ")
+            .append(de.getUid())
+            .append(", ");
+      }
+    }
+
+    return selectBuilder.toString();
+  }
+
   private boolean checkForOwnership(EventQueryParams params) {
     return Optional.ofNullable(params.getProgram())
         .filter(
@@ -1444,6 +1462,11 @@ private StringBuilder dataElementAndFiltersSql(
             }
           }
         }
+      } else {
+        eventDataValuesWhereSql.append(hlp.whereAnd());
+        eventDataValuesWhereSql.append(" (ev.eventdatavalues ?? '");
+        eventDataValuesWhereSql.append(deUid);
+        eventDataValuesWhereSql.append("')");
       }
     }
 

dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/export/event/EventQueryParamsTest.java

@@ -70,13 +70,13 @@ void shouldKeepExistingAttributeFiltersWhenOrderingByAttribute() {
   }
 
   @Test
-  void shouldAddDataElementToOrderAndDataElementsWhenOrderingByDataElement() {
+  void shouldAddDataElementToOrderButNotToDataElementsWhenOrderingByDataElement() {
     EventQueryParams params = new EventQueryParams();
 
     params.orderBy(de1, SortDirection.ASC);
 
     assertEquals(List.of(new Order(de1, SortDirection.ASC)), params.getOrder());
-    assertEquals(Map.of(de1, List.of()), params.getDataElements());
+    assertTrue(params.getDataElements().isEmpty());
     assertFalse(params.hasDataElementFilter());
   }
 

dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/export/event/EventExporterTest.java

@@ -42,6 +42,7 @@
 import java.io.IOException;
 import java.time.ZoneId;
 import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
 import java.util.Map;
@@ -590,6 +591,25 @@ void testExportEventsWhenFilteringByNumericDataElements()
     assertContainsOnly(List.of("D9PbzJY8bJM"), events);
   }
 
+  @Test
+  void shouldFilterByEventsWithGivenDataValuesWhenFilterContainsDataElementUIDsOnly()
+      throws ForbiddenException, BadRequestException {
+    EventOperationParams params =
+        EventOperationParams.builder()
+            .eventParams(EventParams.FALSE)
+            .dataElementFilters(
+                Map.of(
+                    UID.of("GieVkTxp4HH"),
+                    new ArrayList<>(),
+                    UID.of("GieVkTxp4HG"),
+                    new ArrayList<>()))
+            .build();
+
+    List<String> events = getEvents(params);
+
+    assertContainsOnly(List.of("kWjSezkXHVp"), events);
+  }
+
   @Test
   void testEnrollmentEnrolledBeforeSetToBeforeFirstEnrolledAtDate()
       throws ForbiddenException, BadRequestException {

dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportControllerTest.java

@@ -534,7 +534,6 @@ void getDataValuesFileByDataElementIfUserDoesNotHaveReadAccessToDataElement()
     event.getEventDataValues().add(dataValue(de, file.getUid()));
     manager.update(event);
 
-    // this.switchContextToUser(user);
     switchContextToUser(user);
 
     GET("/tracker/events/{eventUid}/dataValues/{dataElementUid}/file", event.getUid(), de.getUid())
@@ -568,6 +567,25 @@ void getDataValuesFileByDataElementIfNoDataValueExists() {
 
     switchContextToUser(user);
 
+    assertEquals(
+        "Event " + event.getUid() + " with data element " + de.getUid() + " could not be found.",
+        GET(
+                "/tracker/events/{eventUid}/dataValues/{dataElementUid}/file",
+                event.getUid(),
+                de.getUid())
+            .error(HttpStatus.NOT_FOUND)
+            .getMessage());
+  }
+
+  @Test
+  void getDataValuesFileByDataElementIfNoDataValueNull() {
+    Event event = event(enrollment(trackedEntity()));
+    DataElement de = dataElement(ValueType.IMAGE);
+
+    event.getEventDataValues().add(dataValue(de, null));
+    manager.flush();
+    switchContextToUser(user);
+
     assertEquals(
         "DataValue for data element " + de.getUid() + " could not be found.",
         GET(