feat: Filter events by DE existence in event [DHIS2-15954]

dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/QueryFilter.java

@@ -30,6 +30,7 @@
 import static org.hisp.dhis.analytics.QueryKey.NV;
 import static org.hisp.dhis.common.QueryOperator.EQ;
 import static org.hisp.dhis.common.QueryOperator.EW;
+import static org.hisp.dhis.common.QueryOperator.EX;
 import static org.hisp.dhis.common.QueryOperator.GE;
 import static org.hisp.dhis.common.QueryOperator.GT;
 import static org.hisp.dhis.common.QueryOperator.IEQ;
@@ -80,6 +81,7 @@ public class QueryFilter {
           .put(EW, unused -> "like")
           .put(NLIKE, unused -> "not like")
           .put(IN, unused -> "in")
+          .put(EX, unused -> "??")
           .build();
 
   protected QueryOperator operator;
@@ -92,6 +94,10 @@ public class QueryFilter {
 
   public QueryFilter() {}
 
+  public QueryFilter(QueryOperator operator) {
+    this.operator = operator;
+  }
+
   public QueryFilter(QueryOperator operator, String filter) {
     this.operator = operator;
     this.filter = filter;

dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/QueryOperator.java

@@ -51,6 +51,7 @@ public enum QueryOperator {
   IN("in", true),
   SW("sw"),
   EW("ew"),
+  EX("??"),
   // Analytics specifics
   IEQ("==", true),
   NE("!=", true),

dhis-2/dhis-api/src/main/java/org/hisp/dhis/programstagefilter/EventDataFilter.java

@@ -65,6 +65,9 @@ public class EventDataFilter implements Serializable {
   /** Like */
   private String like;
 
+  /** Exists */
+  private String ex;
+
   /** If the dataItem is of type date, then date filtering parameters are specified using this. */
   private DateFilterPeriod dateFilter;
 
@@ -158,6 +161,16 @@ public void setLike(String like) {
     this.like = like;
   }
 
+  @JsonProperty
+  @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0)
+  public String getEx() {
+    return ex;
+  }
+
+  public void setEx(String ex) {
+    this.ex = ex;
+  }
+
   @JsonProperty
   @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0)
   public DateFilterPeriod getDateFilter() {

dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/event/DefaultEventService.java

@@ -40,6 +40,8 @@
 import org.hisp.dhis.common.IdentifiableObjectManager;
 import org.hisp.dhis.common.IdentifiableProperty;
 import org.hisp.dhis.common.OrganisationUnitSelectionMode;
+import org.hisp.dhis.common.QueryFilter;
+import org.hisp.dhis.common.QueryOperator;
 import org.hisp.dhis.common.UID;
 import org.hisp.dhis.dataelement.DataElement;
 import org.hisp.dhis.dataelement.DataElementService;
@@ -120,7 +122,8 @@ private FileResource getFileResourceMetadata(UID eventUid, UID dataElementUid)
               .orgUnitMode(OrganisationUnitSelectionMode.ACCESSIBLE)
               .events(Set.of(eventUid))
               .eventParams(EventParams.FALSE)
-              .dataElementFilters(Map.of(dataElementUid, List.of()))
+              .dataElementFilters(
+                  Map.of(dataElementUid, List.of(new QueryFilter(QueryOperator.EX, "true"))))
               .build();
       events = getEvents(operationParams, new PageParams(1, 1, false));
     } catch (BadRequestException e) {

dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/event/JdbcEventStore.java

@@ -1375,6 +1375,12 @@ private StringBuilder dataElementAndFiltersSql(
           .append(" as ")
           .append(deUid);
 
+      if (filterContainsExistenceOperator(filters)) {
+        // Operator EX allows for only one item in the filter list and its value is true or false
+        eventDataValuesWhereSql.append(addExistsFilterCondition(filters.get(0), hlp, deUid));
+        break;
+      }
+
       String optValueTableAs = "opt_" + filterCount;
 
       if (!joinedColumns.contains(deUid) && de.hasOptionSet() && !filters.isEmpty()) {
@@ -1476,6 +1482,22 @@ private StringBuilder dataElementAndFiltersSql(
         .append(" ");
   }
 
+  private String addExistsFilterCondition(QueryFilter queryFilter, SqlHelper hlp, String deUid) {
+    StringBuilder existsBuilder = new StringBuilder();
+
+    existsBuilder.append(hlp.whereAnd());
+    if (!Boolean.parseBoolean(queryFilter.getFilter())) {
+      existsBuilder.append(" not ");
+    }
+    existsBuilder.append(" (ev.eventdatavalues ");
+    existsBuilder.append(queryFilter.getSqlOperator());
+    existsBuilder.append(" '");
+    existsBuilder.append(deUid);
+    existsBuilder.append("')");
+
+    return existsBuilder.toString();
+  }
+
   private String inCondition(QueryFilter filter, String boundParameter, String queryCol) {
     return new StringBuilder()
         .append(" ")
@@ -1490,6 +1512,10 @@ private String inCondition(QueryFilter filter, String boundParameter, String que
         .toString();
   }
 
+  private boolean filterContainsExistenceOperator(List<QueryFilter> filters) {
+    return filters.stream().anyMatch(qf -> qf.getOperator().equals(QueryOperator.EX));
+  }
+
   private String eventStatusSql(
       EventQueryParams params, MapSqlParameterSource mapSqlParameterSource, SqlHelper hlp) {
     StringBuilder stringBuilder = new StringBuilder();

dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/export/event/EventExporterTest.java

@@ -42,7 +42,6 @@
 import java.io.IOException;
 import java.time.ZoneId;
 import java.time.temporal.ChronoUnit;
-import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
 import java.util.Map;
@@ -592,23 +591,41 @@ void testExportEventsWhenFilteringByNumericDataElements()
   }
 
   @Test
-  void shouldFilterByEventsWithGivenDataValuesWhenFilterContainsDataElementUIDsOnly()
+  void shouldFilterByEventsContainingGivenDataValuesWhenFilteringByExistence()
       throws ForbiddenException, BadRequestException {
     EventOperationParams params =
         EventOperationParams.builder()
+            .eventParams(EventParams.FALSE)
             .dataElementFilters(
                 Map.of(
                     UID.of("GieVkTxp4HH"),
-                    new ArrayList<>(),
+                    List.of(new QueryFilter(QueryOperator.EX, "true")),
                     UID.of("GieVkTxp4HG"),
-                    new ArrayList<>()))
+                    List.of(new QueryFilter(QueryOperator.EX, "true"))))
             .build();
 
     List<String> events = getEvents(params);
 
     assertContainsOnly(List.of("kWjSezkXHVp"), events);
   }
 
+  @Test
+  void shouldFilterByEventsNotContainingGivenDataValueWhenFilteringByNonexistence()
+      throws ForbiddenException, BadRequestException {
+    EventOperationParams params =
+        EventOperationParams.builder()
+            .enrollments(UID.of("nxP7UnKhomJ", "TvctPPhpD8z"))
+            .programStage(programStage)
+            .eventParams(EventParams.FALSE)
+            .dataElementFilters(
+                Map.of(UID.of("DATAEL00002"), List.of(new QueryFilter(QueryOperator.EX, "false"))))
+            .build();
+
+    List<String> events = getEvents(params);
+
+    assertContainsOnly(List.of("pTzf9KYMk72"), events);
+  }
+
   @Test
   void testEnrollmentEnrolledBeforeSetToBeforeFirstEnrolledAtDate()
       throws ForbiddenException, BadRequestException {

dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/RequestParamsValidator.java

@@ -277,7 +277,7 @@ public static void validateOrderParams(List<OrderCriteria> order, Set<String> su
   /**
    * Validate the {@code filter} request parameter in change log tracker exporters. Allowed filter
    * values are {@code supportedFieldNames}. Only one field name at a time can be specified. If the
-   * endpoint supports UIDs use {@link #parseFilters(String)}.
+   * endpoint supports UIDs use {@link #parseAttributeFilters(String)}.
    */
   public static void validateFilter(String filter, Set<Pair<String, Class<?>>> supportedFields)
       throws BadRequestException {
@@ -324,12 +324,13 @@ public static void validateFilter(String filter, Set<Pair<String, Class<?>>> sup
 
   /**
    * Parse given {@code input} string representing a filter for an object referenced by a UID like a
-   * tracked entity attribute or data element. Refer to {@link #parseSanitizedFilters(Map, String)}}
-   * for details on the expected input format.
+   * tracked entity attribute. Refer to {@link #parseSanitizedFilters(Map, String)}} for details on
+   * the expected input format.
    *
    * @return filters by UIDs
    */
-  public static Map<UID, List<QueryFilter>> parseFilters(String input) throws BadRequestException {
+  public static Map<UID, List<QueryFilter>> parseAttributeFilters(String input)
+      throws BadRequestException {
     Map<UID, List<QueryFilter>> result = new HashMap<>();
     if (StringUtils.isBlank(input)) {
       return result;
@@ -342,9 +343,30 @@ public static Map<UID, List<QueryFilter>> parseFilters(String input) throws BadR
   }
 
   /**
-   * Accumulate {@link QueryFilter}s per UID by parsing given input string of format
-   * {uid}[:{operator}:{value}]. Only the UID is mandatory. Multiple operator:value pairs are
-   * allowed. A {@link QueryFilter} for each operator:value pair is added to the corresponding UID.
+   * Parse given {@code input} string representing a filter for an object referenced by a UID like a
+   * data element. Refer to {@link #parseSanitizedDataElementFilters(Map, String)}} for details on
+   * the expected input format.
+   *
+   * @return filters by UIDs
+   */
+  public static Map<UID, List<QueryFilter>> parseDataElementFilters(String input)
+      throws BadRequestException {
+    Map<UID, List<QueryFilter>> result = new HashMap<>();
+    if (StringUtils.isBlank(input)) {
+      return result;
+    }
+
+    for (String uidOperatorValue : filterList(input)) {
+      parseSanitizedDataElementFilters(result, uidOperatorValue);
+    }
+    return result;
+  }
+
+  /**
+   * Accumulate {@link QueryFilter}s per TEA UID by parsing given input string of format
+   * {uid}[:{operator}:{value}]. Only the TEA UID is mandatory. Multiple operator:value pairs are
+   * allowed. A {@link QueryFilter} for each operator:value pair is added to the corresponding TEA
+   * UID.
    *
    * @throws BadRequestException filter is neither multiple nor single operator:value format
    */
@@ -362,7 +384,64 @@ private static void parseSanitizedFilters(Map<UID, List<QueryFilter>> result, St
     result.putIfAbsent(uid, new ArrayList<>());
 
     String[] filters = FILTER_ITEM_SPLIT.split(input.substring(uidIndex));
+    validateFilterLength(filters, result, uid, input);
+  }
 
+  /**
+   * Accumulate {@link QueryFilter}s per DE UID by parsing given input string of format
+   * {uid}[:{operator}:{value}]. Only the DE ID is mandatory. Multiple operator:value pairs are
+   * allowed. A {@link QueryFilter} for each operator:value pair is added to the corresponding DE
+   * UID.
+   *
+   * @throws BadRequestException filter is neither multiple nor single operator:value format
+   */
+  private static void parseSanitizedDataElementFilters(
+      Map<UID, List<QueryFilter>> result, String input) throws BadRequestException {
+    int uidIndex = input.indexOf(DIMENSION_NAME_SEP) + 1;
+
+    if (uidIndex == 0 || input.length() == uidIndex) {
+      UID uid = UID.of(input.replace(DIMENSION_NAME_SEP, ""));
+      result.putIfAbsent(uid, List.of(new QueryFilter(QueryOperator.EX, "true")));
+      return;
+    }
+    UID uid = UID.of(input.substring(0, uidIndex - 1));
+    String[] filters = FILTER_ITEM_SPLIT.split(input.substring(uidIndex));
+    validateExistenceOperator(filters, result, input, uid);
+    result.putIfAbsent(uid, new ArrayList<>());
+    validateFilterLength(filters, result, uid, input);
+  }
+
+  private static void validateExistenceOperator(
+      String[] filters, Map<UID, List<QueryFilter>> result, String input, UID uid)
+      throws BadRequestException {
+    boolean hasExistenceOperator =
+        result.containsKey(uid)
+            && result.get(uid).stream().anyMatch(qf -> qf.getOperator().equals(QueryOperator.EX));
+
+    for (int i = 0; i < filters.length; i += 2) {
+      String operator = filters[i];
+      String value = (i + 1 < filters.length) ? filters[i + 1] : null;
+
+      if (hasExistenceOperator
+          || (operator.equalsIgnoreCase(QueryOperator.EX.name()) && result.containsKey(uid))) {
+        throw new BadRequestException(
+            "A data element UID filtering with the operator 'EX' cannot be combined with additional filter criteria: "
+                + input);
+      }
+
+      if (operator.equalsIgnoreCase(QueryOperator.EX.name())
+          && !"true".equalsIgnoreCase(value)
+          && !"false".equalsIgnoreCase(value)) {
+        throw new BadRequestException(
+            "A filter with the operator 'EX' can only have 'true' or 'false' as its value: "
+                + input);
+      }
+    }
+  }
+
+  private static void validateFilterLength(
+      String[] filters, Map<UID, List<QueryFilter>> result, UID uid, String input)
+      throws BadRequestException {
     // single operator
     if (filters.length == 2) {
       result.get(uid).add(operatorValueQueryFilter(filters[0], filters[1], input));

dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventRequestParamsMapper.java

@@ -29,7 +29,8 @@
 
 import static java.util.Collections.emptySet;
 import static org.hisp.dhis.util.ObjectUtils.applyIfNotNull;
-import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.parseFilters;
+import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.parseAttributeFilters;
+import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.parseDataElementFilters;
 import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validateDeprecatedParameter;
 import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validateDeprecatedUidsParameter;
 import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validateOrderParams;
@@ -109,9 +110,10 @@ public EventOperationParams map(
             "event", eventRequestParams.getEvent(), "events", eventRequestParams.getEvents());
 
     validateFilter(eventRequestParams.getFilter(), eventUids);
-    Map<UID, List<QueryFilter>> dataElementFilters = parseFilters(eventRequestParams.getFilter());
+    Map<UID, List<QueryFilter>> dataElementFilters =
+        parseDataElementFilters(eventRequestParams.getFilter());
     Map<UID, List<QueryFilter>> attributeFilters =
-        parseFilters(eventRequestParams.getFilterAttributes());
+        parseAttributeFilters(eventRequestParams.getFilterAttributes());
 
     Set<UID> assignedUsers =
         validateDeprecatedUidsParameter(

dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntityRequestParamsMapper.java

@@ -28,7 +28,7 @@
 package org.hisp.dhis.webapi.controller.tracker.export.trackedentity;
 
 import static org.hisp.dhis.util.ObjectUtils.applyIfNotNull;
-import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.parseFilters;
+import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.parseAttributeFilters;
 import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validateDeprecatedParameter;
 import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validateDeprecatedUidsParameter;
 import static org.hisp.dhis.webapi.controller.tracker.export.RequestParamsValidator.validateOrderParams;
@@ -123,7 +123,8 @@ public TrackedEntityOperationParams map(
     validateOrderParams(trackedEntityRequestParams.getOrder(), ORDERABLE_FIELD_NAMES, "attribute");
     validateRequestParams(trackedEntityRequestParams, trackedEntities);
 
-    Map<UID, List<QueryFilter>> filters = parseFilters(trackedEntityRequestParams.getFilter());
+    Map<UID, List<QueryFilter>> filters =
+        parseAttributeFilters(trackedEntityRequestParams.getFilter());
 
     TrackedEntityOperationParamsBuilder builder =
         TrackedEntityOperationParams.builder()