feat: Filter events by DE existence in event [DHIS2-15954]

dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/QueryFilter.java

@@ -43,6 +43,8 @@
 import static org.hisp.dhis.common.QueryOperator.NIEQ;
 import static org.hisp.dhis.common.QueryOperator.NILIKE;
 import static org.hisp.dhis.common.QueryOperator.NLIKE;
+import static org.hisp.dhis.common.QueryOperator.NNULL;
+import static org.hisp.dhis.common.QueryOperator.NULL;
 import static org.hisp.dhis.common.QueryOperator.SW;
 
 import com.google.common.collect.ImmutableMap;
@@ -80,6 +82,8 @@ public class QueryFilter {
           .put(EW, unused -> "like")
           .put(NLIKE, unused -> "not like")
           .put(IN, unused -> "in")
+          .put(NULL, unused -> NULL.getValue())
+          .put(NNULL, unused -> NNULL.getValue())
           .build();
 
   protected QueryOperator operator;
@@ -92,6 +96,10 @@ public class QueryFilter {
 
   public QueryFilter() {}
 
+  public QueryFilter(QueryOperator operator) {
+    this.operator = operator;
+  }
+
   public QueryFilter(QueryOperator operator, String filter) {
     this.operator = operator;
     this.filter = filter;
@@ -105,10 +113,6 @@ public boolean isFilter() {
     return operator != null && filter != null && !filter.isEmpty();
   }
 
-  public boolean isOperator(QueryOperator op) {
-    return operator != null && operator.equals(op);
-  }
-
   public String getSqlOperator() {
     return getSqlOperator(false);
   }

dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/QueryOperator.java

@@ -51,6 +51,8 @@ public enum QueryOperator {
   IN("in", true),
   SW("sw"),
   EW("ew"),
+  NULL("is null"),
+  NNULL("is not null"),
   // Analytics specifics
   IEQ("==", true),
   NE("!=", true),
@@ -68,6 +70,8 @@ public enum QueryOperator {
 
   private static final Set<QueryOperator> COMPARISON_OPERATORS = EnumSet.of(GT, GE, LT, LE);
 
+  private static final Set<QueryOperator> UNARY_OPERATORS = EnumSet.of(NULL, NNULL);
+
   private final String value;
 
   private final boolean nullAllowed;
@@ -113,4 +117,8 @@ public boolean isIn() {
   public boolean isComparison() {
     return COMPARISON_OPERATORS.contains(this);
   }
+
+  public boolean isUnary() {
+    return UNARY_OPERATORS.contains(this);
+  }
 }

dhis-2/dhis-api/src/main/java/org/hisp/dhis/programstagefilter/EventDataFilter.java

@@ -65,6 +65,12 @@ public class EventDataFilter implements Serializable {
   /** Like */
   private String like;
 
+  /** Null */
+  private String nullFilter;
+
+  /** Not Null */
+  private String notNullFilter;
+
   /** If the dataItem is of type date, then date filtering parameters are specified using this. */
   private DateFilterPeriod dateFilter;
 
@@ -158,6 +164,31 @@ public void setLike(String like) {
     this.like = like;
   }
 
+  @JsonProperty
+  @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0)
+  public String getNull() {
+    return nullFilter;
+  }
+
+  public void setNull(String nullFilter) {
+    if (nullFilter != null) {
+      this.nullFilter = "true";
+    }
+  }
+
+  @JsonProperty("!null")
+  @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0)
+  public String getNotNull() {
+    return notNullFilter;
+  }
+
+  @JsonProperty("!null")
+  public void setNotNull(String notNullFilter) {
+    if (notNullFilter != null) {
+      this.notNullFilter = "true";
+    }
+  }
+
   @JsonProperty
   @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0)
   public DateFilterPeriod getDateFilter() {

dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/event/EventQueryParams.java

@@ -42,6 +42,7 @@
 import org.hisp.dhis.common.AssignedUserQueryParam;
 import org.hisp.dhis.common.OrganisationUnitSelectionMode;
 import org.hisp.dhis.common.QueryFilter;
+import org.hisp.dhis.common.QueryOperator;
 import org.hisp.dhis.common.SortDirection;
 import org.hisp.dhis.common.UID;
 import org.hisp.dhis.dataelement.DataElement;
@@ -473,7 +474,7 @@ public EventQueryParams filterBy(DataElement de, QueryFilter filter) {
   }
 
   public EventQueryParams filterBy(DataElement de) {
-    this.dataElements.putIfAbsent(de, new ArrayList<>());
+    this.dataElements.putIfAbsent(de, List.of(new QueryFilter(QueryOperator.NNULL)));
     return this;
   }
 

dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/event/JdbcEventStore.java

@@ -1287,10 +1287,7 @@ private StringBuilder dataElementAndFiltersSql(
       StringBuilder selectBuilder) {
     int filterCount = 0;
 
-    StringBuilder optionValueJoinBuilder = new StringBuilder();
-    StringBuilder optionValueConditionBuilder = new StringBuilder();
     StringBuilder eventDataValuesWhereSql = new StringBuilder();
-    Set<String> joinedColumns = new HashSet<>();
 
     for (Entry<DataElement, List<QueryFilter>> item : params.getDataElements().entrySet()) {
       ++filterCount;
@@ -1310,105 +1307,57 @@ private StringBuilder dataElementAndFiltersSql(
           .append(" as ")
           .append(deUid);
 
-      String optValueTableAs = "opt_" + filterCount;
-
-      if (!joinedColumns.contains(deUid) && de.hasOptionSet() && !filters.isEmpty()) {
-        String optSetBind = "optset_" + filterCount;
-
-        mapSqlParameterSource.addValue(optSetBind, de.getOptionSet().getId());
-
-        optionValueJoinBuilder
-            .append("inner join optionvalue as ")
-            .append(optValueTableAs)
-            .append(" on lower(")
-            .append(optValueTableAs)
-            .append(".code) = ")
-            .append("lower(")
-            .append(dataValueValueSql)
-            .append(") and ")
-            .append(optValueTableAs)
-            .append(".optionsetid = ")
-            .append(":")
-            .append(optSetBind)
-            .append(" ");
-
-        joinedColumns.add(deUid);
-      }
-
-      if (!filters.isEmpty()) {
-        for (QueryFilter filter : filters) {
-          ++filterCount;
-
-          final String queryCol =
-              de.getValueType().isNumeric()
-                  ? castToNumber(dataValueValueSql)
-                  : lower(dataValueValueSql);
+      for (QueryFilter filter : filters) {
+        ++filterCount;
 
-          String bindParameter = "parameter_" + filterCount;
-          int itemType = de.getValueType().isNumeric() ? Types.NUMERIC : Types.VARCHAR;
+        final String queryCol =
+            de.getValueType().isNumeric()
+                ? castToNumber(dataValueValueSql)
+                : lower(dataValueValueSql);
 
-          if (!de.hasOptionSet()) {
-            eventDataValuesWhereSql.append(hlp.whereAnd());
+        String bindParameter = "parameter_" + filterCount;
+        int itemType = de.getValueType().isNumeric() ? Types.NUMERIC : Types.VARCHAR;
 
-            if (QueryOperator.IN.getValue().equalsIgnoreCase(filter.getSqlOperator())) {
-              mapSqlParameterSource.addValue(
-                  bindParameter,
-                  QueryFilter.getFilterItems(StringUtils.lowerCase(filter.getFilter())),
-                  itemType);
+        eventDataValuesWhereSql.append(hlp.whereAnd());
 
-              eventDataValuesWhereSql.append(inCondition(filter, bindParameter, queryCol));
-            } else {
-              mapSqlParameterSource.addValue(
-                  bindParameter, StringUtils.lowerCase(filter.getSqlBindFilter()), itemType);
-
-              eventDataValuesWhereSql
-                  .append(" ")
-                  .append(queryCol)
-                  .append(" ")
-                  .append(filter.getSqlOperator())
-                  .append(" ")
-                  .append(":")
-                  .append(bindParameter)
-                  .append(" ");
-            }
-          } else {
-            if (QueryOperator.IN.getValue().equalsIgnoreCase(filter.getSqlOperator())) {
-              mapSqlParameterSource.addValue(
-                  bindParameter,
-                  QueryFilter.getFilterItems(StringUtils.lowerCase(filter.getFilter())),
-                  itemType);
-
-              optionValueConditionBuilder.append(" and ");
-              optionValueConditionBuilder.append(inCondition(filter, bindParameter, queryCol));
-            } else {
-              mapSqlParameterSource.addValue(
-                  bindParameter, StringUtils.lowerCase(filter.getSqlBindFilter()), itemType);
-
-              optionValueConditionBuilder
-                  .append("and lower(")
-                  .append(optValueTableAs)
-                  .append(DOT_NAME)
-                  .append(" ")
-                  .append(filter.getSqlOperator())
-                  .append(" ")
-                  .append(":")
-                  .append(bindParameter)
-                  .append(" ");
-            }
-          }
+        if (filter.getOperator().isUnary()) {
+          eventDataValuesWhereSql.append(unaryOperatorCondition(filter.getOperator(), deUid));
+        } else if (QueryOperator.IN.getValue().equalsIgnoreCase(filter.getSqlOperator())) {
+          mapSqlParameterSource.addValue(
+              bindParameter,
+              QueryFilter.getFilterItems(StringUtils.lowerCase(filter.getFilter())),
+              itemType);
+
+          eventDataValuesWhereSql.append(inCondition(filter, bindParameter, queryCol));
+        } else {
+          mapSqlParameterSource.addValue(
+              bindParameter, StringUtils.lowerCase(filter.getSqlBindFilter()), itemType);
+
+          eventDataValuesWhereSql
+              .append(" ")
+              .append(queryCol)
+              .append(" ")
+              .append(filter.getSqlOperator())
+              .append(" ")
+              .append(":")
+              .append(bindParameter)
+              .append(" ");
         }
-      } else {
-        eventDataValuesWhereSql.append(hlp.whereAnd());
-        eventDataValuesWhereSql.append(" (ev.eventdatavalues ?? '");
-        eventDataValuesWhereSql.append(deUid);
-        eventDataValuesWhereSql.append("')");
       }
     }
 
-    return optionValueJoinBuilder
-        .append(optionValueConditionBuilder)
-        .append(eventDataValuesWhereSql)
-        .append(" ");
+    return eventDataValuesWhereSql.append(" ");
+  }
+
+  private String unaryOperatorCondition(QueryOperator queryOperator, String deUid) {
+    return new StringBuilder()
+        .append(" ev.eventdatavalues->")
+        .append("'")
+        .append(deUid)
+        .append("' ")
+        .append(queryOperator.getValue())
+        .append(" ")
+        .toString();
   }
 
   private String inCondition(QueryFilter filter, String boundParameter, String queryCol) {

dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/export/event/EventOperationParamsMapperTest.java

@@ -364,7 +364,9 @@ void shouldMapDataElementFilters() throws BadRequestException, ForbiddenExceptio
             .dataElementFilters(
                 Map.of(
                     UID.of(DE_1_UID),
-                    List.of(new QueryFilter(QueryOperator.EQ, "2")),
+                    List.of(
+                        new QueryFilter(QueryOperator.EQ, "2"),
+                        new QueryFilter(QueryOperator.NNULL)),
                     UID.of(DE_2_UID),
                     List.of(new QueryFilter(QueryOperator.LIKE, "foo"))))
             .build();
@@ -376,7 +378,7 @@ void shouldMapDataElementFilters() throws BadRequestException, ForbiddenExceptio
     Map<DataElement, List<QueryFilter>> expected =
         Map.of(
             de1,
-            List.of(new QueryFilter(QueryOperator.EQ, "2")),
+            List.of(new QueryFilter(QueryOperator.EQ, "2"), new QueryFilter(QueryOperator.NNULL)),
             de2,
             List.of(new QueryFilter(QueryOperator.LIKE, "foo")));
     assertEquals(expected, dataElements);