Add support for partial SQL and partial post filtering in WhereBuilder

deegree-core/deegree-core-base/src/main/java/org/deegree/filter/Filters.java

@@ -35,7 +35,9 @@
  ----------------------------------------------------------------------------*/
 package org.deegree.filter;
 
+import static java.util.Arrays.asList;
 import static org.deegree.filter.Filter.Type.OPERATOR_FILTER;
+import static org.deegree.filter.Operator.Type.LOGICAL;
 
 import java.util.Arrays;
 import java.util.Collections;
@@ -117,6 +119,26 @@ public class Filters {
 
     private static Logger LOG = LoggerFactory.getLogger( Filters.class );
 
+    /**
+     * Extract all operators from an And-filter (which can all be filtered individually, as each operator can only
+     * narrow the result).
+     *
+     * @return list of operators, can be <code>null</code> (not an And-filter)
+     */
+    public static List<Operator> extractAndOperands( final Filter filter ) {
+        if ( filter != null && filter.getType() == OPERATOR_FILTER ) {
+            final OperatorFilter operatorFilter = (OperatorFilter) filter;
+            final Operator operator = operatorFilter.getOperator();
+            if ( operator.getType() == LOGICAL ) {
+                final LogicalOperator logicalOperator = (LogicalOperator) operator;
+                if ( logicalOperator.getSubType() == LogicalOperator.SubType.AND ) {
+                    return asList( logicalOperator.getParams() );
+                }
+            }
+        }
+        return null;
+    }
+
     /**
      * Tries to extract a {@link BBOX} constraint from the given {@link Filter} that can be used as a pre-filtering step
      * to narrow the result set.

deegree-core/deegree-core-sqldialect/deegree-sqldialect-commons/src/main/java/org/deegree/sqldialect/filter/AbstractWhereBuilder.java

@@ -38,6 +38,7 @@
 import static java.sql.Types.BOOLEAN;
 import static org.deegree.commons.tom.primitive.BaseType.STRING;
 import static org.deegree.commons.xml.CommonNamespaces.XSINS;
+import static org.deegree.filter.Filters.extractAndOperands;
 import static org.deegree.filter.Filters.extractPrefilterBBoxConstraint;
 
 import java.sql.ResultSet;
@@ -71,6 +72,7 @@
 import org.deegree.filter.expression.Function;
 import org.deegree.filter.expression.Literal;
 import org.deegree.filter.expression.ValueReference;
+import org.deegree.filter.logical.And;
 import org.deegree.filter.logical.LogicalOperator;
 import org.deegree.filter.logical.Not;
 import org.deegree.filter.sort.SortProperty;
@@ -101,16 +103,8 @@
  * {@link #getPostFilter()}/{@link #getPostSortCriteria()} return not <code>null</code> and the objects extracted from
  * the corresponding {@link ResultSet} must be filtered/sorted in memory to guarantee the requested constraints/order.
  * </p>
- * <p>
- * TODO: Implement partial backend filtering / sorting. Currently, filtering / sorting is performed completely by the
- * database <i>or</i> by the post filter / criteria (if any property name has been encountered that could not be
- * mapped).
- * </p>
  * 
  * @author <a href="mailto:[email protected]">Markus Schneider</a>
- * @author last edited by: $Author: mschneider $
- * 
- * @version $Revision: 31370 $, $Date: 2011-07-28 19:37:13 +0200 (Do, 28. Jul 2011) $
  */
 public abstract class AbstractWhereBuilder {
 
@@ -125,7 +119,7 @@ public abstract class AbstractWhereBuilder {
     protected final SortProperty[] sortCrit;
 
     /** Keeps track of all generated table aliases. */
-    protected final TableAliasManager aliasManager = new TableAliasManager();
+    protected TableAliasManager aliasManager = new TableAliasManager();
 
     /** Keeps track of all successfully mapped property names. */
     protected final List<PropertyNameMapping> propNameMappingList = new ArrayList<PropertyNameMapping>();
@@ -161,37 +155,54 @@ protected AbstractWhereBuilder( SQLDialect dialect, PropertyNameMapper mapper, O
     }
 
     /**
-     * Invokes the building of the internal variables that store filter and sort criteria.
+     * Builds internal variables that store filter and sort criteria.
      * 
-     * @param allowPartialMappings
-     *            if false, any unmappable expression will cause an {@link UnmappableException} to be thrown
+     * @param allowPostFiltering
+     *            if <code>false</code>, any unmappable expression will cause an {@link UnmappableException} to be
+     *            thrown
      * @throws FilterEvaluationException
      * @throws UnmappableException
      *             if allowPartialMappings is false and an expression could not be mapped to the db
      */
-    protected void build( boolean allowPartialMappings )
+    protected void build( final boolean allowPostFiltering )
                             throws FilterEvaluationException, UnmappableException {
 
         if ( filter != null ) {
             try {
                 whereClause = toProtoSQL( filter.getOperator() );
             } catch ( UnmappableException e ) {
-                if ( !allowPartialMappings ) {
+                LOG.trace( "Stack trace:", e );
+                if ( !allowPostFiltering ) {
                     throw e;
                 }
-                LOG.debug( "Unable to map full filter to WHERE-clause. Trying mapping of bbox constraint only." );
-                LOG.trace( "Stack trace:", e );
-
-                BBOX preFilterBBox = extractPrefilterBBoxConstraint( filter );
-                if ( preFilterBBox != null ) {
-                    try {
-                        whereClause = toProtoSQL( preFilterBBox );
-                    } catch ( UnmappableException e2 ) {
-                        LOG.warn( "Unable to map any filter constraints to WHERE-clause. Fallback to full memory filtering." );
-                        LOG.trace( "Stack trace:", e2 );
+                LOG.debug( "Unable to map full filter to WHERE-clause. Separating SQL and post filter parts." );
+                propNameMappingList.clear();
+                aliasManager = new TableAliasManager();
+                final List<Operator> andOperands = extractAndOperands( filter );
+                if ( andOperands == null ) {
+                    postFilter = filter;
+                    LOG.debug( "Not an AND filter. Using complete filter as post filter." );
+                } else {
+                    final List<Operator> mappableOperands = new ArrayList<Operator>();
+                    final List<Operator> unmappableOperands = new ArrayList<Operator>();
+                    for ( final Operator operand : andOperands ) {
+                        if ( isMappable( operand ) ) {
+                            mappableOperands.add( operand );
+                        } else {
+                            unmappableOperands.add( operand );
+                        }
+                    }
+                    addPrefilterBBoxIfNoGeometryConstraintMapped( filter, mappableOperands );
+                    final OperatorFilter sqlFilter = createFilter( mappableOperands );
+                    if ( sqlFilter != null ) {
+                        propNameMappingList.clear();
+                        aliasManager = new TableAliasManager();
+                        whereClause = toProtoSQL( sqlFilter.getOperator() );
                     }
+                    postFilter = createFilter( unmappableOperands );
+                    LOG.debug( "SQL filter part: " + mappableOperands.size() + " operand(s)" );
+                    LOG.debug( "Post filter part: " + unmappableOperands.size() + " operand(s)" );
                 }
-                postFilter = filter;
             } catch ( FilterEvaluationException e ) {
                 throw e;
             } catch ( RuntimeException e ) {
@@ -204,7 +215,7 @@ protected void build( boolean allowPartialMappings )
             try {
                 orderByClause = toProtoSQL( sortCrit );
             } catch ( UnmappableException e ) {
-                if ( !allowPartialMappings ) {
+                if ( !allowPostFiltering ) {
                     throw e;
                 }
                 LOG.debug( "Unable to map sort criteria to ORDER BY-clause. Setting post order criteria.", e );
@@ -219,6 +230,37 @@ protected void build( boolean allowPartialMappings )
         }
     }
 
+    private boolean isMappable( final Operator operand ) {
+        try {
+            toProtoSQL( operand );
+        } catch ( final Exception e ) {
+            return false;
+        }
+        return true;
+    }
+
+    private void addPrefilterBBoxIfNoGeometryConstraintMapped( final OperatorFilter filter,
+                                                               final List<Operator> operands ) {
+        final Filter mappedFilter = createFilter( operands );
+        final BBOX mappedBbox = extractPrefilterBBoxConstraint( mappedFilter );
+        if ( mappedBbox == null ) {
+            final BBOX preFilterBBox = extractPrefilterBBoxConstraint( filter );
+            if ( preFilterBBox != null ) {
+                operands.add( preFilterBBox );
+            }
+        }
+    }
+
+    private OperatorFilter createFilter( List<Operator> andOperands ) {
+        if ( andOperands.isEmpty() ) {
+            return null;
+        }
+        if ( andOperands.size() == 1 ) {
+            return new OperatorFilter( andOperands.get( 0 ) );
+        }
+        return new OperatorFilter( new And( andOperands.toArray( new Operator[andOperands.size()] ) ) );
+    }
+
     /**
      * Returns the expression for the SQL-WHERE clause.
      * 

deegree-core/deegree-core-sqldialect/deegree-sqldialect-commons/src/test/java/org/deegree/sqldialect/filter/MockPropertyNameMapper.java

@@ -0,0 +1,35 @@
+package org.deegree.sqldialect.filter;
+
+import org.deegree.filter.expression.ValueReference;
+
+/**
+ * Mock implementation of {@link PropertyNameMapper} that maps any {@link ValueReference} to a column with the same
+ * name, except for {@link ValueReference}s that start with "UNMAPPABLE_".
+ *
+ * @author <a href="mailto:[email protected]">Markus schneider</a>
+ *
+ * @since 3.4
+ */
+public class MockPropertyNameMapper implements PropertyNameMapper {
+
+    @Override
+    public PropertyNameMapping getSpatialMapping( ValueReference propName, TableAliasManager aliasManager ) {
+        if ( isMappable( propName ) ) {
+            return new PropertyNameMapping( null, null, propName.getAsQName().getLocalPart(), null );
+        }
+        return null;
+    }
+
+    @Override
+    public PropertyNameMapping getMapping( ValueReference propName, TableAliasManager aliasManager ) {
+        if ( isMappable( propName ) ) {
+            return new PropertyNameMapping( null, null, propName.getAsQName().getLocalPart(), null );
+        }
+        return null;
+    }
+
+    private boolean isMappable( final ValueReference propName ) {
+        return !propName.getAsQName().getLocalPart().startsWith( "UNMAPPABLE" );
+    }
+
+}

deegree-core/deegree-core-sqldialect/deegree-sqldialect-commons/src/test/java/org/deegree/sqldialect/filter/MockWhereBuilder.java

@@ -0,0 +1,28 @@
+package org.deegree.sqldialect.filter;
+
+import org.deegree.filter.FilterEvaluationException;
+import org.deegree.filter.OperatorFilter;
+import org.deegree.filter.sort.SortProperty;
+import org.deegree.filter.spatial.SpatialOperator;
+import org.deegree.sqldialect.filter.expression.SQLOperation;
+
+/**
+ * Mock implementation of {@link AbstractWhereBuilder} that uses the {@link MockPropertyNameMapper}.
+ *
+ * @author <a href="mailto:[email protected]">Markus schneider</a>
+ *
+ * @since 3.4
+ */
+class MockWhereBuilder extends AbstractWhereBuilder {
+
+    MockWhereBuilder( final OperatorFilter filter, final SortProperty[] sortCrit ) throws FilterEvaluationException {
+        super( null, new MockPropertyNameMapper(), filter, sortCrit );
+    }
+
+    @Override
+    protected SQLOperation toProtoSQL( final SpatialOperator op )
+                            throws UnmappableException, FilterEvaluationException {
+        return null;
+    }
+
+}

deegree-core/deegree-core-sqldialect/deegree-sqldialect-commons/src/test/java/org/deegree/sqldialect/filter/WhereBuilderTest.java

@@ -0,0 +1,80 @@
+package org.deegree.sqldialect.filter;
+
+import static org.deegree.filter.MatchAction.ANY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+import javax.xml.namespace.QName;
+
+import org.deegree.commons.tom.primitive.PrimitiveValue;
+import org.deegree.filter.FilterEvaluationException;
+import org.deegree.filter.Operator;
+import org.deegree.filter.OperatorFilter;
+import org.deegree.filter.comparison.PropertyIsEqualTo;
+import org.deegree.filter.expression.Literal;
+import org.deegree.filter.expression.ValueReference;
+import org.deegree.filter.logical.And;
+import org.junit.Test;
+
+/**
+ * Tests for {@link AbstractWhereBuilder}.
+ *
+ * @author <a href="mailto:[email protected]">Markus Schneider</a>
+ *
+ * @since 3.4
+ */
+public class WhereBuilderTest {
+
+    @Test
+    public void buildUsingFullMapping()
+                            throws FilterEvaluationException, UnmappableException {
+        final OperatorFilter filter = createAndFilter( 2, 0 );
+        final MockWhereBuilder wb = new MockWhereBuilder( filter, null );
+        wb.build( true );
+
+        // full filter was mapped to SQL
+        final String actualSql = wb.getWhere().getSQL().toString();
+        assertEquals( "(MAPPABLE_0 = ? AND MAPPABLE_1 = ?)", actualSql );
+
+        // post filter is null
+        final OperatorFilter actualPostFilter = wb.getPostFilter();
+        assertNull( actualPostFilter );
+    }
+
+    @Test
+    public void buildUsingPartialMapping()
+                            throws FilterEvaluationException, UnmappableException {
+        final OperatorFilter filter = createAndFilter( 1, 1 );
+        final MockWhereBuilder wb = new MockWhereBuilder( filter, null );
+        wb.build( true );
+
+        // one part of filter was mapped to SQL
+        final String actualSql = wb.getWhere().getSQL().toString();
+        assertEquals( "MAPPABLE_0 = ?", actualSql );
+
+        // remaining part of filter was extracted as post filter
+        final OperatorFilter actualPostFilter = wb.getPostFilter();
+        final PropertyIsEqualTo operator = (PropertyIsEqualTo) actualPostFilter.getOperator();
+        final ValueReference ref = (ValueReference) operator.getParameter1();
+        assertEquals( "UNMAPPABLE_0", ref.getAsQName().getLocalPart() );
+    }
+
+    private OperatorFilter createAndFilter( final int numMappedOperands, final int numUnmappedOperands ) {
+        final Collection<Operator> operators = new ArrayList<Operator>();
+        for ( int i = 0; i < numMappedOperands; i++ ) {
+            final ValueReference param1 = new ValueReference( new QName( "MAPPABLE_" + i ) );
+            final Literal<PrimitiveValue> param2 = new Literal<PrimitiveValue>( "VALUE" );
+            operators.add( new PropertyIsEqualTo( param1, param2, true, ANY ) );
+        }
+        for ( int i = 0; i < numUnmappedOperands; i++ ) {
+            final ValueReference param1 = new ValueReference( new QName( "UNMAPPABLE_" + i ) );
+            final Literal<PrimitiveValue> param2 = new Literal<PrimitiveValue>( "VALUE" );
+            operators.add( new PropertyIsEqualTo( param1, param2, true, ANY ) );
+        }
+        final And and = new And( operators.toArray( new Operator[operators.size()] ) );
+        return new OperatorFilter( and );
+    }
+}