Add support for array operators (#127)

* add support for array operators * update changelog to reflect fix for #123 * add more array tests, make sure cql2_ops table is up-to-date in incremental migration
stac-utils · Jun 22, 2022 · 232634a · 232634a
1 parent 42d512e
commit 232634a
Show file tree

Hide file tree

Showing 9 changed files with 3,150 additions and 3 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,4 +1,9 @@
 # Changelog
+## [v0.6.6]
+### Added
+- Add support for array operators in CQL2 (a_equals, a_contains, a_contained_by, a_overlaps).
+- Add check in loader to make sure that pypgstac and pgstac versions match before loading data [#123](https://github.com/stac-utils/pgstac/issues/123)
+
 ## [v0.6.5]
 ### Fixed
 - Fix for type casting when using the "in" operator [#122](https://github.com/stac-utils/pgstac/issues/122)

diff --git a/pypgstac/pypgstac/migrations/pgstac.0.6.5-0.6.6.sql b/pypgstac/pypgstac/migrations/pgstac.0.6.5-0.6.6.sql
@@ -0,0 +1,216 @@
+SET SEARCH_PATH to pgstac, public;
+set check_function_bodies = off;
+
+CREATE OR REPLACE FUNCTION pgstac.cql2_query(j jsonb, wrapper text DEFAULT NULL::text)
+ RETURNS text
+ LANGUAGE plpgsql
+ STABLE
+AS $function$
+#variable_conflict use_variable
+DECLARE
+    args jsonb := j->'args';
+    arg jsonb;
+    op text := lower(j->>'op');
+    cql2op RECORD;
+    literal text;
+    _wrapper text;
+    leftarg text;
+    rightarg text;
+BEGIN
+    IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN
+        RETURN NULL;
+    END IF;
+    RAISE NOTICE 'CQL2_QUERY: %', j;
+    IF j ? 'filter' THEN
+        RETURN cql2_query(j->'filter');
+    END IF;
+
+    IF j ? 'upper' THEN
+        RETURN  cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper'));
+    END IF;
+
+    IF j ? 'lower' THEN
+        RETURN  cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower'));
+    END IF;
+
+    -- Temporal Query
+    IF op ilike 't_%' or op = 'anyinteracts' THEN
+        RETURN temporal_op_query(op, args);
+    END IF;
+
+    -- If property is a timestamp convert it to text to use with
+    -- general operators
+    IF j ? 'timestamp' THEN
+        RETURN format('%L::timestamptz', to_tstz(j->'timestamp'));
+    END IF;
+    IF j ? 'interval' THEN
+        RAISE EXCEPTION 'Please use temporal operators when using intervals.';
+        RETURN NONE;
+    END IF;
+
+    -- Spatial Query
+    IF op ilike 's_%' or op = 'intersects' THEN
+        RETURN spatial_op_query(op, args);
+    END IF;
+
+    IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN
+        IF args->0 ? 'property' THEN
+            leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path);
+        END IF;
+        IF args->1 ? 'property' THEN
+            rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path);
+        END IF;
+        RETURN FORMAT(
+            '%s %s %s',
+            COALESCE(leftarg, quote_literal(to_text_array(args->0))),
+            CASE op
+                WHEN 'a_equals' THEN '='
+                WHEN 'a_contains' THEN '@>'
+                WHEN 'a_contained_by' THEN '<@'
+                WHEN 'a_overlaps' THEN '&&'
+            END,
+            COALESCE(rightarg, quote_literal(to_text_array(args->1)))
+        );
+    END IF;
+
+    IF op = 'in' THEN
+        RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1;
+        args := jsonb_build_array(args->0) || (args->1);
+        RAISE NOTICE 'IN2 : %', args;
+    END IF;
+
+
+
+    IF op = 'between' THEN
+        args = jsonb_build_array(
+            args->0,
+            args->1->0,
+            args->1->1
+        );
+    END IF;
+
+    -- Make sure that args is an array and run cql2_query on
+    -- each element of the array
+    RAISE NOTICE 'ARGS PRE: %', args;
+    IF j ? 'args' THEN
+        IF jsonb_typeof(args) != 'array' THEN
+            args := jsonb_build_array(args);
+        END IF;
+
+        IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN
+            wrapper := NULL;
+        ELSE
+            -- if any of the arguments are a property, try to get the property_wrapper
+            FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP
+                RAISE NOTICE 'Arg: %', arg;
+                SELECT property_wrapper INTO wrapper
+                FROM queryables
+                WHERE name=(arg->>'property')
+                LIMIT 1;
+                RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper;
+                IF wrapper IS NOT NULL THEN
+                    EXIT;
+                END IF;
+            END LOOP;
+
+            -- if the property was not in queryables, see if any args were numbers
+            IF
+                wrapper IS NULL
+                AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")')
+            THEN
+                wrapper := 'to_float';
+            END IF;
+            wrapper := coalesce(wrapper, 'to_text');
+        END IF;
+
+        SELECT jsonb_agg(cql2_query(a, wrapper))
+            INTO args
+        FROM jsonb_array_elements(args) a;
+    END IF;
+    RAISE NOTICE 'ARGS: %', args;
+
+    IF op IN ('and', 'or') THEN
+        RETURN
+            format(
+                '(%s)',
+                array_to_string(to_text_array(args), format(' %s ', upper(op)))
+            );
+    END IF;
+
+    IF op = 'in' THEN
+        RAISE NOTICE 'IN --  % %', args->0, to_text(args->0);
+        RETURN format(
+            '%s IN (%s)',
+            to_text(args->0),
+            array_to_string((to_text_array(args))[2:], ',')
+        );
+    END IF;
+
+    -- Look up template from cql2_ops
+    IF j ? 'op' THEN
+        SELECT * INTO cql2op FROM cql2_ops WHERE  cql2_ops.op ilike op;
+        IF FOUND THEN
+            -- If specific index set in queryables for a property cast other arguments to that type
+
+            RETURN format(
+                cql2op.template,
+                VARIADIC (to_text_array(args))
+            );
+        ELSE
+            RAISE EXCEPTION 'Operator % Not Supported.', op;
+        END IF;
+    END IF;
+
+
+    IF wrapper IS NOT NULL THEN
+        RAISE NOTICE 'Wrapping % with %', j, wrapper;
+        IF j ? 'property' THEN
+            RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path);
+        ELSE
+            RETURN format('%I(%L)', wrapper, j);
+        END IF;
+    ELSIF j ? 'property' THEN
+        RETURN quote_ident(j->>'property');
+    END IF;
+
+    RETURN quote_literal(to_text(j));
+END;
+$function$
+;
+
+TRUNCATE cql2_ops;
+INSERT INTO cql2_ops (op, template, types) VALUES
+    ('eq', '%s = %s', NULL),
+    ('neq', '%s != %s', NULL),
+    ('ne', '%s != %s', NULL),
+    ('!=', '%s != %s', NULL),
+    ('<>', '%s != %s', NULL),
+    ('lt', '%s < %s', NULL),
+    ('lte', '%s <= %s', NULL),
+    ('gt', '%s > %s', NULL),
+    ('gte', '%s >= %s', NULL),
+    ('le', '%s <= %s', NULL),
+    ('ge', '%s >= %s', NULL),
+    ('=', '%s = %s', NULL),
+    ('<', '%s < %s', NULL),
+    ('<=', '%s <= %s', NULL),
+    ('>', '%s > %s', NULL),
+    ('>=', '%s >= %s', NULL),
+    ('like', '%s LIKE %s', NULL),
+    ('ilike', '%s ILIKE %s', NULL),
+    ('+', '%s + %s', NULL),
+    ('-', '%s - %s', NULL),
+    ('*', '%s * %s', NULL),
+    ('/', '%s / %s', NULL),
+    ('not', 'NOT (%s)', NULL),
+    ('between', '%s BETWEEN %s AND %s', NULL),
+    ('isnull', '%s IS NULL', NULL),
+    ('upper', 'upper(%s)', NULL),
+    ('lower', 'lower(%s)', NULL)
+ON CONFLICT (op) DO UPDATE
+    SET
+        template = EXCLUDED.template
+;
+
+
+SELECT set_version('0.6.6');