fix: null properties (#28297)

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>

ee/clickhouse/queries/test/test_cohort_query.py

@@ -14,6 +14,7 @@
 from posthog.models.filters.filter import Filter
 from posthog.models.property import Property, PropertyGroup
 
+from posthog.models.property_definition import PropertyDefinition
 from posthog.test.base import (
     BaseTest,
     ClickhouseTestMixin,
@@ -3364,3 +3365,55 @@ def test_basic_valid_negation_tree_with_no_negations(self):
         has_pending_neg, has_reg = check_negation_clause(property_group)
         self.assertEqual(has_pending_neg, False)
         self.assertEqual(has_reg, True)
+
+    def test_type_misalignment(self):
+        PropertyDefinition.objects.create(
+            team=self.team,
+            name="createdDate",
+            property_type="DateTime",
+            type=PropertyDefinition.Type.PERSON,
+        )
+
+        _create_person(
+            team_id=self.team.pk,
+            distinct_ids=["p3"],
+            properties={"name": "test2", "email": "[email protected]", "createdDate": "2022-10-11"},
+        )
+
+        cohort = Cohort.objects.create(
+            team=self.team,
+            name="cohort",
+            is_static=False,
+            filters={
+                "properties": {
+                    "type": "OR",
+                    "values": [
+                        {
+                            "type": "OR",
+                            "values": [
+                                {
+                                    "key": "createdDate",
+                                    "type": "person",
+                                    "value": "2022",
+                                    "negation": False,
+                                    "operator": "icontains",
+                                }
+                            ],
+                        }
+                    ],
+                }
+            },
+        )
+
+        filter = Filter(
+            data={
+                "properties": {
+                    "type": "OR",
+                    "values": [{"key": "id", "value": cohort.pk, "type": "cohort"}],
+                }
+            },
+            team=self.team,
+        )
+
+        res, q, params = execute(filter, self.team)
+        assert 1 == len(res)

mypy-baseline.txt

@@ -184,7 +184,6 @@ posthog/hogql/resolver.py:0: error: Item "None" of "SelectQuery | SelectSetQuery
 posthog/hogql/resolver.py:0: error: Incompatible types in assignment (expression has type "Type | Any | None", variable has type "BaseTableType | SelectSetQueryType | SelectQueryType | SelectQueryAliasType | SelectViewType | None")  [assignment]
 posthog/hogql/resolver.py:0: error: Argument 1 to "append" of "list" has incompatible type "BaseTableType | SelectSetQueryType | SelectQueryType | SelectQueryAliasType | SelectViewType | None"; expected "SelectQueryType | SelectSetQueryType"  [arg-type]
 posthog/hogql/resolver.py:0: error: Statement is unreachable  [unreachable]
-posthog/hogql/resolver.py:0: error: Statement is unreachable  [unreachable]
 posthog/hogql/resolver.py:0: error: Item "None" of "Type | None" has no attribute "resolve_constant_type"  [union-attr]
 posthog/hogql/resolver.py:0: error: Item "None" of "Type | None" has no attribute "resolve_constant_type"  [union-attr]
 posthog/hogql/resolver.py:0: error: Item "None" of "Type | None" has no attribute "resolve_constant_type"  [union-attr]

posthog/api/test/__snapshots__/test_query.ambr

@@ -50,7 +50,7 @@
          'a%sd',
          concat(ifNull(toString(events.event), ''), ' ', ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'key'), ''), 'null'), '^"|"$', '')), ''))
   FROM events
-  WHERE and(equals(events.team_id, 99999), ifNull(ilike(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'path'), ''), 'null'), '^"|"$', ''), '%/%'), 0), less(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-01-10 12:14:05.000000', 6, 'UTC')), greater(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-01-09 12:14:00.000000', 6, 'UTC')))
+  WHERE and(equals(events.team_id, 99999), ifNull(ilike(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'path'), ''), 'null'), '^"|"$', '')), '%/%'), 0), less(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-01-10 12:14:05.000000', 6, 'UTC')), greater(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-01-09 12:14:00.000000', 6, 'UTC')))
   ORDER BY events.event ASC
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
@@ -113,7 +113,7 @@
          'a%sd',
          concat(ifNull(toString(events.event), ''), ' ', ifNull(toString(nullIf(nullIf(events.mat_key, ''), 'null')), ''))
   FROM events
-  WHERE and(equals(events.team_id, 99999), ifNull(ilike(nullIf(nullIf(events.mat_path, ''), 'null'), '%/%'), 0), less(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-01-10 12:14:05.000000', 6, 'UTC')), greater(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-01-09 12:14:00.000000', 6, 'UTC')))
+  WHERE and(equals(events.team_id, 99999), ifNull(ilike(toString(nullIf(nullIf(events.mat_path, ''), 'null')), '%/%'), 0), less(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-01-10 12:14:05.000000', 6, 'UTC')), greater(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-01-09 12:14:00.000000', 6, 'UTC')))
   ORDER BY events.event ASC
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,

posthog/api/test/test_hog_function.py

@@ -807,6 +807,9 @@ def test_generates_filters_bytecode(self, *args):
                 "person",
                 1,
                 3,
+                2,
+                "toString",
+                1,
                 20,
                 32,
                 "$pageview",
@@ -827,6 +830,9 @@ def test_generates_filters_bytecode(self, *args):
                 "person",
                 1,
                 3,
+                2,
+                "toString",
+                1,
                 20,
                 32,
                 "$pageview",

posthog/cdp/test/test_filters.py

@@ -101,6 +101,9 @@ def test_filters_events(self):
                 "properties",
                 1,
                 2,
+                2,
+                "toString",
+                1,
                 18,
                 3,
                 2,
@@ -160,6 +163,9 @@ def test_filters_properties(self):
                 "person",
                 1,
                 3,
+                2,
+                "toString",
+                1,
                 18,
                 32,
                 "ben",
@@ -193,6 +199,9 @@ def test_filters_full(self):
                 "person",
                 1,
                 3,
+                2,
+                "toString",
+                1,
                 20,
                 32,
                 "%@posthog.com%",
@@ -204,6 +213,9 @@ def test_filters_full(self):
                 "person",
                 1,
                 3,
+                2,
+                "toString",
+                1,
                 18,
                 32,
                 "ben",
@@ -231,6 +243,9 @@ def test_filters_full(self):
                 "properties",
                 1,
                 2,
+                2,
+                "toString",
+                1,
                 18,
                 3,
                 5,
@@ -244,6 +259,9 @@ def test_filters_full(self):
                 "person",
                 1,
                 3,
+                2,
+                "toString",
+                1,
                 20,
                 32,
                 "%@posthog.com%",
@@ -255,6 +273,9 @@ def test_filters_full(self):
                 "person",
                 1,
                 3,
+                2,
+                "toString",
+                1,
                 18,
                 32,
                 "ben",

posthog/cdp/test/test_site_functions.py

@@ -229,7 +229,7 @@ def test_get_transpiled_function_with_event_filter(self):
         assert "const filterMatches = " in result
         assert '__getGlobal("event") == "$pageview"' in result
         assert (
-            '(ilike(__getProperty(__getProperty(__getGlobal("person"), "properties", true), "email", true), "%@test.com%")'
+            '(ilike(toString(__getProperty(__getProperty(__getGlobal("person"), "properties", true), "email", true)), "%@test.com%")'
             in result
         )
 

posthog/hogql/ast.py

@@ -1,3 +1,4 @@
+import dataclasses
 import inspect
 import sys
 from enum import StrEnum
@@ -569,7 +570,8 @@ def resolve_constant_type(self, context: HogQLContext) -> ConstantType:
         if self.joined_subquery is not None and self.joined_subquery_field_name is not None:
             return self.joined_subquery.resolve_column_constant_type(self.joined_subquery_field_name, context)
 
-        return self.field_type.resolve_constant_type(context)
+        # PropertyTypes are always nullable
+        return dataclasses.replace(self.field_type.resolve_constant_type(context), nullable=True)
 
 
 @dataclass(kw_only=True)

posthog/hogql/property.py

@@ -184,13 +184,13 @@ def _expr_to_compare_op(
     elif operator == PropertyOperator.ICONTAINS:
         return ast.CompareOperation(
             op=ast.CompareOperationOp.ILike,
-            left=expr,
+            left=ast.Call(name="toString", args=[expr]),
             right=ast.Constant(value=f"%{value}%"),
         )
     elif operator == PropertyOperator.NOT_ICONTAINS:
         return ast.CompareOperation(
             op=ast.CompareOperationOp.NotILike,
-            left=expr,
+            left=ast.Call(name="toString", args=[expr]),
             right=ast.Constant(value=f"%{value}%"),
         )
     elif operator == PropertyOperator.REGEX:

posthog/hogql/resolver.py

@@ -1,3 +1,4 @@
+import dataclasses
 from datetime import date, datetime
 from typing import Any, Literal, Optional, cast
 from uuid import UUID
@@ -502,12 +503,8 @@ def visit_call(self, node: ast.Call):
             signatures = HOGQL_CLICKHOUSE_FUNCTIONS[node.name].signatures
             if signatures:
                 for sig_arg_types, sig_return_type in signatures:
-                    if sig_arg_types is None:
-                        return_type = sig_return_type
-                        break
-
-                    if compare_types(arg_types, sig_arg_types):
-                        return_type = sig_return_type
+                    if sig_arg_types is None or compare_types(arg_types, sig_arg_types):
+                        return_type = dataclasses.replace(sig_return_type)
                         break
 
         if return_type is None:

posthog/hogql/test/__snapshots__/test_resolver.ambr

@@ -249,7 +249,7 @@
                     arg_types: [
                       {
                         data_type: "str"
-                        nullable: False
+                        nullable: True
                       }
                     ]
                     name: "toUUID"
@@ -1271,7 +1271,7 @@
                       arg_types: [
                         {
                           data_type: "str"
-                          nullable: False
+                          nullable: True
                         }
                       ]
                       name: "toUUID"
@@ -2008,7 +2008,7 @@
             arg_types: [
               {
                 data_type: "str"
-                nullable: False
+                nullable: True
               }
             ]
             name: "toUUID"
@@ -2187,7 +2187,7 @@
                   arg_types: [
                     {
                       data_type: "str"
-                      nullable: False
+                      nullable: True
                     }
                   ]
                   name: "toUUID"
@@ -3323,7 +3323,7 @@
                     arg_types: [
                       {
                         data_type: "str"
-                        nullable: False
+                        nullable: True
                       }
                     ]
                     name: "toUUID"
@@ -4346,7 +4346,7 @@
                         arg_types: [
                           {
                             data_type: "str"
-                            nullable: False
+                            nullable: True
                           }
                         ]
                         name: "toUUID"
@@ -5043,7 +5043,7 @@
                       arg_types: [
                         {
                           data_type: "str"
-                          nullable: False
+                          nullable: True
                         }
                       ]
                       name: "toUUID"
@@ -5222,7 +5222,7 @@
                             arg_types: [
                               {
                                 data_type: "str"
-                                nullable: False
+                                nullable: True
                               }
                             ]
                             name: "toUUID"
@@ -6231,7 +6231,7 @@
             arg_types: [
               {
                 data_type: "str"
-                nullable: False
+                nullable: True
               }
             ]
             name: "toUUID"
@@ -6410,7 +6410,7 @@
                   arg_types: [
                     {
                       data_type: "str"
-                      nullable: False
+                      nullable: True
                     }
                   ]
                   name: "toUUID"
@@ -7163,7 +7163,7 @@
             arg_types: [
               {
                 data_type: "str"
-                nullable: False
+                nullable: True
               }
             ]
             name: "toUUID"
@@ -7342,7 +7342,7 @@
                   arg_types: [
                     {
                       data_type: "str"
-                      nullable: False
+                      nullable: True
                     }
                   ]
                   name: "toUUID"

posthog/hogql/test/test_filters.py

@@ -153,5 +153,5 @@ def test_replace_filters_test_accounts(self):
         )
         self.assertEqual(
             self._print_ast(select),
-            f"SELECT event FROM events WHERE notILike(person.properties.email, '%posthog.com%') LIMIT {MAX_SELECT_RETURNED_ROWS}",
+            f"SELECT event FROM events WHERE notILike(toString(person.properties.email), '%posthog.com%') LIMIT {MAX_SELECT_RETURNED_ROWS}",
         )

posthog/hogql/test/test_property.py

@@ -146,11 +146,11 @@ def test_property_to_expr_event(self):
         )
         self.assertEqual(
             self._property_to_expr({"type": "event", "key": "a", "value": "3", "operator": "icontains"}),
-            self._parse_expr("properties.a ilike '%3%'"),
+            self._parse_expr("toString(properties.a) ilike '%3%'"),
         )
         self.assertEqual(
             self._property_to_expr({"type": "event", "key": "a", "value": "3", "operator": "not_icontains"}),
-            self._parse_expr("properties.a not ilike '%3%'"),
+            self._parse_expr("toString(properties.a) not ilike '%3%'"),
         )
         self.assertEqual(
             self._property_to_expr({"type": "event", "key": "a", "value": ".*", "operator": "regex"}),
@@ -233,7 +233,7 @@ def test_property_to_expr_event_list(self):
                     "operator": "icontains",
                 }
             ),
-            self._parse_expr("properties.a ilike '%b%' or properties.a ilike '%c%'"),
+            self._parse_expr("toString(properties.a) ilike '%b%' or toString(properties.a) ilike '%c%'"),
         )
         a = self._property_to_expr({"type": "event", "key": "a", "value": ["b", "c"], "operator": "regex"})
         self.assertEqual(
@@ -258,7 +258,7 @@ def test_property_to_expr_event_list(self):
                     "operator": "not_icontains",
                 }
             ),
-            self._parse_expr("properties.a not ilike '%b%' and properties.a not ilike '%c%'"),
+            self._parse_expr("toString(properties.a) not ilike '%b%' and toString(properties.a) not ilike '%c%'"),
         )
         a = self._property_to_expr(
             {
@@ -353,7 +353,7 @@ def test_property_to_expr_element(self):
                     "operator": "icontains",
                 }
             ),
-            self._parse_expr("elements_chain_href ilike '%href-text.%'"),
+            self._parse_expr("toString(elements_chain_href) ilike '%href-text.%'"),
         )
         self.assertEqual(
             self._property_to_expr(