Add support for marking indexes as deferred (#7129)

A deferred index is a kind of index that gets created and updated in a
background process and not immediately.  This is required for indexes
that are slow and/or require calling out to external services.

To implement this we add the ability to specify `deferrability` on an
abstract index as one of three enum values:

* `Required`: concrete indexes of this type must be declared as
  `deferred`
* `Prohibited`: concrete indexes of this type must NOT be declared
  as `deferred`
* `Permitted`: either way is OK.

All current index types are marked as `Prohibited`.

edb/buildmeta.py

@@ -55,7 +55,7 @@
 
 
 # Increment this whenever the database layout or stdlib changes.
-EDGEDB_CATALOG_VERSION = 2024_03_27_00_01
+EDGEDB_CATALOG_VERSION = 2024_03_28_00_00
 EDGEDB_MAJOR_VERSION = 6
 
 

edb/edgeql/ast.py

@@ -1281,6 +1281,7 @@ class ConcreteIndexCommand(IndexCommand):
     kwargs: typing.Dict[str, Expr] = ast.field(factory=dict)
     expr: Expr
     except_expr: typing.Optional[Expr] = None
+    deferred: typing.Optional[bool] = None
 
 
 class CreateConcreteIndex(ConcreteIndexCommand, CreateObject):

edb/edgeql/codegen.py

@@ -1414,6 +1414,13 @@ def _process_special_set(self, node: qlast.SetField) -> List[str]:
                 keywords.extend(('SET', 'OWNED'))
             else:
                 keywords.extend(('DROP', 'OWNED'))
+        elif fname == 'deferred':
+            if node.value is None:
+                keywords.extend(('RESET', 'DEFERRED'))
+            elif self._eval_bool_expr(node.value):
+                keywords.extend(('SET', 'DEFERRED'))
+            else:
+                keywords.extend(('DROP', 'DEFERRED'))
         else:
             raise EdgeQLSourceGeneratorError(
                 'unknown special field: {!r}'.format(fname))
@@ -1993,22 +2000,6 @@ def after_name() -> None:
 
             self._ddl_visit_bases(node)
 
-            if node.commands or node.code:
-                self.write(' {')
-                self._block_ws(1)
-                commands = self._ddl_clean_up_commands(node.commands)
-                self.visit_list(commands, terminator=';')
-                self.new_lines = 1
-
-                if node.code:
-                    self._write_keywords('USING', node.code.language)
-                    self.write(edgeql_quote.dollar_quote_literal(
-                        node.code.code))
-                    self.write(';')
-
-                self._block_ws(-1)
-                self.write('}')
-
         self._visit_CreateObject(node, 'ABSTRACT INDEX', after_name=after_name)
 
     def visit_AlterIndex(self, node: qlast.AlterIndex) -> None:
@@ -2024,9 +2015,13 @@ def visit_IndexCode(self, node: qlast.IndexCode) -> None:
     def visit_CreateConcreteIndex(
         self, node: qlast.CreateConcreteIndex
     ) -> None:
+        keywords = ['DEFERRED', 'INDEX'] if node.deferred else ['INDEX']
         self._visit_CreateObject(
-            node, 'INDEX', named=node.name.name != 'idx',
-            after_name=lambda: self._after_index(node))
+            node,
+            *keywords,
+            named=node.name.name != 'idx',
+            after_name=lambda: self._after_index(node),
+        )
 
     def visit_AlterConcreteIndex(self, node: qlast.AlterConcreteIndex) -> None:
         self._visit_AlterObject(

edb/edgeql/parser/grammar/commondl.py

@@ -184,6 +184,14 @@ def reduce_OnExpr(self, expr):
         pass
 
 
+class OptDeferred(Nonterm):
+    def reduce_empty(self):
+        self.val = None
+
+    def reduce_DEFERRED(self, _):
+        self.val = True
+
+
 class OptExceptExpr(Nonterm):
     def reduce_empty(self):
         self.val = None

edb/edgeql/parser/grammar/ddl.py

@@ -1543,49 +1543,70 @@ def reduce_DropIndex(self, *kids):
         )
 
 
-commands_block(
-    'AlterConcreteIndex',
-    SetFieldStmt,
-    ResetFieldStmt,
-    AlterOwnedStmt,
-    CreateAnnotationValueStmt,
-    AlterAnnotationValueStmt,
-    DropAnnotationValueStmt,
-    opt=False)
-
-
 #
 # CREATE CONCRETE INDEX
 #
 class CreateConcreteIndexStmt(Nonterm, commondl.ProcessIndexMixin):
-    def reduce_CREATE_INDEX_OnExpr_OptExceptExpr_OptCreateCommandsBlock(
-        self, *kids
-    ):
+    def reduce_CreateConcreteDefaultIndex(self, *kids):
+        r"""%reduce CREATE OptDeferred INDEX OnExpr OptExceptExpr
+                    OptCreateCommandsBlock
+        """
         self.val = qlast.CreateConcreteIndex(
             name=qlast.ObjectRef(module='__', name='idx'),
-            expr=kids[2].val,
-            except_expr=kids[3].val,
-            commands=kids[4].val,
+            expr=kids[3].val,
+            except_expr=kids[4].val,
+            deferred=kids[1].val,
+            commands=kids[5].val,
         )
 
     def reduce_CreateConcreteIndex(self, *kids):
-        r"""%reduce CREATE INDEX NodeName \
-                    OptIndexExtArgList OnExpr OptExceptExpr \
-                    OptCreateCommandsBlock \
+        r"""%reduce CREATE OptDeferred INDEX NodeName
+                    OptIndexExtArgList OnExpr OptExceptExpr
+                    OptCreateCommandsBlock
         """
-        kwargs = self._process_arguments(kids[3].val)
+        kwargs = self._process_arguments(kids[4].val)
         self.val = qlast.CreateConcreteIndex(
-            name=kids[2].val,
+            name=kids[3].val,
             kwargs=kwargs,
-            expr=kids[4].val,
-            except_expr=kids[5].val,
-            commands=kids[6].val,
+            expr=kids[5].val,
+            except_expr=kids[6].val,
+            deferred=kids[1].val,
+            commands=kids[7].val,
         )
 
 
 #
 # ALTER CONCRETE INDEX
 #
+
+class AlterDeferredStmt(Nonterm):
+    def reduce_DROP_DEFERRED(self, *kids):
+        self.val = qlast.SetField(
+            name='deferred',
+            value=qlast.BooleanConstant(value='false'),
+            special_syntax=True,
+        )
+
+    def reduce_SET_DEFERRED(self, *kids):
+        self.val = qlast.SetField(
+            name='deferred',
+            value=qlast.BooleanConstant(value='true'),
+            special_syntax=True,
+        )
+
+
+commands_block(
+    'AlterConcreteIndex',
+    SetFieldStmt,
+    ResetFieldStmt,
+    AlterOwnedStmt,
+    AlterDeferredStmt,
+    CreateAnnotationValueStmt,
+    AlterAnnotationValueStmt,
+    DropAnnotationValueStmt,
+    opt=False)
+
+
 class AlterConcreteIndexStmt(Nonterm, commondl.ProcessIndexMixin):
     def reduce_AlterConcreteIndex(self, *kids):
         r"""%reduce ALTER INDEX OnExpr OptExceptExpr \

edb/edgeql/parser/grammar/sdl.py

@@ -659,8 +659,10 @@ def reduce_CreateIndex_CreateFunctionArgs(self, *kids):
 
 
 class ConcreteIndexDeclarationBlock(Nonterm, commondl.ProcessIndexMixin):
-    def reduce_INDEX_OnExpr_OptExceptExpr_CreateConcreteIndexSDLCommandsBlock(
-            self, *kids):
+    def reduce_CreateConcreteAnonymousIndex(self, *kids):
+        r"""%reduce INDEX OnExpr OptExceptExpr
+                    CreateConcreteIndexSDLCommandsBlock
+        """
         _, on_expr, except_expr, commands = kids
         self.val = qlast.CreateConcreteIndex(
             name=qlast.ObjectRef(module='__', name='idx'),
@@ -669,6 +671,19 @@ def reduce_INDEX_OnExpr_OptExceptExpr_CreateConcreteIndexSDLCommandsBlock(
             commands=commands.val,
         )
 
+    def reduce_CreateConcreteAnonymousDeferredIndex(self, *kids):
+        r"""%reduce DEFERRED INDEX OnExpr OptExceptExpr
+                    CreateConcreteIndexSDLCommandsBlock
+        """
+        _, _, on_expr, except_expr, commands = kids
+        self.val = qlast.CreateConcreteIndex(
+            name=qlast.ObjectRef(module='__', name='idx'),
+            expr=on_expr.val,
+            except_expr=except_expr.val,
+            deferred=True,
+            commands=commands.val,
+        )
+
     def reduce_CreateConcreteIndex(self, *kids):
         r"""%reduce INDEX NodeName \
                     OnExpr OptExceptExpr \
@@ -682,6 +697,20 @@ def reduce_CreateConcreteIndex(self, *kids):
             commands=commands.val,
         )
 
+    def reduce_CreateConcreteDeferredIndex(self, *kids):
+        r"""%reduce DEFERRED INDEX NodeName \
+                    OnExpr OptExceptExpr \
+                    CreateConcreteIndexSDLCommandsBlock \
+        """
+        _, _, name, on_expr, except_expr, commands = kids
+        self.val = qlast.CreateConcreteIndex(
+            name=name.val,
+            expr=on_expr.val,
+            except_expr=except_expr.val,
+            deferred=True,
+            commands=commands.val,
+        )
+
     def reduce_CreateConcreteIndexWithArgs(self, *kids):
         r"""%reduce INDEX NodeName IndexExtArgList \
                     OnExpr OptExceptExpr \
@@ -697,6 +726,22 @@ def reduce_CreateConcreteIndexWithArgs(self, *kids):
             commands=commands.val,
         )
 
+    def reduce_CreateConcreteDeferredIndexWithArgs(self, *kids):
+        r"""%reduce DEFERRED INDEX NodeName IndexExtArgList \
+                    OnExpr OptExceptExpr \
+                    CreateConcreteIndexSDLCommandsBlock \
+        """
+        _, _, name, arg_list, on_expr, except_expr, commands = kids
+        kwargs = self._process_arguments(arg_list.val)
+        self.val = qlast.CreateConcreteIndex(
+            name=name.val,
+            kwargs=kwargs,
+            expr=on_expr.val,
+            except_expr=except_expr.val,
+            deferred=True,
+            commands=commands.val,
+        )
+
 
 class ConcreteIndexDeclarationShort(Nonterm, commondl.ProcessIndexMixin):
     def reduce_INDEX_OnExpr_OptExceptExpr(self, *kids):
@@ -707,9 +752,17 @@ def reduce_INDEX_OnExpr_OptExceptExpr(self, *kids):
             except_expr=except_expr.val,
         )
 
+    def reduce_DEFERRED_INDEX_OnExpr_OptExceptExpr(self, *kids):
+        _, _, on_expr, except_expr = kids
+        self.val = qlast.CreateConcreteIndex(
+            name=qlast.ObjectRef(module='__', name='idx'),
+            expr=on_expr.val,
+            except_expr=except_expr.val,
+            deferred=True,
+        )
+
     def reduce_CreateConcreteIndex(self, *kids):
-        r"""%reduce INDEX NodeName \
-                    OnExpr OptExceptExpr \
+        r"""%reduce INDEX NodeName OnExpr OptExceptExpr
         """
         _, name, on_expr, except_expr = kids
         self.val = qlast.CreateConcreteIndex(
@@ -718,6 +771,17 @@ def reduce_CreateConcreteIndex(self, *kids):
             except_expr=except_expr.val,
         )
 
+    def reduce_CreateConcreteDeferredIndex(self, *kids):
+        r"""%reduce DEFERRED INDEX NodeName OnExpr OptExceptExpr
+        """
+        _, _, name, on_expr, except_expr = kids
+        self.val = qlast.CreateConcreteIndex(
+            name=name.val,
+            expr=on_expr.val,
+            except_expr=except_expr.val,
+            deferred=True,
+        )
+
     def reduce_CreateConcreteIndexWithArgs(self, *kids):
         r"""%reduce INDEX NodeName IndexExtArgList \
                     OnExpr OptExceptExpr \
@@ -731,6 +795,20 @@ def reduce_CreateConcreteIndexWithArgs(self, *kids):
             except_expr=except_expr.val,
         )
 
+    def reduce_CreateConcreteDeferredIndexWithArgs(self, *kids):
+        r"""%reduce DEFERRED INDEX NodeName IndexExtArgList
+                    OnExpr OptExceptExpr
+        """
+        _, _, name, arg_list, on_expr, except_expr = kids
+        kwargs = self._process_arguments(arg_list.val)
+        self.val = qlast.CreateConcreteIndex(
+            name=name.val,
+            kwargs=kwargs,
+            expr=on_expr.val,
+            except_expr=except_expr.val,
+            deferred=True,
+        )
+
 
 #
 # Mutation rewrites

edb/edgeql/qltypes.py

@@ -190,6 +190,18 @@ def is_duplicate(self) -> bool:
         return self is Multiplicity.DUPLICATE
 
 
+class IndexDeferrability(s_enum.OrderedEnumMixin, s_enum.StrEnum):
+    Prohibited = 'Prohibited'
+    Permitted = 'Permitted'
+    Required = 'Required'
+
+    def is_deferrable(self) -> bool:
+        return (
+            self is IndexDeferrability.Required
+            or self is IndexDeferrability.Permitted
+        )
+
+
 class AccessPolicyAction(s_enum.StrEnum):
     Allow = 'Allow'
     Deny = 'Deny'

edb/lib/schema.edgeql

@@ -64,6 +64,9 @@ CREATE SCALAR TYPE schema::RewriteKind
 CREATE SCALAR TYPE schema::MigrationGeneratedBy
     EXTENDING enum<DevMode, DDLStatement>;
 
+CREATE SCALAR TYPE schema::IndexDeferrability
+    EXTENDING enum<Prohibited, Permitted, `Required`>;
+
 # Base type for all schema entities.
 CREATE ABSTRACT TYPE schema::Object EXTENDING std::BaseObject {
     CREATE REQUIRED PROPERTY name -> std::str;
@@ -267,6 +270,8 @@ CREATE TYPE schema::Index
 {
     CREATE PROPERTY expr -> std::str;
     CREATE PROPERTY except_expr -> std::str;
+    CREATE PROPERTY deferrability -> schema::IndexDeferrability;
+    CREATE PROPERTY deferred -> std::bool;
     CREATE MULTI LINK params EXTENDING schema::ordered -> schema::Parameter {
         ON TARGET DELETE ALLOW;
     };

edb/schema/delta.py

@@ -2413,6 +2413,7 @@ def _apply_fields_ast(
                         not fop.new_inherited
                         or context.descriptive_mode
                         or self.ast_ignore_ownership()
+                        or self.ast_ignore_field_ownership(fop.property)
                     )
                     and (
                         fop.old_value != new_value
@@ -2906,6 +2907,10 @@ def ast_ignore_ownership(self) -> bool:
         """Whether to force generating an AST even though it isn't owned"""
         return False
 
+    def ast_ignore_field_ownership(self, field: str) -> bool:
+        """Whether to force generating an AST even though it isn't owned"""
+        return False
+
 
 class ObjectCommandContext(CommandContextToken[ObjectCommand[so.Object_T]]):