diff --git a/edb/schema/delta.py b/edb/schema/delta.py index 21f6ffb5627..035aa11388d 100644 --- a/edb/schema/delta.py +++ b/edb/schema/delta.py @@ -1622,6 +1622,12 @@ def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) self.new_types: Set[uuid.UUID] = set() + @classmethod + def from_commands(cls, *cmds: Command) -> DeltaRoot: + delta = DeltaRoot() + delta.update(cmds) + return delta + def apply( self, schema: s_schema.Schema, diff --git a/edb/server/bootstrap.py b/edb/server/bootstrap.py index 26431d58462..f8171430821 100644 --- a/edb/server/bootstrap.py +++ b/edb/server/bootstrap.py @@ -63,9 +63,11 @@ from edb.schema import delta as sd from edb.schema import extensions as s_exts from edb.schema import functions as s_func +from edb.schema import links as s_links from edb.schema import modules as s_mod from edb.schema import name as sn from edb.schema import objects as s_obj +from edb.schema import objtypes as s_objtypes from edb.schema import properties as s_props from edb.schema import reflection as s_refl from edb.schema import schema as s_schema @@ -2673,6 +2675,117 @@ async def _load_schema( ) +def _is_stdlib_target( + t: s_objtypes.ObjectType, + schema: s_schema.Schema, +) -> bool: + if intersection := t.get_intersection_of(schema): + return any((_is_stdlib_target(it, schema) + for it in intersection.objects(schema))) + elif union := t.get_union_of(schema): + return any((_is_stdlib_target(ut, schema) + for ut in union.objects(schema))) + + name = t.get_name(schema) + + if name == sn.QualName('std', 'Object'): + return False + return t.get_name(schema).get_module_name() in s_schema.STD_MODULES + + +async def _fixup_schema( + ctx: BootstrapContext, + schema: s_schema.ChainedSchema, + keys: dict[str, Any], +) -> None: + current_block = dbops.PLTopBlock() + backend_params = ctx.cluster.get_runtime_params() + + # Recompile functions that reference stdlib types (like + # std::BaseObject or schema::Object), since new subtypes may have + # been added. + to_recompile = schema._top_schema.get_objects(type=s_func.Function) + for func in to_recompile: + if func.get_name(schema).get_root_module_name() == s_schema.EXT_MODULE: + continue + # If none of the types referenced in the function are standard + # library types, we don't need to recompile. + if not ( + (expr := func.get_nativecode(schema)) + and expr.refs + and any( + isinstance(dep, s_objtypes.ObjectType) + and _is_stdlib_target(dep, schema) + for dep in expr.refs.objects(schema) + ) + ): + continue + + alter_func = func.init_delta_command( + schema, sd.AlterObject + ) + alter_func.set_attribute_value( + 'nativecode', func.get_nativecode(schema) + ) + alter_func.canonical = True + + # N.B: We are ignoring the schema changes, since we aren't + # updating the schema version. + _, plan, _ = _process_delta_params( + sd.DeltaRoot.from_commands(alter_func), + schema, + backend_params, + stdmode=False, + **keys, + ) + plan.generate(current_block) + + # Regenerate on_target_delete triggers for any links targeting a + # stdlib type. + links = schema._top_schema.get_objects(type=s_links.Link) + for link in links: + if link.get_name(schema).get_root_module_name() == s_schema.EXT_MODULE: + continue + source = link.get_source(schema) + if ( + not source + or not source.is_material_object_type(schema) + or link.get_computable(schema) + or link.get_shortname(schema).name == '__type__' + or not _is_stdlib_target(link.get_target(schema), schema) + ): + continue + + pol = link.get_on_target_delete(schema) + # HACK: Set the policy in a temporary in-memory schema to be + # something else, so that we can set it back to the real value + # and pgdelta will generate code for it. + fake_pol = ( + s_links.LinkTargetDeleteAction.Allow + if pol == s_links.LinkTargetDeleteAction.Restrict + else s_links.LinkTargetDeleteAction.Restrict + ) + fake_schema = link.set_field_value(schema, 'on_target_delete', fake_pol) + + alter_delta, alter_link, _ = link.init_delta_branch( + schema, sd.CommandContext(), sd.AlterObject + ) + alter_link.set_attribute_value('on_target_delete', pol) + + # N.B: We are ignoring the schema changes, since we aren't + # updating the schema version. + _, plan, _ = _process_delta_params( + sd.DeltaRoot.from_commands(alter_delta), + fake_schema, + backend_params, + stdmode=False, + **keys, + ) + plan.generate(current_block) + + await _execute_block(ctx.conn, current_block) + + async def _upgrade_one( ctx: BootstrapContext, state: edbcompiler.CompilerState, @@ -2704,15 +2817,15 @@ async def _upgrade_one( bootstrap_mode=False, # MAYBE? ) + keys: dict[str, Any] = dict( + testmode=True, + allow_dml_in_functions=True, + ) + # Apply the DDL, but *only* execute the schema storage part!! for ddl_cmd in edgeql.parse_block(ddl): current_block = dbops.PLTopBlock() - keys: dict[str, Any] = dict( - testmode=True, - allow_dml_in_functions=True, - ) - if debug.flags.sdl_loading: ddl_cmd.dump_edgeql() @@ -2768,6 +2881,8 @@ async def _upgrade_one( key = 'configspec_ext'; ''').encode('utf-8')) + await _fixup_schema(ctx, schema, keys) + async def _cleanup_one( ctx: BootstrapContext, @@ -2821,7 +2936,7 @@ async def _upgrade_all( # DEBUG VELOCITY HACK: You can add a failing database to EARLY # when trying to upgrade the whole suite. - EARLY: tuple[str, ...] = () + EARLY: tuple[str, ...] = ('dump01',) databases.sort(key=lambda k: (k not in EARLY, k)) for database in databases: diff --git a/tests/inplace-testing/test.sh b/tests/inplace-testing/test.sh index 877305cb60d..703a376416f 100755 --- a/tests/inplace-testing/test.sh +++ b/tests/inplace-testing/test.sh @@ -10,14 +10,18 @@ if ! git diff-index --quiet HEAD --; then exit 1 fi +make parsers + ./tests/inplace-testing/make-and-prep.sh "$DIR" "$@" tar cf "$DIR".tar "$DIR" patch -f -p1 < tests/inplace-testing/upgrade.patch +make parsers + edb server --bootstrap-only --inplace-upgrade "$DIR"/upgrade.json --data-dir "$DIR" tar cf "$DIR"-cooked.tar "$DIR" -edb test --data-dir "$DIR" --use-data-dir-dbs -v +edb test --data-dir "$DIR" --use-data-dir-dbs -v "$@" diff --git a/tests/inplace-testing/upgrade.patch b/tests/inplace-testing/upgrade.patch index ce5ab53a8a8..7f1be91e670 100644 --- a/tests/inplace-testing/upgrade.patch +++ b/tests/inplace-testing/upgrade.patch @@ -1,11 +1,11 @@ diff --git a/edb/buildmeta.py b/edb/buildmeta.py -index a56a33964..855e7730f 100644 +index 4d224737d..c6e1a1106 100644 --- a/edb/buildmeta.py +++ b/edb/buildmeta.py @@ -68,6 +68,12 @@ class MetadataError(Exception): pass - - + + +# HACK: Put this down here so it overrides the above version without +# merge conflicting with them. +EDGEDB_CATALOG_VERSION = 2030_01_01_00_00 @@ -15,14 +15,119 @@ index a56a33964..855e7730f 100644 class BackendVersion(NamedTuple): major: int minor: int +diff --git a/edb/edgeql/ast.py b/edb/edgeql/ast.py +index ab575e310..3ed277fb0 100644 +--- a/edb/edgeql/ast.py ++++ b/edb/edgeql/ast.py +@@ -1150,6 +1150,25 @@ class SetGlobalType(SetField): + reset_value: bool = False + + ++class BlobalCommand(ObjectDDL): ++ ++ __abstract_node__ = True ++ __rust_ignore__ = True ++ object_class: qltypes.SchemaObjectClass = qltypes.SchemaObjectClass.GLOBAL ++ ++ ++class CreateBlobal(CreateObject, BlobalCommand): ++ pass ++ ++ ++class AlterBlobal(AlterObject, BlobalCommand): ++ pass ++ ++ ++class DropBlobal(DropObject, BlobalCommand): ++ pass ++ ++ + class LinkCommand(ObjectDDL): + + __abstract_node__ = True +diff --git a/edb/edgeql/codegen.py b/edb/edgeql/codegen.py +index f65a16364..f80e11079 100644 +--- a/edb/edgeql/codegen.py ++++ b/edb/edgeql/codegen.py +@@ -2385,6 +2385,12 @@ class EdgeQLSourceGenerator(codegen.SourceGenerator): + def visit_DropGlobal(self, node: qlast.DropGlobal) -> None: + self._visit_DropObject(node, 'GLOBAL') + ++ def visit_CreateBlobal(self, node: qlast.CreateGlobal) -> None: ++ self._visit_CreateObject(node, 'PARTITION') ++ ++ def visit_DropBlobal(self, node: qlast.DropGlobal) -> None: ++ self._visit_DropObject(node, 'PARTITION') ++ + def visit_ConfigSet(self, node: qlast.ConfigSet) -> None: + if node.scope == qltypes.ConfigScope.GLOBAL: + self._write_keywords('SET GLOBAL ') +diff --git a/edb/edgeql/parser/grammar/ddl.py b/edb/edgeql/parser/grammar/ddl.py +index 40459e520..03ecf98c5 100644 +--- a/edb/edgeql/parser/grammar/ddl.py ++++ b/edb/edgeql/parser/grammar/ddl.py +@@ -246,6 +246,14 @@ class InnerDDLStmt(Nonterm): + def reduce_DropGlobalStmt(self, *_): + pass + ++ @parsing.inline(0) ++ def reduce_CreateBlobalStmt(self, *_): ++ pass ++ ++ @parsing.inline(0) ++ def reduce_DropBlobalStmt(self, *_): ++ pass ++ + @parsing.inline(0) + def reduce_DropCastStmt(self, *_): + pass +@@ -3431,6 +3439,38 @@ class DropGlobalStmt(Nonterm): + name=kids[2].val + ) + ++# ++# CREATE BLOBAL ++# ++ ++ ++commands_block( ++ 'CreateBlobal', ++ SetFieldStmt, ++ CreateAnnotationValueStmt, ++) ++ ++ ++class CreateBlobalStmt(Nonterm): ++ def reduce_CreateBlobal(self, *kids): ++ """%reduce ++ CREATE PARTITION NodeName ++ OptCreateBlobalCommandsBlock ++ """ ++ self.val = qlast.CreateBlobal( ++ name=kids[2].val, ++ commands=kids[3].val, ++ ) ++ ++ ++class DropBlobalStmt(Nonterm): ++ def reduce_DropBlobal(self, *kids): ++ r"""%reduce DROP PARTITION NodeName""" ++ self.val = qlast.DropBlobal( ++ name=kids[2].val ++ ) ++ ++ + # + # MIGRATIONS + # diff --git a/edb/lib/_testmode.edgeql b/edb/lib/_testmode.edgeql index 508382ecd..addd01b4f 100644 --- a/edb/lib/_testmode.edgeql +++ b/edb/lib/_testmode.edgeql @@ -214,6 +214,15 @@ create extension package _conf VERSION '1.0' { - + # std::_gen_series - + +CREATE FUNCTION +std::_upgrade_test( +) -> std::str @@ -36,10 +141,18 @@ index 508382ecd..addd01b4f 100644 std::_gen_series( `start`: std::int64, diff --git a/edb/lib/schema.edgeql b/edb/lib/schema.edgeql -index 1cd386fc0..9e515f5d0 100644 +index 2d7b92d31..61f64cba0 100644 --- a/edb/lib/schema.edgeql +++ b/edb/lib/schema.edgeql -@@ -536,6 +536,9 @@ CREATE TYPE schema::Global EXTENDING schema::AnnotationSubject { +@@ -533,9 +533,17 @@ CREATE TYPE schema::Global EXTENDING schema::AnnotationSubject { + }; + + ++CREATE TYPE schema::Blobal EXTENDING schema::AnnotationSubject { ++ CREATE PROPERTY required -> std::bool; ++}; ++ ++ CREATE TYPE schema::Function EXTENDING schema::CallableObject, schema::VolatilitySubject { @@ -49,6 +162,49 @@ index 1cd386fc0..9e515f5d0 100644 CREATE PROPERTY preserves_optionality -> std::bool { SET default := false; }; +diff --git a/edb/pgsql/delta.py b/edb/pgsql/delta.py +index 0555dd420..7f99636fe 100644 +--- a/edb/pgsql/delta.py ++++ b/edb/pgsql/delta.py +@@ -748,6 +748,38 @@ class DeleteGlobal( + pass + + ++class BlobalCommand(MetaCommand): ++ pass ++ ++ ++class CreateBlobal( ++ BlobalCommand, ++ adapts=s_globals.CreateBlobal, ++): ++ pass ++ ++ ++class RenameBlobal( ++ BlobalCommand, ++ adapts=s_globals.RenameBlobal, ++): ++ pass ++ ++ ++class AlterBlobal( ++ BlobalCommand, ++ adapts=s_globals.AlterBlobal, ++): ++ pass ++ ++ ++class DeleteBlobal( ++ BlobalCommand, ++ adapts=s_globals.DeleteBlobal, ++): ++ pass ++ ++ + class AccessPolicyCommand(MetaCommand): + pass + diff --git a/edb/schema/functions.py b/edb/schema/functions.py index 9cca3de6d..c22d3f414 100644 --- a/edb/schema/functions.py @@ -56,7 +212,7 @@ index 9cca3de6d..c22d3f414 100644 @@ -1237,6 +1237,27 @@ class Function( data_safe=True, ): - + + ## + test_field_a = so.SchemaField( + str, @@ -84,7 +240,7 @@ index 9cca3de6d..c22d3f414 100644 @@ -1608,6 +1629,10 @@ class FunctionCommand( nativecode.not_compiled() ) - + + if self.has_attribute_value('nativecode'): + code = self.get_attribute_value('nativecode') + self.set_attribute_value('test_nativecode_size', len(code.text)) @@ -92,14 +248,79 @@ index 9cca3de6d..c22d3f414 100644 # Resolving 'nativecode' has side effects on has_dml and # volatility, so force it to happen as part of # canonicalization of attributes. +diff --git a/edb/schema/globals.py b/edb/schema/globals.py +index d8d4ef67b..eb531ec85 100644 +--- a/edb/schema/globals.py ++++ b/edb/schema/globals.py +@@ -618,3 +618,60 @@ class DeleteGlobal( + GlobalCommand, + ): + astnode = qlast.DropGlobal ++ ++ ++class Blobal( ++ so.QualifiedObject, ++ s_anno.AnnotationSubject, ++ qlkind=qltypes.SchemaObjectClass.GLOBAL, ++ data_safe=True, ++): ++ ++ required = so.SchemaField( ++ bool, ++ default=False, ++ compcoef=0.909, ++ allow_ddl_set=True, ++ ) ++ ++ ++class BlobalCommandContext( ++ sd.ObjectCommandContext[so.Object], ++ s_anno.AnnotationSubjectCommandContext ++): ++ pass ++ ++ ++class BlobalCommand( ++ sd.QualifiedObjectCommand[Blobal], ++ context_class=BlobalCommandContext, ++): ++ pass ++ ++ ++class CreateBlobal( ++ sd.CreateObject[Blobal], ++ BlobalCommand, ++): ++ astnode = qlast.CreateBlobal ++ ++ ++class RenameBlobal( ++ sd.RenameObject[Blobal], ++ BlobalCommand, ++): ++ pass ++ ++ ++class AlterBlobal( ++ sd.AlterObject[Blobal], ++ BlobalCommand, ++): ++ astnode = qlast.AlterBlobal ++ ++ ++class DeleteBlobal( ++ sd.DeleteObject[Blobal], ++ BlobalCommand, ++): ++ astnode = qlast.DropBlobal diff --git a/edb/schema/operators.py b/edb/schema/operators.py -index a98cd3fec..bc9a3aea7 100644 +index 856225997..bc9a3aea7 100644 --- a/edb/schema/operators.py +++ b/edb/schema/operators.py @@ -68,12 +68,6 @@ class Operator( code = so.SchemaField( str, default=None, compcoef=0.4) - + - # An unused dummy field. We have this here to make it easier to - # test the *removal* of internal schema fields during in-place - # upgrades. @@ -109,3 +330,79 @@ index a98cd3fec..bc9a3aea7 100644 # If this is a derivative operator, *derivative_of* would # contain the name of the origin operator. # For example, the `std::IN` operator has `std::=` +diff --git a/tests/test_edgeql_select.py b/tests/test_edgeql_select.py +index 1927d16eb..0f1cb0f4b 100644 +--- a/tests/test_edgeql_select.py ++++ b/tests/test_edgeql_select.py +@@ -1960,6 +1960,18 @@ class TestEdgeQLSelect(tb.QueryTestCase): + ]), + ) + ++ async def test_edgeql_select_baseobject_function_01(self): ++ # HACK: special inplace-upgrade test ++ await self.con.execute(''' ++ create partition asdf { set required := true; }; ++ ''') ++ await self.assert_query_result( ++ r''' ++ select all_objects()[is schema::Blobal] { name }; ++ ''', ++ [{"name": "default::asdf"}], ++ ) ++ + async def test_edgeql_select_id_01(self): + # allow assigning id to a computed (#4781) + await self.con.query('SELECT schema::Type { XYZ := .id};') +diff --git a/tests/test_link_target_delete.py b/tests/test_link_target_delete.py +index 8982b3113..c08616dea 100644 +--- a/tests/test_link_target_delete.py ++++ b/tests/test_link_target_delete.py +@@ -307,6 +307,48 @@ class TestLinkTargetDeleteDeclarative(stb.QueryTestCase): + DELETE (SELECT Target1 FILTER .name = 'Target1.1'); + """) + ++ async def test_link_on_target_delete_restrict_schema_01(self): ++ # HACK: special inplace-upgrade test ++ async with self._run_and_rollback(): ++ await self.con.execute(""" ++ create partition asdf2 { set required := true; }; ++ ++ INSERT SchemaSource { ++ name := 'Source1.1', ++ schema_restrict := ( ++ SELECT schema::Blobal LIMIT 1 ++ ) ++ }; ++ """) ++ ++ with self.assertRaisesRegex( ++ edgedb.ConstraintViolationError, ++ 'prohibited by link'): ++ await self.con.execute(""" ++ DROP PARTITION asdf2; ++ """) ++ ++ async def test_link_on_target_delete_restrict_schema_02(self): ++ # HACK: special inplace-upgrade test ++ async with self._run_and_rollback(): ++ await self.con.execute(""" ++ create partition asdf2 { set required := true; }; ++ ++ INSERT SchemaSource { ++ name := 'Source1.1', ++ schema_m_restrict := ( ++ SELECT schema::Blobal LIMIT 1 ++ ) ++ }; ++ """) ++ ++ with self.assertRaisesRegex( ++ edgedb.ConstraintViolationError, ++ 'prohibited by link'): ++ await self.con.execute(""" ++ DROP PARTITION asdf2; ++ """) ++ + async def test_link_on_target_delete_deferred_restrict_01(self): + exception_is_deferred = False + diff --git a/tests/schemas/issues.esdl b/tests/schemas/issues.esdl index de8faec87ef..061f7c67300 100644 --- a/tests/schemas/issues.esdl +++ b/tests/schemas/issues.esdl @@ -161,3 +161,7 @@ function opt_test(tag: bool, x: optional int64) -> int64 using (x ?? -1); function opt_test(tag: int64, x: int64, y: optional int64) -> int64 using (y ?? -1); function opt_test(tag: bool, x: optional int64, y: optional int64) -> int64 using (y ?? -1); + +function all_objects() -> SET OF BaseObject { + USING (BaseObject) +} diff --git a/tests/schemas/link_tgt_del.esdl b/tests/schemas/link_tgt_del.esdl index 738da84a0cc..d56c4ff08a7 100644 --- a/tests/schemas/link_tgt_del.esdl +++ b/tests/schemas/link_tgt_del.esdl @@ -111,3 +111,12 @@ abstract type AbsSource1 extending Named { } type ChildSource1 extending AbsSource1; + +type SchemaSource extending Named { + link schema_restrict -> schema::Object { + on target delete restrict; + } + link schema_m_restrict -> schema::Object { + on target delete restrict; + } +}