From f14f4cef3692ad243a189448008aebad41beb721 Mon Sep 17 00:00:00 2001 From: Devon Campbell Date: Wed, 3 Jan 2024 09:27:15 -0500 Subject: [PATCH 01/25] Remove computed example that was not a computed (#6627) From 35a9388d443fd9a79d728ab119cf9bc77a26296d Mon Sep 17 00:00:00 2001 From: Devon Campbell Date: Wed, 3 Jan 2024 09:27:28 -0500 Subject: [PATCH 02/25] Docs: Fix auth guide links and note (#6629) * Fix built-in UI links * Style note as an admonition instead of signposting it with asterisks --- docs/guides/auth/email_password.rst | 2 +- docs/guides/auth/index.rst | 9 ++++++--- docs/guides/auth/oauth.rst | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/guides/auth/email_password.rst b/docs/guides/auth/email_password.rst index e8dcafab284..405edebc230 100644 --- a/docs/guides/auth/email_password.rst +++ b/docs/guides/auth/email_password.rst @@ -6,7 +6,7 @@ Email and password :edb-alt-title: Integrating EdgeDB Auth's email and password provider -Along with using the ``Built-in UI ``, you can also +Along with using the `Built-in UI `_, you can also create your own UI that calls to your own web application backend. UI considerations diff --git a/docs/guides/auth/index.rst b/docs/guides/auth/index.rst index 1dbe6a9c34b..a86a4ab73ff 100644 --- a/docs/guides/auth/index.rst +++ b/docs/guides/auth/index.rst @@ -229,9 +229,12 @@ provide those values and the ``additional_scope``: Provider to fulfill our minimal data needs. You can pass additional scope here in a space-separated string and we will request that additional scope when getting the authentication token from the - Identity Provider. \********Note:\*******\* We return this - authentication token with this scope from the Identity Provider when - we return our own authentication token. + Identity Provider. + + .. note:: + + We return this authentication token with this scope from the Identity + Provider when we return our own authentication token. You’ll also need to set a callback URL in each provider’s interface. To build this callback URL, you will need the hostname, port, and database name of your diff --git a/docs/guides/auth/oauth.rst b/docs/guides/auth/oauth.rst index db3548f3083..9a18063514f 100644 --- a/docs/guides/auth/oauth.rst +++ b/docs/guides/auth/oauth.rst @@ -6,7 +6,7 @@ OAuth :edb-alt-title: Integrating EdgeDB Auth's OAuth provider -Along with using the ``Built-in UI ``, you can also +Along with using the `Built-in UI `_, you can also create your own UI that calls to your own web application backend. UI considerations From 98f67c818cdbfd172940ed1813240fbae7cecfe6 Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Wed, 3 Jan 2024 16:43:02 -0800 Subject: [PATCH 03/25] Add a couple testmode config tunables for debugging (#6661) * __internal_no_apply_query_rewrites forces apply_query_rewrites to be false, disabling them in the same way that reflection queries do. * Use the "reflection schema" as the base schema instead of the normal std schema. This allows looking at all the schema fields that are hidden in the public introspection schema. Both of these (and *especially* the first) are things that I've manually done in the code enough times that I've gotten fed up with it. --- edb/buildmeta.py | 2 +- edb/lib/_testmode.edgeql | 15 +++++++++++++++ edb/server/compiler/compiler.py | 10 +++++++++- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/edb/buildmeta.py b/edb/buildmeta.py index 585ea2669f0..07aff7d6d42 100644 --- a/edb/buildmeta.py +++ b/edb/buildmeta.py @@ -44,7 +44,7 @@ # Increment this whenever the database layout or stdlib changes. -EDGEDB_CATALOG_VERSION = 2023_12_05_00_00 +EDGEDB_CATALOG_VERSION = 2024_01_03_00_00 EDGEDB_MAJOR_VERSION = 5 diff --git a/edb/lib/_testmode.edgeql b/edb/lib/_testmode.edgeql index e2c15643c08..964f469490d 100644 --- a/edb/lib/_testmode.edgeql +++ b/edb/lib/_testmode.edgeql @@ -78,6 +78,21 @@ ALTER TYPE cfg::AbstractConfig { SET default := false; }; + # Fully suppress apply_query_rewrites, like is done for internal + # reflection queries. + CREATE PROPERTY __internal_no_apply_query_rewrites -> std::bool { + CREATE ANNOTATION cfg::internal := 'true'; + SET default := false; + }; + + # Use the "reflection schema" as the base schema instead of the + # normal std schema. This allows looking at all the schema fields + # that are hidden in the public introspection schema. + CREATE PROPERTY __internal_query_reflschema -> std::bool { + CREATE ANNOTATION cfg::internal := 'true'; + SET default := false; + }; + CREATE PROPERTY __internal_restart -> std::bool { CREATE ANNOTATION cfg::internal := 'true'; CREATE ANNOTATION cfg::system := 'true'; diff --git a/edb/server/compiler/compiler.py b/edb/server/compiler/compiler.py index 37ea8601415..b029d64bfe5 100644 --- a/edb/server/compiler/compiler.py +++ b/edb/server/compiler/compiler.py @@ -1540,6 +1540,8 @@ def _get_compile_options( apply_query_rewrites=( not ctx.bootstrap_mode and not ctx.schema_reflection_mode + and not bool( + _get_config_val(ctx, '__internal_no_apply_query_rewrites')) ), apply_user_access_policies=_get_config_val( ctx, 'apply_access_policies'), @@ -1696,7 +1698,13 @@ def _compile_ql_query( is_explain = explain_data is not None current_tx = ctx.state.current_tx() - schema = current_tx.get_schema(ctx.compiler_state.std_schema) + base_schema = ( + ctx.compiler_state.std_schema + if not _get_config_val(ctx, '__internal_query_reflschema') + else ctx.compiler_state.refl_schema + ) + schema = current_tx.get_schema(base_schema) + options = _get_compile_options(ctx, is_explain=is_explain) ir = qlcompiler.compile_ast_to_ir( ql, From e84475c16865e52931215a7951286f1e38f89751 Mon Sep 17 00:00:00 2001 From: Devon Campbell Date: Thu, 4 Jan 2024 07:14:34 -0500 Subject: [PATCH 04/25] Document EDGEDB_DEBUG_HTTP_INJECT_CORS (#6659) --- docs/reference/environment.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/reference/environment.rst b/docs/reference/environment.rst index 3b772b7c11b..5f3bed3fe03 100644 --- a/docs/reference/environment.rst +++ b/docs/reference/environment.rst @@ -209,6 +209,16 @@ Server variables These variables will work whether you are running EdgeDB inside Docker or not. +EDGEDB_DEBUG_HTTP_INJECT_CORS +............................. + +Set to ``1`` to have EdgeDB send appropriate CORS headers with HTTP responses. + +.. note:: + + This is set to ``1`` by default for EdgeDB Cloud instances. + + .. _ref_reference_envvar_admin_ui: EDGEDB_SERVER_ADMIN_UI From 09a7f9420043eaf1c5109c18ce2e9ab508ba0d2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alja=C5=BE=20Mur=20Er=C5=BEen?= Date: Thu, 4 Jan 2024 18:54:41 +0100 Subject: [PATCH 05/25] Changelog 4.4 (#6651) --- docs/changelog/4_x.rst | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/changelog/4_x.rst b/docs/changelog/4_x.rst index 8d0a0e0785c..05813e3d169 100644 --- a/docs/changelog/4_x.rst +++ b/docs/changelog/4_x.rst @@ -602,7 +602,7 @@ Bug fixes * Optimize the compiler and reduce time of an update test by ~52% (:eql:gh:`#6579`) - + * Always retry system connection on any BackendError (:eql:gh:`#6588`) @@ -638,3 +638,14 @@ Bug fixes * Fix changing a pointer to be computed when there is a subtype (:eql:gh:`#6565`) + +4.4 +=== +* Fix DML-containing FOR when the iterator is an object set with duplicates + (:eql:gh:`#6609`) + +* Fix very slow global introspection query when there are lots of databases + (:eql:gh:`#6633`) + +* Use correct signature for in-memory cache method + (:eql:gh:`#6643`) From 23bb3957ec06f112130a07605f4f68cdb26cf9ba Mon Sep 17 00:00:00 2001 From: Dave MacLeod <56599343+Dhghomon@users.noreply.github.com> Date: Fri, 5 Jan 2024 23:33:06 +0900 Subject: [PATCH 06/25] Add force_database_error documentation (#6667) * Add force_database_error documentation * Add link to error types --- docs/stdlib/cfg.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/stdlib/cfg.rst b/docs/stdlib/cfg.rst index 31d98124bab..cec140f6a70 100644 --- a/docs/stdlib/cfg.rst +++ b/docs/stdlib/cfg.rst @@ -116,6 +116,29 @@ Query behavior UI session, so you won't have to remember to re-enable it when you're done. +:eql:synopsis:`force_database_error -> str` + A hook to force all queries to produce an error. Defaults to 'false'. + + .. note:: + + This parameter takes a ``str`` instead of a ``bool`` to allow more + verbose messages when all queries are forced to fail. The database will + attempt to deserialize this ``str`` into a JSON object that must include + a ``type`` (which must be an EdgeDB + :ref:`error type ` name), and may also include + ``message``, ``hint``, and ``details`` which can be set ad-hoc by + the user. + + For example, the following is valid input: + + ``'{ "type": "QueryError", + "message": "Did not work", + "hint": "Try doing something else", + "details": "Indeed, something went really wrong" }'`` + + As is this: + + ``'{ "type": "UnknownParameterError" }'`` .. _ref_std_cfg_client_connections: From 19308387f2f810e59873dfb5fc40f9955dc36d07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alja=C5=BE=20Mur=20Er=C5=BEen?= Date: Fri, 5 Jan 2024 19:28:10 +0100 Subject: [PATCH 07/25] Minor refactor, change pg_ast to pgast (#6672) --- edb/pgsql/delta.py | 54 +++++++++++++++++++++-------------------- edb/pgsql/schemamech.py | 39 ++++++++++++++--------------- 2 files changed, 48 insertions(+), 45 deletions(-) diff --git a/edb/pgsql/delta.py b/edb/pgsql/delta.py index a49dec641a0..20fd5331929 100644 --- a/edb/pgsql/delta.py +++ b/edb/pgsql/delta.py @@ -83,7 +83,7 @@ from edb.server import config from edb.server.config import ops as config_ops -from . import ast as pg_ast +from . import ast as pgast from .common import qname as q from .common import quote_literal as ql from .common import quote_ident as qi @@ -3640,7 +3640,7 @@ def create_index( # casts, strip them as they mess with the requirement that # index expressions are IMMUTABLE (also indexes expect the # usage of literals and will do their own implicit casts). - if isinstance(kw_sql_tree, pg_ast.TypeCast): + if isinstance(kw_sql_tree, pgast.TypeCast): kw_sql_tree = kw_sql_tree.arg sql = codegen.generate_source(kw_sql_tree) sql_kwarg_exprs[name] = sql @@ -4972,7 +4972,7 @@ def _compile_conversion_expr( backend_runtime_params=context.backend_runtime_params, ) sql_tree = sql_res.ast - assert isinstance(sql_tree, pg_ast.SelectStmt) + assert isinstance(sql_tree, pgast.SelectStmt) if produce_ctes: # ensure the result contains the object id in the second column @@ -4989,47 +4989,47 @@ def _compile_conversion_expr( if check_non_null: # wrap into raise_on_null pointer_name = 'link' if is_link else 'property' - msg = pg_ast.StringConstant( + msg = pgast.StringConstant( val=f"missing value for required {pointer_name}" ) # Concat to string which is a JSON. Great. Equivalent to SQL: # '{"object_id": "' || {obj_id_ref} || '"}' - detail = pg_ast.Expr( + detail = pgast.Expr( name='||', - lexpr=pg_ast.StringConstant(val='{"object_id": "'), - rexpr=pg_ast.Expr( + lexpr=pgast.StringConstant(val='{"object_id": "'), + rexpr=pgast.Expr( name='||', - lexpr=pg_ast.ColumnRef(name=('id', )), - rexpr=pg_ast.StringConstant(val='"}'), + lexpr=pgast.ColumnRef(name=('id',)), + rexpr=pgast.StringConstant(val='"}'), ) ) - column = pg_ast.StringConstant(val=str(pointer.id)) + column = pgast.StringConstant(val=str(pointer.id)) - null_check = pg_ast.FuncCall( + null_check = pgast.FuncCall( name=("edgedb", "raise_on_null"), args=[ - pg_ast.ColumnRef(name=("val", )), - pg_ast.StringConstant(val="not_null_violation"), - pg_ast.NamedFuncArg(name="msg", val=msg), - pg_ast.NamedFuncArg(name="detail", val=detail), - pg_ast.NamedFuncArg(name="column", val=column), + pgast.ColumnRef(name=("val",)), + pgast.StringConstant(val="not_null_violation"), + pgast.NamedFuncArg(name="msg", val=msg), + pgast.NamedFuncArg(name="detail", val=detail), + pgast.NamedFuncArg(name="column", val=column), ], ) inner_colnames = ["val"] - target_list = [pg_ast.ResTarget(val=null_check)] + target_list = [pgast.ResTarget(val=null_check)] if produce_ctes: inner_colnames.append("id") target_list.append( - pg_ast.ResTarget(val=pg_ast.ColumnRef(name=("id", ))) + pgast.ResTarget(val=pgast.ColumnRef(name=("id",))) ) - sql_tree = pg_ast.SelectStmt( + sql_tree = pgast.SelectStmt( target_list=target_list, from_clause=[ - pg_ast.RangeSubselect( + pgast.RangeSubselect( subquery=sql_tree, - alias=pg_ast.Alias( + alias=pgast.Alias( aliasname="_inner", colnames=inner_colnames ) ) @@ -5040,11 +5040,13 @@ def _compile_conversion_expr( if produce_ctes: # convert root query into last CTE - ctes.append(pg_ast.CommonTableExpr( - name="_conv_rel", - aliascolnames=["val", "id"], - query=sql_tree - )) + ctes.append( + pgast.CommonTableExpr( + name="_conv_rel", + aliascolnames=["val", "id"], + query=sql_tree, + ) + ) # compile to SQL ctes_sql = codegen.generate_ctes_source(ctes) diff --git a/edb/pgsql/schemamech.py b/edb/pgsql/schemamech.py index 8bfcede4da5..86263e3b0b3 100644 --- a/edb/pgsql/schemamech.py +++ b/edb/pgsql/schemamech.py @@ -46,7 +46,7 @@ from edb.common import ast from edb.common import parsing -from . import ast as pg_ast +from . import ast as pgast from . import dbops from . import deltadbops from . import common @@ -106,26 +106,29 @@ class ExprDataSources: plain_chunks: Sequence[str] -def _to_source(sql_expr: pg_ast.Base) -> str: +def _to_source(sql_expr: pgast.Base) -> str: src = codegen.generate_source(sql_expr) # ColumnRefs are the most common thing, and they should be safe to # skip parenthesizing, for deuglification purposes. anything else # we put parens around, to be sure. - if not isinstance(sql_expr, pg_ast.ColumnRef): + if not isinstance(sql_expr, pgast.ColumnRef): src = f'({src})' return src def _edgeql_tree_to_expr_data( - sql_expr: pg_ast.Base, refs: Optional[Set[pg_ast.ColumnRef]] = None + sql_expr: pgast.Base, refs: Optional[Set[pgast.ColumnRef]] = None ) -> ExprDataSources: if refs is None: - refs = set(ast.find_children( - sql_expr, pg_ast.ColumnRef, lambda n: len(n.name) == 1)) + refs = set( + ast.find_children( + sql_expr, pgast.ColumnRef, lambda n: len(n.name) == 1 + ) + ) plain_expr = _to_source(sql_expr) - if isinstance(sql_expr, (pg_ast.RowExpr, pg_ast.ImplicitRowExpr)): + if isinstance(sql_expr, (pgast.RowExpr, pgast.ImplicitRowExpr)): chunks = [] for elem in sql_expr.args: @@ -133,7 +136,7 @@ def _edgeql_tree_to_expr_data( else: chunks = [plain_expr] - if isinstance(sql_expr, pg_ast.ColumnRef): + if isinstance(sql_expr, pgast.ColumnRef): refs.add(sql_expr) for ref in refs: @@ -158,12 +161,12 @@ def _edgeql_ref_to_pg_constr( ) -> ExprData: sql_res = compiler.compile_ir_to_sql_tree(tree, singleton_mode=True) - sql_expr: pg_ast.Base - if isinstance(sql_res.ast, pg_ast.SelectStmt): + sql_expr: pgast.Base + if isinstance(sql_res.ast, pgast.SelectStmt): # XXX: use ast pattern matcher for this from_clause = sql_res.ast.from_clause[0] - assert isinstance(from_clause, pg_ast.RelRangeVar) - assert isinstance(from_clause.relation, pg_ast.CommonTableExpr) + assert isinstance(from_clause, pgast.RelRangeVar) + assert isinstance(from_clause.relation, pgast.CommonTableExpr) sql_expr = from_clause.relation.query.target_list[0].val else: sql_expr = sql_res.ast @@ -174,22 +177,20 @@ def _edgeql_ref_to_pg_constr( if isinstance(tree, irast.Set) and isinstance(tree.expr, irast.SelectStmt): tree = tree.expr.result - is_multicol = isinstance(sql_expr, (pg_ast.RowExpr, pg_ast.ImplicitRowExpr)) + is_multicol = isinstance(sql_expr, (pgast.RowExpr, pgast.ImplicitRowExpr)) # Determine if the sequence of references are all simple refs, not # expressions. This influences the type of Postgres constraint used. # - is_trivial = isinstance(sql_expr, pg_ast.ColumnRef) or ( - isinstance(sql_expr, (pg_ast.RowExpr, pg_ast.ImplicitRowExpr)) - and all(isinstance(el, pg_ast.ColumnRef) for el in sql_expr.args) + is_trivial = isinstance(sql_expr, pgast.ColumnRef) or ( + isinstance(sql_expr, (pgast.RowExpr, pgast.ImplicitRowExpr)) + and all(isinstance(el, pgast.ColumnRef) for el in sql_expr.args) ) # Find all field references # refs = set( - ast.find_children( - sql_expr, pg_ast.ColumnRef, lambda n: len(n.name) == 1 - ) + ast.find_children(sql_expr, pgast.ColumnRef, lambda n: len(n.name) == 1) ) if isinstance(subject, s_scalars.ScalarType): From c2d421163a95808b8d4d52ab11f7cd729e341475 Mon Sep 17 00:00:00 2001 From: Victor Petrovykh Date: Thu, 4 Jan 2024 23:47:03 -0500 Subject: [PATCH 08/25] Reduce the size of data for analyze tests. Instead of generating a large dataset to force usage of indexes, disable sequential scan and use a smaller data set. Fixes #6316 --- tests/schemas/explain_setup.edgeql | 18 +- tests/test_edgeql_explain.py | 1163 +++++++++++++++------------- 2 files changed, 645 insertions(+), 536 deletions(-) diff --git a/tests/schemas/explain_setup.edgeql b/tests/schemas/explain_setup.edgeql index 5ff5a1806f1..463ce0ad51a 100644 --- a/tests/schemas/explain_setup.edgeql +++ b/tests/schemas/explain_setup.edgeql @@ -133,10 +133,18 @@ SET { }; -# Make the database more populated so that it uses indexes... -for i in range_unpack(range(1, 1000)) union ( +# Add a function that disables sequential scan. +create function _set_seqscan(val: std::str) -> std::str { + using sql $$ + select set_config('enable_seqscan', val, true) + $$; +}; + + +# Generate some data ... +for i in range_unpack(range(1, 10)) union ( with u := (insert User { name := i }), - for j in range_unpack(range(0, 5)) union ( + for j in range_unpack(range(0, 3)) union ( insert Issue { owner := u, number := (i*100 + j), @@ -150,7 +158,7 @@ update User set { }; -for x in range_unpack(range(0, 900_000)) union ( +for x in range_unpack(range(0, 100)) union ( insert RangeTest { rval := range(-(x * 101419 % 307), x * 201881 % 307), mval := multirange([ @@ -181,7 +189,7 @@ for x in range_unpack(range(0, 900_000)) union ( ); -for x in range_unpack(range(0, 100_000)) union ( +for x in range_unpack(range(0, 100)) union ( insert JSONTest{val := (a:=x, b:=x * 40123 % 10007)} ); diff --git a/tests/test_edgeql_explain.py b/tests/test_edgeql_explain.py index c8edcad0364..49c739bb796 100644 --- a/tests/test_edgeql_explain.py +++ b/tests/test_edgeql_explain.py @@ -29,7 +29,7 @@ class TestEdgeQLExplain(tb.QueryTestCase): - '''Tests for EXPLAIN. + '''Tests for ANALYZE. This is a good way of testing explain functionality, but also this can be used to test indexes. @@ -54,8 +54,12 @@ def assert_plan(self, data, shape, message=None): data, shape, fail=self.fail, message=message) async def explain(self, query, *, execute=True, con=None): + con = (con or self.con) + # Disable sequential scan so that we hit the index even on small + # datasets. + await con.query_single('select _set_seqscan("off")') no_ex = '(execute := False) ' if not execute else '' - return json.loads(await (con or self.con).query_single( + return json.loads(await con.query_single( f'analyze {no_ex}{query}' )) @@ -110,7 +114,6 @@ async def test_edgeql_explain_simple_01(self): "type": "expr", }, ]), - "startup_cost": float, } ], "subplans": [], @@ -138,214 +141,207 @@ async def test_edgeql_explain_with_bound_01(self): ''') shape = { - "contexts": [ - {"buffer_idx": 0, "end": 35, "start": 31, "text": "User"}], - "pipeline": [ - { - "actual_loops": 1, - "actual_rows": 1, - "plan_rows": 1, - "plan_type": "SubqueryScan", - "properties": tb.bag([ - { - "important": False, - "title": "filter", - "type": "expr", - }, - ]), - "startup_cost": float, - "total_cost": float, - } - ], - "subplans": [ + "contexts": [{ + "start": 31, + "end": 35, + "buffer_idx": 0, + "text": "User", + }], + "pipeline": [{ + "plan_rows": 1, + "actual_rows": 1, + "actual_loops": 1, + "plan_type": "SubqueryScan", + "properties": [{ + "title": "filter", + "type": "expr", + "important": False, + }], + # Just validating that these fields appear. This was part of + # the early tests and these fields are something the users may + # rely on and should be part of stable API. + "startup_cost": float, + "total_cost": float, + }], + "subplans": tb.bag([ { - "contexts": [ - { - "buffer_idx": 0, - "end": 115, - "start": 74, - } - ], - "pipeline": [ + "contexts": [{ + "start": 74, + "end": 115, + "buffer_idx": 0, + "text": "elvis := (select U filter .name like 'E%'", + }], + "pipeline": tb.bag([ { - "actual_loops": 1, - "actual_rows": 1, "plan_rows": 1, + "actual_rows": 1, + "actual_loops": 1, "plan_type": "Aggregate", "properties": tb.bag([ { - "important": False, "title": "parent_relationship", "type": "text", - "value": "InitPlan", + "important": False, }, { - "important": False, "title": "subplan_name", "type": "text", - "value": "InitPlan 1 (returns " "$0)", + "important": False, }, { - "important": True, "title": "strategy", - "type": "text", "value": "Plain", + "type": "text", + "important": True, }, { - "important": True, "title": "partial_mode", - "type": "text", "value": "Simple", + "type": "text", + "important": True, }, ]), - "startup_cost": float, - "total_cost": float, }, { - "actual_loops": 1, - "actual_rows": 1, "plan_rows": 1, + "actual_rows": 1, + "actual_loops": 1, "plan_type": "IndexScan", "properties": tb.bag([ { - "important": False, "title": "filter", "type": "expr", + "important": False, }, { - "important": False, "title": "parent_relationship", - "type": "text", "value": "Outer", + "type": "text", + "important": False, }, { - "important": False, "title": "schema", - "type": "text", "value": "edgedbpub", + "type": "text", + "important": False, }, { - "important": False, "title": "alias", + "value": 'User~3', "type": "text", - "value": "User~3", + "important": False, }, { - "important": True, "title": "relation_name", + "value": 'User', "type": "relation", + "important": True, }, { - "important": True, "title": "scan_direction", - "type": "text", "value": "Forward", + "type": "text", + "important": True, }, { - "important": True, "title": "index_name", + "value": + "index of object type 'default::User' " + "on (__subject__.name)", "type": "index", - "value": "index of object type " - "'default::User' on " - "(__subject__.name)", + "important": True, }, { - "important": False, "title": "index_cond", "type": "expr", + "important": False, }, ]), - "startup_cost": float, - "total_cost": float, }, - ], + ]), "subplans": [], }, { - "contexts": [ - { - "buffer_idx": 0, - "end": 173, - "start": 134, - }, - ], - "pipeline": [ + "contexts": [{ + "start": 134, + "end": 173, + "buffer_idx": 0, + "text": "yury := (select U filter .name[0] = 'Y'", + }], + "pipeline": tb.bag([ { - "actual_loops": 1, - "actual_rows": 1, "plan_rows": 1, + "actual_rows": 1, + "actual_loops": 1, "plan_type": "Aggregate", "properties": tb.bag([ { - "important": False, "title": "parent_relationship", "type": "text", - "value": "InitPlan", + "important": False, }, { - "important": False, "title": "subplan_name", "type": "text", - "value": "InitPlan 2 (returns " "$1)", + "important": False, }, { - "important": True, "title": "strategy", - "type": "text", "value": "Plain", + "type": "text", + "important": True, }, { - "important": True, "title": "partial_mode", - "type": "text", "value": "Simple", + "type": "text", + "important": True, }, ]), - "startup_cost": float, - "total_cost": float, }, { - "actual_loops": 1, + "plan_rows": 1, "actual_rows": 1, - "plan_rows": 5, + "actual_loops": 1, "plan_type": "SeqScan", "properties": tb.bag([ { - "important": False, "title": "filter", "type": "expr", + "important": False, }, { - "important": False, "title": "parent_relationship", - "type": "text", "value": "Outer", + "type": "text", + "important": False, }, { - "important": False, "title": "schema", - "type": "text", "value": "edgedbpub", + "type": "text", + "important": False, }, { - "important": False, "title": "alias", + "value": 'User~7', "type": "text", + "important": False, }, { - "important": True, "title": "relation_name", + "value": 'User', "type": "relation", + "important": True, }, ]), - "startup_cost": float, - "total_cost": float, - }, - ], + } + ]), "subplans": [], - }, - ], + } + ]), } + self.assert_plan(res['fine_grained'], shape) async def test_edgeql_explain_multi_link_01(self): @@ -355,393 +351,430 @@ async def test_edgeql_explain_multi_link_01(self): ''') shape = { - "contexts": [{ - "buffer_idx": 0, - "end": 32, - "start": 28, - "text": "User", - }], + "contexts": [ + { + "start": 28, + "end": 32, + "buffer_idx": 0, + "text": "User", + }, + ], "pipeline": [ { - "actual_loops": 1, - "actual_rows": 1, "plan_rows": 1, + "actual_rows": 1, + "actual_loops": 1, "plan_type": "IndexScan", "properties": tb.bag([ { - "important": False, "title": "schema", - "type": "text", "value": "edgedbpub", + "type": "text", + "important": False, }, { - "important": False, "title": "alias", - "type": "text", "value": "User~2", + "type": "text", + "important": False, }, { - "important": True, "title": "relation_name", + "value": "User", "type": "relation", + "important": True, }, { - "important": True, "title": "scan_direction", - "type": "text", "value": "Forward", + "type": "text", + "important": True, }, { - "important": True, "title": "index_name", + "value": + "index of object type 'default::User' " + "on (__subject__.name)", "type": "index", - "value": "index of object type 'default::User' " - "on (__subject__.name)", + "important": True, }, { - "important": False, "title": "index_cond", "type": "expr", + "important": False, }, ]), - } + }, ], "subplans": [ { - "contexts": [{ - "buffer_idx": 0, - "end": 45, - "start": 41, - "text": "todo" - }], - "pipeline": [ + "contexts": [ + { + "start": 41, + "end": 45, + "buffer_idx": 0, + "text": "todo", + }, + ], + "pipeline": tb.bag([ { - "actual_loops": 1, - "actual_rows": 1, "plan_rows": 1, + "actual_rows": 1, + "actual_loops": 1, "plan_type": "Aggregate", "properties": tb.bag([ { - "important": False, "title": "parent_relationship", - "type": "text", "value": "SubPlan", + "type": "text", + "important": False, }, { - "important": False, "title": "subplan_name", "type": "text", - "value": "SubPlan 1", + "important": False, }, { - "important": True, "title": "strategy", - "type": "text", "value": "Plain", + "type": "text", + "important": True, }, { - "important": True, "title": "partial_mode", - "type": "text", "value": "Simple", + "type": "text", + "important": True, }, ]), }, { - "actual_loops": 1, + "plan_rows": 1, "actual_rows": 2, - "plan_rows": 2, + "actual_loops": 1, "plan_type": "NestedLoop", "properties": tb.bag([ { - "important": False, "title": "parent_relationship", - "type": "text", "value": "Outer", + "type": "text", + "important": False, }, { - "important": True, "title": "join_type", - "type": "text", "value": "Inner", + "type": "text", + "important": True, }, ]), }, - ], - "subplans": [ + ]), + "subplans": tb.bag([ { "pipeline": [ { - "actual_loops": 1, + "plan_rows": 1, "actual_rows": 2, - "plan_rows": 2, + "actual_loops": 1, "plan_type": "IndexOnlyScan", - # This has property `heap_fetches` - # that vary on github an locally. - # So skip checking "properties" - } - ], - "subplans": [], - }, - { - "pipeline": [ - { - "actual_loops": 2, - "actual_rows": 1, - "plan_rows": 1, - "plan_type": "IndexScan", "properties": tb.bag([ { - "important": False, "title": "parent_relationship", + "value": "Outer", "type": "text", - "value": "Inner", + "important": False, }, { - "important": False, "title": "schema", - "type": "text", "value": "edgedbpub", + "type": "text", + "important": False, }, { - "important": False, "title": "alias", + "value": "todo~1", "type": "text", - "value": "Issue~1", + "important": False, }, { - "important": True, "title": "relation_name", + "value": "User.todo", "type": "relation", + "important": True, }, { - "important": True, "title": "scan_direction", - "type": "text", "value": "Forward", + "type": "text", + "important": True, }, { - "important": True, "title": "index_name", + "value": + "User.todo forward " + "link index", "type": "index", - "value": "constraint " - "'std::exclusive' " - "of " - "property " - "'id' of " - "object " - "type " - "'default::Issue'", + "important": True, }, { - "important": False, "title": "index_cond", "type": "expr", - "value": '("Issue~1".id ' - "= " - '"todo~1".target)', + "important": False, + }, + { + "title": "heap_fetches", + "type": "float", + "important": False, }, ]), - } + }, ], "subplans": [], }, - ], - } - ], - } - self.assert_plan(res['fine_grained'], shape) - - async def test_edgeql_explain_computed_backlink_01(self): - res = await self.explain(''' - select User { name, owned_issues: {name, number} } - filter .name = 'Elvis'; - ''') - - shape = { - "contexts": [{ - "buffer_idx": 0, - "end": 32, - "start": 28, - "text": "User", - }], - "pipeline": [ - { - "actual_loops": 1, - "actual_rows": 1, - "plan_rows": 1, - "plan_type": "IndexScan", - "properties": tb.bag([ - { - "important": False, - "title": "schema", - "type": "text", - "value": "edgedbpub", - }, - { - "important": False, - "title": "alias", - "type": "text", - "value": "User~2", - }, - { - "important": True, - "title": "relation_name", - "type": "relation", - }, - { - "important": True, - "title": "scan_direction", - "type": "text", - "value": "Forward", - }, - { - "important": True, - "title": "index_name", - "type": "index", - "value": "index of object type 'default::User' " - "on (__subject__.name)", - }, - { - "important": False, - "title": "index_cond", - "type": "expr", - }, - ]), - "startup_cost": float, - "total_cost": float, - } - ], - "subplans": [ - { - "contexts": [ - { - "buffer_idx": 1, - "end": 26, - "start": 0, - "text": ". Date: Wed, 10 Jan 2024 07:12:19 -0800 Subject: [PATCH 09/25] Remove Pydantic warning from FastAPI guide (#6683) --- docs/guides/tutorials/rest_apis_with_fastapi.rst | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/docs/guides/tutorials/rest_apis_with_fastapi.rst b/docs/guides/tutorials/rest_apis_with_fastapi.rst index 9db0cba80f5..4c6452a5c03 100644 --- a/docs/guides/tutorials/rest_apis_with_fastapi.rst +++ b/docs/guides/tutorials/rest_apis_with_fastapi.rst @@ -6,16 +6,6 @@ FastAPI :edb-alt-title: Building a REST API with EdgeDB and FastAPI -.. warning:: - - FastAPI currently has some compatibility issues with Pydantic V2. As they - work to iron out those issues, you may need to fall back to older Pydantic - to run this example project. - - .. code-block:: bash - - $ pip install --force-reinstall -v "pydantic==1.10" - Because FastAPI encourages and facilitates strong typing, it's a natural pairing with EdgeDB. Our Python code generation generates not only typed query functions but result types you can use to annotate your endpoint handler From 6359f3e7cc0a130d6658f7a9af1621f41a7d73f1 Mon Sep 17 00:00:00 2001 From: Devon Campbell Date: Thu, 11 Jan 2024 09:07:28 -0800 Subject: [PATCH 10/25] Note that if..else now supports DML (#6660) * Note that if..else now supports DML * Clarify the workaround is for earlier versions --- docs/edgeql/for.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/edgeql/for.rst b/docs/edgeql/for.rst index 4efca8cfce7..25a112a7def 100644 --- a/docs/edgeql/for.rst +++ b/docs/edgeql/for.rst @@ -74,6 +74,14 @@ A similar approach can be used for bulk updates. Conditional DML --------------- +.. versionadded:: 4.0 + + DML is now supported in ``if..else``. The method of achieving conditional + DML demonstrated below is a workaround for earlier versions of EdgeDB + before this support was introduced in EdgeDB 4.0. If you're on EdgeDB 4.0 + or higher, use :eql:op:`if..else` for a cleaner way to achieve conditional + DML. + DML (i.e., :ref:`insert `, :ref:`update `, :ref:`delete `) is not supported in :eql:op:`if..else`. If you need to do one of these conditionally, you can use a ``for`` loop as a From 45723c1c25b8a0a3dcb9b668030b370802f48e69 Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Fri, 12 Jan 2024 06:39:18 +0800 Subject: [PATCH 11/25] Docs: Some tweaks (#6679) * Update schema.rst remove the extra "space" * Update schema.rst add `int16` * Update edgeql.rst add `int16` * Update constraints.rst User(x) -> Player(o) * Update schema.rst Fix "Line longer than 79 characters" test failure. * Update edgeql.rst Fix "Line longer than 79 characters" test failure. --- docs/datamodel/constraints.rst | 2 +- docs/intro/edgeql.rst | 3 ++- docs/intro/schema.rst | 5 +++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/datamodel/constraints.rst b/docs/datamodel/constraints.rst index 131ce0ece86..ea4917707f0 100644 --- a/docs/datamodel/constraints.rst +++ b/docs/datamodel/constraints.rst @@ -376,7 +376,7 @@ player picks in a color-based memory game: required name: str; } -This constraint ensures that a single ``User`` cannot pick two ``Color``\s at +This constraint ensures that a single ``Player`` cannot pick two ``Color``\s at the same ``@order``. .. _ref_datamodel_constraints_scalars: diff --git a/docs/intro/edgeql.rst b/docs/intro/edgeql.rst index 2dfb99c034e..84e2a3b3b67 100644 --- a/docs/intro/edgeql.rst +++ b/docs/intro/edgeql.rst @@ -31,7 +31,8 @@ EdgeDB has a rich primitive type system consisting of the following data types. * - Booleans - ``bool`` * - Numbers - - ``int32`` ``int64`` ``float32`` ``float64`` ``bigint`` ``decimal`` + - ``int16`` ``int32`` ``int64`` ``float32`` ``float64`` + ``bigint`` ``decimal`` * - UUID - ``uuid`` * - JSON diff --git a/docs/intro/schema.rst b/docs/intro/schema.rst index bb5106a3af4..0402c68c38b 100644 --- a/docs/intro/schema.rst +++ b/docs/intro/schema.rst @@ -22,7 +22,8 @@ types. * - Booleans - ``bool`` * - Numbers - - ``int32`` ``int64`` ``float32`` ``float64`` ``bigint`` ``decimal`` + - ``int16`` ``int32`` ``int64`` ``float32`` ``float64`` + ``bigint`` ``decimal`` * - UUID - ``uuid`` * - JSON @@ -366,7 +367,7 @@ understand backlink syntax is to split it into two parts: ``[is Movie]`` This is a *type filter* that filters out all objects that aren't ``Movie`` objects. A backlink still works without this filter, but could contain any - other number of objects besides `` Movie`` objects. + other number of objects besides ``Movie`` objects. See :ref:`Schema > Computeds > Backlinks `. From 75dccf2a219d00d037a45ddef100a1463fb33c30 Mon Sep 17 00:00:00 2001 From: Frederick Date: Tue, 16 Jan 2024 10:12:35 -0700 Subject: [PATCH 12/25] Update fly.io deployment guide (#6635) There were two main aspects that were broken in this guide. - The fly tooling really wants you to have a `fly.toml` file now. - The 512MB of memory for the postgres VM was not enough for edgedb to bootstrap successfully. --- docs/guides/deployment/fly_io.rst | 50 ++++++++++++++----------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/docs/guides/deployment/fly_io.rst b/docs/guides/deployment/fly_io.rst index b392ad78107..b98d0688c16 100644 --- a/docs/guides/deployment/fly_io.rst +++ b/docs/guides/deployment/fly_io.rst @@ -10,11 +10,6 @@ In this guide we show how to deploy EdgeDB using a `Fly.io `_ PostgreSQL cluster as the backend. The deployment consists of two apps: one running Postgres and the other running EdgeDB. -.. note:: - - At the moment, it isn't possible to expose Fly-hosted EdgeDB instances to - the public internet, only internally to other Fly projects. As such your - application must also be hosted on Fly. Prerequisites ============= @@ -55,7 +50,7 @@ we'll need. There are a couple more environment variables we need to set: .. code-block:: bash $ flyctl secrets set \ - EDGEDB_PASSWORD="$PASSWORD" \ + EDGEDB_SERVER_PASSWORD="$PASSWORD" \ EDGEDB_SERVER_BACKEND_DSN_ENV=DATABASE_URL \ EDGEDB_SERVER_TLS_CERT_MODE=generate_self_signed \ EDGEDB_SERVER_PORT=8080 \ @@ -77,16 +72,20 @@ Let's discuss what's going on with all these secrets. of the default 5656, because Fly.io prefers ``8080`` for its default health checks. -Finally, let's scale the VM as EdgeDB requires a little bit more than the -default Fly.io VM side provides: +Finally, let's configure the VM size as EdgeDB requires a little bit more than +the default Fly.io VM side provides. Put this in a file called ``fly.toml`` in +your current directory.: -.. code-block:: bash +.. code-block:: yaml + + [build] + image = "edgedb/edgedb" + + [[vm]] + memory = "512mb" + cpus = 1 + cpu-kind = "shared" - $ flyctl scale vm shared-cpu-1x --memory=1024 --app $EDB_APP - Scaled VM Type to - shared-cpu-1x - CPU Cores: 1 - Memory: 1 GB Create a PostgreSQL cluster =========================== @@ -128,7 +127,7 @@ this command: .. code-block:: bash - $ flyctl machine update --memory 512 --app $PG_APP -y + $ flyctl machine update --memory 1024 --app $PG_APP -y Searching for image 'flyio/postgres:14.6' remotely... image found: img_0lq747j0ym646x35 Image: registry-1.docker.io/flyio/postgres:14.6 @@ -173,11 +172,12 @@ Everything is set! Time to start EdgeDB. .. code-block:: bash - $ flyctl deploy --image=edgedb/edgedb \ - --remote-only --app $EDB_APP + $ flyctl deploy --remote-only --app $EDB_APP ... - 1 desired, 1 placed, 1 healthy, 0 unhealthy - --> v0 deployed successfully + Finished launching new machines + ------- + ✔ Machine e286630dce9638 [app] was created + ------- That's it! You can now start using the EdgeDB instance located at ``edgedb://myorg-edgedb.internal`` in your Fly.io apps. @@ -185,7 +185,7 @@ That's it! You can now start using the EdgeDB instance located at If deploy did not succeed: -1. make sure you've scaled the EdgeDB VM +1. make sure you've created the ``fly.toml`` file. 2. re-run the ``deploy`` command 3. check the logs for more information: ``flyctl logs --app $EDB_APP`` @@ -268,14 +268,8 @@ From external application If you need to access EdgeDB from outside the Fly.io network, you'll need to configure the Fly.io proxy to let external connections in. -First, save the EdgeDB app config in an **empty directory**: - -.. code-block:: bash - - $ flyctl config save -a $EDB_APP - -A ``fly.toml`` file will be created upon result. Let's make sure our -``[[services]]`` section looks something like this: +Let's make sure the ``[[services]]`` section in our ``fly.toml`` looks +something like this: .. code-block:: toml From 096f791403ef673581303b227af1b0b089cedfb4 Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Tue, 16 Jan 2024 17:15:06 -0800 Subject: [PATCH 13/25] Speed up introspection of system objects like Database and Role (#6665) Currently if there are a lot of roles, doing a query like `select sys::Role { member_of }` is quite slow. This is because the views for both `Role` and `member_of` need to extract metadata for *every* Role and pase it as json, and postgres is planning this as a nested loop. Unfortunately, startup needs to query all these objects, so having lots of databases or roles is a problem. Fix this by forcing sys objects and links to always be materialized into CTEs. This ensures that we do at most one linear scan of each of the views. Since there is no way to hit an index when querying any of these system object views, always doing a linear scan is no worse. For the objects themselves we can leverage the rewrites mechanism. For link tables, we add a slightly lighter weight mechanism to handle it. Reverts the now-unnecessary main part of #6633 but keeps the extension to the patch system. We could actually put annotations on databases now and have it not be *too* slow. This can be backported with an empty `edgeql+schema+globalonly` patch. --- edb/pgsql/compiler/clauses.py | 7 +++- edb/pgsql/compiler/context.py | 8 +++- edb/pgsql/compiler/relctx.py | 66 ++++++++++++++++++++++++++++-- edb/schema/reflection/structure.py | 15 ------- edb/server/bootstrap.py | 6 ++- 5 files changed, 79 insertions(+), 23 deletions(-) diff --git a/edb/pgsql/compiler/clauses.py b/edb/pgsql/compiler/clauses.py index cb5b616a5f2..b8bf0fa4cf0 100644 --- a/edb/pgsql/compiler/clauses.py +++ b/edb/pgsql/compiler/clauses.py @@ -409,8 +409,11 @@ def fini_toplevel( # Type rewrites go first. if stmt.ctes is None: stmt.ctes = [] - stmt.ctes[:0] = list(ctx.param_ctes.values()) - stmt.ctes[:0] = list(ctx.type_ctes.values()) + stmt.ctes[:0] = [ + *ctx.param_ctes.values(), + *ctx.ptr_ctes.values(), + *ctx.type_ctes.values(), + ] if ctx.env.named_param_prefix is None: # Adding unused parameters into a CTE diff --git a/edb/pgsql/compiler/context.py b/edb/pgsql/compiler/context.py index 39ba9294d3b..14a270aa4ca 100644 --- a/edb/pgsql/compiler/context.py +++ b/edb/pgsql/compiler/context.py @@ -220,7 +220,11 @@ class CompilerContextLevel(compiler.ContextLevel): #: CTEs representing decoded parameters param_ctes: Dict[str, pgast.CommonTableExpr] - #: CTEs representing schema types, when rewritten based on access policy + #: CTEs representing pointer tables that we need to force to be + #: materialized for performance reasons. + ptr_ctes: Dict[uuid.UUID, pgast.CommonTableExpr] + + #: CTEs representing types, when rewritten based on access policy type_ctes: Dict[FullRewriteKey, pgast.CommonTableExpr] #: A set of type CTEs currently being generated @@ -322,6 +326,7 @@ def __init__( self.rel = NO_STMT self.rel_hierarchy = {} self.param_ctes = {} + self.ptr_ctes = {} self.type_ctes = {} self.pending_type_ctes = set() self.dml_stmts = {} @@ -361,6 +366,7 @@ def __init__( self.rel = prevlevel.rel self.rel_hierarchy = prevlevel.rel_hierarchy self.param_ctes = prevlevel.param_ctes + self.ptr_ctes = prevlevel.ptr_ctes self.type_ctes = prevlevel.type_ctes self.pending_type_ctes = prevlevel.pending_type_ctes self.dml_stmts = prevlevel.dml_stmts diff --git a/edb/pgsql/compiler/relctx.py b/edb/pgsql/compiler/relctx.py index eb07100fe24..ce266837802 100644 --- a/edb/pgsql/compiler/relctx.py +++ b/edb/pgsql/compiler/relctx.py @@ -1360,6 +1360,21 @@ def _lateral_union_join( type='cross', larg=larg, rarg=rarg) +def _needs_cte(typeref: irast.TypeRef) -> bool: + """Check whether a typeref needs to be forced into a materialized CTE. + + The main use case here is for sys::SystemObjects which are stored as + views that populate their data by parsing JSON metadata embedded in + comments on the SQL system objects. The query plans when fetching multi + links from these objects wind up being pretty pathologically quadratic. + + So instead we force the objects and links into materialized CTEs + so that they *can't* be shoved into nested loops. + """ + assert isinstance(typeref.name_hint, sn.QualName) + return typeref.name_hint.module == 'sys' + + def range_for_material_objtype( typeref: irast.TypeRef, path_id: irast.PathId, @@ -1393,12 +1408,24 @@ def range_for_material_objtype( ) rw_key = (typeref.id, include_descendants) key = rw_key + (dml_source_key,) + force_cte = _needs_cte(typeref) if ( (not ignore_rewrites or is_global) - and (rewrite := ctx.env.type_rewrites.get(rw_key)) is not None + and ( + (rewrite := ctx.env.type_rewrites.get(rw_key)) is not None + or force_cte + ) and rw_key not in ctx.pending_type_ctes and not for_mutation ): + if not rewrite: + # If we are forcing CTE materialization but there is not a + # real rewrite, then create a trivial one. + rewrite = irast.Set( + path_id=irast.PathId.from_typeref(typeref, namespace={'rw'}), + typeref=typeref, + ) + # Don't include overlays in the normal way in trigger mode # when a type cte is used, because we bake the overlays into # the cte instead (and so including them normally could union @@ -1433,9 +1460,9 @@ def range_for_material_objtype( type_rel = sctx.rel else: type_cte = pgast.CommonTableExpr( - name=ctx.env.aliases.get('t'), + name=ctx.env.aliases.get(f't_{typeref.name_hint}'), query=sctx.rel, - materialized=is_global, + materialized=is_global or force_cte, ) ctx.type_ctes[key] = type_cte type_rel = type_cte @@ -1789,6 +1816,36 @@ def range_from_queryset( return rvar +def _make_link_table_cte( + ptrref: irast.PointerRef, + relation: pgast.Relation, + *, + ctx: context.CompilerContextLevel, +) -> pgast.Relation: + if (ptr_cte := ctx.ptr_ctes.get(ptrref.id)) is None: + ptr_select = pgast.SelectStmt( + target_list=[ + pgast.ResTarget(val=pgast.ColumnRef(name=[pgast.Star()])), + ], + from_clause=[pgast.RelRangeVar(relation=relation)], + ) + ptr_cte = pgast.CommonTableExpr( + name=ctx.env.aliases.get(f'p_{ptrref.name}'), + query=ptr_select, + materialized=True, + ) + ctx.ptr_ctes[ptrref.id] = ptr_cte + + # We've set up the CTE to be a perfect drop in replacement for + # the real table (with a select *), so instead of pointing the + # rvar at the CTE (which would require routing path ids), just + # treat it like a base relation. + return pgast.Relation( + name=ptr_cte.name, + type_or_ptr_ref=ptrref, + ) + + def table_from_ptrref( ptrref: irast.PointerRef, ptr_info: pg_types.PointerStorageInfo, @@ -1811,6 +1868,9 @@ def table_from_ptrref( type_or_ptr_ref=ptrref, ) + if aspect == 'inhview' and _needs_cte(ptrref.out_source): + relation = _make_link_table_cte(ptrref, relation, ctx=ctx) + # Pseudo pointers (tuple and type intersection) have no schema id. sobj_id = ptrref.id if isinstance(ptrref, irast.PointerRef) else None rvar = pgast.RelRangeVar( diff --git a/edb/schema/reflection/structure.py b/edb/schema/reflection/structure.py index a21a1edc277..c3f1f08e628 100644 --- a/edb/schema/reflection/structure.py +++ b/edb/schema/reflection/structure.py @@ -805,21 +805,6 @@ def generate_structure( sn.UnqualName(f'{refdict.attr}__internal'), ) - # HACK: sys::Database is an AnnotationSubject, but - # there is no way to actually put annotations on it, - # and fetching them results in some pathological - # quadratic queries where each inner iteration does - # expensive fetching of metadata and JSON decoding. - # Override it to return nothing. - # TODO: For future versions, we can probably just - # drop it. - if ( - str(rschema_name) == 'sys::Database' - and refdict.attr == 'annotations' - ): - props = {} - read_ptr = f'{read_ptr} := {{}}' - for field in props: sfn = field.sname prop_shape_els.append(f'@{sfn}') diff --git a/edb/server/bootstrap.py b/edb/server/bootstrap.py index 31764d5d954..dfba8f7469a 100644 --- a/edb/server/bootstrap.py +++ b/edb/server/bootstrap.py @@ -1287,13 +1287,15 @@ def compile_intro_queries_stdlib( ), ) - local_intro_sql = ' UNION ALL '.join(sql_intro_local_parts) + local_intro_sql = ' UNION ALL '.join( + f'({x})' for x in sql_intro_local_parts) local_intro_sql = f''' WITH intro(c) AS ({local_intro_sql}) SELECT json_agg(intro.c) FROM intro ''' - global_intro_sql = ' UNION ALL '.join(sql_intro_global_parts) + global_intro_sql = ' UNION ALL '.join( + f'({x})' for x in sql_intro_global_parts) global_intro_sql = f''' WITH intro(c) AS ({global_intro_sql}) SELECT json_agg(intro.c) FROM intro From 74cce8f3648db7b68d9879f48a760fecaa199328 Mon Sep 17 00:00:00 2001 From: MiroslavPetrik Date: Wed, 17 Jan 2024 13:15:16 +0100 Subject: [PATCH 14/25] Document edgedb instance credentials (#6685) (#6686) * Document edgedb instance credentials (#6685) * Format line * Fix backticks * Update docs/cli/edgedb_instance/edgedb_instance_credentials.rst Co-authored-by: Devon Campbell * Update docs/cli/edgedb_instance/edgedb_instance_credentials.rst Co-authored-by: Devon Campbell --------- Co-authored-by: Devon Campbell --- .../edgedb_instance_credentials.rst | 39 +++++++++++++++++++ docs/cli/edgedb_instance/index.rst | 3 ++ 2 files changed, 42 insertions(+) create mode 100644 docs/cli/edgedb_instance/edgedb_instance_credentials.rst diff --git a/docs/cli/edgedb_instance/edgedb_instance_credentials.rst b/docs/cli/edgedb_instance/edgedb_instance_credentials.rst new file mode 100644 index 00000000000..99698423215 --- /dev/null +++ b/docs/cli/edgedb_instance/edgedb_instance_credentials.rst @@ -0,0 +1,39 @@ +.. _ref_cli_edgedb_instance_credentials: + + +=========================== +edgedb instance credentials +=========================== + +Display instance credentials. + +.. cli:synopsis:: + + edgedb instance credentials [options] [connection-options] + + +Description +=========== + +``edgedb instance credentials`` is a terminal command for displaying the +credentials of an EdgeDB instance. + + +Options +======= + +:cli:synopsis:`--json` + Output in JSON format. In addition to formatting the credentials as JSON, + this option also includes the password in cleartext and the TLS + certificates. + +:cli:synopsis:`--insecure-dsn` + Output a DSN with password in cleartext. + +Connection Options +================== + +By default, the ``edgedb.toml`` connection is used. + +:cli:synopsis:`` + See :ref:`connection options `. diff --git a/docs/cli/edgedb_instance/index.rst b/docs/cli/edgedb_instance/index.rst index d46b769011a..565a8c98830 100644 --- a/docs/cli/edgedb_instance/index.rst +++ b/docs/cli/edgedb_instance/index.rst @@ -17,6 +17,7 @@ for managing EdgeDB instances. :hidden: edgedb_instance_create + edgedb_instance_credentials edgedb_instance_destroy edgedb_instance_link edgedb_instance_list @@ -35,6 +36,8 @@ for managing EdgeDB instances. * - :ref:`ref_cli_edgedb_instance_create` - Initialize a new server instance + * - :ref:`ref_cli_edgedb_instance_credentials` + - Display instance credentials * - :ref:`ref_cli_edgedb_instance_destroy` - Destroy a server instance and remove the data stored * - :ref:`ref_cli_edgedb_instance_link` From 698a4d5074e7da1a54ceca200d279de03a83bfe7 Mon Sep 17 00:00:00 2001 From: Devon Campbell Date: Wed, 17 Jan 2024 10:44:25 -0500 Subject: [PATCH 15/25] Update deployment guides for v4 (#6702) --- docs/guides/deployment/bare_metal.rst | 18 +++++++++--------- docs/guides/deployment/digitalocean.rst | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/guides/deployment/bare_metal.rst b/docs/guides/deployment/bare_metal.rst index c2762fc279e..35f35d5c588 100644 --- a/docs/guides/deployment/bare_metal.rst +++ b/docs/guides/deployment/bare_metal.rst @@ -72,7 +72,7 @@ default. You can start the server by enabling the unit. .. code-block:: bash - $ sudo systemctl enable --now edgedb-server-3 + $ sudo systemctl enable --now edgedb-server-4 This will start the server on port 5656, and the data directory will be ``/var/lib/edgedb/1/data``. @@ -88,7 +88,7 @@ To set environment variables when running EdgeDB with ``systemctl``, .. code-block:: bash - $ systemctl edit --full edgedb-server-3 + $ systemctl edit --full edgedb-server-4 This opens a ``systemd`` unit file. Set the desired environment variables under the ``[Service]`` section. View the supported environment variables at @@ -104,7 +104,7 @@ Save the file and exit, then restart the service. .. code-block:: bash - $ systemctl restart edgedb-server-3 + $ systemctl restart edgedb-server-4 Set a password @@ -114,14 +114,14 @@ socket directory. You can find this by looking at your system.d unit file. .. code-block:: bash - $ sudo systemctl cat edgedb-server-3 + $ sudo systemctl cat edgedb-server-4 Set a password by connecting from localhost. .. code-block:: bash $ echo -n "> " && read -s PASSWORD - $ RUNSTATE_DIR=$(systemctl show edgedb-server-3 -P ExecStart | \ + $ RUNSTATE_DIR=$(systemctl show edgedb-server-4 -P ExecStart | \ grep -o -m 1 -- "--runstate-dir=[^ ]\+" | \ awk -F "=" '{print $2}') $ sudo edgedb --port 5656 --tls-security insecure --admin \ @@ -147,7 +147,7 @@ You may need to restart the server after changing the listen port or addresses. .. code-block:: bash - $ sudo systemctl restart edgedb-server-3 + $ sudo systemctl restart edgedb-server-4 Link the instance with the CLI @@ -183,7 +183,7 @@ Upgrading EdgeDB intended to manage production instances. When you want to upgrade to the newest point release upgrade the package and -restart the ``edgedb-server-3`` unit. +restart the ``edgedb-server-4`` unit. Debian/Ubuntu LTS @@ -192,7 +192,7 @@ Debian/Ubuntu LTS .. code-block:: bash $ sudo apt-get update && sudo apt-get install --only-upgrade edgedb-3 - $ sudo systemctl restart edgedb-server-3 + $ sudo systemctl restart edgedb-server-4 CentOS/RHEL 7/8 @@ -201,7 +201,7 @@ CentOS/RHEL 7/8 .. code-block:: bash $ sudo yum update edgedb-3 - $ sudo systemctl restart edgedb-server-3 + $ sudo systemctl restart edgedb-server-4 Health Checks ============= diff --git a/docs/guides/deployment/digitalocean.rst b/docs/guides/deployment/digitalocean.rst index 0dd36514ca3..e74df6c7ac9 100644 --- a/docs/guides/deployment/digitalocean.rst +++ b/docs/guides/deployment/digitalocean.rst @@ -210,7 +210,7 @@ Set the security policy to strict. .. code-block:: bash - $ apt-get update && apt-get install --only-upgrade edgedb-server-3 + $ apt-get update && apt-get install --only-upgrade edgedb-server-4 $ systemctl restart edgedb That's it! Refer to the :ref:`Construct the DSN From ba2b94ae05963029b3913fd9a3071d76b17c35f1 Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Wed, 17 Jan 2024 17:07:33 -0800 Subject: [PATCH 16/25] Fix spurious delete/create of constraints with arguments in migrations (#6704) The bug here was that `@index` was not populated in the `params` link for concrete constraints by the reflection writer, and so when we reloaded the schema from the database, it came back in an arbitrary order. This bug has been present for a *long* time, as far as I can tell, but we tended to get "lucky" with the order pg returned the params. This is a strictly *forward* fix. It can be cherry-picked but it will only fix things for constraints created once the fix is applied. I'm working on a backward fix as well. Fixes #6699. --- edb/schema/reflection/writer.py | 7 ++++--- tests/test_edgeql_ddl.py | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/edb/schema/reflection/writer.py b/edb/schema/reflection/writer.py index 30c3e83f5f0..fb80509ddde 100644 --- a/edb/schema/reflection/writer.py +++ b/edb/schema/reflection/writer.py @@ -299,12 +299,13 @@ def _build_object_mutation_shape( # an ObjectKeyDict collection that allow associating objects # with arbitrary values (a transposed ObjectDict). target_expr = f"""assert_distinct(( - FOR v IN {{ json_array_unpack(${var_n}) }} + FOR v IN {{ enumerate(json_array_unpack(${var_n})) }} UNION ( SELECT {target.get_name(schema)} {{ - @value := v[1] + @index := v.0, + @value := v.1[1], }} - FILTER .id = v[0] + FILTER .id = v.1[0] ) ))""" args = props.get('args', []) diff --git a/tests/test_edgeql_ddl.py b/tests/test_edgeql_ddl.py index 0e2577e95ae..cfd740616b5 100644 --- a/tests/test_edgeql_ddl.py +++ b/tests/test_edgeql_ddl.py @@ -10190,6 +10190,26 @@ async def test_edgeql_ddl_constraint_09(self): CREATE TYPE Comment EXTENDING Text; """) + await self.assert_query_result( + """ + select schema::Constraint { + name, params: {name, @index} order by @index + } + filter .name = 'std::max_len_value' + and .subject.name = 'body' + and .subject[is schema::Pointer].source.name ='default::Text'; + """, + [ + { + "name": "std::max_len_value", + "params": [ + {"name": "__subject__", "@index": 0}, + {"name": "max", "@index": 1} + ] + } + ], + ) + await self.con.execute(""" ALTER TYPE Text ALTER PROPERTY body From 0b78e9fa5a1a8359584667af37dfb694ce3d74c8 Mon Sep 17 00:00:00 2001 From: Dijana Pavlovic Date: Thu, 18 Jan 2024 02:07:55 +0100 Subject: [PATCH 17/25] Use updated python version in virtual env (#6691) --- docs/guides/contributing/code.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/guides/contributing/code.rst b/docs/guides/contributing/code.rst index b80b9dea592..842505c9710 100644 --- a/docs/guides/contributing/code.rst +++ b/docs/guides/contributing/code.rst @@ -72,11 +72,11 @@ Python "venv" with all dependencies and commands installed into it. $ git clone --recursive https://github.com/edgedb/edgedb.git -#. Create a Python 3.10 virtual environment and activate it: +#. Create a Python 3.11 virtual environment and activate it: .. code-block:: bash - $ python3.10 -m venv edgedb-dev + $ python3.11 -m venv edgedb-dev $ source edgedb-dev/bin/activate #. Build edgedb (the build will take a while): From 8fad9053339a405919ee2926df0d6795e0a29e42 Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Wed, 17 Jan 2024 22:47:03 -0800 Subject: [PATCH 18/25] Fix some alter+rename combos on pointers in POPULATE MIGRATION (#6666) Doing multiple operations on a pointer and renaming it at the same time in a migration created with POPULATE MIGRATION sometimes fails, because we weren't properly applying renames when generating the AST. This *didn't* happen when using the normal migration flow, because the renames get processed incrementally there. I think I introduced this as a regression in #4007. I also fixed an issue where in a CREATE MIGRATION script, you could refer to the old name of something in a later command. The reason I dealt with that here is because it was preventing schema tests from finding the bug here, since in the schema test the migration script was only checked by applying it with CREATE MIGRATION. (In the actual server, POPULATE MIGRATION explicitly checks the commands validity in a way not afected by this, which is where these failures were happening.) Because of those explicit validity checks, I'm confident that there aren't auto-generated migrations out there that rely on this bug. Fixes #6664. --- edb/schema/delta.py | 11 +++- edb/schema/migrations.py | 17 ++++++ tests/test_edgeql_ddl.py | 19 +++++++ tests/test_schema.py | 112 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 157 insertions(+), 2 deletions(-) diff --git a/edb/schema/delta.py b/edb/schema/delta.py index ba0f050c215..184b888b6dc 100644 --- a/edb/schema/delta.py +++ b/edb/schema/delta.py @@ -2316,8 +2316,15 @@ def _get_ast( astnode = self._get_ast_node(schema, context) if astnode.get_field('name'): - name = sn.shortname_from_fullname(self.classname) - name = context.early_renames.get(name, name) + # We need to be able to catch both renames of the object + # itself, which might have a long name (for pointers, for + # example) as well as an object being referenced by + # shortname, if this is (for example) a concrete + # constraint and the abstract constraint was renamed. + name = context.early_renames.get(self.classname, self.classname) + name = sn.shortname_from_fullname(name) + if self.classname not in context.early_renames: + name = context.early_renames.get(name, name) op = astnode( # type: ignore name=self._deparse_name(schema, context, name), ) diff --git a/edb/schema/migrations.py b/edb/schema/migrations.py index e440f4eb885..a248acf11c9 100644 --- a/edb/schema/migrations.py +++ b/edb/schema/migrations.py @@ -200,6 +200,23 @@ def _cmd_tree_from_ast( return cmd + def apply_subcommands( + self, + schema: s_schema.Schema, + context: sd.CommandContext, + ) -> s_schema.Schema: + assert not self.get_prerequisites() and not self.get_caused() + # Renames shouldn't persist between commands in a migration script. + context.renames.clear() + for op in self.get_subcommands( + include_prerequisites=False, + include_caused=False, + ): + if not isinstance(op, sd.AlterObjectProperty): + schema = op.apply(schema, context=context) + context.renames.clear() + return schema + def _get_ast( self, schema: s_schema.Schema, diff --git a/tests/test_edgeql_ddl.py b/tests/test_edgeql_ddl.py index cfd740616b5..7871993b36c 100644 --- a/tests/test_edgeql_ddl.py +++ b/tests/test_edgeql_ddl.py @@ -13105,6 +13105,25 @@ async def test_edgeql_ddl_create_migration_04(self): # ['test'], # ) + async def test_edgeql_ddl_create_migration_05(self): + await self.con.execute(''' + create type X { create property x -> str; }; + ''') + async with self.assertRaisesRegexTx( + edgedb.InvalidReferenceError, + "property 'x' does not"): + await self.con.execute(''' + CREATE MIGRATION + { + alter type default::X { + alter property x rename to y; + }; + alter type default::X { + alter property x create constraint exclusive; + }; + }; + ''') + async def test_edgeql_ddl_naked_backlink_in_computable(self): await self.con.execute(''' CREATE TYPE User { diff --git a/tests/test_schema.py b/tests/test_schema.py index 135bd3853db..fb32673722d 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -8190,6 +8190,118 @@ def test_schema_migrations_rename_with_stuff_04(self): """ ]) + def test_schema_migrations_rename_and_modify_01(self): + self._assert_migration_equivalence([ + r""" + type Branch{ + property branchURL: std::str { + constraint max_len_value(500); + constraint min_len_value(5); + }; + }; + """, + r""" + type Branch{ + property email: std::str { + constraint max_len_value(50); + constraint min_len_value(5); + }; + }; + """ + ]) + + def test_schema_migrations_rename_and_modify_02(self): + self._assert_migration_equivalence([ + r""" + type X { + obj: Object { + foo: str; + }; + }; + """, + r""" + type X { + obj2: Object { + bar: int64; + }; + }; + """ + ]) + + def test_schema_migrations_rename_and_modify_03(self): + self._assert_migration_equivalence([ + r""" + type Branch{ + property branchName: std::str { + constraint min_len_value(0); + constraint max_len_value(255); + }; + property branchCode: std::int64; + property branchURL: std::str { + constraint max_len_value(500); + constraint regexp("url"); + constraint min_len_value(5); + }; + }; + """, + r""" + type Branch{ + property branchName: std::str { + constraint min_len_value(0); + constraint max_len_value(255); + }; + property branchCode: std::int64; + property phoneNumber: std::str { + constraint min_len_value(5); + constraint max_len_value(50); + constraint regexp(r"phone"); + }; + property email: std::str { + constraint min_len_value(5); + constraint max_len_value(50); + constraint regexp(r"email"); + }; + }; + """ + ]) + + def test_schema_migrations_rename_and_modify_04(self): + self._assert_migration_equivalence([ + r""" + type Branch{ + property branchName: std::str { + constraint min_len_value(0); + constraint max_len_value(255); + }; + property branchCode: std::int64; + property branchURL: std::str { + constraint max_len_value(500); + constraint regexp("url"); + constraint min_len_value(5); + }; + }; + """, + r""" + type Branch2 { + property branchName: std::str { + constraint min_len_value(0); + constraint max_len_value(255); + }; + property branchCode: std::int64; + property phoneNumber: std::str { + constraint min_len_value(5); + constraint max_len_value(50); + constraint regexp(r"phone"); + }; + property email: std::str { + constraint min_len_value(5); + constraint max_len_value(50); + constraint regexp(r"email"); + }; + }; + """ + ]) + def test_schema_migrations_except_01(self): self._assert_migration_equivalence([ r""" From 47771cb8d013f2ca15d65f43d518f4f76f4581da Mon Sep 17 00:00:00 2001 From: Fantix King Date: Fri, 19 Jan 2024 05:11:41 +0900 Subject: [PATCH 19/25] Pin old Sphinx deps to old versions (#6709) The Sphinx dependencies started to mark their Sphinx version requirement at runtime, which stopped old Sphinx 4.* from installing dependencies of the right versions. --- pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 113e2240164..9fa92bed215 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,11 @@ test = [ 'Pygments~=2.10.0', 'Sphinx~=4.2.0', 'sphinxcontrib-asyncio~=0.3.0', + 'sphinxcontrib-applehelp<1.0.8', + 'sphinxcontrib-devhelp<1.0.6', + 'sphinxcontrib-htmlhelp<2.0.5', + 'sphinxcontrib-serializinghtml<1.1.10', + 'sphinxcontrib-qthelp<1.0.7', 'sphinx_code_tabs~=0.5.3', ] From cee9d2d7ba004dc721fe5947ba2a6afc9fb7952a Mon Sep 17 00:00:00 2001 From: Scott Trinh Date: Thu, 18 Jan 2024 16:08:45 -0500 Subject: [PATCH 20/25] Use newly released hishel in-memory cache (#6644) --- edb/server/protocol/auth_ext/http_client.py | 20 ++------------------ pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/edb/server/protocol/auth_ext/http_client.py b/edb/server/protocol/auth_ext/http_client.py index 55c3666836d..8becb135cf0 100644 --- a/edb/server/protocol/auth_ext/http_client.py +++ b/edb/server/protocol/auth_ext/http_client.py @@ -21,8 +21,6 @@ import hishel import httpx -from edb.common import lru - class HttpClient(httpx.AsyncClient): def __init__( @@ -33,7 +31,8 @@ def __init__( self.edgedb_orig_base_url = urllib.parse.quote(base_url, safe='') base_url = edgedb_test_url cache = hishel.AsyncCacheTransport( - transport=httpx.AsyncHTTPTransport(), storage=InMemoryStorage() + transport=httpx.AsyncHTTPTransport(), + storage=hishel.AsyncInMemoryStorage(capacity=5), ) super().__init__(*args, base_url=base_url, transport=cache, **kwargs) @@ -46,18 +45,3 @@ async def get(self, path, *args, **kwargs): if self.edgedb_orig_base_url: path = f'{self.edgedb_orig_base_url}/{path}' return await super().get(path, *args, **kwargs) - - -class InMemoryStorage(hishel._async._storages.AsyncBaseStorage): - def __init__(self, maxsize=5): - super().__init__() - self._storage = lru.LRUMapping(maxsize=maxsize) - - async def store(self, key: str, response, request, metadata): - self._storage[key] = (response, request, metadata) - - async def retreive(self, key: str): - return self._storage.get(key, None) - - async def aclose(self): - pass diff --git a/pyproject.toml b/pyproject.toml index 9fa92bed215..be33386d259 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ dependencies = [ 'setproctitle~=1.2', 'httpx~=0.24.1', - 'hishel==0.0.18', + 'hishel==0.0.21', 'argon2-cffi~=23.1.0', 'aiosmtplib~=2.0', ] From a8e2933dc264f40d333f65c1da5a1d127427f6e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alja=C5=BE=20Mur=20Er=C5=BEen?= Date: Thu, 18 Jan 2024 23:20:40 -0800 Subject: [PATCH 21/25] GH job to check that postgres/ has not been unintentianally changed (#6710) --- .github/workflows/pull-request-target.yml | 39 +++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/pull-request-target.yml diff --git a/.github/workflows/pull-request-target.yml b/.github/workflows/pull-request-target.yml new file mode 100644 index 00000000000..2ab84d3b81e --- /dev/null +++ b/.github/workflows/pull-request-target.yml @@ -0,0 +1,39 @@ +name: Pull Request + +on: + pull_request_target: + types: [opened, edited, synchronize, labeled, closed] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + + test-pr: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: false + + - name: Verify that postgres/ was not changed unintentionally + shell: bash + run: | + required_prefix="Update bundled PostgreSQL" + + title="${{ github.event.pull_request.title }}" + if [[ $title == $required_prefix* ]]; then + exit 0 + fi + + git diff --quiet \ + ${{ github.event.pull_request.base.sha }} \ + ${{ github.event.pull_request.head.sha }} + if [ $? != 0 ]; then + echo "postgres/ submodule has been changed,"\ + "but PR title does not indicate that" 1>&2 + echo "(it should start with '$required_prefix')" 1>&2 + exit 1 + fi From 18c7f6944519d4ccaf98ec309bbcedf613e3b3ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alja=C5=BE=20Mur=20Er=C5=BEen?= Date: Fri, 19 Jan 2024 00:36:44 -0800 Subject: [PATCH 22/25] GH job to check that postgres/ has not been unintentianally changed, take 2 (#6715) --- ...-request-target.yml => pull-request-meta.yml} | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) rename .github/workflows/{pull-request-target.yml => pull-request-meta.yml} (71%) diff --git a/.github/workflows/pull-request-target.yml b/.github/workflows/pull-request-meta.yml similarity index 71% rename from .github/workflows/pull-request-target.yml rename to .github/workflows/pull-request-meta.yml index 2ab84d3b81e..0126615448c 100644 --- a/.github/workflows/pull-request-target.yml +++ b/.github/workflows/pull-request-meta.yml @@ -1,7 +1,7 @@ -name: Pull Request +name: Pull Request Meta on: - pull_request_target: + pull_request: types: [opened, edited, synchronize, labeled, closed] concurrency: @@ -28,12 +28,14 @@ jobs: exit 0 fi - git diff --quiet \ + if git diff --quiet \ ${{ github.event.pull_request.base.sha }} \ - ${{ github.event.pull_request.head.sha }} - if [ $? != 0 ]; then + ${{ github.event.pull_request.head.sha }} -- postgres/ + then + echo 'all ok' + else echo "postgres/ submodule has been changed,"\ - "but PR title does not indicate that" 1>&2 - echo "(it should start with '$required_prefix')" 1>&2 + "but PR title does not indicate that" + echo "(it should start with '$required_prefix')" exit 1 fi From cfa3729307127afb421c6087b849d03b388c0460 Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Sat, 20 Jan 2024 12:15:06 -0800 Subject: [PATCH 23/25] Fix some postgres submodule check issues (#6717) --- .github/workflows/pull-request-meta.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pull-request-meta.yml b/.github/workflows/pull-request-meta.yml index 0126615448c..e6c9c85e3af 100644 --- a/.github/workflows/pull-request-meta.yml +++ b/.github/workflows/pull-request-meta.yml @@ -2,7 +2,7 @@ name: Pull Request Meta on: pull_request: - types: [opened, edited, synchronize, labeled, closed] + types: [opened, edited, synchronize] concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} @@ -19,12 +19,13 @@ jobs: submodules: false - name: Verify that postgres/ was not changed unintentionally + env: + PR_TITLE: ${{ github.event.pull_request.title }} shell: bash run: | required_prefix="Update bundled PostgreSQL" - title="${{ github.event.pull_request.title }}" - if [[ $title == $required_prefix* ]]; then + if [[ "$PR_TITLE" == $required_prefix* ]]; then exit 0 fi From a405e32d1e55814ae99dc097534a61445583f245 Mon Sep 17 00:00:00 2001 From: Scott Trinh Date: Tue, 23 Jan 2024 13:36:23 -0500 Subject: [PATCH 24/25] Add Discord OAuth provider (#6708) --- edb/lib/ext/auth.edgeql | 11 + .../auth_ext/_static/icon_discord.svg | 3 + edb/server/protocol/auth_ext/discord.py | 95 ++++++ edb/server/protocol/auth_ext/oauth.py | 4 +- edb/server/protocol/auth_ext/ui.py | 1 + tests/test_http_ext_auth.py | 281 +++++++++++++++++- 6 files changed, 387 insertions(+), 8 deletions(-) create mode 100644 edb/server/protocol/auth_ext/_static/icon_discord.svg create mode 100644 edb/server/protocol/auth_ext/discord.py diff --git a/edb/lib/ext/auth.edgeql b/edb/lib/ext/auth.edgeql index cc0b8196427..df41ebda009 100644 --- a/edb/lib/ext/auth.edgeql +++ b/edb/lib/ext/auth.edgeql @@ -145,6 +145,17 @@ CREATE EXTENSION PACKAGE auth VERSION '1.0' { }; }; + create type ext::auth::DiscordOAuthProvider + extending ext::auth::OAuthProviderConfig { + alter property name { + set default := 'builtin::oauth_discord'; + }; + + alter property display_name { + set default := 'Discord'; + }; + }; + create type ext::auth::GitHubOAuthProvider extending ext::auth::OAuthProviderConfig { alter property name { diff --git a/edb/server/protocol/auth_ext/_static/icon_discord.svg b/edb/server/protocol/auth_ext/_static/icon_discord.svg new file mode 100644 index 00000000000..3b0b3b1f664 --- /dev/null +++ b/edb/server/protocol/auth_ext/_static/icon_discord.svg @@ -0,0 +1,3 @@ + + + diff --git a/edb/server/protocol/auth_ext/discord.py b/edb/server/protocol/auth_ext/discord.py new file mode 100644 index 00000000000..76b58a033e8 --- /dev/null +++ b/edb/server/protocol/auth_ext/discord.py @@ -0,0 +1,95 @@ +# +# This source file is part of the EdgeDB open source project. +# +# Copyright 2024-present MagicStack Inc. and the EdgeDB authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import urllib.parse +import functools + +from . import base, data, errors + + +class DiscordProvider(base.BaseProvider): + def __init__(self, *args, **kwargs): + super().__init__("discord", "https://discord.com", *args, **kwargs) + self.auth_domain = self.issuer_url + self.api_domain = f"{self.issuer_url}/api/v10" + self.auth_client = functools.partial( + self.http_factory, base_url=self.auth_domain + ) + self.api_client = functools.partial( + self.http_factory, base_url=self.api_domain + ) + + async def get_code_url( + self, state: str, redirect_uri: str, additional_scope: str + ) -> str: + params = { + "client_id": self.client_id, + "scope": f"email identify {additional_scope}", + "state": state, + "redirect_uri": redirect_uri, + "response_type": "code", + } + encoded = urllib.parse.urlencode(params) + return f"{self.auth_domain}/oauth2/authorize?{encoded}" + + async def exchange_code( + self, code: str, redirect_uri: str + ) -> data.OAuthAccessTokenResponse: + async with self.auth_client() as client: + resp = await client.post( + "/api/oauth2/token", + data={ + "grant_type": "authorization_code", + "code": code, + "client_id": self.client_id, + "client_secret": self.client_secret, + "redirect_uri": redirect_uri, + }, + headers={ + "accept": "application/json", + }, + ) + if resp.status_code >= 400: + raise errors.OAuthProviderFailure( + f"Failed to exchange code: {resp.text}" + ) + json = resp.json() + + return data.OAuthAccessTokenResponse(**json) + + async def fetch_user_info( + self, token_response: data.OAuthAccessTokenResponse + ) -> data.UserInfo: + async with self.api_client() as client: + resp = await client.get( + "/users/@me", + headers={ + "Authorization": f"Bearer {token_response.access_token}", + "Accept": "application/json", + "Cache-Control": "no-store", + }, + ) + payload = resp.json() + return data.UserInfo( + sub=str(payload["id"]), + preferred_username=payload.get("username"), + name=payload.get("global_name"), + email=payload.get("email"), + picture=payload.get("avatar"), + ) diff --git a/edb/server/protocol/auth_ext/oauth.py b/edb/server/protocol/auth_ext/oauth.py index 70e5099f2ff..6440df8e83c 100644 --- a/edb/server/protocol/auth_ext/oauth.py +++ b/edb/server/protocol/auth_ext/oauth.py @@ -22,7 +22,7 @@ from typing import Any, Type from edb.server.protocol import execute -from . import github, google, azure, apple +from . import github, google, azure, apple, discord from . import errors, util, data, base, http_client @@ -55,6 +55,8 @@ def __init__( provider_class = azure.AzureProvider case "builtin::oauth_apple": provider_class = apple.AppleProvider + case "builtin::oauth_discord": + provider_class = discord.DiscordProvider case _: raise errors.InvalidData(f"Invalid provider: {provider_name}") diff --git a/edb/server/protocol/auth_ext/ui.py b/edb/server/protocol/auth_ext/ui.py index 6ca3dc2c255..72a9e3b0b40 100644 --- a/edb/server/protocol/auth_ext/ui.py +++ b/edb/server/protocol/auth_ext/ui.py @@ -30,6 +30,7 @@ 'builtin::oauth_google', 'builtin::oauth_apple', 'builtin::oauth_azure', + 'builtin::oauth_discord', ] diff --git a/tests/test_http_ext_auth.py b/tests/test_http_ext_auth.py index a3378e13a03..c4be059a706 100644 --- a/tests/test_http_ext_auth.py +++ b/tests/test_http_ext_auth.py @@ -242,14 +242,19 @@ def handle_request( # Parse and save the request details parsed_path = urllib.parse.urlparse(path) - request_details = { - 'headers': {k.lower(): v for k, v in dict(handler.headers).items()}, - 'query_params': urllib.parse.parse_qs(parsed_path.query), - 'body': handler.rfile.read( - int(handler.headers['Content-Length']) + headers = {k.lower(): v for k, v in dict(handler.headers).items()} + query_params = urllib.parse.parse_qs(parsed_path.query) + if 'content-length' in headers: + body = handler.rfile.read( + int(headers['content-length']) ).decode() - if 'Content-Length' in handler.headers - else None, + else: + body = None + + request_details = { + 'headers': headers, + 'query_params': query_params, + 'body': body, } self.requests[key].append(request_details) @@ -344,6 +349,7 @@ def __exit__(self, *exc): GOOGLE_SECRET = 'c' * 32 AZURE_SECRET = 'c' * 32 APPLE_SECRET = 'c' * 32 +DISCORD_SECRET = 'd' * 32 class TestHttpExtAuth(tb.ExtAuthTestCase): @@ -391,6 +397,12 @@ class TestHttpExtAuth(tb.ExtAuthTestCase): client_id := '{uuid.uuid4()}', }}; + CONFIGURE CURRENT DATABASE + INSERT ext::auth::DiscordOAuthProvider {{ + secret := '{DISCORD_SECRET}', + client_id := '{uuid.uuid4()}', + }}; + CONFIGURE CURRENT DATABASE INSERT ext::auth::EmailPasswordProviderConfig {{ require_verification := false, @@ -983,6 +995,261 @@ async def test_http_auth_ext_github_callback_failure_02(self): "error=access_denied", ) + async def test_http_auth_ext_discord_authorize_01(self): + with MockAuthProvider(), self.http_con() as http_con: + provider_config = await self.get_builtin_provider_config_by_name( + "oauth_discord" + ) + provider_name = provider_config.name + client_id = provider_config.client_id + redirect_to = f"{self.http_addr}/some/path" + challenge = ( + base64.urlsafe_b64encode( + hashlib.sha256( + base64.urlsafe_b64encode(os.urandom(43)).rstrip(b'=') + ).digest() + ) + .rstrip(b'=') + .decode() + ) + query = { + "provider": provider_name, + "redirect_to": redirect_to, + "challenge": challenge, + } + + _, headers, status = self.http_con_request( + http_con, + query, + path="authorize", + ) + + self.assertEqual(status, 302) + + location = headers.get("location") + assert location is not None + url = urllib.parse.urlparse(location) + qs = urllib.parse.parse_qs(url.query, keep_blank_values=True) + self.assertEqual(url.scheme, "https") + self.assertEqual(url.hostname, "discord.com") + self.assertEqual(url.path, "/oauth2/authorize") + self.assertEqual(qs.get("scope"), ["email identify "]) + + state = qs.get("state") + assert state is not None + + claims = await self.extract_jwt_claims(state[0]) + self.assertEqual(claims.get("provider"), provider_name) + self.assertEqual(claims.get("iss"), self.http_addr) + self.assertEqual(claims.get("redirect_to"), redirect_to) + + self.assertEqual( + qs.get("redirect_uri"), [f"{self.http_addr}/callback"] + ) + self.assertEqual(qs.get("client_id"), [client_id]) + + pkce = await self.con.query( + """ + select ext::auth::PKCEChallenge + filter .challenge = $challenge + """, + challenge=challenge, + ) + self.assertEqual(len(pkce), 1) + + _, _, repeat_status = self.http_con_request( + http_con, + query, + path="authorize", + ) + self.assertEqual(repeat_status, 302) + + repeat_pkce = await self.con.query_single( + """ + select ext::auth::PKCEChallenge + filter .challenge = $challenge + """, + challenge=challenge, + ) + self.assertEqual(pkce[0].id, repeat_pkce.id) + + async def test_http_auth_ext_discord_callback_01(self): + with MockAuthProvider() as mock_provider, self.http_con() as http_con: + provider_config = await self.get_builtin_provider_config_by_name( + "oauth_discord" + ) + provider_name = provider_config.name + client_id = provider_config.client_id + client_secret = DISCORD_SECRET + + now = utcnow() + token_request = ( + "POST", + "https://discord.com", + "/api/oauth2/token", + ) + mock_provider.register_route_handler(*token_request)( + ( + json.dumps( + { + "access_token": "discord_access_token", + "scope": "read:user", + "token_type": "bearer", + } + ), + 200, + ) + ) + + user_request = ("GET", "https://discord.com/api/v10", "/users/@me") + mock_provider.register_route_handler(*user_request)( + ( + json.dumps( + { + "id": 1, + "username": "dischord", + "global_name": "Ian MacKaye", + "email": "ian@example.com", + "picture": "https://example.com/example.jpg", + } + ), + 200, + ) + ) + + challenge = ( + base64.urlsafe_b64encode( + hashlib.sha256( + base64.urlsafe_b64encode(os.urandom(43)).rstrip(b'=') + ).digest() + ) + .rstrip(b'=') + .decode() + ) + await self.con.query( + """ + insert ext::auth::PKCEChallenge { + challenge := $challenge, + } + """, + challenge=challenge, + ) + + signing_key = await self.get_signing_key() + + expires_at = now + datetime.timedelta(minutes=5) + state_claims = { + "iss": self.http_addr, + "provider": str(provider_name), + "exp": expires_at.timestamp(), + "redirect_to": f"{self.http_addr}/some/path", + "challenge": challenge, + } + state_token = self.generate_state_value(state_claims, signing_key) + + data, headers, status = self.http_con_request( + http_con, + {"state": state_token, "code": "abc123"}, + path="callback", + ) + + self.assertEqual(data, b"") + self.assertEqual(status, 302) + + location = headers.get("location") + assert location is not None + server_url = urllib.parse.urlparse(self.http_addr) + url = urllib.parse.urlparse(location) + self.assertEqual(url.scheme, server_url.scheme) + self.assertEqual(url.hostname, server_url.hostname) + self.assertEqual(url.path, f"{server_url.path}/some/path") + + requests_for_token = mock_provider.requests[token_request] + self.assertEqual(len(requests_for_token), 1) + + self.assertEqual( + urllib.parse.parse_qs(requests_for_token[0]["body"]), + { + "grant_type": ["authorization_code"], + "code": ["abc123"], + "client_id": [client_id], + "client_secret": [client_secret], + "redirect_uri": [f"{self.http_addr}/callback"], + }, + ) + + requests_for_user = mock_provider.requests[user_request] + self.assertEqual(len(requests_for_user), 1) + self.assertEqual( + requests_for_user[0]["headers"]["authorization"], + "Bearer discord_access_token", + ) + + identity = await self.con.query( + """ + SELECT ext::auth::Identity + FILTER .subject = '1' + AND .issuer = 'https://discord.com' + """ + ) + self.assertEqual(len(identity), 1) + + session_claims = await self.extract_session_claims(headers) + self.assertEqual(session_claims.get("sub"), str(identity[0].id)) + self.assertEqual(session_claims.get("iss"), str(self.http_addr)) + tomorrow = now + datetime.timedelta(hours=25) + self.assertTrue(session_claims.get("exp") > now.timestamp()) + self.assertTrue(session_claims.get("exp") < tomorrow.timestamp()) + + pkce_object = await self.con.query( + """ + SELECT ext::auth::PKCEChallenge + { id, auth_token, refresh_token } + filter .identity.id = $identity_id + """, + identity_id=identity[0].id, + ) + + self.assertEqual(len(pkce_object), 1) + self.assertEqual(pkce_object[0].auth_token, "discord_access_token") + self.assertIsNone(pkce_object[0].refresh_token) + + mock_provider.register_route_handler(*user_request)( + ( + json.dumps( + { + "id": 1, + "login": "octocat", + "name": "monalisa octocat", + "email": "octocat+2@example.com", + "avatar_url": "https://example.com/example.jpg", + "updated_at": now.isoformat(), + } + ), + 200, + ) + ) + (_, new_headers, _) = self.http_con_request( + http_con, + {"state": state_token, "code": "abc123"}, + path="callback", + ) + + same_identity = await self.con.query( + """ + SELECT ext::auth::Identity + FILTER .subject = '1' + AND .issuer = 'https://discord.com' + """ + ) + self.assertEqual(len(same_identity), 1) + self.assertEqual(identity[0].id, same_identity[0].id) + + new_session_claims = await self.extract_session_claims(new_headers) + self.assertTrue( + new_session_claims.get("exp") > session_claims.get("exp") + ) + async def test_http_auth_ext_google_callback_01(self) -> None: with MockAuthProvider() as mock_provider, self.http_con() as http_con: provider_config = await self.get_builtin_provider_config_by_name( From b6cde57359d481d4b407d282aa6316afcd0aed17 Mon Sep 17 00:00:00 2001 From: Dave MacLeod <56599343+Dhghomon@users.noreply.github.com> Date: Wed, 24 Jan 2024 04:07:11 +0900 Subject: [PATCH 25/25] Add schema migration guide, reorder and rename in places (#6656) * Add schema migration guide, reorder and rename in places * Squish tips into single file * Add UI docs * Add UI to index * rv backlink.rst * +images * redo schema viewer * line length * More line length * wax on, wax off * Change code block * . * Rephrase * rv edgeql-repl in places * + alt text * Move migration tip to other page * Links, wording * +tips on relative difficulty * +query editor image * fix dots * Rv UI docs (now in other branch) * Fix query * Change block (not meant to be parsed) * Change to REPL * First migration: explain why questions now asked * Move above "The DDL..." sentence * Rephase why backslash commands are faster * Move output up * Fix type name * First 20 suggestions from code review Co-authored-by: Devon Campbell * Next 20 suggestions from code review Co-authored-by: Devon Campbell * Next 20 suggestions from code review Co-authored-by: Devon Campbell * Next 20 suggestions from code review Co-authored-by: Devon Campbell * Apply suggestions from code review Co-authored-by: Devon Campbell * Line length * Hint that difference is very small * Add section ref * Fix ref, blank space * Add migration edit * Note on when DDL is disabled * typo * Apply suggestions from code review Co-authored-by: Devon Campbell * Explain the grouping query * Apply suggestion from code review Co-authored-by: Devon Campbell * Small hint about commenting out * Inline edgedb watch, rephrase here and there * Multiple migrations: clarify invite to follow along * Add tip on destroying instances * Older migration section: typo + rephrase * More readable link * Update titles We discussed changes along these lines previously, but I think they got lost in the shuffle. --------- Co-authored-by: Devon Campbell --- docs/guides/datamigrations/index.rst | 6 +- docs/guides/index.rst | 8 +- docs/guides/migrations/backlink.rst | 210 ---- docs/guides/migrations/guide.rst | 1363 ++++++++++++++++++++++++ docs/guides/migrations/index.rst | 20 +- docs/guides/migrations/names.rst | 215 ---- docs/guides/migrations/proptolink.rst | 334 ------ docs/guides/migrations/proptype.rst | 201 ---- docs/guides/migrations/recovering.rst | 65 -- docs/guides/migrations/reqlink.rst | 376 ------- docs/guides/migrations/tips.rst | 1410 +++++++++++++++++++++++++ docs/guides/tutorials/index.rst | 6 +- docs/intro/migrations.rst | 13 +- 13 files changed, 2796 insertions(+), 1431 deletions(-) delete mode 100644 docs/guides/migrations/backlink.rst create mode 100644 docs/guides/migrations/guide.rst delete mode 100644 docs/guides/migrations/names.rst delete mode 100644 docs/guides/migrations/proptolink.rst delete mode 100644 docs/guides/migrations/proptype.rst delete mode 100644 docs/guides/migrations/recovering.rst delete mode 100644 docs/guides/migrations/reqlink.rst create mode 100644 docs/guides/migrations/tips.rst diff --git a/docs/guides/datamigrations/index.rst b/docs/guides/datamigrations/index.rst index 13ab5d3e87b..5771b26b1fc 100644 --- a/docs/guides/datamigrations/index.rst +++ b/docs/guides/datamigrations/index.rst @@ -1,8 +1,8 @@ .. _ref_guide_data_migrations: -======================= -Importing existing data -======================= +=================== +Switching to EdgeDB +=================== Perhaps you like EdgeDB so much you want to migrate an existing project to it. Here we'll show you some possible ways to import your data. diff --git a/docs/guides/index.rst b/docs/guides/index.rst index 211f66f59bd..1dc27b2d920 100644 --- a/docs/guides/index.rst +++ b/docs/guides/index.rst @@ -18,10 +18,10 @@ guide! :maxdepth: 1 cloud - auth/index - tutorials/index + cheatsheet/index deployment/index - migrations/index datamigrations/index - cheatsheet/index + tutorials/index + auth/index + migrations/index contributing/index diff --git a/docs/guides/migrations/backlink.rst b/docs/guides/migrations/backlink.rst deleted file mode 100644 index e506f799c30..00000000000 --- a/docs/guides/migrations/backlink.rst +++ /dev/null @@ -1,210 +0,0 @@ -.. _ref_migration_backlink: - -================ -Adding backlinks -================ - -This example shows how to handle a schema that makes use of a -backlink. We'll use a linked-list structure to represent a sequence of -events. - -We'll start with this schema: - -.. code-block:: sdl - :version-lt: 3.0 - - type Event { - required property name -> str; - link prev -> Event; - - # ... more properties and links - } - -.. code-block:: sdl - - type Event { - required name: str; - prev: Event; - - # ... more properties and links - } - -We specify a ``prev`` link because that will make adding a new -``Event`` at the end of the chain easier, since we'll be able to -specify the payload and the chain the ``Event`` should be appended to -in a single :eql:stmt:`insert`. Once we've updated the schema -file we proceed with our first migration: - -.. code-block:: bash - - $ edgedb migration create - did you create object type 'default::Event'? [y,n,l,c,b,s,q,?] - > y - Created ./dbschema/migrations/00001.edgeql, id: - m1v3ahcx5f43y6mlsdmlz2agnf6msbc7rt3zstiqmezaqx4ev2qovq - $ edgedb migrate - Applied m1v3ahcx5f43y6mlsdmlz2agnf6msbc7rt3zstiqmezaqx4ev2qovq - (00001.edgeql) - -We now have a way of chaining events together. We might create a few -events like these: - -.. code-block:: edgeql-repl - - db> select Event { - ... name, - ... prev: { name }, - ... }; - { - default::Event {name: 'setup', prev: {}}, - default::Event {name: 'work', prev: default::Event {name: 'setup'}}, - default::Event {name: 'cleanup', prev: default::Event {name: 'work'}}, - } - -It seems like having a ``next`` link would be useful, too. So we can -define it as a computed link by using :ref:`backlink -` notation: - -.. code-block:: sdl - :version-lt: 3.0 - - type Event { - required property name -> str; - - link prev -> Event; - link next := . y - Created ./dbschema/migrations/00002.edgeql, id: - m1qpukyvw2m4lmomoseni7vdmevk4wzgsbviojacyrqgiyqjp5sdsa - $ edgedb migrate - Applied m1qpukyvw2m4lmomoseni7vdmevk4wzgsbviojacyrqgiyqjp5sdsa - (00002.edgeql) - -Trying out the new link on our existing data gives us: - -.. code-block:: edgeql-repl - - db> select Event { - ... name, - ... prev_name := .prev.name, - ... next_name := .next.name, - ... }; - { - default::Event { - name: 'setup', - prev_name: {}, - next_name: {'work'}, - }, - default::Event { - name: 'work', - prev_name: 'setup', - next_name: {'cleanup'}, - }, - default::Event { - name: 'cleanup', - prev_name: 'work', - next_name: {}, - }, - } - -That's not quite right. The value of ``next_name`` appears to be a set -rather than a singleton. This is because the link ``prev`` is -many-to-one and so ``next`` is one-to-many, making it a *multi* link. -Let's fix that by making the link ``prev`` a one-to-one, after all -we're interested in building event chains, not trees. - -.. code-block:: sdl - :version-lt: 3.0 - - type Event { - required property name -> str; - - link prev -> Event { - constraint exclusive; - }; - link next := . y - Created ./dbschema/migrations/00003.edgeql, id: - m17or2bfywuckdqeornjmjh7c2voxgatspcewyefcd4p2vbdepimoa - $ edgedb migrate - Applied m17or2bfywuckdqeornjmjh7c2voxgatspcewyefcd4p2vbdepimoa - (00003.edgeql) - -The new ``next`` computed link is now inferred as a ``single`` link -and so the query results for ``next_name`` and ``prev_name`` are -symmetrical: - -.. code-block:: edgeql-repl - - db> select Event { - ... name, - ... prev_name := .prev.name, - ... next_name := .next.name, - ... }; - { - default::Event {name: 'setup', prev_name: {}, next_name: 'work'}, - default::Event {name: 'work', prev_name: 'setup', next_name: 'cleanup'}, - default::Event {name: 'cleanup', prev_name: 'work', next_name: {}}, - } diff --git a/docs/guides/migrations/guide.rst b/docs/guides/migrations/guide.rst new file mode 100644 index 00000000000..7187be4d981 --- /dev/null +++ b/docs/guides/migrations/guide.rst @@ -0,0 +1,1363 @@ +.. _ref_migration_guide: + +====== +Basics +====== + +:edb-alt-title: Schema migration basics + +EdgeQL is a strongly-typed language, which means that it moves checks +and verification of your code to compile time as much as possible +instead of performing them at run time. EdgeDB's view is that a schema +should allow you to set types, constraints, expressions, and more so that +you can confidently know what sort of behavior to expect from your data. +Laying a type-safe foundation means a bit more thinking up front, but saves +you all kinds of headaches down the road. + +It's not unlikely though that you'll define your schema up perfectly with +your first try, or that you'll build an application that never needs +its schema revised. When you *do* eventually need to make a change, you will +need to migrate your schema from its current state to a new state. + +The basics of creating a project, modifying its schema, and migrating +it in EdgeDB are pretty easy: + +- Type ``edgedb project init`` to start a project, +- Open the newly created empty schema at ``dbschema/default.esdl`` and add + a simple type like ``SomeType { name: str; }`` inside the empty ``module`` +- Run ``edgedb migration create``, type ``y`` to confirm the change, + then run ``edgedb migrate``, and you are done! You can now + ``insert SomeType;`` in your database to your heart's content. + +.. note:: + + If you ever feel like outright removing and creating an instance anew + during this migration guide, you can use the command + ``edgedb instance destroy -I --force``. And if you want to + remove all existing migrations as well, you can manually delete them inside + your ``/migrations`` folder (otherwise, the CLI will try to apply the + migrations again when you recreate your instance with + ``edgedb migration create``). Once that is done, you will have a blank + slate on which to start over again. + +But many EdgeDB users have needs that go beyond these basics. In addition, +schema migrations are pretty interesting and teach you a lot about +what EdgeDB does behind the scenes. This guide will turn you from +a casual migration user into one with a lot more tools at hand, along +with a deeper understanding of the internals of EdgeDB at the same +time. + +EdgeDB's built-in tools are what make schema migrations easy, and +the way they work is through a pretty interesting interaction between +EdgeDB's SDL (Schema Definition Language) and DDL (Data Definition +Language). The first thing to understand about migrations is the difference +between SDL and DDL, and how they are used. + +SDL: For humans +=============== + +SDL, not DDL, is the primary way for you to create and migrate your +schema in EdgeDB. You don't need to work with DDL to use EdgeDB any +more than you need to know how to change a tire to drive a car. + +SDL is built for humans to read, which is why it is said to be *declarative*. +The 'clar' inside *declarative* is the same word as *clear*, and this +is exactly what declarative means: making it *clear* what you want +the final result to be. An example of a declarative instruction in +real life would be telling a friend to show up at your house at 6416 +Riverside Way. You've declared what the final result should be, but +it's up to your friend to find how to achieve it. + +Now let's look at some real SDL and think about its role in EdgeDB. +Here is a simple example of a schema: + +.. code-block:: sdl + + module default { + type User { + name: str; + } + } + +If you have EdgeDB installed and want to follow along, type ``edgedb +project init`` and copy the above schema into your ``default.esdl`` +file inside the ``/dbschema`` folder it creates. Then save the file. + +.. note:: + + While schema is usually contained inside the ``default.esdl`` file, + you can divide a schema over multiple files if you like. EdgeDB will + combine all ``.esdl`` files inside the ``/dbschema`` folder into a + single schema. + +Type ``edgedb`` to start the EdgeDB REPL, and, into the REPL, type +``describe schema as sdl``. The output will be ``{'module default{};'}`` +— nothing more than the empty ``default`` module. What happened? +Our ``type User`` is nowhere to be found. + +This is the first thing to know about SDL. Like an address to a +person's house, it doesn't *do* anything on its own. With SDL you are +declaring what you want the final result to be: a schema containing a single +type called ``User``, with a property of type ``str`` called ``name``. + +In order for a migration to happen, the EdgeDB server needs to receive +DDL statements telling it what changes to make, in the exact same +way that you give instructions like "turn right at the next intersection" +to your friend who is trying to find your house. In EdgeDB's case, +these commands will start with words like ``create`` and ``drop`` +and ``alter`` to tell it what changes to make. EdgeDB accomplishes +these changes by knowing how to turn your declarative SDL into a schema +migration file that contains the DDL statements to accomplish the +necessary changes. + +DDL: For computers (mostly) +=========================== + +To see what a schema migration file looks like, type ``edgedb migration +create``. Now look inside your ``/dbschema/migrations`` folder. You should +see a file called ``00001.esdl`` with the following, our first view into +what DDL looks like. + +.. code-block:: + + CREATE TYPE default::User { + CREATE PROPERTY name: std::str; + }; + +The declarative schema has now been turned into *imperative* DDL (imperative +meaning "giving orders"), specifically commands telling the database how +to get from the current state to the desired state. Note that, in +contrast to SDL, this code says nothing about the current schema or +its final state. This command would work with the schema of any database +at all that doesn't already have a type called ``User``. + +Let's try one more small migration, in which we decide that we don't +want the ``name`` property anymore. Once again, we are declaring the +final state: a ``User`` type with nothing inside. Update your ``default.esdl`` +to look like this: + +.. code-block:: sdl + + module default { + type User; + } + +As before, typing ``edgedb migration create`` will create a DDL statement to +change the schema from the current state to the one we have declared. This +time we aren't starting from a blank schema, so the stakes are a bit higher. +After all, dropping a property from a type will also drop all existing data +under that property name. Thus, the schema planner will first ask a question +to confirm the change with us. We will learn a lot more about working with +these questions very soon, but in the meantime just press ``y`` to confirm +the change. + +.. code-block:: + + db> did you drop property 'name' of object type 'default::User'? + [y,n,l,c,b,s,q,?] + > y + +Your ``/dbschema/migrations`` folder will now have a new file that contains +the following: + +.. code-block:: + + ALTER TYPE default::User { + DROP PROPERTY name; + }; + +The difference between SDL and DDL is even clearer this time. The DDL +statement alone doesn't give us any indication what the schema looks like; +all anyone could know from this migration script alone is that there is +a ``User`` type inside a module called ``default`` that *doesn't* have +a property called ``name`` anymore. + +.. note:: + + EdgeDB commands inside the REPL use a backslash instead of the ``edgedb`` + command, so you can migrate your schema inside the REPL by typing + ``\migration create`` , followed by ``\migrate``. Not only are the comands + shorter, but they also execute faster. This is because the database client + is already connected to your database when you're inside the REPL, which + is not the case when creating and applying the migration via the CLI. + +Order matters in DDL +-------------------- + +The analogy of a person driving along the road tells us another detail +about DDL: order matters. If you need to first drive two blocks forward +and then turn to the right to reach a destination, that doesn't mean +that you can switch the order around; you can't turn right and *then* +drive two blocks forward and expect to reach the same spot. + +Similarly, if you want add a property to an existing type and the +property's type is a new scalar type, the database will need to create +the new scalar type first. + +Let's take a look at this by first getting EdgeDB to describe our +schema to us. Typing ``describe schema;`` inside the REPL will display +the following DDL statements: + +.. code-block:: + + { + 'create module default if not exists; + create type default::User;', + } + +Thankfully, the DDL statements here are simply the minimum needed +to produce our current schema, not a collection of all the statements +in all of our previous migrations. So while this is a collection of +DDL statements, the DDL produced by ``describe schema`` is just about +as readable as the SDL in your schema. + +If we type ``describe schema as sdl;`` then we'll see the SDL version +of the DDL above: a declarative schema as opposed to statements. + +.. code-block:: sdl + + module default { + type User; + }; + +Now let's add the new scalar type mentioned above and give it to the +``User`` type. Our schema will now look like this: + +.. code-block:: sdl-diff + + module default { + type User { + + name: Name; + } + + scalar type Name extending str; + } + +Note that we are able to define the custom scalar type ``Name`` after we +define the ``User`` type even though we use ``Name`` within that object +because order doesn't matter in SDL. Let's migrate to this new schema +and then use ``describe schema;`` again. You will see the following +statements: + +.. code-block:: + + create module default if not exists; + create scalar type default::Name extending std::str; + create type default::User { + create property name: default::Name; + }; + +The output shows us that the database has gone in the necessary order +to make the schema: first it creates the module, then a scalar type +called ``Name``, and finally the ``User`` type which is now able to +have a property of type ``Name``. + +The output with ``describe schema as sdl;`` is also somewhat similar. +It's SDL, but the order matches that of the DDL statements. + +.. code-block:: sdl + + module default { + scalar type Name extending std::str; + type User { + property name: default::Name; + }; + }; + +Although the schema produced with ``describe schema as sdl;`` may not match +the schema you've written inside ``default.esdl``, it will +show you the order in which statements were needed to reach this final +schema. + +Non-interactive migrations +-------------------------- + +Let's move back to the most basic schema with a single type that +has no properties. + +.. code-block:: sdl + + module default { + type User; + } + +Creating a migration with ``edgedb migration create`` will result +in two questions, one to confirm that we wanted to drop the ``name`` +property, and another to drop the ``Name`` type. + +.. code-block:: bash + + $ edgedb migration create + did you drop property 'name' of object type 'default::User'? + [y,n,l,c,b,s,q,?] + > y + did you drop scalar type 'default::Name'? [y,n,l,c,b,s,q,?] + > y + +This didn't take very long, but you can imagine that it could get +annoying if we had decided to drop ten or more types or properties +and had to say yes to every change. In a case like this, we can use +a non-interactive migration. Let's give that a try. + +First go into your ``/dbschema/migrations`` folder and delete the +most recent ``.edgeql`` file that drops the property ``name`` and +the scalar type ``Name``. Don't worry - the migration hasn't been +applied yet, so you won't confuse the database by deleting it at this +point. And now type ``edgedb migration create --non-interactive``. + +You'll see the same file generated, except that this time there weren't +any questions to answer. A non-interactive migration will work as +long as the database has a high degree of confidence about every change +made, and will fail otherwise. + +A non-interactive migration will fail if we make changes to our schema +that are ambiguous. Let's see if we can make a non-interactive migration +fail by doing just that. Delete the most recent ``.edgeql`` migration +file again, and change the schema to the following that only differs by +a single letter. Can you spot the difference? + +.. code-block:: sdl + + module default { + type User { + nam: Name; + } + scalar type Name extending str; + } + +The only difference from the current schema is that we would like +to change the property name ``name`` to ``nam``, but this time EdgeDB isn't +sure what change we wanted to make. Did we intend to: + +- Change ``name`` to ``nam`` and keep the existing data? +- Drop ``name`` and create a new property called ``nam``? +- Do something else? + +Because of the ambiguity, this non-interactive migration will fail, but with +some pretty helpful output: + +.. code-block:: edgeql-repl + + db> \migration create --non-interactive + EdgeDB intended to apply the following migration: + ALTER TYPE default::User { + ALTER PROPERTY name { + RENAME TO nam; + }; + }; + But confidence is 0.67, below minimum threshold of 0.99999 + Error executing command: EdgeDB is unable to make a decision. + + Please run in interactive mode to confirm changes, or use + `--allow-unsafe` + +As the output suggests, you can add ``--allow-unsafe`` to a non-interactive +migration if you truly want to push the suggestions through regardless +of the migration tool's confidence, but it's more likely in this case +that you would like to interact with the CLI's questions to help it +make a decision. For example, if we had intended to drop the property +``name`` and create a new property ``nam``, we would simply answer +``n`` when it asks us if we intended to *rename* the property. It +then confirms that we are altering the ``User`` type, and finishes +the migration script. + +.. code-block:: edgeql-repl + + db> \migration create + did you rename property 'name' of object type 'default::User' + to 'nam'? [y,n,l,c,b,s,q,?] + > n + did you alter object type 'default::User'? [y,n,l,c,b,s,q,?] + > y + +Afterwards, you can go into the ``.edgeql`` file that was just created +to confirm that these were the changes we wanted to make. It will +look like this: + +.. code-block:: + + CREATE MIGRATION m15hu2pbez5od7fe3shlxwcprbqhvctnfavadccjgjszboy26grgka + ONTO m17m6qjjhtslfkqojvjb4g2vqtzasv5mlbtrqbp6mhwlzv57p5f2uq + { + ALTER TYPE default::User { + CREATE PROPERTY nam: default::Name; + DROP PROPERTY name; + }; + }; + +.. note:: + + See the section on + :ref:`data migrations ` + and migration hashes if you are curious about how migrations are named. + +This migration will alter the ``User`` type by creating a new property and +dropping the old one. If that is what we wanted, then we can now type +``\migrate`` in the REPL or ``edgedb migrate`` at the command line to complete +the migration. + +Questions from the CLI +====================== + +So far we've only learned how to say "yes" or "no" to the CLI's questions +when we migrate a schema, but quite a few other options are presented +when the CLI asks us a question: + +.. code-block:: + + did you create object type 'default::PlayerCharacter'? [y,n,l,c,b,s,q,?] + > y + +The choices ``y`` and ``n`` are obviously "yes" and "no," and you can +probably guess that ``?`` will output help for the available response options, +but the others aren't so clear. Let's go over every option to make sure we +understand them. + +``y`` (or ``yes``) +------------------ + +This will accept the proposed change and move on to the next step. +If it's the last proposed change, the migration will now be created. + +``n`` (or ``no``) +----------------- + +This will reject the proposed change. At this point, the migration +tool will try to suggest a different change if it can, but it won't +always be able to do so. + +We can see this behavior with the same tiny schema change we made +above where we changed a property name from ``name`` to ``nam``. In +the output of that ``migration create``, we see the following: + +- The CLI first asks us if we renamed the property, to which we say "no". +- It then tries to confirm that we have altered the ``User`` type. + We say "no" again. +- The CLI then guesses that maybe we are dropping and creating the + whole ``User`` type instead. This time, we say "yes." +- It then asks us to confirm that we are creating a ``User`` type, + since we have decided to drop the existing one. + +If we say "no" again to the final question, the CLI will throw its hands +up and tell us that it doesn't know what we are trying to do because +there is no way left for it to migrate to the schema that we have +told it to move to. + +Here is what that would look like: + +.. code-block:: + + did you rename property 'name' of object type 'default::User' + to 'nam'? + [y,n,l,c,b,s,q,?] + > n + did you alter object type 'default::User'? [y,n,l,c,b,s,q,?] + > n + did you drop object type 'default::User'? [y,n,l,c,b,s,q,?] + > y + did you create object type 'default::User'? [y,n,l,c,b,s,q,?] + > n + Error executing command: EdgeDB could not resolve migration with + the provided answers. Please retry with different answers. + +``l`` (or ``list``) +------------------- + +This is used to see (list) the actual DDL statements that are being proposed. +When asked the question ``did you alter object type 'default::User'?`` +in the example above, we might be wondering exactly what changes will +be made here. How exactly does the database intend to alter the ``User`` +type if we say "yes?" Simply pressing ``l`` will show it: + +.. code-block:: + + The following DDL statements will be applied: + ALTER TYPE default::User { + CREATE PROPERTY nam: std::str; + DROP PROPERTY name; + }; + +This shows us clear as day that saying "yes" will result in creating +a new property called ``nam`` and dropping the existing ``name`` property. + +So when doubts dwell, press the letter "l!" + +``c`` (or ``confirmed``) +------------------------ + +This simply shows the entire list of statements that have been confirmed. +In other words, this is the migration as it stands at this point. + +``b`` (or ``back``) +------------------- + +This will undo the last confirmation you agreed to and move you back +a step in the migration. If you haven't confirmed any statements yet, +a message will simply appear to let you know that there is nowhere +further back to move to. So pressing ``b`` will never abort a migration. + +The following two keys will stop the migration, but in different ways: + +``s`` (or ``stop``) +------------------- + +This is also known as a 'split'. Pressing ``s`` will complete the +migration at the current point. Any statements that you have applied +will be applied, but the schema will not yet match the schema in your +``.esdl`` file(s). You can easily start another migration to complete +the remaining changes once you have applied the migration that was +just created. This effectively splits the migration into two or more +files. + +``q`` (or ``quit``) +------------------- + +Pressing ``q`` will simply quit without saving any of your progress. + +.. _ref_migration_guide_migrations_and_hashes: + +Data migrations and migration hashes +==================================== + +Sometimes you may want to initialize a database with some default +data, or add some data to a migration that you have just created before +you apply it. + +EdgeDB assumes by default that a migration involves a change to your +schema, so it won't create a migration for you if it doesn't see a +schema change: + +.. code-block:: bash + + $ edgedb migration create + No schema changes detected. + +So how do you create a migration with only data? To do this, just +add ``--allow-empty`` to the command: + +.. code-block:: bash + + $ edgedb migration create --allow-empty + Created myproject/dbschema/migrations/00002.edgeql, + id: m1xseswmheqzxutr55cu66ko4oracannpddujg7gkna2zsjpqm2g3a + +You will now see an empty migration in ``dbschema/migrations`` in which you +can enter some queries. It will look something like this: + +.. code-block:: + + CREATE MIGRATION m1xseswmheqzxutr55cu66ko4oracannpddujg7gkna2zsjpqm2g3a + ONTO m1n5lfw7n74626cverbjwdhcafnhmbezjhwec2rbt46gh3ztoo7mqa + { + }; + +Let's see what happens if we add some queries inside the braces. Assuming +a schema with a simple ``User`` type, we could then add a bunch of queries +such as the following: + +.. code-block:: + + CREATE MIGRATION m1xseswmheqzxutr55cu66ko4oracannpddujg7gkna2zsjpqm2g3a + ONTO m1n5lfw7n74626cverbjwdhcafnhmbezjhwec2rbt46gh3ztoo7mqa + { + insert User { name := 'User 1'}; + insert User { name := 'User 2'}; + delete User filter .name = 'User 2'; + }; + +The problem is, if you save that migration and run ``edgedb migrate``, the CLI +will complain that the migration hash doesn't match what it is supposed to be. +However, it helpfully provides the reason: "Migration names are computed from +the hash of the migration contents." + +Fortunately, it also tells you exactly what the hash (the migration name) +will need to be: + +.. code-block:: + + Error executing command: could not read migrations in + myproject/dbschema/migrations: + + could not read migration file myproject/dbschema/migrations/00002.edgeql: + + Migration name should be: + m13g7j2tqu23yaffv6wkn2adp6hayp76su2qtg2lutdh3mmj5xyk6q, but + m1xseswmheqzxutr55cu66ko4oracannpddujg7gkna2zsjpqm2g3a found instead. + + + Migration names are computed from the hash of the migration contents. + + To proceed you must fix the statement to read as: + CREATE MIGRATION m13g7j2tqu23yaffv6wkn2adp6hayp76su2qtg2lutdh3mmj5xyk6q + ONTO ... + Alternatively, revert the changes to the file. + +If you change the statement to read in exactly the way the output suggests, +the migration will now work. + +That's the manual way to do a data migration, but EdgeDB also has an +``edgedb migration edit`` command that will automate the process for you. +Using ``edgedb migration edit`` will open up the most recent migration for +you to change, and update the migration hash when you close the window. + +Aside from exclusive data migrations, you can also create a migration that +combines schema changes *and* data. This is even easier, since it doesn't even +require appending ``--allow-empty`` to the command. Just do the following: + +1. Change your schema +2. Type ``edgedb migration create`` and respond to the CLI's questions +3. Add your queries to the file (best done on the bottom after the + DDL statements have changed the schema) either manually or using + ``edgedb migration edit`` +4. Type ``edgedb migrate`` to migrate the schema. If you have changed the + schema file manually, copy the suggested name into the migration hash + and type ``edgedb migrate`` again. + +The `EdgeDB tutorial `_ is a good example of a database +set up with both a schema migration and a data migration. Setting +up a database with `schema changes in one file and default data in +a second file `_ is a nice way to separate the two operations +and maintain high readability at the same time. + +Squashing migrations +==================== + +Users often end up making many changes to their schema because +of how effortless it is to do. (And in the next section we will learn +about ``edgedb watch``, which is even more effortless!) This leads to +an interesting side effect: lots of ``.edgeql`` files, many of which +represent trials and approaches that don't end up making it to the +final schema. + +Once you are done, you might want to squash the migrations into a +single file. This is especially nice if you need to frequently initialize +database instances using the same schema, because all migrations are +applied when an instance starts up. You can imagine that the output +would be pretty long if you had dozens and dozens of migration files +to work through: + +.. code-block:: + + Initializing EdgeDB instance... + Applying migrations... + Applied m13brvdizqpva6icpcvmsc3fee2yt5j267uba6jugy6iugcbs2djkq + (00001.edgeql) + Applied m1aildofb3gvhv3jaa5vjlre4pe26locxevqok4semmlgqwu3xayaa + (00002.edgeql) + Applied m1ixxlsdgrlinfijnrbmxdicmpfav33snidudqi7fu4yfhg4nngoza + (00003.edgeql) + Applied m1tsi4amrdbcfjypu72duyckrlvvyb46r3wybd7qnbmem4rjvnbcla + (00004.edgeql) + ...and so on... + Project initialized. + +To squash your migrations, just run ``edgedb migration create`` with the +``--squash`` option. Running this command will first display some helpful +info to keep in mind before committing to the operation: + +.. code-block:: + + Current database revision is: + m16ixoukn7ulqdn7tp6lvx2754hviopanufv2lm6wf4x2borgc3g6a + While squashing migrations is non-destructive, + it may lead to manual work if done incorrectly. + + Items to check before using --squash: + 1. Ensure that `./dbschema` dir is comitted + 2. Ensure that other users of the database have the revision + above or can create database from scratch. + To check a specific instance, run: + edgedb -I migration log --from-db --newest-first --limit 1 + 1. Merge version control branches that contain schema changes + if possible. + + Proceed? [y/n] + +Press ``y`` to squash all of your existing migrations into +a single file. + +Fixups during a squash +---------------------- + +If your schema doesn't match the schema in the database, EdgeDB will +prompt you to create a *fixup* file, which can be useful to, as the CLI +says, "automate upgrading other instances to a squashed revision". +You'll see fixups inside ``/dbschema/fixups``. Their file names +are extremely long because they are simply two migration hashes joined +together by a dash. This means a fixup that begins with + +.. code-block:: + + CREATE MIGRATION + m1v3vqmwif4ml3ucbzi555mjgm4myxs2husqemopo2sz2m7otr22ka + ONTO m16awk2tzhtbupjrzoc4fikgw5okxpfnaazupb6rxudxwin2qfgy5q + +will have a file name a full 116 characters in length. + +The CLI output when using squash along with a fixup is pretty informative +on its own, so let's just walk through the output as you'll see it +in practice. First we'll begin with this schema: + +.. code-block:: sdl + + type User { + name: str; + } + +Then remove ``name: str;`` from the ``User`` type, migrate, put it back +again, and migrate. You can repeat this as many times as you like. +One quick way to "remove" items from your schema that you might want +to restore later is to simply use a ``#`` to comment out the entire line: + +.. code-block:: sdl + + type User { + # name: str; + } + +After a few of these simple migrations, you'll now have multiple files +in your ``/migrations`` folder — none of which were all that useful — and +may be in the mood to squash them into one. + +Next, change to this schema **without migrating it**: + +.. code-block:: sdl + + type User { + name: str; + nickname: str; + } + +Now run ``edgedb migration create --squash``. The output is first +the same as with our previous squash: + +.. code-block:: bash + + $ edgedb migration create --squash + Current database revision: + m16awk2tzhtbupjrzoc4fikgw5okxpfnaazupb6rxudxwin2qfgy5q + While squashing migrations is non-destructive, + it may lead to manual work if done incorrectly. + + Items to check before using --squash: + 1. Ensure that `./dbschema` dir is comitted + 2. Ensure that other users of the database have the revision + above or can create database from scratch. + To check a specific instance, run: + edgedb -I migration log --from-db --newest-first --limit 1 + 3. Merge version control branches that contain schema changes + if possible. + + Proceed? [y/n] + > y + +But after typing ``y``, the CLI will notice that the existing schema +differs from what you have and offers to make a fixup file: + +.. code-block:: + + Your schema differs from the last revision. + A fixup file can be created + to automate upgrading other instances to a squashed revision. + This starts the usual migration creation process. + + Feel free to skip this step if you don't have + other instances to migrate + + Create a fixup file? [y/n] + > y + +You will then see the the same questions that would otherwise show up in +a standard migration: + +.. code-block:: + + db> did you create property 'nickname' of object type 'default::User'? + [y,n,l,c,b,s,q,?] + > y + Squash is complete. + +Finally, the CLI will give some advice on recommended commands when +working with git after doing a squash with a fixup. + +.. code-block:: + + Remember to commit the `dbschema` directory including deleted files + and `fixups` subdirectory. Recommended command: + git add dbschema + + The normal migration process will update your migration history: + edgedb migrate + +We'll take its suggestion to apply the migration: + +.. code-block:: bash + + $ edgedb migrate + + Applied m1v3vqmwif4ml3ucbzi555mjgm4myxs2husqemopo2sz2m7otr22ka + (m16awk2tzhtbupjrzoc4fikgw5okxpfnaazupb6rxudxwin2qfgy5q- + m1oih6aevfcftysukvofwuth2bsuj5aahkdnpabscry7p7ljkgbxma.edgeql) + + +.. note:: + + Squashing is limited to schema changes, so queries inside + data migrations will be discarded during a squash. + +EdgeDB Watch +============ + +Another option when quickly iterating over schema changes is ``edgedb watch``. +This will create a long-running process that keeps track of every time you +save an ``.esdl`` file inside your ``/migrations`` folder, letting you know +if your changes have successfully compiled or not. The ``edgedb watch`` +command itself will show the following input when the process starts up: + +.. code-block:: + + Connecting to EdgeDB instance 'anything' at localhost:10700... + EdgeDB Watch initialized. + Hint: Use `edgedb migration create` and `edgedb migrate --dev-mode` + to apply changes once done. + Monitoring "/home/instancename". + +Unseen to the user, ``edgedb watch`` will begin creating individual migration +scripts for every time you save a change to one of your files. These +are stored as separate "dev mode" migrations, which are sort of like +preliminary migrations that haven't been turned into a standalone +migration script yet. + +We can test this out by starting with this schema: + +.. code-block:: sdl + + module default { + type User { + name: str; + } + } + +Now let's add a single property. Keep an eye on your terminal output and +hit after making a change to the following schema: + +.. code-block:: sdl + + module default { + type User { + name: str; + number: int32; + } + } + +You will see a quick "calculating diff" show up as ``edgedb watch`` checks +to see that the change we made was a valid one. As the change we made was +to a valid schema, the "calculating diff" message will disappear pretty +quickly. + +However, if the schema file you save is incorrect, the output will be a lot +more verbose. Let's add some incorrect syntax to the existing schema: + +.. code-block:: sdl + + module default { + type User { + name: str; + number: int32; + wrong_property: i32; # Should say int32, not i32 + } + } + +Once you hit save, ``edgedb watch`` will suddenly pipe up and inform you +that the schema can't be resolved: + +.. code-block:: + + error: type 'default::i32' does not exist + ┌─ myproject/dbschema/default.esdl:5:25 + │ + 5 │ wrong_property: i32; + │ ^^^ error + + Schema migration error: + cannot proceed until .esdl files are fixed + +Once you correct the ``i32`` type to ``int32``, you will see a message +letting you know that things are okay now. + +.. code-block:: + + Resolved. Schema is up to date now. + +The process will once again quieten down, but will continue to watch your +schema and apply migrations to any changes you make to your schema. + +``edgedb watch`` is best run in a separate instance of your command line so +that you can take care of other tasks — including officially migrating +when you are satisfied with your current schema — without having to +stop the process. + +If you are curious what is happening as ``edgedb watch`` does its thing, +try the following query after you have made some changes. It will return +a few lists of applied migrations, grouped by the way they were generated. + +.. code-block:: + + group schema::Migration { + name, + script + } by .generated_by; + +Some migrations will contain nothing in their ``generated_by`` property, +while those generated by ``edgedb watch`` will have a +``MigrationGeneratedBy.DevMode``. + +.. note:: + + The final option (aside from ``DevMode`` and the empty set) for + ``generated_by`` is ``MigrationGeneratedBy.DDLStatement``, which will + show up if you directly change your schema by using DDL, which is + generally not recommended. + +Once you are satisfied with your changes while running ``edgedb watch``, +just create the migration with ``edgedb migration create`` and then +apply them with one small tweak to the ``migrate`` command: +``edgedb migrate --dev-mode`` to let the CLI know to apply the migrations +made during dev mode that were made by ``edgedb watch``. + +So, you really want to use DDL? +=============================== + +You might have a good reason to use a direct DDL statement or two +to change your schema. How do you make that happen? EdgeDB disables +the usage of DDL by default if you have already carried out a migration +through the recommended migration commands, so this attempt to use DDL +will not work: + +.. code-block:: edgeql-repl + + db> create type MyType; + error: QueryError: bare DDL statements are not + allowed in this database + ┌─ :1:1 + │ + 1 │ create type MyType; + │ ^^^^^^^^^^^^^^^^^^ Use the migration commands instead. + │ + = The `allow_bare_ddl` configuration variable is set to + 'NeverAllow'. The `edgedb migrate` command normally sets + this to avoid accidental schema changes outside of the + migration flow. + +This configuration can be overridden by the following command which +changes the enum ``allow_bare_ddl`` from the default ``NeverAllow`` +to the other option, ``AlwaysAllow``. + +.. code-block:: edgeql-repl + + db> configure current database set allow_bare_ddl := 'AlwaysAllow'; + +Note that the command is ``configure current database`` and not ``configure +instance``, as ``allow_bare_ddl`` is evaluated on the database level. + +That wasn't so bad, so why did the CLI tell us to try to "avoid accidental +schema changes outside of the migration flow?" Why is DDL disabled +after running a migration in the first place? + +So, you really wanted to use DDL but now regret it? +=================================================== + +Let's start out with a very simple schema to see what happens after +DDL is used to directly modify a schema. + +.. code-block:: sdl + + module default { + type User { + name: str; + } + } + +Next, we'll set the current database to allow bare DDL: + +.. code-block:: edgeql-repl + + db> configure current database set allow_bare_ddl := 'AlwaysAllow'; + +And then create a type called ``SomeType`` without any properties: + +.. code-block:: edgeql-repl + + db> create type SomeType; + OK: CREATE TYPE + +Your schema now contains this type, as you can see by typing ``describe +schema`` or ``describe schema as sdl``: + +.. code-block:: + + { + 'module default { + type SomeType; + type User { + property name: std::str; + }; + };', + } + +Great! This type is now inside your schema and you can do whatever +you like with it. + +But this has also ruined the migration flow. Watch what happens when +you try to apply the change: + +.. code-block:: edgeql-repl + + db> \migration create + Error executing command: Database must be updated to + the last migration on the filesystem for + `migration create`. Run: + edgedb migrate + + db> \migrate + Error executing command: database applied migration + history is ahead of migration history in + "myproject/dbschema/migrations" by 1 revision + +Sneakily adding ``SomeType`` into your schema to match won't work +either. The problem is that there *is* a migration already present, +it just doesn't exist inside your ``/migrations`` folder. You can +see it with the following query: + +.. code-block:: edgeql-repl + + db> select schema::Migration {*} + ... filter + ... .generated_by = schema::MigrationGeneratedBy.DDLStatement; + { + schema::Migration { + id: 3882894a-8bb7-11ee-b009-ad814ec6a5f5, + name: 'm1s6oniru3zqepiaxeljt7vcgyynxuwh4ki3zdfr4hfavjozsndfua', + internal: false, + builtin: false, + computed_fields: [], + script: 'SET generated_by := + (schema::MigrationGeneratedBy.DDLStatement); + CREATE TYPE SomeType;', + message: {}, + generated_by: DDLStatement, + }, + } + +Fortunately, the fix is not too hard: we can use the command +``edgedb migration extract``. This command will retrieve the migration(s) +created using DDL and assign each of them a proper file name and hash +inside the ``/dbschema/migrations`` folder, effectively giving them a proper +position inside the migration flow. + +Note that at this point your ``.esdl`` schema will still not match +the database schema, so if you were to type ``edgedb migration create`` +the CLI would then ask you if you want to drop the type that you just +created - because it doesn't exist inside there. So be sure to change +your schema to match the schema inside the database that you have +manually changed via DDL. If in doubt, use ``describe schema as sdl`` +to compare or use ``edgedb migration create`` and check the output. +If the CLI is asking you if you want to drop a type, that means that +you forgot to add it to the schema inside your ``.esdl`` file(s). + + +Multiple migrations to keep data +================================ + +Sometimes you may want to change your schema in a complex way that doesn't +allow you to keep existing data. For example, what if you decide that you +don't need a ``multi`` link anymore but would like to keep some of the +information in the currently linked to objects as an array instead? One +way to make this happen is by migrating more than once. + +Let's give this a try by starting with with a simple ``User`` type that has +a ``friends`` link to other ``User`` objects. (If you've been following along +all this time, a quick migration to this schema will be a breeze.) + +.. code-block:: sdl + + module default { + type User { + name: str; + multi friends: User; + } + } + +First let's insert three ``User`` objects, followed by an update to +make each ``User`` friends with all of the others: + +.. code-block:: edgeql-repl + + db> insert User { + ... name := 'User 1' + ... }; + {default::User {id: d44a19bc-8bc1-11ee-8f28-47d7ec5238fe}} + db> insert User { + ... name := 'User 2' + ... }; + {default::User {id: d5f941c0-8bc1-11ee-8f28-b3f56009a7b0}} + db> insert User { + ... name := 'User 3' + ... }; + {default::User {id: d79cb03e-8bc1-11ee-8f28-43fe3f68004c}} + db> update User set { + ... friends := (select detached User filter User.name != .name) + ... }; + +Now what happens if we now want to change ``multi friends`` to an +``array``? If we were simply changing a scalar property to another +property it would be easy, because EdgeDB would prompt us for a conversion +expression, but a change from a link to a property is different: + +.. code-block:: sdl + + module default { + type User { + name: str; + multi friends: array; + } + } + +Doing a migration as such will just drop the ``friends`` link (along +with its data) and create a new ``friends`` property - without any +data at all. + +To solve this problem, we can do two migrations instead of one. First +we will keep the ``friends`` link, while adding a new property called +``friend_names``: + +.. code-block:: sdl + + module default { + type User { + name: str; + multi friends: User; + friend_names: array; + } + } + +Upon using ``edgedb migration create``, the CLI will simply ask us if we +created a property called ``friend_names``. We haven't applied the migration +yet, so we might as well put the data inside the same migration. A simple +``update`` will do the job! As we learned previously, +``edgedb migration edit`` is the easiest way to add data to a migration. Or +you can manually add the ``update``, try to apply the migration, and change +the migration hash to the output suggested by the CLI. + +.. code-block:: + + CREATE MIGRATION m1hvciatdgpo3a74wagbmwhbunxbridda4qvdbrr3z2a34opks63rq + ONTO m1vktopcva7l6spiinh5e5nnc4dtje4ygw2fhismbmczbyaqbws7jq + { + ALTER TYPE default::User { + CREATE PROPERTY friend_names: array; + }; + update User set { friend_names := array_agg(.friends.name) }; + }; + +Once the migration is applied, we can do a query to confirm that the data +inside ``.friends.name`` when converted to an array is indeed the same as +the data inside the ``friend_names`` property: + +.. code-block:: edgeql-repl + + db> select User { f:= array_agg(.friends.name), friend_names }; + { + default::User { + f: ['User 2', 'User 3'], + friend_names: ['User 2', 'User 3'] + }, + default::User { + f: ['User 1', 'User 3'], + friend_names: ['User 1', 'User 3'] + }, + default::User { + f: ['User 1', 'User 2'], + friend_names: ['User 1', 'User 2'] + }, + } + +Or we could also use the ``all()`` function to confirm that this is the case. + +.. code-block:: edgeql-repl + + db> select all(array_agg(User.friends.name) = User.friend_names); + {true} + +Looks good! And now we can simply remove ``multi friends: User;`` +from our schema and do a final migration. + +Migration internals +=================== + +We've now reached the most optional part of the migrations tutorial, +but an interesting one for those curious about what goes on behind +the scenes during a migration. + +Migrations in EdgeDB before the advent of the ``edgedb project`` flow +were still automated but required more manual work if you didn't +want to accept all of the suggestions provided by the server. This +process is in fact still used to migrate even today; the CLI just +facilitates it by making it easy to respond to the generated suggestions. + +`Early EdgeDB migrations took place inside a transaction `_ +handled by the user that essentially went like this: + +.. code-block:: + + db> start migration to { }; + +This starts the migration, after which the quickest process was to +type ``populate migration`` to accept the statements suggested by +the server, and then ``commit migration`` to finish the process. + +Now, there is another option besides simply typing ``populate migration`` +that allows you to look at and handle the suggestions every step of +the way (in the same way the CLI does today), and this is what we +are going to have some fun with. You can see +`the original migrations RFC `_ if you are curious. + +It is *very* finicky compared to the CLI, resulting in a failed transaction +if any step along the way is different from the expected behavior, +but is an entertaining challenge to try to get right if you want to +truly understand how migrations work in EdgeDB. + +This process requires looking at the server's proposed solutions every +step of the way, and these steps are best seen in JSON format. We can make +this format as readable as possible with the following command: + +.. code-block:: edgeql-repl + + db> \set output-format json-pretty + +First, let's begin with the same same simple schema used in the previous +examples, via the regular ``edgedb migration create`` and ``edgedb migrate`` +commands. + +.. code-block:: sdl + + module default { + type User { + name: str; + } + } + +And, as before, we will make a somewhat ambiguous change by changing +``name`` to ``nam``. + +.. code-block:: sdl + + module default { + type User { + nam: str; + } + } + +And now it's time to give the older migration method a try! To move to this +schema using the old method, we will need to start a migration by pasting our +desired schema into a ``start migration to {};`` block: + +.. code-block:: edgeql-repl + + db> start migration to { + ... module default { + ... type User { + ... nam: str; + ... } + ... } + ... }; + +You should get the output ``OK: START MIGRATION``, followed by a prompt +that ends with ``[tx]`` to show that we are inside of a transaction. +Anything we do here will have no effect on the current registered +schema until we finally commit the migration. + +So now what do we do? We could simply type ``populate migration`` +to accept the server's suggested changes, but let's instead take a +look at them one step at a time. To see the current described change, +type ``describe current migration as json;``. This will generate the +following output: + +.. code-block:: + + { + "parent": "m14opov4ymcbd34x7csurz3mu4u6sik3r7dosz32gist6kpayhdg4q", + "complete": false, + "proposed": { + "prompt": "did you rename property 'name' of object type 'default::User' + to 'nam'?", + "data_safe": true, + "prompt_id": "RenameProperty PROPERTY default::__|name@default|User + TO default::__|nam@default|User", + "confidence": 0.67, + "statements": [{"text": "ALTER TYPE default::User {\n ALTER + PROPERTY name {\n RENAME TO nam;\n };\n};"}], + "required_user_input": [] + }, + "confirmed": [] + } + +The server is telling us with ``"complete": false`` that this suggestion +is not the final step in the migration, that it is 67% confident that +its suggestion is correct, and that we should probably type the following +statement: + +.. code-block:: + + ALTER TYPE default::User { ALTER PROPERTY name { RENAME TO nam; };}; + +Don't forget to remove the newlines (``\n``) from inside the original +suggestion; the transaction will fail if you don't take them out. If the +migration fails at any step, you will see ``[tx]`` change to ``[tx:failed]`` +and you will have to type ``abort migration`` to leave the transaction +and begin the migration again. + +Technically, at this point you are permitted to write any DDL statement +you like and the migration tool will adapt its suggestions to reach +the desired schema. Doing so though is bad practice and is more than likely +to generate an error when you try to commit the migration. +(Even so, give it a try if you're curious.) + +Let's dutifully type the suggested statement above, and then use +``describe current migration as json`` again to see what the current +status of the migration is. This time we see two major differences: +"complete" is now ``true``, meaning that we are at the end of the +proposed migration, and "proposed" does not contain anything. We can +also see our confirmed statement inside "confirmed" at the bottom. + +.. code-block:: + + { + "parent": "m1fgpuxbvd74m6pb72rdikakjv3fv7cftrez7r56qjgonboimp5zoa", + "complete": true, + "proposed": null, + "confirmed": ["ALTER TYPE default::User {\n ALTER PROPERTY name + {\n RENAME TO nam;\n };\n};"] + } + +With this done, you can commit the migration and the migration +will be complete. + +.. code-block:: edgeql-repl + + db[tx]> commit migration; + OK: COMMIT MIGRATION + +Since this migration was created using direct DDL statements, +you will need to use ``edgedb migration extract`` to extract the latest +migration and give it a proper ``.edgeql`` file in the same way we +did above in the "So you really wanted to use DDL but now regret it?" +section. + +.. lint-off + +.. _rfc: https://github.com/edgedb/rfcs/blob/master/text/1000-migrations.rst +.. _transaction: https://www.edgedb.com/docs/reference/ddl/migrations +.. _tutorial: https://www.edgedb.com/tutorial +.. _tutorial_files: https://github.com/edgedb/website/tree/main/content/tutorial/dbschema/migrations + +.. lint-on \ No newline at end of file diff --git a/docs/guides/migrations/index.rst b/docs/guides/migrations/index.rst index 15d77f320d8..d822aa11265 100644 --- a/docs/guides/migrations/index.rst +++ b/docs/guides/migrations/index.rst @@ -1,15 +1,13 @@ .. _ref_guide_advanced_migrations: -================== -Migration Patterns -================== +================= +Schema migrations +================= + +Welcome to the guide to EdgeDB migrations! Let's get started. .. toctree:: - :maxdepth: 3 - - names - backlink - proptype - proptolink - reqlink - recovering + :maxdepth: 2 + + guide + tips diff --git a/docs/guides/migrations/names.rst b/docs/guides/migrations/names.rst deleted file mode 100644 index bcbd734ce4e..00000000000 --- a/docs/guides/migrations/names.rst +++ /dev/null @@ -1,215 +0,0 @@ -.. _ref_migration_names: - -========================== -Making a property required -========================== - -This example shows how a property may evolve to be more and more -strict over time by looking at a user name field. However, similar -evolution may be applicable to other properties that start off with -few restrictions and gradually become more constrained and formalized -as the needs of the project evolve. - -We'll start with a fairly simple schema: - -.. code-block:: sdl - :version-lt: 3.0 - - type User { - property name -> str; - } - -.. code-block:: sdl - - type User { - name: str; - } - -At this stage we don't think that this property needs to be unique or -even required. Perhaps it's only used as a screen name and not as a -way of identifying users. - -.. code-block:: bash - - $ edgedb migration create - did you create object type 'default::User'? [y,n,l,c,b,s,q,?] - > y - Created ./dbschema/migrations/00001.edgeql, id: - m14gwyorqqipfg7riexvbdq5dhgv7x6buqw2jaaulilcmywinmakzq - $ edgedb migrate - Applied m14gwyorqqipfg7riexvbdq5dhgv7x6buqw2jaaulilcmywinmakzq - (00001.edgeql) - -We've got our first migration to set up the schema. Now after using -that for a little while we realize that we want to make ``name`` a -*required property*. So we make the following change in the schema -file: - -.. code-block:: sdl - :version-lt: 3.0 - - type User { - required property name -> str; - } - -.. code-block:: sdl - - type User { - required name: str; - } - -Next we try to migrate: - -.. code-block:: bash - - $ edgedb migration create - did you make property 'name' of object type 'default::User' required? - [y,n,l,c,b,s,q,?] - > y - Please specify an expression to populate existing objects in order to make - property 'name' of object type 'default::User' required: - fill_expr> 'change me' - -Oh! That's right, we can't just make ``name`` *required* because there -could be existing ``User`` objects without a ``name`` at all. So we -need to provide some kind of placeholder value for those cases. We -type ``'change me'`` (although any other string would do, too). This is -different from specifying a ``default`` value since it will be applied -to *existing* objects, whereas the ``default`` applies to *new ones*. - -Unseen to us (unless we take a look at the automatically generated -``.edgeql`` files inside our ``/dbschema`` folder), EdgeDB has created -a migration script that includes the following command to make our -schema change happen. - -.. code-block:: edgeql - - ALTER TYPE default::User { - ALTER PROPERTY name { - SET REQUIRED USING ('change me'); - }; - }; - -We then run :ref:`ref_cli_edgedb_migrate` to apply the changes. - -Next we realize that we actually want to make names unique, perhaps to -avoid confusion or to use them as reliable human-readable identifiers -(unlike ``id``). We update the schema again: - -.. code-block:: sdl - :version-lt: 3.0 - - type User { - required property name -> str { - constraint exclusive; - } - } - -.. code-block:: sdl - - type User { - required name: str { - constraint exclusive; - } - } - -Now we proceed with the migration: - -.. code-block:: bash - - $ edgedb migration create - did you create constraint 'std::exclusive' of property 'name'? - [y,n,l,c,b,s,q,?] - > y - Created ./dbschema/migrations/00003.edgeql, id: - m1dxs3xbk4f3vhmqh6mjzetojafddtwlphp5a3kfbfuyvupjafevya - $ edgedb migrate - edgedb error: ConstraintViolationError: name violates exclusivity - constraint - -Some objects must have the same ``name``, so the migration can't be -applied. We have a couple of options for fixing this: - -1) Review the existing data and manually :eql:stmt:`update` the - entries with duplicate names so that they are unique. -2) Edit the migration to add an :eql:stmt:`update` which will - de-duplicate ``name`` for any potential existing ``User`` objects. - -The first option is good for situations where we want to signal to any -other maintainer of a copy of this project that they need to make a -decision about handling name duplicates in whatever way is appropriate -to them without making an implicit decision once and for all. - -Here we will go with the second option, which is good for situations -where we know enough about the situation that we can make a decision -now and never have to duplicate this effort for any other potential -copies of our project. - -We edit the last migration file ``00003.edgeql``: - -.. code-block:: edgeql-diff - - CREATE MIGRATION m1dxs3xbk4f3vhmqh6mjzetojafddtwlphp5a3kfbfuyvupjafevya - ONTO m1ndhbxx7yudb2dv7zpypl2su2oygyjlggk3olryb5uszofrfml4uq - { - + with U := default::User - + update default::User - + filter U.name = .name and U != default::User - + set { - + # De-duplicate names by appending a random uuid. - + name := .name ++ '_' ++ uuid_generate_v1mc() - + }; - + - ALTER TYPE default::User { - ALTER PROPERTY name { - CREATE CONSTRAINT std::exclusive; - }; - }; - }; - -And then we apply the migration: - -.. code-block:: bash - - $ edgedb migrate - edgedb error: could not read migrations in ./dbschema/migrations: could not - read migration file ./dbschema/migrations/00003.edgeql: migration name - should be `m1t6slgcfne35vir2lcgnqkmaxsxylzvn2hanr6mijbj5esefsp7za` but ` - m1dxs3xbk4f3vhmqh6mjzetojafddtwlphp5a3kfbfuyvupjafevya` is used instead. - Migration names are computed from the hash of the migration contents. To - proceed you must fix the statement to read as: - CREATE MIGRATION m1t6slgcfne35vir2lcgnqkmaxsxylzvn2hanr6mijbj5esefsp7za - ONTO ... - if this migration is not applied to any database. Alternatively, revert the - changes to the file. - -The migration tool detected that we've altered the file and asks us to -update the migration name (acting as a checksum) if this was -deliberate. This is done as a precaution against accidental changes. -Since we've done this on purpose, we can update the file and run -:ref:`ref_cli_edgedb_migrate` again. - -Finally, we evolved our schema all the way from having an optional -property ``name`` all the way to making it both *required* and -*exclusive*. We've worked with the EdgeDB :ref:`migration tools -` to iron out the kinks throughout the -migration process. At this point we take a quick look at the way -duplicate ``User`` objects were resolved to decide whether we need to -do anything more. We can use :eql:func:`re_test` to find names that -look like they are ending in a UUID: - -.. code-block:: edgeql-repl - - db> select User { name } - ... filter - ... re_test('.* [a-z0-9]{8}(-[a-z0-9]{4}){3}-[a-z0-9]{12}$', .name); - { - default::User {name: 'change me bc30d45a-2bcf-11ec-a6c2-6ff21f33a302'}, - default::User {name: 'change me bc30d8a6-2bcf-11ec-a6c2-4f739d559598'}, - } - -Looks like the only duplicates are the users that had no names -originally and that never updated the ``'change me'`` placeholders, so -we can probably let them be for now. In hindsight, it may have been a -good idea to use UUID-based names to populate the empty properties -from the very beginning. diff --git a/docs/guides/migrations/proptolink.rst b/docs/guides/migrations/proptolink.rst deleted file mode 100644 index 5d770e3309a..00000000000 --- a/docs/guides/migrations/proptolink.rst +++ /dev/null @@ -1,334 +0,0 @@ -.. _ref_migration_proptolink: - -============================= -Changing a property to a link -============================= - -This example shows how to change a property into a link. We'll use a -character in an adventure game as the type of data we will evolve. - -Let's start with this schema: - -.. code-block:: sdl - :version-lt: 3.0 - - scalar type CharacterClass extending enum; - - type Character { - required property name -> str; - required property class -> CharacterClass; - } - -.. code-block:: sdl - - scalar type CharacterClass extending enum; - - type Character { - required name: str; - required class: CharacterClass; - } - -We edit the schema file and perform our first migration: - -.. code-block:: bash - - $ edgedb migration create - did you create scalar type 'default::CharacterClass'? [y,n,l,c,b,s,q,?] - > y - did you create object type 'default::Character'? [y,n,l,c,b,s,q,?] - > y - Created ./dbschema/migrations/00001.edgeql, id: - m1fg76t7fbvguwhkmzrx7jwki6jxr6dvkswzeepd5v66oxg27ymkcq - $ edgedb migrate - Applied m1fg76t7fbvguwhkmzrx7jwki6jxr6dvkswzeepd5v66oxg27ymkcq - (00001.edgeql) - -The initial setup may look something like this: - -.. code-block:: edgeql-repl - - db> select Character {name, class}; - { - default::Character {name: 'Alice', class: warrior}, - default::Character {name: 'Billie', class: scholar}, - default::Character {name: 'Cameron', class: rogue}, - } - -After some development work we decide to add more details about the -available classes and encapsulate that information into its own type. -This way instead of a property ``class`` we want to end up with a link -``class`` to the new data structure. Since we cannot just -:eql:op:`cast ` a scalar into an object, we'll need to convert -between the two explicitly. This means that we will need to have both -the old and the new "class" information to begin with: - -.. code-block:: sdl - :version-lt: 3.0 - - scalar type CharacterClass extending enum; - - type NewClass { - required property name -> str; - multi property skills -> str; - } - - type Character { - required property name -> str; - required property class -> CharacterClass; - link new_class -> NewClass; - } - -.. code-block:: sdl - - scalar type CharacterClass extending enum; - - type NewClass { - required name: str; - multi skills: str; - } - - type Character { - required name: str; - required class: CharacterClass; - new_class: NewClass; - } - -We update the schema file and migrate to the new state: - -.. code-block:: bash - - $ edgedb migration create - did you create object type 'default::NewClass'? [y,n,l,c,b,s,q,?] - > y - did you create link 'new_class' of object type 'default::Character'? - [y,n,l,c,b,s,q,?] - > y - Created ./dbschema/migrations/00002.edgeql, id: - m1uttd6f7fpiwiwikhdh6qyijb6pcji747ccg2cyt5357i3wsj3l3q - $ edgedb migrate - Applied m1uttd6f7fpiwiwikhdh6qyijb6pcji747ccg2cyt5357i3wsj3l3q - (00002.edgeql) - -It makes sense to add a data migration as a way of consistently -creating ``NewClass`` objects as well as populating ``new_class`` -links based on the existing ``class`` property. So we first create an -empty migration: - -.. code-block:: bash - - $ edgedb migration create --allow-empty - Created ./dbschema/migrations/00003.edgeql, id: - m1iztxroh3ifoeqmvxncy77whnaei6tp5j3sewyxtrfysronjkxgga - -And then edit the ``00003.edgeql`` file to create and update objects: - -.. code-block:: edgeql-diff - - CREATE MIGRATION m1iztxroh3ifoeqmvxncy77whnaei6tp5j3sewyxtrfysronjkxgga - ONTO m1uttd6f7fpiwiwikhdh6qyijb6pcji747ccg2cyt5357i3wsj3l3q - { - + insert default::NewClass { - + name := 'Warrior', - + skills := {'punch', 'kick', 'run', 'jump'}, - + }; - + insert default::NewClass { - + name := 'Scholar', - + skills := {'read', 'write', 'analyze', 'refine'}, - + }; - + insert default::NewClass { - + name := 'Rogue', - + skills := {'impress', 'sing', 'steal', 'run', 'jump'}, - + }; - + - + update default::Character - + set { - + new_class := assert_single(( - + select default::NewClass - + filter .name ilike default::Character.class - + )), - + }; - }; - -Trying to apply the data migration will produce the following -reminder: - -.. code-block:: bash - - $ edgedb migrate - edgedb error: could not read migrations in ./dbschema/migrations: - could not read migration file ./dbschema/migrations/00003.edgeql: - migration name should be - `m1e3d3eg3j2pr7acie4n5rrhaddyhkiy5kgckd5l7h5ysrpmgwxl5a` but - `m1iztxroh3ifoeqmvxncy77whnaei6tp5j3sewyxtrfysronjkxgga` is used - instead. - Migration names are computed from the hash of the migration - contents. To proceed you must fix the statement to read as: - CREATE MIGRATION m1e3d3eg3j2pr7acie4n5rrhaddyhkiy5kgckd5l7h5ysrpmgwxl5a - ONTO ... - if this migration is not applied to any database. Alternatively, - revert the changes to the file. - -The migration tool detected that we've altered the file and asks us to -update the migration name (acting as a checksum) if this was -deliberate. This is done as a precaution against accidental changes. -Since we've done this on purpose, we can update the file and run -:ref:`ref_cli_edgedb_migrate` again. - -We can see the changes after the data migration is complete: - -.. code-block:: edgeql-repl - - db> select Character { - ... name, - ... class, - ... new_class: { - ... name, - ... } - ... }; - { - default::Character { - name: 'Alice', - class: warrior, - new_class: default::NewClass {name: 'Warrior'}, - }, - default::Character { - name: 'Billie', - class: scholar, - new_class: default::NewClass {name: 'Scholar'}, - }, - default::Character { - name: 'Cameron', - class: rogue, - new_class: default::NewClass {name: 'Rogue'}, - }, - } - -Everything seems to be in order. It is time to clean up the old -property and ``CharacterClass`` :eql:type:`enum`: - -.. code-block:: sdl - :version-lt: 3.0 - - type NewClass { - required property name -> str; - multi property skills -> str; - } - - type Character { - required property name -> str; - link new_class -> NewClass; - } - -.. code-block:: sdl - - type NewClass { - required name: str; - multi skills: str; - } - - type Character { - required name: str; - new_class: NewClass; - } - -The migration tools should have no trouble detecting the things we -just removed: - -.. code-block:: bash - - $ edgedb migration create - did you drop property 'class' of object type 'default::Character'? - [y,n,l,c,b,s,q,?] - > y - did you drop scalar type 'default::CharacterClass'? [y,n,l,c,b,s,q,?] - > y - Created ./dbschema/migrations/00004.edgeql, id: - m1jdnz5bxjj6kjz2pylvudli5rvw4jyr2ilpb4hit3yutwi3bq34ha - $ edgedb migrate - Applied m1jdnz5bxjj6kjz2pylvudli5rvw4jyr2ilpb4hit3yutwi3bq34ha - (00004.edgeql) - -Now that the original property and scalar type are gone, we can rename -the "new" components, so that they become ``class`` link and -``CharacterClass`` type, respectively: - -.. code-block:: sdl - :version-lt: 3.0 - - type CharacterClass { - required property name -> str; - multi property skills -> str; - } - - type Character { - required property name -> str; - link class -> CharacterClass; - } - -.. code-block:: sdl - - type CharacterClass { - required name: str; - multi skills: str; - } - - type Character { - required name: str; - class: CharacterClass; - } - -The migration tools pick up the changes without any issues again. It -may seem tempting to combine the last two steps, but deleting and -renaming in a single step would cause the migration tools to report a -name clash. As a general rule, it is a good idea to never mix renaming -and deleting of closely interacting entities in the same migration. - -.. code-block:: bash - - $ edgedb migration create - did you rename object type 'default::NewClass' to - 'default::CharacterClass'? [y,n,l,c,b,s,q,?] - > y - did you rename link 'new_class' of object type 'default::Character' to - 'class'? [y,n,l,c,b,s,q,?] - > y - Created ./dbschema/migrations/00005.edgeql, id: - m1ra4fhx2erkygbhi7qjxt27yup5aw5hkr5bekn5y5jeam5yn57vsa - $ edgedb migrate - Applied m1ra4fhx2erkygbhi7qjxt27yup5aw5hkr5bekn5y5jeam5yn57vsa - (00005.edgeql) - -Finally, we have replaced the original ``class`` property with a link: - -.. code-block:: edgeql-repl - - db> select Character { - ... name, - ... class: { - ... name, - ... skills, - ... } - ... }; - { - default::Character { - name: 'Alice', - class: default::CharacterClass { - name: 'Warrior', - skills: {'punch', 'kick', 'run', 'jump'}, - }, - }, - default::Character { - name: 'Billie', - class: default::CharacterClass { - name: 'Scholar', - skills: {'read', 'write', 'analyze', 'refine'}, - }, - }, - default::Character { - name: 'Cameron', - class: default::CharacterClass { - name: 'Rogue', - skills: {'impress', 'sing', 'steal', 'run', 'jump'}, - }, - }, - } diff --git a/docs/guides/migrations/proptype.rst b/docs/guides/migrations/proptype.rst deleted file mode 100644 index 164e0555b8f..00000000000 --- a/docs/guides/migrations/proptype.rst +++ /dev/null @@ -1,201 +0,0 @@ -.. _ref_migration_proptype: - -=============================== -Changing the type of a property -=============================== - -This example shows how to change the type of a property. We'll use a -character in an adventure game as the type of data we will evolve. - -Let's start with this schema: - -.. code-block:: sdl - :version-lt: 3.0 - - type Character { - required property name -> str; - required property description -> str; - } - -.. code-block:: sdl - - type Character { - required name: str; - required description: str; - } - -We edit the schema file and perform our first migration: - -.. code-block:: bash - - $ edgedb migration create - did you create object type 'default::Character'? [y,n,l,c,b,s,q,?] - > y - Created ./dbschema/migrations/00001.edgeql, id: - m1paw3ogpsdtxaoywd6pl6beg2g64zj4ykhd43zby4eqh64yjad47a - $ edgedb migrate - Applied m1paw3ogpsdtxaoywd6pl6beg2g64zj4ykhd43zby4eqh64yjad47a - (00001.edgeql) - -The intent is for the ``description`` to provide some text which -serves both as something to be shown to the player as well as -determining some game actions. Se we end up with something like this: - -.. code-block:: edgeql-repl - - db> select Character {name, description}; - { - default::Character {name: 'Alice', description: 'Tall and strong'}, - default::Character {name: 'Billie', description: 'Smart and aloof'}, - default::Character {name: 'Cameron', description: 'Dashing and smooth'}, - } - -However, as we keep developing our game it becomes apparent that this -is less of a "description" and more of a "character class", so at -first we just rename the property to reflect that: - -.. code-block:: sdl - :version-lt: 3.0 - - type Character { - required property name -> str; - required property class -> str; - } - -.. code-block:: sdl - - type Character { - required name: str; - required class: str; - } - -The migration gives us this: - -.. code-block:: bash - - $ edgedb migration create - did you rename property 'description' of object type 'default::Character' - to 'class'? [y,n,l,c,b,s,q,?] - > y - Created ./dbschema/migrations/00002.edgeql, id: - m1ljrgrofsqkvo5hsxc62mnztdhlerxp6ucdto262se6dinhuj4mqq - $ edgedb migrate - Applied m1ljrgrofsqkvo5hsxc62mnztdhlerxp6ucdto262se6dinhuj4mqq - (00002.edgeql) - -EdgeDB detected that the change looked like a property was being -renamed, which we confirmed. Since this was an existing property being -renamed, the data is all preserved: - -.. code-block:: edgeql-repl - - db> select Character {name, class}; - { - default::Character {name: 'Alice', class: 'Tall and strong'}, - default::Character {name: 'Billie', class: 'Smart and aloof'}, - default::Character {name: 'Cameron', class: 'Dashing and smooth'}, - } - -The contents of the ``class`` property are a bit too verbose, so we -decide to update them. In order for this update to be consistently -applied across several developers, we will make it in the form of a -*data migration*: - -.. code-block:: bash - - $ edgedb migration create --allow-empty - Created ./dbschema/migrations/00003.edgeql, id: - m1qv2pdksjxxzlnujfed4b6to2ppuodj3xqax4p3r75yfef7kd7jna - -Now we can edit the file ``00003.edgeql`` directly: - -.. code-block:: edgeql-diff - - CREATE MIGRATION m1qv2pdksjxxzlnujfed4b6to2ppuodj3xqax4p3r75yfef7kd7jna - ONTO m1ljrgrofsqkvo5hsxc62mnztdhlerxp6ucdto262se6dinhuj4mqq - { - + update default::Character - + set { - + class := - + 'warrior' if .class = 'Tall and strong' else - + 'scholar' if .class = 'Smart and aloof' else - + 'rogue' - + }; - }; - -We're ready to apply the migration: - -.. code-block:: bash - - $ edgedb migrate - edgedb error: could not read migrations in ./dbschema/migrations: - could not read migration file ./dbschema/migrations/00003.edgeql: - migration name should be - `m1ryafvp24g5eqjeu65zr4bqf6m3qath3lckfdhoecfncmr7zshehq` - but `m1qv2pdksjxxzlnujfed4b6to2ppuodj3xqax4p3r75yfef7kd7jna` is used - instead. - Migration names are computed from the hash of the migration - contents. To proceed you must fix the statement to read as: - CREATE MIGRATION m1ryafvp24g5eqjeu65zr4bqf6m3qath3lckfdhoecfncmr7zshehq - ONTO ... - if this migration is not applied to any database. Alternatively, - revert the changes to the file. - -The migration tool detected that we've altered the file and asks us to -update the migration name (acting as a checksum) if this was -deliberate. This is done as a precaution against accidental changes. -Since we've done this on purpose, we can update the file and run -:ref:`ref_cli_edgedb_migrate` again. - -As the game becomes more stable there's no reason for the ``class`` to -be a :eql:type:`str` anymore, instead we can use an :eql:type:`enum` -to make sure that we don't accidentally use some invalid value for it. - -.. code-block:: sdl - :version-lt: 3.0 - - scalar type CharacterClass extending enum; - - type Character { - required property name -> str; - required property class -> CharacterClass; - } - -.. code-block:: sdl - - scalar type CharacterClass extending enum; - - type Character { - required name: str; - required class: CharacterClass; - } - -Fortunately, we've already updated the ``class`` strings to match the -:eql:type:`enum` values, so that a simple cast will convert all the -values. If we had not done this earlier we would need to do it now in -order for the type change to work. - -.. code-block:: bash - - $ edgedb migration create - did you create scalar type 'default::CharacterClass'? [y,n,l,c,b,s,q,?] - > y - did you alter the type of property 'class' of object type - 'default::Character'? [y,n,l,c,b,s,q,?] - > y - Created ./dbschema/migrations/00004.edgeql, id: - m1hc4yynkejef2hh7fvymvg3f26nmynpffksg7yvfksqufif6lulgq - $ edgedb migrate - Applied m1hc4yynkejef2hh7fvymvg3f26nmynpffksg7yvfksqufif6lulgq - (00004.edgeql) - -The final migration converted all the ``class`` property values: - -.. code-block:: edgeql-repl - - db> select Character {name, class}; - { - default::Character {name: 'Alice', class: warrior}, - default::Character {name: 'Billie', class: scholar}, - default::Character {name: 'Cameron', class: rogue}, - } diff --git a/docs/guides/migrations/recovering.rst b/docs/guides/migrations/recovering.rst deleted file mode 100644 index 6d6043c7213..00000000000 --- a/docs/guides/migrations/recovering.rst +++ /dev/null @@ -1,65 +0,0 @@ -.. _ref_migration_recovering: - -========================== -Recovering lost migrations -========================== - -Each time you create a migration with :ref:`ref_cli_edgedb_migration_create`, -a file containing the DDL for that migration is created in -``dbschema/migrations``. When you apply a migration with -:ref:`ref_cli_edgedb_migration_apply` or :ref:`ref_cli_edgedb_migrate`, the -database stores a record of the migration it applied. - -On rare occasions, you may find you have deleted your migration files by -mistake. If you don't care about any of your data and don't need to keep your -migration history, you can :ref:`wipe ` your -database and start over, creating a single migration to the current state of -your schema. If that's not an option, all hope is not lost. You can instead -recover your migrations from the database. - -Run this query to see your migrations: - -.. code-block:: edgeql - - select schema::Migration { - name, - script, - parents: {name} - } - -You can rebuild your migrations from the results of this query, either manually -or via a script if you've applied too many of them to recreate by hand. -Migrations in the file system are named sequentially starting from -``00001.edgeql``. They are in this format: - -.. code-block:: edgeql - - CREATE MIGRATION m1rsm66e5pvh5ets2yznutintmqnxluzvgbocspi6umd3ht64e4naq - # ☝️ Replace with migration name - ONTO m1l5esbbycsyqcnx6udxx24riavvyvkskchtekwe7jqx5mmiyli54a - # ☝️ Replace with parent migration name - { - # script - # ☝️ Replace with migration script - }; - -or if this is the first migration: - -.. code-block:: edgeql - - CREATE MIGRATION m1l5esbbycsyqcnx6udxx24riavvyvkskchtekwe7jqx5mmiyli54a - # ☝️ Replace with migration name - ONTO initial - { - # script - # ☝️ Replace with migration script - }; - -Replace the name, script, and parent name with the values from -your ``Migration`` query results. - -You can identify the first migration in your query results as the one with no -object linked on ``parents``. Order the other migrations by chaining the links. -The ``Migration`` with the initial migration linked via ``parents`` is the -second migration — ``00002.edgeql``. The migration linking to the second -migration via ``parents`` is the third migration, and so on). diff --git a/docs/guides/migrations/reqlink.rst b/docs/guides/migrations/reqlink.rst deleted file mode 100644 index f8b3f4cea9d..00000000000 --- a/docs/guides/migrations/reqlink.rst +++ /dev/null @@ -1,376 +0,0 @@ -.. _ref_migration_reqlink: - -====================== -Adding a required link -====================== - -This example shows how to setup a required link. We'll use a -character in an adventure game as the type of data we will evolve. - -Let's start with this schema: - -.. code-block:: sdl - :version-lt: 3.0 - - type Character { - required property name -> str; - } - -.. code-block:: sdl - - type Character { - required name: str; - } - -We edit the schema file and perform our first migration: - -.. code-block:: bash - - $ edgedb migration create - did you create object type 'default::Character'? [y,n,l,c,b,s,q,?] - > y - Created ./dbschema/migrations/00001.edgeql, id: - m1xvu7o4z5f5xfwuun2vee2cryvvzh5lfilwgkulmqpifo5m3dnd6a - $ edgedb migrate - Applied m1xvu7o4z5f5xfwuun2vee2cryvvzh5lfilwgkulmqpifo5m3dnd6a - (00001.edgeql) - -This time around let's practice performing a data migration and set up -our character data. For this purpose we can create an empty migration -and fill it out as we like: - -.. code-block:: bash - - $ edgedb migration create --allow-empty - Created ./dbschema/migrations/00002.edgeql, id: - m1lclvwdpwitjj4xqm45wp74y4wjyadljct5o6bsctlnh5xbto74iq - -We edit the ``00002.edgeql`` file by simply adding the query to add -characters to it. We can use :eql:stmt:`for` to add multiple characters -like this: - -.. code-block:: edgeql-diff - - CREATE MIGRATION m1lclvwdpwitjj4xqm45wp74y4wjyadljct5o6bsctlnh5xbto74iq - ONTO m1xvu7o4z5f5xfwuun2vee2cryvvzh5lfilwgkulmqpifo5m3dnd6a - { - + for name in {'Alice', 'Billie', 'Cameron', 'Dana'} - + union ( - + insert default::Character { - + name := name - + } - + ); - }; - -Trying to apply the data migration will produce the following -reminder: - -.. code-block:: bash - - $ edgedb migrate - edgedb error: could not read migrations in ./dbschema/migrations: - could not read migration file ./dbschema/migrations/00002.edgeql: - migration name should be - `m1juin65wriqmb4vwg23fiyajjxlzj2jyjv5qp36uxenit5y63g2iq` but - `m1lclvwdpwitjj4xqm45wp74y4wjyadljct5o6bsctlnh5xbto74iq` is used instead. - Migration names are computed from the hash of the migration contents. To - proceed you must fix the statement to read as: - CREATE MIGRATION m1juin65wriqmb4vwg23fiyajjxlzj2jyjv5qp36uxenit5y63g2iq - ONTO ... - if this migration is not applied to any database. Alternatively, - revert the changes to the file. - -The migration tool detected that we've altered the file and asks us to -update the migration name (acting as a checksum) if this was -deliberate. This is done as a precaution against accidental changes. -Since we've done this on purpose, we can update the file and run -:ref:`ref_cli_edgedb_migrate` again. - -.. code-block:: edgeql-diff - - - CREATE MIGRATION m1lclvwdpwitjj4xqm45wp74y4wjyadljct5o6bsctlnh5xbto74iq - + CREATE MIGRATION m1juin65wriqmb4vwg23fiyajjxlzj2jyjv5qp36uxenit5y63g2iq - ONTO m1xvu7o4z5f5xfwuun2vee2cryvvzh5lfilwgkulmqpifo5m3dnd6a - { - # ... - }; - -After we apply the data migration we should be able to see the added -characters: - -.. code-block:: edgeql-repl - - db> select Character {name}; - { - default::Character {name: 'Alice'}, - default::Character {name: 'Billie'}, - default::Character {name: 'Cameron'}, - default::Character {name: 'Dana'}, - } - -Let's add a character ``class`` represented by a new type to our -schema and data. Unlike in :ref:`this scenario `, -we will add the ``required link class`` right away, without any intermediate -properties. So we end up with a schema like this: - -.. code-block:: sdl - :version-lt: 3.0 - - type CharacterClass { - required property name -> str; - multi property skills -> str; - } - - type Character { - required property name -> str; - required link class -> CharacterClass; - } - -.. code-block:: sdl - - type CharacterClass { - required name: str; - multi skills: str; - } - - type Character { - required name: str; - required class: CharacterClass; - } - -We go ahead and try to apply this new schema: - -.. code-block:: bash - - $ edgedb migration create - did you create object type 'default::CharacterClass'? [y,n,l,c,b,s,q,?] - > y - did you create link 'class' of object type 'default::Character'? - [y,n,l,c,b,s,q,?] - > y - Please specify an expression to populate existing objects in order to make - link 'class' of object type 'default::Character' required: - fill_expr> - -Uh-oh! Unlike in a situation with a :ref:`required property -`, it's not a good idea to just -:eql:stmt:`insert` a new ``CharacterClass`` object for every -character. So we should abort this migration attempt and rethink -our strategy. We need a separate step where the ``class`` link is -not *required* so that we can write some custom queries to handle -the character classes: - -.. code-block:: sdl - :version-lt: 3.0 - - type CharacterClass { - required property name -> str; - multi property skills -> str; - } - - type Character { - required property name -> str; - link class -> CharacterClass; - } - -.. code-block:: sdl - - type CharacterClass { - required name: str; - multi skills: str; - } - - type Character { - required name: str; - class: CharacterClass; - } - -We can now create a migration for our new schema, but we won't apply -it right away: - -.. code-block:: bash - - $ edgedb migration create - did you create object type 'default::CharacterClass'? [y,n,l,c,b,s,q,?] - > y - did you create link 'class' of object type 'default::Character'? - [y,n,l,c,b,s,q,?] - > y - Created ./dbschema/migrations/00003.edgeql, id: - m1jie3xamsm2b7ygqccwfh2degdi45oc7mwuyzjkanh2qwgiqvi2ya - -We don't need to create a blank migration to add data, we can add our -modifications into the migration that adds the ``class`` link -directly. Doing this makes sense when the schema changes seem to -require the data migration and the two types of changes logically go -together. We will need to create some ``CharacterClass`` objects as -well as :eql:stmt:`update` the ``class`` link on existing -``Character`` objects: - -.. code-block:: edgeql-diff - - CREATE MIGRATION m1jie3xamsm2b7ygqccwfh2degdi45oc7mwuyzjkanh2qwgiqvi2ya - ONTO m1juin65wriqmb4vwg23fiyajjxlzj2jyjv5qp36uxenit5y63g2iq - { - CREATE TYPE default::CharacterClass { - CREATE REQUIRED PROPERTY name -> std::str; - CREATE MULTI PROPERTY skills -> std::str; - }; - ALTER TYPE default::Character { - CREATE LINK class -> default::CharacterClass; - }; - - + insert default::CharacterClass { - + name := 'Warrior', - + skills := {'punch', 'kick', 'run', 'jump'}, - + }; - + insert default::CharacterClass { - + name := 'Scholar', - + skills := {'read', 'write', 'analyze', 'refine'}, - + }; - + insert default::CharacterClass { - + name := 'Rogue', - + skills := {'impress', 'sing', 'steal', 'run', 'jump'}, - + }; - + # All warriors - + update default::Character - + filter .name in {'Alice'} - + set { - + class := assert_single(( - + select default::CharacterClass - + filter .name = 'Warrior' - + )), - + }; - + # All scholars - + update default::Character - + filter .name in {'Billie'} - + set { - + class := assert_single(( - + select default::CharacterClass - + filter .name = 'Scholar' - + )), - + }; - + # All rogues - + update default::Character - + filter .name in {'Cameron', 'Dana'} - + set { - + class := assert_single(( - + select default::CharacterClass - + filter .name = 'Rogue' - + )), - + }; - }; - -In a real game we might have a lot more characters and so a good way -to update them all is to update characters of the same class in bulk. - -Just like before we'll be reminded to fix the migration name since -we've altered the migration file. After fixing the migration hash we -can apply it. Now all our characters should have been assigned their -classes: - -.. code-block:: edgeql-repl - - db> select Character { - ... name, - ... class: { - ... name - ... } - ... }; - { - default::Character { - name: 'Alice', - class: default::CharacterClass {name: 'Warrior'}, - }, - default::Character { - name: 'Billie', - class: default::CharacterClass {name: 'Scholar'}, - }, - default::Character { - name: 'Cameron', - class: default::CharacterClass {name: 'Rogue'}, - }, - default::Character { - name: 'Dana', - class: default::CharacterClass {name: 'Rogue'}, - }, - } - -We're finally ready to make the ``class`` link *required*. We update -the schema: - -.. code-block:: sdl - :version-lt: 3.0 - - type CharacterClass { - required property name -> str; - multi property skills -> str; - } - - type Character { - required property name -> str; - required link class -> CharacterClass; - } - -.. code-block:: sdl - - type CharacterClass { - required name: str; - multi skills: str; - } - - type Character { - required name: str; - required class: CharacterClass; - } - -And we perform our final migration: - -.. code-block:: bash - - $ edgedb migration create - did you make link 'class' of object type 'default::Character' required? - [y,n,l,c,b,s,q,?] - > y - Please specify an expression to populate existing objects in order to - make link 'class' of object type 'default::Character' required: - fill_expr> assert_exists(.class) - Created ./dbschema/migrations/00004.edgeql, id: - m14yblybdo77c7bjtm6nugiy5cs6pl6rnuzo5b27gamy4zhuwjifia - -The migration system doesn't know that we've already assigned ``class`` values -to all the ``Character`` objects, so it still asks us for an expression to be -used in case any of the objects need it. We can use ``assert_exists(.class)`` -here as a way of being explicit about the fact that we expect the values to -already be present. Missing values would have caused an error even without the -``assert_exists`` wrapper, but being explicit may help us capture the intent -and make debugging a little easier if anyone runs into a problem at this step. - -In fact, before applying this migration, let's actually add a new -``Character`` to see what happens: - -.. code-block:: edgeql-repl - - db> insert Character {name := 'Eric'}; - { - default::Character { - id: 9f4ac7a8-ac38-11ec-b076-afefd12d7e66, - }, - } - -Our attempt at migrating fails as we expected: - -.. code-block:: bash - - $ edgedb migrate - edgedb error: MissingRequiredError: missing value for required link - 'class' of object type 'default::Character' - Detail: Failing object id is 'ee604992-c1b1-11ec-ad59-4f878963769f'. - -After removing the bugged ``Character``, we can migrate without any problems: - -.. code-block:: bash - - $ edgedb migrate - Applied m14yblybdo77c7bjtm6nugiy5cs6pl6rnuzo5b27gamy4zhuwjifia - (00004.edgeql) diff --git a/docs/guides/migrations/tips.rst b/docs/guides/migrations/tips.rst new file mode 100644 index 00000000000..0e2f07e56a4 --- /dev/null +++ b/docs/guides/migrations/tips.rst @@ -0,0 +1,1410 @@ +.. _ref_migration_tips: + +==== +Tips +==== + +:edb-alt-title: Schema migration tips + +Adding backlinks +---------------- + +This example shows how to handle a schema that makes use of a +backlink. We'll use a linked-list structure to represent a sequence of +events. + +We'll start with this schema: + +.. code-block:: sdl + :version-lt: 3.0 + + type Event { + required property name -> str; + link prev -> Event; + + # ... more properties and links + } + +.. code-block:: sdl + + type Event { + required name: str; + prev: Event; + + # ... more properties and links + } + +We specify a ``prev`` link because that will make adding a new +``Event`` at the end of the chain easier, since we'll be able to +specify the payload and the chain the ``Event`` should be appended to +in a single :eql:stmt:`insert`. Once we've updated the schema +file we proceed with our first migration: + +.. code-block:: bash + + $ edgedb migration create + did you create object type 'default::Event'? [y,n,l,c,b,s,q,?] + > y + Created ./dbschema/migrations/00001.edgeql, id: + m1v3ahcx5f43y6mlsdmlz2agnf6msbc7rt3zstiqmezaqx4ev2qovq + $ edgedb migrate + Applied m1v3ahcx5f43y6mlsdmlz2agnf6msbc7rt3zstiqmezaqx4ev2qovq + (00001.edgeql) + +We now have a way of chaining events together. We might create a few +events like these: + +.. code-block:: edgeql-repl + + db> select Event { + ... name, + ... prev: { name }, + ... }; + { + default::Event {name: 'setup', prev: {}}, + default::Event {name: 'work', prev: default::Event {name: 'setup'}}, + default::Event {name: 'cleanup', prev: default::Event {name: 'work'}}, + } + +It seems like having a ``next`` link would be useful, too. So we can +define it as a computed link by using :ref:`backlink +` notation: + +.. code-block:: sdl + :version-lt: 3.0 + + type Event { + required property name -> str; + + link prev -> Event; + link next := . y + Created ./dbschema/migrations/00002.edgeql, id: + m1qpukyvw2m4lmomoseni7vdmevk4wzgsbviojacyrqgiyqjp5sdsa + $ edgedb migrate + Applied m1qpukyvw2m4lmomoseni7vdmevk4wzgsbviojacyrqgiyqjp5sdsa + (00002.edgeql) + +Trying out the new link on our existing data gives us: + +.. code-block:: edgeql-repl + + db> select Event { + ... name, + ... prev_name := .prev.name, + ... next_name := .next.name, + ... }; + { + default::Event { + name: 'setup', + prev_name: {}, + next_name: {'work'}, + }, + default::Event { + name: 'work', + prev_name: 'setup', + next_name: {'cleanup'}, + }, + default::Event { + name: 'cleanup', + prev_name: 'work', + next_name: {}, + }, + } + +That's not quite right. The value of ``next_name`` appears to be a set +rather than a singleton. This is because the link ``prev`` is +many-to-one and so ``next`` is one-to-many, making it a *multi* link. +Let's fix that by making the link ``prev`` a one-to-one, after all +we're interested in building event chains, not trees. + +.. code-block:: sdl + :version-lt: 3.0 + + type Event { + required property name -> str; + + link prev -> Event { + constraint exclusive; + }; + link next := . y + Created ./dbschema/migrations/00003.edgeql, id: + m17or2bfywuckdqeornjmjh7c2voxgatspcewyefcd4p2vbdepimoa + $ edgedb migrate + Applied m17or2bfywuckdqeornjmjh7c2voxgatspcewyefcd4p2vbdepimoa + (00003.edgeql) + +The new ``next`` computed link is now inferred as a ``single`` link +and so the query results for ``next_name`` and ``prev_name`` are +symmetrical: + +.. code-block:: edgeql-repl + + db> select Event { + ... name, + ... prev_name := .prev.name, + ... next_name := .next.name, + ... }; + { + default::Event {name: 'setup', prev_name: {}, next_name: 'work'}, + default::Event {name: 'work', prev_name: 'setup', next_name: 'cleanup'}, + default::Event {name: 'cleanup', prev_name: 'work', next_name: {}}, + } + +Making a property required +-------------------------- + +This example shows how a property may evolve to be more and more +strict over time by looking at a user name field. However, similar +evolution may be applicable to other properties that start off with +few restrictions and gradually become more constrained and formalized +as the needs of the project evolve. + +We'll start with a fairly simple schema: + +.. code-block:: sdl + :version-lt: 3.0 + + type User { + property name -> str; + } + +.. code-block:: sdl + + type User { + name: str; + } + +At this stage we don't think that this property needs to be unique or +even required. Perhaps it's only used as a screen name and not as a +way of identifying users. + +.. code-block:: bash + + $ edgedb migration create + did you create object type 'default::User'? [y,n,l,c,b,s,q,?] + > y + Created ./dbschema/migrations/00001.edgeql, id: + m14gwyorqqipfg7riexvbdq5dhgv7x6buqw2jaaulilcmywinmakzq + $ edgedb migrate + Applied m14gwyorqqipfg7riexvbdq5dhgv7x6buqw2jaaulilcmywinmakzq + (00001.edgeql) + +We've got our first migration to set up the schema. Now after using +that for a little while we realize that we want to make ``name`` a +*required property*. So we make the following change in the schema +file: + +.. code-block:: sdl + :version-lt: 3.0 + + type User { + required property name -> str; + } + +.. code-block:: sdl + + type User { + required name: str; + } + +Next we try to migrate: + +.. code-block:: bash + + $ edgedb migration create + did you make property 'name' of object type 'default::User' required? + [y,n,l,c,b,s,q,?] + > y + Please specify an expression to populate existing objects in order to make + property 'name' of object type 'default::User' required: + fill_expr> 'change me' + +Oh! That's right, we can't just make ``name`` *required* because there +could be existing ``User`` objects without a ``name`` at all. So we +need to provide some kind of placeholder value for those cases. We +type ``'change me'`` (although any other string would do, too). This is +different from specifying a ``default`` value since it will be applied +to *existing* objects, whereas the ``default`` applies to *new ones*. + +Unseen to us (unless we take a look at the automatically generated +``.edgeql`` files inside our ``/dbschema`` folder), EdgeDB has created +a migration script that includes the following command to make our +schema change happen. + +.. code-block:: edgeql + + ALTER TYPE default::User { + ALTER PROPERTY name { + SET REQUIRED USING ('change me'); + }; + }; + +We then run :ref:`ref_cli_edgedb_migrate` to apply the changes. + +Next we realize that we actually want to make names unique, perhaps to +avoid confusion or to use them as reliable human-readable identifiers +(unlike ``id``). We update the schema again: + +.. code-block:: sdl + :version-lt: 3.0 + + type User { + required property name -> str { + constraint exclusive; + } + } + +.. code-block:: sdl + + type User { + required name: str { + constraint exclusive; + } + } + +Now we proceed with the migration: + +.. code-block:: bash + + $ edgedb migration create + did you create constraint 'std::exclusive' of property 'name'? + [y,n,l,c,b,s,q,?] + > y + Created ./dbschema/migrations/00003.edgeql, id: + m1dxs3xbk4f3vhmqh6mjzetojafddtwlphp5a3kfbfuyvupjafevya + $ edgedb migrate + edgedb error: ConstraintViolationError: name violates exclusivity + constraint + +Some objects must have the same ``name``, so the migration can't be +applied. We have a couple of options for fixing this: + +1) Review the existing data and manually :eql:stmt:`update` the + entries with duplicate names so that they are unique. +2) Edit the migration to add an :eql:stmt:`update` which will + de-duplicate ``name`` for any potential existing ``User`` objects. + +The first option is good for situations where we want to signal to any +other maintainer of a copy of this project that they need to make a +decision about handling name duplicates in whatever way is appropriate +to them without making an implicit decision once and for all. + +Here we will go with the second option, which is good for situations +where we know enough about the situation that we can make a decision +now and never have to duplicate this effort for any other potential +copies of our project. + +We edit the last migration file ``00003.edgeql``: + +.. code-block:: edgeql-diff + + CREATE MIGRATION m1dxs3xbk4f3vhmqh6mjzetojafddtwlphp5a3kfbfuyvupjafevya + ONTO m1ndhbxx7yudb2dv7zpypl2su2oygyjlggk3olryb5uszofrfml4uq + { + + with U := default::User + + update default::User + + filter U.name = .name and U != default::User + + set { + + # De-duplicate names by appending a random uuid. + + name := .name ++ '_' ++ uuid_generate_v1mc() + + }; + + + ALTER TYPE default::User { + ALTER PROPERTY name { + CREATE CONSTRAINT std::exclusive; + }; + }; + }; + +And then we apply the migration: + +.. code-block:: bash + + $ edgedb migrate + edgedb error: could not read migrations in ./dbschema/migrations: could not + read migration file ./dbschema/migrations/00003.edgeql: migration name + should be `m1t6slgcfne35vir2lcgnqkmaxsxylzvn2hanr6mijbj5esefsp7za` but ` + m1dxs3xbk4f3vhmqh6mjzetojafddtwlphp5a3kfbfuyvupjafevya` is used instead. + Migration names are computed from the hash of the migration contents. To + proceed you must fix the statement to read as: + CREATE MIGRATION m1t6slgcfne35vir2lcgnqkmaxsxylzvn2hanr6mijbj5esefsp7za + ONTO ... + if this migration is not applied to any database. Alternatively, revert the + changes to the file. + +The migration tool detected that we've altered the file and asks us to +update the migration name (acting as a checksum) if this was +deliberate. This is done as a precaution against accidental changes. +Since we've done this on purpose, we can update the file and run +:ref:`ref_cli_edgedb_migrate` again. + +Finally, we evolved our schema all the way from having an optional +property ``name`` all the way to making it both *required* and +*exclusive*. We've worked with the EdgeDB :ref:`migration tools +` to iron out the kinks throughout the +migration process. At this point we take a quick look at the way +duplicate ``User`` objects were resolved to decide whether we need to +do anything more. We can use :eql:func:`re_test` to find names that +look like they are ending in a UUID: + +.. code-block:: edgeql-repl + + db> select User { name } + ... filter + ... re_test('.* [a-z0-9]{8}(-[a-z0-9]{4}){3}-[a-z0-9]{12}$', .name); + { + default::User {name: 'change me bc30d45a-2bcf-11ec-a6c2-6ff21f33a302'}, + default::User {name: 'change me bc30d8a6-2bcf-11ec-a6c2-4f739d559598'}, + } + +Looks like the only duplicates are the users that had no names +originally and that never updated the ``'change me'`` placeholders, so +we can probably let them be for now. In hindsight, it may have been a +good idea to use UUID-based names to populate the empty properties +from the very beginning. + +Changing a property to a link +----------------------------- + +This example shows how to change a property into a link. We'll use a +character in an adventure game as the type of data we will evolve. + +Let's start with this schema: + +.. code-block:: sdl + :version-lt: 3.0 + + scalar type CharacterClass extending enum; + + type Character { + required property name -> str; + required property class -> CharacterClass; + } + +.. code-block:: sdl + + scalar type CharacterClass extending enum; + + type Character { + required name: str; + required class: CharacterClass; + } + +We edit the schema file and perform our first migration: + +.. code-block:: bash + + $ edgedb migration create + did you create scalar type 'default::CharacterClass'? [y,n,l,c,b,s,q,?] + > y + did you create object type 'default::Character'? [y,n,l,c,b,s,q,?] + > y + Created ./dbschema/migrations/00001.edgeql, id: + m1fg76t7fbvguwhkmzrx7jwki6jxr6dvkswzeepd5v66oxg27ymkcq + $ edgedb migrate + Applied m1fg76t7fbvguwhkmzrx7jwki6jxr6dvkswzeepd5v66oxg27ymkcq + (00001.edgeql) + +The initial setup may look something like this: + +.. code-block:: edgeql-repl + + db> select Character {name, class}; + { + default::Character {name: 'Alice', class: warrior}, + default::Character {name: 'Billie', class: scholar}, + default::Character {name: 'Cameron', class: rogue}, + } + +After some development work we decide to add more details about the +available classes and encapsulate that information into its own type. +This way instead of a property ``class`` we want to end up with a link +``class`` to the new data structure. Since we cannot just +:eql:op:`cast ` a scalar into an object, we'll need to convert +between the two explicitly. This means that we will need to have both +the old and the new "class" information to begin with: + +.. code-block:: sdl + :version-lt: 3.0 + + scalar type CharacterClass extending enum; + + type NewClass { + required property name -> str; + multi property skills -> str; + } + + type Character { + required property name -> str; + required property class -> CharacterClass; + link new_class -> NewClass; + } + +.. code-block:: sdl + + scalar type CharacterClass extending enum; + + type NewClass { + required name: str; + multi skills: str; + } + + type Character { + required name: str; + required class: CharacterClass; + new_class: NewClass; + } + +We update the schema file and migrate to the new state: + +.. code-block:: bash + + $ edgedb migration create + did you create object type 'default::NewClass'? [y,n,l,c,b,s,q,?] + > y + did you create link 'new_class' of object type 'default::Character'? + [y,n,l,c,b,s,q,?] + > y + Created ./dbschema/migrations/00002.edgeql, id: + m1uttd6f7fpiwiwikhdh6qyijb6pcji747ccg2cyt5357i3wsj3l3q + $ edgedb migrate + Applied m1uttd6f7fpiwiwikhdh6qyijb6pcji747ccg2cyt5357i3wsj3l3q + (00002.edgeql) + +It makes sense to add a data migration as a way of consistently +creating ``NewClass`` objects as well as populating ``new_class`` +links based on the existing ``class`` property. So we first create an +empty migration: + +.. code-block:: bash + + $ edgedb migration create --allow-empty + Created ./dbschema/migrations/00003.edgeql, id: + m1iztxroh3ifoeqmvxncy77whnaei6tp5j3sewyxtrfysronjkxgga + +And then edit the ``00003.edgeql`` file to create and update objects: + +.. code-block:: edgeql-diff + + CREATE MIGRATION m1iztxroh3ifoeqmvxncy77whnaei6tp5j3sewyxtrfysronjkxgga + ONTO m1uttd6f7fpiwiwikhdh6qyijb6pcji747ccg2cyt5357i3wsj3l3q + { + + insert default::NewClass { + + name := 'Warrior', + + skills := {'punch', 'kick', 'run', 'jump'}, + + }; + + insert default::NewClass { + + name := 'Scholar', + + skills := {'read', 'write', 'analyze', 'refine'}, + + }; + + insert default::NewClass { + + name := 'Rogue', + + skills := {'impress', 'sing', 'steal', 'run', 'jump'}, + + }; + + + + update default::Character + + set { + + new_class := assert_single(( + + select default::NewClass + + filter .name ilike default::Character.class + + )), + + }; + }; + +Trying to apply the data migration will produce the following +reminder: + +.. code-block:: bash + + $ edgedb migrate + edgedb error: could not read migrations in ./dbschema/migrations: + could not read migration file ./dbschema/migrations/00003.edgeql: + migration name should be + `m1e3d3eg3j2pr7acie4n5rrhaddyhkiy5kgckd5l7h5ysrpmgwxl5a` but + `m1iztxroh3ifoeqmvxncy77whnaei6tp5j3sewyxtrfysronjkxgga` is used + instead. + Migration names are computed from the hash of the migration + contents. To proceed you must fix the statement to read as: + CREATE MIGRATION m1e3d3eg3j2pr7acie4n5rrhaddyhkiy5kgckd5l7h5ysrpmgwxl5a + ONTO ... + if this migration is not applied to any database. Alternatively, + revert the changes to the file. + +The migration tool detected that we've altered the file and asks us to +update the migration name (acting as a checksum) if this was +deliberate. This is done as a precaution against accidental changes. +Since we've done this on purpose, we can update the file and run +:ref:`ref_cli_edgedb_migrate` again. + +We can see the changes after the data migration is complete: + +.. code-block:: edgeql-repl + + db> select Character { + ... name, + ... class, + ... new_class: { + ... name, + ... } + ... }; + { + default::Character { + name: 'Alice', + class: warrior, + new_class: default::NewClass {name: 'Warrior'}, + }, + default::Character { + name: 'Billie', + class: scholar, + new_class: default::NewClass {name: 'Scholar'}, + }, + default::Character { + name: 'Cameron', + class: rogue, + new_class: default::NewClass {name: 'Rogue'}, + }, + } + +Everything seems to be in order. It is time to clean up the old +property and ``CharacterClass`` :eql:type:`enum`: + +.. code-block:: sdl + :version-lt: 3.0 + + type NewClass { + required property name -> str; + multi property skills -> str; + } + + type Character { + required property name -> str; + link new_class -> NewClass; + } + +.. code-block:: sdl + + type NewClass { + required name: str; + multi skills: str; + } + + type Character { + required name: str; + new_class: NewClass; + } + +The migration tools should have no trouble detecting the things we +just removed: + +.. code-block:: bash + + $ edgedb migration create + did you drop property 'class' of object type 'default::Character'? + [y,n,l,c,b,s,q,?] + > y + did you drop scalar type 'default::CharacterClass'? [y,n,l,c,b,s,q,?] + > y + Created ./dbschema/migrations/00004.edgeql, id: + m1jdnz5bxjj6kjz2pylvudli5rvw4jyr2ilpb4hit3yutwi3bq34ha + $ edgedb migrate + Applied m1jdnz5bxjj6kjz2pylvudli5rvw4jyr2ilpb4hit3yutwi3bq34ha + (00004.edgeql) + +Now that the original property and scalar type are gone, we can rename +the "new" components, so that they become ``class`` link and +``CharacterClass`` type, respectively: + +.. code-block:: sdl + :version-lt: 3.0 + + type CharacterClass { + required property name -> str; + multi property skills -> str; + } + + type Character { + required property name -> str; + link class -> CharacterClass; + } + +.. code-block:: sdl + + type CharacterClass { + required name: str; + multi skills: str; + } + + type Character { + required name: str; + class: CharacterClass; + } + +The migration tools pick up the changes without any issues again. It +may seem tempting to combine the last two steps, but deleting and +renaming in a single step would cause the migration tools to report a +name clash. As a general rule, it is a good idea to never mix renaming +and deleting of closely interacting entities in the same migration. + +.. code-block:: bash + + $ edgedb migration create + did you rename object type 'default::NewClass' to + 'default::CharacterClass'? [y,n,l,c,b,s,q,?] + > y + did you rename link 'new_class' of object type 'default::Character' to + 'class'? [y,n,l,c,b,s,q,?] + > y + Created ./dbschema/migrations/00005.edgeql, id: + m1ra4fhx2erkygbhi7qjxt27yup5aw5hkr5bekn5y5jeam5yn57vsa + $ edgedb migrate + Applied m1ra4fhx2erkygbhi7qjxt27yup5aw5hkr5bekn5y5jeam5yn57vsa + (00005.edgeql) + +Finally, we have replaced the original ``class`` property with a link: + +.. code-block:: edgeql-repl + + db> select Character { + ... name, + ... class: { + ... name, + ... skills, + ... } + ... }; + { + default::Character { + name: 'Alice', + class: default::CharacterClass { + name: 'Warrior', + skills: {'punch', 'kick', 'run', 'jump'}, + }, + }, + default::Character { + name: 'Billie', + class: default::CharacterClass { + name: 'Scholar', + skills: {'read', 'write', 'analyze', 'refine'}, + }, + }, + default::Character { + name: 'Cameron', + class: default::CharacterClass { + name: 'Rogue', + skills: {'impress', 'sing', 'steal', 'run', 'jump'}, + }, + }, + } + +Changing the type of a property +------------------------------- + +This example shows how to change the type of a property. We'll use a +character in an adventure game as the type of data we will evolve. + +Let's start with this schema: + +.. code-block:: sdl + :version-lt: 3.0 + + type Character { + required property name -> str; + required property description -> str; + } + +.. code-block:: sdl + + type Character { + required name: str; + required description: str; + } + +We edit the schema file and perform our first migration: + +.. code-block:: bash + + $ edgedb migration create + did you create object type 'default::Character'? [y,n,l,c,b,s,q,?] + > y + Created ./dbschema/migrations/00001.edgeql, id: + m1paw3ogpsdtxaoywd6pl6beg2g64zj4ykhd43zby4eqh64yjad47a + $ edgedb migrate + Applied m1paw3ogpsdtxaoywd6pl6beg2g64zj4ykhd43zby4eqh64yjad47a + (00001.edgeql) + +The intent is for the ``description`` to provide some text which +serves both as something to be shown to the player as well as +determining some game actions. Se we end up with something like this: + +.. code-block:: edgeql-repl + + db> select Character {name, description}; + { + default::Character {name: 'Alice', description: 'Tall and strong'}, + default::Character {name: 'Billie', description: 'Smart and aloof'}, + default::Character {name: 'Cameron', description: 'Dashing and smooth'}, + } + +However, as we keep developing our game it becomes apparent that this +is less of a "description" and more of a "character class", so at +first we just rename the property to reflect that: + +.. code-block:: sdl + :version-lt: 3.0 + + type Character { + required property name -> str; + required property class -> str; + } + +.. code-block:: sdl + + type Character { + required name: str; + required class: str; + } + +The migration gives us this: + +.. code-block:: bash + + $ edgedb migration create + did you rename property 'description' of object type 'default::Character' + to 'class'? [y,n,l,c,b,s,q,?] + > y + Created ./dbschema/migrations/00002.edgeql, id: + m1ljrgrofsqkvo5hsxc62mnztdhlerxp6ucdto262se6dinhuj4mqq + $ edgedb migrate + Applied m1ljrgrofsqkvo5hsxc62mnztdhlerxp6ucdto262se6dinhuj4mqq + (00002.edgeql) + +EdgeDB detected that the change looked like a property was being +renamed, which we confirmed. Since this was an existing property being +renamed, the data is all preserved: + +.. code-block:: edgeql-repl + + db> select Character {name, class}; + { + default::Character {name: 'Alice', class: 'Tall and strong'}, + default::Character {name: 'Billie', class: 'Smart and aloof'}, + default::Character {name: 'Cameron', class: 'Dashing and smooth'}, + } + +The contents of the ``class`` property are a bit too verbose, so we +decide to update them. In order for this update to be consistently +applied across several developers, we will make it in the form of a +*data migration*: + +.. code-block:: bash + + $ edgedb migration create --allow-empty + Created ./dbschema/migrations/00003.edgeql, id: + m1qv2pdksjxxzlnujfed4b6to2ppuodj3xqax4p3r75yfef7kd7jna + +Now we can edit the file ``00003.edgeql`` directly: + +.. code-block:: edgeql-diff + + CREATE MIGRATION m1qv2pdksjxxzlnujfed4b6to2ppuodj3xqax4p3r75yfef7kd7jna + ONTO m1ljrgrofsqkvo5hsxc62mnztdhlerxp6ucdto262se6dinhuj4mqq + { + + update default::Character + + set { + + class := + + 'warrior' if .class = 'Tall and strong' else + + 'scholar' if .class = 'Smart and aloof' else + + 'rogue' + + }; + }; + +We're ready to apply the migration: + +.. code-block:: bash + + $ edgedb migrate + edgedb error: could not read migrations in ./dbschema/migrations: + could not read migration file ./dbschema/migrations/00003.edgeql: + migration name should be + `m1ryafvp24g5eqjeu65zr4bqf6m3qath3lckfdhoecfncmr7zshehq` + but `m1qv2pdksjxxzlnujfed4b6to2ppuodj3xqax4p3r75yfef7kd7jna` is used + instead. + Migration names are computed from the hash of the migration + contents. To proceed you must fix the statement to read as: + CREATE MIGRATION m1ryafvp24g5eqjeu65zr4bqf6m3qath3lckfdhoecfncmr7zshehq + ONTO ... + if this migration is not applied to any database. Alternatively, + revert the changes to the file. + +The migration tool detected that we've altered the file and asks us to +update the migration name (acting as a checksum) if this was +deliberate. This is done as a precaution against accidental changes. +Since we've done this on purpose, we can update the file and run +:ref:`ref_cli_edgedb_migrate` again. + +As the game becomes more stable there's no reason for the ``class`` to +be a :eql:type:`str` anymore, instead we can use an :eql:type:`enum` +to make sure that we don't accidentally use some invalid value for it. + +.. code-block:: sdl + :version-lt: 3.0 + + scalar type CharacterClass extending enum; + + type Character { + required property name -> str; + required property class -> CharacterClass; + } + +.. code-block:: sdl + + scalar type CharacterClass extending enum; + + type Character { + required name: str; + required class: CharacterClass; + } + +Fortunately, we've already updated the ``class`` strings to match the +:eql:type:`enum` values, so that a simple cast will convert all the +values. If we had not done this earlier we would need to do it now in +order for the type change to work. + +.. code-block:: bash + + $ edgedb migration create + did you create scalar type 'default::CharacterClass'? [y,n,l,c,b,s,q,?] + > y + did you alter the type of property 'class' of object type + 'default::Character'? [y,n,l,c,b,s,q,?] + > y + Created ./dbschema/migrations/00004.edgeql, id: + m1hc4yynkejef2hh7fvymvg3f26nmynpffksg7yvfksqufif6lulgq + $ edgedb migrate + Applied m1hc4yynkejef2hh7fvymvg3f26nmynpffksg7yvfksqufif6lulgq + (00004.edgeql) + +The final migration converted all the ``class`` property values: + +.. code-block:: edgeql-repl + + db> select Character {name, class}; + { + default::Character {name: 'Alice', class: warrior}, + default::Character {name: 'Billie', class: scholar}, + default::Character {name: 'Cameron', class: rogue}, + } + +Adding a required link +---------------------- + +This example shows how to setup a required link. We'll use a +character in an adventure game as the type of data we will evolve. + +Let's start with this schema: + +.. code-block:: sdl + :version-lt: 3.0 + + type Character { + required property name -> str; + } + +.. code-block:: sdl + + type Character { + required name: str; + } + +We edit the schema file and perform our first migration: + +.. code-block:: bash + + $ edgedb migration create + did you create object type 'default::Character'? [y,n,l,c,b,s,q,?] + > y + Created ./dbschema/migrations/00001.edgeql, id: + m1xvu7o4z5f5xfwuun2vee2cryvvzh5lfilwgkulmqpifo5m3dnd6a + $ edgedb migrate + Applied m1xvu7o4z5f5xfwuun2vee2cryvvzh5lfilwgkulmqpifo5m3dnd6a + (00001.edgeql) + +This time around let's practice performing a data migration and set up +our character data. For this purpose we can create an empty migration +and fill it out as we like: + +.. code-block:: bash + + $ edgedb migration create --allow-empty + Created ./dbschema/migrations/00002.edgeql, id: + m1lclvwdpwitjj4xqm45wp74y4wjyadljct5o6bsctlnh5xbto74iq + +We edit the ``00002.edgeql`` file by simply adding the query to add +characters to it. We can use :eql:stmt:`for` to add multiple characters +like this: + +.. code-block:: edgeql-diff + + CREATE MIGRATION m1lclvwdpwitjj4xqm45wp74y4wjyadljct5o6bsctlnh5xbto74iq + ONTO m1xvu7o4z5f5xfwuun2vee2cryvvzh5lfilwgkulmqpifo5m3dnd6a + { + + for name in {'Alice', 'Billie', 'Cameron', 'Dana'} + + union ( + + insert default::Character { + + name := name + + } + + ); + }; + +Trying to apply the data migration will produce the following +reminder: + +.. code-block:: bash + + $ edgedb migrate + edgedb error: could not read migrations in ./dbschema/migrations: + could not read migration file ./dbschema/migrations/00002.edgeql: + migration name should be + `m1juin65wriqmb4vwg23fiyajjxlzj2jyjv5qp36uxenit5y63g2iq` but + `m1lclvwdpwitjj4xqm45wp74y4wjyadljct5o6bsctlnh5xbto74iq` is used instead. + Migration names are computed from the hash of the migration contents. To + proceed you must fix the statement to read as: + CREATE MIGRATION m1juin65wriqmb4vwg23fiyajjxlzj2jyjv5qp36uxenit5y63g2iq + ONTO ... + if this migration is not applied to any database. Alternatively, + revert the changes to the file. + +The migration tool detected that we've altered the file and asks us to +update the migration name (acting as a checksum) if this was +deliberate. This is done as a precaution against accidental changes. +Since we've done this on purpose, we can update the file and run +:ref:`ref_cli_edgedb_migrate` again. + +.. code-block:: edgeql-diff + + - CREATE MIGRATION m1lclvwdpwitjj4xqm45wp74y4wjyadljct5o6bsctlnh5xbto74iq + + CREATE MIGRATION m1juin65wriqmb4vwg23fiyajjxlzj2jyjv5qp36uxenit5y63g2iq + ONTO m1xvu7o4z5f5xfwuun2vee2cryvvzh5lfilwgkulmqpifo5m3dnd6a + { + # ... + }; + +After we apply the data migration we should be able to see the added +characters: + +.. code-block:: edgeql-repl + + db> select Character {name}; + { + default::Character {name: 'Alice'}, + default::Character {name: 'Billie'}, + default::Character {name: 'Cameron'}, + default::Character {name: 'Dana'}, + } + +Let's add a character ``class`` represented by a new type to our +schema and data. Unlike in the scenario when changing a property +to a link, we will add the ``required link class`` right away, +without any intermediate properties. So we end up with a schema +like this: + +.. code-block:: sdl + :version-lt: 3.0 + + type CharacterClass { + required property name -> str; + multi property skills -> str; + } + + type Character { + required property name -> str; + required link class -> CharacterClass; + } + +.. code-block:: sdl + + type CharacterClass { + required name: str; + multi skills: str; + } + + type Character { + required name: str; + required class: CharacterClass; + } + +We go ahead and try to apply this new schema: + +.. code-block:: bash + + $ edgedb migration create + did you create object type 'default::CharacterClass'? [y,n,l,c,b,s,q,?] + > y + did you create link 'class' of object type 'default::Character'? + [y,n,l,c,b,s,q,?] + > y + Please specify an expression to populate existing objects in order to make + link 'class' of object type 'default::Character' required: + fill_expr> + +Uh-oh! Unlike in a situation with a required property, it's not a good +idea to just :eql:stmt:`insert` a new ``CharacterClass`` object for +every character. So we should abort this migration attempt and rethink +our strategy. We need a separate step where the ``class`` link is +not *required* so that we can write some custom queries to handle +the character classes: + +.. code-block:: sdl + :version-lt: 3.0 + + type CharacterClass { + required property name -> str; + multi property skills -> str; + } + + type Character { + required property name -> str; + link class -> CharacterClass; + } + +.. code-block:: sdl + + type CharacterClass { + required name: str; + multi skills: str; + } + + type Character { + required name: str; + class: CharacterClass; + } + +We can now create a migration for our new schema, but we won't apply +it right away: + +.. code-block:: bash + + $ edgedb migration create + did you create object type 'default::CharacterClass'? [y,n,l,c,b,s,q,?] + > y + did you create link 'class' of object type 'default::Character'? + [y,n,l,c,b,s,q,?] + > y + Created ./dbschema/migrations/00003.edgeql, id: + m1jie3xamsm2b7ygqccwfh2degdi45oc7mwuyzjkanh2qwgiqvi2ya + +We don't need to create a blank migration to add data, we can add our +modifications into the migration that adds the ``class`` link +directly. Doing this makes sense when the schema changes seem to +require the data migration and the two types of changes logically go +together. We will need to create some ``CharacterClass`` objects as +well as :eql:stmt:`update` the ``class`` link on existing +``Character`` objects: + +.. code-block:: edgeql-diff + + CREATE MIGRATION m1jie3xamsm2b7ygqccwfh2degdi45oc7mwuyzjkanh2qwgiqvi2ya + ONTO m1juin65wriqmb4vwg23fiyajjxlzj2jyjv5qp36uxenit5y63g2iq + { + CREATE TYPE default::CharacterClass { + CREATE REQUIRED PROPERTY name -> std::str; + CREATE MULTI PROPERTY skills -> std::str; + }; + ALTER TYPE default::Character { + CREATE LINK class -> default::CharacterClass; + }; + + + insert default::CharacterClass { + + name := 'Warrior', + + skills := {'punch', 'kick', 'run', 'jump'}, + + }; + + insert default::CharacterClass { + + name := 'Scholar', + + skills := {'read', 'write', 'analyze', 'refine'}, + + }; + + insert default::CharacterClass { + + name := 'Rogue', + + skills := {'impress', 'sing', 'steal', 'run', 'jump'}, + + }; + + # All warriors + + update default::Character + + filter .name in {'Alice'} + + set { + + class := assert_single(( + + select default::CharacterClass + + filter .name = 'Warrior' + + )), + + }; + + # All scholars + + update default::Character + + filter .name in {'Billie'} + + set { + + class := assert_single(( + + select default::CharacterClass + + filter .name = 'Scholar' + + )), + + }; + + # All rogues + + update default::Character + + filter .name in {'Cameron', 'Dana'} + + set { + + class := assert_single(( + + select default::CharacterClass + + filter .name = 'Rogue' + + )), + + }; + }; + +In a real game we might have a lot more characters and so a good way +to update them all is to update characters of the same class in bulk. + +Just like before we'll be reminded to fix the migration name since +we've altered the migration file. After fixing the migration hash we +can apply it. Now all our characters should have been assigned their +classes: + +.. code-block:: edgeql-repl + + db> select Character { + ... name, + ... class: { + ... name + ... } + ... }; + { + default::Character { + name: 'Alice', + class: default::CharacterClass {name: 'Warrior'}, + }, + default::Character { + name: 'Billie', + class: default::CharacterClass {name: 'Scholar'}, + }, + default::Character { + name: 'Cameron', + class: default::CharacterClass {name: 'Rogue'}, + }, + default::Character { + name: 'Dana', + class: default::CharacterClass {name: 'Rogue'}, + }, + } + +We're finally ready to make the ``class`` link *required*. We update +the schema: + +.. code-block:: sdl + :version-lt: 3.0 + + type CharacterClass { + required property name -> str; + multi property skills -> str; + } + + type Character { + required property name -> str; + required link class -> CharacterClass; + } + +.. code-block:: sdl + + type CharacterClass { + required name: str; + multi skills: str; + } + + type Character { + required name: str; + required class: CharacterClass; + } + +And we perform our final migration: + +.. code-block:: bash + + $ edgedb migration create + did you make link 'class' of object type 'default::Character' required? + [y,n,l,c,b,s,q,?] + > y + Please specify an expression to populate existing objects in order to + make link 'class' of object type 'default::Character' required: + fill_expr> assert_exists(.class) + Created ./dbschema/migrations/00004.edgeql, id: + m14yblybdo77c7bjtm6nugiy5cs6pl6rnuzo5b27gamy4zhuwjifia + +The migration system doesn't know that we've already assigned ``class`` values +to all the ``Character`` objects, so it still asks us for an expression to be +used in case any of the objects need it. We can use ``assert_exists(.class)`` +here as a way of being explicit about the fact that we expect the values to +already be present. Missing values would have caused an error even without the +``assert_exists`` wrapper, but being explicit may help us capture the intent +and make debugging a little easier if anyone runs into a problem at this step. + +In fact, before applying this migration, let's actually add a new +``Character`` to see what happens: + +.. code-block:: edgeql-repl + + db> insert Character {name := 'Eric'}; + { + default::Character { + id: 9f4ac7a8-ac38-11ec-b076-afefd12d7e66, + }, + } + +Our attempt at migrating fails as we expected: + +.. code-block:: bash + + $ edgedb migrate + edgedb error: MissingRequiredError: missing value for required link + 'class' of object type 'default::Character' + Detail: Failing object id is 'ee604992-c1b1-11ec-ad59-4f878963769f'. + +After removing the bugged ``Character``, we can migrate without any problems: + +.. code-block:: bash + + $ edgedb migrate + Applied m14yblybdo77c7bjtm6nugiy5cs6pl6rnuzo5b27gamy4zhuwjifia + (00004.edgeql) + +Recovering lost migrations +-------------------------- + +Each time you create a migration with :ref:`ref_cli_edgedb_migration_create`, +a file containing the DDL for that migration is created in +``dbschema/migrations``. When you apply a migration with +:ref:`ref_cli_edgedb_migration_apply` or :ref:`ref_cli_edgedb_migrate`, the +database stores a record of the migration it applied. + +On rare occasions, you may find you have deleted your migration files by +mistake. If you don't care about any of your data and don't need to keep your +migration history, you can :ref:`wipe ` your +database and start over, creating a single migration to the current state of +your schema. If that's not an option, all hope is not lost. You can instead +recover your migrations from the database. + +Run this query to see your migrations: + +.. code-block:: edgeql + + select schema::Migration { + name, + script, + parents: {name} + } + +You can rebuild your migrations from the results of this query, either manually +or via a script if you've applied too many of them to recreate by hand. +Migrations in the file system are named sequentially starting from +``00001.edgeql``. They are in this format: + +.. code-block:: edgeql + + CREATE MIGRATION m1rsm66e5pvh5ets2yznutintmqnxluzvgbocspi6umd3ht64e4naq + # ☝️ Replace with migration name + ONTO m1l5esbbycsyqcnx6udxx24riavvyvkskchtekwe7jqx5mmiyli54a + # ☝️ Replace with parent migration name + { + # script + # ☝️ Replace with migration script + }; + +or if this is the first migration: + +.. code-block:: edgeql + + CREATE MIGRATION m1l5esbbycsyqcnx6udxx24riavvyvkskchtekwe7jqx5mmiyli54a + # ☝️ Replace with migration name + ONTO initial + { + # script + # ☝️ Replace with migration script + }; + +Replace the name, script, and parent name with the values from +your ``Migration`` query results. + +You can identify the first migration in your query results as the one with no +object linked on ``parents``. Order the other migrations by chaining the links. +The ``Migration`` with the initial migration linked via ``parents`` is the +second migration — ``00002.edgeql``. The migration linking to the second +migration via ``parents`` is the third migration, and so on). + +Getting the current migration +----------------------------- + +The following query will return the most current migration: + +.. code-block:: edgeql-repl + + db> with + ... module schema, + ... lastMigration := ( + ... select Migration filter not exists .` +- :ref:`Migration tips ` -- :ref:`Making a property required ` -- :ref:`Adding backlinks ` -- :ref:`Changing the type of a property ` -- :ref:`Changing a property to a link ` -- :ref:`Adding a required link ` - -For more information on how migrations work in EdgeDB, check out the :ref:`CLI +Further information can be found in the :ref:`CLI reference ` or the `Beta 1 blog post `_, which -describes the design of the migration system. +describes the design of the migration system. \ No newline at end of file