diff --git a/edb/pgsql/metaschema.py b/edb/pgsql/metaschema.py index 82ea8e9654e..a6719ff4095 100644 --- a/edb/pgsql/metaschema.py +++ b/edb/pgsql/metaschema.py @@ -5887,12 +5887,32 @@ def _generate_sql_information_schema() -> List[dbops.Command]: -- fallback to pointer name, with suffix '_id' for links sp.name || case when sl.id is not null then '_id' else '' end ) AS v_column_name, - COALESCE(spec.position, 2) as position, - isc.* - FROM information_schema.columns isc - JOIN edgedbsql_VER.virtual_tables vt ON vt.id::text = isc.table_name + COALESCE(spec.position, 2) AS position, + (sp.expr IS NOT NULL) AS is_computed, + isc.column_default, + CASE WHEN sp.required OR spec.k IS NOT NULL + THEN 'NO' ELSE 'YES' END AS is_nullable, + + -- HACK: computeds don't have backing rows in isc, + -- so we just default to 'text'. This is wrong. + COALESCE(isc.data_type, 'text') AS data_type + FROM edgedb_VER."_SchemaPointer" sp + LEFT JOIN information_schema.columns isc ON ( + isc.table_name = sp.source::TEXT AND CASE + WHEN length(isc.column_name) = 36 -- if column name is uuid + THEN isc.column_name = sp.id::text -- compare uuids + ELSE isc.column_name = sp.name -- for id, source, target + END + ) + + -- needed for attaching `_id` + LEFT JOIN edgedb_VER."_SchemaLink" sl ON sl.id = sp.id + + -- needed for determining table name + JOIN edgedbsql_VER.virtual_tables vt ON vt.id = sp.source - -- id is duplicated to get id and __type__ columns out of it + -- positions for special pointers + -- duplicate id get both id and __type__ columns out of it LEFT JOIN ( VALUES ('id', 'id', 0), ('id', '__type__', 1), @@ -5900,11 +5920,43 @@ def _generate_sql_information_schema() -> List[dbops.Command]: ('target', 'target', 1) ) spec(k, name, position) ON (spec.k = isc.column_name) - LEFT JOIN edgedb_VER."_SchemaPointer" sp - ON sp.id::text = isc.column_name - LEFT JOIN edgedb_VER."_SchemaLink" sl ON sl.id::text = isc.column_name + WHERE isc.column_name IS NOT NULL -- normal pointers + OR sp.expr IS NOT NULL AND sp.cardinality <> 'Many' -- computeds + + UNION ALL + + -- special case: multi properties source and target + -- (this is needed, because schema does not create pointers for + -- these two columns) + SELECT + vt.schema_name AS vt_table_schema, + vt.table_name AS vt_table_name, + isc.column_name AS v_column_name, + spec.position as position, + FALSE as is_computed, + isc.column_default, + 'NO' as is_nullable, + isc.data_type as data_type + FROM edgedb_VER."_SchemaPointer" sp + JOIN information_schema.columns isc ON isc.table_name = sp.id::TEXT + + -- needed for filtering out links + LEFT JOIN edgedb_VER."_SchemaLink" sl ON sl.id = sp.id + + -- needed for determining table name + JOIN edgedbsql_VER.virtual_tables vt ON vt.id = sp.id + + -- positions for special pointers + JOIN ( + VALUES ('source', 'source', 0), + ('target', 'target', 1) + ) spec(k, name, position) ON (spec.k = isc.column_name) + + WHERE + sl.id IS NULL -- property (non-link) + AND sp.cardinality = 'Many' -- multi + AND sp.expr IS NULL -- non-computed ) t - WHERE v_column_name IS NOT NULL ''' ), ), @@ -6182,32 +6234,32 @@ def _generate_sql_information_schema() -> List[dbops.Command]: UNION ALL - SELECT attrelid, + SELECT pc_oid as attrelid, col_name as attname, - atttypid, - attstattarget, - attlen, + COALESCE(atttypid, 25) as atttypid, -- defaults to TEXT + COALESCE(attstattarget, -1) as attstattarget, + COALESCE(attlen, -1) as attlen, (ROW_NUMBER() OVER ( - PARTITION BY attrelid + PARTITION BY pc_oid ORDER BY col_position, col_name ) - 6)::smallint AS attnum, t.attnum as attnum_internal, - attndims, - attcacheoff, - atttypmod, - attbyval, - attstorage, - attalign, - attnotnull, + COALESCE(attndims, 0) as attndims, + COALESCE(attcacheoff, -1) as attcacheoff, + COALESCE(atttypmod, -1) as atttypmod, + COALESCE(attbyval, FALSE) as attbyval, + COALESCE(attstorage, 'x') as attstorage, + COALESCE(attalign, 'i') as attalign, + required as attnotnull, -- Always report no default, to avoid expr trouble false as atthasdef, - atthasmissing, - attidentity, - attgenerated, - attisdropped, - attislocal, - attinhcount, - attcollation, + COALESCE(atthasmissing, FALSE) as atthasmissing, + COALESCE(attidentity, '') as attidentity, + COALESCE(attgenerated, '') as attgenerated, + COALESCE(attisdropped, FALSE) as attisdropped, + COALESCE(attislocal, TRUE) as attislocal, + COALESCE(attinhcount, 0) as attinhcount, + COALESCE(attcollation, 0) as attcollation, attacl, attoptions, attfdwoptions, @@ -6220,17 +6272,27 @@ def _generate_sql_information_schema() -> List[dbops.Command]: sp.name || case when sl.id is not null then '_id' else '' end, pa.attname -- for system columns ) as col_name, - CASE - WHEN pa.attnum <= 0 THEN pa.attnum - ELSE COALESCE(spec.position, 2) - END as col_position, + COALESCE(spec.position, 2) AS col_position, + (sp.required IS TRUE OR spec.k IS NOT NULL) as required, + pc.oid AS pc_oid, pa.*, pa.tableoid, pa.xmin, pa.cmin, pa.xmax, pa.cmax, pa.ctid - FROM pg_attribute pa - JOIN pg_class pc ON pc.oid = pa.attrelid - JOIN edgedbsql_VER.virtual_tables vt ON vt.backend_id = pc.reltype - -- id is duplicated to get id and __type__ columns out of it + FROM edgedb_VER."_SchemaPointer" sp + JOIN edgedbsql_VER.virtual_tables vt ON vt.id = sp.source + JOIN pg_class pc ON pc.reltype = vt.backend_id + + -- try to find existing pg_attribute (it will not exist for computeds) + LEFT JOIN pg_attribute pa ON ( + pa.attrelid = pc.oid AND CASE + WHEN length(pa.attname) = 36 -- if column name is uuid + THEN pa.attname = sp.id::text -- compare uuids + ELSE pa.attname = sp.name -- for id, source, target + END + ) + + -- positions for special pointers + -- duplicate id get both id and __type__ columns out of it LEFT JOIN ( VALUES ('id', 'id', 0), ('id', '__type__', 1), @@ -6238,10 +6300,56 @@ def _generate_sql_information_schema() -> List[dbops.Command]: ('target', 'target', 1) ) spec(k, name, position) ON (spec.k = pa.attname) - LEFT JOIN edgedb_VER."_SchemaPointer" sp ON sp.id::text = pa.attname - LEFT JOIN edgedb_VER."_SchemaLink" sl ON sl.id::text = pa.attname - -- Filter out internal columns - WHERE pa.attname NOT LIKE '\_\_%\_\_' OR pa.attname = '__type__' + -- needed for attaching `_id` + LEFT JOIN edgedb_VER."_SchemaLink" sl ON sl.id = sp.id + + WHERE pa.attname IS NOT NULL -- non-computed pointers + OR sp.expr IS NOT NULL AND sp.cardinality <> 'Many' -- computeds + + UNION ALL + + -- special case: multi properties source and target + -- (this is needed, because schema does not create pointers for + -- these two columns) + SELECT + pa.attname AS col_name, + spec.position as position, + TRUE as required, + pa.attrelid as pc_oid, + pa.*, + pa.tableoid, pa.xmin, pa.cmin, pa.xmax, pa.cmax, pa.ctid + FROM edgedb_VER."_SchemaPointer" sp + JOIN pg_class pc ON pc.relname = sp.id::TEXT + JOIN pg_attribute pa ON pa.attrelid = pc.oid + + -- needed for filtering out links + LEFT JOIN edgedb_VER."_SchemaLink" sl ON sl.id = sp.id + + -- positions for special pointers + JOIN ( + VALUES ('source', 0), + ('target', 1) + ) spec(k, position) ON (spec.k = pa.attname) + + WHERE + sl.id IS NULL -- property (non-link) + AND sp.cardinality = 'Many' -- multi + AND sp.expr IS NULL -- non-computed + + UNION ALL + + -- special case: system columns + SELECT + pa.attname AS col_name, + pa.attnum as position, + TRUE as required, + pa.attrelid as pc_oid, + pa.*, + pa.tableoid, pa.xmin, pa.cmin, pa.xmax, pa.cmax, pa.ctid + FROM pg_attribute pa + JOIN pg_class pc ON pc.oid = pa.attrelid + JOIN edgedbsql_VER.virtual_tables vt ON vt.backend_id = pc.reltype + WHERE pa.attnum < 0 ) t """, ), diff --git a/edb/pgsql/resolver/command.py b/edb/pgsql/resolver/command.py index dda7408ae63..c56ddbb99c4 100644 --- a/edb/pgsql/resolver/command.py +++ b/edb/pgsql/resolver/command.py @@ -40,6 +40,8 @@ def resolve_CopyStmt(stmt: pgast.CopyStmt, *, ctx: Context) -> pgast.CopyStmt: elif stmt.relation: relation, table = dispatch.resolve_relation(stmt.relation, ctx=ctx) + table.reference_as = ctx.names.get('rel') + if stmt.colnames: col_map: Dict[str, context.Column] = { col.name: col for col in table.columns @@ -54,7 +56,12 @@ def resolve_CopyStmt(stmt: pgast.CopyStmt, *, ctx: Context) -> pgast.CopyStmt: # This is probably a view based on edgedb schema, so wrap it into # a select query. query = pgast.SelectStmt( - from_clause=[pgast.RelRangeVar(relation=relation)], + from_clause=[ + pgast.RelRangeVar( + alias=pgast.Alias(aliasname=table.reference_as), + relation=relation, + ) + ], target_list=[ pgast.ResTarget( val=pg_res_expr.resolve_column_kind(table, c.kind, ctx=ctx) diff --git a/edb/pgsql/resolver/context.py b/edb/pgsql/resolver/context.py index dd0b94fa4ed..96236741e47 100644 --- a/edb/pgsql/resolver/context.py +++ b/edb/pgsql/resolver/context.py @@ -24,8 +24,11 @@ import enum import uuid +from edb.pgsql import ast as pgast + from edb.common import compiler from edb.schema import schema as s_schema +from edb.schema import pointers as s_pointers @dataclass(frozen=True) @@ -129,6 +132,13 @@ class ColumnStaticVal(ColumnKind): val: uuid.UUID +@dataclass(kw_only=True) +class ColumnComputable(ColumnKind): + # An EdgeQL computable property. To get the AST for this column, EdgeQL + # compiler needs to be invoked. + pointer: s_pointers.Pointer + + class ContextSwitchMode(enum.Enum): EMPTY = enum.auto() CHILD = enum.auto() @@ -146,6 +156,9 @@ class ResolverContextLevel(compiler.ContextLevel): # child objects. include_inherited: bool + # List of CTEs to append to the current SELECT statement + cte_to_append: List[pgast.CommonTableExpr] + options: Options def __init__( diff --git a/edb/pgsql/resolver/expr.py b/edb/pgsql/resolver/expr.py index 65e449eebb3..7b26772ce46 100644 --- a/edb/pgsql/resolver/expr.py +++ b/edb/pgsql/resolver/expr.py @@ -1,4 +1,5 @@ # +# # This source file is part of the EdgeDB open source project. # # Copyright 2008-present MagicStack Inc. and the EdgeDB authors. @@ -26,6 +27,14 @@ from edb.pgsql import ast as pgast from edb.pgsql import trampoline +from edb.pgsql import compiler as pgcompiler + +from edb.schema import types as s_types + +from edb.ir import ast as irast + + +from edb.edgeql import compiler as qlcompiler from . import dispatch from . import context @@ -117,6 +126,39 @@ def resolve_column_kind( case context.ColumnStaticVal(val=val): # special case: __type__ static value return _uuid_const(val) + case context.ColumnComputable(pointer=pointer): + + expr = pointer.get_expr(ctx.schema) + assert expr + + source = pointer.get_source(ctx.schema) + assert isinstance(source, s_types.Type) + source_id = irast.PathId.from_type(ctx.schema, source, env=None) + + singletons = [source] + options = qlcompiler.CompilerOptions( + modaliases={None: 'default'}, + anchors={'__source__': source}, + path_prefix_anchor='__source__', + singletons=singletons, + make_globals_empty=True, # TODO: globals in SQL + ) + compiled = expr.compiled(ctx.schema, options=options) + + sql_tree = pgcompiler.compile_ir_to_sql_tree( + compiled.irast, + external_rvars={ + (source_id, 'source'): pgast.RelRangeVar( + alias=pgast.Alias( + aliasname=table.reference_as, + ), + relation=pgast.Relation(name=table.reference_as), + ), + }, + output_format=pgcompiler.OutputFormat.NATIVE_INTERNAL, + ) + assert isinstance(sql_tree.ast, pgast.BaseExpr) + return sql_tree.ast case _: raise NotImplementedError(column) @@ -174,7 +216,7 @@ def _lookup_column( assert tab.reference_as col = context.Column( name=tab.reference_as, - kind=context.ColumnByName(reference_as=tab.reference_as) + kind=context.ColumnByName(reference_as=tab.reference_as), ) return [(context.Table(), col)] except errors.QueryError: @@ -338,9 +380,13 @@ def resolve_SortBy( func_calls_remapping: Dict[Tuple[str, ...], Tuple[str, ...]] = { ('information_schema', '_pg_truetypid'): ( - trampoline.versioned_schema('edgedbsql'), '_pg_truetypid'), + trampoline.versioned_schema('edgedbsql'), + '_pg_truetypid', + ), ('information_schema', '_pg_truetypmod'): ( - trampoline.versioned_schema('edgedbsql'), '_pg_truetypmod'), + trampoline.versioned_schema('edgedbsql'), + '_pg_truetypmod', + ), ('pg_catalog', 'format_type'): ('edgedb', '_format_type'), } diff --git a/edb/pgsql/resolver/relation.py b/edb/pgsql/resolver/relation.py index 6c82f47d977..fcddb875b4c 100644 --- a/edb/pgsql/resolver/relation.py +++ b/edb/pgsql/resolver/relation.py @@ -309,8 +309,6 @@ def public_to_default(s: str) -> str: card = p.get_cardinality(ctx.schema) if card.is_multi(): continue - if p.get_computable(ctx.schema): - continue columns.append(_construct_column(p, ctx, ctx.include_inherited)) else: @@ -404,10 +402,13 @@ def _construct_column( col_name: str kind: context.ColumnKind + if isinstance(p, s_properties.Property): col_name = short_name.name - if p.is_link_source_property(ctx.schema): + if p.get_computable(ctx.schema): + kind = context.ColumnComputable(pointer=p) + elif p.is_link_source_property(ctx.schema): kind = context.ColumnByName(reference_as='source') elif p.is_link_target_property(ctx.schema): kind = context.ColumnByName(reference_as='target') @@ -418,7 +419,11 @@ def _construct_column( kind = context.ColumnByName(reference_as=dbname) elif isinstance(p, s_links.Link): - if short_name.name == '__type__': + + if p.get_computable(ctx.schema): + col_name = short_name.name + '_id' + kind = context.ColumnComputable(pointer=p) + elif short_name.name == '__type__': col_name = '__type__' if not include_inherited: diff --git a/tests/schemas/movies.esdl b/tests/schemas/movies.esdl index d12ae87e029..c229bc0d3f0 100644 --- a/tests/schemas/movies.esdl +++ b/tests/schemas/movies.esdl @@ -16,10 +16,15 @@ # limitations under the License. # +global username_prefix: str; type Person { required first_name: str; last_name: str; + + full_name := __source__.first_name ++ ((' ' ++ .last_name) ?? ''); + favorite_genre := (select Genre filter .name = 'Drama' limit 1); + username := (global username_prefix ?? 'u_') ++ str_lower(.first_name); } type Genre { @@ -39,11 +44,14 @@ type Movie extending Content { director: Person { bar: str; }; + + multi actor_names := __source__.actors.first_name; + multi similar_to := (select Content); } type Book extending Content { required pages: int16; - multi chapters: str; +multi chapters: str; } type novel extending Book { diff --git a/tests/test_sql_query.py b/tests/test_sql_query.py index bd07df80f38..9bec6b101eb 100644 --- a/tests/test_sql_query.py +++ b/tests/test_sql_query.py @@ -21,6 +21,7 @@ import os.path import unittest +from edb.tools import test from edb.testbase import server as tb try: @@ -174,17 +175,21 @@ async def test_sql_query_11(self): JOIN "Genre" g ON "Movie".genre_id = g.id ''' ) - self.assert_shape(res, 2, [ - 'id', - '__type__', - 'director_id', - 'genre_id', - 'release_year', - 'title', - 'id', - '__type__', - 'name' - ]) + self.assert_shape( + res, + 2, + [ + 'id', + '__type__', + 'director_id', + 'genre_id', + 'release_year', + 'title', + 'id', + '__type__', + 'name', + ], + ) async def test_sql_query_12(self): # JOIN USING @@ -719,8 +724,11 @@ async def test_sql_query_introspection_01(self): ['Movie.director', 'bar', 'YES', 3], ['Person', 'id', 'NO', 1], ['Person', '__type__', 'NO', 2], - ['Person', 'first_name', 'NO', 3], - ['Person', 'last_name', 'YES', 4], + ['Person', 'favorite_genre_id', 'YES', 3], + ['Person', 'first_name', 'NO', 4], + ['Person', 'full_name', 'NO', 5], + ['Person', 'last_name', 'YES', 6], + ['Person', 'username', 'NO', 7], ['novel', 'id', 'NO', 1], ['novel', '__type__', 'NO', 2], ['novel', 'foo', 'YES', 3], @@ -791,14 +799,18 @@ async def test_sql_query_introspection_04(self): JOIN pg_namespace n ON n.oid = pc.relnamespace WHERE n.nspname = 'public' AND pc.relname = 'novel' ORDER BY attnum - -- skip the system columns - OFFSET 6 ''' ) self.assertEqual( res, [ + ['novel', 'tableoid', True], + ['novel', 'cmax', True], + ['novel', 'xmax', True], + ['novel', 'cmin', True], + ['novel', 'xmin', True], + ['novel', 'ctid', True], ['novel', 'id', True], ['novel', '__type__', True], ['novel', 'foo', False], @@ -1043,15 +1055,15 @@ async def test_sql_query_copy_04(self): out = io.BytesIO() await self.scon.copy_from_table( - "Person", columns=['first_name'], output=out, # 'full_name' + "Person", columns=['first_name', 'full_name'], output=out, format="csv", delimiter="\t" ) out = io.StringIO(out.getvalue().decode("utf-8")) res = list(csv.reader(out, delimiter="\t")) self.assert_data_shape(res, tb.bag([ - ["Robin"], # "Robin" - ["Steven"], # "Steven Spielberg" - ["Tom"], # "Tom Hanks" + ["Robin", "Robin"], + ["Steven", "Steven Spielberg"], + ["Tom", "Tom Hanks"], ])) async def test_sql_query_error_01(self): @@ -1234,3 +1246,111 @@ async def test_sql_query_pgadmin_hack(self): " WHERE name = 'bytea_output'; " ) await self.scon.execute("SET client_encoding='WIN874';") + + async def test_sql_query_computed_01(self): + # single property + res = await self.squery_values( + """ + SELECT full_name + FROM "Person" p + ORDER BY first_name + """ + ) + self.assertEqual(res, [["Robin"], ["Steven Spielberg"], ["Tom Hanks"]]) + + async def test_sql_query_computed_02(self): + # computeds can only be accessed on the table, not rel vars + with self.assertRaisesRegex( + asyncpg.PostgresError, "cannot find column `full_name`" + ): + await self.squery_values( + """ + SELECT t.full_name + FROM ( + SELECT first_name, last_name + FROM "Person" + ) t + """ + ) + + async def test_sql_query_computed_03(self): + # computed in a sublink + res = await self.squery_values( + """ + SELECT (SELECT 'Hello ' || full_name) as hello + FROM "Person" + ORDER BY first_name DESC + LIMIT 1 + """ + ) + self.assertEqual(res, [["Hello Tom Hanks"]]) + + async def test_sql_query_computed_04(self): + # computed in a lateral + res = await self.squery_values( + """ + SELECT t.hello + FROM "Person", + LATERAL (SELECT ('Hello ' || full_name) as hello) t + ORDER BY first_name DESC + LIMIT 1 + """ + ) + self.assertEqual(res, [["Hello Tom Hanks"]]) + + async def test_sql_query_computed_05(self): + # computed in ORDER BY + res = await self.squery_values( + """ + SELECT first_name + FROM "Person" + ORDER BY full_name + """ + ) + self.assertEqual(res, [["Robin"], ["Steven"], ["Tom"]]) + + async def test_sql_query_computed_06(self): + # globals are empty + res = await self.squery_values( + """ + SELECT username FROM "Person" + ORDER BY first_name LIMIT 1 + """ + ) + self.assertEqual(res, [["u_robin"]]) + + async def test_sql_query_computed_07(self): + # single link + res = await self.scon.fetch( + """ + SELECT favorite_genre_id FROM "Person" + """ + ) + self.assert_shape(res, 3, ['favorite_genre_id']) + + res = await self.squery_values( + """ + SELECT g.name + FROM "Person" p + LEFT JOIN "Genre" g ON (p.favorite_genre_id = g.id) + """ + ) + self.assertEqual(res, [["Drama"], ["Drama"], ["Drama"]]) + + @test.not_implemented("multi computed properties are not implemented") + async def test_sql_query_computed_08(self): + # multi property + await self.scon.fetch( + """ + SELECT actor_names FROM "Movie" + """ + ) + + @test.not_implemented("multi computed links are not implemented") + async def test_sql_query_computed_09(self): + # multi link + await self.scon.fetch( + """ + SELECT similar_to FROM "Movie" + """ + )