From 27286af44303ba63240b21a0d7f6f14f4e69545b Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Mon, 12 Feb 2024 11:21:18 -0800 Subject: [PATCH 1/8] Support session and backend extension configs (#6811) This requires: * Some special handling for reflection, since extension settings don't show up in the views we look at normally * Updating _apply_session_config to work for extension configs * Making StateSerializerFactory able to include extension configs. To test this, I fixed pg_trgm's configs. pg_trgm attempted to have some backend configs, but the config object didn't extend ExtensionConfig and also the backend_setting wasn't json. I added some checks to the extension path to prevent this sort of mistake. Database configs work too, and I tested it manually, but currently the only reliable way to make database configs with backend settings take effect is to restart the server, so no tests for that yet. The other big motivation for this is pgvector probes, which I'll leave for @vpetrovykh to follow up on. --- edb/buildmeta.py | 2 +- edb/edgeql/compiler/config.py | 8 -- edb/lib/ext/pg_trgm.edgeql | 8 +- edb/pgsql/delta.py | 7 +- edb/pgsql/metaschema.py | 158 ++++++++++++++++++++++++++----- edb/server/compiler/sertypes.py | 57 ++++++++--- edb/server/config/ops.py | 2 + edb/server/config/spec.py | 39 ++++++-- tests/test_edgeql_ddl.py | 25 +++-- tests/test_edgeql_ext_pg_trgm.py | 118 +++++++++++++++++++++++ 10 files changed, 348 insertions(+), 76 deletions(-) diff --git a/edb/buildmeta.py b/edb/buildmeta.py index f88193fff04..f9e2ad07bce 100644 --- a/edb/buildmeta.py +++ b/edb/buildmeta.py @@ -44,7 +44,7 @@ # Increment this whenever the database layout or stdlib changes. -EDGEDB_CATALOG_VERSION = 2024_02_01_00_00 +EDGEDB_CATALOG_VERSION = 2024_02_09_00_00 EDGEDB_MAJOR_VERSION = 5 diff --git a/edb/edgeql/compiler/config.py b/edb/edgeql/compiler/config.py index b88d5734803..7cd750ff627 100644 --- a/edb/edgeql/compiler/config.py +++ b/edb/edgeql/compiler/config.py @@ -372,14 +372,6 @@ def _validate_op( ptr = None if isinstance(expr, (qlast.ConfigSet, qlast.ConfigReset)): - # TODO: Fix this. The problem is that it gets lost when serializing it - if is_ext_config and expr.scope == qltypes.ConfigScope.SESSION: - raise errors.UnsupportedFeatureError( - 'SESSION configuration of extension-defined config variables ' - 'is not yet implemented' - ) - - # This error is legit, though if is_ext_config and expr.scope == qltypes.ConfigScope.INSTANCE: raise errors.ConfigurationError( 'INSTANCE configuration of extension-defined config variables ' diff --git a/edb/lib/ext/pg_trgm.edgeql b/edb/lib/ext/pg_trgm.edgeql index db85c4fe91c..bd438429cb0 100644 --- a/edb/lib/ext/pg_trgm.edgeql +++ b/edb/lib/ext/pg_trgm.edgeql @@ -23,10 +23,10 @@ create extension package pg_trgm version '1.6' { create module ext::pg_trgm; - create type ext::pg_trgm::Config extending cfg::ConfigObject { + create type ext::pg_trgm::Config extending cfg::ExtensionConfig { create required property similarity_threshold: std::float32 { create annotation cfg::backend_setting := - "pg_trgm.similarity_threshold"; + '"pg_trgm.similarity_threshold"'; create annotation std::description := "The current similarity threshold that is used by the " ++ "pg_trgm::similar() function, the pg_trgm::gin and " @@ -38,7 +38,7 @@ create extension package pg_trgm version '1.6' { }; create required property word_similarity_threshold: std::float32 { create annotation cfg::backend_setting := - "pg_trgm.word_similarity_threshold"; + '"pg_trgm.word_similarity_threshold"'; create annotation std::description := "The current word similarity threshold that is used by the " ++ "pg_trgrm::word_similar() function. The threshold must be " @@ -50,7 +50,7 @@ create extension package pg_trgm version '1.6' { create required property strict_word_similarity_threshold: std::float32 { create annotation cfg::backend_setting := - "pg_trgm.strict_word_similarity_threshold"; + '"pg_trgm.strict_word_similarity_threshold"'; create annotation std::description := "The current strict word similarity threshold that is used by " ++ "the pg_trgrm::strict_word_similar() function. The " diff --git a/edb/pgsql/delta.py b/edb/pgsql/delta.py index 08e173b4ea2..7ced489e761 100644 --- a/edb/pgsql/delta.py +++ b/edb/pgsql/delta.py @@ -3879,7 +3879,12 @@ def _fixup_configs( from edb.pgsql import metaschema - new_local_spec = config.load_spec_from_schema(schema, only_exts=True) + new_local_spec = config.load_spec_from_schema( + schema, + only_exts=True, + # suppress validation because we might be in an intermediate state + validate=False, + ) spec_json = config.spec_to_json(new_local_spec) self.pgops.add(dbops.Query(textwrap.dedent(f'''\ UPDATE diff --git a/edb/pgsql/metaschema.py b/edb/pgsql/metaschema.py index 38762c23fd5..eaf2babbbb3 100644 --- a/edb/pgsql/metaschema.py +++ b/edb/pgsql/metaschema.py @@ -2975,12 +2975,12 @@ class ConvertPostgresConfigUnitsFunction(dbops.Function): ) WHEN "unit" = '' - THEN trunc("value" * "multiplier")::text::jsonb + THEN ("value" * "multiplier")::text::jsonb ELSE edgedb.raise( NULL::jsonb, msg => ( - 'unknown configutation unit "' || + 'unknown configuration unit "' || COALESCE("unit", '') || '"' ) @@ -3003,6 +3003,54 @@ def __init__(self) -> None: ) +class TypeIDToConfigType(dbops.Function): + """Get a postgres config type from a type id. + + (We typically try to read extension configs straight from the + config tables, but for extension configs those aren't present.) + """ + + config_types = { + 'bool': ['std::bool'], + 'string': ['std::str'], + 'integer': ['std::int16', 'std::int32', 'std::int64'], + 'real': ['std::float32', 'std::float64'], + } + cases = [ + f''' + WHEN "typeid" = '{s_obj.get_known_type_id(t)}' THEN '{ct}' + ''' + for ct, types in config_types.items() + for t in types + ] + scases = '\n'.join(cases) + + text = f""" + SELECT ( + CASE + {scases} + ELSE edgedb.raise( + NULL::text, + msg => ( + 'unknown configuration type "' || "typeid" || '"' + ) + ) + END + ) + """ + + def __init__(self) -> None: + super().__init__( + name=('edgedb', '_type_id_to_config_type'), + args=[ + ('typeid', ('uuid',)), + ], + returns=('text',), + volatility='immutable', + text=self.text, + ) + + class NormalizedPgSettingsView(dbops.View): """Just like `pg_settings` but with the parsed 'unit' column.""" @@ -3082,7 +3130,7 @@ class InterpretConfigValueToJsonFunction(dbops.Function): edgedb.raise( NULL::jsonb, msg => ( - 'unknown configutation type "' || + 'unknown configuration type "' || COALESCE("type", '') || '"' ) @@ -3154,17 +3202,6 @@ class PostgresConfigValueToJsonFunction(dbops.Function): END) FROM - ( - SELECT - epg_settings.vartype AS vartype, - epg_settings.multiplier AS multiplier, - epg_settings.unit AS unit - FROM - edgedb._normalized_pg_settings AS epg_settings - WHERE - epg_settings.name = "setting_name" - ) AS settings, - LATERAL ( SELECT regexp_match( "setting_value", '^(\d+)\s*([a-zA-Z]{0,3})$') AS v @@ -3175,6 +3212,27 @@ class PostgresConfigValueToJsonFunction(dbops.Function): COALESCE(_unit.v[1], "setting_value") AS val, COALESCE(_unit.v[2], '') AS unit ) AS parsed_value + LEFT OUTER JOIN + ( + SELECT + epg_settings.vartype AS vartype, + epg_settings.multiplier AS multiplier, + epg_settings.unit AS unit + FROM + edgedb._normalized_pg_settings AS epg_settings + WHERE + epg_settings.name = "setting_name" + ) AS settings_in ON true + CROSS JOIN LATERAL + ( + SELECT + COALESCE(settings_in.vartype, + edgedb._type_id_to_config_type("setting_typeid")) + as vartype, + COALESCE(settings_in.multiplier, '1') as multiplier, + COALESCE(settings_in.unit, '') as unit + ) as settings + """ def __init__(self) -> None: @@ -3182,6 +3240,7 @@ def __init__(self) -> None: name=('edgedb', '_postgres_config_value_to_json'), args=[ ('setting_name', ('text',)), + ('setting_typeid', ('uuid',)), ('setting_value', ('text',)), ], returns=('jsonb',), @@ -3227,6 +3286,9 @@ class SysConfigFullFunction(dbops.Function): FROM config_spec s ), + config_extension_defaults AS ( + SELECT * FROM config_defaults WHERE name like '%::%' + ), config_sys AS ( SELECT @@ -3274,7 +3336,7 @@ class SysConfigFullFunction(dbops.Function): SELECT spec.name, edgedb._postgres_config_value_to_json( - spec.backend_setting, nameval.value + spec.backend_setting, spec.typeid, nameval.value ) AS value, 'database' AS source, TRUE AS is_backend @@ -3300,7 +3362,8 @@ class SysConfigFullFunction(dbops.Function): LATERAL ( SELECT config_spec.name, - config_spec.backend_setting + config_spec.backend_setting, + config_spec.typeid FROM config_spec WHERE @@ -3315,7 +3378,7 @@ class SysConfigFullFunction(dbops.Function): SELECT spec.name, edgedb._postgres_config_value_to_json( - spec.backend_setting, setting + spec.backend_setting, spec.typeid, setting ) AS value, 'postgres configuration file' AS source, TRUE AS is_backend @@ -3324,7 +3387,8 @@ class SysConfigFullFunction(dbops.Function): LATERAL ( SELECT config_spec.name, - config_spec.backend_setting + config_spec.backend_setting, + config_spec.typeid FROM config_spec WHERE @@ -3342,7 +3406,7 @@ class SysConfigFullFunction(dbops.Function): SELECT spec.name, edgedb._postgres_config_value_to_json( - spec.backend_setting, setting + spec.backend_setting, spec.typeid, setting ) AS value, 'system override' AS source, TRUE AS is_backend @@ -3351,7 +3415,8 @@ class SysConfigFullFunction(dbops.Function): LATERAL ( SELECT config_spec.name, - config_spec.backend_setting + config_spec.backend_setting, + config_spec.typeid FROM config_spec WHERE @@ -3409,6 +3474,25 @@ class SysConfigFullFunction(dbops.Function): ) AS spec ), + -- extension session configs don't show up in any system view, so we + -- check _edgecon_state to see when they are present. + pg_extension_config AS ( + SELECT + config_spec.name, + -- XXX: Or would it be better to just use the json directly? + edgedb._postgres_config_value_to_json( + config_spec.backend_setting, + config_spec.typeid, + current_setting(config_spec.backend_setting, true) + ) AS value, + 'session' AS source, + TRUE AS is_backend + FROM _edgecon_state s + INNER JOIN config_spec + ON s.name = config_spec.name + WHERE s.type = 'B' AND s.name LIKE '%::%' + ), + edge_all_settings AS MATERIALIZED ( SELECT q.* @@ -3432,10 +3516,13 @@ class SysConfigFullFunction(dbops.Function): q.* FROM ( + -- extension defaults aren't in any system views + SELECT * FROM config_extension_defaults UNION ALL SELECT * FROM pg_db_setting UNION ALL SELECT * FROM pg_conf_settings UNION ALL SELECT * FROM pg_auto_conf_settings UNION ALL - SELECT * FROM pg_config + SELECT * FROM pg_config UNION ALL + SELECT * FROM pg_extension_config ) AS q WHERE q.is_backend @@ -3448,12 +3535,15 @@ class SysConfigFullFunction(dbops.Function): q.* FROM ( + -- extension defaults aren't in any system views + SELECT * FROM config_extension_defaults UNION ALL -- config_sys is here, because there -- is no other way to read instance-level -- configuration overrides. SELECT * FROM config_sys UNION ALL SELECT * FROM pg_db_setting UNION ALL - SELECT * FROM pg_config + SELECT * FROM pg_config UNION ALL + SELECT * FROM pg_extension_config ) AS q WHERE q.is_backend @@ -3689,10 +3779,6 @@ def __init__(self) -> None: ) -# TODO: Support extension-defined configs that affect the backend -# Not needed for supporting auth, so can skip temporarily. -# If perf seems to matter, can hardcode things for base config -# and consult json for just extension stuff. class ApplySessionConfigFunction(dbops.Function): """Apply an EdgeDB config setting to the backend, if possible. @@ -3738,6 +3824,18 @@ def __init__(self, config_spec: edbconfig.Spec) -> None: ) ''') + ext_config = ''' + SELECT pg_catalog.set_config( + (s.val->>'backend_setting')::text, + "value"->>0, + false + ) + FROM + edgedbinstdata.instdata as id, + LATERAL jsonb_each(id.json) AS s(key, val) + WHERE id.key = 'configspec_ext' AND s.key = "name" + ''' + variants = "\n".join(variants_list) text = f''' SELECT ( @@ -3756,6 +3854,13 @@ def __init__(self, config_spec: edbconfig.Spec) -> None: END ) + WHEN "name" LIKE '%::%' + THEN + CASE WHEN ({ext_config}) IS NULL + THEN "name" + ELSE "name" + END + ELSE "name" END ) @@ -4526,6 +4631,7 @@ async def bootstrap( dbops.CreateEnum(SysConfigScopeType()), dbops.CreateCompositeType(SysConfigValueType()), dbops.CreateCompositeType(SysConfigEntryType()), + dbops.CreateFunction(TypeIDToConfigType()), dbops.CreateFunction(ConvertPostgresConfigUnitsFunction()), dbops.CreateFunction(InterpretConfigValueToJsonFunction()), dbops.CreateFunction(PostgresConfigValueToJsonFunction()), diff --git a/edb/server/compiler/sertypes.py b/edb/server/compiler/sertypes.py index 1e057f91a59..35e7211702e 100644 --- a/edb/server/compiler/sertypes.py +++ b/edb/server/compiler/sertypes.py @@ -64,6 +64,7 @@ _uint16_packer = cast(Callable[[int], bytes], struct.Struct('!H').pack) _uint8_packer = cast(Callable[[int], bytes], struct.Struct('!B').pack) _int64_struct = struct.Struct('!q') +_float32_struct = struct.Struct('!f') EMPTY_TUPLE_ID = s_obj.get_known_type_id('empty-tuple') @@ -132,6 +133,14 @@ def _decode_int64(data: bytes) -> int: return _int64_struct.unpack(data)[0] # type: ignore [no-any-return] +def _encode_float32(data: float) -> bytes: + return _float32_struct.pack(data) + + +def _decode_float32(data: bytes) -> float: + return _float32_struct.unpack(data)[0] # type: ignore [no-any-return] + + def _string_packer(s: str) -> bytes: s_bytes = s.encode('utf-8') return _uint32_packer(len(s_bytes)) + s_bytes @@ -1635,6 +1644,28 @@ def __init__(self, std_schema: s_schema.Schema, config_spec: config.Spec): schema, config_type = derive_alias( schema, free_obj, 'state_config' ) + config_shape = self._make_config_shape(config_spec, schema) + + self._input_shapes: immutables.Map[ + s_types.Type, + tuple[InputShapeElement, ...], + ] = immutables.Map([ + (config_type, config_shape), + (self._state_type, ( + ("module", str_type, enums.Cardinality.AT_MOST_ONE), + ("aliases", aliases_array, enums.Cardinality.AT_MOST_ONE), + ("config", config_type, enums.Cardinality.AT_MOST_ONE), + )) + ]) + self.config_type = config_type + self._schema = schema + self._contexts: dict[edbdef.ProtocolVersion, Context] = {} + + @staticmethod + def _make_config_shape( + config_spec: config.Spec, + schema: s_schema.Schema, + ) -> tuple[tuple[str, s_types.Type, enums.Cardinality], ...]: config_shape: list[tuple[str, s_types.Type, enums.Cardinality]] = [] for setting in config_spec.values(): @@ -1649,20 +1680,7 @@ def __init__(self, std_schema: s_schema.Schema, config_spec: config.Spec): enums.Cardinality.AT_MOST_ONE, ) ) - - self._input_shapes: immutables.Map[ - s_types.Type, - tuple[InputShapeElement, ...], - ] = immutables.Map([ - (config_type, tuple(sorted(config_shape))), - (self._state_type, ( - ("module", str_type, enums.Cardinality.AT_MOST_ONE), - ("aliases", aliases_array, enums.Cardinality.AT_MOST_ONE), - ("config", config_type, enums.Cardinality.AT_MOST_ONE), - )) - ]) - self._schema = schema - self._contexts: dict[edbdef.ProtocolVersion, Context] = {} + return tuple(sorted(config_shape)) def make( self, @@ -1688,6 +1706,12 @@ def make( ctx.schema = s_schema.ChainedSchema( self._schema, user_schema, global_schema) + # Update the config shape with any extension configs + ext_config_spec = config.load_ext_spec_from_schema( + user_schema, self._schema) + new_config = self._make_config_shape(ext_config_spec, ctx.schema) + full_config = self._input_shapes[self.config_type] + new_config + globals_shape = [] global_reps = {} for g in ctx.schema.get_objects(type=s_globals.Global): @@ -1704,6 +1728,7 @@ def make( self._state_type, self._input_shapes.update({ self.globals_type: tuple(sorted(globals_shape)), + self.config_type: full_config, self._state_type: self._input_shapes[self._state_type] + ( ( "globals", @@ -1889,6 +1914,10 @@ class BaseScalarDesc(SchemaTypeDesc): _encode_int64, _decode_int64, ), + s_obj.get_known_type_id('std::float32'): ( + _encode_float32, + _decode_float32, + ), } def encode(self, data: Any) -> bytes: diff --git a/edb/server/config/ops.py b/edb/server/config/ops.py index df95dcc2ef1..2d50d72cdeb 100644 --- a/edb/server/config/ops.py +++ b/edb/server/config/ops.py @@ -317,6 +317,8 @@ def spec_to_json(spec: spec.Spec): typeid = s_obj.get_known_type_id('std::bool') elif _issubclass(setting.type, int): typeid = s_obj.get_known_type_id('std::int64') + elif _issubclass(setting.type, float): + typeid = s_obj.get_known_type_id('std::float32') elif _issubclass(setting.type, types.ConfigType): typeid = setting.type.get_edgeql_typeid() elif _issubclass(setting.type, statypes.Duration): diff --git a/edb/server/config/spec.py b/edb/server/config/spec.py index 8627b17f3e8..7cb7676d925 100644 --- a/edb/server/config/spec.py +++ b/edb/server/config/spec.py @@ -29,6 +29,7 @@ from edb.edgeql import compiler as qlcompiler from edb.ir import staeval from edb.ir import statypes +from edb.schema import links as s_links from edb.schema import name as sn from edb.schema import objtypes as s_objtypes from edb.schema import scalars as s_scalars @@ -39,7 +40,8 @@ from . import types -SETTING_TYPES = {str, int, bool, statypes.Duration, statypes.ConfigMemory} +SETTING_TYPES = {str, int, bool, float, + statypes.Duration, statypes.ConfigMemory} @dataclasses.dataclass(frozen=True, eq=True) @@ -66,7 +68,8 @@ def __post_init__(self) -> None: not isinstance(self.type, types.ConfigTypeSpec)): raise ValueError( f'invalid config setting {self.name!r}: ' - f'type is expected to be either one of {{str, int, bool}} ' + f'type is expected to be either one of ' + f'{{str, int, bool, float}} ' f'or an edb.server.config.types.ConfigType subclass') if self.set_of: @@ -195,7 +198,9 @@ def __len__(self) -> int: def load_spec_from_schema( - schema: s_schema.Schema, only_exts: bool=False + schema: s_schema.Schema, + only_exts: bool=False, + validate: bool=True, ) -> Spec: settings = [] if not only_exts: @@ -204,6 +209,18 @@ def load_spec_from_schema( settings.extend(load_ext_settings_from_schema(schema)) + # Make sure there aren't any dangling ConfigObject children + if validate: + cfg_object = schema.get('cfg::ConfigObject', type=s_objtypes.ObjectType) + for child in cfg_object.children(schema): + if not schema.get_referrers( + child, scls_type=s_links.Link, field_name='target' + ): + raise RuntimeError( + f'cfg::ConfigObject child {child.get_name(schema)} has no ' + f'links pointing at it (did you mean cfg::ExtensionConfig?)' + ) + return FlatSpec(*settings) @@ -255,11 +272,17 @@ def _load_spec_from_type( else: pytype = staeval.scalar_type_to_python_type(ptype, schema) - attributes = { - a: json.loads(v.get_value(schema)) - for a, v in p.get_annotations(schema).items(schema) - if isinstance(a, sn.QualName) and a.module == 'cfg' - } + attributes = {} + for a, v in p.get_annotations(schema).items(schema): + if isinstance(a, sn.QualName) and a.module == 'cfg': + try: + jv = json.loads(v.get_value(schema)) + except json.JSONDecodeError: + raise RuntimeError( + f'Config annotation {a} on {p.get_name(schema)} ' + f'is not valid json' + ) + attributes[a] = jv ptr_card = p.get_cardinality(schema) set_of = ptr_card.is_multi() diff --git a/tests/test_edgeql_ddl.py b/tests/test_edgeql_ddl.py index cb19712f06c..459a4c37fb9 100644 --- a/tests/test_edgeql_ddl.py +++ b/tests/test_edgeql_ddl.py @@ -16564,22 +16564,19 @@ async def _check(_cfg_obj='Config', **kwargs): "session!"; ''') - # TODO: This should all work, instead! - async with self.assertRaisesRegexTx( - edgedb.UnsupportedFeatureError, ""): - await self.con.execute(''' - configure session set ext::_conf::Config::config_name := - "session!"; - ''') + await self.con.execute(''' + configure session set ext::_conf::Config::config_name := + "session!"; + ''') - await _check( - config_name='session!', - objs=[], - ) + await _check( + config_name='session!', + objs=[], + ) - await self.con.execute(''' - configure session reset ext::_conf::Config::config_name; - ''') + await self.con.execute(''' + configure session reset ext::_conf::Config::config_name; + ''') await _check( config_name='test', diff --git a/tests/test_edgeql_ext_pg_trgm.py b/tests/test_edgeql_ext_pg_trgm.py index 517f6cc1e6e..098fa2f81ff 100644 --- a/tests/test_edgeql_ext_pg_trgm.py +++ b/tests/test_edgeql_ext_pg_trgm.py @@ -304,3 +304,121 @@ async def test_edgeql_ext_pg_trgm_strict_word_similarity(self): qry, index_type="ext::pg_trgm::gist", ) + + async def test_edgeql_ext_pg_trgm_config(self): + # We are going to fiddle with the similarity_threshold config + # and make sure it works right. + + sim_query = """ + WITH similar := ( + SELECT + Gist { + p_str, + sim := ext::pg_trgm::similarity(.p_str, "qwertyu0988") + } + FILTER + ext::pg_trgm::similar(.p_str, "qwertyu0988") + ), + SELECT exists similar and all(similar.sim >= $sim) + """ + + cfg_query = """ + select cfg::Config.extensions[is ext::pg_trgm::Config] + .similarity_threshold; + """ + + await self.assert_query_result( + sim_query, + [True], + variables=dict(sim=0.3), + ) + await self.assert_query_result( + sim_query, + [False], + variables=dict(sim=0.5), + ) + await self.assert_query_result( + sim_query, + [False], + variables=dict(sim=0.9), + ) + + await self.assert_query_result( + cfg_query, + [0.3], + ) + + await self.con.execute(''' + configure session + set ext::pg_trgm::Config::similarity_threshold := 0.5 + ''') + + await self.assert_query_result( + sim_query, + [True], + variables=dict(sim=0.3), + ) + await self.assert_query_result( + sim_query, + [True], + variables=dict(sim=0.5), + ) + await self.assert_query_result( + sim_query, + [False], + variables=dict(sim=0.9), + ) + await self.assert_query_result( + cfg_query, + [0.5], + ) + + await self.con.execute(''' + configure session + set ext::pg_trgm::Config::similarity_threshold := 0.9 + ''') + + await self.assert_query_result( + sim_query, + [True], + variables=dict(sim=0.3), + ) + await self.assert_query_result( + sim_query, + [True], + variables=dict(sim=0.5), + ) + await self.assert_query_result( + sim_query, + [True], + variables=dict(sim=0.9), + ) + await self.assert_query_result( + cfg_query, + [0.9], + ) + + await self.con.execute(''' + configure session + reset ext::pg_trgm::Config::similarity_threshold + ''') + + await self.assert_query_result( + sim_query, + [True], + variables=dict(sim=0.3), + ) + await self.assert_query_result( + sim_query, + [False], + variables=dict(sim=0.5), + ) + await self.assert_query_result( + sim_query, + [False], + variables=dict(sim=0.9), + ) + await self.assert_query_result( + cfg_query, + [0.3], + ) From 017738f33e941a559da9af7836a38a9e83fa086a Mon Sep 17 00:00:00 2001 From: Fantix King Date: Tue, 13 Feb 2024 05:05:43 +0900 Subject: [PATCH 2/8] Drop dead code (#6818) Queries after DDL in a transaction have been running without cache for some time, dropping the dead code for now; maybe we will add it back in the future properly. --- edb/server/dbview/dbview.pxd | 4 ---- edb/server/dbview/dbview.pyx | 36 ++++++------------------------------ 2 files changed, 6 insertions(+), 34 deletions(-) diff --git a/edb/server/dbview/dbview.pxd b/edb/server/dbview/dbview.pxd index 68e7e482103..169df4516cf 100644 --- a/edb/server/dbview/dbview.pxd +++ b/edb/server/dbview/dbview.pxd @@ -138,9 +138,6 @@ cdef class DatabaseConnectionView: tuple _session_state_db_cache tuple _session_state_cache - - object _eql_to_compiled - object _txid object _in_tx_db_config object _in_tx_savepoints @@ -163,7 +160,6 @@ cdef class DatabaseConnectionView: object __weakref__ - cdef _invalidate_local_cache(self) cdef _reset_tx_state(self) cdef clear_tx_error(self) diff --git a/edb/server/dbview/dbview.pyx b/edb/server/dbview/dbview.pyx index 296304ec0fd..d526fc182c4 100644 --- a/edb/server/dbview/dbview.pyx +++ b/edb/server/dbview/dbview.pyx @@ -294,8 +294,6 @@ cdef class Database: cdef class DatabaseConnectionView: - _eql_to_compiled: typing.Mapping[bytes, dbstate.QueryUnitGroup] - def __init__(self, db: Database, *, query_cache, protocol_version): self._db = db @@ -325,16 +323,8 @@ cdef class DatabaseConnectionView: self._last_comp_state = None self._last_comp_state_id = 0 - # Whenever we are in a transaction that had executed a - # DDL command, we use this cache for compiled queries. - self._eql_to_compiled = lru.LRUMapping( - maxsize=defines._MAX_QUERIES_CACHE) - self._reset_tx_state() - cdef _invalidate_local_cache(self): - self._eql_to_compiled.clear() - cdef _reset_tx_state(self): self._txid = None self._in_tx = False @@ -354,7 +344,6 @@ cdef class DatabaseConnectionView: self._in_tx_state_serializer = None self._tx_error = False self._in_tx_dbver = 0 - self._invalidate_local_cache() cdef clear_tx_error(self): self._tx_error = False @@ -379,7 +368,6 @@ cdef class DatabaseConnectionView: self.set_session_config(config) self.set_globals(globals) self.set_state_serializer(state_serializer) - self._invalidate_local_cache() cdef declare_savepoint(self, name, spid): state = ( @@ -729,11 +717,8 @@ cdef class DatabaseConnectionView: cdef cache_compiled_query(self, object key, object query_unit_group): assert query_unit_group.cacheable - key = (key, self.get_modaliases(), self.get_session_config()) - - if self._in_tx_with_ddl: - self._eql_to_compiled[key] = query_unit_group - else: + if not self._in_tx_with_ddl: + key = (key, self.get_modaliases(), self.get_session_config()) self._db._cache_compiled_query(key, query_unit_group) cdef lookup_compiled_query(self, object key): @@ -743,14 +728,10 @@ cdef class DatabaseConnectionView: return None key = (key, self.get_modaliases(), self.get_session_config()) - - if self._in_tx_with_ddl: - query_unit_group = self._eql_to_compiled.get(key) - else: - query_unit_group, qu_dbver = self._db._eql_to_compiled.get( - key, DICTDEFAULT) - if query_unit_group is not None and qu_dbver != self._db.dbver: - query_unit_group = None + query_unit_group, qu_dbver = self._db._eql_to_compiled.get( + key, DICTDEFAULT) + if query_unit_group is not None and qu_dbver != self._db.dbver: + query_unit_group = None return query_unit_group @@ -817,11 +798,6 @@ cdef class DatabaseConnectionView: cdef on_success(self, query_unit, new_types): side_effects = 0 - if query_unit.tx_savepoint_rollback: - # Need to invalidate the cache in case there were - # SET ALIAS or CONFIGURE or DDL commands. - self._invalidate_local_cache() - if not self._in_tx: if new_types: self._db._update_backend_ids(new_types) From 6a192dd9f064873712707e387d4089b744c33f4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alja=C5=BE=20Mur=20Er=C5=BEen?= Date: Mon, 12 Feb 2024 21:57:45 +0100 Subject: [PATCH 3/8] Migrate rust-python crates from cpython to pyo3 (#6816) --- Cargo.lock | 231 ++++++++-- Cargo.toml | 1 + edb/edgeql-parser/Cargo.toml | 4 +- .../edgeql-parser-derive/src/lib.rs | 2 +- .../edgeql-parser-python/Cargo.toml | 2 +- .../edgeql-parser-python/src/errors.rs | 93 +---- .../edgeql-parser-python/src/hash.rs | 61 +-- .../edgeql-parser-python/src/keywords.rs | 49 +-- .../edgeql-parser-python/src/lib.rs | 93 ++--- .../edgeql-parser-python/src/normalize.rs | 14 +- .../edgeql-parser-python/src/parser.rs | 199 +++++---- .../edgeql-parser-python/src/position.rs | 80 ++-- .../edgeql-parser-python/src/pynormalize.rs | 176 ++++---- .../edgeql-parser-python/src/tokenizer.rs | 79 ++-- .../edgeql-parser-python/tests/normalize.rs | 4 +- edb/edgeql-parser/src/ast.rs | 1 + edb/edgeql-parser/src/hash.rs | 6 +- edb/edgeql-parser/src/helpers/bytes.rs | 28 +- edb/edgeql-parser/src/helpers/strings.rs | 24 +- edb/edgeql-parser/src/into_python.rs | 1 + edb/edgeql-parser/src/parser.rs | 8 +- edb/edgeql-parser/src/parser/custom_errors.rs | 4 +- edb/edgeql-parser/src/position.rs | 4 +- edb/edgeql-parser/src/preparser.rs | 4 +- edb/edgeql-parser/src/schema_file.rs | 4 +- edb/edgeql-parser/src/tokenizer.rs | 96 ++--- edb/edgeql-parser/src/validation.rs | 10 +- edb/edgeql-parser/tests/preparser.rs | 2 +- edb/edgeql-parser/tests/tokenizer.rs | 24 +- edb/edgeql/parser/__init__.py | 22 +- edb/edgeql/tokenizer.py | 18 +- edb/graphql-rewrite/Cargo.toml | 4 +- edb/graphql-rewrite/_graphql_rewrite.pyi | 11 + edb/graphql-rewrite/src/lib.rs | 47 ++- edb/graphql-rewrite/src/py_entry.rs | 85 ++++ edb/graphql-rewrite/src/py_exception.rs | 23 + edb/graphql-rewrite/src/py_token.rs | 193 +++++++++ edb/graphql-rewrite/src/pyentry.rs | 249 ----------- edb/graphql-rewrite/src/pyerrors.rs | 297 ------------- edb/graphql-rewrite/src/pytoken.rs | 74 ---- .../src/{entry_point.rs => rewrite.rs} | 393 +++++++++--------- edb/graphql-rewrite/tests/rewrite.rs | 34 +- edb/graphql/extension.pyx | 10 +- edb/tools/parser_demo.py | 13 +- setup.py | 4 +- 45 files changed, 1279 insertions(+), 1502 deletions(-) create mode 100644 edb/graphql-rewrite/_graphql_rewrite.pyi create mode 100644 edb/graphql-rewrite/src/py_entry.rs create mode 100644 edb/graphql-rewrite/src/py_exception.rs create mode 100644 edb/graphql-rewrite/src/py_token.rs delete mode 100644 edb/graphql-rewrite/src/pyentry.rs delete mode 100644 edb/graphql-rewrite/src/pyerrors.rs delete mode 100644 edb/graphql-rewrite/src/pytoken.rs rename edb/graphql-rewrite/src/{entry_point.rs => rewrite.rs} (78%) diff --git a/Cargo.lock b/Cargo.lock index 7ce44d68ebd..1fcffff4d52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,15 +2,6 @@ # It is not intended for manual editing. version = 3 -[[package]] -name = "aho-corasick" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" -dependencies = [ - "memchr", -] - [[package]] name = "append-only-vec" version = "0.1.2" @@ -148,18 +139,6 @@ dependencies = [ "libc", ] -[[package]] -name = "cpython" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3052106c29da7390237bc2310c1928335733b286287754ea85e6093d2495280e" -dependencies = [ - "libc", - "num-traits", - "paste", - "python3-sys", -] - [[package]] name = "crypto-common" version = "0.1.6" @@ -245,12 +224,12 @@ dependencies = [ "base32", "bigdecimal", "bumpalo", - "cpython", "edgeql-parser-derive", "indexmap", "memchr", "num-bigint 0.3.3", "phf", + "pyo3", "serde", "serde_json", "sha2", @@ -277,12 +256,12 @@ dependencies = [ "bitcode", "blake2", "bytes", - "cpython", "edgedb-protocol", "edgeql-parser", "indexmap", "num-bigint 0.4.3", "once_cell", + "pyo3", "serde", "serde_json", ] @@ -314,11 +293,11 @@ name = "graphql-rewrite" version = "0.1.0" dependencies = [ "combine", - "cpython", "edb-graphql-parser", "num-bigint 0.4.3", "num-traits", "pretty_assertions", + "pyo3", "thiserror", ] @@ -344,6 +323,12 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "indoc" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" + [[package]] name = "itertools" version = "0.4.19" @@ -362,6 +347,16 @@ version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +[[package]] +name = "lock_api" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.17" @@ -377,6 +372,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + [[package]] name = "num-bigint" version = "0.2.6" @@ -455,10 +459,27 @@ dependencies = [ ] [[package]] -name = "paste" -version = "1.0.12" +name = "parking_lot" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] [[package]] name = "phf" @@ -524,13 +545,64 @@ dependencies = [ ] [[package]] -name = "python3-sys" -version = "0.7.1" +name = "pyo3" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a89dc7a5850d0e983be1ec2a463a171d20990487c3cfcd68b5363f1ee3d6fe0" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "memoffset", + "parking_lot", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07426f0d8fe5a601f26293f300afd1a7b1ed5e78b2a705870c5f30893c5163be" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f8b50d72fb3015735aa403eebf19bbd72c093bfeeae24ee798be5f2f1aab52" +checksum = "dbb7dec17e17766b46bca4f1a4215a85006b4c2ecde122076c562dd058da6cf1" dependencies = [ "libc", - "regex", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f738b4e40d50b5711957f142878cfa0f28e054aa0ebdfc3fd137a843f74ed3" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn 2.0.16", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fc910d4851847827daf9d6cdd4a823fbdaab5b8818325c5e97a86da79e8881f" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.16", ] [[package]] @@ -558,22 +630,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" [[package]] -name = "regex" -version = "1.8.1" +name = "redox_syscall" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", + "bitflags", ] -[[package]] -name = "regex-syntax" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" - [[package]] name = "residua-zigzag" version = "0.1.0" @@ -586,6 +650,12 @@ version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "serde" version = "1.0.163" @@ -635,6 +705,12 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" +[[package]] +name = "smallvec" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" + [[package]] name = "snafu" version = "0.7.4" @@ -685,6 +761,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "target-lexicon" +version = "0.12.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69758bda2e78f098e4ccb393021a0963bb3442eac05f135c30f61b7370bbafae" + [[package]] name = "thiserror" version = "1.0.40" @@ -723,6 +805,12 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +[[package]] +name = "unindent" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" + [[package]] name = "unreachable" version = "1.0.0" @@ -828,6 +916,63 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "yansi" version = "0.5.1" diff --git a/Cargo.toml b/Cargo.toml index 41e1681b941..6c9a653dff0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "edb/edgeql-parser/edgeql-parser-python", "edb/graphql-rewrite", ] +resolver = "2" [profile.release] debug = true diff --git a/edb/edgeql-parser/Cargo.toml b/edb/edgeql-parser/Cargo.toml index 06e581981e7..214f4f63ed6 100644 --- a/edb/edgeql-parser/Cargo.toml +++ b/edb/edgeql-parser/Cargo.toml @@ -20,7 +20,7 @@ serde = { version = "1.0.106", features = ["derive"], optional = true } thiserror = "1.0.23" unicode-width = "0.1.8" edgeql-parser-derive = { path = "edgeql-parser-derive", optional = true } -cpython = { version = "0.7.0", optional = true } +pyo3 = { version = "0.20.2", optional = true } indexmap = "1.9.3" serde_json = { version = "1.0", features = ["preserve_order"] } bumpalo = { version = "3.13.0", features = ["collections"] } @@ -30,6 +30,6 @@ append-only-vec = "0.1.2" [features] default = [] wasm-lexer = ["wasm-bindgen", "serde"] -python = ["cpython", "serde", "edgeql-parser-derive"] +python = ["pyo3", "serde", "edgeql-parser-derive"] [lib] diff --git a/edb/edgeql-parser/edgeql-parser-derive/src/lib.rs b/edb/edgeql-parser/edgeql-parser-derive/src/lib.rs index 480bf1f4a72..bd14bda7a1e 100644 --- a/edb/edgeql-parser/edgeql-parser-derive/src/lib.rs +++ b/edb/edgeql-parser/edgeql-parser-derive/src/lib.rs @@ -190,7 +190,7 @@ fn find_attr<'a>(attrs: &'a [Attribute], name: &'static str) -> Option<&'a Attri }) } -fn is_option<'a>(ty: &Type) -> bool { +fn is_option(ty: &Type) -> bool { let Type::Path(TypePath { path, .. }) = ty else { return false; }; diff --git a/edb/edgeql-parser/edgeql-parser-python/Cargo.toml b/edb/edgeql-parser/edgeql-parser-python/Cargo.toml index 8e53ade0ddb..10c6e6735a2 100644 --- a/edb/edgeql-parser/edgeql-parser-python/Cargo.toml +++ b/edb/edgeql-parser/edgeql-parser-python/Cargo.toml @@ -16,7 +16,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" indexmap = "1.9.3" once_cell = "1.18.0" -cpython = { version = "0.7.0", features = ["extension-module"] } +pyo3 = { version = "0.20.2", features = ["extension-module"] } bitcode = { version = "0.4.0", features = ["serde"] } [dependencies.edgedb-protocol] diff --git a/edb/edgeql-parser/edgeql-parser-python/src/errors.rs b/edb/edgeql-parser/edgeql-parser-python/src/errors.rs index c691a779602..6c22a3c2604 100644 --- a/edb/edgeql-parser/edgeql-parser-python/src/errors.rs +++ b/edb/edgeql-parser/edgeql-parser-python/src/errors.rs @@ -1,90 +1,18 @@ -use crate::cpython::PythonObjectWithTypeObject; - -use cpython::exc::Exception; -use cpython::{ - PyClone, PyErr, PyList, PyObject, PyResult, PyType, Python, PythonObject, ToPyObject, -}; use edgeql_parser::tokenizer::Error; +use pyo3::prelude::*; +use pyo3::{create_exception, exceptions}; -// can't use py_exception macro because that fails on dotted module name -pub struct SyntaxError(PyObject); - -pyobject_newtype!(SyntaxError); - -impl SyntaxError { - pub fn new(py: Python, args: T) -> PyErr { - PyErr::new::(py, args) - } -} - -impl cpython::PythonObjectWithCheckedDowncast for SyntaxError { - #[inline] - fn downcast_from( - py: Python, - obj: PyObject, - ) -> Result { - if SyntaxError::type_object(py).is_instance(py, &obj) { - Ok(unsafe { PythonObject::unchecked_downcast_from(obj) }) - } else { - Err(cpython::PythonObjectDowncastError::new( - py, - "SyntaxError", - SyntaxError::type_object(py), - )) - } - } +create_exception!(_edgeql_parser, SyntaxError, exceptions::PyException); - #[inline] - fn downcast_borrow_from<'a, 'p>( - py: Python<'p>, - obj: &'a PyObject, - ) -> Result<&'a SyntaxError, cpython::PythonObjectDowncastError<'p>> { - if SyntaxError::type_object(py).is_instance(py, obj) { - Ok(unsafe { PythonObject::unchecked_downcast_borrow_from(obj) }) - } else { - Err(cpython::PythonObjectDowncastError::new( - py, - "SyntaxError", - SyntaxError::type_object(py), - )) - } - } -} - -impl cpython::PythonObjectWithTypeObject for SyntaxError { - #[inline] - fn type_object(py: Python) -> PyType { - unsafe { - static mut TYPE_OBJECT: *mut cpython::_detail::ffi::PyTypeObject = - 0 as *mut cpython::_detail::ffi::PyTypeObject; - - if TYPE_OBJECT.is_null() { - TYPE_OBJECT = PyErr::new_type( - py, - "edb._edgeql_parser.SyntaxError", - Some(PythonObject::into_object(py.get_type::())), - None, - ) - .as_type_ptr(); - } +#[pyclass] +pub struct ParserResult { + #[pyo3(get)] + pub out: PyObject, - PyType::from_type_ptr(py, TYPE_OBJECT) - } - } + #[pyo3(get)] + pub errors: PyObject, } -py_class!(pub class ParserResult |py| { - data _out: PyObject; - data _errors: PyList; - - def out(&self) -> PyResult { - Ok(self._out(py).clone_ref(py)) - } - def errors(&self) -> PyResult { - Ok(self._errors(py).clone_ref(py)) - } -}); - pub fn parser_error_into_tuple(py: Python, error: Error) -> PyObject { ( error.message, @@ -92,6 +20,5 @@ pub fn parser_error_into_tuple(py: Python, error: Error) -> PyObject { error.hint, error.details, ) - .into_py_object(py) - .into_object() + .into_py(py) } diff --git a/edb/edgeql-parser/edgeql-parser-python/src/hash.rs b/edb/edgeql-parser/edgeql-parser-python/src/hash.rs index ec2f8e3956d..34cf12be7d3 100644 --- a/edb/edgeql-parser/edgeql-parser-python/src/hash.rs +++ b/edb/edgeql-parser/edgeql-parser-python/src/hash.rs @@ -1,40 +1,45 @@ use std::cell::RefCell; -use cpython::exc::RuntimeError; -use cpython::{PyErr, PyObject, PyResult, PyString}; - use edgeql_parser::hash; +use pyo3::{exceptions::PyRuntimeError, prelude::*, types::PyString}; use crate::errors::SyntaxError; -py_class!(pub class Hasher |py| { - data _hasher: RefCell>; - @staticmethod - def start_migration(parent_id: &PyString) -> PyResult { - let hasher = hash::Hasher::start_migration(&parent_id.to_string(py)?); - Hasher::create_instance(py, RefCell::new(Some(hasher))) +#[pyclass] +pub struct Hasher { + _hasher: RefCell>, +} + +#[pymethods] +impl Hasher { + #[staticmethod] + fn start_migration(parent_id: &PyString) -> PyResult { + let hasher = hash::Hasher::start_migration(parent_id.to_str()?); + Ok(Hasher { + _hasher: RefCell::new(Some(hasher)), + }) } - def add_source(&self, data: &PyString) -> PyResult { - let text = data.to_string(py)?; - let mut cell = self._hasher(py).borrow_mut(); - let hasher = cell.as_mut().ok_or_else(|| { - PyErr::new::(py, - ("cannot add source after finish",)) + + fn add_source(&self, py: Python, data: &PyString) -> PyResult { + let text = data.to_str()?; + let mut cell = self._hasher.borrow_mut(); + let hasher = cell + .as_mut() + .ok_or_else(|| PyRuntimeError::new_err(("cannot add source after finish",)))?; + + hasher.add_source(text).map_err(|e| match e { + hash::Error::Tokenizer(msg, pos) => { + SyntaxError::new_err((msg, (pos.offset, py.None()), py.None(), py.None())) + } })?; - hasher.add_source(&text) - .map_err(|e| match e { - hash::Error::Tokenizer(msg, pos) => { - SyntaxError::new(py, (msg, (pos.offset, py.None()), py.None(), py.None())) - } - })?; Ok(py.None()) } - def make_migration_id(&self) -> PyResult { - let mut cell = self._hasher(py).borrow_mut(); - let hasher = cell.take().ok_or_else(|| { - PyErr::new::(py, - ("cannot do migration id twice",)) - })?; + + fn make_migration_id(&self) -> PyResult { + let mut cell = self._hasher.borrow_mut(); + let hasher = cell + .take() + .ok_or_else(|| PyRuntimeError::new_err(("cannot do migration id twice",)))?; Ok(hasher.make_migration_id()) } -}); +} diff --git a/edb/edgeql-parser/edgeql-parser-python/src/keywords.rs b/edb/edgeql-parser/edgeql-parser-python/src/keywords.rs index dd2a0fc531e..74c722fbbb0 100644 --- a/edb/edgeql-parser/edgeql-parser-python/src/keywords.rs +++ b/edb/edgeql-parser/edgeql-parser-python/src/keywords.rs @@ -1,4 +1,7 @@ -use cpython::{Python, ObjectProtocol, PyResult, PyString, PyList, PyObject}; +use pyo3::{ + prelude::*, + types::{PyList, PyString}, +}; use edgeql_parser::keywords; @@ -9,31 +12,29 @@ pub struct AllKeywords { pub partial: PyObject, } - pub fn get_keywords(py: Python) -> PyResult { - let py_intern = py.import("sys")?.get(py, "intern")?; - let py_frozenset = py.import("builtins")?.get(py, "frozenset")?; - let intern = |s: &str| -> PyResult { - let py_str = PyString::new(py, s); - py_intern.call(py, (py_str,), None) - }; - let current = keywords::CURRENT_RESERVED_KEYWORDS - .iter().copied().map(&intern) - .collect::,_>>()?; - let unreserved = keywords::UNRESERVED_KEYWORDS - .iter().copied().map(&intern) - .collect::,_>>()?; - let future = keywords::FUTURE_RESERVED_KEYWORDS - .iter().copied().map(&intern) - .collect::,_>>()?; - let partial = keywords::PARTIAL_RESERVED_KEYWORDS - .iter().copied().map(&intern) - .collect::,_>>()?; + let intern = py.import("sys")?.getattr("intern")?; + let frozen = py.import("builtins")?.getattr("frozenset")?; + + let current = prepare_keywords(py, keywords::CURRENT_RESERVED_KEYWORDS.iter(), intern)?; + let unreserved = prepare_keywords(py, keywords::UNRESERVED_KEYWORDS.iter(), intern)?; + let future = prepare_keywords(py, keywords::FUTURE_RESERVED_KEYWORDS.iter(), intern)?; + let partial = prepare_keywords(py, keywords::PARTIAL_RESERVED_KEYWORDS.iter(), intern)?; Ok(AllKeywords { - current: py_frozenset.call(py, (PyList::new(py, ¤t),), None)?, - unreserved: py_frozenset.call(py, (PyList::new(py, &unreserved),), None)?, - future: py_frozenset.call(py, (PyList::new(py, &future),), None)?, - partial: py_frozenset.call(py, (PyList::new(py, &partial),), None)?, + current: frozen.call((PyList::new(py, ¤t),), None)?.into(), + unreserved: frozen.call((PyList::new(py, &unreserved),), None)?.into(), + future: frozen.call((PyList::new(py, &future),), None)?.into(), + partial: frozen.call((PyList::new(py, &partial),), None)?.into(), }) } +fn prepare_keywords<'py, I: Iterator>( + py: Python<'py>, + keyword_set: I, + intern: &'py PyAny, +) -> Result, PyErr> { + keyword_set + .cloned() + .map(|s: &str| intern.call((PyString::new(py, s),), None)) + .collect() +} diff --git a/edb/edgeql-parser/edgeql-parser-python/src/lib.rs b/edb/edgeql-parser/edgeql-parser-python/src/lib.rs index 11e2e1d17b0..ce8a88da069 100644 --- a/edb/edgeql-parser/edgeql-parser-python/src/lib.rs +++ b/edb/edgeql-parser/edgeql-parser-python/src/lib.rs @@ -1,8 +1,3 @@ -#[macro_use] -extern crate cpython; - -use cpython::{PyObject, PyString}; - mod errors; mod hash; mod keywords; @@ -12,60 +7,38 @@ mod position; mod pynormalize; mod tokenizer; -use errors::{SyntaxError, ParserResult}; -use parser::{parse, preload_spec, save_spec, CSTNode, Production}; -use position::{offset_of_line, SourcePoint}; -use pynormalize::normalize; -use tokenizer::{get_fn_unpickle_token, tokenize, OpaqueToken}; +use pyo3::prelude::*; + +/// Rust bindings to the edgeql-parser crate +#[pymodule] +fn _edgeql_parser(py: Python, m: &PyModule) -> PyResult<()> { + m.add("SyntaxError", py.get_type::())?; + m.add("ParserResult", py.get_type::())?; + + m.add_class::()?; + + let keywords = keywords::get_keywords(py)?; + m.add("unreserved_keywords", keywords.unreserved)?; + m.add("partial_reserved_keywords", keywords.partial)?; + m.add("future_reserved_keywords", keywords.future)?; + m.add("current_reserved_keywords", keywords.current)?; + + m.add_class::()?; + m.add_function(wrap_pyfunction!(pynormalize::normalize, m)?)?; + + m.add_function(wrap_pyfunction!(parser::parse, m)?)?; + m.add_function(wrap_pyfunction!(parser::preload_spec, m)?)?; + m.add_function(wrap_pyfunction!(parser::save_spec, m)?)?; + m.add_class::()?; + m.add_class::()?; + + m.add_function(wrap_pyfunction!(position::offset_of_line, m)?)?; + m.add("SourcePoint", py.get_type::())?; -py_module_initializer!( - _edgeql_parser, - init_edgeql_parser, - PyInit__edgeql_parser, - |py, m| { - tokenizer::init_module(py); - let keywords = keywords::get_keywords(py)?; - m.add( - py, - "__doc__", - "Rust enhancements for edgeql language parser", - )?; + m.add_class::()?; + m.add_function(wrap_pyfunction!(tokenizer::tokenize, m)?)?; + m.add_function(wrap_pyfunction!(tokenizer::unpickle_token, m)?)?; + tokenizer::fini_module(py, m); - m.add(py, "tokenize", py_fn!(py, tokenize(data: &PyString)))?; - m.add(py, "_unpickle_token", get_fn_unpickle_token(py))?; - m.add(py, "Token", py.get_type::())?; - m.add(py, "SyntaxError", py.get_type::())?; - m.add(py, "ParserResult", py.get_type::())?; - m.add(py, "Entry", py.get_type::())?; - m.add(py, "SourcePoint", py.get_type::())?; - m.add(py, "normalize", py_fn!(py, normalize(query: &PyString)))?; - m.add( - py, - "offset_of_line", - py_fn!(py, offset_of_line(text: &str, line: usize)), - )?; - m.add(py, "Hasher", py.get_type::())?; - m.add(py, "unreserved_keywords", keywords.unreserved)?; - m.add(py, "partial_reserved_keywords", keywords.partial)?; - m.add(py, "future_reserved_keywords", keywords.future)?; - m.add(py, "current_reserved_keywords", keywords.current)?; - m.add( - py, - "parse", - py_fn!(py, parse(parser_name: &PyString, data: PyObject)), - )?; - m.add( - py, - "preload_spec", - py_fn!(py, preload_spec(spec_filepath: &PyString)), - )?; - m.add( - py, - "save_spec", - py_fn!(py, save_spec(py_spec: &PyString, dst: &PyString)), - )?; - m.add(py, "CSTNode", py.get_type::())?; - m.add(py, "Production", py.get_type::())?; - Ok(()) - } -); + Ok(()) +} diff --git a/edb/edgeql-parser/edgeql-parser-python/src/normalize.rs b/edb/edgeql-parser/edgeql-parser-python/src/normalize.rs index 9be90a12c14..a1c96ec564f 100644 --- a/edb/edgeql-parser/edgeql-parser-python/src/normalize.rs +++ b/edb/edgeql-parser/edgeql-parser-python/src/normalize.rs @@ -155,7 +155,7 @@ pub fn normalize(text: &str) -> Result { all_variables.push(variables); let processed_source = serialize_tokens(&rewritten_tokens[..]); - return Ok(Entry { + Ok(Entry { hash: hash(&processed_source), processed_source, named_args, @@ -166,7 +166,7 @@ pub fn normalize(text: &str) -> Result { }, tokens: rewritten_tokens, variables: all_variables, - }); + }) } fn is_operator(token: &Token) -> bool { @@ -200,7 +200,7 @@ fn serialize_tokens(tokens: &[Token]) -> String { buf.push_str(&token.text); needs_space = !is_operator(token); } - return buf; + buf } fn scan_vars<'x, 'y: 'x, I>(tokens: I) -> Option<(bool, usize)> @@ -233,7 +233,7 @@ where fn hash(text: &str) -> [u8; 64] { let mut result = [0u8; 64]; result.copy_from_slice(&Blake2b512::new_with_prefix(text.as_bytes()).finalize()); - return result; + result } /// Produces tokens corresponding to ($var) @@ -253,17 +253,17 @@ mod test { use super::scan_vars; use edgeql_parser::tokenizer::{Token, Tokenizer}; - fn tokenize<'x>(s: &'x str) -> Vec { + fn tokenize(s: &str) -> Vec { let mut r = Vec::new(); let mut s = Tokenizer::new(s); loop { match s.next() { - Some(Ok(x)) => r.push(x.into()), + Some(Ok(x)) => r.push(x), None => break, Some(Err(e)) => panic!("Parse error at {}: {}", s.current_pos(), e.message), } } - return r; + r } #[test] diff --git a/edb/edgeql-parser/edgeql-parser-python/src/parser.rs b/edb/edgeql-parser/edgeql-parser-python/src/parser.rs index 1fc73d5d60a..37459262259 100644 --- a/edb/edgeql-parser/edgeql-parser-python/src/parser.rs +++ b/edb/edgeql-parser/edgeql-parser-python/src/parser.rs @@ -1,20 +1,23 @@ -use cpython::exc::AssertionError; -use cpython::{ - ObjectProtocol, PyClone, PyInt, PyList, PyNone, PyObject, PyResult, PyString, PyTuple, Python, - PythonObject, PythonObjectWithCheckedDowncast, ToPyObject, -}; use once_cell::sync::OnceCell; use edgeql_parser::parser; +use pyo3::exceptions::PyAssertionError; +use pyo3::prelude::*; +use pyo3::types::{PyList, PyString, PyTuple}; use crate::errors::{parser_error_into_tuple, ParserResult}; use crate::pynormalize::value_to_py_object; use crate::tokenizer::OpaqueToken; -pub fn parse(py: Python, start_token_name: &PyString, tokens: PyObject) -> PyResult { - let start_token_name = start_token_name.to_string(py).unwrap(); +#[pyfunction] +pub fn parse( + py: Python, + start_token_name: &PyString, + tokens: PyObject, +) -> PyResult<(ParserResult, PyObject)> { + let start_token_name = start_token_name.to_string(); - let (spec, productions) = get_spec(py)?; + let (spec, productions) = get_spec()?; let tokens = downcast_tokens(py, &start_token_name, tokens)?; @@ -29,69 +32,56 @@ pub fn parse(py: Python, start_token_name: &PyString, tokens: PyObject) -> PyRes .collect::>(); let errors = PyList::new(py, &errors); - let res = ParserResult::create_instance(py, cst.into_py_object(py), errors)?; + let res = ParserResult { + out: cst.into_py(py), + errors: errors.into(), + }; - Ok((res, productions).into_py_object(py)) + Ok((res, productions.clone())) } -py_class!(pub class CSTNode |py| { - data _production: PyObject; - data _terminal: PyObject; - - def production(&self) -> PyResult { - Ok(self._production(py).clone_ref(py)) - } - def terminal(&self) -> PyResult { - Ok(self._terminal(py).clone_ref(py)) - } -}); - -py_class!(pub class Production |py| { - data _id: PyInt; - data _args: PyList; - - def id(&self) -> PyResult { - Ok(self._id(py).clone_ref(py)) - } - def args(&self) -> PyResult { - Ok(self._args(py).clone_ref(py)) - } -}); +#[pyclass] +pub struct CSTNode { + #[pyo3(get)] + production: PyObject, + #[pyo3(get)] + terminal: PyObject, +} -py_class!(pub class Terminal |py| { - data _text: PyString; - data _value: PyObject; - data _start: u64; - data _end: u64; +#[pyclass] +pub struct Production { + #[pyo3(get)] + id: usize, + #[pyo3(get)] + args: PyObject, +} - def text(&self) -> PyResult { - Ok(self._text(py).clone_ref(py)) - } - def value(&self) -> PyResult { - Ok(self._value(py).clone_ref(py)) - } - def start(&self) -> PyResult { - Ok(*self._start(py)) - } - def end(&self) -> PyResult { - Ok(*self._end(py)) - } -}); +#[pyclass] +pub struct Terminal { + #[pyo3(get)] + text: String, + #[pyo3(get)] + value: PyObject, + #[pyo3(get)] + start: u64, + #[pyo3(get)] + end: u64, +} static PARSER_SPECS: OnceCell<(parser::Spec, PyObject)> = OnceCell::new(); -fn downcast_tokens<'a>( +fn downcast_tokens( py: Python, start_token_name: &str, token_list: PyObject, ) -> PyResult> { - let tokens = PyList::downcast_from(py, token_list)?; + let tokens: &PyList = token_list.downcast(py)?; - let mut buf = Vec::with_capacity(tokens.len(py) + 1); + let mut buf = Vec::with_capacity(tokens.len() + 1); buf.push(parser::Terminal::from_start_name(start_token_name)); - for token in tokens.iter(py) { - let token = OpaqueToken::downcast_from(py, token)?; - let token = token.inner(py); + for token in tokens.iter() { + let token: &PyCell = token.downcast()?; + let token = token.borrow().inner.clone(); buf.push(parser::Terminal::from_token(token)); } @@ -105,25 +95,23 @@ fn downcast_tokens<'a>( Ok(buf) } -fn get_spec(py: Python) -> Result<&'static (parser::Spec, PyObject), cpython::PyErr> { +fn get_spec() -> PyResult<&'static (parser::Spec, PyObject)> { if let Some(x) = PARSER_SPECS.get() { - return Ok(x); + Ok(x) } else { - return Err(cpython::PyErr::new::( - py, - ("grammar spec not loaded",), - )); + Err(PyAssertionError::new_err(("grammar spec not loaded",))) } } /// Loads the grammar specification from file and caches it in memory. -pub fn preload_spec(py: Python, spec_filepath: &PyString) -> PyResult { +#[pyfunction] +pub fn preload_spec(py: Python, spec_filepath: &PyString) -> PyResult<()> { if PARSER_SPECS.get().is_some() { - return Ok(PyNone); + return Ok(()); } - let spec_filepath = spec_filepath.to_string(py)?; - let bytes = std::fs::read(spec_filepath.as_ref()) + let spec_filepath = spec_filepath.to_string(); + let bytes = std::fs::read(&spec_filepath) .unwrap_or_else(|e| panic!("Cannot read grammar spec from {spec_filepath} ({e})")); let spec: parser::Spec = bitcode::deserialize::(&bytes) @@ -132,21 +120,22 @@ pub fn preload_spec(py: Python, spec_filepath: &PyString) -> PyResult { let productions = load_productions(py, &spec)?; let _ = PARSER_SPECS.set((spec, productions)); - Ok(PyNone) + Ok(()) } /// Serialize the grammar specification and write it to a file. /// /// Called from setup.py. -pub fn save_spec(py: Python, spec_json: &PyString, dst: &PyString) -> PyResult { - let spec_json = spec_json.to_string(py).unwrap(); +#[pyfunction] +pub fn save_spec(spec_json: &PyString, dst: &PyString) -> PyResult<()> { + let spec_json = spec_json.to_string(); let spec: parser::SpecSerializable = serde_json::from_str(&spec_json).unwrap(); let spec_bitcode = bitcode::serialize(&spec).unwrap(); - let dst = dst.to_string(py)?.to_string(); + let dst = dst.to_string(); std::fs::write(dst, spec_bitcode).ok().unwrap(); - Ok(PyNone) + Ok(()) } fn load_productions(py: Python<'_>, spec: &parser::Spec) -> PyResult { @@ -154,47 +143,53 @@ fn load_productions(py: Python<'_>, spec: &parser::Spec) -> PyResult { let grammar_mod = py.import(grammar_name)?; let load_productions = py .import("edb.common.parsing")? - .get(py, "load_spec_productions")?; + .getattr("load_spec_productions")?; - let productions = load_productions.call(py, (&spec.production_names, grammar_mod), None)?; - Ok(productions) + let production_names: Vec<_> = spec + .production_names + .iter() + .map(|(a, b)| PyTuple::new(py, [a, b])) + .collect(); + + let productions = load_productions.call((production_names, grammar_mod), None)?; + Ok(productions.into()) } fn to_py_cst<'a>(cst: &'a parser::CSTNode<'a>, py: Python) -> PyResult { - match cst { - parser::CSTNode::Empty => CSTNode::create_instance(py, py.None(), py.None()), - parser::CSTNode::Terminal(token) => CSTNode::create_instance( - py, - py.None(), - Terminal::create_instance( - py, - token.text.to_py_object(py), - if let Some(val) = &token.value { + Ok(match cst { + parser::CSTNode::Empty => CSTNode { + production: py.None(), + terminal: py.None(), + }, + parser::CSTNode::Terminal(token) => CSTNode { + production: py.None(), + terminal: Terminal { + text: token.text.clone(), + value: if let Some(val) = &token.value { value_to_py_object(py, val)? } else { py.None() }, - token.span.start, - token.span.end, - )? - .into_object(), - ), - parser::CSTNode::Production(prod) => CSTNode::create_instance( - py, - Production::create_instance( - py, - prod.id.into_py_object(py), - PyList::new( + start: token.span.start, + end: token.span.end, + } + .into_py(py), + }, + parser::CSTNode::Production(prod) => CSTNode { + production: Production { + id: prod.id, + args: PyList::new( py, prod.args .iter() - .map(|a| to_py_cst(a, py).map(|x| x.into_object())) + .map(|a| to_py_cst(a, py).map(|x| x.into_py(py))) .collect::>>()? .as_slice(), - ), - )? - .into_object(), - py.None(), - ), - } + ) + .into(), + } + .into_py(py), + terminal: py.None(), + }, + }) } diff --git a/edb/edgeql-parser/edgeql-parser-python/src/position.rs b/edb/edgeql-parser/edgeql-parser-python/src/position.rs index fecd0318bcb..5954e06f965 100644 --- a/edb/edgeql-parser/edgeql-parser-python/src/position.rs +++ b/edb/edgeql-parser/edgeql-parser-python/src/position.rs @@ -1,48 +1,62 @@ - -use cpython::exc::{RuntimeError, IndexError}; -use cpython::{py_class, PyErr, PyResult, PyInt, PyBytes, PyList, ToPyObject}; -use cpython::{Python, PyObject}; +use pyo3::{ + exceptions::{PyIndexError, PyRuntimeError}, + prelude::*, + types::PyBytes, +}; use edgeql_parser::position::InflatedPos; -py_class!(pub class SourcePoint |py| { - data _position: InflatedPos; - @classmethod def from_offsets(_cls, data: PyBytes, offsets: PyObject) - -> PyResult - { +#[pyclass] +pub struct SourcePoint { + _position: InflatedPos, +} + +#[pymethods] +impl SourcePoint { + #[staticmethod] + fn from_offsets(py: Python, data: &PyBytes, offsets: PyObject) -> PyResult { let mut list: Vec = offsets.extract(py)?; - let data: &[u8] = data.data(py); + let data: &[u8] = data.as_bytes(); list.sort(); let result = InflatedPos::from_offsets(data, &list) - .map_err(|e| PyErr::new::(py, e.to_string()))?; - Ok(result.into_iter() - .map(|pos| SourcePoint::create_instance(py, pos)) - .collect::, _>>()? - .to_py_object(py)) + .map_err(|e| PyRuntimeError::new_err(e.to_string()))?; + + Ok(result + .into_iter() + .map(|_position| SourcePoint { _position }) + .collect::>() + .into_py(py)) } - @property def line(&self) -> PyResult { - Ok((self._position(py).line + 1).to_py_object(py)) + + #[getter] + fn line(&self) -> u64 { + self._position.line + 1 } - @property def zero_based_line(&self) -> PyResult { - Ok((self._position(py).line).to_py_object(py)) + #[getter] + fn zero_based_line(&self) -> u64 { + self._position.line } - @property def column(&self) -> PyResult { - Ok((self._position(py).column + 1).to_py_object(py)) + #[getter] + fn column(&self) -> u64 { + self._position.column + 1 } - @property def utf16column(&self) -> PyResult { - Ok((self._position(py).utf16column).to_py_object(py)) + #[getter] + fn utf16column(&self) -> u64 { + self._position.utf16column } - @property def offset(&self) -> PyResult { - Ok((self._position(py).offset).to_py_object(py)) + #[getter] + fn offset(&self) -> u64 { + self._position.offset } - @property def char_offset(&self) -> PyResult { - Ok((self._position(py).char_offset).to_py_object(py)) + #[getter] + fn char_offset(&self) -> u64 { + self._position.char_offset } -}); +} fn _offset_of_line(text: &str, target: usize) -> Option { let mut was_lf = false; - let mut line = 0; // this assumes line found by rfind + let mut line = 0; // this assumes line found by rfind for (idx, &byte) in text.as_bytes().iter().enumerate() { if line >= target { return Some(idx); @@ -74,13 +88,11 @@ fn _offset_of_line(text: &str, target: usize) -> Option { Some(text.len()) } -pub fn offset_of_line(py: Python, text: &str, target: usize) -> PyResult -{ +#[pyfunction] +pub fn offset_of_line(text: &str, target: usize) -> PyResult { match _offset_of_line(text, target) { Some(offset) => Ok(offset), - None => { - Err(PyErr::new::(py, "line number is too large")) - } + None => Err(PyIndexError::new_err("line number is too large")), } } diff --git a/edb/edgeql-parser/edgeql-parser-python/src/pynormalize.rs b/edb/edgeql-parser/edgeql-parser-python/src/pynormalize.rs index 95bd1bb1724..feaa66a6b94 100644 --- a/edb/edgeql-parser/edgeql-parser-python/src/pynormalize.rs +++ b/edb/edgeql-parser/edgeql-parser-python/src/pynormalize.rs @@ -1,64 +1,98 @@ use std::convert::TryFrom; use bigdecimal::Num; -use cpython::exc::AssertionError; -use cpython::{PyBytes, PyErr, PyInt, PythonObject, ToPyObject}; -use cpython::{PyClone, PyDict, PyList, PyResult, PyString, Python}; -use cpython::{PyFloat, PyObject}; use bytes::{BufMut, Bytes, BytesMut}; use edgedb_protocol::codec; use edgedb_protocol::model::{BigInt, Decimal}; use edgeql_parser::tokenizer::Value; +use pyo3::exceptions::PyAssertionError; +use pyo3::prelude::*; +use pyo3::types::{PyBytes, PyDict, PyFloat, PyList, PyLong, PyString}; use crate::errors::SyntaxError; use crate::normalize::{normalize as _normalize, Error, Variable}; use crate::tokenizer::tokens_to_py; -py_class!(pub class Entry |py| { - data _key: PyBytes; - data _processed_source: String; - data _tokens: PyList; - data _extra_blobs: PyList; - data _extra_named: bool; - data _first_extra: Option; - data _extra_counts: PyList; - data _variables: Vec>; - def key(&self) -> PyResult { - Ok(self._key(py).clone_ref(py)) +#[pyfunction] +pub fn normalize(py: Python<'_>, text: &PyString) -> PyResult { + let text = text.to_string(); + match _normalize(&text) { + Ok(entry) => { + let blobs = + serialize_all(py, &entry.variables).map_err(PyAssertionError::new_err)?; + let counts: Vec<_> = entry + .variables + .iter() + .map(|x| x.len().into_py(py)) + .collect(); + + Ok(Entry { + key: PyBytes::new(py, &entry.hash[..]).into(), + tokens: tokens_to_py(py, entry.tokens)?, + extra_blobs: blobs.into(), + extra_named: entry.named_args, + first_extra: entry.first_arg, + extra_counts: PyList::new(py, &counts[..]).into(), + variables: entry.variables, + }) + } + Err(Error::Tokenizer(msg, pos)) => { + Err(SyntaxError::new_err(( + msg, + (pos, py.None()), + py.None(), + py.None(), + ))) + } + Err(Error::Assertion(msg, pos)) => { + Err(PyAssertionError::new_err(format!("{}: {}", pos, msg))) + } } - def variables(&self) -> PyResult { +} + +#[pyclass] +pub struct Entry { + #[pyo3(get)] + key: PyObject, + + #[pyo3(get)] + tokens: PyObject, + + #[pyo3(get)] + extra_blobs: PyObject, + + extra_named: bool, + + #[pyo3(get)] + first_extra: Option, + + #[pyo3(get)] + extra_counts: PyObject, + + variables: Vec>, +} + +#[pymethods] +impl Entry { + fn get_variables(&self, py: Python) -> PyResult { let vars = PyDict::new(py); - let named = *self._extra_named(py); - let first = match self._first_extra(py) { + let first = match self.first_extra { Some(first) => first, - None => return Ok(vars), + None => return Ok(vars.to_object(py)), }; - for (idx, var) in self._variables(py).iter().flatten().enumerate() { - let s = if named { + for (idx, var) in self.variables.iter().flatten().enumerate() { + let s = if self.extra_named { format!("__edb_arg_{}", first + idx) } else { (first + idx).to_string() }; - vars.set_item( - py, s.to_py_object(py), value_to_py_object(py, &var.value)? - )?; + vars.set_item(s.into_py(py), value_to_py_object(py, &var.value)?)?; } - Ok(vars) - } - def tokens(&self) -> PyResult { - Ok(self._tokens(py).clone_ref(py)) - } - def first_extra(&self) -> PyResult> { - Ok(self._first_extra(py).map(|x| x.to_py_object(py))) - } - def extra_counts(&self) -> PyResult { - Ok(self._extra_counts(py).to_py_object(py)) - } - def extra_blobs(&self) -> PyResult { - Ok(self._extra_blobs(py).clone_ref(py)) + + Ok(vars.to_object(py)) } -}); +} pub fn serialize_extra(variables: &[Variable]) -> Result { use edgedb_protocol::codec::Codec; @@ -122,64 +156,32 @@ pub fn serialize_extra(variables: &[Variable]) -> Result { Ok(buf.freeze()) } -pub fn serialize_all(py: Python<'_>, variables: &[Vec]) -> Result { +pub fn serialize_all<'a>( + py: Python<'a>, + variables: &[Vec], +) -> Result<&'a PyList, String> { let mut buf = Vec::with_capacity(variables.len()); for vars in variables { let bytes = serialize_extra(vars)?; - let pybytes = PyBytes::new(py, &bytes).into_object(); + let pybytes = PyBytes::new(py, &bytes).as_ref(); buf.push(pybytes); } - Ok(PyList::new(py, &buf[..])) -} - -pub fn normalize(py: Python<'_>, text: &PyString) -> PyResult { - let text = text.to_string(py)?; - match _normalize(&text) { - Ok(entry) => { - let blobs = serialize_all(py, &entry.variables) - .map_err(|e| PyErr::new::(py, e))?; - let counts: Vec<_> = entry - .variables - .iter() - .map(|x| x.len().to_py_object(py).into_object()) - .collect(); - - Ok(Entry::create_instance( - py, - /* key: */ PyBytes::new(py, &entry.hash[..]), - /* processed_source: */ entry.processed_source, - /* tokens: */ tokens_to_py(py, entry.tokens)?, - /* extra_blobs: */ blobs, - /* extra_named: */ entry.named_args, - /* first_extra: */ entry.first_arg, - /* extra_counts: */ PyList::new(py, &counts[..]), - /* variables: */ entry.variables, - )?) - } - Err(Error::Tokenizer(msg, pos)) => { - return Err(SyntaxError::new( - py, - (msg, (pos, py.None()), py.None(), py.None()), - )) - } - Err(Error::Assertion(msg, pos)) => { - return Err(PyErr::new::( - py, - format!("{}: {}", pos, msg), - )); - } - } + Ok(PyList::new(py, buf.as_slice())) } pub fn value_to_py_object(py: Python, val: &Value) -> PyResult { Ok(match val { - Value::Int(v) => v.to_py_object(py).into_object(), - Value::String(v) => v.to_py_object(py).into_object(), - Value::Float(v) => v.to_py_object(py).into_object(), + Value::Int(v) => v.into_py(py), + Value::String(v) => v.into_py(py), + Value::Float(v) => v.into_py(py), Value::BigInt(v) => py - .get_type::() - .call(py, (v, 16.to_py_object(py)), None)?, - Value::Decimal(v) => py.get_type::().call(py, (v.to_string(),), None)?, - Value::Bytes(v) => PyBytes::new(py, v).into_object(), + .get_type::() + .call((v, 16.into_py(py)), None)? + .into(), + Value::Decimal(v) => py + .get_type::() + .call((v.to_string(),), None)? + .into(), + Value::Bytes(v) => PyBytes::new(py, v).into(), }) } diff --git a/edb/edgeql-parser/edgeql-parser-python/src/tokenizer.rs b/edb/edgeql-parser/edgeql-parser-python/src/tokenizer.rs index 0a9910a314e..bb0956d4a92 100644 --- a/edb/edgeql-parser/edgeql-parser-python/src/tokenizer.rs +++ b/edb/edgeql-parser/edgeql-parser-python/src/tokenizer.rs @@ -1,13 +1,13 @@ -use cpython::{PyBytes, PyClone, PyResult, PyString, Python, PythonObject}; -use cpython::{PyList, PyObject, PyTuple, ToPyObject}; - use edgeql_parser::tokenizer::{Token, Tokenizer}; use once_cell::sync::OnceCell; +use pyo3::prelude::*; +use pyo3::types::{PyBytes, PyList, PyString}; use crate::errors::{parser_error_into_tuple, ParserResult}; +#[pyfunction] pub fn tokenize(py: Python, s: &PyString) -> PyResult { - let data = s.to_string(py)?; + let data = s.to_string(); let mut token_stream = Tokenizer::new(&data[..]).validated_values().with_eof(); @@ -28,68 +28,69 @@ pub fn tokenize(py: Python, s: &PyString) -> PyResult { let tokens = tokens_to_py(py, tokens)?; - let errors = PyList::new(py, errors.as_slice()).to_py_object(py); + let errors = PyList::new(py, errors.as_slice()).into_py(py); - ParserResult::create_instance(py, tokens.into_object(), errors) + Ok(ParserResult { + out: tokens.into_py(py), + errors, + }) } // An opaque wrapper around [edgeql_parser::tokenizer::Token]. // Supports Python pickle serialization. -py_class!(pub class OpaqueToken |py| { - data _inner: Token<'static>; +#[pyclass] +pub struct OpaqueToken { + pub inner: Token<'static>, +} - def __repr__(&self) -> PyResult { - Ok(PyString::new(py, &self._inner(py).to_string())) +#[pymethods] +impl OpaqueToken { + fn __repr__(&self) -> PyResult { + Ok(self.inner.to_string()) } - def __reduce__(&self) -> PyResult { - let data = bitcode::serialize(self._inner(py)).unwrap(); - - return Ok(( - get_fn_unpickle_token(py), - ( - PyBytes::new(py, &data), - ), - ).to_py_object(py)) + fn __reduce__(&self, py: Python) -> PyResult<(PyObject, (PyObject,))> { + let data = bitcode::serialize(&self.inner).unwrap(); + + let tok = get_unpickle_token_fn(py); + Ok((tok, (PyBytes::new(py, &data).to_object(py),))) } -}); +} -pub fn tokens_to_py(py: Python, rust_tokens: Vec) -> PyResult { +pub fn tokens_to_py(py: Python<'_>, rust_tokens: Vec) -> PyResult { let mut buf = Vec::with_capacity(rust_tokens.len()); for tok in rust_tokens { - let py_tok = OpaqueToken::create_instance(py, tok.cloned())?.into_object(); + let py_tok = OpaqueToken { + inner: tok.cloned(), + }; - buf.push(py_tok); + buf.push(py_tok.into_py(py)); } - Ok(PyList::new(py, &buf[..])) + Ok(PyList::new(py, &buf[..]).into_py(py)) } /// To support pickle serialization of OpaqueTokens, we need to provide a /// deserialization function in __reduce__ methods. /// This function must not be inlined and must be globally accessible. /// To achieve this, we expose it a part of the module definition -/// (`_unpickle_token`) and save reference to is in the `FN_UNPICKLE_TOKEN`. +/// (`unpickle_token`) and save reference to is in the `FN_UNPICKLE_TOKEN`. /// /// A bit hackly, but it works. static FN_UNPICKLE_TOKEN: OnceCell = OnceCell::new(); -pub fn init_module(py: Python) { +pub fn fini_module(py: Python, m: &PyModule) { + let _unpickle_token = m.getattr("unpickle_token").unwrap(); FN_UNPICKLE_TOKEN - .set(py_fn!(py, _unpickle_token(bytes: &PyBytes))) + .set(_unpickle_token.to_object(py)) .expect("module is already initialized"); } -pub fn _unpickle_token(py: Python, bytes: &PyBytes) -> PyResult { - let token = bitcode::deserialize(bytes.data(py)).unwrap(); - OpaqueToken::create_instance(py, token) +#[pyfunction] +pub fn unpickle_token(bytes: &PyBytes) -> PyResult { + let token = bitcode::deserialize(bytes.as_bytes()).unwrap(); + Ok(OpaqueToken { inner: token }) } -pub fn get_fn_unpickle_token(py: Python) -> PyObject { - let py_function = FN_UNPICKLE_TOKEN.get().expect("module initialized"); - return py_function.clone_ref(py); -} - -impl OpaqueToken { - pub(super) fn inner(&self, py: Python) -> Token { - self._inner(py).clone() - } +fn get_unpickle_token_fn(py: Python) -> PyObject { + let py_function = FN_UNPICKLE_TOKEN.get().expect("module uninitialized"); + py_function.clone_ref(py) } diff --git a/edb/edgeql-parser/edgeql-parser-python/tests/normalize.rs b/edb/edgeql-parser/edgeql-parser-python/tests/normalize.rs index 5551322a723..da7fdabd3db 100644 --- a/edb/edgeql-parser/edgeql-parser-python/tests/normalize.rs +++ b/edb/edgeql-parser/edgeql-parser-python/tests/normalize.rs @@ -40,9 +40,9 @@ fn test_int() { #[test] fn test_str() { - let entry = normalize(r###" + let entry = normalize(r#" SELECT "x" + "yy" - "###).unwrap(); + "#).unwrap(); assert_eq!(entry.processed_source, "SELECT $0+$1"); assert_eq!(entry.variables, vec![vec![ diff --git a/edb/edgeql-parser/src/ast.rs b/edb/edgeql-parser/src/ast.rs index 768072d66b8..4c6a8d7d4eb 100644 --- a/edb/edgeql-parser/src/ast.rs +++ b/edb/edgeql-parser/src/ast.rs @@ -4,6 +4,7 @@ //! Abstract Syntax Tree for EdgeQL #![allow(non_camel_case_types)] +#![cfg(never)] // TODO: migrate cpython-rust to pyo3 use indexmap::IndexMap; diff --git a/edb/edgeql-parser/src/hash.rs b/edb/edgeql-parser/src/hash.rs index d435d2be4c8..be6ef6e3a12 100644 --- a/edb/edgeql-parser/src/hash.rs +++ b/edb/edgeql-parser/src/hash.rs @@ -22,7 +22,7 @@ impl Hasher { me.hasher.update(b"CREATE\0MIGRATION\0ONTO\0"); me.hasher.update(parent_id.as_bytes()); me.hasher.update(b"\0{\0"); - return me; + me } pub fn add_source(&mut self, data: &str) -> Result<&mut Self, Error> { let mut parser = &mut Tokenizer::new(data); @@ -44,7 +44,7 @@ impl Hasher { let hash = base32::encode( base32::Alphabet::RFC4648 { padding: false }, &self.hasher.finalize()); - return format!("m1{}", hash.to_ascii_lowercase()); + format!("m1{}", hash.to_ascii_lowercase()) } } @@ -55,7 +55,7 @@ mod test { fn hash(initial: &str, text: &str) -> String { let mut hasher = Hasher::start_migration(initial); hasher.add_source(text).unwrap(); - return hasher.make_migration_id(); + hasher.make_migration_id() } #[test] diff --git a/edb/edgeql-parser/src/helpers/bytes.rs b/edb/edgeql-parser/src/helpers/bytes.rs index b0670ec507e..08162cf7212 100644 --- a/edb/edgeql-parser/src/helpers/bytes.rs +++ b/edb/edgeql-parser/src/helpers/bytes.rs @@ -70,18 +70,18 @@ fn unquote_bytes_inner(s: &str) -> Result, String> { #[test] fn simple_bytes() { - assert_eq!(unquote_bytes_inner(r#"\x09"#).unwrap(), b"\x09"); - assert_eq!(unquote_bytes_inner(r#"\x0A"#).unwrap(), b"\x0A"); - assert_eq!(unquote_bytes_inner(r#"\x0D"#).unwrap(), b"\x0D"); - assert_eq!(unquote_bytes_inner(r#"\x20"#).unwrap(), b"\x20"); - assert_eq!(unquote_bytes(r#"b'\x09'"#).unwrap(), b"\x09"); - assert_eq!(unquote_bytes(r#"b'\x0A'"#).unwrap(), b"\x0A"); - assert_eq!(unquote_bytes(r#"b'\x0D'"#).unwrap(), b"\x0D"); - assert_eq!(unquote_bytes(r#"b'\x20'"#).unwrap(), b"\x20"); - assert_eq!(unquote_bytes(r#"br'\x09'"#).unwrap(), b"\\x09"); - assert_eq!(unquote_bytes(r#"br'\x0A'"#).unwrap(), b"\\x0A"); - assert_eq!(unquote_bytes(r#"br'\x0D'"#).unwrap(), b"\\x0D"); - assert_eq!(unquote_bytes(r#"br'\x20'"#).unwrap(), b"\\x20"); + assert_eq!(unquote_bytes_inner(r"\x09").unwrap(), b"\x09"); + assert_eq!(unquote_bytes_inner(r"\x0A").unwrap(), b"\x0A"); + assert_eq!(unquote_bytes_inner(r"\x0D").unwrap(), b"\x0D"); + assert_eq!(unquote_bytes_inner(r"\x20").unwrap(), b"\x20"); + assert_eq!(unquote_bytes(r"b'\x09'").unwrap(), b"\x09"); + assert_eq!(unquote_bytes(r"b'\x0A'").unwrap(), b"\x0A"); + assert_eq!(unquote_bytes(r"b'\x0D'").unwrap(), b"\x0D"); + assert_eq!(unquote_bytes(r"b'\x20'").unwrap(), b"\x20"); + assert_eq!(unquote_bytes(r"br'\x09'").unwrap(), b"\\x09"); + assert_eq!(unquote_bytes(r"br'\x0A'").unwrap(), b"\\x0A"); + assert_eq!(unquote_bytes(r"br'\x0D'").unwrap(), b"\\x0D"); + assert_eq!(unquote_bytes(r"br'\x20'").unwrap(), b"\\x20"); } #[test] @@ -118,8 +118,8 @@ aa \ #[test] fn complex_bytes() { - assert_eq!(unquote_bytes_inner(r#"\x09 hello \x0A there"#).unwrap(), + assert_eq!(unquote_bytes_inner(r"\x09 hello \x0A there").unwrap(), b"\x09 hello \x0A there"); - assert_eq!(unquote_bytes(r#"br'\x09 hello \x0A there'"#).unwrap(), + assert_eq!(unquote_bytes(r"br'\x09 hello \x0A there'").unwrap(), b"\\x09 hello \\x0A there"); } diff --git a/edb/edgeql-parser/src/helpers/strings.rs b/edb/edgeql-parser/src/helpers/strings.rs index 0cbd0c7c94d..b6ef9c874e3 100644 --- a/edb/edgeql-parser/src/helpers/strings.rs +++ b/edb/edgeql-parser/src/helpers/strings.rs @@ -32,7 +32,7 @@ pub fn quote_name(s: &str) -> Cow { s.push('`'); s.push_str(&escaped); s.push('`'); - return s.into(); + s.into() } pub fn quote_string(s: &str) -> String { @@ -57,7 +57,7 @@ pub fn quote_string(s: &str) -> String { } } buf.push('"'); - return buf; + buf } pub fn unquote_string(value: &str) -> Result, UnquoteError> { @@ -169,21 +169,21 @@ fn _unquote_string(s: &str) -> Result { #[test] fn unquote_unicode_string() { - assert_eq!(_unquote_string(r#"\x09"#).unwrap(), "\u{09}"); - assert_eq!(_unquote_string(r#"\u000A"#).unwrap(), "\u{000A}"); - assert_eq!(_unquote_string(r#"\u000D"#).unwrap(), "\u{000D}"); - assert_eq!(_unquote_string(r#"\u0020"#).unwrap(), "\u{0020}"); - assert_eq!(_unquote_string(r#"\uFFFF"#).unwrap(), "\u{FFFF}"); + assert_eq!(_unquote_string(r"\x09").unwrap(), "\u{09}"); + assert_eq!(_unquote_string(r"\u000A").unwrap(), "\u{000A}"); + assert_eq!(_unquote_string(r"\u000D").unwrap(), "\u{000D}"); + assert_eq!(_unquote_string(r"\u0020").unwrap(), "\u{0020}"); + assert_eq!(_unquote_string(r"\uFFFF").unwrap(), "\u{FFFF}"); } #[test] fn unquote_string_error() { - assert_eq!(_unquote_string(r#"\x00"#).unwrap_err(), + assert_eq!(_unquote_string(r"\x00").unwrap_err(), "invalid string literal: \ invalid escape sequence '\\x0' (only non-null ascii allowed)"); - assert_eq!(_unquote_string(r#"\u0000"#).unwrap_err(), + assert_eq!(_unquote_string(r"\u0000").unwrap_err(), "invalid string literal: invalid escape sequence '\\u0000'"); - assert_eq!(_unquote_string(r#"\U00000000"#).unwrap_err(), + assert_eq!(_unquote_string(r"\U00000000").unwrap_err(), "invalid string literal: invalid escape sequence '\\U00000000'"); } @@ -213,10 +213,10 @@ fn test_quote_string() { #[test] fn complex_strings() { - assert_eq!(_unquote_string(r#"\u0009 hello \u000A there"#).unwrap(), + assert_eq!(_unquote_string(r"\u0009 hello \u000A there").unwrap(), "\u{0009} hello \u{000A} there"); - assert_eq!(_unquote_string(r#"\x62:\u2665:\U000025C6"#).unwrap(), + assert_eq!(_unquote_string(r"\x62:\u2665:\U000025C6").unwrap(), "\u{62}:\u{2665}:\u{25C6}"); } diff --git a/edb/edgeql-parser/src/into_python.rs b/edb/edgeql-parser/src/into_python.rs index 928bbe8f03b..4f7bc8ac618 100644 --- a/edb/edgeql-parser/src/into_python.rs +++ b/edb/edgeql-parser/src/into_python.rs @@ -1,3 +1,4 @@ +#![cfg(never)] // TODO: migrate cpython-rust to pyo3 use indexmap::IndexMap; use cpython::{ diff --git a/edb/edgeql-parser/src/parser.rs b/edb/edgeql-parser/src/parser.rs index 6592207059d..1b7d6fb4d82 100644 --- a/edb/edgeql-parser/src/parser.rs +++ b/edb/edgeql-parser/src/parser.rs @@ -246,7 +246,9 @@ pub struct Reduce { /// must manually drop. This is why Terminal has a special vec arena that does /// Drop. #[derive(Debug, Clone, Copy)] +#[derive(Default)] pub enum CSTNode<'a> { + #[default] Empty, Terminal(&'a Terminal), Production(Production<'a>), @@ -565,11 +567,7 @@ impl std::fmt::Display for Terminal { } } -impl<'a> Default for CSTNode<'a> { - fn default() -> Self { - CSTNode::Empty - } -} + impl Terminal { pub fn from_token(token: Token) -> Self { diff --git a/edb/edgeql-parser/src/parser/custom_errors.rs b/edb/edgeql-parser/src/parser/custom_errors.rs index 9e23283746f..a4d42c761d9 100644 --- a/edb/edgeql-parser/src/parser/custom_errors.rs +++ b/edb/edgeql-parser/src/parser/custom_errors.rs @@ -250,7 +250,7 @@ impl<'s> Parser<'s> { /// D - X /// E /// ``` - fn compare_stack<'a>(&self, expected: &[Cond], top_offset: usize, ctx: &Context) -> bool { + fn compare_stack(&self, expected: &[Cond], top_offset: usize, ctx: &Context) -> bool { let mut current = self.get_from_top(top_offset); for validator in expected.iter().rev() { @@ -369,7 +369,7 @@ pub fn post_process(errors: Vec) -> Vec { { let last = new_errors.pop().unwrap(); let text = last.message.strip_prefix("Unexpected keyword '").unwrap(); - let text = text.strip_suffix("'").unwrap(); + let text = text.strip_suffix('\'').unwrap(); new_errors.push(unexpected_reserved_keyword(text, last.span)); continue; diff --git a/edb/edgeql-parser/src/position.rs b/edb/edgeql-parser/src/position.rs index cd35751c79d..a5d8bdd02b2 100644 --- a/edb/edgeql-parser/src/position.rs +++ b/edb/edgeql-parser/src/position.rs @@ -101,7 +101,7 @@ fn new_lines_in_fragment(data: &[u8]) -> u64 { } } } - return lines; + lines } impl InflatedPos { @@ -144,7 +144,7 @@ impl InflatedPos { char_offset: prefix_s.chars().count() as u64, }); } - return Ok(result); + Ok(result) } pub fn deflate(self) -> Pos { diff --git a/edb/edgeql-parser/src/preparser.rs b/edb/edgeql-parser/src/preparser.rs index 6c4286eec40..0440fb9e21f 100644 --- a/edb/edgeql-parser/src/preparser.rs +++ b/edb/edgeql-parser/src/preparser.rs @@ -154,11 +154,11 @@ pub fn full_statement(data: &[u8], continuation: Option) b'}' | b')' | b']' if braces_buf.last() == Some(b) => { braces_buf.pop(); } - b';' if braces_buf.len() == 0 => return Ok(idx+1), + b';' if braces_buf.is_empty() => return Ok(idx+1), _ => continue, } } - return Err(Continuation { position: data.len(), braces: braces_buf }); + Err(Continuation { position: data.len(), braces: braces_buf }) } /// Returns true if the text has no partial statements diff --git a/edb/edgeql-parser/src/schema_file.rs b/edb/edgeql-parser/src/schema_file.rs index 70a335be1de..a52962a4a2a 100644 --- a/edb/edgeql-parser/src/schema_file.rs +++ b/edb/edgeql-parser/src/schema_file.rs @@ -94,8 +94,8 @@ mod test { .map(|_| String::new()) .map_err(|e| { let s = e.to_string(); - assert!(s != ""); - return s + assert!(!s.is_empty()); + s }) .unwrap_or_else(|e| e) } diff --git a/edb/edgeql-parser/src/tokenizer.rs b/edb/edgeql-parser/src/tokenizer.rs index 75adb917528..b130e732ac0 100644 --- a/edb/edgeql-parser/src/tokenizer.rs +++ b/edb/edgeql-parser/src/tokenizer.rs @@ -291,63 +291,63 @@ impl<'a> Tokenizer<'a> { match cur_char { ':' => match iter.next() { - Some((_, '=')) => return Ok((Assign, 2)), - Some((_, ':')) => return Ok((Namespace, 2)), - _ => return Ok((Colon, 1)), + Some((_, '=')) => Ok((Assign, 2)), + Some((_, ':')) => Ok((Namespace, 2)), + _ => Ok((Colon, 1)), }, '-' => match iter.next() { - Some((_, '>')) => return Ok((Arrow, 2)), - Some((_, '=')) => return Ok((SubAssign, 2)), - _ => return Ok((Sub, 1)), + Some((_, '>')) => Ok((Arrow, 2)), + Some((_, '=')) => Ok((SubAssign, 2)), + _ => Ok((Sub, 1)), }, '>' => match iter.next() { - Some((_, '=')) => return Ok((GreaterEq, 2)), - _ => return Ok((Greater, 1)), + Some((_, '=')) => Ok((GreaterEq, 2)), + _ => Ok((Greater, 1)), }, '<' => match iter.next() { - Some((_, '=')) => return Ok((LessEq, 2)), - _ => return Ok((Less, 1)), + Some((_, '=')) => Ok((LessEq, 2)), + _ => Ok((Less, 1)), }, '+' => match iter.next() { - Some((_, '=')) => return Ok((AddAssign, 2)), - Some((_, '+')) => return Ok((Concat, 2)), - _ => return Ok((Add, 1)), + Some((_, '=')) => Ok((AddAssign, 2)), + Some((_, '+')) => Ok((Concat, 2)), + _ => Ok((Add, 1)), }, '/' => match iter.next() { - Some((_, '/')) => return Ok((FloorDiv, 2)), - _ => return Ok((Div, 1)), + Some((_, '/')) => Ok((FloorDiv, 2)), + _ => Ok((Div, 1)), }, '.' => match iter.next() { - Some((_, '<')) => return Ok((BackwardLink, 2)), - _ => return Ok((Dot, 1)), + Some((_, '<')) => Ok((BackwardLink, 2)), + _ => Ok((Dot, 1)), }, '?' => match iter.next() { - Some((_, '?')) => return Ok((Coalesce, 2)), - Some((_, '=')) => return Ok((NotDistinctFrom, 2)), + Some((_, '?')) => Ok((Coalesce, 2)), + Some((_, '=')) => Ok((NotDistinctFrom, 2)), Some((_, '!')) => { if let Some((_, '=')) = iter.next() { - return Ok((DistinctFrom, 3)); + Ok((DistinctFrom, 3)) } else { - return Err(Error::new( + Err(Error::new( "`?!` is not an operator, \ did you mean `?!=` ?", - )); + )) } } _ => { - return Err(Error::new( + Err(Error::new( "Bare `?` is not an operator, \ did you mean `?=` or `??` ?", )) } }, '!' => match iter.next() { - Some((_, '=')) => return Ok((NotEq, 2)), + Some((_, '=')) => Ok((NotEq, 2)), _ => { - return Err(Error::new( + Err(Error::new( "Bare `!` is not an operator, \ did you mean `!=`?", - )); + )) } }, '"' | '\'' => self.parse_string(0, false, false), @@ -389,26 +389,26 @@ impl<'a> Tokenizer<'a> { } check_prohibited(c, false)?; } - return Err(Error::new("unterminated backtick name")); + Err(Error::new("unterminated backtick name")) } - '=' => return Ok((Eq, 1)), - ',' => return Ok((Comma, 1)), - '(' => return Ok((OpenParen, 1)), - ')' => return Ok((CloseParen, 1)), - '[' => return Ok((OpenBracket, 1)), - ']' => return Ok((CloseBracket, 1)), - '{' => return Ok((OpenBrace, 1)), - '}' => return Ok((CloseBrace, 1)), - ';' => return Ok((Semicolon, 1)), + '=' => Ok((Eq, 1)), + ',' => Ok((Comma, 1)), + '(' => Ok((OpenParen, 1)), + ')' => Ok((CloseParen, 1)), + '[' => Ok((OpenBracket, 1)), + ']' => Ok((CloseBracket, 1)), + '{' => Ok((OpenBrace, 1)), + '}' => Ok((CloseBrace, 1)), + ';' => Ok((Semicolon, 1)), '*' => match iter.next() { - Some((_, '*')) => return Ok((DoubleSplat, 2)), - _ => return Ok((Mul, 1)), + Some((_, '*')) => Ok((DoubleSplat, 2)), + _ => Ok((Mul, 1)), }, - '%' => return Ok((Modulo, 1)), - '^' => return Ok((Pow, 1)), - '&' => return Ok((Ampersand, 1)), - '|' => return Ok((Pipe, 1)), - '@' => return Ok((At, 1)), + '%' => Ok((Modulo, 1)), + '^' => Ok((Pow, 1)), + '&' => Ok((Ampersand, 1)), + '|' => Ok((Pipe, 1)), + '@' => Ok((At, 1)), c if c == '_' || c.is_alphabetic() => { let end_idx = loop { match iter.next() { @@ -446,7 +446,7 @@ impl<'a> Tokenizer<'a> { }; let val = &tail[..end_idx]; if let Some(keyword) = self.as_keyword(val) { - return Ok((Keyword(keyword), end_idx)); + Ok((Keyword(keyword), end_idx)) } else if val.starts_with("__") && val.ends_with("__") { return Err(Error::new( "identifiers surrounded by double \ @@ -589,7 +589,7 @@ impl<'a> Tokenizer<'a> { ))); } } - return Ok((Parameter, end_idx)); + Ok((Parameter, end_idx)) } '\\' => match iter.next() { Some((_, '(')) => { @@ -815,9 +815,9 @@ impl<'a> Tokenizer<'a> { let suffix = &self.buf[self.off + soff..self.off + end]; if suffix == "n" { if decimal { - return Ok((DecimalConst, end)); + Ok((DecimalConst, end)) } else { - return Ok((BigIntConst, end)); + Ok((BigIntConst, end)) } } else { let suffix = if suffix.len() > 8 { @@ -919,7 +919,7 @@ impl<'a> Tokenizer<'a> { self.keyword_buf.clear(); self.keyword_buf.push_str(s); self.keyword_buf.make_ascii_lowercase(); - return keywords::lookup_all(&self.keyword_buf); + keywords::lookup_all(&self.keyword_buf) } } diff --git a/edb/edgeql-parser/src/validation.rs b/edb/edgeql-parser/src/validation.rs index 5059af31f8a..c50e69cf172 100644 --- a/edb/edgeql-parser/src/validation.rs +++ b/edb/edgeql-parser/src/validation.rs @@ -123,7 +123,7 @@ impl<'a> Validator<'a> { } _ => {} } - return None; + None } fn peek_keyword(&mut self, kw: &'static str) -> bool { @@ -151,7 +151,7 @@ pub fn parse_value(token: &Token) -> Result, String> { } DecimalConst => { return text[..text.len() - 1] - .replace("_", "") + .replace('_', "") .parse() .map(Value::Decimal) .map(Some) @@ -159,7 +159,7 @@ pub fn parse_value(token: &Token) -> Result, String> { } FloatConst => { return text - .replace("_", "") + .replace('_', "") .parse::() .map_err(|e| format!("can't parse std::float64: {}", e)) .and_then(|num| { @@ -185,13 +185,13 @@ pub fn parse_value(token: &Token) -> Result, String> { // i64 as absolute (positive) value. // Python has no problem of representing such a positive // value, though. - return u64::from_str(&text.replace("_", "")) + return u64::from_str(&text.replace('_', "")) .map(|x| Some(Value::Int(x as i64))) .map_err(|e| format!("error reading int: {}", e)); } BigIntConst => { return text[..text.len() - 1] - .replace("_", "") + .replace('_', "") .parse::() .map_err(|e| format!("error reading bigint: {}", e)) // this conversion to decimal and back to string diff --git a/edb/edgeql-parser/tests/preparser.rs b/edb/edgeql-parser/tests/preparser.rs index fa6ef895953..38a66c07fac 100644 --- a/edb/edgeql-parser/tests/preparser.rs +++ b/edb/edgeql-parser/tests/preparser.rs @@ -36,7 +36,7 @@ fn test_raw_string() { #[test] fn test_raw_byte_string() { test_statement(br#"select rb"\"; some trailer"#, 13); - test_statement(br#"select br'hello\'; some trailer"#, 18); + test_statement(br"select br'hello\'; some trailer", 18); } #[test] diff --git a/edb/edgeql-parser/tests/tokenizer.rs b/edb/edgeql-parser/tests/tokenizer.rs index f8f520e6604..96b62bf3d9b 100644 --- a/edb/edgeql-parser/tests/tokenizer.rs +++ b/edb/edgeql-parser/tests/tokenizer.rs @@ -11,7 +11,7 @@ fn tok_str(s: &str) -> Vec { Some(Err(e)) => panic!("Parse error at {}: {}", e.span.start, e.message), } } - return r; + r } fn tok_typ(s: &str) -> Vec { @@ -24,7 +24,7 @@ fn tok_typ(s: &str) -> Vec { Some(Err(e)) => panic!("Parse error at {}: {}", e.span.start, e.message), } } - return r; + r } fn tok_err(s: &str) -> String { @@ -631,22 +631,22 @@ fn strings() { assert_eq!(tok_str(r#" "h\"ello" "#), [r#""h\"ello""#]); assert_eq!(tok_typ(r#" "h\"ello" "#), [Str]); - assert_eq!(tok_str(r#" 'h\'ello' "#), [r#"'h\'ello'"#]); - assert_eq!(tok_typ(r#" 'h\'ello' "#), [Str]); + assert_eq!(tok_str(r" 'h\'ello' "), [r"'h\'ello'"]); + assert_eq!(tok_typ(r" 'h\'ello' "), [Str]); assert_eq!(tok_str(r#" r"hello\" "#), [r#"r"hello\""#]); assert_eq!(tok_typ(r#" r"hello\" "#), [Str]); - assert_eq!(tok_str(r#" r'hello\' "#), [r#"r'hello\'"#]); - assert_eq!(tok_typ(r#" r'hello\' "#), [Str]); + assert_eq!(tok_str(r" r'hello\' "), [r"r'hello\'"]); + assert_eq!(tok_typ(r" r'hello\' "), [Str]); assert_eq!(tok_str(r#" b"h\"ello" "#), [r#"b"h\"ello""#]); assert_eq!(tok_typ(r#" b"h\"ello" "#), [BinStr]); - assert_eq!(tok_str(r#" b'h\'ello' "#), [r#"b'h\'ello'"#]); - assert_eq!(tok_typ(r#" b'h\'ello' "#), [BinStr]); + assert_eq!(tok_str(r" b'h\'ello' "), [r"b'h\'ello'"]); + assert_eq!(tok_typ(r" b'h\'ello' "), [BinStr]); assert_eq!(tok_str(r#" rb"hello\" "#), [r#"rb"hello\""#]); assert_eq!(tok_typ(r#" rb"hello\" "#), [BinStr]); - assert_eq!(tok_str(r#" rb'hello\' "#), [r#"rb'hello\'"#]); - assert_eq!(tok_typ(r#" rb'hello\' "#), [BinStr]); - assert_eq!(tok_str(r#" `hello\` "#), [r#"`hello\`"#]); - assert_eq!(tok_typ(r#" `hello\` "#), [Ident]); + assert_eq!(tok_str(r" rb'hello\' "), [r"rb'hello\'"]); + assert_eq!(tok_typ(r" rb'hello\' "), [BinStr]); + assert_eq!(tok_str(r" `hello\` "), [r"`hello\`"]); + assert_eq!(tok_typ(r" `hello\` "), [Ident]); assert_eq!(tok_str(r#" `hel``lo` "#), [r#"`hel``lo`"#]); assert_eq!(tok_typ(r#" `hel``lo` "#), [Ident]); diff --git a/edb/edgeql/parser/__init__.py b/edb/edgeql/parser/__init__.py index d61cc25a1ce..c80e034e505 100644 --- a/edb/edgeql/parser/__init__.py +++ b/edb/edgeql/parser/__init__.py @@ -135,7 +135,7 @@ def parse( start_name = start_token.__name__[2:] result, productions = rust_parser.parse(start_name, source.tokens()) - if len(result.errors()) > 0: + if len(result.errors) > 0: # TODO: emit multiple errors # Heuristic to pick the error: @@ -143,7 +143,7 @@ def parse( # - first encountered, # - Unexpected before Missing, # - original order. - errs: List[ParserError] = result.errors() + errs: List[ParserError] = result.errors unexpected = [e for e in errs if e[0].startswith('Unexpected')] if ( len(unexpected) == 1 @@ -173,7 +173,7 @@ def parse( ) return _cst_to_ast( - result.out(), + result.out, productions, source, filename, @@ -206,26 +206,26 @@ def _cst_to_ast( if isinstance(node, rust_parser.CSTNode): # this would be the body of the original recursion function - if terminal := node.terminal(): + if terminal := node.terminal: # Terminal is simple: just convert to parsing.Token context = parsing.ParserContext( name=filename, buffer=source.text(), - start=terminal.start(), - end=terminal.end(), + start=terminal.start, + end=terminal.end, ) result.append( parsing.Token( - terminal.text(), terminal.value(), context + terminal.text, terminal.value, context ) ) - elif production := node.production(): + elif production := node.production: # Production needs to first process all args, then # call the appropriate method. # (this is all in reverse, because stacks) stack.append(production) - args = list(production.args()) + args = list(production.args) args.reverse() stack.extend(args) else: @@ -233,13 +233,13 @@ def _cst_to_ast( elif isinstance(node, rust_parser.Production): # production args are done, get them out of result stack - len_args = len(node.args()) + len_args = len(node.args) split_at = len(result) - len_args args = result[split_at:] result = result[0:split_at] # find correct method to call - production_id = node.id() + production_id = node.id production = productions[production_id] non_term_type, method = production diff --git a/edb/edgeql/tokenizer.py b/edb/edgeql/tokenizer.py index b106edc7582..f6df7219f3c 100644 --- a/edb/edgeql/tokenizer.py +++ b/edb/edgeql/tokenizer.py @@ -68,12 +68,12 @@ def __repr__(self): class NormalizedSource(Source): def __init__(self, normalized: ql_parser.Entry, text: str) -> None: self._text = text - self._cache_key = normalized.key() - self._tokens = normalized.tokens() - self._variables = normalized.variables() - self._first_extra = normalized.first_extra() - self._extra_counts = normalized.extra_counts() - self._extra_blobs = normalized.extra_blobs() + self._cache_key = normalized.key + self._tokens = normalized.tokens + self._variables = normalized.get_variables() + self._first_extra = normalized.first_extra + self._extra_counts = normalized.extra_counts + self._extra_blobs = normalized.extra_blobs def text(self) -> str: return self._text @@ -132,9 +132,9 @@ def inflate_position( def _tokenize(eql: str) -> List[ql_parser.Token]: result = ql_parser.tokenize(eql) - if len(result.errors()) > 0: + if len(result.errors) > 0: # TODO: emit multiple errors - error = result.errors()[0] + error = result.errors[0] message, span, hint, details = error position = inflate_position(eql, span) @@ -144,7 +144,7 @@ def _tokenize(eql: str) -> List[ql_parser.Token]: message, position=position, hint=hint, details=details ) - return result.out() + return result.out def _normalize(eql: str) -> ql_parser.Entry: diff --git a/edb/graphql-rewrite/Cargo.toml b/edb/graphql-rewrite/Cargo.toml index d8f553a1106..07298934f36 100644 --- a/edb/graphql-rewrite/Cargo.toml +++ b/edb/graphql-rewrite/Cargo.toml @@ -13,8 +13,8 @@ num-bigint = "0.4.3" num-traits = "0.2.11" edb-graphql-parser = { git="https://github.com/edgedb/graphql-parser" } -[dependencies.cpython] -version = "0.7.0" +[dependencies.pyo3] +version = "0.20.2" features = ["extension-module"] [dev-dependencies] diff --git a/edb/graphql-rewrite/_graphql_rewrite.pyi b/edb/graphql-rewrite/_graphql_rewrite.pyi new file mode 100644 index 00000000000..8beace45df6 --- /dev/null +++ b/edb/graphql-rewrite/_graphql_rewrite.pyi @@ -0,0 +1,11 @@ +from typing import * + +class Entry: + key: str + key_vars: List[str] + variables: Dict[str, Any] + substitutions: Dict[str, Tuple[str, int, int]] + + def tokens(self) -> List[Tuple[Any, int, int, int, int, Any]]: ... + +def rewrite(operation: Optional[str], text: str) -> Entry: ... diff --git a/edb/graphql-rewrite/src/lib.rs b/edb/graphql-rewrite/src/lib.rs index 0981ae382e2..bff5018ee70 100644 --- a/edb/graphql-rewrite/src/lib.rs +++ b/edb/graphql-rewrite/src/lib.rs @@ -1,10 +1,41 @@ -#[macro_use] extern crate cpython; - -mod pytoken; -mod pyentry; -mod pyerrors; -mod entry_point; +mod py_entry; +mod py_exception; +mod py_token; +mod rewrite; mod token_vec; -pub use entry_point::{rewrite, Variable, Value}; -pub use pytoken::{PyToken, PyTokenKind}; +pub use py_token::{PyToken, PyTokenKind}; +pub use rewrite::{rewrite, Value, Variable}; + +use py_exception::{AssertionError, LexingError, NotFoundError, QueryError, SyntaxError}; +use pyo3::{prelude::*, types::PyString}; + +/// Rust optimizer for graphql queries +#[pymodule] +fn _graphql_rewrite(py: Python, m: &PyModule) -> PyResult<()> { + m.add_function(wrap_pyfunction!(py_rewrite, m)?)?; + m.add_class::()?; + m.add("LexingError", py.get_type::())?; + m.add("SyntaxError", py.get_type::())?; + m.add("NotFoundError", py.get_type::())?; + m.add("AssertionError", py.get_type::())?; + m.add("QueryError", py.get_type::())?; + Ok(()) +} + +#[pyo3::pyfunction(name = "rewrite")] +#[pyo3(signature = (operation, text))] +fn py_rewrite( + py: Python<'_>, + operation: Option<&PyString>, + text: &PyString, +) -> PyResult { + // convert args + let operation = operation.map(|x| x.to_string()); + let text = text.to_string(); + + match rewrite::rewrite(operation.as_ref().map(|x| &x[..]), &text) { + Ok(entry) => py_entry::convert_entry(py, entry), + Err(e) => Err(py_exception::convert_error(e)), + } +} diff --git a/edb/graphql-rewrite/src/py_entry.rs b/edb/graphql-rewrite/src/py_entry.rs new file mode 100644 index 00000000000..59b59e7c9a9 --- /dev/null +++ b/edb/graphql-rewrite/src/py_entry.rs @@ -0,0 +1,85 @@ +use pyo3::prelude::*; +use pyo3::types::{PyDict, PyList, PyLong, PyString, PyTuple, PyType}; + +use edb_graphql_parser::position::Pos; + +use crate::py_token::{self, PyToken}; +use crate::rewrite::{self, Value}; + +#[pyclass] +pub struct Entry { + #[pyo3(get)] + key: PyObject, + #[pyo3(get)] + key_vars: PyObject, + #[pyo3(get)] + variables: PyObject, + #[pyo3(get)] + substitutions: PyObject, + _tokens: Vec, + _end_pos: Pos, +} + +#[pymethods] +impl Entry { + fn tokens(&self, py: Python, kinds: PyObject) -> PyResult { + py_token::convert_tokens(py, &self._tokens, &self._end_pos, kinds) + } +} + +pub fn convert_entry(py: Python<'_>, entry: rewrite::Entry) -> PyResult { + // import decimal + let decimal_cls = PyModule::import(py, "decimal")?.getattr("Decimal")?; + + let vars = PyDict::new(py); + let substitutions = PyDict::new(py); + for (idx, var) in entry.variables.iter().enumerate() { + let s = format!("_edb_arg__{}", idx).to_object(py); + + vars.set_item(s.clone_ref(py), value_to_py(py, &var.value, decimal_cls)?)?; + + substitutions.set_item( + s.clone_ref(py), + ( + &var.token.value, + var.token.position.map(|x| x.line), + var.token.position.map(|x| x.column), + ), + )?; + } + for (name, var) in &entry.defaults { + vars.set_item(name.into_py(py), value_to_py(py, &var.value, decimal_cls)?)? + } + let key_vars = PyList::new( + py, + &entry + .key_vars + .iter() + .map(|v| v.into_py(py)) + .collect::>(), + ); + Ok(Entry { + key: PyString::new(py, &entry.key).into(), + key_vars: key_vars.into(), + variables: vars.into_py(py), + substitutions: substitutions.into(), + _tokens: entry.tokens, + _end_pos: entry.end_pos, + }) +} + +fn value_to_py(py: Python, value: &Value, decimal_cls: &PyAny) -> PyResult { + let v = match value { + Value::Str(ref v) => PyString::new(py, v).into(), + Value::Int32(v) => v.into_py(py), + Value::Int64(v) => v.into_py(py), + Value::Decimal(v) => decimal_cls + .call(PyTuple::new(py, &[v.into_py(py)]), None)? + .into(), + Value::BigInt(ref v) => PyType::new::(py) + .call(PyTuple::new(py, &[v.into_py(py)]), None)? + .into(), + Value::Boolean(b) => b.into_py(py), + }; + Ok(v) +} diff --git a/edb/graphql-rewrite/src/py_exception.rs b/edb/graphql-rewrite/src/py_exception.rs new file mode 100644 index 00000000000..6db76bfccd2 --- /dev/null +++ b/edb/graphql-rewrite/src/py_exception.rs @@ -0,0 +1,23 @@ +use pyo3::{create_exception, exceptions::PyException, PyErr}; + +use crate::rewrite::Error; + +create_exception!(_graphql_rewrite, LexingError, PyException); + +create_exception!(_graphql_rewrite, SyntaxError, PyException); + +create_exception!(_graphql_rewrite, NotFoundError, PyException); + +create_exception!(_graphql_rewrite, AssertionError, PyException); + +create_exception!(_graphql_rewrite, QueryError, PyException); + +pub fn convert_error(error: Error) -> PyErr { + match error { + Error::Lexing(e) => LexingError::new_err(e), + Error::Syntax(e) => SyntaxError::new_err(e.to_string()), + Error::NotFound(e) => NotFoundError::new_err(e), + Error::Query(e) => QueryError::new_err(e), + Error::Assertion(e) => AssertionError::new_err(e), + } +} diff --git a/edb/graphql-rewrite/src/py_token.rs b/edb/graphql-rewrite/src/py_token.rs new file mode 100644 index 00000000000..d49965bbc59 --- /dev/null +++ b/edb/graphql-rewrite/src/py_token.rs @@ -0,0 +1,193 @@ +use edb_graphql_parser::common::{unquote_block_string, unquote_string}; +use edb_graphql_parser::position::Pos; +use edb_graphql_parser::tokenizer::Token; +use pyo3::prelude::*; +use pyo3::types::{PyList, PyTuple}; +use std::borrow::Cow; + +use crate::py_exception::LexingError; +use crate::rewrite::Error; + +#[derive(Debug, PartialEq, Copy, Clone)] +pub enum PyTokenKind { + Sof, + Eof, + Bang, + Dollar, + ParenL, + ParenR, + Spread, + Colon, + Equals, + At, + BracketL, + BracketR, + BraceL, + Pipe, + BraceR, + Name, + Int, + Float, + String, + BlockString, +} + +#[derive(Debug, PartialEq, Clone)] +pub struct PyToken { + pub kind: PyTokenKind, + pub value: Cow<'static, str>, + pub position: Option, +} + +impl PyToken { + pub fn new((token, position): &(Token<'_>, Pos)) -> Result { + use edb_graphql_parser::tokenizer::Kind::*; + use PyTokenKind as T; + + let (kind, value) = match (token.kind, token.value) { + (IntValue, val) => (T::Int, Cow::Owned(val.into())), + (FloatValue, val) => (T::Float, Cow::Owned(val.into())), + (StringValue, val) => (T::String, Cow::Owned(val.into())), + (BlockString, val) => (T::BlockString, Cow::Owned(val.into())), + (Name, val) => (T::Name, Cow::Owned(val.into())), + (Punctuator, "!") => (T::Bang, "!".into()), + (Punctuator, "$") => (T::Dollar, "$".into()), + (Punctuator, "(") => (T::ParenL, "(".into()), + (Punctuator, ")") => (T::ParenR, ")".into()), + (Punctuator, "...") => (T::Spread, "...".into()), + (Punctuator, ":") => (T::Colon, ":".into()), + (Punctuator, "=") => (T::Equals, "=".into()), + (Punctuator, "@") => (T::At, "@".into()), + (Punctuator, "[") => (T::BracketL, "[".into()), + (Punctuator, "]") => (T::BracketR, "]".into()), + (Punctuator, "{") => (T::BraceL, "{".into()), + (Punctuator, "}") => (T::BraceR, "}".into()), + (Punctuator, "|") => (T::Pipe, "|".into()), + (Punctuator, _) => Err(Error::Assertion("unsupported punctuator".into()))?, + }; + Ok(PyToken { + kind, + value, + position: Some(*position), + }) + } +} + +pub fn convert_tokens( + py: Python, + tokens: &[PyToken], + end_pos: &Pos, + kinds: PyObject, +) -> PyResult { + use PyTokenKind as K; + + let sof = kinds.getattr(py, "SOF")?; + let eof = kinds.getattr(py, "EOF")?; + let bang = kinds.getattr(py, "BANG")?; + let bang_v: PyObject = "!".into_py(py); + let dollar = kinds.getattr(py, "DOLLAR")?; + let dollar_v: PyObject = "$".into_py(py); + let paren_l = kinds.getattr(py, "PAREN_L")?; + let paren_l_v: PyObject = "(".into_py(py); + let paren_r = kinds.getattr(py, "PAREN_R")?; + let paren_r_v: PyObject = ")".into_py(py); + let spread = kinds.getattr(py, "SPREAD")?; + let spread_v: PyObject = "...".into_py(py); + let colon = kinds.getattr(py, "COLON")?; + let colon_v: PyObject = ":".into_py(py); + let equals = kinds.getattr(py, "EQUALS")?; + let equals_v: PyObject = "=".into_py(py); + let at = kinds.getattr(py, "AT")?; + let at_v: PyObject = "@".into_py(py); + let bracket_l = kinds.getattr(py, "BRACKET_L")?; + let bracket_l_v: PyObject = "[".into_py(py); + let bracket_r = kinds.getattr(py, "BRACKET_R")?; + let bracket_r_v: PyObject = "]".into_py(py); + let brace_l = kinds.getattr(py, "BRACE_L")?; + let brace_l_v: PyObject = "{".into_py(py); + let pipe = kinds.getattr(py, "PIPE")?; + let pipe_v: PyObject = "|".into_py(py); + let brace_r = kinds.getattr(py, "BRACE_R")?; + let brace_r_v: PyObject = "}".into_py(py); + let name = kinds.getattr(py, "NAME")?; + let int = kinds.getattr(py, "INT")?; + let float = kinds.getattr(py, "FLOAT")?; + let string = kinds.getattr(py, "STRING")?; + let block_string = kinds.getattr(py, "BLOCK_STRING")?; + + let mut elems: Vec = Vec::with_capacity(tokens.len()); + + let start_of_file = [ + sof.clone_ref(py), + 0u32.into_py(py), + 0u32.into_py(py), + 0u32.into_py(py), + 0u32.into_py(py), + py.None(), + ]; + elems.push(PyTuple::new(py, &start_of_file).into()); + + for token in tokens { + let (kind, value) = match token.kind { + K::Sof => (sof.clone_ref(py), py.None()), + K::Eof => (eof.clone_ref(py), py.None()), + K::Bang => (bang.clone_ref(py), bang_v.clone_ref(py)), + K::Dollar => (dollar.clone_ref(py), dollar_v.clone_ref(py)), + K::ParenL => (paren_l.clone_ref(py), paren_l_v.clone_ref(py)), + K::ParenR => (paren_r.clone_ref(py), paren_r_v.clone_ref(py)), + K::Spread => (spread.clone_ref(py), spread_v.clone_ref(py)), + K::Colon => (colon.clone_ref(py), colon_v.clone_ref(py)), + K::Equals => (equals.clone_ref(py), equals_v.clone_ref(py)), + K::At => (at.clone_ref(py), at_v.clone_ref(py)), + K::BracketL => (bracket_l.clone_ref(py), bracket_l_v.clone_ref(py)), + K::BracketR => (bracket_r.clone_ref(py), bracket_r_v.clone_ref(py)), + K::BraceL => (brace_l.clone_ref(py), brace_l_v.clone_ref(py)), + K::Pipe => (pipe.clone_ref(py), pipe_v.clone_ref(py)), + K::BraceR => (brace_r.clone_ref(py), brace_r_v.clone_ref(py)), + K::Name => (name.clone_ref(py), token.value.clone().into_py(py)), + K::Int => (int.clone_ref(py), token.value.clone().into_py(py)), + K::Float => (float.clone_ref(py), token.value.clone().into_py(py)), + K::String => { + // graphql-core 3 receives unescaped strings from the lexer + let v = unquote_string(&token.value) + .map_err(|e| LexingError::new_err(e.to_string()))? + .into_py(py); + (string.clone_ref(py), v) + } + K::BlockString => { + // graphql-core 3 receives unescaped strings from the lexer + let v = unquote_block_string(&token.value) + .map_err(|e| LexingError::new_err(e.to_string()))? + .into_py(py); + (block_string.clone_ref(py), v) + } + }; + let token_tuple = [ + kind, + token.position.map(|x| x.character).into_py(py), + token + .position + .map(|x| x.character + token.value.chars().count()) + .into_py(py), + token.position.map(|x| x.line).into_py(py), + token.position.map(|x| x.column).into_py(py), + value, + ]; + elems.push(PyTuple::new(py, &token_tuple).into()); + } + elems.push( + PyTuple::new( + py, + &[ + eof.clone_ref(py), + end_pos.character.into_py(py), + end_pos.line.into_py(py), + end_pos.column.into_py(py), + end_pos.character.into_py(py), + py.None(), + ], + ) + .into(), + ); + Ok(PyList::new(py, &elems[..]).into()) +} diff --git a/edb/graphql-rewrite/src/pyentry.rs b/edb/graphql-rewrite/src/pyentry.rs deleted file mode 100644 index 62a53e72782..00000000000 --- a/edb/graphql-rewrite/src/pyentry.rs +++ /dev/null @@ -1,249 +0,0 @@ -use cpython::{Python, PyClone, ToPyObject, PythonObject, ObjectProtocol}; -use cpython::{PyString, PyResult, PyTuple, PyDict, PyList, PyObject, PyInt}; -use cpython::{PyModule, PyType, FromPyObject}; - -use edb_graphql_parser::common::{unquote_string, unquote_block_string}; -use edb_graphql_parser::position::Pos; - -use crate::pyerrors::{LexingError, SyntaxError, NotFoundError, AssertionError}; -use crate::pyerrors::{QueryError}; -use crate::entry_point::{Value, Error}; -use crate::pytoken::PyToken; -use crate::entry_point; - - -py_class!(pub class Entry |py| { - data _key: PyString; - data _key_vars: PyList; - data _variables: PyDict; - data _substitutions: PyDict; - data _tokens: Vec; - data _end_pos: Pos; - def key(&self) -> PyResult { - Ok(self._key(py).clone_ref(py)) - } - def key_vars(&self) -> PyResult { - Ok(self._key_vars(py).clone_ref(py)) - } - def variables(&self) -> PyResult { - Ok(self._variables(py).clone_ref(py)) - } - def substitutions(&self) -> PyResult { - Ok(self._substitutions(py).clone_ref(py)) - } - def tokens(&self, kinds: PyObject) -> PyResult { - use crate::pytoken::PyTokenKind as K; - - let sof = kinds.get_item(py, "SOF")?; - let eof = kinds.get_item(py, "EOF")?; - let bang = kinds.get_item(py, "BANG")?; - let bang_v = "!".to_py_object(py).into_object(); - let dollar = kinds.get_item(py, "DOLLAR")?; - let dollar_v = "$".to_py_object(py).into_object(); - let paren_l = kinds.get_item(py, "PAREN_L")?; - let paren_l_v = "(".to_py_object(py).into_object(); - let paren_r = kinds.get_item(py, "PAREN_R")?; - let paren_r_v = ")".to_py_object(py).into_object(); - let spread = kinds.get_item(py, "SPREAD")?; - let spread_v = "...".to_py_object(py).into_object(); - let colon = kinds.get_item(py, "COLON")?; - let colon_v = ":".to_py_object(py).into_object(); - let equals = kinds.get_item(py, "EQUALS")?; - let equals_v = "=".to_py_object(py).into_object(); - let at = kinds.get_item(py, "AT")?; - let at_v = "@".to_py_object(py).into_object(); - let bracket_l = kinds.get_item(py, "BRACKET_L")?; - let bracket_l_v = "[".to_py_object(py).into_object(); - let bracket_r = kinds.get_item(py, "BRACKET_R")?; - let bracket_r_v = "]".to_py_object(py).into_object(); - let brace_l = kinds.get_item(py, "BRACE_L")?; - let brace_l_v = "{".to_py_object(py).into_object(); - let pipe = kinds.get_item(py, "PIPE")?; - let pipe_v = "|".to_py_object(py).into_object(); - let brace_r = kinds.get_item(py, "BRACE_R")?; - let brace_r_v = "}".to_py_object(py).into_object(); - let name = kinds.get_item(py, "NAME")?; - let int = kinds.get_item(py, "INT")?; - let float = kinds.get_item(py, "FLOAT")?; - let string = kinds.get_item(py, "STRING")?; - let block_string = kinds.get_item(py, "BLOCK_STRING")?; - - let tokens = self._tokens(py); - let mut elems = Vec::with_capacity(tokens.len()); - elems.push(PyTuple::new(py, &[ - sof.clone_ref(py), - 0u32.to_py_object(py).into_object(), - 0u32.to_py_object(py).into_object(), - 0u32.to_py_object(py).into_object(), - 0u32.to_py_object(py).into_object(), - py.None(), - ]).into_object()); - for el in tokens { - let (kind, value) = match el.kind { - K::Sof => (sof.clone_ref(py), py.None()), - K::Eof => (eof.clone_ref(py), py.None()), - K::Bang => (bang.clone_ref(py), bang_v.clone_ref(py)), - K::Dollar => (dollar.clone_ref(py), dollar_v.clone_ref(py)), - K::ParenL => (paren_l.clone_ref(py), paren_l_v.clone_ref(py)), - K::ParenR => (paren_r.clone_ref(py), paren_r_v.clone_ref(py)), - K::Spread => (spread.clone_ref(py), spread_v.clone_ref(py)), - K::Colon => (colon.clone_ref(py), colon_v.clone_ref(py)), - K::Equals => (equals.clone_ref(py), equals_v.clone_ref(py)), - K::At => (at.clone_ref(py), at_v.clone_ref(py)), - K::BracketL => (bracket_l.clone_ref(py), - bracket_l_v.clone_ref(py)), - K::BracketR => (bracket_r.clone_ref(py), - bracket_r_v.clone_ref(py)), - K::BraceL => (brace_l.clone_ref(py), brace_l_v.clone_ref(py)), - K::Pipe => (pipe.clone_ref(py), pipe_v.clone_ref(py)), - K::BraceR => (brace_r.clone_ref(py), brace_r_v.clone_ref(py)), - K::Name => (name.clone_ref(py), - el.value.to_py_object(py).into_object()), - K::Int => (int.clone_ref(py), - el.value.to_py_object(py).into_object()), - K::Float => (float.clone_ref(py), - el.value.to_py_object(py).into_object()), - K::String => { - // graphql-core 3 receives unescaped strings from the lexer - let v = unquote_string(&el.value) - .map_err(|e| LexingError::new(py, e.to_string()))? - .to_py_object(py).into_object(); - (string.clone_ref(py), v) - } - K::BlockString => { - // graphql-core 3 receives unescaped strings from the lexer - let v = unquote_block_string(&el.value) - .map_err(|e| LexingError::new(py, e.to_string()))? - .to_py_object(py).into_object(); - (block_string.clone_ref(py), v) - } - }; - elems.push(PyTuple::new(py, &[ - kind, - el.position.map(|x| x.character) - .to_py_object(py).into_object(), - el.position.map(|x| x.character + el.value.chars().count()) - .to_py_object(py).into_object(), - el.position.map(|x| x.line) - .to_py_object(py).into_object(), - el.position.map(|x| x.column) - .to_py_object(py).into_object(), - value, - ]).into_object()); - } - let pos = self._end_pos(py); - let end_off = pos.character.to_py_object(py).into_object(); - elems.push(PyTuple::new(py, &[ - eof.clone_ref(py), - end_off.clone_ref(py), - pos.line.to_py_object(py).into_object(), - pos.column.to_py_object(py).into_object(), - end_off, - py.None(), - ]).into_object()); - Ok(PyList::new(py, &elems[..])) - } -}); - -fn init_module(_py: Python<'_>) { -} - -fn value_to_py(py: Python<'_>, value: &Value, decimal: &PyType) - -> PyResult -{ - let v = match value { - Value::Str(ref v) => { - PyString::new(py, v).into_object() - } - Value::Int32(v) => { - v.to_py_object(py).into_object() - } - Value::Int64(v) => { - v.to_py_object(py).into_object() - } - Value::Decimal(v) => { - decimal.call(py, - PyTuple::new(py, &[ - v.to_py_object(py).into_object(), - ]), - None)? - } - Value::BigInt(ref v) => { - py.get_type::() - .call(py, - PyTuple::new(py, &[ - v.to_py_object(py).into_object(), - ]), - None)? - } - Value::Boolean(b) => { - b.to_py_object(py).into_object() - } - }; - Ok(v) -} - -fn rewrite(py: Python<'_>, operation: Option<&PyString>, text: &PyString) - -> PyResult -{ - let decimal = PyType::extract(py, - &PyModule::import(py, "decimal")?.get(py, "Decimal")?)?; - let oper = operation.map(|x| x.to_string(py)).transpose()?; - let text = text.to_string(py)?; - match entry_point::rewrite(oper.as_ref().map(|x| &x[..]), &text) { - Ok(entry) => { - let vars = PyDict::new(py); - let substitutions = PyDict::new(py); - for (idx, var) in entry.variables.iter().enumerate() { - let s = format!("_edb_arg__{}", idx).to_py_object(py); - vars.set_item(py, - s.clone_ref(py), - value_to_py(py, &var.value, &decimal)?)?; - substitutions.set_item(py, s.clone_ref(py), ( - &var.token.value, - var.token.position.map(|x| x.line), - var.token.position.map(|x| x.column), - ).to_py_object(py).into_object())?; - } - for (name, var) in &entry.defaults { - vars.set_item(py, - name.to_py_object(py), - value_to_py(py, &var.value, &decimal)?)? - } - let key_vars = PyList::new(py, - &entry.key_vars.iter() - .map(|v| v.to_py_object(py).into_object()) - .collect::>()); - Entry::create_instance(py, - PyString::new(py, &entry.key), - key_vars, - vars, - substitutions, - entry.tokens, - entry.end_pos, - ) - } - Err(Error::Lexing(e)) => Err(LexingError::new(py, e)), - Err(Error::Syntax(e)) => Err(SyntaxError::new(py, e.to_string())), - Err(Error::NotFound(e)) => Err(NotFoundError::new(py, e)), - Err(Error::Query(e)) => Err(QueryError::new(py, e)), - Err(Error::Assertion(e)) - => Err(AssertionError::new(py, e)), - } -} - -py_module_initializer!( - _graphql_rewrite, init_graphql_rewrite, PyInit__graphql_rewrite, - |py, m| { - init_module(py); - m.add(py, "__doc__", "Rust optimizer for graphql queries")?; - m.add(py, "rewrite", - py_fn!(py, rewrite(option: Option<&PyString>, data: &PyString)))?; - m.add(py, "Entry", py.get_type::())?; - m.add(py, "LexingError", py.get_type::())?; - m.add(py, "SyntaxError", py.get_type::())?; - m.add(py, "NotFoundError", py.get_type::())?; - m.add(py, "AssertionError", py.get_type::())?; - m.add(py, "QueryError", py.get_type::())?; - Ok(()) - }); diff --git a/edb/graphql-rewrite/src/pyerrors.rs b/edb/graphql-rewrite/src/pyerrors.rs deleted file mode 100644 index 4de063980db..00000000000 --- a/edb/graphql-rewrite/src/pyerrors.rs +++ /dev/null @@ -1,297 +0,0 @@ -use cpython::{PyObject, ToPyObject, Python, PyErr, PythonObject, PyType}; -use cpython::exc::Exception; -use crate::cpython::PythonObjectWithTypeObject; - - -// can't use py_exception macro because that fails on dotted module name -pub struct LexingError(PyObject); -pub struct SyntaxError(PyObject); -pub struct NotFoundError(PyObject); -pub struct AssertionError(PyObject); -pub struct QueryError(PyObject); - -pyobject_newtype!(LexingError); -pyobject_newtype!(SyntaxError); -pyobject_newtype!(NotFoundError); -pyobject_newtype!(AssertionError); -pyobject_newtype!(QueryError); - -impl LexingError { - pub fn new(py: Python, args: T) -> PyErr { - PyErr::new::(py, args) - } -} - -impl cpython::PythonObjectWithCheckedDowncast for LexingError { - #[inline] - fn downcast_from(py: Python, obj: PyObject) - -> Result - { - if LexingError::type_object(py).is_instance(py, &obj) { - Ok(unsafe { PythonObject::unchecked_downcast_from(obj) }) - } else { - Err(cpython::PythonObjectDowncastError::new(py, - "LexingError", - LexingError::type_object(py), - )) - } - } - - #[inline] - fn downcast_borrow_from<'a, 'p>(py: Python<'p>, obj: &'a PyObject) - -> Result<&'a LexingError, cpython::PythonObjectDowncastError<'p>> - { - if LexingError::type_object(py).is_instance(py, obj) { - Ok(unsafe { PythonObject::unchecked_downcast_borrow_from(obj) }) - } else { - Err(cpython::PythonObjectDowncastError::new(py, - "LexingError", - LexingError::type_object(py), - )) - } - } -} - -impl cpython::PythonObjectWithTypeObject for LexingError { - #[inline] - fn type_object(py: Python) -> PyType { - unsafe { - static mut TYPE_OBJECT: *mut cpython::_detail::ffi::PyTypeObject - = 0 as *mut cpython::_detail::ffi::PyTypeObject; - - if TYPE_OBJECT.is_null() { - TYPE_OBJECT = PyErr::new_type( - py, - "edb._graphql_rewrite.LexingError", - Some(PythonObject::into_object(py.get_type::())), - None).as_type_ptr(); - } - - PyType::from_type_ptr(py, TYPE_OBJECT) - } - } -} - -impl SyntaxError { - pub fn new(py: Python, args: T) -> PyErr { - PyErr::new::(py, args) - } -} - -impl cpython::PythonObjectWithCheckedDowncast for SyntaxError { - #[inline] - fn downcast_from(py: Python, obj: PyObject) - -> Result - { - if SyntaxError::type_object(py).is_instance(py, &obj) { - Ok(unsafe { PythonObject::unchecked_downcast_from(obj) }) - } else { - Err(cpython::PythonObjectDowncastError::new(py, - "SyntaxError", - SyntaxError::type_object(py), - )) - } - } - - #[inline] - fn downcast_borrow_from<'a, 'p>(py: Python<'p>, obj: &'a PyObject) - -> Result<&'a SyntaxError, cpython::PythonObjectDowncastError<'p>> - { - if SyntaxError::type_object(py).is_instance(py, obj) { - Ok(unsafe { PythonObject::unchecked_downcast_borrow_from(obj) }) - } else { - Err(cpython::PythonObjectDowncastError::new(py, - "SyntaxError", - SyntaxError::type_object(py), - )) - } - } -} - -impl cpython::PythonObjectWithTypeObject for SyntaxError { - #[inline] - fn type_object(py: Python) -> PyType { - unsafe { - static mut TYPE_OBJECT: *mut cpython::_detail::ffi::PyTypeObject - = 0 as *mut cpython::_detail::ffi::PyTypeObject; - - if TYPE_OBJECT.is_null() { - TYPE_OBJECT = PyErr::new_type( - py, - "edb._graphql_rewrite.SyntaxError", - Some(PythonObject::into_object(py.get_type::())), - None).as_type_ptr(); - } - - PyType::from_type_ptr(py, TYPE_OBJECT) - } - } -} - -impl NotFoundError { - pub fn new(py: Python, args: T) -> PyErr { - PyErr::new::(py, args) - } -} - -impl cpython::PythonObjectWithCheckedDowncast for NotFoundError { - #[inline] - fn downcast_from(py: Python, obj: PyObject) - -> Result - { - if NotFoundError::type_object(py).is_instance(py, &obj) { - Ok(unsafe { PythonObject::unchecked_downcast_from(obj) }) - } else { - Err(cpython::PythonObjectDowncastError::new(py, - "NotFoundError", - NotFoundError::type_object(py), - )) - } - } - - #[inline] - fn downcast_borrow_from<'a, 'p>(py: Python<'p>, obj: &'a PyObject) - -> Result<&'a NotFoundError, cpython::PythonObjectDowncastError<'p>> - { - if NotFoundError::type_object(py).is_instance(py, obj) { - Ok(unsafe { PythonObject::unchecked_downcast_borrow_from(obj) }) - } else { - Err(cpython::PythonObjectDowncastError::new(py, - "NotFoundError", - NotFoundError::type_object(py), - )) - } - } -} - -impl cpython::PythonObjectWithTypeObject for NotFoundError { - #[inline] - fn type_object(py: Python) -> PyType { - unsafe { - static mut TYPE_OBJECT: *mut cpython::_detail::ffi::PyTypeObject - = 0 as *mut cpython::_detail::ffi::PyTypeObject; - - if TYPE_OBJECT.is_null() { - TYPE_OBJECT = PyErr::new_type( - py, - "edb._graphql_rewrite.NotFoundError", - Some(PythonObject::into_object(py.get_type::())), - None).as_type_ptr(); - } - - PyType::from_type_ptr(py, TYPE_OBJECT) - } - } -} - -impl AssertionError { - pub fn new(py: Python, args: T) -> PyErr { - PyErr::new::(py, args) - } -} - -impl cpython::PythonObjectWithCheckedDowncast for AssertionError { - #[inline] - fn downcast_from(py: Python, obj: PyObject) - -> Result - { - if AssertionError::type_object(py).is_instance(py, &obj) { - Ok(unsafe { PythonObject::unchecked_downcast_from(obj) }) - } else { - Err(cpython::PythonObjectDowncastError::new(py, - "AssertionError", - AssertionError::type_object(py), - )) - } - } - - #[inline] - fn downcast_borrow_from<'a, 'p>(py: Python<'p>, obj: &'a PyObject) - -> Result<&'a AssertionError, cpython::PythonObjectDowncastError<'p>> - { - if AssertionError::type_object(py).is_instance(py, obj) { - Ok(unsafe { PythonObject::unchecked_downcast_borrow_from(obj) }) - } else { - Err(cpython::PythonObjectDowncastError::new(py, - "AssertionError", - AssertionError::type_object(py), - )) - } - } -} - -impl cpython::PythonObjectWithTypeObject for AssertionError { - #[inline] - fn type_object(py: Python) -> PyType { - unsafe { - static mut TYPE_OBJECT: *mut cpython::_detail::ffi::PyTypeObject - = 0 as *mut cpython::_detail::ffi::PyTypeObject; - - if TYPE_OBJECT.is_null() { - TYPE_OBJECT = PyErr::new_type( - py, - "edb._graphql_rewrite.AssertionError", - Some(PythonObject::into_object(py.get_type::())), - None).as_type_ptr(); - } - - PyType::from_type_ptr(py, TYPE_OBJECT) - } - } -} - -impl QueryError { - pub fn new(py: Python, args: T) -> PyErr { - PyErr::new::(py, args) - } -} - -impl cpython::PythonObjectWithCheckedDowncast for QueryError { - #[inline] - fn downcast_from(py: Python, obj: PyObject) - -> Result - { - if QueryError::type_object(py).is_instance(py, &obj) { - Ok(unsafe { PythonObject::unchecked_downcast_from(obj) }) - } else { - Err(cpython::PythonObjectDowncastError::new(py, - "QueryError", - QueryError::type_object(py), - )) - } - } - - #[inline] - fn downcast_borrow_from<'a, 'p>(py: Python<'p>, obj: &'a PyObject) - -> Result<&'a QueryError, cpython::PythonObjectDowncastError<'p>> - { - if QueryError::type_object(py).is_instance(py, obj) { - Ok(unsafe { PythonObject::unchecked_downcast_borrow_from(obj) }) - } else { - Err(cpython::PythonObjectDowncastError::new(py, - "QueryError", - QueryError::type_object(py), - )) - } - } -} - -impl cpython::PythonObjectWithTypeObject for QueryError { - #[inline] - fn type_object(py: Python) -> PyType { - unsafe { - static mut TYPE_OBJECT: *mut cpython::_detail::ffi::PyTypeObject - = 0 as *mut cpython::_detail::ffi::PyTypeObject; - - if TYPE_OBJECT.is_null() { - TYPE_OBJECT = PyErr::new_type( - py, - "edb._graphql_rewrite.QueryError", - Some(PythonObject::into_object(py.get_type::())), - None).as_type_ptr(); - } - - PyType::from_type_ptr(py, TYPE_OBJECT) - } - } -} diff --git a/edb/graphql-rewrite/src/pytoken.rs b/edb/graphql-rewrite/src/pytoken.rs deleted file mode 100644 index decd2edebeb..00000000000 --- a/edb/graphql-rewrite/src/pytoken.rs +++ /dev/null @@ -1,74 +0,0 @@ -use std::borrow::Cow; - -use edb_graphql_parser::tokenizer::Token; -use edb_graphql_parser::position::Pos; - -use crate::entry_point::Error; - - -#[derive(Debug, PartialEq, Copy, Clone)] -pub enum PyTokenKind { - Sof, - Eof, - Bang, - Dollar, - ParenL, - ParenR, - Spread, - Colon, - Equals, - At, - BracketL, - BracketR, - BraceL, - Pipe, - BraceR, - Name, - Int, - Float, - String, - BlockString, -} - - -#[derive(Debug, PartialEq, Clone)] -pub struct PyToken { - pub kind: PyTokenKind, - pub value: Cow<'static, str>, - pub position: Option, -} - -impl PyToken { - pub fn new((token, position): &(Token<'_>, Pos)) -> Result - { - use edb_graphql_parser::tokenizer::Kind::*; - use PyTokenKind as T; - - let (kind, value) = match (token.kind, token.value) { - (IntValue, val) => (T::Int, Cow::Owned(val.into())), - (FloatValue, val) => (T::Float, Cow::Owned(val.into())), - (StringValue, val) => (T::String, Cow::Owned(val.into())), - (BlockString, val) => (T::BlockString, Cow::Owned(val.into())), - (Name, val) => (T::Name, Cow::Owned(val.into())), - (Punctuator, "!") => (T::Bang, "!".into()), - (Punctuator, "$") => (T::Dollar, "$".into()), - (Punctuator, "(") => (T::ParenL, "(".into()), - (Punctuator, ")") => (T::ParenR, ")".into()), - (Punctuator, "...") => (T::Spread, "...".into()), - (Punctuator, ":") => (T::Colon, ":".into()), - (Punctuator, "=") => (T::Equals, "=".into()), - (Punctuator, "@") => (T::At, "@".into()), - (Punctuator, "[") => (T::BracketL, "[".into()), - (Punctuator, "]") => (T::BracketR, "]".into()), - (Punctuator, "{") => (T::BraceL, "{".into()), - (Punctuator, "}") => (T::BraceR, "}".into()), - (Punctuator, "|") => (T::Pipe, "|".into()), - (Punctuator, _) - => Err(Error::Assertion("unsupported punctuator".into()))?, - }; - Ok(PyToken { - kind, value, - position: Some(*position), - }) - } -} diff --git a/edb/graphql-rewrite/src/entry_point.rs b/edb/graphql-rewrite/src/rewrite.rs similarity index 78% rename from edb/graphql-rewrite/src/entry_point.rs rename to edb/graphql-rewrite/src/rewrite.rs index 75d82aa950e..8bf9d5e70ee 100644 --- a/edb/graphql-rewrite/src/entry_point.rs +++ b/edb/graphql-rewrite/src/rewrite.rs @@ -2,21 +2,20 @@ use std::collections::{BTreeMap, BTreeSet, HashSet}; use combine::stream::{Positioned, StreamOnce}; +use edb_graphql_parser::common::{unquote_string, Type, Value as GqlValue}; use edb_graphql_parser::position::Pos; +use edb_graphql_parser::query::{parse_query, Document, ParseError}; use edb_graphql_parser::query::{Definition, Directive}; -use edb_graphql_parser::query::{Operation, InsertVars, InsertVarsKind}; -use edb_graphql_parser::query::{Document, parse_query, ParseError}; -use edb_graphql_parser::tokenizer::Kind::{StringValue, BlockString}; -use edb_graphql_parser::tokenizer::Kind::{IntValue, FloatValue}; -use edb_graphql_parser::tokenizer::Kind::{Punctuator, Name}; -use edb_graphql_parser::tokenizer::{TokenStream, Token}; -use edb_graphql_parser::common::{unquote_string, Type, Value as GqlValue}; +use edb_graphql_parser::query::{InsertVars, InsertVarsKind, Operation}; +use edb_graphql_parser::tokenizer::Kind::{BlockString, StringValue}; +use edb_graphql_parser::tokenizer::Kind::{FloatValue, IntValue}; +use edb_graphql_parser::tokenizer::Kind::{Name, Punctuator}; +use edb_graphql_parser::tokenizer::{Token, TokenStream}; use edb_graphql_parser::visitor::Visit; -use crate::pytoken::{PyToken, PyTokenKind}; +use crate::py_token::{PyToken, PyTokenKind}; use crate::token_vec::TokenVec; - #[derive(Debug, PartialEq)] pub enum Value { Str(String), @@ -42,7 +41,6 @@ pub enum Error { Query(String), } - #[derive(Debug)] pub struct Entry { pub key: String, @@ -53,158 +51,26 @@ pub struct Entry { pub end_pos: Pos, } -impl From for Error { - fn from(v: ParseError) -> Error { - Error::Syntax(v) - } -} - -impl<'a> From,Token<'a>>> for Error { - fn from(v: combine::easy::Error,Token<'a>>) -> Error { - Error::Lexing(v.to_string()) - } -} - -fn token_array(s: &str) -> Result<(Vec<(Token, Pos)>, Pos), Error> { - let mut lexer = TokenStream::new(s); - let mut tokens = Vec::new(); - let mut pos = lexer.position(); - loop { - match lexer.uncons() { - Ok(token) => { - tokens.push((token, pos)); - pos = lexer.position(); - } - Err(ref e) if e == &combine::easy::Error::end_of_input() => break, - Err(e) => panic!("Parse error at {}: {}", lexer.position(), e), - } - } - return Ok((tokens, lexer.position())); -} - -fn find_operation<'a>(document: &'a Document<'a, &'a str>, - operation: &str) - -> Option<&'a Operation<'a, &'a str>> -{ - for def in &document.definitions { - let res = match def { - Definition::Operation(ref op) if op.name == Some(operation) => op, - _ => continue, - }; - return Some(res); - } - return None; -} - -fn insert_args(dest: &mut Vec, ins: &InsertVars, args: Vec) { - use crate::pytoken::PyTokenKind as P; - - if args.is_empty() { - return; - } - if ins.kind == InsertVarsKind::Query { - dest.push(PyToken { - kind: P::Name, - value: "query".into(), - position: None, - }); - } - if ins.kind != InsertVarsKind::Normal { - dest.push(PyToken { - kind: P::ParenL, - value: "(".into(), - position: None, - }); - } - dest.extend(args); - if ins.kind != InsertVarsKind::Normal { - dest.push(PyToken { - kind: P::ParenR, - value: ")".into(), - position: None, - }); - } -} - -fn type_name<'x>(var_type: &'x Type<'x, &'x str>) -> Option<&'x str> { - match var_type { - Type::NamedType(t) => Some(t), - Type::NonNullType(b) => type_name(b), - _ => None, - } -} - -fn push_var_definition(args: &mut Vec, var_name: &str, - var_type: &'static str) -{ - use crate::pytoken::PyTokenKind as P; - - args.push(PyToken { - kind: P::Dollar, - value: "$".into(), - position: None, - }); - args.push(PyToken { - kind: P::Name, - value: var_name.to_owned().into(), - position: None, - }); - args.push(PyToken { - kind: P::Colon, - value: ":".into(), - position: None, - }); - args.push(PyToken { - kind: P::Name, - value: var_type.into(), - position: None, - }); - args.push(PyToken { - kind: P::Bang, - value: "!".into(), - position: None, - }); -} - -pub fn visit_directives<'x>(key_vars: &mut BTreeSet, - value_positions: &mut HashSet, - oper: &'x Operation<'x, &'x str>) -{ - for dir in oper.selection_set.visit::>() { - if dir.name == "include" || dir.name == "skip" { - for arg in &dir.arguments { - match arg.value { - GqlValue::Variable(vname) => { - key_vars.insert(vname.to_string()); - } - GqlValue::Boolean(_) => { - value_positions.insert(arg.value_position.token); - } - _ => {} - } - } - } - } -} - pub fn rewrite(operation: Option<&str>, s: &str) -> Result { + use crate::py_token::PyTokenKind as P; use edb_graphql_parser::query::Value as G; use Value::*; - use crate::pytoken::PyTokenKind as P; let document: Document<'_, &str> = parse_query(s).map_err(Error::Syntax)?; let oper = if let Some(oper_name) = operation { find_operation(&document, oper_name) - .ok_or_else(|| Error::NotFound( - format!("no operation {:?} found", operation)))? + .ok_or_else(|| Error::NotFound(format!("no operation {:?} found", operation)))? } else { let mut oper = None; for def in &document.definitions { match def { Definition::Operation(ref op) => { if oper.is_some() { - return Err(Error::NotFound("Multiple operations \ - found. Please specify operation name".into()))?; + return Err(Error::NotFound( + "Multiple operations \ + found. Please specify operation name" + .into(), + ))?; } else { oper = Some(op); } @@ -228,18 +94,17 @@ pub fn rewrite(operation: Option<&str>, s: &str) -> Result { for var in &oper.variable_definitions { if var.name.starts_with("_edb_arg__") { return Err(Error::Query( - "Variables starting with '_edb_arg__' are prohibited".into())); + "Variables starting with '_edb_arg__' are prohibited".into(), + )); } if let Some(ref dvalue) = var.default_value { let value = match (&dvalue.value, type_name(&var.var_type)) { (G::String(ref s), Some("String")) => Str(s.clone()), - (G::Int(ref s), Some("Int")) | (G::Int(ref s), Some("Int32")) - => { + (G::Int(ref s), Some("Int")) | (G::Int(ref s), Some("Int32")) => { let value = match s.as_i64() { - Some(v) - if v <= i32::max_value() as i64 - && v >= i32::min_value() as i64 - => v, + Some(v) if v <= i32::max_value() as i64 && v >= i32::min_value() as i64 => { + v + } // Ignore bad values. Let graphql solver handle that _ => continue, }; @@ -253,18 +118,10 @@ pub fn rewrite(operation: Option<&str>, s: &str) -> Result { }; Int64(value) } - (G::Int(ref s), Some("Bigint")) => { - BigInt(s.as_bigint().to_string()) - } - (G::Float(s), Some("Float")) => { - Decimal(s.clone()) - } - (G::Float(s), Some("Decimal")) => { - Decimal(s.clone()) - } - (G::Boolean(s), Some("Boolean")) => { - Boolean(*s) - } + (G::Int(ref s), Some("Bigint")) => BigInt(s.as_bigint().to_string()), + (G::Float(s), Some("Float")) => Decimal(s.clone()), + (G::Float(s), Some("Decimal")) => Decimal(s.clone()), + (G::Boolean(s), Some("Boolean")) => Boolean(*s), // other types are unsupported _ => continue, }; @@ -279,20 +136,24 @@ pub fn rewrite(operation: Option<&str>, s: &str) -> Result { }); } // first token is needed for errors, others are discarded - let pair = src_tokens.drain_to(dvalue.span.1.token) - .next().expect("at least one token of default value"); - defaults.insert(var.name.to_owned(), Variable { - value, - token: PyToken::new(pair)?, - }); + let pair = src_tokens + .drain_to(dvalue.span.1.token) + .next() + .expect("at least one token of default value"); + defaults.insert( + var.name.to_owned(), + Variable { + value, + token: PyToken::new(pair)?, + }, + ); } } for tok in src_tokens.drain_to(oper.insert_variables.position.token) { tokens.push(PyToken::new(tok)?); } let mut args = Vec::new(); - let mut tmp = Vec::with_capacity( - oper.selection_set.span.1.token - tokens.len()); + let mut tmp = Vec::with_capacity(oper.selection_set.span.1.token - tokens.len()); for tok in src_tokens.drain_to(oper.selection_set.span.0.token) { tmp.push(PyToken::new(tok)?); } @@ -320,10 +181,10 @@ pub fn rewrite(operation: Option<&str>, s: &str) -> Result { IntValue => { if token.value == "1" { if pos.token > 2 - && all_src_tokens[pos.token-1].0.kind == Punctuator - && all_src_tokens[pos.token-1].0.value == ":" - && all_src_tokens[pos.token-2].0.kind == Name - && all_src_tokens[pos.token-2].0.value == "first" + && all_src_tokens[pos.token - 1].0.kind == Punctuator + && all_src_tokens[pos.token - 1].0.value == ":" + && all_src_tokens[pos.token - 2].0.kind == Name + && all_src_tokens[pos.token - 2].0.value == "first" { // skip `first: 1` as this is used to fetch singleton // properties from queries where literal `LIMIT 1` @@ -343,11 +204,8 @@ pub fn rewrite(operation: Option<&str>, s: &str) -> Result { value: var_name.clone().into(), position: None, }); - let (value, typ) = if let Ok(val) = token.value.parse::() - { - if val <= i32::max_value() as i64 - && val >= i32::min_value() as i64 - { + let (value, typ) = if let Ok(val) = token.value.parse::() { + if val <= i32::max_value() as i64 && val >= i32::min_value() as i64 { (Value::Int32(val as i32), "Int") } else { (Value::Int64(val), "Int64") @@ -414,7 +272,7 @@ pub fn rewrite(operation: Option<&str>, s: &str) -> Result { tokens.push(PyToken::new(tok)?); } - return Ok(Entry { + Ok(Entry { key: join_tokens(&tokens), key_vars, variables, @@ -424,26 +282,159 @@ pub fn rewrite(operation: Option<&str>, s: &str) -> Result { }) } -fn join_tokens<'a, I: IntoIterator>(tokens: I) -> String { +impl From for Error { + fn from(v: ParseError) -> Error { + Error::Syntax(v) + } +} + +impl<'a> From, Token<'a>>> for Error { + fn from(v: combine::easy::Error, Token<'a>>) -> Error { + Error::Lexing(v.to_string()) + } +} + +fn token_array(s: &str) -> Result<(Vec<(Token, Pos)>, Pos), Error> { + let mut lexer = TokenStream::new(s); + let mut tokens = Vec::new(); + let mut pos = lexer.position(); + loop { + match lexer.uncons() { + Ok(token) => { + tokens.push((token, pos)); + pos = lexer.position(); + } + Err(ref e) if e == &combine::easy::Error::end_of_input() => break, + Err(e) => panic!("Parse error at {}: {}", lexer.position(), e), + } + } + Ok((tokens, lexer.position())) +} + +fn find_operation<'a>( + document: &'a Document<'a, &'a str>, + operation: &str, +) -> Option<&'a Operation<'a, &'a str>> { + for def in &document.definitions { + let res = match def { + Definition::Operation(ref op) if op.name == Some(operation) => op, + _ => continue, + }; + return Some(res); + } + None +} + +fn insert_args(dest: &mut Vec, ins: &InsertVars, args: Vec) { + use crate::py_token::PyTokenKind as P; + + if args.is_empty() { + return; + } + if ins.kind == InsertVarsKind::Query { + dest.push(PyToken { + kind: P::Name, + value: "query".into(), + position: None, + }); + } + if ins.kind != InsertVarsKind::Normal { + dest.push(PyToken { + kind: P::ParenL, + value: "(".into(), + position: None, + }); + } + dest.extend(args); + if ins.kind != InsertVarsKind::Normal { + dest.push(PyToken { + kind: P::ParenR, + value: ")".into(), + position: None, + }); + } +} + +fn type_name<'x>(var_type: &'x Type<'x, &'x str>) -> Option<&'x str> { + match var_type { + Type::NamedType(t) => Some(t), + Type::NonNullType(b) => type_name(b), + _ => None, + } +} + +fn push_var_definition(args: &mut Vec, var_name: &str, var_type: &'static str) { + use crate::py_token::PyTokenKind as P; + + args.push(PyToken { + kind: P::Dollar, + value: "$".into(), + position: None, + }); + args.push(PyToken { + kind: P::Name, + value: var_name.to_owned().into(), + position: None, + }); + args.push(PyToken { + kind: P::Colon, + value: ":".into(), + position: None, + }); + args.push(PyToken { + kind: P::Name, + value: var_type.into(), + position: None, + }); + args.push(PyToken { + kind: P::Bang, + value: "!".into(), + position: None, + }); +} + +fn visit_directives<'x>( + key_vars: &mut BTreeSet, + value_positions: &mut HashSet, + oper: &'x Operation<'x, &'x str>, +) { + for dir in oper.selection_set.visit::>() { + if dir.name == "include" || dir.name == "skip" { + for arg in &dir.arguments { + match arg.value { + GqlValue::Variable(vname) => { + key_vars.insert(vname.to_string()); + } + GqlValue::Boolean(_) => { + value_positions.insert(arg.value_position.token); + } + _ => {} + } + } + } + } +} + +fn join_tokens<'a, I: IntoIterator>(tokens: I) -> String { let mut buf = String::new(); let mut needs_whitespace = false; for token in tokens { match (token.kind, needs_whitespace) { // space before puncutators is optional - (PyTokenKind::ParenL, true) => {}, - (PyTokenKind::ParenR, true) => {}, - (PyTokenKind::Spread, true) => {}, - (PyTokenKind::Colon, true) => {}, - (PyTokenKind::Equals, true) => {}, - (PyTokenKind::At, true) => {}, - (PyTokenKind::BracketL, true) => {}, - (PyTokenKind::BracketR, true) => {}, - (PyTokenKind::BraceL, true) => {}, - (PyTokenKind::BraceR, true) => {}, - (PyTokenKind::Pipe, true) => {}, - (PyTokenKind::Bang, true) => {}, + (PyTokenKind::ParenL, true) => {} + (PyTokenKind::ParenR, true) => {} + (PyTokenKind::Spread, true) => {} + (PyTokenKind::Colon, true) => {} + (PyTokenKind::Equals, true) => {} + (PyTokenKind::At, true) => {} + (PyTokenKind::BracketL, true) => {} + (PyTokenKind::BracketR, true) => {} + (PyTokenKind::BraceL, true) => {} + (PyTokenKind::BraceR, true) => {} + (PyTokenKind::Pipe, true) => {} + (PyTokenKind::Bang, true) => {} (_, true) => buf.push(' '), - (_, false) => {}, + (_, false) => {} } buf.push_str(&token.value); needs_whitespace = match token.kind { @@ -469,5 +460,5 @@ fn join_tokens<'a, I: IntoIterator>(tokens: I) -> String { PyTokenKind::Sof => unreachable!(), }; } - return buf; + buf } diff --git a/edb/graphql-rewrite/tests/rewrite.rs b/edb/graphql-rewrite/tests/rewrite.rs index e62ea4de82b..7a535356362 100644 --- a/edb/graphql-rewrite/tests/rewrite.rs +++ b/edb/graphql-rewrite/tests/rewrite.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; -use edb_graphql_parser::{Pos}; +use edb_graphql_parser::Pos; use graphql_rewrite::{rewrite, Variable, Value}; use graphql_rewrite::{PyToken, PyTokenKind}; @@ -8,13 +8,13 @@ use graphql_rewrite::{PyToken, PyTokenKind}; #[test] fn test_no_args() { - let entry = rewrite(None, r###" + let entry = rewrite(None, r#" query { object(filter: {field: {eq: "test"}}) { field } } - "###).unwrap(); + "#).unwrap(); assert_eq!(entry.key, "\ query($_edb_arg__0:String!){\ object(filter:{field:{eq:$_edb_arg__0}}){\ @@ -37,13 +37,13 @@ fn test_no_args() { #[test] fn test_no_query() { - let entry = rewrite(None, r###" + let entry = rewrite(None, r#" { object(filter: {field: {eq: "test"}}) { field } } - "###).unwrap(); + "#).unwrap(); assert_eq!(entry.key, "\ query($_edb_arg__0:String!){\ object(filter:{field:{eq:$_edb_arg__0}}){\ @@ -66,13 +66,13 @@ fn test_no_query() { #[test] fn test_no_name() { - let entry = rewrite(None, r###" + let entry = rewrite(None, r#" query($x: String) { object(filter: {field: {eq: "test"}}, y: $x) { field } } - "###).unwrap(); + "#).unwrap(); assert_eq!(entry.key, "\ query($x:String $_edb_arg__0:String!){\ object(filter:{field:{eq:$_edb_arg__0}}y:$x){\ @@ -95,13 +95,13 @@ fn test_no_name() { #[test] fn test_name_args() { - let entry = rewrite(Some("Hello"), r###" + let entry = rewrite(Some("Hello"), r#" query Hello($x: String, $y: String!) { object(filter: {field: {eq: "test"}}, x: $x, y: $y) { field } } - "###).unwrap(); + "#).unwrap(); assert_eq!(entry.key, "\ query Hello($x:String $y:String!$_edb_arg__0:String!){\ object(filter:{field:{eq:$_edb_arg__0}}x:$x y:$y){\ @@ -124,13 +124,13 @@ fn test_name_args() { #[test] fn test_name() { - let entry = rewrite(Some("Hello"), r###" + let entry = rewrite(Some("Hello"), r#" query Hello { object(filter: {field: {eq: "test"}}) { field } } - "###).unwrap(); + "#).unwrap(); assert_eq!(entry.key, "\ query Hello($_edb_arg__0:String!){\ object(filter:{field:{eq:$_edb_arg__0}}){\ @@ -153,13 +153,13 @@ fn test_name() { #[test] fn test_default_name() { - let entry = rewrite(None, r###" + let entry = rewrite(None, r#" query Hello { object(filter: {field: {eq: "test"}}) { field } } - "###).unwrap(); + "#).unwrap(); assert_eq!(entry.key, "\ query Hello($_edb_arg__0:String!){\ object(filter:{field:{eq:$_edb_arg__0}}){\ @@ -182,7 +182,7 @@ fn test_default_name() { #[test] fn test_other() { - let entry = rewrite(Some("Hello"), r###" + let entry = rewrite(Some("Hello"), r#" query Other { object(filter: {field: {eq: "test1"}}) { field @@ -193,7 +193,7 @@ fn test_other() { field } } - "###).unwrap(); + "#).unwrap(); assert_eq!(entry.key, "\ query Other{\ object(filter:{field:{eq:\"test1\"}}){\ @@ -221,13 +221,13 @@ fn test_other() { #[test] fn test_defaults() { - let entry = rewrite(Some("Hello"), r###" + let entry = rewrite(Some("Hello"), r#" query Hello($x: String = "xxx", $y: String! = "yyy") { object(filter: {field: {eq: "test"}}, x: $x, y: $y) { field } } - "###).unwrap(); + "#).unwrap(); assert_eq!(entry.key, "\ query Hello($x:String!$y:String!$_edb_arg__0:String!){\ object(filter:{field:{eq:$_edb_arg__0}}x:$x y:$y){\ diff --git a/edb/graphql/extension.pyx b/edb/graphql/extension.pyx index e19b1eb4a9c..f9045e819c9 100644 --- a/edb/graphql/extension.pyx +++ b/edb/graphql/extension.pyx @@ -271,10 +271,10 @@ async def _execute(db, tenant, query, operation_name, variables, globals): try: rewritten = _graphql_rewrite.rewrite(operation_name, query) - vars = rewritten.variables().copy() + vars = rewritten.variables.copy() if variables: vars.update(variables) - key_var_names = rewritten.key_vars() + key_var_names = rewritten.key_vars # on bad queries the following line can trigger KeyError key_vars = tuple(vars[k] for k in key_var_names) except _graphql_rewrite.QueryError as e: @@ -291,11 +291,11 @@ async def _execute(db, tenant, query, operation_name, variables, globals): key_var_names = [] key_vars = () else: - prepared_query = rewritten.key() + prepared_query = rewritten.key if debug.flags.graphql_compile: debug.header('GraphQL optimized query') - print(rewritten.key()) + print(rewritten) print(f'key_vars: {key_var_names}') print(f'variables: {vars}') @@ -320,7 +320,7 @@ async def _execute(db, tenant, query, operation_name, variables, globals): tenant, query, rewritten.tokens(gql_lexer.TokenKind), - rewritten.substitutions(), + rewritten.substitutions, operation_name, vars, ) diff --git a/edb/tools/parser_demo.py b/edb/tools/parser_demo.py index 8dab75c9c27..b5e31adc64f 100644 --- a/edb/tools/parser_demo.py +++ b/edb/tools/parser_demo.py @@ -30,6 +30,7 @@ @edbcommands.command("parser-demo") def main(): + qlparser.preload_spec() for q in QUERIES[-10:]: sdl = q.startswith('sdl') @@ -37,8 +38,8 @@ def main(): q = q[3:] try: - # s = tokenizer.NormalizedSource.from_string(q) - source = tokenizer.Source.from_string(q) + source = tokenizer.NormalizedSource.from_string(q) + # source = tokenizer.Source.from_string(q) except BaseException as e: print('Error during tokenization:') print(e) @@ -52,11 +53,11 @@ def main(): print('-' * 30) print() - for index, error in enumerate(result.errors()): + for index, error in enumerate(result.errors): message, span, hint, details = error (start, end) = tokenizer.inflate_span(source.text(), span) - print(f'Error [{index+1}/{len(result.errors())}]:') + print(f'Error [{index+1}/{len(result.errors)}]:') print( '\n'.join( source.text().splitlines()[(start.line - 1) : end.line] @@ -74,9 +75,9 @@ def main(): print(f' Hint: {hint}') print() - if result.out(): + if result.out: try: - ast = qlparser._cst_to_ast(result.out(), productions).val + ast = qlparser._cst_to_ast(result.out, productions).val except BaseException: ast = None if ast: diff --git a/setup.py b/setup.py index 041923552f6..6600a77ed11 100644 --- a/setup.py +++ b/setup.py @@ -1083,12 +1083,12 @@ def _version(): setuptools_rust.RustExtension( "edb._edgeql_parser", path="edb/edgeql-parser/edgeql-parser-python/Cargo.toml", - binding=setuptools_rust.Binding.RustCPython, + binding=setuptools_rust.Binding.PyO3, ), setuptools_rust.RustExtension( "edb._graphql_rewrite", path="edb/graphql-rewrite/Cargo.toml", - binding=setuptools_rust.Binding.RustCPython, + binding=setuptools_rust.Binding.PyO3, ), ], ) From b174d8d0be14f1d02e865d706448155e803faf95 Mon Sep 17 00:00:00 2001 From: Fantix King Date: Tue, 13 Feb 2024 06:41:51 +0900 Subject: [PATCH 4/8] Fix query cache dbver issue with concurrent DDL (#6819) During a query compilation, if the schema is changed, the server was caching the compiled query under a wrong dbver, causing future queries to fail. See the test for issue reproduction. --- edb/server/compiler/compiler.py | 9 +++++ edb/server/dbview/dbview.pxd | 6 ++-- edb/server/dbview/dbview.pyx | 20 +++++++---- tests/test_server_proto.py | 60 +++++++++++++++++++++++++++++++++ 4 files changed, 86 insertions(+), 9 deletions(-) diff --git a/edb/server/compiler/compiler.py b/edb/server/compiler/compiler.py index 8e53e8911a1..8fff7daf5ea 100644 --- a/edb/server/compiler/compiler.py +++ b/edb/server/compiler/compiler.py @@ -26,6 +26,7 @@ import hashlib import pickle import textwrap +import time import uuid import immutables @@ -2335,6 +2336,14 @@ def _try_compile( ctx: CompileContext, source: edgeql.Source, ) -> dbstate.QueryUnitGroup: + if _get_config_val(ctx, '__internal_testmode'): + # This is a bad but simple way to emulate a slow compilation for tests. + # Ideally, we should have a testmode function that is hooked to sleep + # as `simple_special_case`, or wait for a notification from the test. + sentinel = "# EDGEDB_TEST_COMPILER_SLEEP = " + text = source.text() + if text.startswith(sentinel): + time.sleep(float(text[len(sentinel):text.index("\n")])) default_cardinality = enums.Cardinality.NO_RESULT statements = edgeql.parse_block(source) diff --git a/edb/server/dbview/dbview.pxd b/edb/server/dbview/dbview.pxd index 169df4516cf..dd26c7ae50f 100644 --- a/edb/server/dbview/dbview.pxd +++ b/edb/server/dbview/dbview.pxd @@ -94,7 +94,7 @@ cdef class Database: cdef schedule_config_update(self) cdef _invalidate_caches(self) - cdef _cache_compiled_query(self, key, query_unit) + cdef _cache_compiled_query(self, key, compiled, int dbver) cdef _new_view(self, query_cache, protocol_version) cdef _remove_view(self, view) cdef _update_backend_ids(self, new_types) @@ -171,7 +171,9 @@ cdef class DatabaseConnectionView: cpdef in_tx(self) cpdef in_tx_error(self) - cdef cache_compiled_query(self, object key, object query_unit) + cdef cache_compiled_query( + self, object key, object query_unit_group, int dbver + ) cdef lookup_compiled_query(self, object key) cdef tx_error(self) diff --git a/edb/server/dbview/dbview.pyx b/edb/server/dbview/dbview.pyx index d526fc182c4..bb56c62372d 100644 --- a/edb/server/dbview/dbview.pyx +++ b/edb/server/dbview/dbview.pyx @@ -230,15 +230,18 @@ cdef class Database: self._sql_to_compiled.clear() self._index.invalidate_caches() - cdef _cache_compiled_query(self, key, compiled: dbstate.QueryUnitGroup): + cdef _cache_compiled_query( + self, key, compiled: dbstate.QueryUnitGroup, int dbver + ): + # `dbver` must be the schema version `compiled` was compiled upon assert compiled.cacheable - existing, dbver = self._eql_to_compiled.get(key, DICTDEFAULT) - if existing is not None and dbver == self.dbver: + existing, existing_dbver = self._eql_to_compiled.get(key, DICTDEFAULT) + if existing is not None and existing_dbver == self.dbver: # We already have a cached query for a more recent DB version. return - self._eql_to_compiled[key] = compiled, self.dbver + self._eql_to_compiled[key] = compiled, dbver def cache_compiled_sql(self, key, compiled: list[str]): existing, dbver = self._sql_to_compiled.get(key, DICTDEFAULT) @@ -714,12 +717,14 @@ cdef class DatabaseConnectionView: cpdef in_tx_error(self): return self._tx_error - cdef cache_compiled_query(self, object key, object query_unit_group): + cdef cache_compiled_query( + self, object key, object query_unit_group, int dbver + ): assert query_unit_group.cacheable if not self._in_tx_with_ddl: key = (key, self.get_modaliases(), self.get_session_config()) - self._db._cache_compiled_query(key, query_unit_group) + self._db._cache_compiled_query(key, query_unit_group, dbver) cdef lookup_compiled_query(self, object key): if (self._tx_error or @@ -975,6 +980,7 @@ cdef class DatabaseConnectionView: if query_unit_group is None: # Cache miss; need to compile this query. cached = False + dbver = self._db.dbver try: query_unit_group = await self._compile(query_req) @@ -1016,7 +1022,7 @@ cdef class DatabaseConnectionView: if cached_globally: self.server.system_compile_cache[query_req] = query_unit_group else: - self.cache_compiled_query(query_req, query_unit_group) + self.cache_compiled_query(query_req, query_unit_group, dbver) if use_metrics: metrics.edgeql_query_compilations.inc( diff --git a/tests/test_server_proto.py b/tests/test_server_proto.py index d773522c7f6..8262bbb9f7d 100644 --- a/tests/test_server_proto.py +++ b/tests/test_server_proto.py @@ -20,6 +20,7 @@ import decimal import http import json +import time import uuid import unittest @@ -2827,6 +2828,65 @@ async def test_server_proto_query_cache_invalidate_09(self): finally: await self.con.query('ROLLBACK') + async def test_server_proto_query_cache_invalidate_10(self): + typename = 'CacheInv_10' + + con1 = self.con + con2 = await self.connect(database=con1.dbname) + try: + await con2.execute(f''' + CREATE TYPE {typename} {{ + CREATE REQUIRED PROPERTY prop1 -> std::str; + }}; + + INSERT {typename} {{ + prop1 := 'aaa' + }}; + ''') + + sleep = 3 + query = ( + f"# EDGEDB_TEST_COMPILER_SLEEP = {sleep}\n" + f"SELECT {typename}.prop1" + ) + task = self.loop.create_task(con1.query(query)) + + start = time.monotonic() + await con2.execute(f''' + DELETE (SELECT {typename}); + + ALTER TYPE {typename} {{ + DROP PROPERTY prop1; + }}; + + ALTER TYPE {typename} {{ + CREATE REQUIRED PROPERTY prop1 -> std::int64; + }}; + + INSERT {typename} {{ + prop1 := 123 + }}; + ''') + if time.monotonic() - start > sleep: + self.skipTest("The host is too slow for this test.") + # If this happens too much, consider increasing the sleep time + + # ISE is NOT the right expected result - a proper EdgeDB error is. + # FIXME in https://github.com/edgedb/edgedb/issues/6820 + with self.assertRaisesRegex( + edgedb.errors.InternalServerError, + "column .* does not exist", + ): + await task + + self.assertEqual( + await con1.query(query), + edgedb.Set([123]), + ) + + finally: + await con2.aclose() + async def test_server_proto_backend_tid_propagation_01(self): async with self._run_and_rollback(): await self.con.execute(''' From 06cd3d398980e139232727c527cc0bd1469d594b Mon Sep 17 00:00:00 2001 From: Elvis Pranskevichus Date: Mon, 12 Feb 2024 14:23:20 -0800 Subject: [PATCH 5/8] Use `asyncio.TaskGroup` instead of the original in-tree implementation (#6821) The `edb.common.TaskGroup` stuff has been upstreamed to Python since 3.11, and since that's the minimum required version, switch over to using the implementation from the standard library. Also, drop `MultiError` as we now have the `ExceptionGroup` --- edb/common/exceptions.py | 6 +- edb/common/multi_error.py | 45 -- edb/common/supervisor.py | 7 +- edb/common/taskgroup.py | 293 ------------- edb/server/compiler_pool/pool.py | 3 +- edb/server/compiler_pool/server.py | 3 +- edb/server/multitenant.py | 8 +- edb/server/protocol/auth_ext/pkce.py | 3 +- edb/server/protocol/binary.pyx | 4 +- edb/server/server.py | 13 +- edb/server/tenant.py | 9 +- edb/testbase/server.py | 3 +- tests/common/test_signalctl.py | 6 +- tests/common/test_supervisor.py | 4 +- tests/common/test_taskgroup.py | 602 --------------------------- tests/test_server_ops.py | 3 +- tests/test_server_pool.py | 9 +- tests/test_server_proto.py | 23 +- 18 files changed, 43 insertions(+), 1001 deletions(-) delete mode 100644 edb/common/multi_error.py delete mode 100644 edb/common/taskgroup.py delete mode 100644 tests/common/test_taskgroup.py diff --git a/edb/common/exceptions.py b/edb/common/exceptions.py index 5d81d8e25e9..8b54c0b96af 100644 --- a/edb/common/exceptions.py +++ b/edb/common/exceptions.py @@ -21,8 +21,6 @@ import sys -from . import multi_error - def _get_contexts(ex, *, auto_init=False): try: @@ -93,8 +91,8 @@ def __init__(self, hint=None, details=None): def _is_internal_error(exc): - if isinstance(exc, multi_error.MultiError): - return any(_is_internal_error(e) for e in exc.__errors__) + if isinstance(exc, ExceptionGroup): + return any(_is_internal_error(e) for e in exc.exceptions) # This is pretty cheesy but avoids needing to import our edgedb # exceptions or do anything elaborate with contexts. return type(exc).__name__ == 'InternalServerError' diff --git a/edb/common/multi_error.py b/edb/common/multi_error.py deleted file mode 100644 index a3e6e8b9065..00000000000 --- a/edb/common/multi_error.py +++ /dev/null @@ -1,45 +0,0 @@ -# -# This source file is part of the EdgeDB open source project. -# -# Copyright 2016-present MagicStack Inc. and the EdgeDB authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -from __future__ import annotations - -import textwrap -import traceback - - -class MultiError(Exception): - - def __init__(self, msg, *args, errors=()): - if errors: - types = set(type(e).__name__ for e in errors) - msg = f'{msg}; {len(errors)} sub errors: ({", ".join(types)})' - for er in errors: - exc_fmt = traceback.format_exception(er) - msg += f'\n + {exc_fmt[0]}' - er_tb = ''.join(exc_fmt[1:]) - er_tb = textwrap.indent(er_tb, ' | ', lambda _: True) - msg += f'{er_tb}\n' - super().__init__(msg, *args) - self.__errors__ = tuple(errors) - - def get_error_types(self): - return {type(e) for e in self.__errors__} - - def __reduce__(self): - return (type(self), (self.args,), {'__errors__': self.__errors__}) diff --git a/edb/common/supervisor.py b/edb/common/supervisor.py index 129d6e7afa8..6eae9f0e497 100644 --- a/edb/common/supervisor.py +++ b/edb/common/supervisor.py @@ -23,8 +23,6 @@ import asyncio import itertools -from . import taskgroup - class Supervisor: @@ -95,10 +93,9 @@ async def wait(self): # cycles (bad for GC); let's not keep a reference to # a bunch of them. errors = self._errors - self._errors = None + self._errors = [] - me = taskgroup.TaskGroupError('unhandled errors in a Supervisor', - errors=errors) + me = ExceptionGroup('unhandled errors in a Supervisor', errors) raise me from None async def _wait(self): diff --git a/edb/common/taskgroup.py b/edb/common/taskgroup.py deleted file mode 100644 index a9d126bca90..00000000000 --- a/edb/common/taskgroup.py +++ /dev/null @@ -1,293 +0,0 @@ -# -# This source file is part of the EdgeDB open source project. -# -# Copyright 2016-present MagicStack Inc. and the EdgeDB authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -from __future__ import annotations - -import asyncio -import itertools -import sys -import types - -from . import multi_error - - -class TaskGroup: - - def __init__(self, *, name=None): - if name is None: - self._name = f'tg-{_name_counter()}' - else: - self._name = str(name) - - self._entered = False - self._exiting = False - self._aborting = False - self._loop = None - self._parent_task = None - self._parent_cancel_requested = False - self._tasks = set() - self._unfinished_tasks = 0 - self._errors = [] - self._base_error = None - self._on_completed_fut = None - - def get_name(self): - return self._name - - def __repr__(self): - msg = f'= (3, 8): - - # In Python 3.8 Tasks propagate all exceptions correctly, - # except for KeybaordInterrupt and SystemExit which are - # still considered special. - - def _is_base_error(self, exc: BaseException) -> bool: - assert isinstance(exc, BaseException) - return isinstance(exc, (SystemExit, KeyboardInterrupt)) - - else: - - # In Python prior to 3.8 all BaseExceptions are special and - # are bypassing the proper propagation through async/await - # code, essentially aborting the execution. - - def _is_base_error(self, exc: BaseException) -> bool: - assert isinstance(exc, BaseException) - return not isinstance(exc, Exception) - - def _patch_task(self, task): - # In Python 3.8 we'll need proper API on asyncio.Task to - # make TaskGroups possible. We need to be able to access - # information about task cancellation, more specifically, - # we need a flag to say if a task was cancelled or not. - # We also need to be able to flip that flag. - - if sys.version_info >= (3, 9): - def _task_cancel(self, msg=None): - self.__cancel_requested__ = True - return asyncio.Task.cancel(self, msg) - else: - def _task_cancel(self): - self.__cancel_requested__ = True - return asyncio.Task.cancel(self) - - if hasattr(task, '__cancel_requested__'): - return - - task.__cancel_requested__ = False - # confirm that we were successful at adding the new attribute: - assert not task.__cancel_requested__ - - task.cancel = types.MethodType(_task_cancel, task) - - def _abort(self): - self._aborting = True - - for t in self._tasks: - if not t.done(): - t.cancel() - - def _on_task_done(self, task): - self._tasks.discard(task) - self._unfinished_tasks -= 1 - assert self._unfinished_tasks >= 0 - - if self._exiting and not self._unfinished_tasks: - if not self._on_completed_fut.done(): - self._on_completed_fut.set_result(True) - - if task.cancelled(): - return - - exc = task.exception() - if exc is None: - return - - self._errors.append(exc) - if self._is_base_error(exc) and self._base_error is None: - self._base_error = exc - - if self._parent_task.done(): - # Not sure if this case is possible, but we want to handle - # it anyways. - self._loop.call_exception_handler({ - 'message': f'Task {task!r} has errored out but its parent ' - f'task {self._parent_task} is already completed', - 'exception': exc, - 'task': task, - }) - return - - self._abort() - if not self._parent_task.__cancel_requested__: - # If parent task *is not* being cancelled, it means that we want - # to manually cancel it to abort whatever is being run right now - # in the TaskGroup. But we want to mark parent task as - # "not cancelled" later in __aexit__. Example situation that - # we need to handle: - # - # async def foo(): - # try: - # async with TaskGroup() as g: - # g.create_task(crash_soon()) - # await something # <- this needs to be canceled - # # by the TaskGroup, e.g. - # # foo() needs to be cancelled - # except Exception: - # # Ignore any exceptions raised in the TaskGroup - # pass - # await something_else # this line has to be called - # # after TaskGroup is finished. - self._parent_cancel_requested = True - self._parent_task.cancel() - - -class TaskGroupError(multi_error.MultiError): - pass - - -_name_counter = itertools.count(1).__next__ diff --git a/edb/server/compiler_pool/pool.py b/edb/server/compiler_pool/pool.py index 97cb7300028..284973cc495 100644 --- a/edb/server/compiler_pool/pool.py +++ b/edb/server/compiler_pool/pool.py @@ -37,7 +37,6 @@ import immutables from edb.common import debug -from edb.common import taskgroup from edb.pgsql import params as pgparams @@ -943,7 +942,7 @@ def __init__(self, *, pool_size, **kwargs): self._max_num_workers = pool_size async def _start(self): - async with taskgroup.TaskGroup() as g: + async with asyncio.TaskGroup() as g: for _i in range(self._pool_size): g.create_task(self._create_worker()) diff --git a/edb/server/compiler_pool/server.py b/edb/server/compiler_pool/server.py index 2b7d6fd92c3..596d159cebd 100644 --- a/edb/server/compiler_pool/server.py +++ b/edb/server/compiler_pool/server.py @@ -37,7 +37,6 @@ from edb.common import debug from edb.common import markup -from edb.common import taskgroup from .. import metrics from .. import args as srvargs @@ -610,7 +609,7 @@ async def server_main( ) await pool.start() try: - async with taskgroup.TaskGroup() as tg: + async with asyncio.TaskGroup() as tg: tg.create_task( _run_server( loop, diff --git a/edb/server/multitenant.py b/edb/server/multitenant.py index bf6ae70ccca..84b7e55730c 100644 --- a/edb/server/multitenant.py +++ b/edb/server/multitenant.py @@ -35,7 +35,6 @@ from edb import errors from edb.common import retryloop from edb.common import signalctl -from edb.common import taskgroup from edb.pgsql import params as pgparams from edb.server import compiler as edbcompiler @@ -79,7 +78,7 @@ class MultiTenantServer(server.BaseServer): _tenants: dict[str, edbtenant.Tenant] _admin_tenant: edbtenant.Tenant | None - _task_group: taskgroup.TaskGroup | None + _task_group: asyncio.TaskGroup | None _task_serial: int def __init__( @@ -106,7 +105,7 @@ def __init__( self._tenants = {} self._admin_tenant = None - self._task_group = taskgroup.TaskGroup() + self._task_group = asyncio.TaskGroup() self._task_serial = 0 self._sys_queries = sys_queries self._report_config_typedesc = report_config_typedesc @@ -153,7 +152,8 @@ def _get_backend_runtime_params(self) -> pgparams.BackendRuntimeParams: async def stop(self): await super().stop() - await self._task_group.__aexit__(*sys.exc_info()) + if self._task_group is not None: + await self._task_group.__aexit__(*sys.exc_info()) try: for tenant in self._tenants.values(): tenant.stop() diff --git a/edb/server/protocol/auth_ext/pkce.py b/edb/server/protocol/auth_ext/pkce.py index 5d001263ba1..57cd33a9fc5 100644 --- a/edb/server/protocol/auth_ext/pkce.py +++ b/edb/server/protocol/auth_ext/pkce.py @@ -24,7 +24,6 @@ import logging import dataclasses -from edb.common import taskgroup from edb.ir import statypes from edb.server.protocol import execute @@ -154,7 +153,7 @@ async def delete(db, id: str) -> None: async def _gc(tenant: edbtenant.Tenant): try: - async with taskgroup.TaskGroup() as g: + async with asyncio.TaskGroup() as g: for db in tenant.iter_dbs(): if "auth" in db.extensions: g.create_task( diff --git a/edb/server/protocol/binary.pyx b/edb/server/protocol/binary.pyx index fc8cf4f3fb5..007c4ccf4e0 100644 --- a/edb/server/protocol/binary.pyx +++ b/edb/server/protocol/binary.pyx @@ -77,7 +77,7 @@ from edb.schema import objects as s_obj from edb import errors from edb.errors import base as base_errors, EdgeQLSyntaxError -from edb.common import debug, taskgroup +from edb.common import debug from edb.common import context as pctx from edb.protocol import messages @@ -1367,7 +1367,7 @@ cdef class EdgeConnection(frontend.FrontendConnection): blocks_queue = collections.deque(blocks) output_queue = asyncio.Queue(maxsize=2) - async with taskgroup.TaskGroup() as g: + async with asyncio.TaskGroup() as g: g.create_task(pgcon.dump( blocks_queue, output_queue, diff --git a/edb/server/server.py b/edb/server/server.py index 5fb9b4d51bc..4beff7647ac 100644 --- a/edb/server/server.py +++ b/edb/server/server.py @@ -44,7 +44,6 @@ from edb.common import devmode from edb.common import lru from edb.common import secretkey -from edb.common import taskgroup from edb.common import windowedsum from edb.schema import reflection as s_refl @@ -689,7 +688,7 @@ async def _start_servers( else: start_tasks = {} try: - async with taskgroup.TaskGroup() as g: + async with asyncio.TaskGroup() as g: if sockets: for host, sock in zip(hosts, sockets): start_tasks[host] = g.create_task( @@ -712,9 +711,9 @@ async def _start_servers( raise servers.update({ - host: fut.result() + host: srv for host, fut in start_tasks.items() - if fut.result() is not None + if (srv := fut.result()) is not None }) # Fail if none of the servers can be started, except when the admin @@ -887,7 +886,7 @@ def get_jws_key(self) -> jwk.JWK | None: return self._jws_key async def _stop_servers(self, servers): - async with taskgroup.TaskGroup() as g: + async with asyncio.TaskGroup() as g: for srv in servers: srv.close() g.create_task(srv.wait_closed()) @@ -1316,7 +1315,7 @@ async def _maybe_patch(self) -> None: dbnames = await self.get_dbnames(syscon) - async with taskgroup.TaskGroup(name='apply patches') as g: + async with asyncio.TaskGroup() as g: # Cap the parallelism used when applying patches, to avoid # having huge numbers of in flight patches that make # little visible progress in the logs. @@ -1738,7 +1737,7 @@ async def _resolve_interfaces( hosts: Sequence[str] ) -> Tuple[Sequence[str], bool, bool]: - async with taskgroup.TaskGroup() as g: + async with asyncio.TaskGroup() as g: resolve_tasks = { host: g.create_task(_resolve_host(host)) for host in hosts diff --git a/edb/server/tenant.py b/edb/server/tenant.py index 659924de0a1..ee8a44cddb9 100644 --- a/edb/server/tenant.py +++ b/edb/server/tenant.py @@ -34,7 +34,6 @@ from edb import errors from edb.common import retryloop -from edb.common import taskgroup from . import args as srvargs from . import config @@ -75,7 +74,7 @@ class Tenant(ha_base.ClusterProtocol): _accepting_connections: bool __loop: asyncio.AbstractEventLoop - _task_group: taskgroup.TaskGroup | None + _task_group: asyncio.TaskGroup | None _tasks: Set[asyncio.Task] _accept_new_tasks: bool _file_watch_finalizers: list[Callable[[], None]] @@ -388,7 +387,7 @@ def reload_state_file(_file_modified, _event): async def start_accepting_new_tasks(self) -> None: assert self._task_group is None - self._task_group = taskgroup.TaskGroup() + self._task_group = asyncio.TaskGroup() await self._task_group.__aenter__() self._accept_new_tasks = True await self._cluster.start_watching(self.on_switch_over) @@ -933,7 +932,7 @@ async def _introspect_dbs(self) -> None: async with self.use_sys_pgcon() as syscon: dbnames = await self._server.get_dbnames(syscon) - async with taskgroup.TaskGroup(name="introspect DB extensions") as g: + async with asyncio.TaskGroup() as g: for dbname in dbnames: # There's a risk of the DB being dropped by another server # between us building the list of databases and loading @@ -1316,7 +1315,7 @@ async def task(): async with self.use_sys_pgcon() as syscon: dbnames = set(await self._server.get_dbnames(syscon)) - tg = taskgroup.TaskGroup(name="new database introspection") + tg = asyncio.TaskGroup() async with tg as g: for dbname in dbnames: if not self._dbindex.has_db(dbname): diff --git a/edb/testbase/server.py b/edb/testbase/server.py index 98534e38e15..89bf9d54393 100644 --- a/edb/testbase/server.py +++ b/edb/testbase/server.py @@ -62,7 +62,6 @@ from edb.common import debug from edb.common import retryloop from edb.common import secretkey -from edb.common import taskgroup from edb.protocol import protocol as test_protocol from edb.testbase import serutils @@ -1852,7 +1851,7 @@ async def setup_test_cases( if verbose: print(f' -> {dbname}: OK', flush=True) else: - async with taskgroup.TaskGroup(name='setup test cases') as g: + async with asyncio.TaskGroup() as g: # Use a semaphore to limit the concurrency of bootstrap # tasks to the number of jobs (bootstrap is heavy, having # more tasks than `--jobs` won't necessarily make diff --git a/tests/common/test_signalctl.py b/tests/common/test_signalctl.py index 387c0bd21b2..b88a84ec4ee 100644 --- a/tests/common/test_signalctl.py +++ b/tests/common/test_signalctl.py @@ -354,7 +354,7 @@ async def _subtask2(): self.notify_parent(3) async def _task(): - async with taskgroup.TaskGroup() as tg: + async with asyncio.TaskGroup() as tg: tg.create_task(_subtask1()) tg.create_task(_subtask2()) @@ -363,9 +363,7 @@ async def _task(): await sc.wait_for(_task()) """ - async with spawn( - test_prog, global_prog="from edb.common import taskgroup" - ) as p: + async with spawn(test_prog) as p: await self.wait_for_child(p, 1) await self.wait_for_child(p, 1) p.terminate() diff --git a/tests/common/test_supervisor.py b/tests/common/test_supervisor.py index 17f6b722bc2..9e19adb1510 100644 --- a/tests/common/test_supervisor.py +++ b/tests/common/test_supervisor.py @@ -20,7 +20,6 @@ import asyncio from edb.common import supervisor -from edb.common import taskgroup from edb.testbase import server as tb @@ -118,8 +117,7 @@ async def runner(): NUM += 10 - with self.assertRaisesRegex(taskgroup.TaskGroupError, - r'1 sub errors: \(ZeroDivisionError\)'): + with self.assertRaisesRegex(ExceptionGroup, r'1 sub-exception'): await self.loop.create_task(runner()) self.assertEqual(NUM, 0) diff --git a/tests/common/test_taskgroup.py b/tests/common/test_taskgroup.py deleted file mode 100644 index 712050ac974..00000000000 --- a/tests/common/test_taskgroup.py +++ /dev/null @@ -1,602 +0,0 @@ -# -# This source file is part of the EdgeDB open source project. -# -# Copyright 2016-present MagicStack Inc. and the EdgeDB authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -import asyncio - -from edb.common import taskgroup -from edb.testbase import server as tb - - -class MyExc(Exception): - pass - - -class TestTaskGroup(tb.TestCase): - - async def test_taskgroup_01(self): - - async def foo1(): - await asyncio.sleep(0.1) - return 42 - - async def foo2(): - await asyncio.sleep(0.2) - return 11 - - async with taskgroup.TaskGroup() as g: - t1 = g.create_task(foo1()) - t2 = g.create_task(foo2()) - - self.assertEqual(t1.result(), 42) - self.assertEqual(t2.result(), 11) - - async def test_taskgroup_02(self): - - async def foo1(): - await asyncio.sleep(0.1) - return 42 - - async def foo2(): - await asyncio.sleep(0.2) - return 11 - - async with taskgroup.TaskGroup() as g: - t1 = g.create_task(foo1()) - await asyncio.sleep(0.15) - t2 = g.create_task(foo2()) - - self.assertEqual(t1.result(), 42) - self.assertEqual(t2.result(), 11) - - async def test_taskgroup_03(self): - - async def foo1(): - await asyncio.sleep(1) - return 42 - - async def foo2(): - await asyncio.sleep(0.2) - return 11 - - async with taskgroup.TaskGroup() as g: - t1 = g.create_task(foo1()) - await asyncio.sleep(0.15) - # cancel t1 explicitly, i.e. everything should continue - # working as expected. - t1.cancel() - - t2 = g.create_task(foo2()) - - self.assertTrue(t1.cancelled()) - self.assertEqual(t2.result(), 11) - - async def test_taskgroup_04(self): - - NUM = 0 - t2_cancel = False - t2 = None - - async def foo1(): - await asyncio.sleep(0.1) - 1 / 0 - - async def foo2(): - nonlocal NUM, t2_cancel - try: - await asyncio.sleep(1) - except asyncio.CancelledError: - t2_cancel = True - raise - NUM += 1 - - async def runner(): - nonlocal NUM, t2 - - async with taskgroup.TaskGroup() as g: - g.create_task(foo1()) - t2 = g.create_task(foo2()) - - NUM += 10 - - with self.assertRaisesRegex(taskgroup.TaskGroupError, - r'1 sub errors: \(ZeroDivisionError\)'): - await self.loop.create_task(runner()) - - self.assertEqual(NUM, 0) - self.assertTrue(t2_cancel) - self.assertTrue(t2.cancelled()) - - async def test_taskgroup_05(self): - - NUM = 0 - t2_cancel = False - runner_cancel = False - - async def foo1(): - await asyncio.sleep(0.1) - 1 / 0 - - async def foo2(): - nonlocal NUM, t2_cancel - try: - await asyncio.sleep(5) - except asyncio.CancelledError: - t2_cancel = True - raise - NUM += 1 - - async def runner(): - nonlocal NUM, runner_cancel - - async with taskgroup.TaskGroup() as g: - g.create_task(foo1()) - g.create_task(foo1()) - g.create_task(foo1()) - g.create_task(foo2()) - try: - await asyncio.sleep(10) - except asyncio.CancelledError: - runner_cancel = True - raise - - NUM += 10 - - # The 3 foo1 sub tasks can be racy when the host is busy - if the - # cancellation happens in the middle, we'll see partial sub errors here - with self.assertRaisesRegex( - taskgroup.TaskGroupError, - r'(1|2|3) sub errors: \(ZeroDivisionError\)', - ): - await self.loop.create_task(runner()) - - self.assertEqual(NUM, 0) - self.assertTrue(t2_cancel) - self.assertTrue(runner_cancel) - - async def test_taskgroup_06(self): - - NUM = 0 - - async def foo(): - nonlocal NUM - try: - await asyncio.sleep(5) - except asyncio.CancelledError: - NUM += 1 - raise - - async def runner(): - async with taskgroup.TaskGroup() as g: - for _ in range(5): - g.create_task(foo()) - - r = self.loop.create_task(runner()) - await asyncio.sleep(0.1) - - self.assertFalse(r.done()) - r.cancel() - with self.assertRaises(asyncio.CancelledError): - await r - - self.assertEqual(NUM, 5) - - async def test_taskgroup_07(self): - - NUM = 0 - - async def foo(): - nonlocal NUM - try: - await asyncio.sleep(5) - except asyncio.CancelledError: - NUM += 1 - raise - - async def runner(): - nonlocal NUM - async with taskgroup.TaskGroup() as g: - for _ in range(5): - g.create_task(foo()) - - try: - await asyncio.sleep(10) - except asyncio.CancelledError: - NUM += 10 - raise - - r = self.loop.create_task(runner()) - await asyncio.sleep(0.1) - - self.assertFalse(r.done()) - r.cancel() - with self.assertRaises(asyncio.CancelledError): - await r - - self.assertEqual(NUM, 15) - - async def test_taskgroup_08(self): - - async def foo(): - await asyncio.sleep(0.1) - 1 / 0 - - async def runner(): - async with taskgroup.TaskGroup() as g: - for _ in range(5): - g.create_task(foo()) - - try: - await asyncio.sleep(10) - except asyncio.CancelledError: - raise - - r = self.loop.create_task(runner()) - await asyncio.sleep(0.1) - - self.assertFalse(r.done()) - r.cancel() - with self.assertRaises(asyncio.CancelledError): - await r - - async def test_taskgroup_09(self): - - t1 = t2 = None - - async def foo1(): - await asyncio.sleep(1) - return 42 - - async def foo2(): - await asyncio.sleep(2) - return 11 - - async def runner(): - nonlocal t1, t2 - async with taskgroup.TaskGroup() as g: - t1 = g.create_task(foo1()) - t2 = g.create_task(foo2()) - await asyncio.sleep(0.1) - 1 / 0 - - try: - await runner() - except taskgroup.TaskGroupError as t: - self.assertEqual(t.get_error_types(), {ZeroDivisionError}) - else: - self.fail('TaskGroupError was not raised') - - self.assertTrue(t1.cancelled()) - self.assertTrue(t2.cancelled()) - - async def test_taskgroup_10(self): - - t1 = t2 = None - - async def foo1(): - await asyncio.sleep(1) - return 42 - - async def foo2(): - await asyncio.sleep(2) - return 11 - - async def runner(): - nonlocal t1, t2 - async with taskgroup.TaskGroup() as g: - t1 = g.create_task(foo1()) - t2 = g.create_task(foo2()) - 1 / 0 - - try: - await runner() - except taskgroup.TaskGroupError as t: - self.assertEqual(t.get_error_types(), {ZeroDivisionError}) - else: - self.fail('TaskGroupError was not raised') - - self.assertTrue(t1.cancelled()) - self.assertTrue(t2.cancelled()) - - async def test_taskgroup_11(self): - - async def foo(): - await asyncio.sleep(0.1) - 1 / 0 - - async def runner(): - async with taskgroup.TaskGroup(): - async with taskgroup.TaskGroup() as g2: - for _ in range(5): - g2.create_task(foo()) - - try: - await asyncio.sleep(10) - except asyncio.CancelledError: - raise - - r = self.loop.create_task(runner()) - await asyncio.sleep(0.1) - - self.assertFalse(r.done()) - r.cancel() - with self.assertRaises(asyncio.CancelledError): - await r - - async def test_taskgroup_12(self): - - async def foo(): - await asyncio.sleep(0.1) - 1 / 0 - - async def runner(): - async with taskgroup.TaskGroup() as g1: - g1.create_task(asyncio.sleep(10)) - - async with taskgroup.TaskGroup() as g2: - for _ in range(5): - g2.create_task(foo()) - - try: - await asyncio.sleep(10) - except asyncio.CancelledError: - raise - - r = self.loop.create_task(runner()) - await asyncio.sleep(0.1) - - self.assertFalse(r.done()) - r.cancel() - with self.assertRaises(asyncio.CancelledError): - await r - - async def test_taskgroup_13(self): - - async def crash_after(t): - await asyncio.sleep(t) - raise ValueError(t) - - async def runner(): - async with taskgroup.TaskGroup(name='g1') as g1: - g1.create_task(crash_after(0.1)) - - async with taskgroup.TaskGroup(name='g2') as g2: - g2.create_task(crash_after(0.2)) - - r = self.loop.create_task(runner()) - with self.assertRaisesRegex(taskgroup.TaskGroupError, r'1 sub errors'): - await r - - async def test_taskgroup_14(self): - - async def crash_after(t): - await asyncio.sleep(t) - raise ValueError(t) - - async def runner(): - async with taskgroup.TaskGroup(name='g1') as g1: - g1.create_task(crash_after(0.2)) - - async with taskgroup.TaskGroup(name='g2') as g2: - g2.create_task(crash_after(0.1)) - - r = self.loop.create_task(runner()) - with self.assertRaisesRegex(taskgroup.TaskGroupError, r'1 sub errors'): - await r - - async def test_taskgroup_15(self): - - async def crash_soon(): - await asyncio.sleep(0.3) - 1 / 0 - - async def runner(): - async with taskgroup.TaskGroup(name='g1') as g1: - g1.create_task(crash_soon()) - try: - await asyncio.sleep(10) - except asyncio.CancelledError: - await asyncio.sleep(0.5) - raise - - r = self.loop.create_task(runner()) - await asyncio.sleep(0.1) - - self.assertFalse(r.done()) - r.cancel() - with self.assertRaises(asyncio.CancelledError): - await r - - async def test_taskgroup_16(self): - - async def crash_soon(): - await asyncio.sleep(0.3) - 1 / 0 - - async def nested_runner(): - async with taskgroup.TaskGroup(name='g1') as g1: - g1.create_task(crash_soon()) - try: - await asyncio.sleep(10) - except asyncio.CancelledError: - await asyncio.sleep(0.5) - raise - - async def runner(): - t = self.loop.create_task(nested_runner()) - await t - - r = self.loop.create_task(runner()) - await asyncio.sleep(0.1) - - self.assertFalse(r.done()) - r.cancel() - with self.assertRaises(asyncio.CancelledError): - await r - - async def test_taskgroup_17(self): - NUM = 0 - - async def runner(): - nonlocal NUM - async with taskgroup.TaskGroup(): - try: - await asyncio.sleep(10) - except asyncio.CancelledError: - NUM += 10 - raise - - r = self.loop.create_task(runner()) - await asyncio.sleep(0.1) - - self.assertFalse(r.done()) - r.cancel() - with self.assertRaises(asyncio.CancelledError): - await r - - self.assertEqual(NUM, 10) - - async def test_taskgroup_18(self): - NUM = 0 - - async def runner(): - nonlocal NUM - async with taskgroup.TaskGroup(): - try: - await asyncio.sleep(10) - except asyncio.CancelledError: - NUM += 10 - # This isn't a good idea, but we have to support - # this weird case. - raise MyExc - - r = self.loop.create_task(runner()) - await asyncio.sleep(0.1) - - self.assertFalse(r.done()) - r.cancel() - - try: - await r - except taskgroup.TaskGroupError as t: - self.assertEqual(t.get_error_types(), {MyExc}) - else: - self.fail('TaskGroupError was not raised') - - self.assertEqual(NUM, 10) - - async def test_taskgroup_19(self): - async def crash_soon(): - await asyncio.sleep(0.1) - 1 / 0 - - async def nested(): - try: - await asyncio.sleep(10) - finally: - raise MyExc - - async def runner(): - async with taskgroup.TaskGroup() as g: - g.create_task(crash_soon()) - await nested() - - r = self.loop.create_task(runner()) - try: - await r - except taskgroup.TaskGroupError as t: - self.assertEqual(t.get_error_types(), {MyExc, ZeroDivisionError}) - else: - self.fail('TasgGroupError was not raised') - - async def test_taskgroup_20(self): - async def crash_soon(): - await asyncio.sleep(0.1) - 1 / 0 - - async def nested(): - try: - await asyncio.sleep(10) - finally: - raise KeyboardInterrupt - - async def runner(): - async with taskgroup.TaskGroup() as g: - g.create_task(crash_soon()) - await nested() - - with self.assertRaises(KeyboardInterrupt): - await runner() - - async def _test_taskgroup_21(self): - # This test doesn't work as asyncio, currently, doesn't - # know how to handle BaseExceptions. - - async def crash_soon(): - await asyncio.sleep(0.1) - raise KeyboardInterrupt - - async def nested(): - try: - await asyncio.sleep(10) - finally: - raise TypeError - - async def runner(): - async with taskgroup.TaskGroup() as g: - g.create_task(crash_soon()) - await nested() - - with self.assertRaises(KeyboardInterrupt): - await runner() - - async def test_taskgroup_22(self): - - async def foo1(): - await asyncio.sleep(1) - return 42 - - async def foo2(): - await asyncio.sleep(2) - return 11 - - async def runner(): - async with taskgroup.TaskGroup() as g: - g.create_task(foo1()) - g.create_task(foo2()) - - r = self.loop.create_task(runner()) - await asyncio.sleep(0.05) - r.cancel() - - with self.assertRaises(asyncio.CancelledError): - await r - - async def test_taskgroup_23(self): - - async def do_job(delay): - await asyncio.sleep(delay) - - async with taskgroup.TaskGroup() as g: - for count in range(10): - await asyncio.sleep(0.1) - g.create_task(do_job(0.3)) - if count == 5: - self.assertLess(len(g._tasks), 5) - await asyncio.sleep(1.35) - self.assertEqual(len(g._tasks), 0) diff --git a/tests/test_server_ops.py b/tests/test_server_ops.py index bd441593159..29a7b2cf6e2 100644 --- a/tests/test_server_ops.py +++ b/tests/test_server_ops.py @@ -41,7 +41,6 @@ from edb import protocol from edb.common import devmode -from edb.common import taskgroup from edb.protocol import protocol as edb_protocol # type: ignore from edb.server import args, pgcluster, pgconnparams from edb.server import cluster as edbcluster @@ -519,7 +518,7 @@ async def test(pgdata_path, tenant): await cluster.start() try: - async with taskgroup.TaskGroup() as tg: + async with asyncio.TaskGroup() as tg: tg.create_task(test(td, 'tenant1')) tg.create_task(test(td, 'tenant2')) finally: diff --git a/tests/test_server_pool.py b/tests/test_server_pool.py index 013ba922ac6..60cab695203 100644 --- a/tests/test_server_pool.py +++ b/tests/test_server_pool.py @@ -52,7 +52,6 @@ import unittest import unittest.mock -from edb.common import taskgroup from edb.server import connpool from edb.server.connpool import pool as pool_impl @@ -510,7 +509,7 @@ def on_stats(stat): TICK_EVERY = 0.001 started_at = time.monotonic() - async with taskgroup.TaskGroup() as g: + async with asyncio.TaskGroup() as g: elapsed = 0 while elapsed < spec.duration: elapsed = time.monotonic() - started_at @@ -616,7 +615,7 @@ async def _base_test_single( getters = 0 TICK_EVERY = 0.001 started_at = time.monotonic() - async with taskgroup.TaskGroup() as g: + async with asyncio.TaskGroup() as g: elapsed = 0 while elapsed < total_duration * TIME_SCALE: elapsed = time.monotonic() - started_at @@ -1355,7 +1354,7 @@ async def test(delay: float): max_capacity=10, ) - async with taskgroup.TaskGroup() as g: + async with asyncio.TaskGroup() as g: g.create_task(q0(pool, event)) await asyncio.sleep(delay) g.create_task(q1(pool, event)) @@ -1390,7 +1389,7 @@ async def test(delay: float): max_capacity=5, ) - async with taskgroup.TaskGroup() as g: + async with asyncio.TaskGroup() as g: for _ in range(4): g.create_task(q('A', pool, wait_event=e1)) diff --git a/tests/test_server_proto.py b/tests/test_server_proto.py index 8262bbb9f7d..be627bc59ef 100644 --- a/tests/test_server_proto.py +++ b/tests/test_server_proto.py @@ -27,7 +27,6 @@ import edgedb from edb.common import devmode -from edb.common import taskgroup as tg from edb.common import asyncutil from edb.testbase import server as tb from edb.server.compiler import enums @@ -869,7 +868,7 @@ async def test_server_proto_wait_cancel_01(self): 'select sys::_advisory_lock($0)', lock_key) try: - async with tg.TaskGroup() as g: + async with asyncio.TaskGroup() as g: async def exec_to_fail(): with self.assertRaises(edgedb.ClientConnectionClosedError): @@ -3256,7 +3255,7 @@ async def test_server_proto_concurrent_ddl(self): typename_prefix = 'ConcurrentDDL' ntasks = 5 - async with tg.TaskGroup() as g: + async with asyncio.TaskGroup() as g: cons_tasks = [ g.create_task(self.connect(database=self.con.dbname)) for _ in range(ntasks) @@ -3265,7 +3264,7 @@ async def test_server_proto_concurrent_ddl(self): cons = [c.result() for c in cons_tasks] try: - async with tg.TaskGroup() as g: + async with asyncio.TaskGroup() as g: for i, con in enumerate(cons): # deferred_shield ensures that none of the # operations get cancelled, which allows us to @@ -3279,15 +3278,15 @@ async def test_server_proto_concurrent_ddl(self): prop1 := {i} }}; '''))) - except tg.TaskGroupError as e: + except ExceptionGroup as e: self.assertIn( edgedb.TransactionSerializationError, - e.get_error_types(), + [type(e) for e in e.exceptions], ) else: self.fail("TransactionSerializationError not raised") finally: - async with tg.TaskGroup() as g: + async with asyncio.TaskGroup() as g: for con in cons: g.create_task(con.aclose()) @@ -3302,7 +3301,7 @@ async def test_server_proto_concurrent_global_ddl(self): ntasks = 5 - async with tg.TaskGroup() as g: + async with asyncio.TaskGroup() as g: cons_tasks = [ g.create_task(self.connect(database=self.con.dbname)) for _ in range(ntasks) @@ -3311,7 +3310,7 @@ async def test_server_proto_concurrent_global_ddl(self): cons = [c.result() for c in cons_tasks] try: - async with tg.TaskGroup() as g: + async with asyncio.TaskGroup() as g: for i, con in enumerate(cons): # deferred_shield ensures that none of the # operations get cancelled, which allows us to @@ -3319,15 +3318,15 @@ async def test_server_proto_concurrent_global_ddl(self): g.create_task(asyncutil.deferred_shield(con.execute(f''' CREATE SUPERUSER ROLE concurrent_{i} '''))) - except tg.TaskGroupError as e: + except ExceptionGroup as e: self.assertIn( edgedb.TransactionSerializationError, - e.get_error_types(), + [type(e) for e in e.exceptions], ) else: self.fail("TransactionSerializationError not raised") finally: - async with tg.TaskGroup() as g: + async with asyncio.TaskGroup() as g: for con in cons: g.create_task(con.aclose()) From 1f90b3a8d098b3744490f5dc34adf50561bd6b08 Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Mon, 12 Feb 2024 14:32:22 -0800 Subject: [PATCH 6/8] Fix another `DROP DATABASE` flake in the tests (#6822) In #6782, we got rid of test suite retry loops on DROP DATABASE. There was one cause of retries I missed, which was if the admin connection got closed for being idle. Fix this by not bothering to hold the admin connection open for the entire test run. --- edb/testbase/server.py | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/edb/testbase/server.py b/edb/testbase/server.py index 89bf9d54393..408c89a3d6a 100644 --- a/edb/testbase/server.py +++ b/edb/testbase/server.py @@ -1045,7 +1045,6 @@ class DatabaseTestCase(ConnectedTestCase): async def setup_and_connect(cls): dbname = cls.get_database_name() - cls.admin_conn = None cls.con = None class_set_up = os.environ.get('EDGEDB_TEST_CASES_SET_UP', 'run') @@ -1053,16 +1052,17 @@ async def setup_and_connect(cls): # Only open an extra admin connection if necessary. if class_set_up == 'run': script = f'CREATE DATABASE {dbname};' - cls.admin_conn = await cls.connect() - await cls.admin_conn.execute(script) + admin_conn = await cls.connect() + await admin_conn.execute(script) + await admin_conn.aclose() elif class_set_up == 'inplace': dbname = edgedb_defines.EDGEDB_SUPERUSER_DB elif cls.uses_database_copies(): - cls.admin_conn = await cls.connect() + admin_conn = await cls.connect() - orig_testmode = await cls.admin_conn.query( + orig_testmode = await admin_conn.query( 'SELECT cfg::Config.__internal_testmode', ) if not orig_testmode: @@ -1072,7 +1072,7 @@ async def setup_and_connect(cls): # Enable testmode to unblock the template database syntax below. if not orig_testmode: - await cls.admin_conn.execute( + await admin_conn.execute( 'CONFIGURE SESSION SET __internal_testmode := true;', ) @@ -1087,7 +1087,7 @@ async def create_db(): timeout=30, ): async with tr: - await cls.admin_conn.execute( + await admin_conn.execute( f''' CREATE DATABASE {qlquote.quote_ident(dbname)} FROM {qlquote.quote_ident(base_db_name)} @@ -1096,10 +1096,12 @@ async def create_db(): await create_db() if not orig_testmode: - await cls.admin_conn.execute( + await admin_conn.execute( 'CONFIGURE SESSION SET __internal_testmode := false;', ) + await admin_conn.aclose() + cls.con = await cls.connect(database=dbname) if class_set_up != 'skip': @@ -1126,19 +1128,18 @@ async def teardown_and_disconnect(cls): if class_set_up == 'inplace': await cls.tearDownSingleDB() finally: - try: - await cls.con.aclose() + await cls.con.aclose() - if class_set_up == 'inplace': - pass - - elif class_set_up == 'run' or cls.uses_database_copies(): - dbname = qlquote.quote_ident(cls.get_database_name()) - await drop_db(cls.admin_conn, dbname) + if class_set_up == 'inplace': + pass - finally: - if cls.admin_conn is not None: - await cls.admin_conn.aclose() + elif class_set_up == 'run' or cls.uses_database_copies(): + dbname = qlquote.quote_ident(cls.get_database_name()) + admin_conn = await cls.connect() + try: + await drop_db(admin_conn, dbname) + finally: + await admin_conn.aclose() @classmethod def get_database_name(cls): From e574b18bca5bce9cfedef2c5d18ad0037d3bd4d3 Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Mon, 12 Feb 2024 15:00:45 -0800 Subject: [PATCH 7/8] Put a limit on in-flight branch creations (#6824) This keeps us from blowing the budget too badly; see the comment. --- edb/server/tenant.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/edb/server/tenant.py b/edb/server/tenant.py index ee8a44cddb9..f298de40917 100644 --- a/edb/server/tenant.py +++ b/edb/server/tenant.py @@ -162,6 +162,8 @@ def __init__( # DB state will be initialized in init(). self._dbindex = None + self._branch_sem = asyncio.Semaphore(value=1) + self._roles = immutables.Map() self._sys_auth = tuple() self._jwt_sub_allowlist_file = None @@ -1192,16 +1194,25 @@ async def on_after_create_db_from_template( real_src_dbname = common.get_database_backend_name( src_dbname, tenant_id=self._tenant_id) - async with self.direct_pgcon(tgt_dbname) as con: - await bootstrap.create_branch( - self._cluster, - self._server._refl_schema, - con, - real_src_dbname, - real_tgt_dbname, - mode, - self._server._sys_queries['backend_id_fixup'], - ) + # HACK: Limit the maximum number of in-flight branch + # creations. This is because branches use up to 3 concurrent + # connections (one direct, two via pg_dump/pg_restore), and so + # it can substantially blow our budget if many are in flight. + # The right way to handle this issue would probably be to use + # the connection pool to reserve the connections, but we would + # need to carefully consider deadlock concerns if we want to + # allow tasks to acquire multiple pool connections. + async with self._branch_sem: + async with self.direct_pgcon(tgt_dbname) as con: + await bootstrap.create_branch( + self._cluster, + self._server._refl_schema, + con, + real_src_dbname, + real_tgt_dbname, + mode, + self._server._sys_queries['backend_id_fixup'], + ) logger.info('Finished copy from %s to %s', src_dbname, tgt_dbname) From b6885212ccaf6a647570ed1ae065416e81757208 Mon Sep 17 00:00:00 2001 From: Elvis Pranskevichus Date: Mon, 12 Feb 2024 15:08:48 -0800 Subject: [PATCH 8/8] Avoid rebuilding nightlies of the same SCM revision (#6814) Port of edgedb/edgedb-cli#913 Co-authored-by: Fantix King --- .github/workflows.src/build.inc.yml | 70 +++- .github/workflows.src/build.targets.yml | 24 +- .github/workflows/dryrun.yml | 429 +++++++++++++++++++++++- .github/workflows/nightly.yml | 429 +++++++++++++++++++++++- .github/workflows/release.yml | 28 +- .github/workflows/testing.yml | 28 +- 6 files changed, 1002 insertions(+), 6 deletions(-) diff --git a/.github/workflows.src/build.inc.yml b/.github/workflows.src/build.inc.yml index 8f534c46d77..e14c5fcf8d8 100644 --- a/.github/workflows.src/build.inc.yml +++ b/.github/workflows.src/build.inc.yml @@ -3,6 +3,11 @@ runs-on: ubuntu-latest outputs: branch: ${{ steps.whichver.outputs.branch }} +<% if subdist == "nightly" %> +<% for tgt in targets.linux + targets.macos %> + if_<< tgt.name.replace('-', '_') >>: ${{ steps.scm.outputs.if_<< tgt.name.replace('-', '_') >> }} +<% endfor %> +<% endif %> steps: - uses: actions/checkout@v3 @@ -13,12 +18,72 @@ echo branch="${branch}" >> $GITHUB_OUTPUT id: whichver +<% if subdist == "nightly" %> + - name: Determine SCM revision + id: scm + shell: bash + run: | + rev=$(git rev-parse HEAD) + jq_filter='.packages[] | select(.basename == "edgedb-server") | select(.architecture == $ARCH) | .version_details.metadata.scm_revision | . as $rev | select(($rev != null) and ($REV | startswith($rev)))' +<% for tgt in targets.linux %> + val=true +<% if tgt.family == "debian" %> + idx_file=<< tgt.platform_version >>.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/apt/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "<< tgt.arch >>" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing << tgt.name >>' + val=false + fi +<% elif tgt.family == "redhat" %> + idx_file=el<< tgt.platform_version >>.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/rpm/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "<< tgt.arch >>" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing << tgt.name >>' + val=false + fi +<% elif tgt.family == "generic" %> + out=$(curl -s https://packages.edgedb.com/archive/.jsonindexes/<< tgt.platform_version >>-unknown-linux-<< "{}".format(tgt.platform_libc) if tgt.platform_libc else "gnu" >>.nightly.json | jq -r --arg REV "$rev" --arg ARCH "<< tgt.arch >>" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing << tgt.name >>' + val=false + fi +<% endif %> + echo if_<< tgt.name.replace('-', '_') >>="$val" >> $GITHUB_OUTPUT +<% endfor %> +<% for tgt in targets.macos %> + val=true +<% if tgt.platform == "macos" %> + out=$(curl -s https://packages.edgedb.com/archive/.jsonindexes/<< tgt.platform_version >>-apple-darwin.nightly.json | jq -r --arg REV "$rev" --arg ARCH "<< tgt.arch >>" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing << tgt.platform >>-<< tgt.platform_version >>' + val=false + fi +<% elif tgt.platform == "win" %> + out=$(curl -s https://packages.edgedb.com/archive/.jsonindexes/<< tgt.platform_version >>-pc-windows-msvc.nightly.json | jq -r --arg REV "$rev" --arg ARCH "<< tgt.arch >>" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing << tgt.platform >>-<< tgt.platform_version >>' + val=false + fi +<% endif %> + echo if_<< tgt.name.replace('-', '_') >>="$val" >> $GITHUB_OUTPUT +<% endfor %> +<% endif %> + <%- for tgt in targets.linux %> <%- set plat_id = tgt.platform + ("{}".format(tgt.platform_libc) if tgt.platform_libc else "") + ("-{}".format(tgt.platform_version) if tgt.platform_version else "") %> build-<< tgt.name >>: runs-on: << tgt.runs_on if tgt.runs_on else "ubuntu-latest" >> needs: prep +<% if subdist == "nightly" %> + if: needs.prep.outputs.if_<< tgt.name.replace('-', '_') >> == 'true' +<% endif %> steps: - name: Build @@ -55,6 +120,9 @@ build-<< tgt.name >>: runs-on: << tgt.runs_on if tgt.runs_on else "macos-latest" >> needs: prep +<% if subdist == "nightly" %> + if: needs.prep.outputs.if_<< tgt.name.replace('-', '_') >> == 'true' +<% endif %> steps: - uses: actions/checkout@v3 @@ -89,7 +157,7 @@ <%- endif %> PKG_PLATFORM: "<< tgt.platform >>" PKG_PLATFORM_VERSION: "<< tgt.platform_version >>" - PKG_PLATFORM_ARCH: "<< tgt.platform_arch if tgt.platform_arch else '' >>" + PKG_PLATFORM_ARCH: "<< tgt.arch if tgt.arch else '' >>" METAPKG_GIT_CACHE: disabled <%- if tgt.family == "generic" %> BUILD_GENERIC: true diff --git a/.github/workflows.src/build.targets.yml b/.github/workflows.src/build.targets.yml index 85b503cde3c..ce8f7e1c2cc 100644 --- a/.github/workflows.src/build.targets.yml +++ b/.github/workflows.src/build.targets.yml @@ -6,109 +6,130 @@ publications: targets: linux: - name: debian-buster-x86_64 + arch: x86_64 platform: debian platform_version: buster family: debian runs_on: [self-hosted, linux, x64] docker_arch: linux/amd64 - name: debian-buster-aarch64 + arch: aarch64 platform: debian platform_version: buster family: debian runs_on: [self-hosted, linux, arm64] docker_arch: linux/arm64 - name: debian-bullseye-x86_64 + arch: x86_64 platform: debian platform_version: bullseye family: debian runs_on: [self-hosted, linux, x64] - name: debian-bullseye-aarch64 + arch: aarch64 platform: debian platform_version: bullseye family: debian runs_on: [self-hosted, linux, arm64] - name: debian-bookworm-x86_64 + arch: x86_64 platform: debian platform_version: bookworm family: debian runs_on: [self-hosted, linux, x64] - name: debian-bookworm-aarch64 + arch: aarch64 platform: debian platform_version: bookworm family: debian runs_on: [self-hosted, linux, arm64] - name: ubuntu-bionic-x86_64 + arch: x86_64 platform: ubuntu platform_version: bionic family: debian runs_on: [self-hosted, linux, x64] - name: ubuntu-bionic-aarch64 + arch: aarch64 platform: ubuntu platform_version: bionic family: debian runs_on: [self-hosted, linux, arm64] - name: ubuntu-focal-x86_64 + arch: x86_64 platform: ubuntu platform_version: focal family: debian runs_on: [self-hosted, linux, x64] - name: ubuntu-focal-aarch64 + arch: aarch64 platform: ubuntu platform_version: focal family: debian runs_on: [self-hosted, linux, arm64] - name: ubuntu-jammy-x86_64 + arch: x86_64 platform: ubuntu platform_version: jammy family: debian runs_on: [self-hosted, linux, x64] - name: ubuntu-jammy-aarch64 + arch: aarch64 platform: ubuntu platform_version: jammy family: debian runs_on: [self-hosted, linux, arm64] - name: centos-7-x86_64 + arch: x86_64 platform: centos platform_version: 7 family: redhat runs_on: [self-hosted, linux, x64] - name: centos-8-x86_64 + arch: x86_64 platform: centos platform_version: 8 family: redhat runs_on: [self-hosted, linux, x64] - name: centos-8-aarch64 + arch: aarch64 platform: centos platform_version: 8 family: redhat runs_on: [self-hosted, linux, arm64] - name: rockylinux-9-x86_64 + arch: x86_64 platform: rockylinux platform_version: 9 family: redhat runs_on: [self-hosted, linux, x64] - name: rockylinux-9-aarch64 + arch: aarch64 platform: rockylinux platform_version: 9 family: redhat runs_on: [self-hosted, linux, arm64] - name: linux-x86_64 + arch: x86_64 platform: linux platform_version: x86_64 family: generic runs_on: [self-hosted, linux, x64] - name: linux-aarch64 + arch: aarch64 platform: linux platform_version: aarch64 family: generic runs_on: [self-hosted, linux, arm64] - name: linuxmusl-x86_64 + arch: x86_64 platform: linux platform_version: x86_64 platform_libc: musl family: generic runs_on: [self-hosted, linux, x64] - name: linuxmusl-aarch64 + arch: aarch64 platform: linux platform_version: aarch64 platform_libc: musl @@ -117,12 +138,13 @@ targets: macos: - name: macos-x86_64 + arch: x86_64 platform: macos platform_version: x86_64 family: generic - platform_arch: x86_64 runs_on: [self-hosted, macOS, X64] - name: macos-aarch64 + arch: aarch64 platform: macos platform_version: aarch64 family: generic diff --git a/.github/workflows/dryrun.yml b/.github/workflows/dryrun.yml index 50426b7c4f2..4eb0520bf9d 100644 --- a/.github/workflows/dryrun.yml +++ b/.github/workflows/dryrun.yml @@ -9,6 +9,55 @@ jobs: runs-on: ubuntu-latest outputs: branch: ${{ steps.whichver.outputs.branch }} + + + if_debian_buster_x86_64: ${{ steps.scm.outputs.if_debian_buster_x86_64 }} + + if_debian_buster_aarch64: ${{ steps.scm.outputs.if_debian_buster_aarch64 }} + + if_debian_bullseye_x86_64: ${{ steps.scm.outputs.if_debian_bullseye_x86_64 }} + + if_debian_bullseye_aarch64: ${{ steps.scm.outputs.if_debian_bullseye_aarch64 }} + + if_debian_bookworm_x86_64: ${{ steps.scm.outputs.if_debian_bookworm_x86_64 }} + + if_debian_bookworm_aarch64: ${{ steps.scm.outputs.if_debian_bookworm_aarch64 }} + + if_ubuntu_bionic_x86_64: ${{ steps.scm.outputs.if_ubuntu_bionic_x86_64 }} + + if_ubuntu_bionic_aarch64: ${{ steps.scm.outputs.if_ubuntu_bionic_aarch64 }} + + if_ubuntu_focal_x86_64: ${{ steps.scm.outputs.if_ubuntu_focal_x86_64 }} + + if_ubuntu_focal_aarch64: ${{ steps.scm.outputs.if_ubuntu_focal_aarch64 }} + + if_ubuntu_jammy_x86_64: ${{ steps.scm.outputs.if_ubuntu_jammy_x86_64 }} + + if_ubuntu_jammy_aarch64: ${{ steps.scm.outputs.if_ubuntu_jammy_aarch64 }} + + if_centos_7_x86_64: ${{ steps.scm.outputs.if_centos_7_x86_64 }} + + if_centos_8_x86_64: ${{ steps.scm.outputs.if_centos_8_x86_64 }} + + if_centos_8_aarch64: ${{ steps.scm.outputs.if_centos_8_aarch64 }} + + if_rockylinux_9_x86_64: ${{ steps.scm.outputs.if_rockylinux_9_x86_64 }} + + if_rockylinux_9_aarch64: ${{ steps.scm.outputs.if_rockylinux_9_aarch64 }} + + if_linux_x86_64: ${{ steps.scm.outputs.if_linux_x86_64 }} + + if_linux_aarch64: ${{ steps.scm.outputs.if_linux_aarch64 }} + + if_linuxmusl_x86_64: ${{ steps.scm.outputs.if_linuxmusl_x86_64 }} + + if_linuxmusl_aarch64: ${{ steps.scm.outputs.if_linuxmusl_aarch64 }} + + if_macos_x86_64: ${{ steps.scm.outputs.if_macos_x86_64 }} + + if_macos_aarch64: ${{ steps.scm.outputs.if_macos_aarch64 }} + + steps: - uses: actions/checkout@v3 @@ -19,10 +68,322 @@ jobs: echo branch="${branch}" >> $GITHUB_OUTPUT id: whichver + + - name: Determine SCM revision + id: scm + shell: bash + run: | + rev=$(git rev-parse HEAD) + jq_filter='.packages[] | select(.basename == "edgedb-server") | select(.architecture == $ARCH) | .version_details.metadata.scm_revision | . as $rev | select(($rev != null) and ($REV | startswith($rev)))' + + val=true + + idx_file=buster.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/apt/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "x86_64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing debian-buster-x86_64' + val=false + fi + + echo if_debian_buster_x86_64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=buster.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/apt/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "aarch64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing debian-buster-aarch64' + val=false + fi + + echo if_debian_buster_aarch64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=bullseye.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/apt/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "x86_64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing debian-bullseye-x86_64' + val=false + fi + + echo if_debian_bullseye_x86_64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=bullseye.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/apt/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "aarch64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing debian-bullseye-aarch64' + val=false + fi + + echo if_debian_bullseye_aarch64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=bookworm.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/apt/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "x86_64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing debian-bookworm-x86_64' + val=false + fi + + echo if_debian_bookworm_x86_64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=bookworm.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/apt/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "aarch64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing debian-bookworm-aarch64' + val=false + fi + + echo if_debian_bookworm_aarch64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=bionic.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/apt/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "x86_64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing ubuntu-bionic-x86_64' + val=false + fi + + echo if_ubuntu_bionic_x86_64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=bionic.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/apt/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "aarch64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing ubuntu-bionic-aarch64' + val=false + fi + + echo if_ubuntu_bionic_aarch64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=focal.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/apt/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "x86_64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing ubuntu-focal-x86_64' + val=false + fi + + echo if_ubuntu_focal_x86_64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=focal.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/apt/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "aarch64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing ubuntu-focal-aarch64' + val=false + fi + + echo if_ubuntu_focal_aarch64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=jammy.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/apt/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "x86_64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing ubuntu-jammy-x86_64' + val=false + fi + + echo if_ubuntu_jammy_x86_64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=jammy.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/apt/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "aarch64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing ubuntu-jammy-aarch64' + val=false + fi + + echo if_ubuntu_jammy_aarch64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=el7.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/rpm/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "x86_64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing centos-7-x86_64' + val=false + fi + + echo if_centos_7_x86_64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=el8.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/rpm/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "x86_64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing centos-8-x86_64' + val=false + fi + + echo if_centos_8_x86_64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=el8.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/rpm/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "aarch64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing centos-8-aarch64' + val=false + fi + + echo if_centos_8_aarch64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=el9.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/rpm/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "x86_64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing rockylinux-9-x86_64' + val=false + fi + + echo if_rockylinux_9_x86_64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=el9.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/rpm/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "aarch64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing rockylinux-9-aarch64' + val=false + fi + + echo if_rockylinux_9_aarch64="$val" >> $GITHUB_OUTPUT + + val=true + + out=$(curl -s https://packages.edgedb.com/archive/.jsonindexes/x86_64-unknown-linux-gnu.nightly.json | jq -r --arg REV "$rev" --arg ARCH "x86_64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing linux-x86_64' + val=false + fi + + echo if_linux_x86_64="$val" >> $GITHUB_OUTPUT + + val=true + + out=$(curl -s https://packages.edgedb.com/archive/.jsonindexes/aarch64-unknown-linux-gnu.nightly.json | jq -r --arg REV "$rev" --arg ARCH "aarch64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing linux-aarch64' + val=false + fi + + echo if_linux_aarch64="$val" >> $GITHUB_OUTPUT + + val=true + + out=$(curl -s https://packages.edgedb.com/archive/.jsonindexes/x86_64-unknown-linux-musl.nightly.json | jq -r --arg REV "$rev" --arg ARCH "x86_64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing linuxmusl-x86_64' + val=false + fi + + echo if_linuxmusl_x86_64="$val" >> $GITHUB_OUTPUT + + val=true + + out=$(curl -s https://packages.edgedb.com/archive/.jsonindexes/aarch64-unknown-linux-musl.nightly.json | jq -r --arg REV "$rev" --arg ARCH "aarch64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing linuxmusl-aarch64' + val=false + fi + + echo if_linuxmusl_aarch64="$val" >> $GITHUB_OUTPUT + + + val=true + + out=$(curl -s https://packages.edgedb.com/archive/.jsonindexes/x86_64-apple-darwin.nightly.json | jq -r --arg REV "$rev" --arg ARCH "x86_64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing macos-x86_64' + val=false + fi + + echo if_macos_x86_64="$val" >> $GITHUB_OUTPUT + + val=true + + out=$(curl -s https://packages.edgedb.com/archive/.jsonindexes/aarch64-apple-darwin.nightly.json | jq -r --arg REV "$rev" --arg ARCH "aarch64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing macos-aarch64' + val=false + fi + + echo if_macos_aarch64="$val" >> $GITHUB_OUTPUT + + + build-debian-buster-x86_64: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + if: needs.prep.outputs.if_debian_buster_x86_64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/debian-buster@master @@ -44,6 +405,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + if: needs.prep.outputs.if_debian_buster_aarch64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/debian-buster@master @@ -65,6 +429,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + if: needs.prep.outputs.if_debian_bullseye_x86_64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/debian-bullseye@master @@ -86,6 +453,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + if: needs.prep.outputs.if_debian_bullseye_aarch64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/debian-bullseye@master @@ -107,6 +477,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + if: needs.prep.outputs.if_debian_bookworm_x86_64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/debian-bookworm@master @@ -128,6 +501,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + if: needs.prep.outputs.if_debian_bookworm_aarch64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/debian-bookworm@master @@ -149,6 +525,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + if: needs.prep.outputs.if_ubuntu_bionic_x86_64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/ubuntu-bionic@master @@ -170,6 +549,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + if: needs.prep.outputs.if_ubuntu_bionic_aarch64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/ubuntu-bionic@master @@ -191,6 +573,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + if: needs.prep.outputs.if_ubuntu_focal_x86_64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/ubuntu-focal@master @@ -212,6 +597,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + if: needs.prep.outputs.if_ubuntu_focal_aarch64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/ubuntu-focal@master @@ -233,6 +621,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + if: needs.prep.outputs.if_ubuntu_jammy_x86_64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/ubuntu-jammy@master @@ -254,6 +645,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + if: needs.prep.outputs.if_ubuntu_jammy_aarch64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/ubuntu-jammy@master @@ -275,6 +669,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + if: needs.prep.outputs.if_centos_7_x86_64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/centos-7@master @@ -296,6 +693,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + if: needs.prep.outputs.if_centos_8_x86_64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/centos-8@master @@ -317,6 +717,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + if: needs.prep.outputs.if_centos_8_aarch64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/centos-8@master @@ -338,6 +741,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + if: needs.prep.outputs.if_rockylinux_9_x86_64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/rockylinux-9@master @@ -359,6 +765,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + if: needs.prep.outputs.if_rockylinux_9_aarch64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/rockylinux-9@master @@ -380,6 +789,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + if: needs.prep.outputs.if_linux_x86_64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/linux-x86_64@master @@ -402,6 +814,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + if: needs.prep.outputs.if_linux_aarch64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/linux-aarch64@master @@ -424,6 +839,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + if: needs.prep.outputs.if_linuxmusl_x86_64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/linuxmusl-x86_64@master @@ -447,6 +865,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + if: needs.prep.outputs.if_linuxmusl_aarch64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/linuxmusl-aarch64@master @@ -470,6 +891,9 @@ jobs: runs-on: ['self-hosted', 'macOS', 'X64'] needs: prep + if: needs.prep.outputs.if_macos_x86_64 == 'true' + + steps: - uses: actions/checkout@v3 with: @@ -513,6 +937,9 @@ jobs: runs-on: ['self-hosted', 'macOS', 'ARM64'] needs: prep + if: needs.prep.outputs.if_macos_aarch64 == 'true' + + steps: - uses: actions/checkout@v3 with: @@ -541,7 +968,7 @@ jobs: PKG_SUBDIST: "nightly" PKG_PLATFORM: "macos" PKG_PLATFORM_VERSION: "aarch64" - PKG_PLATFORM_ARCH: "" + PKG_PLATFORM_ARCH: "aarch64" METAPKG_GIT_CACHE: disabled BUILD_GENERIC: true run: | diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 8a0b0b7540c..648be336113 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -14,6 +14,55 @@ jobs: runs-on: ubuntu-latest outputs: branch: ${{ steps.whichver.outputs.branch }} + + + if_debian_buster_x86_64: ${{ steps.scm.outputs.if_debian_buster_x86_64 }} + + if_debian_buster_aarch64: ${{ steps.scm.outputs.if_debian_buster_aarch64 }} + + if_debian_bullseye_x86_64: ${{ steps.scm.outputs.if_debian_bullseye_x86_64 }} + + if_debian_bullseye_aarch64: ${{ steps.scm.outputs.if_debian_bullseye_aarch64 }} + + if_debian_bookworm_x86_64: ${{ steps.scm.outputs.if_debian_bookworm_x86_64 }} + + if_debian_bookworm_aarch64: ${{ steps.scm.outputs.if_debian_bookworm_aarch64 }} + + if_ubuntu_bionic_x86_64: ${{ steps.scm.outputs.if_ubuntu_bionic_x86_64 }} + + if_ubuntu_bionic_aarch64: ${{ steps.scm.outputs.if_ubuntu_bionic_aarch64 }} + + if_ubuntu_focal_x86_64: ${{ steps.scm.outputs.if_ubuntu_focal_x86_64 }} + + if_ubuntu_focal_aarch64: ${{ steps.scm.outputs.if_ubuntu_focal_aarch64 }} + + if_ubuntu_jammy_x86_64: ${{ steps.scm.outputs.if_ubuntu_jammy_x86_64 }} + + if_ubuntu_jammy_aarch64: ${{ steps.scm.outputs.if_ubuntu_jammy_aarch64 }} + + if_centos_7_x86_64: ${{ steps.scm.outputs.if_centos_7_x86_64 }} + + if_centos_8_x86_64: ${{ steps.scm.outputs.if_centos_8_x86_64 }} + + if_centos_8_aarch64: ${{ steps.scm.outputs.if_centos_8_aarch64 }} + + if_rockylinux_9_x86_64: ${{ steps.scm.outputs.if_rockylinux_9_x86_64 }} + + if_rockylinux_9_aarch64: ${{ steps.scm.outputs.if_rockylinux_9_aarch64 }} + + if_linux_x86_64: ${{ steps.scm.outputs.if_linux_x86_64 }} + + if_linux_aarch64: ${{ steps.scm.outputs.if_linux_aarch64 }} + + if_linuxmusl_x86_64: ${{ steps.scm.outputs.if_linuxmusl_x86_64 }} + + if_linuxmusl_aarch64: ${{ steps.scm.outputs.if_linuxmusl_aarch64 }} + + if_macos_x86_64: ${{ steps.scm.outputs.if_macos_x86_64 }} + + if_macos_aarch64: ${{ steps.scm.outputs.if_macos_aarch64 }} + + steps: - uses: actions/checkout@v3 @@ -24,10 +73,322 @@ jobs: echo branch="${branch}" >> $GITHUB_OUTPUT id: whichver + + - name: Determine SCM revision + id: scm + shell: bash + run: | + rev=$(git rev-parse HEAD) + jq_filter='.packages[] | select(.basename == "edgedb-server") | select(.architecture == $ARCH) | .version_details.metadata.scm_revision | . as $rev | select(($rev != null) and ($REV | startswith($rev)))' + + val=true + + idx_file=buster.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/apt/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "x86_64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing debian-buster-x86_64' + val=false + fi + + echo if_debian_buster_x86_64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=buster.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/apt/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "aarch64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing debian-buster-aarch64' + val=false + fi + + echo if_debian_buster_aarch64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=bullseye.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/apt/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "x86_64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing debian-bullseye-x86_64' + val=false + fi + + echo if_debian_bullseye_x86_64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=bullseye.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/apt/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "aarch64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing debian-bullseye-aarch64' + val=false + fi + + echo if_debian_bullseye_aarch64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=bookworm.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/apt/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "x86_64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing debian-bookworm-x86_64' + val=false + fi + + echo if_debian_bookworm_x86_64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=bookworm.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/apt/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "aarch64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing debian-bookworm-aarch64' + val=false + fi + + echo if_debian_bookworm_aarch64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=bionic.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/apt/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "x86_64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing ubuntu-bionic-x86_64' + val=false + fi + + echo if_ubuntu_bionic_x86_64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=bionic.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/apt/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "aarch64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing ubuntu-bionic-aarch64' + val=false + fi + + echo if_ubuntu_bionic_aarch64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=focal.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/apt/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "x86_64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing ubuntu-focal-x86_64' + val=false + fi + + echo if_ubuntu_focal_x86_64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=focal.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/apt/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "aarch64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing ubuntu-focal-aarch64' + val=false + fi + + echo if_ubuntu_focal_aarch64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=jammy.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/apt/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "x86_64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing ubuntu-jammy-x86_64' + val=false + fi + + echo if_ubuntu_jammy_x86_64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=jammy.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/apt/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "aarch64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing ubuntu-jammy-aarch64' + val=false + fi + + echo if_ubuntu_jammy_aarch64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=el7.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/rpm/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "x86_64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing centos-7-x86_64' + val=false + fi + + echo if_centos_7_x86_64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=el8.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/rpm/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "x86_64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing centos-8-x86_64' + val=false + fi + + echo if_centos_8_x86_64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=el8.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/rpm/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "aarch64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing centos-8-aarch64' + val=false + fi + + echo if_centos_8_aarch64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=el9.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/rpm/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "x86_64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing rockylinux-9-x86_64' + val=false + fi + + echo if_rockylinux_9_x86_64="$val" >> $GITHUB_OUTPUT + + val=true + + idx_file=el9.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/rpm/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "aarch64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing rockylinux-9-aarch64' + val=false + fi + + echo if_rockylinux_9_aarch64="$val" >> $GITHUB_OUTPUT + + val=true + + out=$(curl -s https://packages.edgedb.com/archive/.jsonindexes/x86_64-unknown-linux-gnu.nightly.json | jq -r --arg REV "$rev" --arg ARCH "x86_64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing linux-x86_64' + val=false + fi + + echo if_linux_x86_64="$val" >> $GITHUB_OUTPUT + + val=true + + out=$(curl -s https://packages.edgedb.com/archive/.jsonindexes/aarch64-unknown-linux-gnu.nightly.json | jq -r --arg REV "$rev" --arg ARCH "aarch64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing linux-aarch64' + val=false + fi + + echo if_linux_aarch64="$val" >> $GITHUB_OUTPUT + + val=true + + out=$(curl -s https://packages.edgedb.com/archive/.jsonindexes/x86_64-unknown-linux-musl.nightly.json | jq -r --arg REV "$rev" --arg ARCH "x86_64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing linuxmusl-x86_64' + val=false + fi + + echo if_linuxmusl_x86_64="$val" >> $GITHUB_OUTPUT + + val=true + + out=$(curl -s https://packages.edgedb.com/archive/.jsonindexes/aarch64-unknown-linux-musl.nightly.json | jq -r --arg REV "$rev" --arg ARCH "aarch64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing linuxmusl-aarch64' + val=false + fi + + echo if_linuxmusl_aarch64="$val" >> $GITHUB_OUTPUT + + + val=true + + out=$(curl -s https://packages.edgedb.com/archive/.jsonindexes/x86_64-apple-darwin.nightly.json | jq -r --arg REV "$rev" --arg ARCH "x86_64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing macos-x86_64' + val=false + fi + + echo if_macos_x86_64="$val" >> $GITHUB_OUTPUT + + val=true + + out=$(curl -s https://packages.edgedb.com/archive/.jsonindexes/aarch64-apple-darwin.nightly.json | jq -r --arg REV "$rev" --arg ARCH "aarch64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing macos-aarch64' + val=false + fi + + echo if_macos_aarch64="$val" >> $GITHUB_OUTPUT + + + build-debian-buster-x86_64: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + if: needs.prep.outputs.if_debian_buster_x86_64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/debian-buster@master @@ -49,6 +410,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + if: needs.prep.outputs.if_debian_buster_aarch64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/debian-buster@master @@ -70,6 +434,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + if: needs.prep.outputs.if_debian_bullseye_x86_64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/debian-bullseye@master @@ -91,6 +458,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + if: needs.prep.outputs.if_debian_bullseye_aarch64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/debian-bullseye@master @@ -112,6 +482,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + if: needs.prep.outputs.if_debian_bookworm_x86_64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/debian-bookworm@master @@ -133,6 +506,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + if: needs.prep.outputs.if_debian_bookworm_aarch64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/debian-bookworm@master @@ -154,6 +530,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + if: needs.prep.outputs.if_ubuntu_bionic_x86_64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/ubuntu-bionic@master @@ -175,6 +554,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + if: needs.prep.outputs.if_ubuntu_bionic_aarch64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/ubuntu-bionic@master @@ -196,6 +578,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + if: needs.prep.outputs.if_ubuntu_focal_x86_64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/ubuntu-focal@master @@ -217,6 +602,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + if: needs.prep.outputs.if_ubuntu_focal_aarch64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/ubuntu-focal@master @@ -238,6 +626,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + if: needs.prep.outputs.if_ubuntu_jammy_x86_64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/ubuntu-jammy@master @@ -259,6 +650,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + if: needs.prep.outputs.if_ubuntu_jammy_aarch64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/ubuntu-jammy@master @@ -280,6 +674,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + if: needs.prep.outputs.if_centos_7_x86_64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/centos-7@master @@ -301,6 +698,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + if: needs.prep.outputs.if_centos_8_x86_64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/centos-8@master @@ -322,6 +722,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + if: needs.prep.outputs.if_centos_8_aarch64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/centos-8@master @@ -343,6 +746,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + if: needs.prep.outputs.if_rockylinux_9_x86_64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/rockylinux-9@master @@ -364,6 +770,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + if: needs.prep.outputs.if_rockylinux_9_aarch64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/rockylinux-9@master @@ -385,6 +794,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + if: needs.prep.outputs.if_linux_x86_64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/linux-x86_64@master @@ -407,6 +819,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + if: needs.prep.outputs.if_linux_aarch64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/linux-aarch64@master @@ -429,6 +844,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + if: needs.prep.outputs.if_linuxmusl_x86_64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/linuxmusl-x86_64@master @@ -452,6 +870,9 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + if: needs.prep.outputs.if_linuxmusl_aarch64 == 'true' + + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/linuxmusl-aarch64@master @@ -475,6 +896,9 @@ jobs: runs-on: ['self-hosted', 'macOS', 'X64'] needs: prep + if: needs.prep.outputs.if_macos_x86_64 == 'true' + + steps: - uses: actions/checkout@v3 with: @@ -518,6 +942,9 @@ jobs: runs-on: ['self-hosted', 'macOS', 'ARM64'] needs: prep + if: needs.prep.outputs.if_macos_aarch64 == 'true' + + steps: - uses: actions/checkout@v3 with: @@ -546,7 +973,7 @@ jobs: PKG_SUBDIST: "nightly" PKG_PLATFORM: "macos" PKG_PLATFORM_VERSION: "aarch64" - PKG_PLATFORM_ARCH: "" + PKG_PLATFORM_ARCH: "aarch64" METAPKG_GIT_CACHE: disabled BUILD_GENERIC: true run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2c1131b637d..c93627ecc9e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,6 +9,7 @@ jobs: runs-on: ubuntu-latest outputs: branch: ${{ steps.whichver.outputs.branch }} + steps: - uses: actions/checkout@v3 @@ -19,10 +20,13 @@ jobs: echo branch="${branch}" >> $GITHUB_OUTPUT id: whichver + + build-debian-buster-x86_64: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/debian-buster@master @@ -44,6 +48,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/debian-buster@master @@ -65,6 +70,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/debian-bullseye@master @@ -86,6 +92,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/debian-bullseye@master @@ -107,6 +114,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/debian-bookworm@master @@ -128,6 +136,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/debian-bookworm@master @@ -149,6 +158,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/ubuntu-bionic@master @@ -170,6 +180,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/ubuntu-bionic@master @@ -191,6 +202,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/ubuntu-focal@master @@ -212,6 +224,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/ubuntu-focal@master @@ -233,6 +246,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/ubuntu-jammy@master @@ -254,6 +268,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/ubuntu-jammy@master @@ -275,6 +290,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/centos-7@master @@ -296,6 +312,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/centos-8@master @@ -317,6 +334,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/centos-8@master @@ -338,6 +356,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/rockylinux-9@master @@ -359,6 +378,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/rockylinux-9@master @@ -380,6 +400,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/linux-x86_64@master @@ -402,6 +423,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/linux-aarch64@master @@ -424,6 +446,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/linuxmusl-x86_64@master @@ -447,6 +470,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/linuxmusl-aarch64@master @@ -470,6 +494,7 @@ jobs: runs-on: ['self-hosted', 'macOS', 'X64'] needs: prep + steps: - uses: actions/checkout@v3 with: @@ -513,6 +538,7 @@ jobs: runs-on: ['self-hosted', 'macOS', 'ARM64'] needs: prep + steps: - uses: actions/checkout@v3 with: @@ -541,7 +567,7 @@ jobs: PKG_REVISION: "" PKG_PLATFORM: "macos" PKG_PLATFORM_VERSION: "aarch64" - PKG_PLATFORM_ARCH: "" + PKG_PLATFORM_ARCH: "aarch64" METAPKG_GIT_CACHE: disabled BUILD_GENERIC: true run: | diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 1b9556ec60e..21f67dcd130 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -9,6 +9,7 @@ jobs: runs-on: ubuntu-latest outputs: branch: ${{ steps.whichver.outputs.branch }} + steps: - uses: actions/checkout@v3 @@ -19,10 +20,13 @@ jobs: echo branch="${branch}" >> $GITHUB_OUTPUT id: whichver + + build-debian-buster-x86_64: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/debian-buster@master @@ -45,6 +49,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/debian-buster@master @@ -67,6 +72,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/debian-bullseye@master @@ -89,6 +95,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/debian-bullseye@master @@ -111,6 +118,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/debian-bookworm@master @@ -133,6 +141,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/debian-bookworm@master @@ -155,6 +164,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/ubuntu-bionic@master @@ -177,6 +187,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/ubuntu-bionic@master @@ -199,6 +210,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/ubuntu-focal@master @@ -221,6 +233,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/ubuntu-focal@master @@ -243,6 +256,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/ubuntu-jammy@master @@ -265,6 +279,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/ubuntu-jammy@master @@ -287,6 +302,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/centos-7@master @@ -309,6 +325,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/centos-8@master @@ -331,6 +348,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/centos-8@master @@ -353,6 +371,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/rockylinux-9@master @@ -375,6 +394,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/rockylinux-9@master @@ -397,6 +417,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/linux-x86_64@master @@ -420,6 +441,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/linux-aarch64@master @@ -443,6 +465,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'x64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/linuxmusl-x86_64@master @@ -467,6 +490,7 @@ jobs: runs-on: ['self-hosted', 'linux', 'arm64'] needs: prep + steps: - name: Build uses: edgedb/edgedb-pkg/integration/linux/build/linuxmusl-aarch64@master @@ -491,6 +515,7 @@ jobs: runs-on: ['self-hosted', 'macOS', 'X64'] needs: prep + steps: - uses: actions/checkout@v3 with: @@ -535,6 +560,7 @@ jobs: runs-on: ['self-hosted', 'macOS', 'ARM64'] needs: prep + steps: - uses: actions/checkout@v3 with: @@ -564,7 +590,7 @@ jobs: PKG_SUBDIST: "testing" PKG_PLATFORM: "macos" PKG_PLATFORM_VERSION: "aarch64" - PKG_PLATFORM_ARCH: "" + PKG_PLATFORM_ARCH: "aarch64" METAPKG_GIT_CACHE: disabled BUILD_GENERIC: true run: |