diff --git a/edb/buildmeta.py b/edb/buildmeta.py index f88193fff04..f9e2ad07bce 100644 --- a/edb/buildmeta.py +++ b/edb/buildmeta.py @@ -44,7 +44,7 @@ # Increment this whenever the database layout or stdlib changes. -EDGEDB_CATALOG_VERSION = 2024_02_01_00_00 +EDGEDB_CATALOG_VERSION = 2024_02_09_00_00 EDGEDB_MAJOR_VERSION = 5 diff --git a/edb/edgeql/compiler/config.py b/edb/edgeql/compiler/config.py index b88d5734803..7cd750ff627 100644 --- a/edb/edgeql/compiler/config.py +++ b/edb/edgeql/compiler/config.py @@ -372,14 +372,6 @@ def _validate_op( ptr = None if isinstance(expr, (qlast.ConfigSet, qlast.ConfigReset)): - # TODO: Fix this. The problem is that it gets lost when serializing it - if is_ext_config and expr.scope == qltypes.ConfigScope.SESSION: - raise errors.UnsupportedFeatureError( - 'SESSION configuration of extension-defined config variables ' - 'is not yet implemented' - ) - - # This error is legit, though if is_ext_config and expr.scope == qltypes.ConfigScope.INSTANCE: raise errors.ConfigurationError( 'INSTANCE configuration of extension-defined config variables ' diff --git a/edb/lib/ext/pg_trgm.edgeql b/edb/lib/ext/pg_trgm.edgeql index db85c4fe91c..bd438429cb0 100644 --- a/edb/lib/ext/pg_trgm.edgeql +++ b/edb/lib/ext/pg_trgm.edgeql @@ -23,10 +23,10 @@ create extension package pg_trgm version '1.6' { create module ext::pg_trgm; - create type ext::pg_trgm::Config extending cfg::ConfigObject { + create type ext::pg_trgm::Config extending cfg::ExtensionConfig { create required property similarity_threshold: std::float32 { create annotation cfg::backend_setting := - "pg_trgm.similarity_threshold"; + '"pg_trgm.similarity_threshold"'; create annotation std::description := "The current similarity threshold that is used by the " ++ "pg_trgm::similar() function, the pg_trgm::gin and " @@ -38,7 +38,7 @@ create extension package pg_trgm version '1.6' { }; create required property word_similarity_threshold: std::float32 { create annotation cfg::backend_setting := - "pg_trgm.word_similarity_threshold"; + '"pg_trgm.word_similarity_threshold"'; create annotation std::description := "The current word similarity threshold that is used by the " ++ "pg_trgrm::word_similar() function. The threshold must be " @@ -50,7 +50,7 @@ create extension package pg_trgm version '1.6' { create required property strict_word_similarity_threshold: std::float32 { create annotation cfg::backend_setting := - "pg_trgm.strict_word_similarity_threshold"; + '"pg_trgm.strict_word_similarity_threshold"'; create annotation std::description := "The current strict word similarity threshold that is used by " ++ "the pg_trgrm::strict_word_similar() function. The " diff --git a/edb/pgsql/delta.py b/edb/pgsql/delta.py index 08e173b4ea2..7ced489e761 100644 --- a/edb/pgsql/delta.py +++ b/edb/pgsql/delta.py @@ -3879,7 +3879,12 @@ def _fixup_configs( from edb.pgsql import metaschema - new_local_spec = config.load_spec_from_schema(schema, only_exts=True) + new_local_spec = config.load_spec_from_schema( + schema, + only_exts=True, + # suppress validation because we might be in an intermediate state + validate=False, + ) spec_json = config.spec_to_json(new_local_spec) self.pgops.add(dbops.Query(textwrap.dedent(f'''\ UPDATE diff --git a/edb/pgsql/metaschema.py b/edb/pgsql/metaschema.py index 38762c23fd5..eaf2babbbb3 100644 --- a/edb/pgsql/metaschema.py +++ b/edb/pgsql/metaschema.py @@ -2975,12 +2975,12 @@ class ConvertPostgresConfigUnitsFunction(dbops.Function): ) WHEN "unit" = '' - THEN trunc("value" * "multiplier")::text::jsonb + THEN ("value" * "multiplier")::text::jsonb ELSE edgedb.raise( NULL::jsonb, msg => ( - 'unknown configutation unit "' || + 'unknown configuration unit "' || COALESCE("unit", '') || '"' ) @@ -3003,6 +3003,54 @@ def __init__(self) -> None: ) +class TypeIDToConfigType(dbops.Function): + """Get a postgres config type from a type id. + + (We typically try to read extension configs straight from the + config tables, but for extension configs those aren't present.) + """ + + config_types = { + 'bool': ['std::bool'], + 'string': ['std::str'], + 'integer': ['std::int16', 'std::int32', 'std::int64'], + 'real': ['std::float32', 'std::float64'], + } + cases = [ + f''' + WHEN "typeid" = '{s_obj.get_known_type_id(t)}' THEN '{ct}' + ''' + for ct, types in config_types.items() + for t in types + ] + scases = '\n'.join(cases) + + text = f""" + SELECT ( + CASE + {scases} + ELSE edgedb.raise( + NULL::text, + msg => ( + 'unknown configuration type "' || "typeid" || '"' + ) + ) + END + ) + """ + + def __init__(self) -> None: + super().__init__( + name=('edgedb', '_type_id_to_config_type'), + args=[ + ('typeid', ('uuid',)), + ], + returns=('text',), + volatility='immutable', + text=self.text, + ) + + class NormalizedPgSettingsView(dbops.View): """Just like `pg_settings` but with the parsed 'unit' column.""" @@ -3082,7 +3130,7 @@ class InterpretConfigValueToJsonFunction(dbops.Function): edgedb.raise( NULL::jsonb, msg => ( - 'unknown configutation type "' || + 'unknown configuration type "' || COALESCE("type", '') || '"' ) @@ -3154,17 +3202,6 @@ class PostgresConfigValueToJsonFunction(dbops.Function): END) FROM - ( - SELECT - epg_settings.vartype AS vartype, - epg_settings.multiplier AS multiplier, - epg_settings.unit AS unit - FROM - edgedb._normalized_pg_settings AS epg_settings - WHERE - epg_settings.name = "setting_name" - ) AS settings, - LATERAL ( SELECT regexp_match( "setting_value", '^(\d+)\s*([a-zA-Z]{0,3})$') AS v @@ -3175,6 +3212,27 @@ class PostgresConfigValueToJsonFunction(dbops.Function): COALESCE(_unit.v[1], "setting_value") AS val, COALESCE(_unit.v[2], '') AS unit ) AS parsed_value + LEFT OUTER JOIN + ( + SELECT + epg_settings.vartype AS vartype, + epg_settings.multiplier AS multiplier, + epg_settings.unit AS unit + FROM + edgedb._normalized_pg_settings AS epg_settings + WHERE + epg_settings.name = "setting_name" + ) AS settings_in ON true + CROSS JOIN LATERAL + ( + SELECT + COALESCE(settings_in.vartype, + edgedb._type_id_to_config_type("setting_typeid")) + as vartype, + COALESCE(settings_in.multiplier, '1') as multiplier, + COALESCE(settings_in.unit, '') as unit + ) as settings + """ def __init__(self) -> None: @@ -3182,6 +3240,7 @@ def __init__(self) -> None: name=('edgedb', '_postgres_config_value_to_json'), args=[ ('setting_name', ('text',)), + ('setting_typeid', ('uuid',)), ('setting_value', ('text',)), ], returns=('jsonb',), @@ -3227,6 +3286,9 @@ class SysConfigFullFunction(dbops.Function): FROM config_spec s ), + config_extension_defaults AS ( + SELECT * FROM config_defaults WHERE name like '%::%' + ), config_sys AS ( SELECT @@ -3274,7 +3336,7 @@ class SysConfigFullFunction(dbops.Function): SELECT spec.name, edgedb._postgres_config_value_to_json( - spec.backend_setting, nameval.value + spec.backend_setting, spec.typeid, nameval.value ) AS value, 'database' AS source, TRUE AS is_backend @@ -3300,7 +3362,8 @@ class SysConfigFullFunction(dbops.Function): LATERAL ( SELECT config_spec.name, - config_spec.backend_setting + config_spec.backend_setting, + config_spec.typeid FROM config_spec WHERE @@ -3315,7 +3378,7 @@ class SysConfigFullFunction(dbops.Function): SELECT spec.name, edgedb._postgres_config_value_to_json( - spec.backend_setting, setting + spec.backend_setting, spec.typeid, setting ) AS value, 'postgres configuration file' AS source, TRUE AS is_backend @@ -3324,7 +3387,8 @@ class SysConfigFullFunction(dbops.Function): LATERAL ( SELECT config_spec.name, - config_spec.backend_setting + config_spec.backend_setting, + config_spec.typeid FROM config_spec WHERE @@ -3342,7 +3406,7 @@ class SysConfigFullFunction(dbops.Function): SELECT spec.name, edgedb._postgres_config_value_to_json( - spec.backend_setting, setting + spec.backend_setting, spec.typeid, setting ) AS value, 'system override' AS source, TRUE AS is_backend @@ -3351,7 +3415,8 @@ class SysConfigFullFunction(dbops.Function): LATERAL ( SELECT config_spec.name, - config_spec.backend_setting + config_spec.backend_setting, + config_spec.typeid FROM config_spec WHERE @@ -3409,6 +3474,25 @@ class SysConfigFullFunction(dbops.Function): ) AS spec ), + -- extension session configs don't show up in any system view, so we + -- check _edgecon_state to see when they are present. + pg_extension_config AS ( + SELECT + config_spec.name, + -- XXX: Or would it be better to just use the json directly? + edgedb._postgres_config_value_to_json( + config_spec.backend_setting, + config_spec.typeid, + current_setting(config_spec.backend_setting, true) + ) AS value, + 'session' AS source, + TRUE AS is_backend + FROM _edgecon_state s + INNER JOIN config_spec + ON s.name = config_spec.name + WHERE s.type = 'B' AND s.name LIKE '%::%' + ), + edge_all_settings AS MATERIALIZED ( SELECT q.* @@ -3432,10 +3516,13 @@ class SysConfigFullFunction(dbops.Function): q.* FROM ( + -- extension defaults aren't in any system views + SELECT * FROM config_extension_defaults UNION ALL SELECT * FROM pg_db_setting UNION ALL SELECT * FROM pg_conf_settings UNION ALL SELECT * FROM pg_auto_conf_settings UNION ALL - SELECT * FROM pg_config + SELECT * FROM pg_config UNION ALL + SELECT * FROM pg_extension_config ) AS q WHERE q.is_backend @@ -3448,12 +3535,15 @@ class SysConfigFullFunction(dbops.Function): q.* FROM ( + -- extension defaults aren't in any system views + SELECT * FROM config_extension_defaults UNION ALL -- config_sys is here, because there -- is no other way to read instance-level -- configuration overrides. SELECT * FROM config_sys UNION ALL SELECT * FROM pg_db_setting UNION ALL - SELECT * FROM pg_config + SELECT * FROM pg_config UNION ALL + SELECT * FROM pg_extension_config ) AS q WHERE q.is_backend @@ -3689,10 +3779,6 @@ def __init__(self) -> None: ) -# TODO: Support extension-defined configs that affect the backend -# Not needed for supporting auth, so can skip temporarily. -# If perf seems to matter, can hardcode things for base config -# and consult json for just extension stuff. class ApplySessionConfigFunction(dbops.Function): """Apply an EdgeDB config setting to the backend, if possible. @@ -3738,6 +3824,18 @@ def __init__(self, config_spec: edbconfig.Spec) -> None: ) ''') + ext_config = ''' + SELECT pg_catalog.set_config( + (s.val->>'backend_setting')::text, + "value"->>0, + false + ) + FROM + edgedbinstdata.instdata as id, + LATERAL jsonb_each(id.json) AS s(key, val) + WHERE id.key = 'configspec_ext' AND s.key = "name" + ''' + variants = "\n".join(variants_list) text = f''' SELECT ( @@ -3756,6 +3854,13 @@ def __init__(self, config_spec: edbconfig.Spec) -> None: END ) + WHEN "name" LIKE '%::%' + THEN + CASE WHEN ({ext_config}) IS NULL + THEN "name" + ELSE "name" + END + ELSE "name" END ) @@ -4526,6 +4631,7 @@ async def bootstrap( dbops.CreateEnum(SysConfigScopeType()), dbops.CreateCompositeType(SysConfigValueType()), dbops.CreateCompositeType(SysConfigEntryType()), + dbops.CreateFunction(TypeIDToConfigType()), dbops.CreateFunction(ConvertPostgresConfigUnitsFunction()), dbops.CreateFunction(InterpretConfigValueToJsonFunction()), dbops.CreateFunction(PostgresConfigValueToJsonFunction()), diff --git a/edb/server/compiler/sertypes.py b/edb/server/compiler/sertypes.py index 1e057f91a59..35e7211702e 100644 --- a/edb/server/compiler/sertypes.py +++ b/edb/server/compiler/sertypes.py @@ -64,6 +64,7 @@ _uint16_packer = cast(Callable[[int], bytes], struct.Struct('!H').pack) _uint8_packer = cast(Callable[[int], bytes], struct.Struct('!B').pack) _int64_struct = struct.Struct('!q') +_float32_struct = struct.Struct('!f') EMPTY_TUPLE_ID = s_obj.get_known_type_id('empty-tuple') @@ -132,6 +133,14 @@ def _decode_int64(data: bytes) -> int: return _int64_struct.unpack(data)[0] # type: ignore [no-any-return] +def _encode_float32(data: float) -> bytes: + return _float32_struct.pack(data) + + +def _decode_float32(data: bytes) -> float: + return _float32_struct.unpack(data)[0] # type: ignore [no-any-return] + + def _string_packer(s: str) -> bytes: s_bytes = s.encode('utf-8') return _uint32_packer(len(s_bytes)) + s_bytes @@ -1635,6 +1644,28 @@ def __init__(self, std_schema: s_schema.Schema, config_spec: config.Spec): schema, config_type = derive_alias( schema, free_obj, 'state_config' ) + config_shape = self._make_config_shape(config_spec, schema) + + self._input_shapes: immutables.Map[ + s_types.Type, + tuple[InputShapeElement, ...], + ] = immutables.Map([ + (config_type, config_shape), + (self._state_type, ( + ("module", str_type, enums.Cardinality.AT_MOST_ONE), + ("aliases", aliases_array, enums.Cardinality.AT_MOST_ONE), + ("config", config_type, enums.Cardinality.AT_MOST_ONE), + )) + ]) + self.config_type = config_type + self._schema = schema + self._contexts: dict[edbdef.ProtocolVersion, Context] = {} + + @staticmethod + def _make_config_shape( + config_spec: config.Spec, + schema: s_schema.Schema, + ) -> tuple[tuple[str, s_types.Type, enums.Cardinality], ...]: config_shape: list[tuple[str, s_types.Type, enums.Cardinality]] = [] for setting in config_spec.values(): @@ -1649,20 +1680,7 @@ def __init__(self, std_schema: s_schema.Schema, config_spec: config.Spec): enums.Cardinality.AT_MOST_ONE, ) ) - - self._input_shapes: immutables.Map[ - s_types.Type, - tuple[InputShapeElement, ...], - ] = immutables.Map([ - (config_type, tuple(sorted(config_shape))), - (self._state_type, ( - ("module", str_type, enums.Cardinality.AT_MOST_ONE), - ("aliases", aliases_array, enums.Cardinality.AT_MOST_ONE), - ("config", config_type, enums.Cardinality.AT_MOST_ONE), - )) - ]) - self._schema = schema - self._contexts: dict[edbdef.ProtocolVersion, Context] = {} + return tuple(sorted(config_shape)) def make( self, @@ -1688,6 +1706,12 @@ def make( ctx.schema = s_schema.ChainedSchema( self._schema, user_schema, global_schema) + # Update the config shape with any extension configs + ext_config_spec = config.load_ext_spec_from_schema( + user_schema, self._schema) + new_config = self._make_config_shape(ext_config_spec, ctx.schema) + full_config = self._input_shapes[self.config_type] + new_config + globals_shape = [] global_reps = {} for g in ctx.schema.get_objects(type=s_globals.Global): @@ -1704,6 +1728,7 @@ def make( self._state_type, self._input_shapes.update({ self.globals_type: tuple(sorted(globals_shape)), + self.config_type: full_config, self._state_type: self._input_shapes[self._state_type] + ( ( "globals", @@ -1889,6 +1914,10 @@ class BaseScalarDesc(SchemaTypeDesc): _encode_int64, _decode_int64, ), + s_obj.get_known_type_id('std::float32'): ( + _encode_float32, + _decode_float32, + ), } def encode(self, data: Any) -> bytes: diff --git a/edb/server/config/ops.py b/edb/server/config/ops.py index df95dcc2ef1..2d50d72cdeb 100644 --- a/edb/server/config/ops.py +++ b/edb/server/config/ops.py @@ -317,6 +317,8 @@ def spec_to_json(spec: spec.Spec): typeid = s_obj.get_known_type_id('std::bool') elif _issubclass(setting.type, int): typeid = s_obj.get_known_type_id('std::int64') + elif _issubclass(setting.type, float): + typeid = s_obj.get_known_type_id('std::float32') elif _issubclass(setting.type, types.ConfigType): typeid = setting.type.get_edgeql_typeid() elif _issubclass(setting.type, statypes.Duration): diff --git a/edb/server/config/spec.py b/edb/server/config/spec.py index 8627b17f3e8..7cb7676d925 100644 --- a/edb/server/config/spec.py +++ b/edb/server/config/spec.py @@ -29,6 +29,7 @@ from edb.edgeql import compiler as qlcompiler from edb.ir import staeval from edb.ir import statypes +from edb.schema import links as s_links from edb.schema import name as sn from edb.schema import objtypes as s_objtypes from edb.schema import scalars as s_scalars @@ -39,7 +40,8 @@ from . import types -SETTING_TYPES = {str, int, bool, statypes.Duration, statypes.ConfigMemory} +SETTING_TYPES = {str, int, bool, float, + statypes.Duration, statypes.ConfigMemory} @dataclasses.dataclass(frozen=True, eq=True) @@ -66,7 +68,8 @@ def __post_init__(self) -> None: not isinstance(self.type, types.ConfigTypeSpec)): raise ValueError( f'invalid config setting {self.name!r}: ' - f'type is expected to be either one of {{str, int, bool}} ' + f'type is expected to be either one of ' + f'{{str, int, bool, float}} ' f'or an edb.server.config.types.ConfigType subclass') if self.set_of: @@ -195,7 +198,9 @@ def __len__(self) -> int: def load_spec_from_schema( - schema: s_schema.Schema, only_exts: bool=False + schema: s_schema.Schema, + only_exts: bool=False, + validate: bool=True, ) -> Spec: settings = [] if not only_exts: @@ -204,6 +209,18 @@ def load_spec_from_schema( settings.extend(load_ext_settings_from_schema(schema)) + # Make sure there aren't any dangling ConfigObject children + if validate: + cfg_object = schema.get('cfg::ConfigObject', type=s_objtypes.ObjectType) + for child in cfg_object.children(schema): + if not schema.get_referrers( + child, scls_type=s_links.Link, field_name='target' + ): + raise RuntimeError( + f'cfg::ConfigObject child {child.get_name(schema)} has no ' + f'links pointing at it (did you mean cfg::ExtensionConfig?)' + ) + return FlatSpec(*settings) @@ -255,11 +272,17 @@ def _load_spec_from_type( else: pytype = staeval.scalar_type_to_python_type(ptype, schema) - attributes = { - a: json.loads(v.get_value(schema)) - for a, v in p.get_annotations(schema).items(schema) - if isinstance(a, sn.QualName) and a.module == 'cfg' - } + attributes = {} + for a, v in p.get_annotations(schema).items(schema): + if isinstance(a, sn.QualName) and a.module == 'cfg': + try: + jv = json.loads(v.get_value(schema)) + except json.JSONDecodeError: + raise RuntimeError( + f'Config annotation {a} on {p.get_name(schema)} ' + f'is not valid json' + ) + attributes[a] = jv ptr_card = p.get_cardinality(schema) set_of = ptr_card.is_multi() diff --git a/tests/test_edgeql_ddl.py b/tests/test_edgeql_ddl.py index cb19712f06c..459a4c37fb9 100644 --- a/tests/test_edgeql_ddl.py +++ b/tests/test_edgeql_ddl.py @@ -16564,22 +16564,19 @@ async def _check(_cfg_obj='Config', **kwargs): "session!"; ''') - # TODO: This should all work, instead! - async with self.assertRaisesRegexTx( - edgedb.UnsupportedFeatureError, ""): - await self.con.execute(''' - configure session set ext::_conf::Config::config_name := - "session!"; - ''') + await self.con.execute(''' + configure session set ext::_conf::Config::config_name := + "session!"; + ''') - await _check( - config_name='session!', - objs=[], - ) + await _check( + config_name='session!', + objs=[], + ) - await self.con.execute(''' - configure session reset ext::_conf::Config::config_name; - ''') + await self.con.execute(''' + configure session reset ext::_conf::Config::config_name; + ''') await _check( config_name='test', diff --git a/tests/test_edgeql_ext_pg_trgm.py b/tests/test_edgeql_ext_pg_trgm.py index 517f6cc1e6e..098fa2f81ff 100644 --- a/tests/test_edgeql_ext_pg_trgm.py +++ b/tests/test_edgeql_ext_pg_trgm.py @@ -304,3 +304,121 @@ async def test_edgeql_ext_pg_trgm_strict_word_similarity(self): qry, index_type="ext::pg_trgm::gist", ) + + async def test_edgeql_ext_pg_trgm_config(self): + # We are going to fiddle with the similarity_threshold config + # and make sure it works right. + + sim_query = """ + WITH similar := ( + SELECT + Gist { + p_str, + sim := ext::pg_trgm::similarity(.p_str, "qwertyu0988") + } + FILTER + ext::pg_trgm::similar(.p_str, "qwertyu0988") + ), + SELECT exists similar and all(similar.sim >= $sim) + """ + + cfg_query = """ + select cfg::Config.extensions[is ext::pg_trgm::Config] + .similarity_threshold; + """ + + await self.assert_query_result( + sim_query, + [True], + variables=dict(sim=0.3), + ) + await self.assert_query_result( + sim_query, + [False], + variables=dict(sim=0.5), + ) + await self.assert_query_result( + sim_query, + [False], + variables=dict(sim=0.9), + ) + + await self.assert_query_result( + cfg_query, + [0.3], + ) + + await self.con.execute(''' + configure session + set ext::pg_trgm::Config::similarity_threshold := 0.5 + ''') + + await self.assert_query_result( + sim_query, + [True], + variables=dict(sim=0.3), + ) + await self.assert_query_result( + sim_query, + [True], + variables=dict(sim=0.5), + ) + await self.assert_query_result( + sim_query, + [False], + variables=dict(sim=0.9), + ) + await self.assert_query_result( + cfg_query, + [0.5], + ) + + await self.con.execute(''' + configure session + set ext::pg_trgm::Config::similarity_threshold := 0.9 + ''') + + await self.assert_query_result( + sim_query, + [True], + variables=dict(sim=0.3), + ) + await self.assert_query_result( + sim_query, + [True], + variables=dict(sim=0.5), + ) + await self.assert_query_result( + sim_query, + [True], + variables=dict(sim=0.9), + ) + await self.assert_query_result( + cfg_query, + [0.9], + ) + + await self.con.execute(''' + configure session + reset ext::pg_trgm::Config::similarity_threshold + ''') + + await self.assert_query_result( + sim_query, + [True], + variables=dict(sim=0.3), + ) + await self.assert_query_result( + sim_query, + [False], + variables=dict(sim=0.5), + ) + await self.assert_query_result( + sim_query, + [False], + variables=dict(sim=0.9), + ) + await self.assert_query_result( + cfg_query, + [0.3], + )