Skip to content

Commit

Permalink
Support session and backend extension configs (#6811)
Browse files Browse the repository at this point in the history 
This requires:
 * Some special handling for reflection, since extension settings
   don't show up in the views we look at normally
 * Updating _apply_session_config to work for extension configs
 * Making StateSerializerFactory able to include extension configs.

To test this, I fixed pg_trgm's configs. pg_trgm attempted to have
some backend configs, but the config object didn't extend
ExtensionConfig and also the backend_setting wasn't json.  I added
some checks to the extension path to prevent this sort of mistake.

Database configs work too, and I tested it manually, but currently the
only reliable way to make database configs with backend settings take
effect is to restart the server, so no tests for that yet.

The other big motivation for this is pgvector probes, which I'll leave
for @vpetrovykh to follow up on.
  • Loading branch information
@msullivan
msullivan authored Feb 12, 2024
1 parent 95b8f6d commit 27286af
Show file tree
Hide file tree
Showing 10 changed files with 348 additions and 76 deletions.
2 changes: 1 addition & 1 deletion edb/buildmeta.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@


# Increment this whenever the database layout or stdlib changes.
EDGEDB_CATALOG_VERSION = 2024_02_01_00_00
EDGEDB_CATALOG_VERSION = 2024_02_09_00_00
EDGEDB_MAJOR_VERSION = 5


Expand Down
8 changes: 0 additions & 8 deletions edb/edgeql/compiler/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -372,14 +372,6 @@ def _validate_op(
ptr = None

if isinstance(expr, (qlast.ConfigSet, qlast.ConfigReset)):
# TODO: Fix this. The problem is that it gets lost when serializing it
if is_ext_config and expr.scope == qltypes.ConfigScope.SESSION:
raise errors.UnsupportedFeatureError(
'SESSION configuration of extension-defined config variables '
'is not yet implemented'
)

# This error is legit, though
if is_ext_config and expr.scope == qltypes.ConfigScope.INSTANCE:
raise errors.ConfigurationError(
'INSTANCE configuration of extension-defined config variables '
Expand Down
8 changes: 4 additions & 4 deletions edb/lib/ext/pg_trgm.edgeql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,10 @@ create extension package pg_trgm version '1.6' {

create module ext::pg_trgm;

create type ext::pg_trgm::Config extending cfg::ConfigObject {
create type ext::pg_trgm::Config extending cfg::ExtensionConfig {
create required property similarity_threshold: std::float32 {
create annotation cfg::backend_setting :=
"pg_trgm.similarity_threshold";
'"pg_trgm.similarity_threshold"';
create annotation std::description :=
"The current similarity threshold that is used by the "
++ "pg_trgm::similar() function, the pg_trgm::gin and "
Expand All @@ -38,7 +38,7 @@ create extension package pg_trgm version '1.6' {
};
create required property word_similarity_threshold: std::float32 {
create annotation cfg::backend_setting :=
"pg_trgm.word_similarity_threshold";
'"pg_trgm.word_similarity_threshold"';
create annotation std::description :=
"The current word similarity threshold that is used by the "
++ "pg_trgrm::word_similar() function. The threshold must be "
Expand All @@ -50,7 +50,7 @@ create extension package pg_trgm version '1.6' {
create required property strict_word_similarity_threshold: std::float32
{
create annotation cfg::backend_setting :=
"pg_trgm.strict_word_similarity_threshold";
'"pg_trgm.strict_word_similarity_threshold"';
create annotation std::description :=
"The current strict word similarity threshold that is used by "
++ "the pg_trgrm::strict_word_similar() function. The "
Expand Down
7 changes: 6 additions & 1 deletion edb/pgsql/delta.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3879,7 +3879,12 @@ def _fixup_configs(

from edb.pgsql import metaschema

new_local_spec = config.load_spec_from_schema(schema, only_exts=True)
new_local_spec = config.load_spec_from_schema(
schema,
only_exts=True,
# suppress validation because we might be in an intermediate state
validate=False,
)
spec_json = config.spec_to_json(new_local_spec)
self.pgops.add(dbops.Query(textwrap.dedent(f'''\
UPDATE
Expand Down
158 changes: 132 additions & 26 deletions edb/pgsql/metaschema.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2975,12 +2975,12 @@ class ConvertPostgresConfigUnitsFunction(dbops.Function):
)
WHEN "unit" = ''
THEN trunc("value" * "multiplier")::text::jsonb
THEN ("value" * "multiplier")::text::jsonb
ELSE edgedb.raise(
NULL::jsonb,
msg => (
'unknown configutation unit "' ||
'unknown configuration unit "' ||
COALESCE("unit", '<NULL>') ||
'"'
)
Expand All @@ -3003,6 +3003,54 @@ def __init__(self) -> None:
)


class TypeIDToConfigType(dbops.Function):
"""Get a postgres config type from a type id.
(We typically try to read extension configs straight from the
config tables, but for extension configs those aren't present.)
"""

config_types = {
'bool': ['std::bool'],
'string': ['std::str'],
'integer': ['std::int16', 'std::int32', 'std::int64'],
'real': ['std::float32', 'std::float64'],
}
cases = [
f'''
WHEN "typeid" = '{s_obj.get_known_type_id(t)}' THEN '{ct}'
'''
for ct, types in config_types.items()
for t in types
]
scases = '\n'.join(cases)

text = f"""
SELECT (
CASE
{scases}
ELSE edgedb.raise(
NULL::text,
msg => (
'unknown configuration type "' || "typeid" || '"'
)
)
END
)
"""

def __init__(self) -> None:
super().__init__(
name=('edgedb', '_type_id_to_config_type'),
args=[
('typeid', ('uuid',)),
],
returns=('text',),
volatility='immutable',
text=self.text,
)


class NormalizedPgSettingsView(dbops.View):
"""Just like `pg_settings` but with the parsed 'unit' column."""

Expand Down Expand Up @@ -3082,7 +3130,7 @@ class InterpretConfigValueToJsonFunction(dbops.Function):
edgedb.raise(
NULL::jsonb,
msg => (
'unknown configutation type "' ||
'unknown configuration type "' ||
COALESCE("type", '<NULL>') ||
'"'
)
Expand Down Expand Up @@ -3154,17 +3202,6 @@ class PostgresConfigValueToJsonFunction(dbops.Function):
END)
FROM
(
SELECT
epg_settings.vartype AS vartype,
epg_settings.multiplier AS multiplier,
epg_settings.unit AS unit
FROM
edgedb._normalized_pg_settings AS epg_settings
WHERE
epg_settings.name = "setting_name"
) AS settings,
LATERAL (
SELECT regexp_match(
"setting_value", '^(\d+)\s*([a-zA-Z]{0,3})$') AS v
Expand All @@ -3175,13 +3212,35 @@ class PostgresConfigValueToJsonFunction(dbops.Function):
COALESCE(_unit.v[1], "setting_value") AS val,
COALESCE(_unit.v[2], '') AS unit
) AS parsed_value
LEFT OUTER JOIN
(
SELECT
epg_settings.vartype AS vartype,
epg_settings.multiplier AS multiplier,
epg_settings.unit AS unit
FROM
edgedb._normalized_pg_settings AS epg_settings
WHERE
epg_settings.name = "setting_name"
) AS settings_in ON true
CROSS JOIN LATERAL
(
SELECT
COALESCE(settings_in.vartype,
edgedb._type_id_to_config_type("setting_typeid"))
as vartype,
COALESCE(settings_in.multiplier, '1') as multiplier,
COALESCE(settings_in.unit, '') as unit
) as settings
"""

def __init__(self) -> None:
super().__init__(
name=('edgedb', '_postgres_config_value_to_json'),
args=[
('setting_name', ('text',)),
('setting_typeid', ('uuid',)),
('setting_value', ('text',)),
],
returns=('jsonb',),
Expand Down Expand Up @@ -3227,6 +3286,9 @@ class SysConfigFullFunction(dbops.Function):
FROM
config_spec s
),
config_extension_defaults AS (
SELECT * FROM config_defaults WHERE name like '%::%'
),
config_sys AS (
SELECT
Expand Down Expand Up @@ -3274,7 +3336,7 @@ class SysConfigFullFunction(dbops.Function):
SELECT
spec.name,
edgedb._postgres_config_value_to_json(
spec.backend_setting, nameval.value
spec.backend_setting, spec.typeid, nameval.value
) AS value,
'database' AS source,
TRUE AS is_backend
Expand All @@ -3300,7 +3362,8 @@ class SysConfigFullFunction(dbops.Function):
LATERAL (
SELECT
config_spec.name,
config_spec.backend_setting
config_spec.backend_setting,
config_spec.typeid
FROM
config_spec
WHERE
Expand All @@ -3315,7 +3378,7 @@ class SysConfigFullFunction(dbops.Function):
SELECT
spec.name,
edgedb._postgres_config_value_to_json(
spec.backend_setting, setting
spec.backend_setting, spec.typeid, setting
) AS value,
'postgres configuration file' AS source,
TRUE AS is_backend
Expand All @@ -3324,7 +3387,8 @@ class SysConfigFullFunction(dbops.Function):
LATERAL (
SELECT
config_spec.name,
config_spec.backend_setting
config_spec.backend_setting,
config_spec.typeid
FROM
config_spec
WHERE
Expand All @@ -3342,7 +3406,7 @@ class SysConfigFullFunction(dbops.Function):
SELECT
spec.name,
edgedb._postgres_config_value_to_json(
spec.backend_setting, setting
spec.backend_setting, spec.typeid, setting
) AS value,
'system override' AS source,
TRUE AS is_backend
Expand All @@ -3351,7 +3415,8 @@ class SysConfigFullFunction(dbops.Function):
LATERAL (
SELECT
config_spec.name,
config_spec.backend_setting
config_spec.backend_setting,
config_spec.typeid
FROM
config_spec
WHERE
Expand Down Expand Up @@ -3409,6 +3474,25 @@ class SysConfigFullFunction(dbops.Function):
) AS spec
),
-- extension session configs don't show up in any system view, so we
-- check _edgecon_state to see when they are present.
pg_extension_config AS (
SELECT
config_spec.name,
-- XXX: Or would it be better to just use the json directly?
edgedb._postgres_config_value_to_json(
config_spec.backend_setting,
config_spec.typeid,
current_setting(config_spec.backend_setting, true)
) AS value,
'session' AS source,
TRUE AS is_backend
FROM _edgecon_state s
INNER JOIN config_spec
ON s.name = config_spec.name
WHERE s.type = 'B' AND s.name LIKE '%::%'
),
edge_all_settings AS MATERIALIZED (
SELECT
q.*
Expand All @@ -3432,10 +3516,13 @@ class SysConfigFullFunction(dbops.Function):
q.*
FROM
(
-- extension defaults aren't in any system views
SELECT * FROM config_extension_defaults UNION ALL
SELECT * FROM pg_db_setting UNION ALL
SELECT * FROM pg_conf_settings UNION ALL
SELECT * FROM pg_auto_conf_settings UNION ALL
SELECT * FROM pg_config
SELECT * FROM pg_config UNION ALL
SELECT * FROM pg_extension_config
) AS q
WHERE
q.is_backend
Expand All @@ -3448,12 +3535,15 @@ class SysConfigFullFunction(dbops.Function):
q.*
FROM
(
-- extension defaults aren't in any system views
SELECT * FROM config_extension_defaults UNION ALL
-- config_sys is here, because there
-- is no other way to read instance-level
-- configuration overrides.
SELECT * FROM config_sys UNION ALL
SELECT * FROM pg_db_setting UNION ALL
SELECT * FROM pg_config
SELECT * FROM pg_config UNION ALL
SELECT * FROM pg_extension_config
) AS q
WHERE
q.is_backend
Expand Down Expand Up @@ -3689,10 +3779,6 @@ def __init__(self) -> None:
)


# TODO: Support extension-defined configs that affect the backend
# Not needed for supporting auth, so can skip temporarily.
# If perf seems to matter, can hardcode things for base config
# and consult json for just extension stuff.
class ApplySessionConfigFunction(dbops.Function):
"""Apply an EdgeDB config setting to the backend, if possible.
Expand Down Expand Up @@ -3738,6 +3824,18 @@ def __init__(self, config_spec: edbconfig.Spec) -> None:
)
''')

ext_config = '''
SELECT pg_catalog.set_config(
(s.val->>'backend_setting')::text,
"value"->>0,
false
)
FROM
edgedbinstdata.instdata as id,
LATERAL jsonb_each(id.json) AS s(key, val)
WHERE id.key = 'configspec_ext' AND s.key = "name"
'''

variants = "\n".join(variants_list)
text = f'''
SELECT (
Expand All @@ -3756,6 +3854,13 @@ def __init__(self, config_spec: edbconfig.Spec) -> None:
END
)
WHEN "name" LIKE '%::%'
THEN
CASE WHEN ({ext_config}) IS NULL
THEN "name"
ELSE "name"
END
ELSE "name"
END
)
Expand Down Expand Up @@ -4526,6 +4631,7 @@ async def bootstrap(
dbops.CreateEnum(SysConfigScopeType()),
dbops.CreateCompositeType(SysConfigValueType()),
dbops.CreateCompositeType(SysConfigEntryType()),
dbops.CreateFunction(TypeIDToConfigType()),
dbops.CreateFunction(ConvertPostgresConfigUnitsFunction()),
dbops.CreateFunction(InterpretConfigValueToJsonFunction()),
dbops.CreateFunction(PostgresConfigValueToJsonFunction()),
Expand Down
Loading

0 comments on commit 27286af

Please sign in to comment.