Persistent cache

edb/graphql/extension.pyx

@@ -251,7 +251,7 @@ async def compile(
 
 
 async def _execute(db, tenant, query, operation_name, variables, globals):
-    dbver = db.dbver
+    ver = db.schema_version
     query_cache = tenant.server._http_query_cache
 
     if variables:
@@ -299,7 +299,7 @@ async def _execute(db, tenant, query, operation_name, variables, globals):
             print(f'key_vars: {key_var_names}')
             print(f'variables: {vars}')
 
-    cache_key = ('graphql', prepared_query, key_vars, operation_name, dbver)
+    cache_key = ('graphql', prepared_query, key_vars, operation_name, ver)
     use_prep_stmt = False
 
     entry: CacheEntry = None
@@ -308,7 +308,7 @@ async def _execute(db, tenant, query, operation_name, variables, globals):
 
     if isinstance(entry, CacheRedirect):
         key_vars2 = tuple(vars[k] for k in entry.key_vars)
-        cache_key2 = (prepared_query, key_vars2, operation_name, dbver)
+        cache_key2 = (prepared_query, key_vars2, operation_name, ver)
         entry = query_cache.get(cache_key2, None)
 
     await db.introspection()
@@ -343,7 +343,7 @@ async def _execute(db, tenant, query, operation_name, variables, globals):
             query_cache[cache_key] = redir
             key_vars2 = tuple(vars[k] for k in key_var_names)
             cache_key2 = (
-                'graphql', prepared_query, key_vars2, operation_name, dbver
+                'graphql', prepared_query, key_vars2, operation_name, ver
             )
             query_cache[cache_key2] = qug, gql_op
         else:

edb/server/compiler/compiler.py

@@ -26,6 +26,7 @@
 import hashlib
 import pickle
 import textwrap
+import time
 import uuid
 
 import immutables
@@ -64,6 +65,7 @@
 from edb.schema import roles as s_role
 from edb.schema import schema as s_schema
 from edb.schema import types as s_types
+from edb.schema import version as s_ver
 
 from edb.pgsql import ast as pgast
 from edb.pgsql import compiler as pg_compiler
@@ -1001,6 +1003,7 @@ def parse_user_schema_db_config(
         )
         return dbstate.ParsedDatabase(
             user_schema_pickle=pickle.dumps(user_schema, -1),
+            schema_version=_get_schema_version(user_schema),
             database_config=db_config,
             ext_config_settings=ext_config_settings,
             protocol_version=defines.CURRENT_PROTOCOL,
@@ -1497,6 +1500,11 @@ def _compile_schema_storage_stmt(
         ctx.state.current_tx().update_schema(schema)
 
 
+def _get_schema_version(user_schema: s_schema.Schema) -> uuid.UUID:
+    ver = user_schema.get_global(s_ver.SchemaVersion, "__schema_version__")
+    return ver.get_version(user_schema)
+
+
 def _compile_ql_script(
     ctx: CompileContext,
     eql: str,
@@ -2324,6 +2332,11 @@ def _try_compile(
     ctx: CompileContext,
     source: edgeql.Source,
 ) -> dbstate.QueryUnitGroup:
+    if _get_config_val(ctx, '__internal_testmode'):
+        sentinel = "# EDGEDB_TEST_COMPILER_SLEEP = "
+        text = source.text()
+        if text.startswith(sentinel):
+            time.sleep(float(text[len(sentinel):text.index("\n")]))
 
     default_cardinality = enums.Cardinality.NO_RESULT
     statements = edgeql.parse_block(source)
@@ -2437,7 +2450,10 @@ def _try_compile(
             if not ctx.dump_restore_mode:
                 if comp.user_schema is not None:
                     final_user_schema = comp.user_schema
-                    unit.user_schema = pickle.dumps(comp.user_schema, -1)
+                    unit.user_schema_pv = (
+                        pickle.dumps(comp.user_schema, -1),
+                        _get_schema_version(comp.user_schema),
+                    )
                     unit.extensions, unit.ext_config_settings = (
                         _extract_extensions(ctx, comp.user_schema)
                     )
@@ -2462,7 +2478,10 @@ def _try_compile(
             if not ctx.dump_restore_mode:
                 if comp.user_schema is not None:
                     final_user_schema = comp.user_schema
-                    unit.user_schema = pickle.dumps(comp.user_schema, -1)
+                    unit.user_schema_pv = (
+                        pickle.dumps(comp.user_schema, -1),
+                        _get_schema_version(comp.user_schema),
+                    )
                     unit.extensions, unit.ext_config_settings = (
                         _extract_extensions(ctx, comp.user_schema)
                     )
@@ -2500,7 +2519,10 @@ def _try_compile(
             if not ctx.dump_restore_mode:
                 if comp.user_schema is not None:
                     final_user_schema = comp.user_schema
-                    unit.user_schema = pickle.dumps(comp.user_schema, -1)
+                    unit.user_schema_pv = (
+                        pickle.dumps(comp.user_schema, -1),
+                        _get_schema_version(comp.user_schema),
+                    )
                     unit.extensions, unit.ext_config_settings = (
                         _extract_extensions(ctx, comp.user_schema)
                     )
@@ -2620,7 +2642,7 @@ def _try_compile(
             unit.cardinality is enums.Cardinality.NO_RESULT
         )
         if unit.cacheable and (
-            unit.config_ops or unit.modaliases or unit.user_schema or
+            unit.config_ops or unit.modaliases or unit.user_schema_pv or
             unit.cached_reflection
         ):
             raise errors.InternalServerError(

edb/server/compiler/dbstate.py

@@ -310,9 +310,9 @@ class QueryUnit:
         dataclasses.field(default_factory=list))
     modaliases: Optional[immutables.Map[Optional[str], str]] = None
 
-    # If present, represents the future schema state after
-    # the command is run. The schema is pickled.
-    user_schema: Optional[bytes] = None
+    # If present, represents the future schema state after the command is run.
+    # The schema is pickled, followed by its version (_pv).
+    user_schema_pv: Optional[Tuple[bytes, uuid.UUID]] = None
     cached_reflection: Optional[bytes] = None
     extensions: Optional[set[str]] = None
     ext_config_settings: Optional[list[config.Setting]] = None
@@ -473,6 +473,7 @@ class SQLQueryUnit:
 @dataclasses.dataclass
 class ParsedDatabase:
     user_schema_pickle: bytes
+    schema_version: uuid.UUID
     database_config: immutables.Map[str, config.SettingValue]
     ext_config_settings: list[config.Setting]
 

edb/server/dbview/dbview.pxd

@@ -84,7 +84,7 @@ cdef class Database:
         readonly object user_config_spec
 
         readonly str name
-        readonly object dbver
+        readonly object schema_version
         readonly object db_config
         readonly bytes user_schema_pickle
         readonly object reflection_cache
@@ -94,13 +94,14 @@ cdef class Database:
     cdef schedule_config_update(self)
 
     cdef _invalidate_caches(self)
-    cdef _cache_compiled_query(self, key, query_unit)
+    cdef _cache_compiled_query(self, key, compiled, schema_version)
     cdef _new_view(self, query_cache, protocol_version)
     cdef _remove_view(self, view)
     cdef _update_backend_ids(self, new_types)
     cdef _set_and_signal_new_user_schema(
         self,
         new_schema_pickle,
+        schema_version,
         extensions,
         ext_config_settings,
         reflection_cache=?,
@@ -119,7 +120,7 @@ cdef class DatabaseConnectionView:
         object _protocol_version
 
         object _db_config_temp
-        object _db_config_dbver
+        object _db_config_schema_version
 
         # State properties
         object _config
@@ -138,17 +139,13 @@ cdef class DatabaseConnectionView:
         tuple _session_state_db_cache
         tuple _session_state_cache
 
-
-        object _eql_to_compiled
-
         object _txid
         object _in_tx_db_config
         object _in_tx_savepoints
-        object _in_tx_user_schema_pickle
+        object _in_tx_user_schema_pv
         object _in_tx_user_config_spec
         object _in_tx_global_schema_pickle
         object _in_tx_new_types
-        int _in_tx_dbver
         bint _in_tx
         bint _in_tx_with_ddl
         bint _in_tx_with_sysconfig
@@ -163,7 +160,6 @@ cdef class DatabaseConnectionView:
 
         object __weakref__
 
-    cdef _invalidate_local_cache(self)
     cdef _reset_tx_state(self)
 
     cdef clear_tx_error(self)
@@ -175,7 +171,9 @@ cdef class DatabaseConnectionView:
     cpdef in_tx(self)
     cpdef in_tx_error(self)
 
-    cdef cache_compiled_query(self, object key, object query_unit)
+    cdef cache_compiled_query(
+        self, object key, object query_unit_group, schema_version
+    )
     cdef lookup_compiled_query(self, object key)
 
     cdef tx_error(self)
@@ -188,7 +186,7 @@ cdef class DatabaseConnectionView:
     cdef on_success(self, query_unit, new_types)
     cdef commit_implicit_tx(
         self,
-        user_schema,
+        user_schema_pv,
         extensions,
         ext_config_settings,
         global_schema,