Skip to content

Commit

Permalink
Use dataset type preload throughout unit tests
Browse files Browse the repository at this point in the history
Trigger dataset type preload the first time a connection is made to the Butler server in each unit test, to better match the conditions that will exist in the actual server.
  • Loading branch information
dhirving committed Dec 10, 2024
1 parent d15e0bc commit efa3711
Show file tree
Hide file tree
Showing 4 changed files with 20 additions and 11 deletions.
9 changes: 4 additions & 5 deletions python/lsst/daf/butler/_labeled_butler_factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ def __init__(self, repositories: Mapping[str, str] | None = None) -> None:
self._initialization_locks = NamedLocks()

# This may be overridden by unit tests.
self._preload_direct_butler_cache = True
self._preload_unsafe_direct_butler_caches = True

def bind(self, access_token: str | None) -> LabeledButlerFactoryProtocol:
"""Create a callable factory function for generating Butler instances
Expand Down Expand Up @@ -161,7 +161,7 @@ def _create_butler_factory_function(self, label: str) -> _FactoryFunction:

match butler_type:
case ButlerType.DIRECT:
return _create_direct_butler_factory(config, self._preload_direct_butler_cache)
return _create_direct_butler_factory(config, self._preload_unsafe_direct_butler_caches)
case ButlerType.REMOTE:
return _create_remote_butler_factory(config)
case _:
Expand All @@ -177,7 +177,7 @@ def _get_config_uri(self, label: str) -> ResourcePathExpression:
return config_uri


def _create_direct_butler_factory(config: ButlerConfig, preload_cache: bool) -> _FactoryFunction:
def _create_direct_butler_factory(config: ButlerConfig, preload_unsafe_caches: bool) -> _FactoryFunction:
import lsst.daf.butler.direct_butler

# Create a 'template' Butler that will be cloned when callers request an
Expand All @@ -187,8 +187,7 @@ def _create_direct_butler_factory(config: ButlerConfig, preload_cache: bool) ->

# Load caches so that data is available in cloned instances without
# needing to refetch it from the database for every instance.
if preload_cache:
butler._preload_cache()
butler._preload_cache(load_dimension_record_cache=preload_unsafe_caches)

def create_butler(access_token: str | None) -> Butler:
# Access token is ignored because DirectButler does not use Gafaelfawr
Expand Down
4 changes: 2 additions & 2 deletions python/lsst/daf/butler/direct_butler/_direct_butler.py
Original file line number Diff line number Diff line change
Expand Up @@ -2339,9 +2339,9 @@ def _query_all_datasets_by_page(
pages = query_all_datasets(self, query, args)
yield iter(page.data for page in pages)

def _preload_cache(self) -> None:
def _preload_cache(self, *, load_dimension_record_cache: bool = True) -> None:
"""Immediately load caches that are used for common operations."""
self._registry.preload_cache()
self._registry.preload_cache(load_dimension_record_cache=load_dimension_record_cache)

_config: ButlerConfig
"""Configuration for this Butler instance."""
Expand Down
16 changes: 13 additions & 3 deletions python/lsst/daf/butler/registry/sql_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -2482,11 +2482,21 @@ def make_datastore_tables(self, tables: Mapping[str, DatastoreOpaqueTable]) -> N
pass
self._datastore_record_classes = datastore_record_classes

def preload_cache(self) -> None:
"""Immediately load caches that are used for common operations."""
self.dimension_record_cache.preload_cache()
def preload_cache(self, *, load_dimension_record_cache: bool) -> None:
"""Immediately load caches that are used for common operations.
Parameters
----------
load_dimension_record_cache : `bool`
If True, preload the dimension record cache. When this cache is
preloaded, subsequent external changes to governor dimension
records will not be visible to this Butler.
"""
self._managers.datasets.preload_cache()

if load_dimension_record_cache:
self.dimension_record_cache.preload_cache()

@property
def obsCoreTableManager(self) -> ObsCoreTableManager | None:
"""The ObsCore manager instance for this registry
Expand Down
2 changes: 1 addition & 1 deletion python/lsst/daf/butler/tests/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ def create_test_server(
# instrument records etc during setup. So configure the
# factory to disable this preloading and re-fetch the records
# as needed.
server_butler_factory._preload_direct_butler_cache = False
server_butler_factory._preload_unsafe_direct_butler_caches = False
app.dependency_overrides[butler_factory_dependency] = lambda: server_butler_factory

# Using TestClient in a context manager ensures that it uses
Expand Down

0 comments on commit efa3711

Please sign in to comment.