Merge pull request #892 from lsst/tickets/DM-41043

DM-41043: internal package reorganization

doc/changes/DM-41043.misc.md

@@ -0,0 +1,5 @@
+Reorganize internal subpackages, rename modules, and adjust symbol lifting.
+
+This includes moving some symbols that we had always intended to be private (or
+public only to other middleware packages) that were not clearly marked as such
+(e.g. with leading underscores) before.

doc/lsst.daf.butler/index.rst

@@ -101,18 +101,8 @@ Python API reference
 
 .. automodapi:: lsst.daf.butler
    :no-main-docstr:
-
-.. automodapi:: lsst.daf.butler.registry
-   :no-main-docstr:
-
-.. automodapi:: lsst.daf.butler.registry.interfaces
-   :no-main-docstr:
-
-.. automodapi:: lsst.daf.butler.registry.queries
-   :no-main-docstr:
-
-.. automodapi:: lsst.daf.butler.registry.wildcards
-   :no-main-docstr:
+   :skip: CollectionSearch
+   :skip: RegistryConfig
 
 Example datastores
 ------------------
@@ -149,57 +139,130 @@ Example formatters
    :no-main-docstr:
    :headings: ^"
 
+Test utilities
+--------------
+
+.. automodapi:: lsst.daf.butler.tests
+   :no-main-docstr:
+   :headings: ^"
+   :no-inheritance-diagram:
+
+Middleware-Internal API
+=======================
+
+.. warning::
+
+   These symbols are used throughout the middleware system and may be used in advanced middleware extensions (e.g. third-party `Datastore` implementations), but are not considered fully public interfaces in terms of stability guarantees.
+
+Datastore utilities
+-------------------
+
+.. automodapi:: lsst.daf.butler.datastore
+   :no-main-docstr:
+   :headings: ^"
+   :skip: Datastore
+
+Registry utilities and interfaces
+---------------------------------
+
+.. automodapi:: lsst.daf.butler.registry
+   :no-main-docstr:
+   :headings: ^"
+   :skip: Registry
+   :skip: CollectionType
+
+.. automodapi:: lsst.daf.butler.registry.interfaces
+   :headings: ^"
+   :no-main-docstr:
+
+.. automodapi:: lsst.daf.butler.registry.queries
+   :headings: ^"
+   :no-main-docstr:
+
+.. automodapi:: lsst.daf.butler.registry.wildcards
+   :headings: ^"
+   :no-main-docstr:
+
 Database backends
 -----------------
 
 .. automodapi:: lsst.daf.butler.registry.databases.sqlite
    :no-main-docstr:
    :headings: ^"
+
 .. automodapi:: lsst.daf.butler.registry.databases.postgresql
    :no-main-docstr:
    :headings: ^"
 
-Support API
------------
+General utilities
+-----------------
 
-.. automodapi:: lsst.daf.butler.core.utils
+.. automodapi:: lsst.daf.butler.ddl
    :no-main-docstr:
    :headings: ^"
-.. automodapi:: lsst.daf.butler.core.repoRelocation
+
+.. automodapi:: lsst.daf.butler.instrument
    :no-main-docstr:
    :headings: ^"
-   :include-all-objects:
 
-Test utilities
---------------
+.. automodapi:: lsst.daf.butler.json
+   :no-main-docstr:
+   :headings: ^"
 
-.. automodapi:: lsst.daf.butler.tests
+.. automodapi:: lsst.daf.butler.logging
    :no-main-docstr:
-   :no-inheritance-diagram:
+   :headings: ^"
 
-Command Line Interface API
---------------------------
+.. automodapi:: lsst.daf.butler.mapping_factory
+   :no-main-docstr:
+   :headings: ^"
 
-.. warning::
-   The command line interface API (everything in ``lsst.daf.butler.cli``) is for only for developer use to write command line interfaces, and is not intended for general use.
+.. automodapi:: lsst.daf.butler.persistence_context
+   :no-main-docstr:
+   :headings: ^"
 
-.. automodapi:: lsst.daf.butler.cli.butler
+.. automodapi:: lsst.daf.butler.progress
    :no-main-docstr:
+   :headings: ^"
+
+.. automodapi:: lsst.daf.butler.repo_relocation
+   :no-main-docstr:
+   :headings: ^"
+   :include-all-objects:
+
+.. automodapi:: lsst.daf.butler.time_utils
+   :no-main-docstr:
+   :headings: ^"
+
+.. automodapi:: lsst.daf.butler.utils
+   :no-main-docstr:
+   :headings: ^"
+
+Command-Line Interface Utilities
+--------------------------------
 
+.. automodapi:: lsst.daf.butler.cli.butler
+   :no-main-docstr:
+   :headings: ^"
 
 .. automodapi:: lsst.daf.butler.cli.cmd
    :no-main-docstr:
    :include-all-objects:
+   :headings: ^"
 
 .. automodapi:: lsst.daf.butler.cli.opt
    :no-main-docstr:
    :include-all-objects:
+   :headings: ^"
 
 .. automodapi:: lsst.daf.butler.cli.utils
    :no-main-docstr:
+   :headings: ^"
 
 .. automodapi:: lsst.daf.butler.cli.cliLog
    :no-main-docstr:
+   :headings: ^"
 
 .. automodapi:: lsst.daf.butler.cli.progress
    :no-main-docstr:
+   :headings: ^"

python/lsst/daf/butler/__init__.py

@@ -1,20 +1,88 @@
+# This file is part of daf_butler.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This software is dual licensed under the GNU General Public License and also
+# under a 3-clause BSD license. Recipients may choose which of these licenses
+# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
+# respectively.  If you choose the GPL option then the following text applies
+# (but note that there is still no warranty even if you opt for BSD instead):
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 """
 Data Access Butler
 """
 
 # Some components are not auto-imported since they can have additional runtime
 # dependencies.
 
+from . import logging  # most symbols are helpers only
+from . import progress  # most symbols are only used by handler implementors
+from . import ddl, time_utils
 from ._butler import *
-from ._butlerConfig import *
-from ._butlerRepoIndex import *
+from ._butler_config import *
+from ._butler_repo_index import *
+from ._column_categorization import *
+from ._column_tags import *
+from ._column_type_info import *
+from ._config import *
+from ._config_support import LookupKey
+from ._dataset_association import *
 from ._dataset_existence import *
+from ._dataset_ref import *
+from ._dataset_type import *
 from ._deferredDatasetHandle import *
+from ._exceptions import *
+from ._file_dataset import *
+from ._file_descriptor import *
+from ._formatter import *
+
+# Do not import 'instrument' or 'json' at all by default.
 from ._limited_butler import *
+from ._location import *
+from ._named import *
+from ._quantum import *
 from ._quantum_backed import *
-from .core import *
+from ._storage_class import *
+from ._storage_class_delegate import *
+from ._timespan import *
+from ._topology import *
+
+# Only lift 'Datastore' itself to this scope.
+from .datastore import Datastore
+from .dimensions import *
+
+# Only export 'ButlerLogRecords' from 'logging', import the module as-is for
+# other symbols. ButlerLogRecords is the fundamental type stored in datastores.
+from .logging import ButlerLogRecords
 
+# Do not import or lift symbols from mapping_factory and persistence_content,
+# as those are internal.
+# Only lift 'Progress' from 'progess'; the module is imported as-is above
+from .progress import Progress
+
+# Do not import or lift symbols from 'server' or 'server_models'.
 # Import the registry subpackage directly for other symbols.
 from .registry import CollectionSearch, CollectionType, Registry, RegistryConfig
 from .transfers import RepoExportContext, YamlRepoExportBackend, YamlRepoImportBackend
 from .version import *
+
+# Do not import or lift symbols from 'repo_relocation'.
+
+# Do not export the utility routines from utils.

python/lsst/daf/butler/_butler.py

@@ -52,38 +52,31 @@
 from lsst.utils.logging import VERBOSE, getLogger
 from sqlalchemy.exc import IntegrityError
 
-from ._butlerConfig import ButlerConfig
-from ._butlerRepoIndex import ButlerRepoIndex
+from ._butler_config import ButlerConfig
+from ._butler_repo_index import ButlerRepoIndex
+from ._config import Config, ConfigSubset
 from ._dataset_existence import DatasetExistence
+from ._dataset_ref import DatasetIdGenEnum, DatasetRef
+from ._dataset_type import DatasetType
 from ._deferredDatasetHandle import DeferredDatasetHandle
+from ._exceptions import ValidationError
+from ._file_dataset import FileDataset
 from ._limited_butler import LimitedButler
 from ._registry_shim import RegistryShim
-from .core import (
-    Config,
-    ConfigSubset,
+from ._storage_class import StorageClass, StorageClassFactory
+from ._timespan import Timespan
+from .datastore import DatasetRefURIs, Datastore, NullDatastore
+from .dimensions import (
     DataCoordinate,
     DataId,
     DataIdValue,
-    DatasetIdGenEnum,
-    DatasetRef,
-    DatasetRefURIs,
-    DatasetType,
-    Datastore,
     Dimension,
     DimensionConfig,
     DimensionElement,
     DimensionRecord,
     DimensionUniverse,
-    FileDataset,
-    NullDatastore,
-    Progress,
-    StorageClass,
-    StorageClassFactory,
-    Timespan,
-    ValidationError,
 )
-from .core.repoRelocation import BUTLER_ROOT_TAG
-from .core.utils import transactional
+from .progress import Progress
 from .registry import (
     CollectionType,
     ConflictingDefinitionError,
@@ -96,7 +89,9 @@
     _ButlerRegistry,
     _RegistryFactory,
 )
+from .repo_relocation import BUTLER_ROOT_TAG
 from .transfers import RepoExportContext
+from .utils import transactional
 
 if TYPE_CHECKING:
     from lsst.resources import ResourceHandleProtocol

python/lsst/daf/butler/_butlerConfig.py → python/lsst/daf/butler/_butler_config.py

@@ -38,8 +38,10 @@
 
 from lsst.resources import ResourcePath, ResourcePathExpression
 
-from ._butlerRepoIndex import ButlerRepoIndex
-from .core import Config, DatastoreConfig, StorageClassConfig
+from ._butler_repo_index import ButlerRepoIndex
+from ._config import Config
+from ._storage_class import StorageClassConfig
+from .datastore import DatastoreConfig
 from .registry import RegistryConfig
 from .transfers import RepoTransferFormatConfig
 

python/lsst/daf/butler/_butlerRepoIndex.py → python/lsst/daf/butler/_butler_repo_index.py

@@ -34,7 +34,7 @@
 
 from lsst.resources import ResourcePath
 
-from .core import Config
+from ._config import Config
 
 
 class ButlerRepoIndex:

python/lsst/daf/butler/core/_column_type_info.py → python/lsst/daf/butler/_column_type_info.py

@@ -40,8 +40,8 @@
 
 from . import ddl
 from ._column_tags import DatasetColumnTag, DimensionKeyColumnTag, DimensionRecordColumnTag
+from ._timespan import TimespanDatabaseRepresentation
 from .dimensions import Dimension, DimensionUniverse
-from .timespan import TimespanDatabaseRepresentation
 
 LogicalColumn = sqlalchemy.sql.ColumnElement | TimespanDatabaseRepresentation
 """A type alias for the types used to represent columns in SQL relations.
@@ -77,7 +77,7 @@ class ColumnTypeInfo:
 
     ingest_date_dtype: type[ddl.AstropyTimeNsecTai] | type[sqlalchemy.TIMESTAMP]
     """Type of the ``ingest_date`` column, can be either
-    `~lsst.daf.butler.core.ddl.AstropyTimeNsecTai` or `sqlalchemy.TIMESTAMP`.
+    `~lsst.daf.butler.ddl.AstropyTimeNsecTai` or `sqlalchemy.TIMESTAMP`.
     """
 
     @property

python/lsst/daf/butler/core/configSupport.py → python/lsst/daf/butler/_config_support.py

@@ -39,7 +39,7 @@
 from .dimensions import DimensionGraph
 
 if TYPE_CHECKING:
-    from .config import Config
+    from ._config import Config
     from .dimensions import Dimension, DimensionUniverse
 
 log = logging.getLogger(__name__)

python/lsst/daf/butler/core/datasets/association.py → python/lsst/daf/butler/_dataset_association.py

@@ -27,11 +27,13 @@
 
 from __future__ import annotations
 
+__all__ = ("DatasetAssociation",)
+
 from dataclasses import dataclass
 from typing import Any
 
-from ..timespan import Timespan
-from .ref import DatasetRef
+from ._dataset_ref import DatasetRef
+from ._timespan import Timespan
 
 
 @dataclass(frozen=True, eq=True)